Summary: This PR implements the MergeOperator for Cassandra Row Values. Closes https://github.com/facebook/rocksdb/pull/2289 Differential Revision: D5055464 Pulled By: scv119 fbshipit-source-id: 45f276ef8cbc4704279202f6a20c64889bc1adefmain
Chen Shen
8 years ago
committed by
Facebook Github Bot
parent
2c98b06bff
commit
cbd825deea
@ -0,0 +1,47 @@ |
||||
// Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
// This source code is also licensed under the GPLv2 license found in the
|
||||
// COPYING file in the root directory of this source tree.
|
||||
|
||||
#include <stdio.h> |
||||
#include <stdlib.h> |
||||
#include <jni.h> |
||||
#include <string> |
||||
#include <memory> |
||||
|
||||
#include "include/org_rocksdb_CassandraValueMergeOperator.h" |
||||
#include "rocksjni/portal.h" |
||||
#include "rocksdb/db.h" |
||||
#include "rocksdb/options.h" |
||||
#include "rocksdb/statistics.h" |
||||
#include "rocksdb/memtablerep.h" |
||||
#include "rocksdb/table.h" |
||||
#include "rocksdb/slice_transform.h" |
||||
#include "rocksdb/merge_operator.h" |
||||
#include "utilities/merge_operators/cassandra/merge_operator.h" |
||||
|
||||
/*
|
||||
* Class: org_rocksdb_CassandraValueMergeOperator |
||||
* Method: newSharedCassandraValueMergeOperator |
||||
* Signature: ()J |
||||
*/ |
||||
jlong Java_org_rocksdb_CassandraValueMergeOperator_newSharedCassandraValueMergeOperator |
||||
(JNIEnv* env, jclass jclazz) { |
||||
auto* sptr_string_append_op = new std::shared_ptr<rocksdb::MergeOperator>( |
||||
rocksdb::CassandraValueMergeOperator::CreateSharedInstance()); |
||||
return reinterpret_cast<jlong>(sptr_string_append_op); |
||||
} |
||||
|
||||
/*
|
||||
* Class: org_rocksdb_CassandraValueMergeOperator |
||||
* Method: disposeInternal |
||||
* Signature: (J)V |
||||
*/ |
||||
void Java_org_rocksdb_CassandraValueMergeOperator_disposeInternal( |
||||
JNIEnv* env, jobject jobj, jlong jhandle) { |
||||
auto* sptr_string_append_op = |
||||
reinterpret_cast<std::shared_ptr<rocksdb::MergeOperator>* >(jhandle); |
||||
delete sptr_string_append_op; // delete std::shared_ptr
|
||||
} |
||||
@ -0,0 +1,22 @@ |
||||
// Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
// This source code is also licensed under the GPLv2 license found in the
|
||||
// COPYING file in the root directory of this source tree.
|
||||
|
||||
package org.rocksdb; |
||||
|
||||
/** |
||||
* CassandraValueMergeOperator is a merge operator that merges two cassandra wide column |
||||
* values. |
||||
*/ |
||||
public class CassandraValueMergeOperator extends MergeOperator { |
||||
public CassandraValueMergeOperator() { |
||||
super(newSharedCassandraValueMergeOperator()); |
||||
} |
||||
|
||||
private native static long newSharedCassandraValueMergeOperator(); |
||||
|
||||
@Override protected final native void disposeInternal(final long handle); |
||||
} |
||||
@ -0,0 +1,134 @@ |
||||
// Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
// This source code is also licensed under the GPLv2 license found in the
|
||||
// COPYING file in the root directory of this source tree.
|
||||
|
||||
#include <iostream> |
||||
|
||||
#include "rocksdb/db.h" |
||||
#include "rocksdb/merge_operator.h" |
||||
#include "rocksdb/utilities/db_ttl.h" |
||||
#include "util/testharness.h" |
||||
#include "util/random.h" |
||||
#include "utilities/merge_operators.h" |
||||
#include "utilities/merge_operators/cassandra/merge_operator.h" |
||||
#include "utilities/merge_operators/cassandra/test_utils.h" |
||||
|
||||
using namespace rocksdb; |
||||
|
||||
namespace rocksdb { |
||||
namespace cassandra { |
||||
|
||||
// Path to the database on file system
|
||||
const std::string kDbName = test::TmpDir() + "/cassandramerge_test"; |
||||
|
||||
class CassandraStore { |
||||
public: |
||||
explicit CassandraStore(std::shared_ptr<DB> db) |
||||
: db_(db), |
||||
merge_option_(), |
||||
get_option_() { |
||||
assert(db); |
||||
} |
||||
|
||||
bool Append(const std::string& key, const RowValue& val){ |
||||
std::string result; |
||||
val.Serialize(&result); |
||||
Slice valSlice(result.data(), result.size()); |
||||
auto s = db_->Merge(merge_option_, key, valSlice); |
||||
|
||||
if (s.ok()) { |
||||
return true; |
||||
} else { |
||||
std::cerr << "ERROR " << s.ToString() << std::endl; |
||||
return false; |
||||
} |
||||
} |
||||
|
||||
std::tuple<bool, RowValue> Get(const std::string& key){ |
||||
std::string result; |
||||
auto s = db_->Get(get_option_, key, &result); |
||||
|
||||
if (s.ok()) { |
||||
return std::make_tuple(true, |
||||
RowValue::Deserialize(result.data(), |
||||
result.size())); |
||||
} |
||||
|
||||
if (!s.IsNotFound()) { |
||||
std::cerr << "ERROR " << s.ToString() << std::endl; |
||||
} |
||||
|
||||
return std::make_tuple(false, RowValue(0, 0)); |
||||
} |
||||
|
||||
private: |
||||
std::shared_ptr<DB> db_; |
||||
WriteOptions merge_option_; |
||||
ReadOptions get_option_; |
||||
}; |
||||
|
||||
|
||||
// The class for unit-testing
|
||||
class CassandraMergeTest : public testing::Test { |
||||
public: |
||||
CassandraMergeTest() { |
||||
DestroyDB(kDbName, Options()); // Start each test with a fresh DB
|
||||
} |
||||
|
||||
std::shared_ptr<DB> OpenDb() { |
||||
DB* db; |
||||
Options options; |
||||
options.create_if_missing = true; |
||||
options.merge_operator.reset(new CassandraValueMergeOperator()); |
||||
EXPECT_OK(DB::Open(options, kDbName, &db)); |
||||
return std::shared_ptr<DB>(db); |
||||
} |
||||
}; |
||||
|
||||
// THE TEST CASES BEGIN HERE
|
||||
|
||||
TEST_F(CassandraMergeTest, SimpleTest) { |
||||
auto db = OpenDb(); |
||||
CassandraStore store(db); |
||||
|
||||
store.Append("k1", CreateTestRowValue({ |
||||
std::make_tuple(kTombstone, 0, 5), |
||||
std::make_tuple(kColumn, 1, 8), |
||||
std::make_tuple(kExpiringColumn, 2, 5), |
||||
})); |
||||
store.Append("k1",CreateTestRowValue({ |
||||
std::make_tuple(kColumn, 0, 2), |
||||
std::make_tuple(kExpiringColumn, 1, 5), |
||||
std::make_tuple(kTombstone, 2, 7), |
||||
std::make_tuple(kExpiringColumn, 7, 17), |
||||
})); |
||||
store.Append("k1", CreateTestRowValue({ |
||||
std::make_tuple(kExpiringColumn, 0, 6), |
||||
std::make_tuple(kTombstone, 1, 5), |
||||
std::make_tuple(kColumn, 2, 4), |
||||
std::make_tuple(kTombstone, 11, 11), |
||||
})); |
||||
|
||||
auto ret = store.Get("k1"); |
||||
|
||||
ASSERT_TRUE(std::get<0>(ret)); |
||||
RowValue& merged = std::get<1>(ret); |
||||
EXPECT_EQ(merged.columns_.size(), 5); |
||||
VerifyRowValueColumns(merged.columns_, 0, kExpiringColumn, 0, 6); |
||||
VerifyRowValueColumns(merged.columns_, 1, kColumn, 1, 8); |
||||
VerifyRowValueColumns(merged.columns_, 2, kTombstone, 2, 7); |
||||
VerifyRowValueColumns(merged.columns_, 3, kExpiringColumn, 7, 17); |
||||
VerifyRowValueColumns(merged.columns_, 4, kTombstone, 11, 11); |
||||
} |
||||
|
||||
|
||||
} // namespace cassandra
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) { |
||||
::testing::InitGoogleTest(&argc, argv); |
||||
return RUN_ALL_TESTS(); |
||||
} |
||||
@ -0,0 +1,296 @@ |
||||
// Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
// This source code is also licensed under the GPLv2 license found in the
|
||||
// COPYING file in the root directory of this source tree.
|
||||
|
||||
#include "format.h" |
||||
|
||||
#include <algorithm> |
||||
#include <map> |
||||
#include <memory> |
||||
|
||||
#include "utilities/merge_operators/cassandra/serialize.h" |
||||
|
||||
namespace rocksdb { |
||||
namespace cassandra { |
||||
namespace { |
||||
const int32_t kDefaultLocalDeletionTime = |
||||
std::numeric_limits<int32_t>::max(); |
||||
const int64_t kDefaultMarkedForDeleteAt = |
||||
std::numeric_limits<int64_t>::min(); |
||||
} |
||||
|
||||
ColumnBase::ColumnBase(int8_t mask, int8_t index) |
||||
: mask_(mask), index_(index) {} |
||||
|
||||
std::size_t ColumnBase::Size() const { |
||||
return sizeof(mask_) + sizeof(index_); |
||||
} |
||||
|
||||
int8_t ColumnBase::Mask() const { |
||||
return mask_; |
||||
} |
||||
|
||||
int8_t ColumnBase::Index() const { |
||||
return index_; |
||||
} |
||||
|
||||
void ColumnBase::Serialize(std::string* dest) const { |
||||
rocksdb::cassandra::Serialize<int8_t>(mask_, dest); |
||||
rocksdb::cassandra::Serialize<int8_t>(index_, dest); |
||||
} |
||||
|
||||
std::unique_ptr<ColumnBase> ColumnBase::Deserialize(const char* src, |
||||
std::size_t offset) { |
||||
int8_t mask = rocksdb::cassandra::Deserialize<int8_t>(src, offset); |
||||
if ((mask & ColumnTypeMask::DELETION_MASK) != 0) { |
||||
return Tombstone::Deserialize(src, offset); |
||||
} else if ((mask & ColumnTypeMask::EXPIRATION_MASK) != 0) { |
||||
return ExpiringColumn::Deserialize(src, offset); |
||||
} else { |
||||
return Column::Deserialize(src, offset); |
||||
} |
||||
} |
||||
|
||||
Column::Column( |
||||
int8_t mask, |
||||
int8_t index, |
||||
int64_t timestamp, |
||||
int32_t value_size, |
||||
const char* value |
||||
) : ColumnBase(mask, index), timestamp_(timestamp), |
||||
value_size_(value_size), value_(value) {} |
||||
|
||||
int64_t Column::Timestamp() const { |
||||
return timestamp_; |
||||
} |
||||
|
||||
std::size_t Column::Size() const { |
||||
return ColumnBase::Size() + sizeof(timestamp_) + sizeof(value_size_) |
||||
+ value_size_; |
||||
} |
||||
|
||||
void Column::Serialize(std::string* dest) const { |
||||
ColumnBase::Serialize(dest); |
||||
rocksdb::cassandra::Serialize<int64_t>(timestamp_, dest); |
||||
rocksdb::cassandra::Serialize<int32_t>(value_size_, dest); |
||||
dest->append(value_, value_size_); |
||||
} |
||||
|
||||
std::unique_ptr<Column> Column::Deserialize(const char *src, |
||||
std::size_t offset) { |
||||
int8_t mask = rocksdb::cassandra::Deserialize<int8_t>(src, offset); |
||||
offset += sizeof(mask); |
||||
int8_t index = rocksdb::cassandra::Deserialize<int8_t>(src, offset); |
||||
offset += sizeof(index); |
||||
int64_t timestamp = rocksdb::cassandra::Deserialize<int64_t>(src, offset); |
||||
offset += sizeof(timestamp); |
||||
int32_t value_size = rocksdb::cassandra::Deserialize<int32_t>(src, offset); |
||||
offset += sizeof(value_size); |
||||
return std::unique_ptr<Column>( |
||||
new Column(mask, index, timestamp, value_size, src + offset)); |
||||
} |
||||
|
||||
ExpiringColumn::ExpiringColumn( |
||||
int8_t mask, |
||||
int8_t index, |
||||
int64_t timestamp, |
||||
int32_t value_size, |
||||
const char* value, |
||||
int32_t ttl |
||||
) : Column(mask, index, timestamp, value_size, value), |
||||
ttl_(ttl) {} |
||||
|
||||
std::size_t ExpiringColumn::Size() const { |
||||
return Column::Size() + sizeof(ttl_); |
||||
} |
||||
|
||||
void ExpiringColumn::Serialize(std::string* dest) const { |
||||
Column::Serialize(dest); |
||||
rocksdb::cassandra::Serialize<int32_t>(ttl_, dest); |
||||
} |
||||
|
||||
std::unique_ptr<ExpiringColumn> ExpiringColumn::Deserialize( |
||||
const char *src, |
||||
std::size_t offset) { |
||||
int8_t mask = rocksdb::cassandra::Deserialize<int8_t>(src, offset); |
||||
offset += sizeof(mask); |
||||
int8_t index = rocksdb::cassandra::Deserialize<int8_t>(src, offset); |
||||
offset += sizeof(index); |
||||
int64_t timestamp = rocksdb::cassandra::Deserialize<int64_t>(src, offset); |
||||
offset += sizeof(timestamp); |
||||
int32_t value_size = rocksdb::cassandra::Deserialize<int32_t>(src, offset); |
||||
offset += sizeof(value_size); |
||||
const char* value = src + offset; |
||||
offset += value_size; |
||||
int32_t ttl = rocksdb::cassandra::Deserialize<int32_t>(src, offset); |
||||
return std::unique_ptr<ExpiringColumn>( |
||||
new ExpiringColumn(mask, index, timestamp, value_size, value, ttl)); |
||||
} |
||||
|
||||
Tombstone::Tombstone( |
||||
int8_t mask, |
||||
int8_t index, |
||||
int32_t local_deletion_time, |
||||
int64_t marked_for_delete_at |
||||
) : ColumnBase(mask, index), local_deletion_time_(local_deletion_time), |
||||
marked_for_delete_at_(marked_for_delete_at) {} |
||||
|
||||
int64_t Tombstone::Timestamp() const { |
||||
return marked_for_delete_at_; |
||||
} |
||||
|
||||
std::size_t Tombstone::Size() const { |
||||
return ColumnBase::Size() + sizeof(local_deletion_time_) |
||||
+ sizeof(marked_for_delete_at_); |
||||
} |
||||
|
||||
void Tombstone::Serialize(std::string* dest) const { |
||||
ColumnBase::Serialize(dest); |
||||
rocksdb::cassandra::Serialize<int32_t>(local_deletion_time_, dest); |
||||
rocksdb::cassandra::Serialize<int64_t>(marked_for_delete_at_, dest); |
||||
} |
||||
|
||||
std::unique_ptr<Tombstone> Tombstone::Deserialize(const char *src, |
||||
std::size_t offset) { |
||||
int8_t mask = rocksdb::cassandra::Deserialize<int8_t>(src, offset); |
||||
offset += sizeof(mask); |
||||
int8_t index = rocksdb::cassandra::Deserialize<int8_t>(src, offset); |
||||
offset += sizeof(index); |
||||
int32_t local_deletion_time = |
||||
rocksdb::cassandra::Deserialize<int32_t>(src, offset); |
||||
offset += sizeof(int32_t); |
||||
int64_t marked_for_delete_at = |
||||
rocksdb::cassandra::Deserialize<int64_t>(src, offset); |
||||
return std::unique_ptr<Tombstone>( |
||||
new Tombstone(mask, index, local_deletion_time, marked_for_delete_at)); |
||||
} |
||||
|
||||
RowValue::RowValue(int32_t local_deletion_time, int64_t marked_for_delete_at) |
||||
: local_deletion_time_(local_deletion_time), |
||||
marked_for_delete_at_(marked_for_delete_at), columns_(), |
||||
last_modified_time_(0) {} |
||||
|
||||
RowValue::RowValue(std::vector<std::unique_ptr<ColumnBase>> columns, |
||||
int64_t last_modified_time) |
||||
: local_deletion_time_(kDefaultLocalDeletionTime), |
||||
marked_for_delete_at_(kDefaultMarkedForDeleteAt), |
||||
columns_(std::move(columns)), last_modified_time_(last_modified_time) {} |
||||
|
||||
std::size_t RowValue::Size() const { |
||||
std::size_t size = sizeof(local_deletion_time_) |
||||
+ sizeof(marked_for_delete_at_); |
||||
for (const auto& column : columns_) { |
||||
size += column -> Size(); |
||||
} |
||||
return size; |
||||
} |
||||
|
||||
int64_t RowValue::LastModifiedTime() const { |
||||
if (IsTombstone()) { |
||||
return marked_for_delete_at_; |
||||
} else { |
||||
return last_modified_time_; |
||||
} |
||||
} |
||||
|
||||
bool RowValue::IsTombstone() const { |
||||
return marked_for_delete_at_ > kDefaultMarkedForDeleteAt; |
||||
} |
||||
|
||||
void RowValue::Serialize(std::string* dest) const { |
||||
rocksdb::cassandra::Serialize<int32_t>(local_deletion_time_, dest); |
||||
rocksdb::cassandra::Serialize<int64_t>(marked_for_delete_at_, dest); |
||||
for (const auto& column : columns_) { |
||||
column -> Serialize(dest); |
||||
} |
||||
} |
||||
|
||||
RowValue RowValue::Deserialize(const char *src, std::size_t size) { |
||||
std::size_t offset = 0; |
||||
assert(size >= sizeof(local_deletion_time_) + sizeof(marked_for_delete_at_)); |
||||
int32_t local_deletion_time = |
||||
rocksdb::cassandra::Deserialize<int32_t>(src, offset); |
||||
offset += sizeof(int32_t); |
||||
int64_t marked_for_delete_at = |
||||
rocksdb::cassandra::Deserialize<int64_t>(src, offset); |
||||
offset += sizeof(int64_t); |
||||
if (offset == size) { |
||||
return RowValue(local_deletion_time, marked_for_delete_at); |
||||
} |
||||
|
||||
assert(local_deletion_time == kDefaultLocalDeletionTime); |
||||
assert(marked_for_delete_at == kDefaultMarkedForDeleteAt); |
||||
std::vector<std::unique_ptr<ColumnBase>> columns; |
||||
int64_t last_modified_time = 0; |
||||
while (offset < size) { |
||||
auto c = ColumnBase::Deserialize(src, offset); |
||||
offset += c -> Size(); |
||||
assert(offset <= size); |
||||
last_modified_time = std::max(last_modified_time, c -> Timestamp()); |
||||
columns.push_back(std::move(c)); |
||||
} |
||||
|
||||
return RowValue(std::move(columns), last_modified_time); |
||||
} |
||||
|
||||
// Merge multiple row values into one.
|
||||
// For each column in rows with same index, we pick the one with latest
|
||||
// timestamp. And we also take row tombstone into consideration, by iterating
|
||||
// each row from reverse timestamp order, and stop once we hit the first
|
||||
// row tombstone.
|
||||
RowValue RowValue::Merge(std::vector<RowValue>&& values) { |
||||
assert(values.size() > 0); |
||||
if (values.size() == 1) { |
||||
return std::move(values[0]); |
||||
} |
||||
|
||||
// Merge columns by their last modified time, and skip once we hit
|
||||
// a row tombstone.
|
||||
std::sort(values.begin(), values.end(), |
||||
[](const RowValue& r1, const RowValue& r2) { |
||||
return r1.LastModifiedTime() > r2.LastModifiedTime(); |
||||
}); |
||||
|
||||
std::map<int8_t, std::unique_ptr<ColumnBase>> merged_columns; |
||||
int64_t tombstone_timestamp = 0; |
||||
|
||||
for (auto& value : values) { |
||||
if (value.IsTombstone()) { |
||||
if (merged_columns.size() == 0) { |
||||
return std::move(value); |
||||
} |
||||
tombstone_timestamp = value.LastModifiedTime(); |
||||
break; |
||||
} |
||||
for (auto& column : value.columns_) { |
||||
int8_t index = column->Index(); |
||||
if (merged_columns.find(index) == merged_columns.end()) { |
||||
merged_columns[index] = std::move(column); |
||||
} else { |
||||
if (column->Timestamp() > merged_columns[index]->Timestamp()) { |
||||
merged_columns[index] = std::move(column); |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
int64_t last_modified_time = 0; |
||||
std::vector<std::unique_ptr<ColumnBase>> columns; |
||||
for (auto& pair: merged_columns) { |
||||
// For some row, its last_modified_time > row tombstone_timestamp, but
|
||||
// it might have rows whose timestamp is ealier than tombstone, so we
|
||||
// ned to filter these rows.
|
||||
if (pair.second->Timestamp() <= tombstone_timestamp) { |
||||
continue; |
||||
} |
||||
last_modified_time = std::max(last_modified_time, pair.second->Timestamp()); |
||||
columns.push_back(std::move(pair.second)); |
||||
} |
||||
return RowValue(std::move(columns), last_modified_time); |
||||
} |
||||
|
||||
} // namepsace cassandrda
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,179 @@ |
||||
// Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
// This source code is also licensed under the GPLv2 license found in the
|
||||
// COPYING file in the root directory of this source tree.
|
||||
|
||||
/**
|
||||
* The encoding of Cassandra Row Value. |
||||
* |
||||
* A Cassandra Row Value could either be a row tombstone, |
||||
* or contains multiple columns, it has following fields: |
||||
* |
||||
* struct row_value { |
||||
* int32_t local_deletion_time; // Time in second when the row is deleted,
|
||||
* // only used for Cassandra tombstone gc.
|
||||
* int64_t marked_for_delete_at; // Ms that marked this row is deleted.
|
||||
* struct column_base columns[]; // For non tombstone row, all columns
|
||||
* // are stored here.
|
||||
* } |
||||
* |
||||
* If the local_deletion_time and marked_for_delete_at is set, then this is |
||||
* a tombstone, otherwise it contains multiple columns. |
||||
* |
||||
* There are three type of Columns: Normal Column, Expiring Column and Column |
||||
* Tombstone, which have following fields: |
||||
* |
||||
* // Identify the type of the column.
|
||||
* enum mask { |
||||
* DELETION_MASK = 0x01, |
||||
* EXPIRATION_MASK = 0x02, |
||||
* }; |
||||
* |
||||
* struct column { |
||||
* int8_t mask = 0; |
||||
* int8_t index; |
||||
* int64_t timestamp; |
||||
* int32_t value_length; |
||||
* char value[value_length]; |
||||
* } |
||||
* |
||||
* struct expiring_column { |
||||
* int8_t mask = mask.EXPIRATION_MASK; |
||||
* int8_t index; |
||||
* int64_t timestamp; |
||||
* int32_t value_length; |
||||
* char value[value_length]; |
||||
* int32_t ttl; |
||||
* } |
||||
* |
||||
* struct tombstone_column { |
||||
* int8_t mask = mask.DELETION_MASK; |
||||
* int8_t index; |
||||
* int32_t local_deletion_time; // Similar to row_value's field.
|
||||
* int64_t marked_for_delete_at; |
||||
* } |
||||
*/ |
||||
|
||||
#pragma once |
||||
#include <vector> |
||||
#include <memory> |
||||
#include "rocksdb/merge_operator.h" |
||||
#include "rocksdb/slice.h" |
||||
#include "util/testharness.h" |
||||
|
||||
namespace rocksdb { |
||||
namespace cassandra { |
||||
|
||||
// Identify the type of the column.
|
||||
enum ColumnTypeMask { |
||||
DELETION_MASK = 0x01, |
||||
EXPIRATION_MASK = 0x02, |
||||
}; |
||||
|
||||
class ColumnBase { |
||||
public: |
||||
ColumnBase(int8_t mask, int8_t index); |
||||
virtual ~ColumnBase() = default; |
||||
|
||||
virtual int64_t Timestamp() const = 0; |
||||
virtual int8_t Mask() const; |
||||
virtual int8_t Index() const; |
||||
virtual std::size_t Size() const; |
||||
virtual void Serialize(std::string* dest) const; |
||||
|
||||
static std::unique_ptr<ColumnBase> Deserialize(const char* src, |
||||
std::size_t offset); |
||||
|
||||
private: |
||||
int8_t mask_; |
||||
int8_t index_; |
||||
}; |
||||
|
||||
class Column : public ColumnBase { |
||||
public: |
||||
Column(int8_t mask, int8_t index, int64_t timestamp, |
||||
int32_t value_size, const char* value); |
||||
|
||||
virtual int64_t Timestamp() const override; |
||||
virtual std::size_t Size() const override; |
||||
virtual void Serialize(std::string* dest) const override; |
||||
|
||||
static std::unique_ptr<Column> Deserialize(const char* src, |
||||
std::size_t offset); |
||||
|
||||
private: |
||||
int64_t timestamp_; |
||||
int32_t value_size_; |
||||
const char* value_; |
||||
}; |
||||
|
||||
class ExpiringColumn : public Column { |
||||
public: |
||||
ExpiringColumn(int8_t mask, int8_t index, int64_t timestamp, |
||||
int32_t value_size, const char* value, int32_t ttl); |
||||
|
||||
virtual std::size_t Size() const override; |
||||
virtual void Serialize(std::string* dest) const override; |
||||
|
||||
static std::unique_ptr<ExpiringColumn> Deserialize(const char* src, |
||||
std::size_t offset); |
||||
|
||||
private: |
||||
int32_t ttl_; |
||||
}; |
||||
|
||||
class Tombstone : public ColumnBase { |
||||
public: |
||||
Tombstone(int8_t mask, int8_t index, |
||||
int32_t local_deletion_time, int64_t marked_for_delete_at); |
||||
|
||||
virtual int64_t Timestamp() const override; |
||||
virtual std::size_t Size() const override; |
||||
virtual void Serialize(std::string* dest) const override; |
||||
|
||||
static std::unique_ptr<Tombstone> Deserialize(const char* src, |
||||
std::size_t offset); |
||||
|
||||
private: |
||||
int32_t local_deletion_time_; |
||||
int64_t marked_for_delete_at_; |
||||
}; |
||||
|
||||
class RowValue { |
||||
public: |
||||
// Create a Row Tombstone.
|
||||
RowValue(int32_t local_deletion_time, int64_t marked_for_delete_at); |
||||
// Create a Row containing columns.
|
||||
RowValue(std::vector<std::unique_ptr<ColumnBase>> columns, |
||||
int64_t last_modified_time); |
||||
RowValue(const RowValue& that) = delete; |
||||
RowValue(RowValue&& that) noexcept = default; |
||||
RowValue& operator=(const RowValue& that) = delete; |
||||
RowValue& operator=(RowValue&& that) = default; |
||||
|
||||
std::size_t Size() const;; |
||||
bool IsTombstone() const; |
||||
// For Tombstone this returns the marked_for_delete_at_,
|
||||
// otherwise it returns the max timestamp of containing columns.
|
||||
int64_t LastModifiedTime() const; |
||||
void Serialize(std::string* dest) const; |
||||
|
||||
static RowValue Deserialize(const char* src, std::size_t size); |
||||
// Merge multiple rows according to their timestamp.
|
||||
static RowValue Merge(std::vector<RowValue>&& values); |
||||
|
||||
private: |
||||
int32_t local_deletion_time_; |
||||
int64_t marked_for_delete_at_; |
||||
std::vector<std::unique_ptr<ColumnBase>> columns_; |
||||
int64_t last_modified_time_; |
||||
|
||||
FRIEND_TEST(RowValueMergeTest, Merge); |
||||
FRIEND_TEST(RowValueMergeTest, MergeWithRowTombstone); |
||||
FRIEND_TEST(CassandraMergeTest, SimpleTest); |
||||
}; |
||||
|
||||
} // namepsace cassandrda
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,306 @@ |
||||
// Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
// This source code is also licensed under the GPLv2 license found in the
|
||||
// COPYING file in the root directory of this source tree.
|
||||
|
||||
#include <cstring> |
||||
#include <memory> |
||||
#include "util/testharness.h" |
||||
#include "utilities/merge_operators/cassandra/format.h" |
||||
#include "utilities/merge_operators/cassandra/serialize.h" |
||||
|
||||
using namespace rocksdb::cassandra; |
||||
|
||||
namespace rocksdb { |
||||
namespace cassandra { |
||||
|
||||
TEST(ColumnTest, Column) { |
||||
char data[4] = {'d', 'a', 't', 'a'}; |
||||
int8_t mask = 0; |
||||
int8_t index = 1; |
||||
int64_t timestamp = 1494022807044; |
||||
Column c = Column(mask, index, timestamp, sizeof(data), data); |
||||
|
||||
EXPECT_EQ(c.Index(), index); |
||||
EXPECT_EQ(c.Timestamp(), timestamp); |
||||
EXPECT_EQ(c.Size(), 14 + sizeof(data)); |
||||
|
||||
// Verify the serialization.
|
||||
std::string dest; |
||||
dest.reserve(c.Size() * 2); |
||||
c.Serialize(&dest); |
||||
|
||||
EXPECT_EQ(dest.size(), c.Size()); |
||||
std::size_t offset = 0; |
||||
EXPECT_EQ(Deserialize<int8_t>(dest.c_str(), offset), mask); |
||||
offset += sizeof(int8_t); |
||||
EXPECT_EQ(Deserialize<int8_t>(dest.c_str(), offset), index); |
||||
offset += sizeof(int8_t); |
||||
EXPECT_EQ(Deserialize<int64_t>(dest.c_str(), offset), timestamp); |
||||
offset += sizeof(int64_t); |
||||
EXPECT_EQ(Deserialize<int32_t>(dest.c_str(), offset), sizeof(data)); |
||||
offset += sizeof(int32_t); |
||||
EXPECT_TRUE(std::memcmp(data, dest.c_str() + offset, sizeof(data)) == 0); |
||||
|
||||
// Verify the deserialization.
|
||||
std::unique_ptr<Column> c1 = Column::Deserialize(dest.c_str(), 0); |
||||
EXPECT_EQ(c1->Index(), index); |
||||
EXPECT_EQ(c1->Timestamp(), timestamp); |
||||
EXPECT_EQ(c1->Size(), 14 + sizeof(data)); |
||||
|
||||
c1->Serialize(&dest); |
||||
EXPECT_EQ(dest.size(), 2 * c.Size()); |
||||
EXPECT_TRUE( |
||||
std::memcmp(dest.c_str(), dest.c_str() + c.Size(), c.Size()) == 0); |
||||
|
||||
// Verify the ColumnBase::Deserialization.
|
||||
std::unique_ptr<ColumnBase> c2 = |
||||
ColumnBase::Deserialize(dest.c_str(), c.Size()); |
||||
c2->Serialize(&dest); |
||||
EXPECT_EQ(dest.size(), 3 * c.Size()); |
||||
EXPECT_TRUE( |
||||
std::memcmp(dest.c_str() + c.Size(), dest.c_str() + c.Size() * 2, c.Size()) |
||||
== 0); |
||||
} |
||||
|
||||
TEST(ExpiringColumnTest, ExpiringColumn) { |
||||
char data[4] = {'d', 'a', 't', 'a'}; |
||||
int8_t mask = ColumnTypeMask::EXPIRATION_MASK; |
||||
int8_t index = 3; |
||||
int64_t timestamp = 1494022807044; |
||||
int32_t ttl = 3600; |
||||
ExpiringColumn c = ExpiringColumn(mask, index, timestamp, |
||||
sizeof(data), data, ttl); |
||||
|
||||
EXPECT_EQ(c.Index(), index); |
||||
EXPECT_EQ(c.Timestamp(), timestamp); |
||||
EXPECT_EQ(c.Size(), 18 + sizeof(data)); |
||||
|
||||
// Verify the serialization.
|
||||
std::string dest; |
||||
dest.reserve(c.Size() * 2); |
||||
c.Serialize(&dest); |
||||
|
||||
EXPECT_EQ(dest.size(), c.Size()); |
||||
std::size_t offset = 0; |
||||
EXPECT_EQ(Deserialize<int8_t>(dest.c_str(), offset), mask); |
||||
offset += sizeof(int8_t); |
||||
EXPECT_EQ(Deserialize<int8_t>(dest.c_str(), offset), index); |
||||
offset += sizeof(int8_t); |
||||
EXPECT_EQ(Deserialize<int64_t>(dest.c_str(), offset), timestamp); |
||||
offset += sizeof(int64_t); |
||||
EXPECT_EQ(Deserialize<int32_t>(dest.c_str(), offset), sizeof(data)); |
||||
offset += sizeof(int32_t); |
||||
EXPECT_TRUE(std::memcmp(data, dest.c_str() + offset, sizeof(data)) == 0); |
||||
offset += sizeof(data); |
||||
EXPECT_EQ(Deserialize<int32_t>(dest.c_str(), offset), ttl); |
||||
|
||||
// Verify the deserialization.
|
||||
std::unique_ptr<ExpiringColumn> c1 = |
||||
ExpiringColumn::Deserialize(dest.c_str(), 0); |
||||
EXPECT_EQ(c1->Index(), index); |
||||
EXPECT_EQ(c1->Timestamp(), timestamp); |
||||
EXPECT_EQ(c1->Size(), 18 + sizeof(data)); |
||||
|
||||
c1->Serialize(&dest); |
||||
EXPECT_EQ(dest.size(), 2 * c.Size()); |
||||
EXPECT_TRUE( |
||||
std::memcmp(dest.c_str(), dest.c_str() + c.Size(), c.Size()) == 0); |
||||
|
||||
// Verify the ColumnBase::Deserialization.
|
||||
std::unique_ptr<ColumnBase> c2 = |
||||
ColumnBase::Deserialize(dest.c_str(), c.Size()); |
||||
c2->Serialize(&dest); |
||||
EXPECT_EQ(dest.size(), 3 * c.Size()); |
||||
EXPECT_TRUE( |
||||
std::memcmp(dest.c_str() + c.Size(), dest.c_str() + c.Size() * 2, c.Size()) |
||||
== 0); |
||||
} |
||||
|
||||
TEST(TombstoneTest, Tombstone) { |
||||
int8_t mask = ColumnTypeMask::DELETION_MASK; |
||||
int8_t index = 2; |
||||
int32_t local_deletion_time = 1494022807; |
||||
int64_t marked_for_delete_at = 1494022807044; |
||||
Tombstone c = Tombstone(mask, index, local_deletion_time, |
||||
marked_for_delete_at); |
||||
|
||||
EXPECT_EQ(c.Index(), index); |
||||
EXPECT_EQ(c.Timestamp(), marked_for_delete_at); |
||||
EXPECT_EQ(c.Size(), 14); |
||||
|
||||
// Verify the serialization.
|
||||
std::string dest; |
||||
dest.reserve(c.Size() * 2); |
||||
c.Serialize(&dest); |
||||
|
||||
EXPECT_EQ(dest.size(), c.Size()); |
||||
std::size_t offset = 0; |
||||
EXPECT_EQ(Deserialize<int8_t>(dest.c_str(), offset), mask); |
||||
offset += sizeof(int8_t); |
||||
EXPECT_EQ(Deserialize<int8_t>(dest.c_str(), offset), index); |
||||
offset += sizeof(int8_t); |
||||
EXPECT_EQ(Deserialize<int32_t>(dest.c_str(), offset), local_deletion_time); |
||||
offset += sizeof(int32_t); |
||||
EXPECT_EQ(Deserialize<int64_t>(dest.c_str(), offset), marked_for_delete_at); |
||||
|
||||
// Verify the deserialization.
|
||||
std::unique_ptr<Tombstone> c1 = Tombstone::Deserialize(dest.c_str(), 0); |
||||
EXPECT_EQ(c1->Index(), index); |
||||
EXPECT_EQ(c1->Timestamp(), marked_for_delete_at); |
||||
EXPECT_EQ(c1->Size(), 14); |
||||
|
||||
c1->Serialize(&dest); |
||||
EXPECT_EQ(dest.size(), 2 * c.Size()); |
||||
EXPECT_TRUE( |
||||
std::memcmp(dest.c_str(), dest.c_str() + c.Size(), c.Size()) == 0); |
||||
|
||||
// Verify the ColumnBase::Deserialization.
|
||||
std::unique_ptr<ColumnBase> c2 = |
||||
ColumnBase::Deserialize(dest.c_str(), c.Size()); |
||||
c2->Serialize(&dest); |
||||
EXPECT_EQ(dest.size(), 3 * c.Size()); |
||||
EXPECT_TRUE( |
||||
std::memcmp(dest.c_str() + c.Size(), dest.c_str() + c.Size() * 2, c.Size()) |
||||
== 0); |
||||
} |
||||
|
||||
TEST(RowValueTest, RowTombstone) { |
||||
int32_t local_deletion_time = 1494022807; |
||||
int64_t marked_for_delete_at = 1494022807044; |
||||
RowValue r = RowValue(local_deletion_time, marked_for_delete_at); |
||||
|
||||
EXPECT_EQ(r.Size(), 12); |
||||
EXPECT_EQ(r.IsTombstone(), true); |
||||
EXPECT_EQ(r.LastModifiedTime(), marked_for_delete_at); |
||||
|
||||
// Verify the serialization.
|
||||
std::string dest; |
||||
dest.reserve(r.Size() * 2); |
||||
r.Serialize(&dest); |
||||
|
||||
EXPECT_EQ(dest.size(), r.Size()); |
||||
std::size_t offset = 0; |
||||
EXPECT_EQ(Deserialize<int32_t>(dest.c_str(), offset), local_deletion_time); |
||||
offset += sizeof(int32_t); |
||||
EXPECT_EQ(Deserialize<int64_t>(dest.c_str(), offset), marked_for_delete_at); |
||||
|
||||
// Verify the deserialization.
|
||||
RowValue r1 = RowValue::Deserialize(dest.c_str(), r.Size()); |
||||
EXPECT_EQ(r1.Size(), 12); |
||||
EXPECT_EQ(r1.IsTombstone(), true); |
||||
EXPECT_EQ(r1.LastModifiedTime(), marked_for_delete_at); |
||||
|
||||
r1.Serialize(&dest); |
||||
EXPECT_EQ(dest.size(), 2 * r.Size()); |
||||
EXPECT_TRUE( |
||||
std::memcmp(dest.c_str(), dest.c_str() + r.Size(), r.Size()) == 0); |
||||
} |
||||
|
||||
TEST(RowValueTest, RowWithColumns) { |
||||
std::vector<std::unique_ptr<ColumnBase>> columns; |
||||
int64_t last_modified_time = 1494022807048; |
||||
std::size_t columns_data_size = 0; |
||||
|
||||
char e_data[5] = {'e', 'd', 'a', 't', 'a'}; |
||||
int8_t e_index = 0; |
||||
int64_t e_timestamp = 1494022807044; |
||||
int32_t e_ttl = 3600; |
||||
columns.push_back(std::unique_ptr<ExpiringColumn>( |
||||
new ExpiringColumn(ColumnTypeMask::EXPIRATION_MASK, e_index, |
||||
e_timestamp, sizeof(e_data), e_data, e_ttl))); |
||||
columns_data_size += columns[0]->Size(); |
||||
|
||||
char c_data[4] = {'d', 'a', 't', 'a'}; |
||||
int8_t c_index = 1; |
||||
int64_t c_timestamp = 1494022807048; |
||||
columns.push_back(std::unique_ptr<Column>( |
||||
new Column(0, c_index, c_timestamp, sizeof(c_data), c_data))); |
||||
columns_data_size += columns[1]->Size(); |
||||
|
||||
int8_t t_index = 2; |
||||
int32_t t_local_deletion_time = 1494022801; |
||||
int64_t t_marked_for_delete_at = 1494022807043; |
||||
columns.push_back(std::unique_ptr<Tombstone>( |
||||
new Tombstone(ColumnTypeMask::DELETION_MASK, |
||||
t_index, t_local_deletion_time, t_marked_for_delete_at))); |
||||
columns_data_size += columns[2]->Size(); |
||||
|
||||
RowValue r = RowValue(std::move(columns), last_modified_time); |
||||
|
||||
EXPECT_EQ(r.Size(), columns_data_size + 12); |
||||
EXPECT_EQ(r.IsTombstone(), false); |
||||
EXPECT_EQ(r.LastModifiedTime(), last_modified_time); |
||||
|
||||
// Verify the serialization.
|
||||
std::string dest; |
||||
dest.reserve(r.Size() * 2); |
||||
r.Serialize(&dest); |
||||
|
||||
EXPECT_EQ(dest.size(), r.Size()); |
||||
std::size_t offset = 0; |
||||
EXPECT_EQ(Deserialize<int32_t>(dest.c_str(), offset), |
||||
std::numeric_limits<int32_t>::max()); |
||||
offset += sizeof(int32_t); |
||||
EXPECT_EQ(Deserialize<int64_t>(dest.c_str(), offset), |
||||
std::numeric_limits<int64_t>::min()); |
||||
offset += sizeof(int64_t); |
||||
|
||||
// Column0: ExpiringColumn
|
||||
EXPECT_EQ(Deserialize<int8_t>(dest.c_str(), offset), |
||||
ColumnTypeMask::EXPIRATION_MASK); |
||||
offset += sizeof(int8_t); |
||||
EXPECT_EQ(Deserialize<int8_t>(dest.c_str(), offset), e_index); |
||||
offset += sizeof(int8_t); |
||||
EXPECT_EQ(Deserialize<int64_t>(dest.c_str(), offset), e_timestamp); |
||||
offset += sizeof(int64_t); |
||||
EXPECT_EQ(Deserialize<int32_t>(dest.c_str(), offset), sizeof(e_data)); |
||||
offset += sizeof(int32_t); |
||||
EXPECT_TRUE(std::memcmp(e_data, dest.c_str() + offset, sizeof(e_data)) == 0); |
||||
offset += sizeof(e_data); |
||||
EXPECT_EQ(Deserialize<int32_t>(dest.c_str(), offset), e_ttl); |
||||
offset += sizeof(int32_t); |
||||
|
||||
// Column1: Column
|
||||
EXPECT_EQ(Deserialize<int8_t>(dest.c_str(), offset), 0); |
||||
offset += sizeof(int8_t); |
||||
EXPECT_EQ(Deserialize<int8_t>(dest.c_str(), offset), c_index); |
||||
offset += sizeof(int8_t); |
||||
EXPECT_EQ(Deserialize<int64_t>(dest.c_str(), offset), c_timestamp); |
||||
offset += sizeof(int64_t); |
||||
EXPECT_EQ(Deserialize<int32_t>(dest.c_str(), offset), sizeof(c_data)); |
||||
offset += sizeof(int32_t); |
||||
EXPECT_TRUE(std::memcmp(c_data, dest.c_str() + offset, sizeof(c_data)) == 0); |
||||
offset += sizeof(c_data); |
||||
|
||||
// Column2: Tombstone
|
||||
EXPECT_EQ(Deserialize<int8_t>(dest.c_str(), offset), |
||||
ColumnTypeMask::DELETION_MASK); |
||||
offset += sizeof(int8_t); |
||||
EXPECT_EQ(Deserialize<int8_t>(dest.c_str(), offset), t_index); |
||||
offset += sizeof(int8_t); |
||||
EXPECT_EQ(Deserialize<int32_t>(dest.c_str(), offset), t_local_deletion_time); |
||||
offset += sizeof(int32_t); |
||||
EXPECT_EQ(Deserialize<int64_t>(dest.c_str(), offset), t_marked_for_delete_at); |
||||
|
||||
// Verify the deserialization.
|
||||
RowValue r1 = RowValue::Deserialize(dest.c_str(), r.Size()); |
||||
EXPECT_EQ(r1.Size(), columns_data_size + 12); |
||||
EXPECT_EQ(r1.IsTombstone(), false); |
||||
EXPECT_EQ(r1.LastModifiedTime(), last_modified_time); |
||||
|
||||
r1.Serialize(&dest); |
||||
EXPECT_EQ(dest.size(), 2 * r.Size()); |
||||
EXPECT_TRUE( |
||||
std::memcmp(dest.c_str(), dest.c_str() + r.Size(), r.Size()) == 0); |
||||
} |
||||
|
||||
} // namespace cassandra
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) { |
||||
::testing::InitGoogleTest(&argc, argv); |
||||
return RUN_ALL_TESTS(); |
||||
} |
||||
@ -0,0 +1,103 @@ |
||||
// Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
// This source code is also licensed under the GPLv2 license found in the
|
||||
// COPYING file in the root directory of this source tree.
|
||||
|
||||
#include "merge_operator.h" |
||||
|
||||
#include <memory> |
||||
#include <assert.h> |
||||
|
||||
#include "rocksdb/slice.h" |
||||
#include "rocksdb/merge_operator.h" |
||||
#include "utilities/merge_operators.h" |
||||
#include "utilities/merge_operators/cassandra/format.h" |
||||
|
||||
namespace rocksdb { |
||||
namespace cassandra { |
||||
|
||||
// Implementation for the merge operation (merges two Cassandra values)
|
||||
bool CassandraValueMergeOperator::FullMergeV2( |
||||
const MergeOperationInput& merge_in, |
||||
MergeOperationOutput* merge_out) const { |
||||
// Clear the *new_value for writing.
|
||||
merge_out->new_value.clear(); |
||||
|
||||
if (merge_in.existing_value == nullptr && merge_in.operand_list.size() == 1) { |
||||
// Only one operand
|
||||
merge_out->existing_operand = merge_in.operand_list.back(); |
||||
return true; |
||||
} |
||||
|
||||
std::vector<RowValue> row_values; |
||||
if (merge_in.existing_value) { |
||||
row_values.push_back( |
||||
RowValue::Deserialize(merge_in.existing_value->data(), |
||||
merge_in.existing_value->size())); |
||||
} |
||||
|
||||
for (auto& operand : merge_in.operand_list) { |
||||
row_values.push_back(RowValue::Deserialize(operand.data(), operand.size())); |
||||
} |
||||
|
||||
RowValue merged = RowValue::Merge(std::move(row_values)); |
||||
merge_out->new_value.reserve(merged.Size()); |
||||
merged.Serialize(&(merge_out->new_value)); |
||||
|
||||
return true; |
||||
} |
||||
|
||||
// Implementation for the merge operation (merges two Cassandra values)
|
||||
bool CassandraValueMergeOperator::PartialMerge(const Slice& key, |
||||
const Slice& left_operand, |
||||
const Slice& right_operand, |
||||
std::string* new_value, |
||||
Logger* logger) const { |
||||
// Clear the *new_value for writing.
|
||||
assert(new_value); |
||||
new_value->clear(); |
||||
|
||||
std::vector<RowValue> row_values; |
||||
row_values.push_back(RowValue::Deserialize(left_operand.data(), |
||||
left_operand.size())); |
||||
row_values.push_back(RowValue::Deserialize(right_operand.data(), |
||||
right_operand.size())); |
||||
RowValue merged = RowValue::Merge(std::move(row_values)); |
||||
new_value->reserve(merged.Size()); |
||||
merged.Serialize(new_value); |
||||
return true; |
||||
} |
||||
|
||||
bool CassandraValueMergeOperator::PartialMergeMulti( |
||||
const Slice& key, |
||||
const std::deque<Slice>& operand_list, |
||||
std::string* new_value, |
||||
Logger* logger) const { |
||||
// Clear the *new_value for writing.
|
||||
assert(new_value); |
||||
new_value->clear(); |
||||
|
||||
std::vector<RowValue> row_values; |
||||
for (auto& operand : operand_list) { |
||||
row_values.push_back(RowValue::Deserialize(operand.data(), operand.size())); |
||||
} |
||||
RowValue merged = RowValue::Merge(std::move(row_values)); |
||||
new_value->reserve(merged.Size()); |
||||
merged.Serialize(new_value); |
||||
return true; |
||||
} |
||||
|
||||
const char* CassandraValueMergeOperator::Name() const { |
||||
return "CassandraValueMergeOperator"; |
||||
} |
||||
|
||||
} // namespace cassandra
|
||||
|
||||
std::shared_ptr<MergeOperator> |
||||
MergeOperators::CreateCassandraMergeOperator() { |
||||
return std::make_shared<rocksdb::cassandra::CassandraValueMergeOperator>(); |
||||
} |
||||
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,39 @@ |
||||
// Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
// This source code is also licensed under the GPLv2 license found in the
|
||||
// COPYING file in the root directory of this source tree.
|
||||
|
||||
#pragma once |
||||
#include "rocksdb/merge_operator.h" |
||||
#include "rocksdb/slice.h" |
||||
|
||||
namespace rocksdb { |
||||
namespace cassandra { |
||||
|
||||
/**
|
||||
* A MergeOperator for rocksdb that implements Cassandra row value merge. |
||||
*/ |
||||
class CassandraValueMergeOperator : public MergeOperator { |
||||
public: |
||||
static std::shared_ptr<MergeOperator> CreateSharedInstance(); |
||||
|
||||
virtual bool FullMergeV2(const MergeOperationInput& merge_in, |
||||
MergeOperationOutput* merge_out) const override; |
||||
|
||||
virtual bool PartialMerge(const Slice& key, |
||||
const Slice& left_operand, |
||||
const Slice& right_operand, |
||||
std::string* new_value, |
||||
Logger* logger) const override; |
||||
|
||||
virtual bool PartialMergeMulti(const Slice& key, |
||||
const std::deque<Slice>& operand_list, |
||||
std::string* new_value, |
||||
Logger* logger) const override; |
||||
|
||||
virtual const char* Name() const override; |
||||
}; |
||||
} // namespace cassandra
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,114 @@ |
||||
// Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
// This source code is also licensed under the GPLv2 license found in the
|
||||
// COPYING file in the root directory of this source tree.
|
||||
|
||||
#include <memory> |
||||
#include "util/testharness.h" |
||||
#include "utilities/merge_operators/cassandra/format.h" |
||||
#include "utilities/merge_operators/cassandra/test_utils.h" |
||||
|
||||
namespace rocksdb { |
||||
namespace cassandra { |
||||
|
||||
TEST(RowValueMergeTest, Merge) { |
||||
std::vector<RowValue> row_values; |
||||
row_values.push_back( |
||||
CreateTestRowValue({ |
||||
std::make_tuple(kTombstone, 0, 5), |
||||
std::make_tuple(kColumn, 1, 8), |
||||
std::make_tuple(kExpiringColumn, 2, 5), |
||||
}) |
||||
); |
||||
|
||||
row_values.push_back( |
||||
CreateTestRowValue({ |
||||
std::make_tuple(kColumn, 0, 2), |
||||
std::make_tuple(kExpiringColumn, 1, 5), |
||||
std::make_tuple(kTombstone, 2, 7), |
||||
std::make_tuple(kExpiringColumn, 7, 17), |
||||
}) |
||||
); |
||||
|
||||
row_values.push_back( |
||||
CreateTestRowValue({ |
||||
std::make_tuple(kExpiringColumn, 0, 6), |
||||
std::make_tuple(kTombstone, 1, 5), |
||||
std::make_tuple(kColumn, 2, 4), |
||||
std::make_tuple(kTombstone, 11, 11), |
||||
}) |
||||
); |
||||
|
||||
RowValue merged = RowValue::Merge(std::move(row_values)); |
||||
EXPECT_FALSE(merged.IsTombstone()); |
||||
EXPECT_EQ(merged.columns_.size(), 5); |
||||
VerifyRowValueColumns(merged.columns_, 0, kExpiringColumn, 0, 6); |
||||
VerifyRowValueColumns(merged.columns_, 1, kColumn, 1, 8); |
||||
VerifyRowValueColumns(merged.columns_, 2, kTombstone, 2, 7); |
||||
VerifyRowValueColumns(merged.columns_, 3, kExpiringColumn, 7, 17); |
||||
VerifyRowValueColumns(merged.columns_, 4, kTombstone, 11, 11); |
||||
} |
||||
|
||||
TEST(RowValueMergeTest, MergeWithRowTombstone) { |
||||
std::vector<RowValue> row_values; |
||||
|
||||
// A row tombstone.
|
||||
row_values.push_back( |
||||
CreateRowTombstone(11) |
||||
); |
||||
|
||||
// This row's timestamp is smaller than tombstone.
|
||||
row_values.push_back( |
||||
CreateTestRowValue({ |
||||
std::make_tuple(kColumn, 0, 5), |
||||
std::make_tuple(kColumn, 1, 6), |
||||
}) |
||||
); |
||||
|
||||
// Some of the column's row is smaller, some is larger.
|
||||
row_values.push_back( |
||||
CreateTestRowValue({ |
||||
std::make_tuple(kColumn, 2, 10), |
||||
std::make_tuple(kColumn, 3, 12), |
||||
}) |
||||
); |
||||
|
||||
// All of the column's rows are larger than tombstone.
|
||||
row_values.push_back( |
||||
CreateTestRowValue({ |
||||
std::make_tuple(kColumn, 4, 13), |
||||
std::make_tuple(kColumn, 5, 14), |
||||
}) |
||||
); |
||||
|
||||
RowValue merged = RowValue::Merge(std::move(row_values)); |
||||
EXPECT_FALSE(merged.IsTombstone()); |
||||
EXPECT_EQ(merged.columns_.size(), 3); |
||||
VerifyRowValueColumns(merged.columns_, 0, kColumn, 3, 12); |
||||
VerifyRowValueColumns(merged.columns_, 1, kColumn, 4, 13); |
||||
VerifyRowValueColumns(merged.columns_, 2, kColumn, 5, 14); |
||||
|
||||
// If the tombstone's timestamp is the latest, then it returns a
|
||||
// row tombstone.
|
||||
row_values.push_back( |
||||
CreateRowTombstone(15) |
||||
); |
||||
|
||||
row_values.push_back( |
||||
CreateRowTombstone(17) |
||||
); |
||||
|
||||
merged = RowValue::Merge(std::move(row_values)); |
||||
EXPECT_TRUE(merged.IsTombstone()); |
||||
EXPECT_EQ(merged.LastModifiedTime(), 17); |
||||
} |
||||
|
||||
} // namespace cassandra
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) { |
||||
::testing::InitGoogleTest(&argc, argv); |
||||
return RUN_ALL_TESTS(); |
||||
} |
||||
@ -0,0 +1,77 @@ |
||||
// Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
// This source code is also licensed under the GPLv2 license found in the
|
||||
// COPYING file in the root directory of this source tree.
|
||||
|
||||
/**
|
||||
* Helper functions which serialize and deserialize integers |
||||
* into bytes in big endian. |
||||
*/ |
||||
|
||||
#pragma once |
||||
|
||||
namespace rocksdb { |
||||
namespace cassandra { |
||||
namespace { |
||||
const int64_t kCharMask = 0xFFLL; |
||||
const int32_t kBitsPerByte = 8; |
||||
} |
||||
|
||||
template<typename T> |
||||
void Serialize(T val, std::string* dest); |
||||
|
||||
template<typename T> |
||||
T Deserialize(const char* src, std::size_t offset=0); |
||||
|
||||
// Specializations
|
||||
template<> |
||||
inline void Serialize<int8_t>(int8_t t, std::string* dest) { |
||||
dest->append(1, static_cast<char>(t & kCharMask)); |
||||
} |
||||
|
||||
template<> |
||||
inline void Serialize<int32_t>(int32_t t, std::string* dest) { |
||||
for (unsigned long i = 0; i < sizeof(int32_t); i++) { |
||||
dest->append(1, static_cast<char>( |
||||
(t >> (sizeof(int32_t) - 1 - i) * kBitsPerByte) & kCharMask)); |
||||
} |
||||
} |
||||
|
||||
template<> |
||||
inline void Serialize<int64_t>(int64_t t, std::string* dest) { |
||||
for (unsigned long i = 0; i < sizeof(int64_t); i++) { |
||||
dest->append( |
||||
1, static_cast<char>( |
||||
(t >> (sizeof(int64_t) - 1 - i) * kBitsPerByte) & kCharMask)); |
||||
} |
||||
} |
||||
|
||||
template<> |
||||
inline int8_t Deserialize<int8_t>(const char* src, std::size_t offset) { |
||||
return static_cast<int8_t>(src[offset]); |
||||
} |
||||
|
||||
template<> |
||||
inline int32_t Deserialize<int32_t>(const char* src, std::size_t offset) { |
||||
int32_t result = 0; |
||||
for (unsigned long i = 0; i < sizeof(int32_t); i++) { |
||||
result |= static_cast<int32_t>(static_cast<unsigned char>(src[offset + i])) |
||||
<< ((sizeof(int32_t) - 1 - i) * kBitsPerByte); |
||||
} |
||||
return result; |
||||
} |
||||
|
||||
template<> |
||||
inline int64_t Deserialize<int64_t>(const char* src, std::size_t offset) { |
||||
int64_t result = 0; |
||||
for (unsigned long i = 0; i < sizeof(int64_t); i++) { |
||||
result |= static_cast<int64_t>(static_cast<unsigned char>(src[offset + i])) |
||||
<< ((sizeof(int64_t) - 1 - i) * kBitsPerByte); |
||||
} |
||||
return result; |
||||
} |
||||
|
||||
} // namepsace cassandrda
|
||||
} // namespace rocksdb
|
||||
@ -0,0 +1,190 @@ |
||||
// Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
// This source code is also licensed under the GPLv2 license found in the
|
||||
// COPYING file in the root directory of this source tree.
|
||||
|
||||
#include "util/testharness.h" |
||||
#include "utilities/merge_operators/cassandra/serialize.h" |
||||
|
||||
using namespace rocksdb::cassandra; |
||||
|
||||
namespace rocksdb { |
||||
namespace cassandra { |
||||
|
||||
TEST(SerializeTest, SerializeI64) { |
||||
std::string dest; |
||||
Serialize<int64_t>(0, &dest); |
||||
EXPECT_EQ( |
||||
std::string( |
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00'}), |
||||
dest); |
||||
|
||||
dest.clear(); |
||||
Serialize<int64_t>(1, &dest); |
||||
EXPECT_EQ( |
||||
std::string( |
||||
{'\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x01'}), |
||||
dest); |
||||
|
||||
|
||||
dest.clear(); |
||||
Serialize<int64_t>(-1, &dest); |
||||
EXPECT_EQ( |
||||
std::string( |
||||
{'\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff'}), |
||||
dest); |
||||
|
||||
dest.clear(); |
||||
Serialize<int64_t>(9223372036854775807, &dest); |
||||
EXPECT_EQ( |
||||
std::string( |
||||
{'\x7f', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff', '\xff'}), |
||||
dest); |
||||
|
||||
dest.clear(); |
||||
Serialize<int64_t>(-9223372036854775807, &dest); |
||||
EXPECT_EQ( |
||||
std::string( |
||||
{'\x80', '\x00', '\x00', '\x00', '\x00', '\x00', '\x00', '\x01'}), |
||||
dest); |
||||
} |
||||
|
||||
TEST(SerializeTest, DeserializeI64) { |
||||
std::string dest; |
||||
std::size_t offset = dest.size(); |
||||
Serialize<int64_t>(0, &dest); |
||||
EXPECT_EQ(0, Deserialize<int64_t>(dest.c_str(), offset)); |
||||
|
||||
offset = dest.size(); |
||||
Serialize<int64_t>(1, &dest); |
||||
EXPECT_EQ(1, Deserialize<int64_t>(dest.c_str(), offset)); |
||||
|
||||
offset = dest.size(); |
||||
Serialize<int64_t>(-1, &dest); |
||||
EXPECT_EQ(-1, Deserialize<int64_t>(dest.c_str(), offset)); |
||||
|
||||
offset = dest.size(); |
||||
Serialize<int64_t>(-9223372036854775807, &dest); |
||||
EXPECT_EQ(-9223372036854775807, Deserialize<int64_t>(dest.c_str(), offset)); |
||||
|
||||
offset = dest.size(); |
||||
Serialize<int64_t>(9223372036854775807, &dest); |
||||
EXPECT_EQ(9223372036854775807, Deserialize<int64_t>(dest.c_str(), offset)); |
||||
} |
||||
|
||||
TEST(SerializeTest, SerializeI32) { |
||||
std::string dest; |
||||
Serialize<int32_t>(0, &dest); |
||||
EXPECT_EQ( |
||||
std::string( |
||||
{'\x00', '\x00', '\x00', '\x00'}), |
||||
dest); |
||||
|
||||
dest.clear(); |
||||
Serialize<int32_t>(1, &dest); |
||||
EXPECT_EQ( |
||||
std::string( |
||||
{'\x00', '\x00', '\x00', '\x01'}), |
||||
dest); |
||||
|
||||
|
||||
dest.clear(); |
||||
Serialize<int32_t>(-1, &dest); |
||||
EXPECT_EQ( |
||||
std::string( |
||||
{'\xff', '\xff', '\xff', '\xff'}), |
||||
dest); |
||||
|
||||
dest.clear(); |
||||
Serialize<int32_t>(2147483647, &dest); |
||||
EXPECT_EQ( |
||||
std::string( |
||||
{'\x7f', '\xff', '\xff', '\xff'}), |
||||
dest); |
||||
|
||||
dest.clear(); |
||||
Serialize<int32_t>(-2147483648LL, &dest); |
||||
EXPECT_EQ( |
||||
std::string( |
||||
{'\x80', '\x00', '\x00', '\x00'}), |
||||
dest); |
||||
} |
||||
|
||||
TEST(SerializeTest, DeserializeI32) { |
||||
std::string dest; |
||||
std::size_t offset = dest.size(); |
||||
Serialize<int32_t>(0, &dest); |
||||
EXPECT_EQ(0, Deserialize<int32_t>(dest.c_str(), offset)); |
||||
|
||||
offset = dest.size(); |
||||
Serialize<int32_t>(1, &dest); |
||||
EXPECT_EQ(1, Deserialize<int32_t>(dest.c_str(), offset)); |
||||
|
||||
offset = dest.size(); |
||||
Serialize<int32_t>(-1, &dest); |
||||
EXPECT_EQ(-1, Deserialize<int32_t>(dest.c_str(), offset)); |
||||
|
||||
offset = dest.size(); |
||||
Serialize<int32_t>(2147483647, &dest); |
||||
EXPECT_EQ(2147483647, Deserialize<int32_t>(dest.c_str(), offset)); |
||||
|
||||
offset = dest.size(); |
||||
Serialize<int32_t>(-2147483648LL, &dest); |
||||
EXPECT_EQ(-2147483648LL, Deserialize<int32_t>(dest.c_str(), offset)); |
||||
} |
||||
|
||||
TEST(SerializeTest, SerializeI8) { |
||||
std::string dest; |
||||
Serialize<int8_t>(0, &dest); |
||||
EXPECT_EQ(std::string({'\x00'}), dest); |
||||
|
||||
dest.clear(); |
||||
Serialize<int8_t>(1, &dest); |
||||
EXPECT_EQ(std::string({'\x01'}), dest); |
||||
|
||||
|
||||
dest.clear(); |
||||
Serialize<int8_t>(-1, &dest); |
||||
EXPECT_EQ(std::string({'\xff'}), dest); |
||||
|
||||
dest.clear(); |
||||
Serialize<int8_t>(127, &dest); |
||||
EXPECT_EQ(std::string({'\x7f'}), dest); |
||||
|
||||
dest.clear(); |
||||
Serialize<int8_t>(-128, &dest); |
||||
EXPECT_EQ(std::string({'\x80'}), dest); |
||||
} |
||||
|
||||
TEST(SerializeTest, DeserializeI8) { |
||||
std::string dest; |
||||
std::size_t offset = dest.size(); |
||||
Serialize<int8_t>(0, &dest); |
||||
EXPECT_EQ(0, Deserialize<int8_t>(dest.c_str(), offset)); |
||||
|
||||
offset = dest.size(); |
||||
Serialize<int8_t>(1, &dest); |
||||
EXPECT_EQ(1, Deserialize<int8_t>(dest.c_str(), offset)); |
||||
|
||||
offset = dest.size(); |
||||
Serialize<int8_t>(-1, &dest); |
||||
EXPECT_EQ(-1, Deserialize<int8_t>(dest.c_str(), offset)); |
||||
|
||||
offset = dest.size(); |
||||
Serialize<int8_t>(127, &dest); |
||||
EXPECT_EQ(127, Deserialize<int8_t>(dest.c_str(), offset)); |
||||
|
||||
offset = dest.size(); |
||||
Serialize<int8_t>(-128, &dest); |
||||
EXPECT_EQ(-128, Deserialize<int8_t>(dest.c_str(), offset)); |
||||
} |
||||
|
||||
} // namespace cassandra
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) { |
||||
::testing::InitGoogleTest(&argc, argv); |
||||
return RUN_ALL_TESTS(); |
||||
} |
||||
@ -0,0 +1,65 @@ |
||||
// Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
// This source code is also licensed under the GPLv2 license found in the
|
||||
// COPYING file in the root directory of this source tree.
|
||||
|
||||
#include "test_utils.h" |
||||
|
||||
namespace rocksdb { |
||||
namespace cassandra { |
||||
const char kData[] = {'d', 'a', 't', 'a'}; |
||||
const char kExpiringData[] = {'e', 'd', 'a', 't', 'a'}; |
||||
const int32_t kLocalDeletionTime = 1; |
||||
const int32_t kTtl = 100; |
||||
const int8_t kColumn = 0; |
||||
const int8_t kTombstone = 1; |
||||
const int8_t kExpiringColumn = 2; |
||||
|
||||
std::unique_ptr<ColumnBase> CreateTestColumn(int8_t mask, |
||||
int8_t index, |
||||
int64_t timestamp) { |
||||
if ((mask & ColumnTypeMask::DELETION_MASK) != 0) { |
||||
return std::unique_ptr<Tombstone>(new Tombstone( |
||||
mask, index, kLocalDeletionTime, timestamp)); |
||||
} else if ((mask & ColumnTypeMask::EXPIRATION_MASK) != 0) { |
||||
return std::unique_ptr<ExpiringColumn>(new ExpiringColumn( |
||||
mask, index, timestamp, sizeof(kExpiringData), kExpiringData, kTtl)); |
||||
} else { |
||||
return std::unique_ptr<Column>( |
||||
new Column(mask, index, timestamp, sizeof(kData), kData)); |
||||
} |
||||
} |
||||
|
||||
RowValue CreateTestRowValue( |
||||
std::vector<std::tuple<int8_t, int8_t, int64_t>> column_specs) { |
||||
std::vector<std::unique_ptr<ColumnBase>> columns; |
||||
int64_t last_modified_time = 0; |
||||
for (auto spec: column_specs) { |
||||
auto c = CreateTestColumn(std::get<0>(spec), std::get<1>(spec), |
||||
std::get<2>(spec)); |
||||
last_modified_time = std::max(last_modified_time, c -> Timestamp()); |
||||
columns.push_back(std::move(c)); |
||||
} |
||||
return RowValue(std::move(columns), last_modified_time); |
||||
} |
||||
|
||||
RowValue CreateRowTombstone(int64_t timestamp) { |
||||
return RowValue(kLocalDeletionTime, timestamp); |
||||
} |
||||
|
||||
void VerifyRowValueColumns( |
||||
std::vector<std::unique_ptr<ColumnBase>> &columns, |
||||
std::size_t index_of_vector, |
||||
int8_t expected_mask, |
||||
int8_t expected_index, |
||||
int64_t expected_timestamp |
||||
) { |
||||
EXPECT_EQ(expected_timestamp, columns[index_of_vector]->Timestamp()); |
||||
EXPECT_EQ(expected_mask, columns[index_of_vector]->Mask()); |
||||
EXPECT_EQ(expected_index, columns[index_of_vector]->Index()); |
||||
} |
||||
|
||||
} |
||||
} |
||||
@ -0,0 +1,43 @@ |
||||
// Copyright (c) 2017-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under the BSD-style license found in the
|
||||
// LICENSE file in the root directory of this source tree. An additional grant
|
||||
// of patent rights can be found in the PATENTS file in the same directory.
|
||||
// This source code is also licensed under the GPLv2 license found in the
|
||||
// COPYING file in the root directory of this source tree.
|
||||
|
||||
#pragma once |
||||
#include <memory> |
||||
#include "util/testharness.h" |
||||
#include "utilities/merge_operators/cassandra/format.h" |
||||
#include "utilities/merge_operators/cassandra/serialize.h" |
||||
|
||||
namespace rocksdb { |
||||
namespace cassandra { |
||||
extern const char kData[]; |
||||
extern const char kExpiringData[]; |
||||
extern const int32_t kLocalDeletionTime; |
||||
extern const int32_t kTtl; |
||||
extern const int8_t kColumn; |
||||
extern const int8_t kTombstone; |
||||
extern const int8_t kExpiringColumn; |
||||
|
||||
|
||||
std::unique_ptr<ColumnBase> CreateTestColumn(int8_t mask, |
||||
int8_t index, |
||||
int64_t timestamp); |
||||
|
||||
RowValue CreateTestRowValue( |
||||
std::vector<std::tuple<int8_t, int8_t, int64_t>> column_specs); |
||||
|
||||
RowValue CreateRowTombstone(int64_t timestamp); |
||||
|
||||
void VerifyRowValueColumns( |
||||
std::vector<std::unique_ptr<ColumnBase>> &columns, |
||||
std::size_t index_of_vector, |
||||
int8_t expected_mask, |
||||
int8_t expected_index, |
||||
int64_t expected_timestamp |
||||
); |
||||
|
||||
} |
||||
} |
||||
Loading…
Reference in new issue