Summary: Experiments on column-aware encodings. Supported features: 1) extract data blocks from SST file and encode with specified encodings; 2) Decode encoded data back into row format; 3) Directly extract data blocks and write in row format (without prefix encoding); 4) Get column distribution statistics for column format; 5) Dump data blocks separated by columns in human-readable format. There is still on-going work on this diff. More refactoring is necessary. Test Plan: Wrote tests in `column_aware_encoding_test.cc`. More tests should be added. Reviewers: sdong Reviewed By: sdong Subscribers: arahut, andrewkr, dhruba Differential Revision: https://reviews.facebook.net/D60027main
omegaga
9 years ago
parent
c116b47804
commit
d51dc96a79
@ -0,0 +1,240 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#include "utilities/col_buf_decoder.h" |
||||||
|
#include <cstring> |
||||||
|
#include <string> |
||||||
|
#include "port/port.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
ColBufDecoder::~ColBufDecoder() {} |
||||||
|
|
||||||
|
namespace { |
||||||
|
|
||||||
|
inline uint64_t EncodeFixed64WithEndian(uint64_t val, bool big_endian) { |
||||||
|
if (big_endian && port::kLittleEndian) { |
||||||
|
val = le64toh(val); |
||||||
|
val = htobe64(val); |
||||||
|
} else if (!big_endian && !port::kLittleEndian) { |
||||||
|
val = be64toh(val); |
||||||
|
val = htole64(val); |
||||||
|
} |
||||||
|
return val; |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace
|
||||||
|
ColBufDecoder* ColBufDecoder::NewColBufDecoder( |
||||||
|
const ColDeclaration& col_declaration) { |
||||||
|
if (col_declaration.col_type == "FixedLength") { |
||||||
|
return new FixedLengthColBufDecoder( |
||||||
|
col_declaration.size, col_declaration.col_compression_type, |
||||||
|
col_declaration.nullable, col_declaration.big_endian); |
||||||
|
} else if (col_declaration.col_type == "VariableLength") { |
||||||
|
return new VariableLengthColBufDecoder(); |
||||||
|
} else if (col_declaration.col_type == "VariableChunk") { |
||||||
|
return new VariableChunkColBufDecoder(col_declaration.col_compression_type); |
||||||
|
} else if (col_declaration.col_type == "LongFixedLength") { |
||||||
|
return new LongFixedLengthColBufDecoder(col_declaration.size, |
||||||
|
col_declaration.nullable); |
||||||
|
} |
||||||
|
// Unrecognized column type
|
||||||
|
return nullptr; |
||||||
|
} |
||||||
|
|
||||||
|
namespace { |
||||||
|
|
||||||
|
void ReadVarint64(const char** src_ptr, uint64_t* val_ptr) { |
||||||
|
const char* q = GetVarint64Ptr(*src_ptr, *src_ptr + 10, val_ptr); |
||||||
|
assert(q != nullptr); |
||||||
|
*src_ptr = q; |
||||||
|
} |
||||||
|
} // namespace
|
||||||
|
|
||||||
|
size_t FixedLengthColBufDecoder::Init(const char* src) { |
||||||
|
remain_runs_ = 0; |
||||||
|
last_val_ = 0; |
||||||
|
// Dictionary initialization
|
||||||
|
dict_vec_.clear(); |
||||||
|
const char* orig_src = src; |
||||||
|
if (col_compression_type_ == kColDict || |
||||||
|
col_compression_type_ == kColRleDict) { |
||||||
|
const char* q; |
||||||
|
size_t dict_size; |
||||||
|
// Bypass limit
|
||||||
|
q = GetVarint64Ptr(src, src + 10, &dict_size); |
||||||
|
assert(q != nullptr); |
||||||
|
src = q; |
||||||
|
|
||||||
|
uint64_t dict_key; |
||||||
|
for (size_t i = 0; i < dict_size; ++i) { |
||||||
|
// Bypass limit
|
||||||
|
ReadVarint64(&src, &dict_key); |
||||||
|
|
||||||
|
dict_key = EncodeFixed64WithEndian(dict_key, big_endian_); |
||||||
|
dict_vec_.push_back(dict_key); |
||||||
|
} |
||||||
|
} |
||||||
|
return src - orig_src; |
||||||
|
} |
||||||
|
|
||||||
|
size_t FixedLengthColBufDecoder::Decode(const char* src, char** dest) { |
||||||
|
uint64_t read_val = 0; |
||||||
|
const char* orig_src = src; |
||||||
|
const char* src_limit = src + 20; |
||||||
|
if (nullable_) { |
||||||
|
bool not_null; |
||||||
|
not_null = *src; |
||||||
|
src += 1; |
||||||
|
if (!not_null) { |
||||||
|
return 1; |
||||||
|
} |
||||||
|
} |
||||||
|
if (IsRunLength(col_compression_type_)) { |
||||||
|
if (remain_runs_ == 0) { |
||||||
|
const char* q; |
||||||
|
run_val_ = 0; |
||||||
|
if (col_compression_type_ == kColRle) { |
||||||
|
memcpy(&run_val_, src, size_); |
||||||
|
src += size_; |
||||||
|
} else { |
||||||
|
q = GetVarint64Ptr(src, src_limit, &run_val_); |
||||||
|
assert(q != nullptr); |
||||||
|
src = q; |
||||||
|
} |
||||||
|
|
||||||
|
q = GetVarint64Ptr(src, src_limit, &remain_runs_); |
||||||
|
assert(q != nullptr); |
||||||
|
src = q; |
||||||
|
|
||||||
|
if (col_compression_type_ != kColRleDeltaVarint && |
||||||
|
col_compression_type_ != kColRleDict) { |
||||||
|
run_val_ = EncodeFixed64WithEndian(run_val_, big_endian_); |
||||||
|
} |
||||||
|
} |
||||||
|
read_val = run_val_; |
||||||
|
} else { |
||||||
|
if (col_compression_type_ == kColNoCompression) { |
||||||
|
memcpy(&read_val, src, size_); |
||||||
|
src += size_; |
||||||
|
} else { |
||||||
|
// Assume a column does not exceed 8 bytes here
|
||||||
|
const char* q = GetVarint64Ptr(src, src_limit, &read_val); |
||||||
|
assert(q != nullptr); |
||||||
|
src = q; |
||||||
|
} |
||||||
|
if (col_compression_type_ != kColDeltaVarint && |
||||||
|
col_compression_type_ != kColDict) { |
||||||
|
read_val = EncodeFixed64WithEndian(read_val, big_endian_); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
uint64_t write_val = read_val; |
||||||
|
if (col_compression_type_ == kColDeltaVarint || |
||||||
|
col_compression_type_ == kColRleDeltaVarint) { |
||||||
|
// does not support 64 bit
|
||||||
|
|
||||||
|
uint64_t mask = (write_val & 1) ? (~0UL) : 0; |
||||||
|
int64_t delta = (write_val >> 1) ^ mask; |
||||||
|
write_val = last_val_ + delta; |
||||||
|
|
||||||
|
uint64_t tmp = write_val; |
||||||
|
write_val = EncodeFixed64WithEndian(write_val, big_endian_); |
||||||
|
last_val_ = tmp; |
||||||
|
} else if (col_compression_type_ == kColRleDict || |
||||||
|
col_compression_type_ == kColDict) { |
||||||
|
size_t dict_val = read_val; |
||||||
|
assert(dict_val < dict_vec_.size()); |
||||||
|
write_val = dict_vec_[dict_val]; |
||||||
|
} |
||||||
|
|
||||||
|
// dest->append(reinterpret_cast<char*>(&write_val), size_);
|
||||||
|
memcpy(*dest, reinterpret_cast<char*>(&write_val), size_); |
||||||
|
*dest += size_; |
||||||
|
if (IsRunLength(col_compression_type_)) { |
||||||
|
--remain_runs_; |
||||||
|
} |
||||||
|
return src - orig_src; |
||||||
|
} |
||||||
|
|
||||||
|
size_t LongFixedLengthColBufDecoder::Decode(const char* src, char** dest) { |
||||||
|
if (nullable_) { |
||||||
|
bool not_null; |
||||||
|
not_null = *src; |
||||||
|
src += 1; |
||||||
|
if (!not_null) { |
||||||
|
return 1; |
||||||
|
} |
||||||
|
} |
||||||
|
memcpy(*dest, src, size_); |
||||||
|
*dest += size_; |
||||||
|
return size_ + 1; |
||||||
|
} |
||||||
|
|
||||||
|
size_t VariableLengthColBufDecoder::Decode(const char* src, char** dest) { |
||||||
|
uint8_t len; |
||||||
|
len = *src; |
||||||
|
memcpy(dest, reinterpret_cast<char*>(&len), 1); |
||||||
|
*dest += 1; |
||||||
|
src += 1; |
||||||
|
memcpy(*dest, src, len); |
||||||
|
*dest += len; |
||||||
|
return len + 1; |
||||||
|
} |
||||||
|
|
||||||
|
size_t VariableChunkColBufDecoder::Init(const char* src) { |
||||||
|
// Dictionary initialization
|
||||||
|
dict_vec_.clear(); |
||||||
|
const char* orig_src = src; |
||||||
|
if (col_compression_type_ == kColDict) { |
||||||
|
const char* q; |
||||||
|
size_t dict_size; |
||||||
|
// Bypass limit
|
||||||
|
q = GetVarint64Ptr(src, src + 10, &dict_size); |
||||||
|
assert(q != nullptr); |
||||||
|
src = q; |
||||||
|
|
||||||
|
uint64_t dict_key; |
||||||
|
for (size_t i = 0; i < dict_size; ++i) { |
||||||
|
// Bypass limit
|
||||||
|
ReadVarint64(&src, &dict_key); |
||||||
|
dict_vec_.push_back(dict_key); |
||||||
|
} |
||||||
|
} |
||||||
|
return src - orig_src; |
||||||
|
} |
||||||
|
|
||||||
|
size_t VariableChunkColBufDecoder::Decode(const char* src, char** dest) { |
||||||
|
const char* orig_src = src; |
||||||
|
uint64_t size = 0; |
||||||
|
ReadVarint64(&src, &size); |
||||||
|
int64_t full_chunks = size / 8; |
||||||
|
uint64_t chunk_buf; |
||||||
|
size_t chunk_size = 8; |
||||||
|
for (int64_t i = 0; i < full_chunks + 1; ++i) { |
||||||
|
chunk_buf = 0; |
||||||
|
if (i == full_chunks) { |
||||||
|
chunk_size = size % 8; |
||||||
|
} |
||||||
|
if (col_compression_type_ == kColDict) { |
||||||
|
size_t dict_val; |
||||||
|
ReadVarint64(&src, &dict_val); |
||||||
|
assert(dict_val < dict_vec_.size()); |
||||||
|
chunk_buf = dict_vec_[dict_val]; |
||||||
|
} else { |
||||||
|
memcpy(&chunk_buf, src, chunk_size); |
||||||
|
src += chunk_size; |
||||||
|
} |
||||||
|
memcpy(*dest, reinterpret_cast<char*>(&chunk_buf), 8); |
||||||
|
*dest += 8; |
||||||
|
uint8_t mask = 0xFF - 8 + chunk_size; |
||||||
|
memcpy(*dest, reinterpret_cast<char*>(&mask), 1); |
||||||
|
*dest += 1; |
||||||
|
} |
||||||
|
|
||||||
|
return src - orig_src; |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,117 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
#include <endian.h> |
||||||
|
#include <cstdio> |
||||||
|
#include <cstring> |
||||||
|
#include <memory> |
||||||
|
#include <string> |
||||||
|
#include <unordered_map> |
||||||
|
#include <vector> |
||||||
|
#include "util/coding.h" |
||||||
|
#include "utilities/col_buf_encoder.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
struct ColDeclaration; |
||||||
|
|
||||||
|
// ColBufDecoder is a class to decode column buffers. It can be populated from a
|
||||||
|
// ColDeclaration. Before starting decoding, a Init() method should be called.
|
||||||
|
// Each time it takes a column value into Decode() method.
|
||||||
|
class ColBufDecoder { |
||||||
|
public: |
||||||
|
virtual ~ColBufDecoder() = 0; |
||||||
|
virtual size_t Init(const char* src) { return 0; } |
||||||
|
virtual size_t Decode(const char* src, char** dest) = 0; |
||||||
|
static ColBufDecoder* NewColBufDecoder(const ColDeclaration& col_declaration); |
||||||
|
|
||||||
|
protected: |
||||||
|
std::string buffer_; |
||||||
|
static inline bool IsRunLength(ColCompressionType type) { |
||||||
|
return type == kColRle || type == kColRleVarint || |
||||||
|
type == kColRleDeltaVarint || type == kColRleDict; |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
class FixedLengthColBufDecoder : public ColBufDecoder { |
||||||
|
public: |
||||||
|
explicit FixedLengthColBufDecoder( |
||||||
|
size_t size, ColCompressionType col_compression_type = kColNoCompression, |
||||||
|
bool nullable = false, bool big_endian = false) |
||||||
|
: size_(size), |
||||||
|
col_compression_type_(col_compression_type), |
||||||
|
nullable_(nullable), |
||||||
|
big_endian_(big_endian) {} |
||||||
|
|
||||||
|
size_t Init(const char* src) override; |
||||||
|
size_t Decode(const char* src, char** dest) override; |
||||||
|
~FixedLengthColBufDecoder() {} |
||||||
|
|
||||||
|
private: |
||||||
|
size_t size_; |
||||||
|
ColCompressionType col_compression_type_; |
||||||
|
bool nullable_; |
||||||
|
bool big_endian_; |
||||||
|
|
||||||
|
// for decoding
|
||||||
|
std::vector<uint64_t> dict_vec_; |
||||||
|
uint64_t remain_runs_; |
||||||
|
uint64_t run_val_; |
||||||
|
uint64_t last_val_; |
||||||
|
}; |
||||||
|
|
||||||
|
class LongFixedLengthColBufDecoder : public ColBufDecoder { |
||||||
|
public: |
||||||
|
LongFixedLengthColBufDecoder(size_t size, bool nullable) |
||||||
|
: size_(size), nullable_(nullable) {} |
||||||
|
|
||||||
|
size_t Decode(const char* src, char** dest) override; |
||||||
|
~LongFixedLengthColBufDecoder() {} |
||||||
|
|
||||||
|
private: |
||||||
|
size_t size_; |
||||||
|
bool nullable_; |
||||||
|
}; |
||||||
|
|
||||||
|
class VariableLengthColBufDecoder : public ColBufDecoder { |
||||||
|
public: |
||||||
|
size_t Decode(const char* src, char** dest) override; |
||||||
|
~VariableLengthColBufDecoder() {} |
||||||
|
}; |
||||||
|
|
||||||
|
class VariableChunkColBufDecoder : public VariableLengthColBufDecoder { |
||||||
|
public: |
||||||
|
size_t Init(const char* src) override; |
||||||
|
size_t Decode(const char* src, char** dest) override; |
||||||
|
explicit VariableChunkColBufDecoder(ColCompressionType col_compression_type) |
||||||
|
: col_compression_type_(col_compression_type) {} |
||||||
|
VariableChunkColBufDecoder() : col_compression_type_(kColNoCompression) {} |
||||||
|
|
||||||
|
private: |
||||||
|
ColCompressionType col_compression_type_; |
||||||
|
std::unordered_map<uint64_t, uint64_t> dictionary_; |
||||||
|
std::vector<uint64_t> dict_vec_; |
||||||
|
}; |
||||||
|
|
||||||
|
struct KVPairColBufDecoders { |
||||||
|
std::vector<std::unique_ptr<ColBufDecoder>> key_col_bufs; |
||||||
|
std::vector<std::unique_ptr<ColBufDecoder>> value_col_bufs; |
||||||
|
std::unique_ptr<ColBufDecoder> value_checksum_buf; |
||||||
|
|
||||||
|
explicit KVPairColBufDecoders(const KVPairColDeclarations& kvp_cd) { |
||||||
|
for (auto kcd : *kvp_cd.key_col_declarations) { |
||||||
|
key_col_bufs.emplace_back( |
||||||
|
std::move(ColBufDecoder::NewColBufDecoder(kcd))); |
||||||
|
} |
||||||
|
for (auto vcd : *kvp_cd.value_col_declarations) { |
||||||
|
value_col_bufs.emplace_back( |
||||||
|
std::move(ColBufDecoder::NewColBufDecoder(vcd))); |
||||||
|
} |
||||||
|
value_checksum_buf.reset( |
||||||
|
ColBufDecoder::NewColBufDecoder(*kvp_cd.value_checksum_declaration)); |
||||||
|
} |
||||||
|
}; |
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,213 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#include "utilities/col_buf_encoder.h" |
||||||
|
#include <cstring> |
||||||
|
#include <string> |
||||||
|
#include "port/port.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
ColBufEncoder::~ColBufEncoder() {} |
||||||
|
|
||||||
|
namespace { |
||||||
|
|
||||||
|
inline uint64_t DecodeFixed64WithEndian(uint64_t val, bool big_endian) { |
||||||
|
if (big_endian && port::kLittleEndian) { |
||||||
|
val = be64toh(val); |
||||||
|
val = htole64(val); |
||||||
|
} else if (!big_endian && !port::kLittleEndian) { |
||||||
|
val = le64toh(val); |
||||||
|
val = htobe64(val); |
||||||
|
} |
||||||
|
return val; |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
const std::string &ColBufEncoder::GetData() { return buffer_; } |
||||||
|
|
||||||
|
ColBufEncoder *ColBufEncoder::NewColBufEncoder( |
||||||
|
const ColDeclaration &col_declaration) { |
||||||
|
if (col_declaration.col_type == "FixedLength") { |
||||||
|
return new FixedLengthColBufEncoder( |
||||||
|
col_declaration.size, col_declaration.col_compression_type, |
||||||
|
col_declaration.nullable, col_declaration.big_endian); |
||||||
|
} else if (col_declaration.col_type == "VariableLength") { |
||||||
|
return new VariableLengthColBufEncoder(); |
||||||
|
} else if (col_declaration.col_type == "VariableChunk") { |
||||||
|
return new VariableChunkColBufEncoder(col_declaration.col_compression_type); |
||||||
|
} else if (col_declaration.col_type == "LongFixedLength") { |
||||||
|
return new LongFixedLengthColBufEncoder(col_declaration.size, |
||||||
|
col_declaration.nullable); |
||||||
|
} |
||||||
|
// Unrecognized column type
|
||||||
|
return nullptr; |
||||||
|
} |
||||||
|
|
||||||
|
size_t FixedLengthColBufEncoder::Append(const char *buf) { |
||||||
|
if (nullable_) { |
||||||
|
if (buf == nullptr) { |
||||||
|
buffer_.append(1, 0); |
||||||
|
return 0; |
||||||
|
} else { |
||||||
|
buffer_.append(1, 1); |
||||||
|
} |
||||||
|
} |
||||||
|
uint64_t read_val = 0; |
||||||
|
memcpy(&read_val, buf, size_); |
||||||
|
if (big_endian_) { |
||||||
|
read_val = DecodeFixed64WithEndian(read_val, big_endian_); |
||||||
|
} |
||||||
|
|
||||||
|
// Determine write value
|
||||||
|
uint64_t write_val = read_val; |
||||||
|
if (col_compression_type_ == kColDeltaVarint || |
||||||
|
col_compression_type_ == kColRleDeltaVarint) { |
||||||
|
int64_t delta = read_val - last_val_; |
||||||
|
// Encode signed delta value
|
||||||
|
delta = (delta << 1) ^ (delta >> 63); |
||||||
|
write_val = delta; |
||||||
|
last_val_ = read_val; |
||||||
|
} else if (col_compression_type_ == kColDict || |
||||||
|
col_compression_type_ == kColRleDict) { |
||||||
|
auto iter = dictionary_.find(read_val); |
||||||
|
uint64_t dict_val; |
||||||
|
if (iter == dictionary_.end()) { |
||||||
|
// Add new entry to dictionary
|
||||||
|
dict_val = dictionary_.size(); |
||||||
|
dictionary_.insert(std::make_pair(read_val, dict_val)); |
||||||
|
dict_vec_.push_back(read_val); |
||||||
|
} else { |
||||||
|
dict_val = iter->second; |
||||||
|
} |
||||||
|
write_val = dict_val; |
||||||
|
} |
||||||
|
|
||||||
|
// Write into buffer
|
||||||
|
if (IsRunLength(col_compression_type_)) { |
||||||
|
if (run_length_ == -1) { |
||||||
|
// First element
|
||||||
|
run_val_ = write_val; |
||||||
|
run_length_ = 1; |
||||||
|
} else if (write_val != run_val_) { |
||||||
|
// End of run
|
||||||
|
// Write run value
|
||||||
|
if (col_compression_type_ == kColRle) { |
||||||
|
buffer_.append(reinterpret_cast<char *>(&run_val_), size_); |
||||||
|
} else { |
||||||
|
PutVarint64(&buffer_, run_val_); |
||||||
|
} |
||||||
|
// Write run length
|
||||||
|
PutVarint64(&buffer_, run_length_); |
||||||
|
run_val_ = write_val; |
||||||
|
run_length_ = 1; |
||||||
|
} else { |
||||||
|
run_length_++; |
||||||
|
} |
||||||
|
} else { // non run-length encodings
|
||||||
|
if (col_compression_type_ == kColNoCompression) { |
||||||
|
buffer_.append(reinterpret_cast<char *>(&write_val), size_); |
||||||
|
} else { |
||||||
|
PutVarint64(&buffer_, write_val); |
||||||
|
} |
||||||
|
} |
||||||
|
return size_; |
||||||
|
} |
||||||
|
|
||||||
|
void FixedLengthColBufEncoder::Finish() { |
||||||
|
if (col_compression_type_ == kColDict || |
||||||
|
col_compression_type_ == kColRleDict) { |
||||||
|
std::string header; |
||||||
|
PutVarint64(&header, dict_vec_.size()); |
||||||
|
// Put dictionary in the header
|
||||||
|
for (auto item : dict_vec_) { |
||||||
|
PutVarint64(&header, item); |
||||||
|
} |
||||||
|
buffer_ = header + buffer_; |
||||||
|
} |
||||||
|
if (IsRunLength(col_compression_type_)) { |
||||||
|
// Finish last run value
|
||||||
|
if (col_compression_type_ == kColRle) { |
||||||
|
buffer_.append(reinterpret_cast<char *>(&run_val_), size_); |
||||||
|
} else { |
||||||
|
PutVarint64(&buffer_, run_val_); |
||||||
|
} |
||||||
|
PutVarint64(&buffer_, run_length_); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
size_t LongFixedLengthColBufEncoder::Append(const char *buf) { |
||||||
|
if (nullable_) { |
||||||
|
if (buf == nullptr) { |
||||||
|
buffer_.append(1, 0); |
||||||
|
return 0; |
||||||
|
} else { |
||||||
|
buffer_.append(1, 1); |
||||||
|
} |
||||||
|
} |
||||||
|
buffer_.append(buf, size_); |
||||||
|
return size_; |
||||||
|
} |
||||||
|
|
||||||
|
void LongFixedLengthColBufEncoder::Finish() {} |
||||||
|
|
||||||
|
size_t VariableLengthColBufEncoder::Append(const char *buf) { |
||||||
|
uint8_t length = 0; |
||||||
|
length = *buf; |
||||||
|
buffer_.append(buf, 1); |
||||||
|
buf += 1; |
||||||
|
buffer_.append(buf, length); |
||||||
|
return length + 1; |
||||||
|
} |
||||||
|
|
||||||
|
void VariableLengthColBufEncoder::Finish() {} |
||||||
|
|
||||||
|
size_t VariableChunkColBufEncoder::Append(const char *buf) { |
||||||
|
const char *orig_buf = buf; |
||||||
|
uint8_t mark = 0xFF; |
||||||
|
size_t length = 0; |
||||||
|
std::string tmp_buffer; |
||||||
|
while (mark == 0xFF) { |
||||||
|
uint64_t val; |
||||||
|
memcpy(&val, buf, 8); |
||||||
|
buf += 8; |
||||||
|
mark = *buf; |
||||||
|
buf += 1; |
||||||
|
int8_t chunk_size = 8 - (0xFF - mark); |
||||||
|
if (col_compression_type_ == kColDict) { |
||||||
|
auto iter = dictionary_.find(val); |
||||||
|
size_t dict_val; |
||||||
|
if (iter == dictionary_.end()) { |
||||||
|
dict_val = dictionary_.size(); |
||||||
|
dictionary_.insert(std::make_pair(val, dict_val)); |
||||||
|
dict_vec_.push_back(val); |
||||||
|
} else { |
||||||
|
dict_val = iter->second; |
||||||
|
} |
||||||
|
PutVarint64(&tmp_buffer, dict_val); |
||||||
|
} else { |
||||||
|
tmp_buffer.append(reinterpret_cast<char *>(&val), chunk_size); |
||||||
|
} |
||||||
|
length += chunk_size; |
||||||
|
} |
||||||
|
|
||||||
|
PutVarint64(&buffer_, length); |
||||||
|
buffer_.append(tmp_buffer); |
||||||
|
return buf - orig_buf; |
||||||
|
} |
||||||
|
|
||||||
|
void VariableChunkColBufEncoder::Finish() { |
||||||
|
if (col_compression_type_ == kColDict) { |
||||||
|
std::string header; |
||||||
|
PutVarint64(&header, dict_vec_.size()); |
||||||
|
for (auto item : dict_vec_) { |
||||||
|
PutVarint64(&header, item); |
||||||
|
} |
||||||
|
buffer_ = header + buffer_; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,220 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
#include <endian.h> |
||||||
|
#include <cstdio> |
||||||
|
#include <cstring> |
||||||
|
#include <memory> |
||||||
|
#include <string> |
||||||
|
#include <unordered_map> |
||||||
|
#include <vector> |
||||||
|
#include "util/coding.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
enum ColCompressionType { |
||||||
|
kColNoCompression, |
||||||
|
kColRle, |
||||||
|
kColVarint, |
||||||
|
kColRleVarint, |
||||||
|
kColDeltaVarint, |
||||||
|
kColRleDeltaVarint, |
||||||
|
kColDict, |
||||||
|
kColRleDict |
||||||
|
}; |
||||||
|
|
||||||
|
struct ColDeclaration; |
||||||
|
|
||||||
|
// ColBufEncoder is a class to encode column buffers. It can be populated from a
|
||||||
|
// ColDeclaration. Each time it takes a column value into Append() method to
|
||||||
|
// encode the column and store it into an internal buffer. After all rows for
|
||||||
|
// this column are consumed, a Finish() should be called to add header and
|
||||||
|
// remaining data.
|
||||||
|
class ColBufEncoder { |
||||||
|
public: |
||||||
|
// Read a column, encode data and append into internal buffer.
|
||||||
|
virtual size_t Append(const char *buf) = 0; |
||||||
|
virtual ~ColBufEncoder() = 0; |
||||||
|
// Get the internal column buffer. Should only be called after Finish().
|
||||||
|
const std::string &GetData(); |
||||||
|
// Finish encoding. Add header and remaining data.
|
||||||
|
virtual void Finish() = 0; |
||||||
|
// Populate a ColBufEncoder from ColDeclaration.
|
||||||
|
static ColBufEncoder *NewColBufEncoder(const ColDeclaration &col_declaration); |
||||||
|
|
||||||
|
protected: |
||||||
|
std::string buffer_; |
||||||
|
static inline bool IsRunLength(ColCompressionType type) { |
||||||
|
return type == kColRle || type == kColRleVarint || |
||||||
|
type == kColRleDeltaVarint || type == kColRleDict; |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
// Encoder for fixed length column buffer. In fixed length column buffer, the
|
||||||
|
// size of the column should not exceed 8 bytes.
|
||||||
|
// The following encodings are supported:
|
||||||
|
// Varint: Variable length integer. See util/coding.h for more details
|
||||||
|
// Rle (Run length encoding): encode a sequence of contiguous value as
|
||||||
|
// [run_value][run_length]. Can be combined with Varint
|
||||||
|
// Delta: Encode value to its delta with its adjacent entry. Use varint to
|
||||||
|
// possibly reduce stored bytes. Can be combined with Rle.
|
||||||
|
// Dictionary: Use a dictionary to record all possible values in the block and
|
||||||
|
// encode them with an ID started from 0. IDs are encoded as varint. A column
|
||||||
|
// with dictionary encoding will have a header to store all actual values,
|
||||||
|
// ordered by their dictionary value, and the data will be replaced by
|
||||||
|
// dictionary value. Can be combined with Rle.
|
||||||
|
class FixedLengthColBufEncoder : public ColBufEncoder { |
||||||
|
public: |
||||||
|
explicit FixedLengthColBufEncoder( |
||||||
|
size_t size, ColCompressionType col_compression_type = kColNoCompression, |
||||||
|
bool nullable = false, bool big_endian = false) |
||||||
|
: size_(size), |
||||||
|
col_compression_type_(col_compression_type), |
||||||
|
nullable_(nullable), |
||||||
|
big_endian_(big_endian), |
||||||
|
last_val_(0), |
||||||
|
run_length_(-1), |
||||||
|
run_val_(0) {} |
||||||
|
|
||||||
|
size_t Append(const char *buf) override; |
||||||
|
void Finish() override; |
||||||
|
~FixedLengthColBufEncoder() {} |
||||||
|
|
||||||
|
private: |
||||||
|
size_t size_; |
||||||
|
ColCompressionType col_compression_type_; |
||||||
|
// If set as true, the input value can be null (represented as nullptr). When
|
||||||
|
// nullable is true, use one more byte before actual value to indicate if the
|
||||||
|
// current value is null.
|
||||||
|
bool nullable_; |
||||||
|
// If set as true, input value will be treated as big endian encoded.
|
||||||
|
bool big_endian_; |
||||||
|
|
||||||
|
// for encoding
|
||||||
|
uint64_t last_val_; |
||||||
|
int16_t run_length_; |
||||||
|
uint64_t run_val_; |
||||||
|
// Map to store dictionary for dictionary encoding
|
||||||
|
std::unordered_map<uint64_t, uint64_t> dictionary_; |
||||||
|
// Vector of dictionary keys.
|
||||||
|
std::vector<uint64_t> dict_vec_; |
||||||
|
}; |
||||||
|
|
||||||
|
// Long fixed length column buffer is a variant of fixed length buffer to hold
|
||||||
|
// fixed length buffer with more than 8 bytes. We do not support any special
|
||||||
|
// encoding schemes in LongFixedLengthColBufEncoder.
|
||||||
|
class LongFixedLengthColBufEncoder : public ColBufEncoder { |
||||||
|
public: |
||||||
|
LongFixedLengthColBufEncoder(size_t size, bool nullable) |
||||||
|
: size_(size), nullable_(nullable) {} |
||||||
|
size_t Append(const char *buf) override; |
||||||
|
void Finish() override; |
||||||
|
|
||||||
|
~LongFixedLengthColBufEncoder() {} |
||||||
|
|
||||||
|
private: |
||||||
|
size_t size_; |
||||||
|
bool nullable_; |
||||||
|
}; |
||||||
|
|
||||||
|
// Variable length column buffer holds a format of variable length column. In
|
||||||
|
// this format, a column is composed of one byte length k, followed by data with
|
||||||
|
// k bytes long data.
|
||||||
|
class VariableLengthColBufEncoder : public ColBufEncoder { |
||||||
|
public: |
||||||
|
size_t Append(const char *buf) override; |
||||||
|
void Finish() override; |
||||||
|
|
||||||
|
~VariableLengthColBufEncoder() {} |
||||||
|
}; |
||||||
|
|
||||||
|
// Variable chunk column buffer holds another format of variable length column.
|
||||||
|
// In this format, a column contains multiple chunks of data, each of which is
|
||||||
|
// composed of 8 bytes long data, and one byte as a mask to indicate whether we
|
||||||
|
// have more data to come. If no more data coming, the mask is set as 0xFF. If
|
||||||
|
// the chunk is the last chunk and has only k valid bytes, the mask is set as
|
||||||
|
// 0xFF - (8 - k).
|
||||||
|
class VariableChunkColBufEncoder : public VariableLengthColBufEncoder { |
||||||
|
public: |
||||||
|
size_t Append(const char *buf) override; |
||||||
|
void Finish() override; |
||||||
|
explicit VariableChunkColBufEncoder(ColCompressionType col_compression_type) |
||||||
|
: col_compression_type_(col_compression_type) {} |
||||||
|
VariableChunkColBufEncoder() : col_compression_type_(kColNoCompression) {} |
||||||
|
|
||||||
|
private: |
||||||
|
ColCompressionType col_compression_type_; |
||||||
|
// Map to store dictionary for dictionary encoding
|
||||||
|
std::unordered_map<uint64_t, uint64_t> dictionary_; |
||||||
|
// Vector of dictionary keys.
|
||||||
|
std::vector<uint64_t> dict_vec_; |
||||||
|
}; |
||||||
|
|
||||||
|
// ColDeclaration declares a column's type, algorithm of column-aware encoding,
|
||||||
|
// and other column data like endian and nullability.
|
||||||
|
struct ColDeclaration { |
||||||
|
explicit ColDeclaration( |
||||||
|
std::string _col_type, |
||||||
|
ColCompressionType _col_compression_type = kColNoCompression, |
||||||
|
size_t _size = 0, bool _nullable = false, bool _big_endian = false) |
||||||
|
: col_type(_col_type), |
||||||
|
col_compression_type(_col_compression_type), |
||||||
|
size(_size), |
||||||
|
nullable(_nullable), |
||||||
|
big_endian(_big_endian) {} |
||||||
|
std::string col_type; |
||||||
|
ColCompressionType col_compression_type; |
||||||
|
size_t size; |
||||||
|
bool nullable; |
||||||
|
bool big_endian; |
||||||
|
}; |
||||||
|
|
||||||
|
// KVPairColDeclarations is a class to hold column declaration of columns in
|
||||||
|
// key and value.
|
||||||
|
struct KVPairColDeclarations { |
||||||
|
std::vector<ColDeclaration> *key_col_declarations; |
||||||
|
std::vector<ColDeclaration> *value_col_declarations; |
||||||
|
ColDeclaration *value_checksum_declaration; |
||||||
|
KVPairColDeclarations(std::vector<ColDeclaration> *_key_col_declarations, |
||||||
|
std::vector<ColDeclaration> *_value_col_declarations, |
||||||
|
ColDeclaration *_value_checksum_declaration) |
||||||
|
: key_col_declarations(_key_col_declarations), |
||||||
|
value_col_declarations(_value_col_declarations), |
||||||
|
value_checksum_declaration(_value_checksum_declaration) {} |
||||||
|
}; |
||||||
|
|
||||||
|
// Similar to KVPairDeclarations, KVPairColBufEncoders is used to hold column
|
||||||
|
// buffer encoders of all columns in key and value.
|
||||||
|
struct KVPairColBufEncoders { |
||||||
|
std::vector<std::unique_ptr<ColBufEncoder>> key_col_bufs; |
||||||
|
std::vector<std::unique_ptr<ColBufEncoder>> value_col_bufs; |
||||||
|
std::unique_ptr<ColBufEncoder> value_checksum_buf; |
||||||
|
|
||||||
|
explicit KVPairColBufEncoders(const KVPairColDeclarations &kvp_cd) { |
||||||
|
for (auto kcd : *kvp_cd.key_col_declarations) { |
||||||
|
key_col_bufs.emplace_back( |
||||||
|
std::move(ColBufEncoder::NewColBufEncoder(kcd))); |
||||||
|
} |
||||||
|
for (auto vcd : *kvp_cd.value_col_declarations) { |
||||||
|
value_col_bufs.emplace_back( |
||||||
|
std::move(ColBufEncoder::NewColBufEncoder(vcd))); |
||||||
|
} |
||||||
|
value_checksum_buf.reset( |
||||||
|
ColBufEncoder::NewColBufEncoder(*kvp_cd.value_checksum_declaration)); |
||||||
|
} |
||||||
|
|
||||||
|
// Helper function to call Finish()
|
||||||
|
void Finish() { |
||||||
|
for (auto &col_buf : key_col_bufs) { |
||||||
|
col_buf->Finish(); |
||||||
|
} |
||||||
|
for (auto &col_buf : value_col_bufs) { |
||||||
|
col_buf->Finish(); |
||||||
|
} |
||||||
|
value_checksum_buf->Finish(); |
||||||
|
} |
||||||
|
}; |
||||||
|
} // namespace rocksdb
|
||||||
@ -0,0 +1,171 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
#include <cstdio> |
||||||
|
|
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
#ifdef GFLAGS |
||||||
|
|
||||||
|
#include <gflags/gflags.h> |
||||||
|
#include <vector> |
||||||
|
#include "rocksdb/env.h" |
||||||
|
#include "rocksdb/options.h" |
||||||
|
#include "table/block_based_table_builder.h" |
||||||
|
#include "table/block_based_table_reader.h" |
||||||
|
#include "table/format.h" |
||||||
|
#include "tools/sst_dump_tool_imp.h" |
||||||
|
#include "util/compression.h" |
||||||
|
#include "util/stop_watch.h" |
||||||
|
#include "utilities/col_buf_encoder.h" |
||||||
|
#include "utilities/column_aware_encoding_util.h" |
||||||
|
|
||||||
|
using GFLAGS::ParseCommandLineFlags; |
||||||
|
DEFINE_string(encoded_file, "", "file to store encoded data blocks"); |
||||||
|
DEFINE_string(decoded_file, "", |
||||||
|
"file to store decoded data blocks after encoding"); |
||||||
|
DEFINE_string(format, "col", "Output Format. Can be 'row' or 'col'"); |
||||||
|
// TODO(jhli): option `col` should be removed and replaced by general
|
||||||
|
// column specifications.
|
||||||
|
DEFINE_string(index_type, "col", "Index type. Can be 'primary' or 'secondary'"); |
||||||
|
DEFINE_string(dump_file, "", |
||||||
|
"Dump data blocks separated by columns in human-readable format"); |
||||||
|
DEFINE_bool(decode, false, "Deocde blocks after they are encoded"); |
||||||
|
DEFINE_bool(stat, false, |
||||||
|
"Print column distribution statistics. Cannot decode in this mode"); |
||||||
|
DEFINE_string(compression_type, "kNoCompression", |
||||||
|
"The compression algorithm used to compress data blocks"); |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class ColumnAwareEncodingExp { |
||||||
|
public: |
||||||
|
static void Run(const std::string& sst_file) { |
||||||
|
bool decode = FLAGS_decode; |
||||||
|
if (FLAGS_decoded_file.size() > 0) { |
||||||
|
decode = true; |
||||||
|
} |
||||||
|
if (FLAGS_stat) { |
||||||
|
decode = false; |
||||||
|
} |
||||||
|
|
||||||
|
ColumnAwareEncodingReader reader(sst_file); |
||||||
|
std::vector<ColDeclaration>* key_col_declarations; |
||||||
|
std::vector<ColDeclaration>* value_col_declarations; |
||||||
|
ColDeclaration* value_checksum_declaration; |
||||||
|
if (FLAGS_index_type == "primary") { |
||||||
|
ColumnAwareEncodingReader::GetColDeclarationsPrimary( |
||||||
|
&key_col_declarations, &value_col_declarations, |
||||||
|
&value_checksum_declaration); |
||||||
|
} else { |
||||||
|
ColumnAwareEncodingReader::GetColDeclarationsSecondary( |
||||||
|
&key_col_declarations, &value_col_declarations, |
||||||
|
&value_checksum_declaration); |
||||||
|
} |
||||||
|
KVPairColDeclarations kvp_cd(key_col_declarations, value_col_declarations, |
||||||
|
value_checksum_declaration); |
||||||
|
|
||||||
|
if (!FLAGS_dump_file.empty()) { |
||||||
|
std::vector<KVPairBlock> kv_pair_blocks; |
||||||
|
reader.GetKVPairsFromDataBlocks(&kv_pair_blocks); |
||||||
|
reader.DumpDataColumns(FLAGS_dump_file, kvp_cd, kv_pair_blocks); |
||||||
|
return; |
||||||
|
} |
||||||
|
std::unordered_map<std::string, CompressionType> compressions = { |
||||||
|
{"kNoCompression", CompressionType::kNoCompression}, |
||||||
|
{"kZlibCompression", CompressionType::kZlibCompression}, |
||||||
|
{"kZSTDNotFinalCompression", |
||||||
|
CompressionType::kZSTDNotFinalCompression}}; |
||||||
|
|
||||||
|
// Find Compression
|
||||||
|
CompressionType compression_type = compressions[FLAGS_compression_type]; |
||||||
|
EnvOptions env_options; |
||||||
|
if (CompressionTypeSupported(compression_type)) { |
||||||
|
fprintf(stdout, "[%s]\n", FLAGS_compression_type.c_str()); |
||||||
|
unique_ptr<WritableFile> encoded_out_file; |
||||||
|
|
||||||
|
std::unique_ptr<Env> env(NewMemEnv(Env::Default())); |
||||||
|
if (!FLAGS_encoded_file.empty()) { |
||||||
|
env->NewWritableFile(FLAGS_encoded_file, &encoded_out_file, |
||||||
|
env_options); |
||||||
|
} |
||||||
|
|
||||||
|
std::vector<KVPairBlock> kv_pair_blocks; |
||||||
|
reader.GetKVPairsFromDataBlocks(&kv_pair_blocks); |
||||||
|
|
||||||
|
std::vector<std::string> encoded_blocks; |
||||||
|
StopWatchNano sw(env.get(), true); |
||||||
|
if (FLAGS_format == "col") { |
||||||
|
reader.EncodeBlocks(kvp_cd, encoded_out_file.get(), compression_type, |
||||||
|
kv_pair_blocks, &encoded_blocks, FLAGS_stat); |
||||||
|
} else { // row format
|
||||||
|
reader.EncodeBlocksToRowFormat(encoded_out_file.get(), compression_type, |
||||||
|
kv_pair_blocks, &encoded_blocks); |
||||||
|
} |
||||||
|
if (encoded_out_file != nullptr) { |
||||||
|
uint64_t size = 0; |
||||||
|
env->GetFileSize(FLAGS_encoded_file, &size); |
||||||
|
fprintf(stdout, "File size: %lu\n", size); |
||||||
|
} |
||||||
|
uint64_t encode_time = sw.ElapsedNanosSafe(false /* reset */); |
||||||
|
fprintf(stdout, "Encode time:%lu\n", encode_time); |
||||||
|
if (decode) { |
||||||
|
unique_ptr<WritableFile> decoded_out_file; |
||||||
|
if (!FLAGS_decoded_file.empty()) { |
||||||
|
env->NewWritableFile(FLAGS_decoded_file, &decoded_out_file, |
||||||
|
env_options); |
||||||
|
} |
||||||
|
sw.Start(); |
||||||
|
if (FLAGS_format == "col") { |
||||||
|
reader.DecodeBlocks(kvp_cd, decoded_out_file.get(), &encoded_blocks); |
||||||
|
} else { |
||||||
|
reader.DecodeBlocksFromRowFormat(decoded_out_file.get(), |
||||||
|
&encoded_blocks); |
||||||
|
} |
||||||
|
uint64_t decode_time = sw.ElapsedNanosSafe(true /* reset */); |
||||||
|
fprintf(stdout, "Decode time:%lu\n", decode_time); |
||||||
|
} |
||||||
|
} else { |
||||||
|
fprintf(stdout, "Unsupported compression type: %s.\n", |
||||||
|
FLAGS_compression_type.c_str()); |
||||||
|
} |
||||||
|
delete key_col_declarations; |
||||||
|
delete value_col_declarations; |
||||||
|
delete value_checksum_declaration; |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
|
|
||||||
|
int main(int argc, char** argv) { |
||||||
|
int arg_idx = ParseCommandLineFlags(&argc, &argv, true); |
||||||
|
if (arg_idx >= argc) { |
||||||
|
fprintf(stdout, "SST filename required.\n"); |
||||||
|
exit(1); |
||||||
|
} |
||||||
|
std::string sst_file(argv[arg_idx]); |
||||||
|
if (FLAGS_format != "row" && FLAGS_format != "col") { |
||||||
|
fprintf(stderr, "Format must be 'row' or 'col'\n"); |
||||||
|
exit(1); |
||||||
|
} |
||||||
|
if (FLAGS_index_type != "primary" && FLAGS_index_type != "secondary") { |
||||||
|
fprintf(stderr, "Format must be 'primary' or 'secondary'\n"); |
||||||
|
exit(1); |
||||||
|
} |
||||||
|
rocksdb::ColumnAwareEncodingExp::Run(sst_file); |
||||||
|
return 0; |
||||||
|
} |
||||||
|
|
||||||
|
#else |
||||||
|
int main() { |
||||||
|
fprintf(stderr, "Please install gflags to run rocksdb tools\n"); |
||||||
|
return 1; |
||||||
|
} |
||||||
|
#endif // GFLAGS
|
||||||
|
#else |
||||||
|
int main(int argc, char** argv) { |
||||||
|
fprintf(stderr, "Not supported in lite mode.\n"); |
||||||
|
return 1; |
||||||
|
} |
||||||
|
#endif // ROCKSDB_LITE
|
||||||
@ -0,0 +1,193 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
|
||||||
|
#include <vector> |
||||||
|
#include "util/testharness.h" |
||||||
|
#include "util/testutil.h" |
||||||
|
#include "utilities/col_buf_decoder.h" |
||||||
|
#include "utilities/col_buf_encoder.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class ColumnAwareEncodingTest : public testing::Test { |
||||||
|
public: |
||||||
|
ColumnAwareEncodingTest() {} |
||||||
|
|
||||||
|
~ColumnAwareEncodingTest() {} |
||||||
|
}; |
||||||
|
|
||||||
|
TEST_F(ColumnAwareEncodingTest, NoCompressionEncodeDecode) { |
||||||
|
std::unique_ptr<ColBufEncoder> col_buf_encoder( |
||||||
|
new FixedLengthColBufEncoder(8, kColNoCompression, false, true)); |
||||||
|
std::string str_buf; |
||||||
|
uint64_t val = 0x0102030405060708; |
||||||
|
for (int i = 0; i < 4; ++i) { |
||||||
|
str_buf.append(reinterpret_cast<char*>(&val), 8); |
||||||
|
} |
||||||
|
const char* str_buf_ptr = str_buf.c_str(); |
||||||
|
for (int i = 0; i < 4; ++i) { |
||||||
|
col_buf_encoder->Append(str_buf_ptr); |
||||||
|
} |
||||||
|
col_buf_encoder->Finish(); |
||||||
|
const std::string& encoded_data = col_buf_encoder->GetData(); |
||||||
|
ASSERT_EQ(encoded_data.size(), 32); |
||||||
|
|
||||||
|
const char* encoded_data_ptr = encoded_data.c_str(); |
||||||
|
uint64_t encoded_val; |
||||||
|
for (int i = 0; i < 4; ++i) { |
||||||
|
memcpy(&encoded_val, encoded_data_ptr, 8); |
||||||
|
ASSERT_EQ(encoded_val, 0x0807060504030201); |
||||||
|
encoded_data_ptr += 8; |
||||||
|
} |
||||||
|
|
||||||
|
std::unique_ptr<ColBufDecoder> col_buf_decoder( |
||||||
|
new FixedLengthColBufDecoder(8, kColNoCompression, false, true)); |
||||||
|
encoded_data_ptr = encoded_data.c_str(); |
||||||
|
encoded_data_ptr += col_buf_decoder->Init(encoded_data_ptr); |
||||||
|
char* decoded_data = new char[100]; |
||||||
|
char* decoded_data_base = decoded_data; |
||||||
|
for (int i = 0; i < 4; ++i) { |
||||||
|
encoded_data_ptr += |
||||||
|
col_buf_decoder->Decode(encoded_data_ptr, &decoded_data); |
||||||
|
} |
||||||
|
|
||||||
|
ASSERT_EQ(4 * 8, decoded_data - decoded_data_base); |
||||||
|
decoded_data = decoded_data_base; |
||||||
|
for (int i = 0; i < 4; ++i) { |
||||||
|
uint64_t decoded_val; |
||||||
|
decoded_val = 0; |
||||||
|
memcpy(&decoded_val, decoded_data, 8); |
||||||
|
ASSERT_EQ(decoded_val, val); |
||||||
|
decoded_data += 8; |
||||||
|
} |
||||||
|
delete[] decoded_data_base; |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(ColumnAwareEncodingTest, RleEncodeDecode) { |
||||||
|
std::unique_ptr<ColBufEncoder> col_buf_encoder( |
||||||
|
new FixedLengthColBufEncoder(8, kColRle, false, true)); |
||||||
|
std::string str_buf; |
||||||
|
uint64_t val = 0x0102030405060708; |
||||||
|
for (int i = 0; i < 4; ++i) { |
||||||
|
str_buf.append(reinterpret_cast<char*>(&val), 8); |
||||||
|
} |
||||||
|
const char* str_buf_ptr = str_buf.c_str(); |
||||||
|
for (int i = 0; i < 4; ++i) { |
||||||
|
str_buf_ptr += col_buf_encoder->Append(str_buf_ptr); |
||||||
|
} |
||||||
|
col_buf_encoder->Finish(); |
||||||
|
const std::string& encoded_data = col_buf_encoder->GetData(); |
||||||
|
ASSERT_EQ(encoded_data.size(), 9); |
||||||
|
|
||||||
|
const char* encoded_data_ptr = encoded_data.c_str(); |
||||||
|
uint64_t encoded_val; |
||||||
|
memcpy(&encoded_val, encoded_data_ptr, 8); |
||||||
|
ASSERT_EQ(encoded_val, 0x0807060504030201); |
||||||
|
|
||||||
|
std::unique_ptr<ColBufDecoder> col_buf_decoder( |
||||||
|
new FixedLengthColBufDecoder(8, kColRle, false, true)); |
||||||
|
char* decoded_data = new char[100]; |
||||||
|
char* decoded_data_base = decoded_data; |
||||||
|
encoded_data_ptr += col_buf_decoder->Init(encoded_data_ptr); |
||||||
|
for (int i = 0; i < 4; ++i) { |
||||||
|
encoded_data_ptr += |
||||||
|
col_buf_decoder->Decode(encoded_data_ptr, &decoded_data); |
||||||
|
} |
||||||
|
ASSERT_EQ(4 * 8, decoded_data - decoded_data_base); |
||||||
|
decoded_data = decoded_data_base; |
||||||
|
for (int i = 0; i < 4; ++i) { |
||||||
|
uint64_t decoded_val; |
||||||
|
decoded_val = 0; |
||||||
|
memcpy(&decoded_val, decoded_data, 8); |
||||||
|
ASSERT_EQ(decoded_val, val); |
||||||
|
decoded_data += 8; |
||||||
|
} |
||||||
|
delete[] decoded_data_base; |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(ColumnAwareEncodingTest, DeltaEncodeDecode) { |
||||||
|
std::unique_ptr<ColBufEncoder> col_buf_encoder( |
||||||
|
new FixedLengthColBufEncoder(8, kColDeltaVarint, false, true)); |
||||||
|
std::string str_buf; |
||||||
|
uint64_t val = 0x0102030405060708; |
||||||
|
uint64_t val2 = 0x0202030405060708; |
||||||
|
const char* str_buf_ptr; |
||||||
|
for (int i = 0; i < 2; ++i) { |
||||||
|
str_buf = std::string(reinterpret_cast<char*>(&val), 8); |
||||||
|
str_buf_ptr = str_buf.c_str(); |
||||||
|
col_buf_encoder->Append(str_buf_ptr); |
||||||
|
|
||||||
|
str_buf = std::string(reinterpret_cast<char*>(&val2), 8); |
||||||
|
str_buf_ptr = str_buf.c_str(); |
||||||
|
col_buf_encoder->Append(str_buf_ptr); |
||||||
|
} |
||||||
|
col_buf_encoder->Finish(); |
||||||
|
const std::string& encoded_data = col_buf_encoder->GetData(); |
||||||
|
ASSERT_EQ(encoded_data.size(), 9 + 3); |
||||||
|
|
||||||
|
std::unique_ptr<ColBufDecoder> col_buf_decoder( |
||||||
|
new FixedLengthColBufDecoder(8, kColDeltaVarint, false, true)); |
||||||
|
char* decoded_data = new char[100]; |
||||||
|
char* decoded_data_base = decoded_data; |
||||||
|
const char* encoded_data_ptr = encoded_data.c_str(); |
||||||
|
encoded_data_ptr += col_buf_decoder->Init(encoded_data_ptr); |
||||||
|
for (int i = 0; i < 4; ++i) { |
||||||
|
encoded_data_ptr += |
||||||
|
col_buf_decoder->Decode(encoded_data_ptr, &decoded_data); |
||||||
|
} |
||||||
|
ASSERT_EQ(4 * 8, decoded_data - decoded_data_base); |
||||||
|
decoded_data = decoded_data_base; |
||||||
|
for (int i = 0; i < 2; ++i) { |
||||||
|
uint64_t decoded_val; |
||||||
|
memcpy(&decoded_val, decoded_data, 8); |
||||||
|
ASSERT_EQ(decoded_val, val); |
||||||
|
decoded_data += 8; |
||||||
|
memcpy(&decoded_val, decoded_data, 8); |
||||||
|
ASSERT_EQ(decoded_val, val2); |
||||||
|
decoded_data += 8; |
||||||
|
} |
||||||
|
delete[] decoded_data_base; |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(ColumnAwareEncodingTest, ChunkBufEncodeDecode) { |
||||||
|
std::unique_ptr<ColBufEncoder> col_buf_encoder( |
||||||
|
new VariableChunkColBufEncoder(kColDict)); |
||||||
|
std::string buf("12345678\377\1\0\0\0\0\0\0\0\376", 18); |
||||||
|
col_buf_encoder->Append(buf.c_str()); |
||||||
|
col_buf_encoder->Finish(); |
||||||
|
const std::string& encoded_data = col_buf_encoder->GetData(); |
||||||
|
const char* str_ptr = encoded_data.c_str(); |
||||||
|
|
||||||
|
std::unique_ptr<ColBufDecoder> col_buf_decoder( |
||||||
|
new VariableChunkColBufDecoder(kColDict)); |
||||||
|
str_ptr += col_buf_decoder->Init(str_ptr); |
||||||
|
char* decoded_data = new char[100]; |
||||||
|
char* decoded_data_base = decoded_data; |
||||||
|
col_buf_decoder->Decode(str_ptr, &decoded_data); |
||||||
|
for (size_t i = 0; i < buf.size(); ++i) { |
||||||
|
ASSERT_EQ(buf[i], decoded_data_base[i]); |
||||||
|
} |
||||||
|
delete[] decoded_data_base; |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
|
|
||||||
|
int main(int argc, char** argv) { |
||||||
|
::testing::InitGoogleTest(&argc, argv); |
||||||
|
return RUN_ALL_TESTS(); |
||||||
|
} |
||||||
|
|
||||||
|
#else |
||||||
|
|
||||||
|
#include <cstdio> |
||||||
|
|
||||||
|
int main() { |
||||||
|
fprintf(stderr, |
||||||
|
"SKIPPED as column aware encoding experiment is not enabled in " |
||||||
|
"ROCKSDB_LITE\n"); |
||||||
|
} |
||||||
|
#endif // ROCKSDB_LITE
|
||||||
@ -0,0 +1,490 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
|
||||||
|
#include "utilities/column_aware_encoding_util.h" |
||||||
|
|
||||||
|
#ifndef __STDC_FORMAT_MACROS |
||||||
|
#define __STDC_FORMAT_MACROS |
||||||
|
#endif |
||||||
|
|
||||||
|
#include <stddef.h> |
||||||
|
#include <stdint.h> |
||||||
|
#include <stdio.h> |
||||||
|
#include <algorithm> |
||||||
|
#include <utility> |
||||||
|
#include <vector> |
||||||
|
#include "include/rocksdb/comparator.h" |
||||||
|
#include "include/rocksdb/slice.h" |
||||||
|
#include "rocksdb/env.h" |
||||||
|
#include "rocksdb/status.h" |
||||||
|
#include "table/block_based_table_builder.h" |
||||||
|
#include "table/block_based_table_factory.h" |
||||||
|
#include "table/format.h" |
||||||
|
#include "table/table_reader.h" |
||||||
|
#include "util/coding.h" |
||||||
|
#include "utilities/col_buf_decoder.h" |
||||||
|
#include "utilities/col_buf_encoder.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
ColumnAwareEncodingReader::ColumnAwareEncodingReader( |
||||||
|
const std::string& file_path) |
||||||
|
: file_name_(file_path), |
||||||
|
ioptions_(options_), |
||||||
|
internal_comparator_(BytewiseComparator()) { |
||||||
|
InitTableReader(file_name_); |
||||||
|
} |
||||||
|
|
||||||
|
void ColumnAwareEncodingReader::InitTableReader(const std::string& file_path) { |
||||||
|
std::unique_ptr<RandomAccessFile> file; |
||||||
|
uint64_t file_size; |
||||||
|
options_.env->NewRandomAccessFile(file_path, &file, soptions_); |
||||||
|
options_.env->GetFileSize(file_path, &file_size); |
||||||
|
|
||||||
|
file_.reset(new RandomAccessFileReader(std::move(file))); |
||||||
|
|
||||||
|
options_.comparator = &internal_comparator_; |
||||||
|
options_.table_factory = std::make_shared<BlockBasedTableFactory>(); |
||||||
|
shared_ptr<BlockBasedTableFactory> block_table_factory = |
||||||
|
std::dynamic_pointer_cast<BlockBasedTableFactory>(options_.table_factory); |
||||||
|
|
||||||
|
std::unique_ptr<TableReader> table_reader; |
||||||
|
block_table_factory->NewTableReader( |
||||||
|
TableReaderOptions(ioptions_, soptions_, internal_comparator_, |
||||||
|
/*skip_filters=*/false), |
||||||
|
std::move(file_), file_size, &table_reader, /*enable_prefetch=*/false); |
||||||
|
|
||||||
|
table_reader_.reset(dynamic_cast<BlockBasedTable*>(table_reader.release())); |
||||||
|
} |
||||||
|
|
||||||
|
void ColumnAwareEncodingReader::GetKVPairsFromDataBlocks( |
||||||
|
std::vector<KVPairBlock>* kv_pair_blocks) { |
||||||
|
table_reader_->GetKVPairsFromDataBlocks(kv_pair_blocks); |
||||||
|
} |
||||||
|
|
||||||
|
void ColumnAwareEncodingReader::DecodeBlocks( |
||||||
|
const KVPairColDeclarations& kvp_col_declarations, WritableFile* out_file, |
||||||
|
const std::vector<std::string>* blocks) { |
||||||
|
char* decoded_content_base = new char[16384]; |
||||||
|
Options options; |
||||||
|
ImmutableCFOptions ioptions(options); |
||||||
|
for (auto& block : *blocks) { |
||||||
|
KVPairColBufDecoders kvp_col_bufs(kvp_col_declarations); |
||||||
|
auto& key_col_bufs = kvp_col_bufs.key_col_bufs; |
||||||
|
auto& value_col_bufs = kvp_col_bufs.value_col_bufs; |
||||||
|
auto& value_checksum_buf = kvp_col_bufs.value_checksum_buf; |
||||||
|
|
||||||
|
auto& slice_final_with_bit = block; |
||||||
|
uint32_t format_version = 2; |
||||||
|
Slice compression_dict; |
||||||
|
BlockContents contents; |
||||||
|
const char* content_ptr; |
||||||
|
|
||||||
|
CompressionType type = |
||||||
|
(CompressionType)slice_final_with_bit[slice_final_with_bit.size() - 1]; |
||||||
|
if (type != kNoCompression) { |
||||||
|
UncompressBlockContents(slice_final_with_bit.c_str(), |
||||||
|
slice_final_with_bit.size() - 1, &contents, |
||||||
|
format_version, compression_dict, ioptions); |
||||||
|
content_ptr = contents.data.data(); |
||||||
|
} else { |
||||||
|
content_ptr = slice_final_with_bit.data(); |
||||||
|
} |
||||||
|
|
||||||
|
size_t num_kv_pairs; |
||||||
|
const char* header_content_ptr = content_ptr; |
||||||
|
num_kv_pairs = DecodeFixed64(header_content_ptr); |
||||||
|
|
||||||
|
header_content_ptr += sizeof(size_t); |
||||||
|
size_t num_key_columns = key_col_bufs.size(); |
||||||
|
size_t num_value_columns = value_col_bufs.size(); |
||||||
|
std::vector<const char*> key_content_ptr(num_key_columns); |
||||||
|
std::vector<const char*> value_content_ptr(num_value_columns); |
||||||
|
const char* checksum_content_ptr; |
||||||
|
|
||||||
|
size_t num_columns = num_key_columns + num_value_columns; |
||||||
|
const char* col_content_ptr = |
||||||
|
header_content_ptr + sizeof(size_t) * num_columns; |
||||||
|
|
||||||
|
// Read headers
|
||||||
|
for (size_t i = 0; i < num_key_columns; ++i) { |
||||||
|
key_content_ptr[i] = col_content_ptr; |
||||||
|
key_content_ptr[i] += key_col_bufs[i]->Init(key_content_ptr[i]); |
||||||
|
size_t offset; |
||||||
|
offset = DecodeFixed64(header_content_ptr); |
||||||
|
header_content_ptr += sizeof(size_t); |
||||||
|
col_content_ptr += offset; |
||||||
|
} |
||||||
|
for (size_t i = 0; i < num_value_columns; ++i) { |
||||||
|
value_content_ptr[i] = col_content_ptr; |
||||||
|
value_content_ptr[i] += value_col_bufs[i]->Init(value_content_ptr[i]); |
||||||
|
size_t offset; |
||||||
|
offset = DecodeFixed64(header_content_ptr); |
||||||
|
header_content_ptr += sizeof(size_t); |
||||||
|
col_content_ptr += offset; |
||||||
|
} |
||||||
|
checksum_content_ptr = col_content_ptr; |
||||||
|
checksum_content_ptr += value_checksum_buf->Init(checksum_content_ptr); |
||||||
|
|
||||||
|
// Decode block
|
||||||
|
char* decoded_content = decoded_content_base; |
||||||
|
for (size_t j = 0; j < num_kv_pairs; ++j) { |
||||||
|
for (size_t i = 0; i < num_key_columns; ++i) { |
||||||
|
key_content_ptr[i] += |
||||||
|
key_col_bufs[i]->Decode(key_content_ptr[i], &decoded_content); |
||||||
|
} |
||||||
|
for (size_t i = 0; i < num_value_columns; ++i) { |
||||||
|
value_content_ptr[i] += |
||||||
|
value_col_bufs[i]->Decode(value_content_ptr[i], &decoded_content); |
||||||
|
} |
||||||
|
checksum_content_ptr += |
||||||
|
value_checksum_buf->Decode(checksum_content_ptr, &decoded_content); |
||||||
|
} |
||||||
|
|
||||||
|
size_t offset = decoded_content - decoded_content_base; |
||||||
|
Slice output_content(decoded_content, offset); |
||||||
|
|
||||||
|
if (out_file != nullptr) { |
||||||
|
out_file->Append(output_content); |
||||||
|
} |
||||||
|
} |
||||||
|
delete[] decoded_content_base; |
||||||
|
} |
||||||
|
|
||||||
|
void ColumnAwareEncodingReader::DecodeBlocksFromRowFormat( |
||||||
|
WritableFile* out_file, const std::vector<std::string>* blocks) { |
||||||
|
Options options; |
||||||
|
ImmutableCFOptions ioptions(options); |
||||||
|
for (auto& block : *blocks) { |
||||||
|
auto& slice_final_with_bit = block; |
||||||
|
uint32_t format_version = 2; |
||||||
|
Slice compression_dict; |
||||||
|
BlockContents contents; |
||||||
|
std::string decoded_content; |
||||||
|
|
||||||
|
CompressionType type = |
||||||
|
(CompressionType)slice_final_with_bit[slice_final_with_bit.size() - 1]; |
||||||
|
if (type != kNoCompression) { |
||||||
|
UncompressBlockContents(slice_final_with_bit.c_str(), |
||||||
|
slice_final_with_bit.size() - 1, &contents, |
||||||
|
format_version, compression_dict, ioptions); |
||||||
|
decoded_content = std::string(contents.data.data(), contents.data.size()); |
||||||
|
} else { |
||||||
|
decoded_content = std::move(slice_final_with_bit); |
||||||
|
} |
||||||
|
|
||||||
|
if (out_file != nullptr) { |
||||||
|
out_file->Append(decoded_content); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
void ColumnAwareEncodingReader::DumpDataColumns( |
||||||
|
const std::string& filename, |
||||||
|
const KVPairColDeclarations& kvp_col_declarations, |
||||||
|
const std::vector<KVPairBlock>& kv_pair_blocks) { |
||||||
|
KVPairColBufEncoders kvp_col_bufs(kvp_col_declarations); |
||||||
|
auto& key_col_bufs = kvp_col_bufs.key_col_bufs; |
||||||
|
auto& value_col_bufs = kvp_col_bufs.value_col_bufs; |
||||||
|
auto& value_checksum_buf = kvp_col_bufs.value_checksum_buf; |
||||||
|
|
||||||
|
FILE* fp = fopen(filename.c_str(), "w"); |
||||||
|
size_t block_id = 1; |
||||||
|
for (auto& kv_pairs : kv_pair_blocks) { |
||||||
|
fprintf(fp, "---------------- Block: %-4lu ----------------\n", block_id); |
||||||
|
for (auto& kv_pair : kv_pairs) { |
||||||
|
const auto& key = kv_pair.first; |
||||||
|
const auto& value = kv_pair.second; |
||||||
|
size_t value_offset = 0; |
||||||
|
|
||||||
|
const char* key_ptr = key.data(); |
||||||
|
for (auto& buf : key_col_bufs) { |
||||||
|
size_t col_size = buf->Append(key_ptr); |
||||||
|
std::string tmp_buf(key_ptr, col_size); |
||||||
|
Slice col(tmp_buf); |
||||||
|
fprintf(fp, "%s ", col.ToString(true).c_str()); |
||||||
|
key_ptr += col_size; |
||||||
|
} |
||||||
|
fprintf(fp, "|"); |
||||||
|
|
||||||
|
const char* value_ptr = value.data(); |
||||||
|
for (auto& buf : value_col_bufs) { |
||||||
|
size_t col_size = buf->Append(value_ptr); |
||||||
|
std::string tmp_buf(value_ptr, col_size); |
||||||
|
Slice col(tmp_buf); |
||||||
|
fprintf(fp, " %s", col.ToString(true).c_str()); |
||||||
|
value_ptr += col_size; |
||||||
|
value_offset += col_size; |
||||||
|
} |
||||||
|
|
||||||
|
if (value_offset < value.size()) { |
||||||
|
size_t col_size = value_checksum_buf->Append(value_ptr); |
||||||
|
std::string tmp_buf(value_ptr, col_size); |
||||||
|
Slice col(tmp_buf); |
||||||
|
fprintf(fp, "|%s", col.ToString(true).c_str()); |
||||||
|
} else { |
||||||
|
value_checksum_buf->Append(nullptr); |
||||||
|
} |
||||||
|
fprintf(fp, "\n"); |
||||||
|
} |
||||||
|
block_id++; |
||||||
|
} |
||||||
|
fclose(fp); |
||||||
|
} |
||||||
|
|
||||||
|
namespace { |
||||||
|
|
||||||
|
void CompressDataBlock(const std::string& output_content, Slice* slice_final, |
||||||
|
CompressionType* type, std::string* compressed_output) { |
||||||
|
CompressionOptions compression_opts; |
||||||
|
uint32_t format_version = 2; // hard-coded version
|
||||||
|
Slice compression_dict; |
||||||
|
*slice_final = |
||||||
|
CompressBlock(output_content, compression_opts, type, format_version, |
||||||
|
compression_dict, compressed_output); |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
void ColumnAwareEncodingReader::EncodeBlocksToRowFormat( |
||||||
|
WritableFile* out_file, CompressionType compression_type, |
||||||
|
const std::vector<KVPairBlock>& kv_pair_blocks, |
||||||
|
std::vector<std::string>* blocks) { |
||||||
|
std::string output_content; |
||||||
|
for (auto& kv_pairs : kv_pair_blocks) { |
||||||
|
output_content.clear(); |
||||||
|
std::string last_key; |
||||||
|
size_t counter = 0; |
||||||
|
const size_t block_restart_interval = 16; |
||||||
|
for (auto& kv_pair : kv_pairs) { |
||||||
|
const auto& key = kv_pair.first; |
||||||
|
const auto& value = kv_pair.second; |
||||||
|
|
||||||
|
Slice last_key_piece(last_key); |
||||||
|
size_t shared = 0; |
||||||
|
if (counter >= block_restart_interval) { |
||||||
|
counter = 0; |
||||||
|
} else { |
||||||
|
const size_t min_length = std::min(last_key_piece.size(), key.size()); |
||||||
|
while ((shared < min_length) && last_key_piece[shared] == key[shared]) { |
||||||
|
shared++; |
||||||
|
} |
||||||
|
} |
||||||
|
const size_t non_shared = key.size() - shared; |
||||||
|
output_content.append(key.c_str() + shared, non_shared); |
||||||
|
output_content.append(value); |
||||||
|
|
||||||
|
last_key.resize(shared); |
||||||
|
last_key.append(key.data() + shared, non_shared); |
||||||
|
counter++; |
||||||
|
} |
||||||
|
Slice slice_final; |
||||||
|
auto type = compression_type; |
||||||
|
std::string compressed_output; |
||||||
|
CompressDataBlock(output_content, &slice_final, &type, &compressed_output); |
||||||
|
|
||||||
|
if (out_file != nullptr) { |
||||||
|
out_file->Append(slice_final); |
||||||
|
} |
||||||
|
|
||||||
|
// Add a bit in the end for decoding
|
||||||
|
std::string slice_final_with_bit(slice_final.data(), slice_final.size()); |
||||||
|
slice_final_with_bit.append(reinterpret_cast<char*>(&type), 1); |
||||||
|
blocks->push_back( |
||||||
|
std::string(slice_final_with_bit.data(), slice_final_with_bit.size())); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
Status ColumnAwareEncodingReader::EncodeBlocks( |
||||||
|
const KVPairColDeclarations& kvp_col_declarations, WritableFile* out_file, |
||||||
|
CompressionType compression_type, |
||||||
|
const std::vector<KVPairBlock>& kv_pair_blocks, |
||||||
|
std::vector<std::string>* blocks, bool print_column_stat) { |
||||||
|
std::vector<size_t> key_col_sizes( |
||||||
|
kvp_col_declarations.key_col_declarations->size(), 0); |
||||||
|
std::vector<size_t> value_col_sizes( |
||||||
|
kvp_col_declarations.value_col_declarations->size(), 0); |
||||||
|
size_t value_checksum_size = 0; |
||||||
|
|
||||||
|
for (auto& kv_pairs : kv_pair_blocks) { |
||||||
|
KVPairColBufEncoders kvp_col_bufs(kvp_col_declarations); |
||||||
|
auto& key_col_bufs = kvp_col_bufs.key_col_bufs; |
||||||
|
auto& value_col_bufs = kvp_col_bufs.value_col_bufs; |
||||||
|
auto& value_checksum_buf = kvp_col_bufs.value_checksum_buf; |
||||||
|
|
||||||
|
size_t num_kv_pairs = 0; |
||||||
|
for (auto& kv_pair : kv_pairs) { |
||||||
|
const auto& key = kv_pair.first; |
||||||
|
const auto& value = kv_pair.second; |
||||||
|
size_t value_offset = 0; |
||||||
|
num_kv_pairs++; |
||||||
|
|
||||||
|
const char* key_ptr = key.data(); |
||||||
|
for (auto& buf : key_col_bufs) { |
||||||
|
size_t col_size = buf->Append(key_ptr); |
||||||
|
key_ptr += col_size; |
||||||
|
} |
||||||
|
|
||||||
|
const char* value_ptr = value.data(); |
||||||
|
for (auto& buf : value_col_bufs) { |
||||||
|
size_t col_size = buf->Append(value_ptr); |
||||||
|
value_ptr += col_size; |
||||||
|
value_offset += col_size; |
||||||
|
} |
||||||
|
|
||||||
|
if (value_offset < value.size()) { |
||||||
|
value_checksum_buf->Append(value_ptr); |
||||||
|
} else { |
||||||
|
value_checksum_buf->Append(nullptr); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
kvp_col_bufs.Finish(); |
||||||
|
// Get stats
|
||||||
|
// Compress and write a block
|
||||||
|
if (print_column_stat) { |
||||||
|
for (size_t i = 0; i < key_col_bufs.size(); ++i) { |
||||||
|
Slice slice_final; |
||||||
|
auto type = compression_type; |
||||||
|
std::string compressed_output; |
||||||
|
CompressDataBlock(key_col_bufs[i]->GetData(), &slice_final, &type, |
||||||
|
&compressed_output); |
||||||
|
out_file->Append(slice_final); |
||||||
|
key_col_sizes[i] += slice_final.size(); |
||||||
|
} |
||||||
|
for (size_t i = 0; i < value_col_bufs.size(); ++i) { |
||||||
|
Slice slice_final; |
||||||
|
auto type = compression_type; |
||||||
|
std::string compressed_output; |
||||||
|
CompressDataBlock(value_col_bufs[i]->GetData(), &slice_final, &type, |
||||||
|
&compressed_output); |
||||||
|
out_file->Append(slice_final); |
||||||
|
value_col_sizes[i] += slice_final.size(); |
||||||
|
} |
||||||
|
Slice slice_final; |
||||||
|
auto type = compression_type; |
||||||
|
std::string compressed_output; |
||||||
|
CompressDataBlock(value_checksum_buf->GetData(), &slice_final, &type, |
||||||
|
&compressed_output); |
||||||
|
out_file->Append(slice_final); |
||||||
|
value_checksum_size += slice_final.size(); |
||||||
|
} else { |
||||||
|
std::string output_content; |
||||||
|
// Write column sizes
|
||||||
|
PutFixed64(&output_content, num_kv_pairs); |
||||||
|
for (auto& buf : key_col_bufs) { |
||||||
|
size_t size = buf->GetData().size(); |
||||||
|
PutFixed64(&output_content, size); |
||||||
|
} |
||||||
|
for (auto& buf : value_col_bufs) { |
||||||
|
size_t size = buf->GetData().size(); |
||||||
|
PutFixed64(&output_content, size); |
||||||
|
} |
||||||
|
// Write data
|
||||||
|
for (auto& buf : key_col_bufs) { |
||||||
|
output_content.append(buf->GetData()); |
||||||
|
} |
||||||
|
for (auto& buf : value_col_bufs) { |
||||||
|
output_content.append(buf->GetData()); |
||||||
|
} |
||||||
|
output_content.append(value_checksum_buf->GetData()); |
||||||
|
|
||||||
|
Slice slice_final; |
||||||
|
auto type = compression_type; |
||||||
|
std::string compressed_output; |
||||||
|
CompressDataBlock(output_content, &slice_final, &type, |
||||||
|
&compressed_output); |
||||||
|
|
||||||
|
if (out_file != nullptr) { |
||||||
|
out_file->Append(slice_final); |
||||||
|
} |
||||||
|
|
||||||
|
// Add a bit in the end for decoding
|
||||||
|
std::string slice_final_with_bit(slice_final.data(), |
||||||
|
slice_final.size() + 1); |
||||||
|
slice_final_with_bit[slice_final.size()] = static_cast<char>(type); |
||||||
|
blocks->push_back(std::string(slice_final_with_bit.data(), |
||||||
|
slice_final_with_bit.size())); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
if (print_column_stat) { |
||||||
|
size_t total_size = 0; |
||||||
|
for (size_t i = 0; i < key_col_sizes.size(); ++i) |
||||||
|
total_size += key_col_sizes[i]; |
||||||
|
for (size_t i = 0; i < value_col_sizes.size(); ++i) |
||||||
|
total_size += value_col_sizes[i]; |
||||||
|
total_size += value_checksum_size; |
||||||
|
|
||||||
|
for (size_t i = 0; i < key_col_sizes.size(); ++i) |
||||||
|
printf("Key col %lu size: %lu percentage %lf%%\n", i, key_col_sizes[i], |
||||||
|
100.0 * key_col_sizes[i] / total_size); |
||||||
|
for (size_t i = 0; i < value_col_sizes.size(); ++i) |
||||||
|
printf("Value col %lu size: %lu percentage %lf%%\n", i, |
||||||
|
value_col_sizes[i], 100.0 * value_col_sizes[i] / total_size); |
||||||
|
printf("Value checksum size: %lu percentage %lf%%\n", value_checksum_size, |
||||||
|
100.0 * value_checksum_size / total_size); |
||||||
|
} |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
void ColumnAwareEncodingReader::GetColDeclarationsPrimary( |
||||||
|
std::vector<ColDeclaration>** key_col_declarations, |
||||||
|
std::vector<ColDeclaration>** value_col_declarations, |
||||||
|
ColDeclaration** value_checksum_declaration) { |
||||||
|
*key_col_declarations = new std::vector<ColDeclaration>{ |
||||||
|
ColDeclaration("FixedLength", ColCompressionType::kColRleVarint, 4, false, |
||||||
|
true), |
||||||
|
ColDeclaration("FixedLength", ColCompressionType::kColRleDeltaVarint, 8, |
||||||
|
false, true), |
||||||
|
ColDeclaration("FixedLength", ColCompressionType::kColDeltaVarint, 8, |
||||||
|
false, true), |
||||||
|
ColDeclaration("FixedLength", ColCompressionType::kColDeltaVarint, 8, |
||||||
|
false, true), |
||||||
|
ColDeclaration("FixedLength", ColCompressionType::kColRleVarint, 8)}; |
||||||
|
|
||||||
|
*value_col_declarations = new std::vector<ColDeclaration>{ |
||||||
|
ColDeclaration("FixedLength", ColCompressionType::kColRleVarint, 4), |
||||||
|
ColDeclaration("FixedLength", ColCompressionType::kColRleVarint, 4), |
||||||
|
ColDeclaration("FixedLength", ColCompressionType::kColRle, 1), |
||||||
|
ColDeclaration("VariableLength"), |
||||||
|
ColDeclaration("FixedLength", ColCompressionType::kColDeltaVarint, 4), |
||||||
|
ColDeclaration("FixedLength", ColCompressionType::kColRleVarint, 8)}; |
||||||
|
*value_checksum_declaration = new ColDeclaration( |
||||||
|
"LongFixedLength", ColCompressionType::kColNoCompression, 9, |
||||||
|
true /* nullable */); |
||||||
|
} |
||||||
|
|
||||||
|
void ColumnAwareEncodingReader::GetColDeclarationsSecondary( |
||||||
|
std::vector<ColDeclaration>** key_col_declarations, |
||||||
|
std::vector<ColDeclaration>** value_col_declarations, |
||||||
|
ColDeclaration** value_checksum_declaration) { |
||||||
|
*key_col_declarations = new std::vector<ColDeclaration>{ |
||||||
|
ColDeclaration("FixedLength", ColCompressionType::kColRleVarint, 4, false, |
||||||
|
true), |
||||||
|
ColDeclaration("FixedLength", ColCompressionType::kColDeltaVarint, 8, |
||||||
|
false, true), |
||||||
|
ColDeclaration("FixedLength", ColCompressionType::kColRleDeltaVarint, 8, |
||||||
|
false, true), |
||||||
|
ColDeclaration("FixedLength", ColCompressionType::kColRle, 1), |
||||||
|
ColDeclaration("FixedLength", ColCompressionType::kColDeltaVarint, 4, |
||||||
|
false, true), |
||||||
|
ColDeclaration("FixedLength", ColCompressionType::kColDeltaVarint, 8, |
||||||
|
false, true), |
||||||
|
ColDeclaration("FixedLength", ColCompressionType::kColRleVarint, 8, false, |
||||||
|
true), |
||||||
|
ColDeclaration("VariableChunk", ColCompressionType::kColNoCompression), |
||||||
|
ColDeclaration("FixedLength", ColCompressionType::kColRleVarint, 8)}; |
||||||
|
*value_col_declarations = new std::vector<ColDeclaration>(); |
||||||
|
*value_checksum_declaration = new ColDeclaration( |
||||||
|
"LongFixedLength", ColCompressionType::kColNoCompression, 9, |
||||||
|
true /* nullable */); |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
|
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
@ -0,0 +1,80 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
#pragma once |
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
|
||||||
|
#include <string> |
||||||
|
#include <vector> |
||||||
|
#include "db/dbformat.h" |
||||||
|
#include "include/rocksdb/env.h" |
||||||
|
#include "include/rocksdb/immutable_options.h" |
||||||
|
#include "include/rocksdb/listener.h" |
||||||
|
#include "include/rocksdb/options.h" |
||||||
|
#include "include/rocksdb/status.h" |
||||||
|
#include "table/block_based_table_reader.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
struct ColDeclaration; |
||||||
|
struct KVPairColDeclarations; |
||||||
|
|
||||||
|
class ColumnAwareEncodingReader { |
||||||
|
public: |
||||||
|
explicit ColumnAwareEncodingReader(const std::string& file_name); |
||||||
|
|
||||||
|
void GetKVPairsFromDataBlocks(std::vector<KVPairBlock>* kv_pair_blocks); |
||||||
|
|
||||||
|
void EncodeBlocksToRowFormat(WritableFile* out_file, |
||||||
|
CompressionType compression_type, |
||||||
|
const std::vector<KVPairBlock>& kv_pair_blocks, |
||||||
|
std::vector<std::string>* blocks); |
||||||
|
|
||||||
|
void DecodeBlocksFromRowFormat(WritableFile* out_file, |
||||||
|
const std::vector<std::string>* blocks); |
||||||
|
|
||||||
|
void DumpDataColumns(const std::string& filename, |
||||||
|
const KVPairColDeclarations& kvp_col_declarations, |
||||||
|
const std::vector<KVPairBlock>& kv_pair_blocks); |
||||||
|
|
||||||
|
Status EncodeBlocks(const KVPairColDeclarations& kvp_col_declarations, |
||||||
|
WritableFile* out_file, CompressionType compression_type, |
||||||
|
const std::vector<KVPairBlock>& kv_pair_blocks, |
||||||
|
std::vector<std::string>* blocks, bool print_column_stat); |
||||||
|
|
||||||
|
void DecodeBlocks(const KVPairColDeclarations& kvp_col_declarations, |
||||||
|
WritableFile* out_file, |
||||||
|
const std::vector<std::string>* blocks); |
||||||
|
|
||||||
|
static void GetColDeclarationsPrimary( |
||||||
|
std::vector<ColDeclaration>** key_col_declarations, |
||||||
|
std::vector<ColDeclaration>** value_col_declarations, |
||||||
|
ColDeclaration** value_checksum_declaration); |
||||||
|
|
||||||
|
static void GetColDeclarationsSecondary( |
||||||
|
std::vector<ColDeclaration>** key_col_declarations, |
||||||
|
std::vector<ColDeclaration>** value_col_declarations, |
||||||
|
ColDeclaration** value_checksum_declaration); |
||||||
|
|
||||||
|
private: |
||||||
|
// Init the TableReader for the sst file
|
||||||
|
void InitTableReader(const std::string& file_path); |
||||||
|
|
||||||
|
std::string file_name_; |
||||||
|
EnvOptions soptions_; |
||||||
|
|
||||||
|
Options options_; |
||||||
|
|
||||||
|
Status init_result_; |
||||||
|
std::unique_ptr<BlockBasedTable> table_reader_; |
||||||
|
std::unique_ptr<RandomAccessFileReader> file_; |
||||||
|
|
||||||
|
const ImmutableCFOptions ioptions_; |
||||||
|
InternalKeyComparator internal_comparator_; |
||||||
|
std::unique_ptr<TableProperties> table_properties_; |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
|
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
Loading…
Reference in new issue