Split BlobFileState into an immutable and a mutable part (#6502)
Summary: It's never too soon to refactor something. The patch splits the recently introduced (`VersionEdit` related) `BlobFileState` into two classes `BlobFileAddition` and `BlobFileGarbage`. The idea is that once blob files are closed, they are immutable, and the only thing that changes is the amount of garbage in them. In the new design, `BlobFileAddition` contains the immutable attributes (currently, the count and total size of all blobs, checksum method, and checksum value), while `BlobFileGarbage` contains the mutable GC-related information elements (count and total size of garbage blobs). This is a better fit for the GC logic and is more consistent with how SST files are handled. Pull Request resolved: https://github.com/facebook/rocksdb/pull/6502 Test Plan: `make check` Differential Revision: D20348352 Pulled By: ltamasi fbshipit-source-id: ff93f0121e80ab15e0e0a6525ba0d6af16a0e008main
Levi Tamasi
5 years ago
committed by
Facebook Github Bot
parent
4028eba67b
commit
f5bc3b99d5
@ -0,0 +1,16 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <cstdint> |
||||||
|
|
||||||
|
#include "rocksdb/rocksdb_namespace.h" |
||||||
|
|
||||||
|
namespace ROCKSDB_NAMESPACE { |
||||||
|
|
||||||
|
constexpr uint64_t kInvalidBlobFileNumber = 0; |
||||||
|
|
||||||
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
@ -0,0 +1,65 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <cassert> |
||||||
|
#include <cstdint> |
||||||
|
#include <iosfwd> |
||||||
|
#include <string> |
||||||
|
|
||||||
|
#include "db/blob_constants.h" |
||||||
|
#include "rocksdb/rocksdb_namespace.h" |
||||||
|
|
||||||
|
namespace ROCKSDB_NAMESPACE { |
||||||
|
|
||||||
|
class JSONWriter; |
||||||
|
class Slice; |
||||||
|
class Status; |
||||||
|
|
||||||
|
class BlobFileAddition { |
||||||
|
public: |
||||||
|
BlobFileAddition() = default; |
||||||
|
|
||||||
|
BlobFileAddition(uint64_t blob_file_number, uint64_t total_blob_count, |
||||||
|
uint64_t total_blob_bytes, std::string checksum_method, |
||||||
|
std::string checksum_value) |
||||||
|
: blob_file_number_(blob_file_number), |
||||||
|
total_blob_count_(total_blob_count), |
||||||
|
total_blob_bytes_(total_blob_bytes), |
||||||
|
checksum_method_(std::move(checksum_method)), |
||||||
|
checksum_value_(std::move(checksum_value)) { |
||||||
|
assert(checksum_method_.empty() == checksum_value_.empty()); |
||||||
|
} |
||||||
|
|
||||||
|
uint64_t GetBlobFileNumber() const { return blob_file_number_; } |
||||||
|
uint64_t GetTotalBlobCount() const { return total_blob_count_; } |
||||||
|
uint64_t GetTotalBlobBytes() const { return total_blob_bytes_; } |
||||||
|
const std::string& GetChecksumMethod() const { return checksum_method_; } |
||||||
|
const std::string& GetChecksumValue() const { return checksum_value_; } |
||||||
|
|
||||||
|
void EncodeTo(std::string* output) const; |
||||||
|
Status DecodeFrom(Slice* input); |
||||||
|
|
||||||
|
std::string DebugString() const; |
||||||
|
std::string DebugJSON() const; |
||||||
|
|
||||||
|
private: |
||||||
|
uint64_t blob_file_number_ = kInvalidBlobFileNumber; |
||||||
|
uint64_t total_blob_count_ = 0; |
||||||
|
uint64_t total_blob_bytes_ = 0; |
||||||
|
std::string checksum_method_; |
||||||
|
std::string checksum_value_; |
||||||
|
}; |
||||||
|
|
||||||
|
bool operator==(const BlobFileAddition& lhs, const BlobFileAddition& rhs); |
||||||
|
bool operator!=(const BlobFileAddition& lhs, const BlobFileAddition& rhs); |
||||||
|
|
||||||
|
std::ostream& operator<<(std::ostream& os, |
||||||
|
const BlobFileAddition& blob_file_addition); |
||||||
|
JSONWriter& operator<<(JSONWriter& jw, |
||||||
|
const BlobFileAddition& blob_file_addition); |
||||||
|
|
||||||
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
@ -0,0 +1,206 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
|
#include "db/blob_file_addition.h" |
||||||
|
|
||||||
|
#include <cstdint> |
||||||
|
#include <cstring> |
||||||
|
#include <string> |
||||||
|
|
||||||
|
#include "test_util/sync_point.h" |
||||||
|
#include "test_util/testharness.h" |
||||||
|
#include "util/coding.h" |
||||||
|
|
||||||
|
namespace ROCKSDB_NAMESPACE { |
||||||
|
|
||||||
|
class BlobFileAdditionTest : public testing::Test { |
||||||
|
public: |
||||||
|
static void TestEncodeDecode(const BlobFileAddition& blob_file_addition) { |
||||||
|
std::string encoded; |
||||||
|
blob_file_addition.EncodeTo(&encoded); |
||||||
|
|
||||||
|
BlobFileAddition decoded; |
||||||
|
Slice input(encoded); |
||||||
|
ASSERT_OK(decoded.DecodeFrom(&input)); |
||||||
|
|
||||||
|
ASSERT_EQ(blob_file_addition, decoded); |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
TEST_F(BlobFileAdditionTest, Empty) { |
||||||
|
BlobFileAddition blob_file_addition; |
||||||
|
|
||||||
|
ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), kInvalidBlobFileNumber); |
||||||
|
ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), 0); |
||||||
|
ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(), 0); |
||||||
|
ASSERT_TRUE(blob_file_addition.GetChecksumMethod().empty()); |
||||||
|
ASSERT_TRUE(blob_file_addition.GetChecksumValue().empty()); |
||||||
|
|
||||||
|
TestEncodeDecode(blob_file_addition); |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(BlobFileAdditionTest, NonEmpty) { |
||||||
|
constexpr uint64_t blob_file_number = 123; |
||||||
|
constexpr uint64_t total_blob_count = 2; |
||||||
|
constexpr uint64_t total_blob_bytes = 123456; |
||||||
|
const std::string checksum_method("SHA1"); |
||||||
|
const std::string checksum_value("bdb7f34a59dfa1592ce7f52e99f98c570c525cbd"); |
||||||
|
|
||||||
|
BlobFileAddition blob_file_addition(blob_file_number, total_blob_count, |
||||||
|
total_blob_bytes, checksum_method, |
||||||
|
checksum_value); |
||||||
|
|
||||||
|
ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), blob_file_number); |
||||||
|
ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), total_blob_count); |
||||||
|
ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(), total_blob_bytes); |
||||||
|
ASSERT_EQ(blob_file_addition.GetChecksumMethod(), checksum_method); |
||||||
|
ASSERT_EQ(blob_file_addition.GetChecksumValue(), checksum_value); |
||||||
|
|
||||||
|
TestEncodeDecode(blob_file_addition); |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(BlobFileAdditionTest, DecodeErrors) { |
||||||
|
std::string str; |
||||||
|
Slice slice(str); |
||||||
|
|
||||||
|
BlobFileAddition blob_file_addition; |
||||||
|
|
||||||
|
{ |
||||||
|
const Status s = blob_file_addition.DecodeFrom(&slice); |
||||||
|
ASSERT_TRUE(s.IsCorruption()); |
||||||
|
ASSERT_TRUE(std::strstr(s.getState(), "blob file number")); |
||||||
|
} |
||||||
|
|
||||||
|
constexpr uint64_t blob_file_number = 123; |
||||||
|
PutVarint64(&str, blob_file_number); |
||||||
|
slice = str; |
||||||
|
|
||||||
|
{ |
||||||
|
const Status s = blob_file_addition.DecodeFrom(&slice); |
||||||
|
ASSERT_TRUE(s.IsCorruption()); |
||||||
|
ASSERT_TRUE(std::strstr(s.getState(), "total blob count")); |
||||||
|
} |
||||||
|
|
||||||
|
constexpr uint64_t total_blob_count = 4567; |
||||||
|
PutVarint64(&str, total_blob_count); |
||||||
|
slice = str; |
||||||
|
|
||||||
|
{ |
||||||
|
const Status s = blob_file_addition.DecodeFrom(&slice); |
||||||
|
ASSERT_TRUE(s.IsCorruption()); |
||||||
|
ASSERT_TRUE(std::strstr(s.getState(), "total blob bytes")); |
||||||
|
} |
||||||
|
|
||||||
|
constexpr uint64_t total_blob_bytes = 12345678; |
||||||
|
PutVarint64(&str, total_blob_bytes); |
||||||
|
slice = str; |
||||||
|
|
||||||
|
{ |
||||||
|
const Status s = blob_file_addition.DecodeFrom(&slice); |
||||||
|
ASSERT_TRUE(s.IsCorruption()); |
||||||
|
ASSERT_TRUE(std::strstr(s.getState(), "checksum method")); |
||||||
|
} |
||||||
|
|
||||||
|
constexpr char checksum_method[] = "SHA1"; |
||||||
|
PutLengthPrefixedSlice(&str, checksum_method); |
||||||
|
slice = str; |
||||||
|
|
||||||
|
{ |
||||||
|
const Status s = blob_file_addition.DecodeFrom(&slice); |
||||||
|
ASSERT_TRUE(s.IsCorruption()); |
||||||
|
ASSERT_TRUE(std::strstr(s.getState(), "checksum value")); |
||||||
|
} |
||||||
|
|
||||||
|
constexpr char checksum_value[] = "bdb7f34a59dfa1592ce7f52e99f98c570c525cbd"; |
||||||
|
PutLengthPrefixedSlice(&str, checksum_value); |
||||||
|
slice = str; |
||||||
|
|
||||||
|
{ |
||||||
|
const Status s = blob_file_addition.DecodeFrom(&slice); |
||||||
|
ASSERT_TRUE(s.IsCorruption()); |
||||||
|
ASSERT_TRUE(std::strstr(s.getState(), "custom field tag")); |
||||||
|
} |
||||||
|
|
||||||
|
constexpr uint32_t custom_tag = 2; |
||||||
|
PutVarint32(&str, custom_tag); |
||||||
|
slice = str; |
||||||
|
|
||||||
|
{ |
||||||
|
const Status s = blob_file_addition.DecodeFrom(&slice); |
||||||
|
ASSERT_TRUE(s.IsCorruption()); |
||||||
|
ASSERT_TRUE(std::strstr(s.getState(), "custom field value")); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(BlobFileAdditionTest, ForwardCompatibleCustomField) { |
||||||
|
SyncPoint::GetInstance()->SetCallBack( |
||||||
|
"BlobFileAddition::EncodeTo::CustomFields", [&](void* arg) { |
||||||
|
std::string* output = static_cast<std::string*>(arg); |
||||||
|
|
||||||
|
constexpr uint32_t forward_compatible_tag = 2; |
||||||
|
PutVarint32(output, forward_compatible_tag); |
||||||
|
|
||||||
|
PutLengthPrefixedSlice(output, "deadbeef"); |
||||||
|
}); |
||||||
|
SyncPoint::GetInstance()->EnableProcessing(); |
||||||
|
|
||||||
|
constexpr uint64_t blob_file_number = 678; |
||||||
|
constexpr uint64_t total_blob_count = 9999; |
||||||
|
constexpr uint64_t total_blob_bytes = 100000000; |
||||||
|
const std::string checksum_method("CRC32"); |
||||||
|
const std::string checksum_value("3d87ff57"); |
||||||
|
|
||||||
|
BlobFileAddition blob_file_addition(blob_file_number, total_blob_count, |
||||||
|
total_blob_bytes, checksum_method, |
||||||
|
checksum_value); |
||||||
|
|
||||||
|
TestEncodeDecode(blob_file_addition); |
||||||
|
|
||||||
|
SyncPoint::GetInstance()->DisableProcessing(); |
||||||
|
SyncPoint::GetInstance()->ClearAllCallBacks(); |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(BlobFileAdditionTest, ForwardIncompatibleCustomField) { |
||||||
|
SyncPoint::GetInstance()->SetCallBack( |
||||||
|
"BlobFileAddition::EncodeTo::CustomFields", [&](void* arg) { |
||||||
|
std::string* output = static_cast<std::string*>(arg); |
||||||
|
|
||||||
|
constexpr uint32_t forward_incompatible_tag = (1 << 6) + 1; |
||||||
|
PutVarint32(output, forward_incompatible_tag); |
||||||
|
|
||||||
|
PutLengthPrefixedSlice(output, "foobar"); |
||||||
|
}); |
||||||
|
SyncPoint::GetInstance()->EnableProcessing(); |
||||||
|
|
||||||
|
constexpr uint64_t blob_file_number = 456; |
||||||
|
constexpr uint64_t total_blob_count = 100; |
||||||
|
constexpr uint64_t total_blob_bytes = 2000000; |
||||||
|
const std::string checksum_method("CRC32B"); |
||||||
|
const std::string checksum_value("6dbdf23a"); |
||||||
|
|
||||||
|
BlobFileAddition blob_file_addition(blob_file_number, total_blob_count, |
||||||
|
total_blob_bytes, checksum_method, |
||||||
|
checksum_value); |
||||||
|
|
||||||
|
std::string encoded; |
||||||
|
blob_file_addition.EncodeTo(&encoded); |
||||||
|
|
||||||
|
BlobFileAddition decoded_blob_file_addition; |
||||||
|
Slice input(encoded); |
||||||
|
const Status s = decoded_blob_file_addition.DecodeFrom(&input); |
||||||
|
|
||||||
|
ASSERT_TRUE(s.IsCorruption()); |
||||||
|
ASSERT_TRUE(std::strstr(s.getState(), "Forward incompatible")); |
||||||
|
|
||||||
|
SyncPoint::GetInstance()->DisableProcessing(); |
||||||
|
SyncPoint::GetInstance()->ClearAllCallBacks(); |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
|
|
||||||
|
int main(int argc, char** argv) { |
||||||
|
::testing::InitGoogleTest(&argc, argv); |
||||||
|
return RUN_ALL_TESTS(); |
||||||
|
} |
||||||
@ -0,0 +1,138 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
|
#include "db/blob_file_garbage.h" |
||||||
|
|
||||||
|
#include <ostream> |
||||||
|
#include <sstream> |
||||||
|
|
||||||
|
#include "logging/event_logger.h" |
||||||
|
#include "rocksdb/slice.h" |
||||||
|
#include "rocksdb/status.h" |
||||||
|
#include "test_util/sync_point.h" |
||||||
|
#include "util/coding.h" |
||||||
|
|
||||||
|
namespace ROCKSDB_NAMESPACE { |
||||||
|
|
||||||
|
namespace { |
||||||
|
|
||||||
|
// Tags for custom fields. Note that these get persisted in the manifest,
|
||||||
|
// so existing tags should not be modified.
|
||||||
|
enum CustomFieldTags : uint32_t { |
||||||
|
kEndMarker, |
||||||
|
|
||||||
|
// Add forward compatible fields here
|
||||||
|
|
||||||
|
/////////////////////////////////////////////////////////////////////
|
||||||
|
|
||||||
|
kForwardIncompatibleMask = 1 << 6, |
||||||
|
|
||||||
|
// Add forward incompatible fields here
|
||||||
|
}; |
||||||
|
|
||||||
|
} // anonymous namespace
|
||||||
|
|
||||||
|
void BlobFileGarbage::EncodeTo(std::string* output) const { |
||||||
|
PutVarint64(output, blob_file_number_); |
||||||
|
PutVarint64(output, garbage_blob_count_); |
||||||
|
PutVarint64(output, garbage_blob_bytes_); |
||||||
|
|
||||||
|
// Encode any custom fields here. The format to use is a Varint32 tag (see
|
||||||
|
// CustomFieldTags above) followed by a length prefixed slice. Unknown custom
|
||||||
|
// fields will be ignored during decoding unless they're in the forward
|
||||||
|
// incompatible range.
|
||||||
|
|
||||||
|
TEST_SYNC_POINT_CALLBACK("BlobFileGarbage::EncodeTo::CustomFields", output); |
||||||
|
|
||||||
|
PutVarint32(output, kEndMarker); |
||||||
|
} |
||||||
|
|
||||||
|
Status BlobFileGarbage::DecodeFrom(Slice* input) { |
||||||
|
constexpr char class_name[] = "BlobFileGarbage"; |
||||||
|
|
||||||
|
if (!GetVarint64(input, &blob_file_number_)) { |
||||||
|
return Status::Corruption(class_name, "Error decoding blob file number"); |
||||||
|
} |
||||||
|
|
||||||
|
if (!GetVarint64(input, &garbage_blob_count_)) { |
||||||
|
return Status::Corruption(class_name, "Error decoding garbage blob count"); |
||||||
|
} |
||||||
|
|
||||||
|
if (!GetVarint64(input, &garbage_blob_bytes_)) { |
||||||
|
return Status::Corruption(class_name, "Error decoding garbage blob bytes"); |
||||||
|
} |
||||||
|
|
||||||
|
while (true) { |
||||||
|
uint32_t custom_field_tag = 0; |
||||||
|
if (!GetVarint32(input, &custom_field_tag)) { |
||||||
|
return Status::Corruption(class_name, "Error decoding custom field tag"); |
||||||
|
} |
||||||
|
|
||||||
|
if (custom_field_tag == kEndMarker) { |
||||||
|
break; |
||||||
|
} |
||||||
|
|
||||||
|
if (custom_field_tag & kForwardIncompatibleMask) { |
||||||
|
return Status::Corruption( |
||||||
|
class_name, "Forward incompatible custom field encountered"); |
||||||
|
} |
||||||
|
|
||||||
|
Slice custom_field_value; |
||||||
|
if (!GetLengthPrefixedSlice(input, &custom_field_value)) { |
||||||
|
return Status::Corruption(class_name, |
||||||
|
"Error decoding custom field value"); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
std::string BlobFileGarbage::DebugString() const { |
||||||
|
std::ostringstream oss; |
||||||
|
|
||||||
|
oss << *this; |
||||||
|
|
||||||
|
return oss.str(); |
||||||
|
} |
||||||
|
|
||||||
|
std::string BlobFileGarbage::DebugJSON() const { |
||||||
|
JSONWriter jw; |
||||||
|
|
||||||
|
jw << *this; |
||||||
|
|
||||||
|
jw.EndObject(); |
||||||
|
|
||||||
|
return jw.Get(); |
||||||
|
} |
||||||
|
|
||||||
|
bool operator==(const BlobFileGarbage& lhs, const BlobFileGarbage& rhs) { |
||||||
|
return lhs.GetBlobFileNumber() == rhs.GetBlobFileNumber() && |
||||||
|
lhs.GetGarbageBlobCount() == rhs.GetGarbageBlobCount() && |
||||||
|
lhs.GetGarbageBlobBytes() == rhs.GetGarbageBlobBytes(); |
||||||
|
} |
||||||
|
|
||||||
|
bool operator!=(const BlobFileGarbage& lhs, const BlobFileGarbage& rhs) { |
||||||
|
return !(lhs == rhs); |
||||||
|
} |
||||||
|
|
||||||
|
std::ostream& operator<<(std::ostream& os, |
||||||
|
const BlobFileGarbage& blob_file_garbage) { |
||||||
|
os << "blob_file_number: " << blob_file_garbage.GetBlobFileNumber() |
||||||
|
<< " garbage_blob_count: " << blob_file_garbage.GetGarbageBlobCount() |
||||||
|
<< " garbage_blob_bytes: " << blob_file_garbage.GetGarbageBlobBytes(); |
||||||
|
|
||||||
|
return os; |
||||||
|
} |
||||||
|
|
||||||
|
JSONWriter& operator<<(JSONWriter& jw, |
||||||
|
const BlobFileGarbage& blob_file_garbage) { |
||||||
|
jw << "BlobFileNumber" << blob_file_garbage.GetBlobFileNumber() |
||||||
|
<< "GarbageBlobCount" << blob_file_garbage.GetGarbageBlobCount() |
||||||
|
<< "GarbageBlobBytes" << blob_file_garbage.GetGarbageBlobBytes(); |
||||||
|
|
||||||
|
return jw; |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
@ -0,0 +1,55 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <cstdint> |
||||||
|
#include <iosfwd> |
||||||
|
#include <string> |
||||||
|
|
||||||
|
#include "db/blob_constants.h" |
||||||
|
#include "rocksdb/rocksdb_namespace.h" |
||||||
|
|
||||||
|
namespace ROCKSDB_NAMESPACE { |
||||||
|
|
||||||
|
class JSONWriter; |
||||||
|
class Slice; |
||||||
|
class Status; |
||||||
|
|
||||||
|
class BlobFileGarbage { |
||||||
|
public: |
||||||
|
BlobFileGarbage() = default; |
||||||
|
|
||||||
|
BlobFileGarbage(uint64_t blob_file_number, uint64_t garbage_blob_count, |
||||||
|
uint64_t garbage_blob_bytes) |
||||||
|
: blob_file_number_(blob_file_number), |
||||||
|
garbage_blob_count_(garbage_blob_count), |
||||||
|
garbage_blob_bytes_(garbage_blob_bytes) {} |
||||||
|
|
||||||
|
uint64_t GetBlobFileNumber() const { return blob_file_number_; } |
||||||
|
uint64_t GetGarbageBlobCount() const { return garbage_blob_count_; } |
||||||
|
uint64_t GetGarbageBlobBytes() const { return garbage_blob_bytes_; } |
||||||
|
|
||||||
|
void EncodeTo(std::string* output) const; |
||||||
|
Status DecodeFrom(Slice* input); |
||||||
|
|
||||||
|
std::string DebugString() const; |
||||||
|
std::string DebugJSON() const; |
||||||
|
|
||||||
|
private: |
||||||
|
uint64_t blob_file_number_ = kInvalidBlobFileNumber; |
||||||
|
uint64_t garbage_blob_count_ = 0; |
||||||
|
uint64_t garbage_blob_bytes_ = 0; |
||||||
|
}; |
||||||
|
|
||||||
|
bool operator==(const BlobFileGarbage& lhs, const BlobFileGarbage& rhs); |
||||||
|
bool operator!=(const BlobFileGarbage& lhs, const BlobFileGarbage& rhs); |
||||||
|
|
||||||
|
std::ostream& operator<<(std::ostream& os, |
||||||
|
const BlobFileGarbage& blob_file_garbage); |
||||||
|
JSONWriter& operator<<(JSONWriter& jw, |
||||||
|
const BlobFileGarbage& blob_file_garbage); |
||||||
|
|
||||||
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
@ -0,0 +1,173 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
|
||||||
|
#include "db/blob_file_garbage.h" |
||||||
|
|
||||||
|
#include <cstdint> |
||||||
|
#include <cstring> |
||||||
|
#include <string> |
||||||
|
|
||||||
|
#include "test_util/sync_point.h" |
||||||
|
#include "test_util/testharness.h" |
||||||
|
#include "util/coding.h" |
||||||
|
|
||||||
|
namespace ROCKSDB_NAMESPACE { |
||||||
|
|
||||||
|
class BlobFileGarbageTest : public testing::Test { |
||||||
|
public: |
||||||
|
static void TestEncodeDecode(const BlobFileGarbage& blob_file_garbage) { |
||||||
|
std::string encoded; |
||||||
|
blob_file_garbage.EncodeTo(&encoded); |
||||||
|
|
||||||
|
BlobFileGarbage decoded; |
||||||
|
Slice input(encoded); |
||||||
|
ASSERT_OK(decoded.DecodeFrom(&input)); |
||||||
|
|
||||||
|
ASSERT_EQ(blob_file_garbage, decoded); |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
TEST_F(BlobFileGarbageTest, Empty) { |
||||||
|
BlobFileGarbage blob_file_garbage; |
||||||
|
|
||||||
|
ASSERT_EQ(blob_file_garbage.GetBlobFileNumber(), kInvalidBlobFileNumber); |
||||||
|
ASSERT_EQ(blob_file_garbage.GetGarbageBlobCount(), 0); |
||||||
|
ASSERT_EQ(blob_file_garbage.GetGarbageBlobBytes(), 0); |
||||||
|
|
||||||
|
TestEncodeDecode(blob_file_garbage); |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(BlobFileGarbageTest, NonEmpty) { |
||||||
|
constexpr uint64_t blob_file_number = 123; |
||||||
|
constexpr uint64_t garbage_blob_count = 1; |
||||||
|
constexpr uint64_t garbage_blob_bytes = 9876; |
||||||
|
|
||||||
|
BlobFileGarbage blob_file_garbage(blob_file_number, garbage_blob_count, |
||||||
|
garbage_blob_bytes); |
||||||
|
|
||||||
|
ASSERT_EQ(blob_file_garbage.GetBlobFileNumber(), blob_file_number); |
||||||
|
ASSERT_EQ(blob_file_garbage.GetGarbageBlobCount(), garbage_blob_count); |
||||||
|
ASSERT_EQ(blob_file_garbage.GetGarbageBlobBytes(), garbage_blob_bytes); |
||||||
|
|
||||||
|
TestEncodeDecode(blob_file_garbage); |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(BlobFileGarbageTest, DecodeErrors) { |
||||||
|
std::string str; |
||||||
|
Slice slice(str); |
||||||
|
|
||||||
|
BlobFileGarbage blob_file_garbage; |
||||||
|
|
||||||
|
{ |
||||||
|
const Status s = blob_file_garbage.DecodeFrom(&slice); |
||||||
|
ASSERT_TRUE(s.IsCorruption()); |
||||||
|
ASSERT_TRUE(std::strstr(s.getState(), "blob file number")); |
||||||
|
} |
||||||
|
|
||||||
|
constexpr uint64_t blob_file_number = 123; |
||||||
|
PutVarint64(&str, blob_file_number); |
||||||
|
slice = str; |
||||||
|
|
||||||
|
{ |
||||||
|
const Status s = blob_file_garbage.DecodeFrom(&slice); |
||||||
|
ASSERT_TRUE(s.IsCorruption()); |
||||||
|
ASSERT_TRUE(std::strstr(s.getState(), "garbage blob count")); |
||||||
|
} |
||||||
|
|
||||||
|
constexpr uint64_t garbage_blob_count = 4567; |
||||||
|
PutVarint64(&str, garbage_blob_count); |
||||||
|
slice = str; |
||||||
|
|
||||||
|
{ |
||||||
|
const Status s = blob_file_garbage.DecodeFrom(&slice); |
||||||
|
ASSERT_TRUE(s.IsCorruption()); |
||||||
|
ASSERT_TRUE(std::strstr(s.getState(), "garbage blob bytes")); |
||||||
|
} |
||||||
|
|
||||||
|
constexpr uint64_t garbage_blob_bytes = 12345678; |
||||||
|
PutVarint64(&str, garbage_blob_bytes); |
||||||
|
slice = str; |
||||||
|
|
||||||
|
{ |
||||||
|
const Status s = blob_file_garbage.DecodeFrom(&slice); |
||||||
|
ASSERT_TRUE(s.IsCorruption()); |
||||||
|
ASSERT_TRUE(std::strstr(s.getState(), "custom field tag")); |
||||||
|
} |
||||||
|
|
||||||
|
constexpr uint32_t custom_tag = 2; |
||||||
|
PutVarint32(&str, custom_tag); |
||||||
|
slice = str; |
||||||
|
|
||||||
|
{ |
||||||
|
const Status s = blob_file_garbage.DecodeFrom(&slice); |
||||||
|
ASSERT_TRUE(s.IsCorruption()); |
||||||
|
ASSERT_TRUE(std::strstr(s.getState(), "custom field value")); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(BlobFileGarbageTest, ForwardCompatibleCustomField) { |
||||||
|
SyncPoint::GetInstance()->SetCallBack( |
||||||
|
"BlobFileGarbage::EncodeTo::CustomFields", [&](void* arg) { |
||||||
|
std::string* output = static_cast<std::string*>(arg); |
||||||
|
|
||||||
|
constexpr uint32_t forward_compatible_tag = 2; |
||||||
|
PutVarint32(output, forward_compatible_tag); |
||||||
|
|
||||||
|
PutLengthPrefixedSlice(output, "deadbeef"); |
||||||
|
}); |
||||||
|
SyncPoint::GetInstance()->EnableProcessing(); |
||||||
|
|
||||||
|
constexpr uint64_t blob_file_number = 678; |
||||||
|
constexpr uint64_t garbage_blob_count = 9999; |
||||||
|
constexpr uint64_t garbage_blob_bytes = 100000000; |
||||||
|
|
||||||
|
BlobFileGarbage blob_file_garbage(blob_file_number, garbage_blob_count, |
||||||
|
garbage_blob_bytes); |
||||||
|
|
||||||
|
TestEncodeDecode(blob_file_garbage); |
||||||
|
|
||||||
|
SyncPoint::GetInstance()->DisableProcessing(); |
||||||
|
SyncPoint::GetInstance()->ClearAllCallBacks(); |
||||||
|
} |
||||||
|
|
||||||
|
TEST_F(BlobFileGarbageTest, ForwardIncompatibleCustomField) { |
||||||
|
SyncPoint::GetInstance()->SetCallBack( |
||||||
|
"BlobFileGarbage::EncodeTo::CustomFields", [&](void* arg) { |
||||||
|
std::string* output = static_cast<std::string*>(arg); |
||||||
|
|
||||||
|
constexpr uint32_t forward_incompatible_tag = (1 << 6) + 1; |
||||||
|
PutVarint32(output, forward_incompatible_tag); |
||||||
|
|
||||||
|
PutLengthPrefixedSlice(output, "foobar"); |
||||||
|
}); |
||||||
|
SyncPoint::GetInstance()->EnableProcessing(); |
||||||
|
|
||||||
|
constexpr uint64_t blob_file_number = 456; |
||||||
|
constexpr uint64_t garbage_blob_count = 100; |
||||||
|
constexpr uint64_t garbage_blob_bytes = 2000000; |
||||||
|
|
||||||
|
BlobFileGarbage blob_file_garbage(blob_file_number, garbage_blob_count, |
||||||
|
garbage_blob_bytes); |
||||||
|
|
||||||
|
std::string encoded; |
||||||
|
blob_file_garbage.EncodeTo(&encoded); |
||||||
|
|
||||||
|
BlobFileGarbage decoded_blob_file_addition; |
||||||
|
Slice input(encoded); |
||||||
|
const Status s = decoded_blob_file_addition.DecodeFrom(&input); |
||||||
|
|
||||||
|
ASSERT_TRUE(s.IsCorruption()); |
||||||
|
ASSERT_TRUE(std::strstr(s.getState(), "Forward incompatible")); |
||||||
|
|
||||||
|
SyncPoint::GetInstance()->DisableProcessing(); |
||||||
|
SyncPoint::GetInstance()->ClearAllCallBacks(); |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
|
|
||||||
|
int main(int argc, char** argv) { |
||||||
|
::testing::InitGoogleTest(&argc, argv); |
||||||
|
return RUN_ALL_TESTS(); |
||||||
|
} |
||||||
@ -1,102 +0,0 @@ |
|||||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
||||||
// This source code is licensed under both the GPLv2 (found in the
|
|
||||||
// COPYING file in the root directory) and Apache 2.0 License
|
|
||||||
// (found in the LICENSE.Apache file in the root directory).
|
|
||||||
|
|
||||||
#pragma once |
|
||||||
|
|
||||||
#include "rocksdb/rocksdb_namespace.h" |
|
||||||
|
|
||||||
#include <cassert> |
|
||||||
#include <cstdint> |
|
||||||
#include <iosfwd> |
|
||||||
#include <string> |
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE { |
|
||||||
|
|
||||||
constexpr uint64_t kInvalidBlobFileNumber = 0; |
|
||||||
|
|
||||||
class JSONWriter; |
|
||||||
class Slice; |
|
||||||
class Status; |
|
||||||
|
|
||||||
class BlobFileState { |
|
||||||
public: |
|
||||||
BlobFileState() = default; |
|
||||||
|
|
||||||
BlobFileState(uint64_t blob_file_number, uint64_t total_blob_count, |
|
||||||
uint64_t total_blob_bytes, std::string checksum_method, |
|
||||||
std::string checksum_value) |
|
||||||
: blob_file_number_(blob_file_number), |
|
||||||
total_blob_count_(total_blob_count), |
|
||||||
total_blob_bytes_(total_blob_bytes), |
|
||||||
checksum_method_(std::move(checksum_method)), |
|
||||||
checksum_value_(std::move(checksum_value)) { |
|
||||||
assert(checksum_method_.empty() == checksum_value_.empty()); |
|
||||||
} |
|
||||||
|
|
||||||
BlobFileState(uint64_t blob_file_number, uint64_t total_blob_count, |
|
||||||
uint64_t total_blob_bytes, uint64_t garbage_blob_count, |
|
||||||
uint64_t garbage_blob_bytes, std::string checksum_method, |
|
||||||
std::string checksum_value) |
|
||||||
: blob_file_number_(blob_file_number), |
|
||||||
total_blob_count_(total_blob_count), |
|
||||||
total_blob_bytes_(total_blob_bytes), |
|
||||||
garbage_blob_count_(garbage_blob_count), |
|
||||||
garbage_blob_bytes_(garbage_blob_bytes), |
|
||||||
checksum_method_(std::move(checksum_method)), |
|
||||||
checksum_value_(std::move(checksum_value)) { |
|
||||||
assert(checksum_method_.empty() == checksum_value_.empty()); |
|
||||||
assert(garbage_blob_count_ <= total_blob_count_); |
|
||||||
assert(garbage_blob_bytes_ <= total_blob_bytes_); |
|
||||||
} |
|
||||||
|
|
||||||
uint64_t GetBlobFileNumber() const { return blob_file_number_; } |
|
||||||
|
|
||||||
uint64_t GetTotalBlobCount() const { return total_blob_count_; } |
|
||||||
uint64_t GetTotalBlobBytes() const { return total_blob_bytes_; } |
|
||||||
|
|
||||||
void AddGarbageBlob(uint64_t size) { |
|
||||||
assert(garbage_blob_count_ < total_blob_count_); |
|
||||||
assert(garbage_blob_bytes_ + size <= total_blob_bytes_); |
|
||||||
|
|
||||||
++garbage_blob_count_; |
|
||||||
garbage_blob_bytes_ += size; |
|
||||||
} |
|
||||||
|
|
||||||
uint64_t GetGarbageBlobCount() const { return garbage_blob_count_; } |
|
||||||
uint64_t GetGarbageBlobBytes() const { return garbage_blob_bytes_; } |
|
||||||
|
|
||||||
bool IsObsolete() const { |
|
||||||
assert(garbage_blob_count_ <= total_blob_count_); |
|
||||||
|
|
||||||
return !(garbage_blob_count_ < total_blob_count_); |
|
||||||
} |
|
||||||
|
|
||||||
const std::string& GetChecksumMethod() const { return checksum_method_; } |
|
||||||
const std::string& GetChecksumValue() const { return checksum_value_; } |
|
||||||
|
|
||||||
void EncodeTo(std::string* output) const; |
|
||||||
Status DecodeFrom(Slice* input); |
|
||||||
|
|
||||||
std::string DebugString() const; |
|
||||||
std::string DebugJSON() const; |
|
||||||
|
|
||||||
private: |
|
||||||
uint64_t blob_file_number_ = kInvalidBlobFileNumber; |
|
||||||
uint64_t total_blob_count_ = 0; |
|
||||||
uint64_t total_blob_bytes_ = 0; |
|
||||||
uint64_t garbage_blob_count_ = 0; |
|
||||||
uint64_t garbage_blob_bytes_ = 0; |
|
||||||
std::string checksum_method_; |
|
||||||
std::string checksum_value_; |
|
||||||
}; |
|
||||||
|
|
||||||
bool operator==(const BlobFileState& lhs, const BlobFileState& rhs); |
|
||||||
bool operator!=(const BlobFileState& lhs, const BlobFileState& rhs); |
|
||||||
|
|
||||||
std::ostream& operator<<(std::ostream& os, |
|
||||||
const BlobFileState& blob_file_state); |
|
||||||
JSONWriter& operator<<(JSONWriter& jw, const BlobFileState& blob_file_state); |
|
||||||
|
|
||||||
} // namespace ROCKSDB_NAMESPACE
|
|
||||||
@ -1,284 +0,0 @@ |
|||||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
||||||
// This source code is licensed under both the GPLv2 (found in the
|
|
||||||
// COPYING file in the root directory) and Apache 2.0 License
|
|
||||||
// (found in the LICENSE.Apache file in the root directory).
|
|
||||||
|
|
||||||
#include "db/blob_file_state.h" |
|
||||||
#include "test_util/sync_point.h" |
|
||||||
#include "test_util/testharness.h" |
|
||||||
#include "util/coding.h" |
|
||||||
|
|
||||||
#include <cstdint> |
|
||||||
#include <cstring> |
|
||||||
#include <string> |
|
||||||
|
|
||||||
namespace ROCKSDB_NAMESPACE { |
|
||||||
|
|
||||||
class BlobFileStateTest : public testing::Test { |
|
||||||
public: |
|
||||||
static void TestEncodeDecode(const BlobFileState& blob_file_state) { |
|
||||||
std::string encoded; |
|
||||||
blob_file_state.EncodeTo(&encoded); |
|
||||||
|
|
||||||
BlobFileState decoded; |
|
||||||
Slice input(encoded); |
|
||||||
ASSERT_OK(decoded.DecodeFrom(&input)); |
|
||||||
|
|
||||||
ASSERT_EQ(blob_file_state, decoded); |
|
||||||
} |
|
||||||
}; |
|
||||||
|
|
||||||
TEST_F(BlobFileStateTest, Empty) { |
|
||||||
BlobFileState blob_file_state; |
|
||||||
|
|
||||||
ASSERT_EQ(blob_file_state.GetBlobFileNumber(), kInvalidBlobFileNumber); |
|
||||||
ASSERT_EQ(blob_file_state.GetTotalBlobCount(), 0); |
|
||||||
ASSERT_EQ(blob_file_state.GetTotalBlobBytes(), 0); |
|
||||||
ASSERT_EQ(blob_file_state.GetGarbageBlobCount(), 0); |
|
||||||
ASSERT_EQ(blob_file_state.GetGarbageBlobBytes(), 0); |
|
||||||
ASSERT_TRUE(blob_file_state.IsObsolete()); |
|
||||||
ASSERT_TRUE(blob_file_state.GetChecksumMethod().empty()); |
|
||||||
ASSERT_TRUE(blob_file_state.GetChecksumValue().empty()); |
|
||||||
|
|
||||||
TestEncodeDecode(blob_file_state); |
|
||||||
} |
|
||||||
|
|
||||||
TEST_F(BlobFileStateTest, NonEmpty) { |
|
||||||
constexpr uint64_t blob_file_number = 123; |
|
||||||
constexpr uint64_t total_blob_count = 2; |
|
||||||
constexpr uint64_t total_blob_bytes = 123456; |
|
||||||
constexpr uint64_t garbage_blob_count = 1; |
|
||||||
constexpr uint64_t garbage_blob_bytes = 9876; |
|
||||||
const std::string checksum_method("SHA1"); |
|
||||||
const std::string checksum_value("bdb7f34a59dfa1592ce7f52e99f98c570c525cbd"); |
|
||||||
|
|
||||||
BlobFileState blob_file_state( |
|
||||||
blob_file_number, total_blob_count, total_blob_bytes, garbage_blob_count, |
|
||||||
garbage_blob_bytes, checksum_method, checksum_value); |
|
||||||
|
|
||||||
ASSERT_EQ(blob_file_state.GetBlobFileNumber(), blob_file_number); |
|
||||||
ASSERT_EQ(blob_file_state.GetTotalBlobCount(), total_blob_count); |
|
||||||
ASSERT_EQ(blob_file_state.GetTotalBlobBytes(), total_blob_bytes); |
|
||||||
ASSERT_EQ(blob_file_state.GetGarbageBlobCount(), garbage_blob_count); |
|
||||||
ASSERT_EQ(blob_file_state.GetGarbageBlobBytes(), garbage_blob_bytes); |
|
||||||
ASSERT_FALSE(blob_file_state.IsObsolete()); |
|
||||||
ASSERT_EQ(blob_file_state.GetChecksumMethod(), checksum_method); |
|
||||||
ASSERT_EQ(blob_file_state.GetChecksumValue(), checksum_value); |
|
||||||
|
|
||||||
TestEncodeDecode(blob_file_state); |
|
||||||
} |
|
||||||
|
|
||||||
TEST_F(BlobFileStateTest, AddGarbageBlob) { |
|
||||||
constexpr uint64_t blob_file_number = 123; |
|
||||||
constexpr uint64_t total_blob_count = 2; |
|
||||||
constexpr uint64_t total_blob_bytes = 123456; |
|
||||||
const std::string checksum_method("MD5"); |
|
||||||
const std::string checksum_value("d8f72233c67a68c5ec2bd51c6be7556e"); |
|
||||||
|
|
||||||
BlobFileState blob_file_state(blob_file_number, total_blob_count, |
|
||||||
total_blob_bytes, checksum_method, |
|
||||||
checksum_value); |
|
||||||
|
|
||||||
ASSERT_EQ(blob_file_state.GetBlobFileNumber(), blob_file_number); |
|
||||||
ASSERT_EQ(blob_file_state.GetTotalBlobCount(), total_blob_count); |
|
||||||
ASSERT_EQ(blob_file_state.GetTotalBlobBytes(), total_blob_bytes); |
|
||||||
ASSERT_EQ(blob_file_state.GetGarbageBlobCount(), 0); |
|
||||||
ASSERT_EQ(blob_file_state.GetGarbageBlobBytes(), 0); |
|
||||||
ASSERT_FALSE(blob_file_state.IsObsolete()); |
|
||||||
ASSERT_EQ(blob_file_state.GetChecksumMethod(), checksum_method); |
|
||||||
ASSERT_EQ(blob_file_state.GetChecksumValue(), checksum_value); |
|
||||||
|
|
||||||
TestEncodeDecode(blob_file_state); |
|
||||||
|
|
||||||
blob_file_state.AddGarbageBlob(123000); |
|
||||||
|
|
||||||
ASSERT_EQ(blob_file_state.GetBlobFileNumber(), blob_file_number); |
|
||||||
ASSERT_EQ(blob_file_state.GetTotalBlobCount(), total_blob_count); |
|
||||||
ASSERT_EQ(blob_file_state.GetTotalBlobBytes(), total_blob_bytes); |
|
||||||
ASSERT_EQ(blob_file_state.GetGarbageBlobCount(), 1); |
|
||||||
ASSERT_EQ(blob_file_state.GetGarbageBlobBytes(), 123000); |
|
||||||
ASSERT_FALSE(blob_file_state.IsObsolete()); |
|
||||||
ASSERT_EQ(blob_file_state.GetChecksumMethod(), checksum_method); |
|
||||||
ASSERT_EQ(blob_file_state.GetChecksumValue(), checksum_value); |
|
||||||
|
|
||||||
TestEncodeDecode(blob_file_state); |
|
||||||
|
|
||||||
blob_file_state.AddGarbageBlob(456); |
|
||||||
|
|
||||||
ASSERT_EQ(blob_file_state.GetBlobFileNumber(), blob_file_number); |
|
||||||
ASSERT_EQ(blob_file_state.GetTotalBlobCount(), total_blob_count); |
|
||||||
ASSERT_EQ(blob_file_state.GetTotalBlobBytes(), total_blob_bytes); |
|
||||||
ASSERT_EQ(blob_file_state.GetGarbageBlobCount(), total_blob_count); |
|
||||||
ASSERT_EQ(blob_file_state.GetGarbageBlobBytes(), total_blob_bytes); |
|
||||||
ASSERT_TRUE(blob_file_state.IsObsolete()); |
|
||||||
ASSERT_EQ(blob_file_state.GetChecksumMethod(), checksum_method); |
|
||||||
ASSERT_EQ(blob_file_state.GetChecksumValue(), checksum_value); |
|
||||||
|
|
||||||
TestEncodeDecode(blob_file_state); |
|
||||||
} |
|
||||||
|
|
||||||
TEST_F(BlobFileStateTest, DecodeErrors) { |
|
||||||
std::string str; |
|
||||||
Slice slice(str); |
|
||||||
|
|
||||||
BlobFileState blob_file_state; |
|
||||||
|
|
||||||
{ |
|
||||||
const Status s = blob_file_state.DecodeFrom(&slice); |
|
||||||
ASSERT_TRUE(s.IsCorruption()); |
|
||||||
ASSERT_TRUE(std::strstr(s.getState(), "blob file number")); |
|
||||||
} |
|
||||||
|
|
||||||
constexpr uint64_t blob_file_number = 123; |
|
||||||
PutVarint64(&str, blob_file_number); |
|
||||||
slice = str; |
|
||||||
|
|
||||||
{ |
|
||||||
const Status s = blob_file_state.DecodeFrom(&slice); |
|
||||||
ASSERT_TRUE(s.IsCorruption()); |
|
||||||
ASSERT_TRUE(std::strstr(s.getState(), "total blob count")); |
|
||||||
} |
|
||||||
|
|
||||||
constexpr uint64_t total_blob_count = 4567; |
|
||||||
PutVarint64(&str, total_blob_count); |
|
||||||
slice = str; |
|
||||||
|
|
||||||
{ |
|
||||||
const Status s = blob_file_state.DecodeFrom(&slice); |
|
||||||
ASSERT_TRUE(s.IsCorruption()); |
|
||||||
ASSERT_TRUE(std::strstr(s.getState(), "total blob bytes")); |
|
||||||
} |
|
||||||
|
|
||||||
constexpr uint64_t total_blob_bytes = 12345678; |
|
||||||
PutVarint64(&str, total_blob_bytes); |
|
||||||
slice = str; |
|
||||||
|
|
||||||
{ |
|
||||||
const Status s = blob_file_state.DecodeFrom(&slice); |
|
||||||
ASSERT_TRUE(s.IsCorruption()); |
|
||||||
ASSERT_TRUE(std::strstr(s.getState(), "garbage blob count")); |
|
||||||
} |
|
||||||
|
|
||||||
constexpr uint64_t garbage_blob_count = 1234; |
|
||||||
PutVarint64(&str, garbage_blob_count); |
|
||||||
slice = str; |
|
||||||
|
|
||||||
{ |
|
||||||
const Status s = blob_file_state.DecodeFrom(&slice); |
|
||||||
ASSERT_TRUE(s.IsCorruption()); |
|
||||||
ASSERT_TRUE(std::strstr(s.getState(), "garbage blob bytes")); |
|
||||||
} |
|
||||||
|
|
||||||
constexpr uint64_t garbage_blob_bytes = 5678; |
|
||||||
PutVarint64(&str, garbage_blob_bytes); |
|
||||||
slice = str; |
|
||||||
|
|
||||||
{ |
|
||||||
const Status s = blob_file_state.DecodeFrom(&slice); |
|
||||||
ASSERT_TRUE(s.IsCorruption()); |
|
||||||
ASSERT_TRUE(std::strstr(s.getState(), "checksum method")); |
|
||||||
} |
|
||||||
|
|
||||||
constexpr char checksum_method[] = "SHA1"; |
|
||||||
PutLengthPrefixedSlice(&str, checksum_method); |
|
||||||
slice = str; |
|
||||||
|
|
||||||
{ |
|
||||||
const Status s = blob_file_state.DecodeFrom(&slice); |
|
||||||
ASSERT_TRUE(s.IsCorruption()); |
|
||||||
ASSERT_TRUE(std::strstr(s.getState(), "checksum value")); |
|
||||||
} |
|
||||||
|
|
||||||
constexpr char checksum_value[] = "bdb7f34a59dfa1592ce7f52e99f98c570c525cbd"; |
|
||||||
PutLengthPrefixedSlice(&str, checksum_value); |
|
||||||
slice = str; |
|
||||||
|
|
||||||
{ |
|
||||||
const Status s = blob_file_state.DecodeFrom(&slice); |
|
||||||
ASSERT_TRUE(s.IsCorruption()); |
|
||||||
ASSERT_TRUE(std::strstr(s.getState(), "custom field tag")); |
|
||||||
} |
|
||||||
|
|
||||||
constexpr uint32_t custom_tag = 2; |
|
||||||
PutVarint32(&str, custom_tag); |
|
||||||
slice = str; |
|
||||||
|
|
||||||
{ |
|
||||||
const Status s = blob_file_state.DecodeFrom(&slice); |
|
||||||
ASSERT_TRUE(s.IsCorruption()); |
|
||||||
ASSERT_TRUE(std::strstr(s.getState(), "custom field value")); |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
TEST_F(BlobFileStateTest, ForwardCompatibleCustomField) { |
|
||||||
SyncPoint::GetInstance()->SetCallBack( |
|
||||||
"BlobFileState::EncodeTo::CustomFields", [&](void* arg) { |
|
||||||
std::string* output = static_cast<std::string*>(arg); |
|
||||||
|
|
||||||
constexpr uint32_t forward_compatible_tag = 2; |
|
||||||
PutVarint32(output, forward_compatible_tag); |
|
||||||
|
|
||||||
PutLengthPrefixedSlice(output, "deadbeef"); |
|
||||||
}); |
|
||||||
SyncPoint::GetInstance()->EnableProcessing(); |
|
||||||
|
|
||||||
constexpr uint64_t blob_file_number = 678; |
|
||||||
constexpr uint64_t total_blob_count = 9999; |
|
||||||
constexpr uint64_t total_blob_bytes = 100000000; |
|
||||||
constexpr uint64_t garbage_blob_count = 3333; |
|
||||||
constexpr uint64_t garbage_blob_bytes = 2500000; |
|
||||||
const std::string checksum_method("CRC32"); |
|
||||||
const std::string checksum_value("3d87ff57"); |
|
||||||
|
|
||||||
BlobFileState blob_file_state( |
|
||||||
blob_file_number, total_blob_count, total_blob_bytes, garbage_blob_count, |
|
||||||
garbage_blob_bytes, checksum_method, checksum_value); |
|
||||||
|
|
||||||
TestEncodeDecode(blob_file_state); |
|
||||||
|
|
||||||
SyncPoint::GetInstance()->DisableProcessing(); |
|
||||||
SyncPoint::GetInstance()->ClearAllCallBacks(); |
|
||||||
} |
|
||||||
|
|
||||||
TEST_F(BlobFileStateTest, ForwardIncompatibleCustomField) { |
|
||||||
SyncPoint::GetInstance()->SetCallBack( |
|
||||||
"BlobFileState::EncodeTo::CustomFields", [&](void* arg) { |
|
||||||
std::string* output = static_cast<std::string*>(arg); |
|
||||||
|
|
||||||
constexpr uint32_t forward_incompatible_tag = (1 << 6) + 1; |
|
||||||
PutVarint32(output, forward_incompatible_tag); |
|
||||||
|
|
||||||
PutLengthPrefixedSlice(output, "foobar"); |
|
||||||
}); |
|
||||||
SyncPoint::GetInstance()->EnableProcessing(); |
|
||||||
|
|
||||||
constexpr uint64_t blob_file_number = 456; |
|
||||||
constexpr uint64_t total_blob_count = 100; |
|
||||||
constexpr uint64_t total_blob_bytes = 2000000; |
|
||||||
const std::string checksum_method("CRC32B"); |
|
||||||
const std::string checksum_value("6dbdf23a"); |
|
||||||
|
|
||||||
BlobFileState blob_file_state(blob_file_number, total_blob_count, |
|
||||||
total_blob_bytes, checksum_method, |
|
||||||
checksum_value); |
|
||||||
|
|
||||||
std::string encoded; |
|
||||||
blob_file_state.EncodeTo(&encoded); |
|
||||||
|
|
||||||
BlobFileState decoded_blob_file_state; |
|
||||||
Slice input(encoded); |
|
||||||
const Status s = decoded_blob_file_state.DecodeFrom(&input); |
|
||||||
|
|
||||||
ASSERT_TRUE(s.IsCorruption()); |
|
||||||
ASSERT_TRUE(std::strstr(s.getState(), "Forward incompatible")); |
|
||||||
|
|
||||||
SyncPoint::GetInstance()->DisableProcessing(); |
|
||||||
SyncPoint::GetInstance()->ClearAllCallBacks(); |
|
||||||
} |
|
||||||
|
|
||||||
} // namespace ROCKSDB_NAMESPACE
|
|
||||||
|
|
||||||
int main(int argc, char** argv) { |
|
||||||
::testing::InitGoogleTest(&argc, argv); |
|
||||||
return RUN_ALL_TESTS(); |
|
||||||
} |
|
||||||
Loading…
Reference in new issue