Introduce options.check_flush_compaction_key_order (#7467)
Summary: Introduce an new option options.check_flush_compaction_key_order, by default set to true, which checks key order of flush and compaction, and fail the operation if the order is violated. Also did minor refactor hash checking code, which consolidates the hashing logic to a vlidation class, where the key ordering logic is added. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7467 Test Plan: Add unit tests to validate the check can catch reordering in flush and compaction, and can be properly disabled. Reviewed By: riversand963 Differential Revision: D24010683 fbshipit-source-id: 8dd6292d2cda8006054e9ded7cfa4bf405f0527cmain
sdong
4 years ago
committed by
Facebook GitHub Bot
parent
3e745053b7
commit
7508175558
@ -0,0 +1,30 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
//
|
||||||
|
#include "db/output_validator.h" |
||||||
|
|
||||||
|
namespace ROCKSDB_NAMESPACE { |
||||||
|
Status OutputValidator::Add(const Slice& key, const Slice& value) { |
||||||
|
if (enable_hash_) { |
||||||
|
// Generate a rolling 64-bit hash of the key and values
|
||||||
|
paranoid_hash_ = Hash64(key.data(), key.size(), paranoid_hash_); |
||||||
|
paranoid_hash_ = Hash64(value.data(), value.size(), paranoid_hash_); |
||||||
|
} |
||||||
|
if (enable_order_check_) { |
||||||
|
TEST_SYNC_POINT_CALLBACK("OutputValidator::Add:order_check", |
||||||
|
/*arg=*/nullptr); |
||||||
|
if (key.size() < kNumInternalBytes) { |
||||||
|
return Status::Corruption( |
||||||
|
"Compaction tries to write a key without internal bytes."); |
||||||
|
} |
||||||
|
// prev_key_ starts with empty.
|
||||||
|
if (!prev_key_.empty() && icmp_.Compare(key, prev_key_) < 0) { |
||||||
|
return Status::Corruption("Compaction sees out-of-order keys."); |
||||||
|
} |
||||||
|
prev_key_.assign(key.data(), key.size()); |
||||||
|
} |
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
@ -0,0 +1,45 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under both the GPLv2 (found in the
|
||||||
|
// COPYING file in the root directory) and Apache 2.0 License
|
||||||
|
// (found in the LICENSE.Apache file in the root directory).
|
||||||
|
//
|
||||||
|
#pragma once |
||||||
|
#include "db/dbformat.h" |
||||||
|
#include "rocksdb/status.h" |
||||||
|
#include "test_util/sync_point.h" |
||||||
|
#include "util/hash.h" |
||||||
|
|
||||||
|
namespace ROCKSDB_NAMESPACE { |
||||||
|
// A class that validates key/value that is inserted to an SST file.
|
||||||
|
// Pass every key/value of the file using OutputValidator::Add()
|
||||||
|
// and the class validates key order and optionally calculate a hash
|
||||||
|
// of all the key and value.
|
||||||
|
class OutputValidator { |
||||||
|
public: |
||||||
|
explicit OutputValidator(const InternalKeyComparator& icmp, |
||||||
|
bool enable_order_check, bool enable_hash) |
||||||
|
: icmp_(icmp), |
||||||
|
enable_order_check_(enable_order_check), |
||||||
|
enable_hash_(enable_hash) {} |
||||||
|
|
||||||
|
// Add a key to the KV sequence, and return whether the key follows
|
||||||
|
// criteria, e.g. key is ordered.
|
||||||
|
Status Add(const Slice& key, const Slice& value); |
||||||
|
|
||||||
|
// Compare result of two key orders are the same. It can be used
|
||||||
|
// to compare the keys inserted into a file, and what is read back.
|
||||||
|
// Return true if the validation passes.
|
||||||
|
bool CompareValidator(const OutputValidator& other_validator) { |
||||||
|
return GetHash() == other_validator.GetHash(); |
||||||
|
} |
||||||
|
|
||||||
|
private: |
||||||
|
uint64_t GetHash() const { return paranoid_hash_; } |
||||||
|
|
||||||
|
const InternalKeyComparator& icmp_; |
||||||
|
std::string prev_key_; |
||||||
|
uint64_t paranoid_hash_ = 0; |
||||||
|
bool enable_order_check_; |
||||||
|
bool enable_hash_; |
||||||
|
}; |
||||||
|
} // namespace ROCKSDB_NAMESPACE
|
||||||
Loading…
Reference in new issue