Extend VerifyFileChecksums API for blob files (#7979)

Summary:
Extend VerifyFileChecksums API to verify blob files in case of
use_file_checksum.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7979

Test Plan: New unit test db_blob_corruption_test

Reviewed By: ltamasi

Differential Revision: D26534040

Pulled By: akankshamahajan15

fbshipit-source-id: 7dc5951a3df9d265ea1265e0122b43c966856ade
main
Akanksha Mahajan 4 years ago committed by Facebook GitHub Bot
parent daca92c17a
commit 46cf5fbfdd
  1. 1
      CMakeLists.txt
  2. 3
      Makefile
  3. 7
      TARGETS
  4. 81
      db/blob/db_blob_corruption_test.cc
  5. 2
      db/corruption_test.cc
  6. 42
      db/db_impl/db_impl.cc
  7. 7
      db/db_impl/db_impl.h
  8. 1
      src.mk

@ -1061,6 +1061,7 @@ if(WITH_TESTS)
db/blob/blob_file_garbage_test.cc db/blob/blob_file_garbage_test.cc
db/blob/blob_file_reader_test.cc db/blob/blob_file_reader_test.cc
db/blob/db_blob_basic_test.cc db/blob/db_blob_basic_test.cc
db/blob/db_blob_corruption_test.cc
db/blob/db_blob_index_test.cc db/blob/db_blob_index_test.cc
db/column_family_test.cc db/column_family_test.cc
db/compact_files_test.cc db/compact_files_test.cc

@ -592,6 +592,7 @@ ifdef ASSERT_STATUS_CHECKED
db_log_iter_test \ db_log_iter_test \
db_bloom_filter_test \ db_bloom_filter_test \
db_blob_basic_test \ db_blob_basic_test \
db_blob_corruption_test \
db_blob_index_test \ db_blob_index_test \
db_block_cache_test \ db_block_cache_test \
db_compaction_test \ db_compaction_test \
@ -2059,6 +2060,8 @@ io_tracer_parser_test: $(OBJ_DIR)/tools/io_tracer_parser_test.o $(OBJ_DIR)/tools
io_tracer_parser: $(OBJ_DIR)/tools/io_tracer_parser.o $(TOOLS_LIBRARY) $(LIBRARY) io_tracer_parser: $(OBJ_DIR)/tools/io_tracer_parser.o $(TOOLS_LIBRARY) $(LIBRARY)
$(AM_LINK) $(AM_LINK)
db_blob_corruption_test: $(OBJ_DIR)/db/blob/db_blob_corruption_test.o $(TEST_LIBRARY) $(LIBRARY)
$(AM_LINK)
#------------------------------------------------- #-------------------------------------------------
# make install related stuff # make install related stuff
PREFIX ?= /usr/local PREFIX ?= /usr/local

@ -1149,6 +1149,13 @@ ROCKS_TESTS = [
[], [],
[], [],
], ],
[
"db_blob_corruption_test",
"db/blob/db_blob_corruption_test.cc",
"serial",
[],
[],
],
[ [
"db_blob_index_test", "db_blob_index_test",
"db/blob/db_blob_index_test.cc", "db/blob/db_blob_index_test.cc",

@ -0,0 +1,81 @@
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
// This source code is licensed under both the GPLv2 (found in the
// COPYING file in the root directory) and Apache 2.0 License
// (found in the LICENSE.Apache file in the root directory).
#include "db/db_test_util.h"
#include "port/stack_trace.h"
#include "test_util/sync_point.h"
namespace ROCKSDB_NAMESPACE {
class DBBlobCorruptionTest : public DBTestBase {
protected:
DBBlobCorruptionTest()
: DBTestBase("/db_blob_corruption_test", /* env_do_fsync */ false) {}
void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) {
// Pick file to corrupt
std::vector<std::string> filenames;
ASSERT_OK(env_->GetChildren(dbname_, &filenames));
uint64_t number;
FileType type;
std::string fname;
uint64_t picked_number = kInvalidBlobFileNumber;
for (size_t i = 0; i < filenames.size(); i++) {
if (ParseFileName(filenames[i], &number, &type) && type == filetype &&
number > picked_number) { // Pick latest file
fname = dbname_ + "/" + filenames[i];
picked_number = number;
}
}
ASSERT_TRUE(!fname.empty()) << filetype;
ASSERT_OK(test::CorruptFile(env_, fname, offset, bytes_to_corrupt));
}
};
#ifndef ROCKSDB_LITE
TEST_F(DBBlobCorruptionTest, VerifyWholeBlobFileChecksum) {
Options options = GetDefaultOptions();
options.enable_blob_files = true;
options.min_blob_size = 0;
options.create_if_missing = true;
options.file_checksum_gen_factory =
ROCKSDB_NAMESPACE::GetFileChecksumGenCrc32cFactory();
Reopen(options);
ASSERT_OK(Put(Slice("key_1"), Slice("blob_value_1")));
ASSERT_OK(Flush());
ASSERT_OK(Put(Slice("key_2"), Slice("blob_value_2")));
ASSERT_OK(Flush());
ASSERT_OK(db_->VerifyFileChecksums(ReadOptions()));
Close();
Corrupt(kBlobFile, 0, 2);
ASSERT_OK(TryReopen(options));
int count{0};
SyncPoint::GetInstance()->SetCallBack(
"DBImpl::VerifyFullFileChecksum:mismatch", [&](void* arg) {
const Status* s = static_cast<Status*>(arg);
ASSERT_NE(s, nullptr);
++count;
ASSERT_NOK(*s);
});
SyncPoint::GetInstance()->EnableProcessing();
ASSERT_TRUE(db_->VerifyFileChecksums(ReadOptions()).IsCorruption());
ASSERT_EQ(1, count);
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
}
#endif // !ROCKSDB_LITE
} // namespace ROCKSDB_NAMESPACE
int main(int argc, char** argv) {
ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

@ -882,7 +882,7 @@ TEST_F(CorruptionTest, VerifyWholeTableChecksum) {
SyncPoint::GetInstance()->ClearAllCallBacks(); SyncPoint::GetInstance()->ClearAllCallBacks();
int count{0}; int count{0};
SyncPoint::GetInstance()->SetCallBack( SyncPoint::GetInstance()->SetCallBack(
"DBImpl::VerifySstFileChecksum:mismatch", [&](void* arg) { "DBImpl::VerifyFullFileChecksum:mismatch", [&](void* arg) {
auto* s = reinterpret_cast<Status*>(arg); auto* s = reinterpret_cast<Status*>(arg);
ASSERT_NE(s, nullptr); ASSERT_NE(s, nullptr);
++count; ++count;

@ -4855,17 +4855,37 @@ Status DBImpl::VerifyChecksumInternal(const ReadOptions& read_options,
std::string fname = TableFileName(cfd->ioptions()->cf_paths, std::string fname = TableFileName(cfd->ioptions()->cf_paths,
fd.GetNumber(), fd.GetPathId()); fd.GetNumber(), fd.GetPathId());
if (use_file_checksum) { if (use_file_checksum) {
s = VerifySstFileChecksum(*fmeta, fname, read_options); s = VerifyFullFileChecksum(fmeta->file_checksum,
fmeta->file_checksum_func_name, fname,
read_options);
} else { } else {
s = ROCKSDB_NAMESPACE::VerifySstFileChecksum(opts, file_options_, s = ROCKSDB_NAMESPACE::VerifySstFileChecksum(opts, file_options_,
read_options, fname); read_options, fname);
} }
} }
} }
if (s.ok() && use_file_checksum) {
const auto& blob_files = vstorage->GetBlobFiles();
for (const auto& pair : blob_files) {
const uint64_t blob_file_number = pair.first;
const auto& meta = pair.second;
assert(meta);
const std::string blob_file_name = BlobFileName(
cfd->ioptions()->cf_paths.front().path, blob_file_number);
s = VerifyFullFileChecksum(meta->GetChecksumValue(),
meta->GetChecksumMethod(), blob_file_name,
read_options);
if (!s.ok()) {
break;
}
}
}
if (!s.ok()) { if (!s.ok()) {
break; break;
} }
} }
bool defer_purge = bool defer_purge =
immutable_db_options().avoid_unnecessary_blocking_io; immutable_db_options().avoid_unnecessary_blocking_io;
{ {
@ -4890,29 +4910,31 @@ Status DBImpl::VerifyChecksumInternal(const ReadOptions& read_options,
return s; return s;
} }
Status DBImpl::VerifySstFileChecksum(const FileMetaData& fmeta, Status DBImpl::VerifyFullFileChecksum(const std::string& file_checksum_expected,
const std::string& fname, const std::string& func_name_expected,
const ReadOptions& read_options) { const std::string& fname,
const ReadOptions& read_options) {
Status s; Status s;
if (fmeta.file_checksum == kUnknownFileChecksum) { if (file_checksum_expected == kUnknownFileChecksum) {
return s; return s;
} }
std::string file_checksum; std::string file_checksum;
std::string func_name; std::string func_name;
s = ROCKSDB_NAMESPACE::GenerateOneFileChecksum( s = ROCKSDB_NAMESPACE::GenerateOneFileChecksum(
fs_.get(), fname, immutable_db_options_.file_checksum_gen_factory.get(), fs_.get(), fname, immutable_db_options_.file_checksum_gen_factory.get(),
fmeta.file_checksum_func_name, &file_checksum, &func_name, func_name_expected, &file_checksum, &func_name,
read_options.readahead_size, immutable_db_options_.allow_mmap_reads, read_options.readahead_size, immutable_db_options_.allow_mmap_reads,
io_tracer_, immutable_db_options_.rate_limiter.get()); io_tracer_, immutable_db_options_.rate_limiter.get());
if (s.ok()) { if (s.ok()) {
assert(fmeta.file_checksum_func_name == func_name); assert(func_name_expected == func_name);
if (file_checksum != fmeta.file_checksum) { if (file_checksum != file_checksum_expected) {
std::ostringstream oss; std::ostringstream oss;
oss << fname << " file checksum mismatch, "; oss << fname << " file checksum mismatch, ";
oss << "expecting " << Slice(fmeta.file_checksum).ToString(/*hex=*/true); oss << "expecting "
<< Slice(file_checksum_expected).ToString(/*hex=*/true);
oss << ", but actual " << Slice(file_checksum).ToString(/*hex=*/true); oss << ", but actual " << Slice(file_checksum).ToString(/*hex=*/true);
s = Status::Corruption(oss.str()); s = Status::Corruption(oss.str());
TEST_SYNC_POINT_CALLBACK("DBImpl::VerifySstFileChecksum:mismatch", &s); TEST_SYNC_POINT_CALLBACK("DBImpl::VerifyFullFileChecksum:mismatch", &s);
} }
} }
return s; return s;

@ -447,9 +447,10 @@ class DBImpl : public DB {
Status VerifyChecksumInternal(const ReadOptions& read_options, Status VerifyChecksumInternal(const ReadOptions& read_options,
bool use_file_checksum); bool use_file_checksum);
Status VerifySstFileChecksum(const FileMetaData& fmeta, Status VerifyFullFileChecksum(const std::string& file_checksum_expected,
const std::string& fpath, const std::string& func_name_expected,
const ReadOptions& read_options); const std::string& fpath,
const ReadOptions& read_options);
using DB::StartTrace; using DB::StartTrace;
virtual Status StartTrace( virtual Status StartTrace(

@ -377,6 +377,7 @@ TEST_MAIN_SOURCES = \
db/blob/blob_file_garbage_test.cc \ db/blob/blob_file_garbage_test.cc \
db/blob/blob_file_reader_test.cc \ db/blob/blob_file_reader_test.cc \
db/blob/db_blob_basic_test.cc \ db/blob/db_blob_basic_test.cc \
db/blob/db_blob_corruption_test.cc \
db/blob/db_blob_index_test.cc \ db/blob/db_blob_index_test.cc \
db/column_family_test.cc \ db/column_family_test.cc \
db/compact_files_test.cc \ db/compact_files_test.cc \

Loading…
Cancel
Save