Update SstFileWriter fuzzer to iterate and check all key-value pairs (#7761)

Summary:
as title

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7761

Test Plan: cd fuzz && make sst_file_writer_fuzzer && ./sst_file_writer_fuzzer

Reviewed By: pdillinger

Differential Revision: D25430802

Pulled By: cheng-chang

fbshipit-source-id: 01436307df6f4c434bb608f44e1c8e4a1119f94f
main
Cheng Chang 4 years ago committed by Facebook GitHub Bot
parent b1ee191405
commit 491779514e
  1. 33
      fuzz/Makefile
  2. 2
      fuzz/README.md
  3. 26
      fuzz/db_map_fuzzer.cc
  4. 1
      fuzz/proto/db_operation.proto
  5. 171
      fuzz/sst_file_writer_fuzzer.cc
  6. 23
      fuzz/util.h

@ -3,7 +3,9 @@
# COPYING file in the root directory) and Apache 2.0 License # COPYING file in the root directory) and Apache 2.0 License
# (found in the LICENSE.Apache file in the root directory). # (found in the LICENSE.Apache file in the root directory).
CWD = $(shell pwd) ROOT_DIR = $(abspath $(shell pwd)/../)
include $(ROOT_DIR)/make_config.mk
PROTOBUF_CFLAGS = `pkg-config --cflags protobuf` PROTOBUF_CFLAGS = `pkg-config --cflags protobuf`
PROTOBUF_LDFLAGS = `pkg-config --libs protobuf` PROTOBUF_LDFLAGS = `pkg-config --libs protobuf`
@ -11,17 +13,17 @@ PROTOBUF_LDFLAGS = `pkg-config --libs protobuf`
PROTOBUF_MUTATOR_CFLAGS = `pkg-config --cflags libprotobuf-mutator` PROTOBUF_MUTATOR_CFLAGS = `pkg-config --cflags libprotobuf-mutator`
PROTOBUF_MUTATOR_LDFLAGS = `pkg-config --libs libprotobuf-mutator` PROTOBUF_MUTATOR_LDFLAGS = `pkg-config --libs libprotobuf-mutator`
ROCKSDB_INCLUDE_DIR = $(CWD)/../include ROCKSDB_INCLUDE_DIR = $(ROOT_DIR)/include
ROCKSDB_LIB_DIR = $(CWD)/.. ROCKSDB_LIB_DIR = $(ROOT_DIR)
PROTO_IN = $(CWD)/proto PROTO_IN = $(ROOT_DIR)/fuzz/proto
PROTO_OUT = $(CWD)/proto/gen PROTO_OUT = $(ROOT_DIR)/fuzz/proto/gen
ifneq ($(FUZZ_ENV), ossfuzz) ifneq ($(FUZZ_ENV), ossfuzz)
CC = clang++ CC = clang++
CCFLAGS += -std=c++14 -Wall -fsanitize=address,fuzzer CCFLAGS += -Wall -fsanitize=address,fuzzer
CFLAGS += $(PROTOBUF_CFLAGS) $(PROTOBUF_MUTATOR_CFLAGS) -I$(PROTO_OUT) -I$(ROCKSDB_INCLUDE_DIR) CFLAGS += $(PLATFORM_CXXFLAGS) $(PROTOBUF_CFLAGS) $(PROTOBUF_MUTATOR_CFLAGS) -I$(PROTO_OUT) -I$(ROCKSDB_INCLUDE_DIR) -I$(ROCKSDB_LIB_DIR)
LDFLAGS += $(PROTOBUF_LDFLAGS) $(PROTOBUF_MUTATOR_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb -lz -lbz2 LDFLAGS += $(PLATFORM_LDFLAGS) $(PROTOBUF_LDFLAGS) $(PROTOBUF_MUTATOR_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb
else else
# OSS-Fuzz sets various environment flags that are used for compilation. # OSS-Fuzz sets various environment flags that are used for compilation.
# These environment flags depend on which type of sanitizer build is being # These environment flags depend on which type of sanitizer build is being
@ -36,14 +38,11 @@ else
# LIB_FUZZING_ENGINE="-fsanitize=fuzzer" # LIB_FUZZING_ENGINE="-fsanitize=fuzzer"
CC = $(CXX) CC = $(CXX)
CCFLAGS = $(CXXFLAGS) CCFLAGS = $(CXXFLAGS)
CFLAGS += $(PROTOBUF_CFLAGS) $(PROTOBUF_MUTATOR_CFLAGS) -I$(PROTO_OUT) -I$(ROCKSDB_INCLUDE_DIR) CFLAGS += $(PROTOBUF_CFLAGS) $(PROTOBUF_MUTATOR_CFLAGS) -I$(PROTO_OUT) -I$(ROCKSDB_INCLUDE_DIR) -I$(ROCKSDB_LIB_DIR)
LDFLAGS += $(LIB_FUZZING_ENGINE) $(PROTOBUF_MUTATOR_LDFLAGS) $(PROTOBUF_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb -lz -lbz2 LDFLAGS += $(PLATFORM_LDFLAGS) $(LIB_FUZZING_ENGINE) $(PROTOBUF_MUTATOR_LDFLAGS) $(PROTOBUF_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb
endif endif
.PHONY: librocksdb gen_proto .PHONY: gen_proto
librocksdb:
cd $(CWD)/.. && make static_lib
gen_proto: gen_proto:
mkdir -p $(PROTO_OUT) mkdir -p $(PROTO_OUT)
@ -52,11 +51,11 @@ gen_proto:
--cpp_out=$(PROTO_OUT) \ --cpp_out=$(PROTO_OUT) \
$(PROTO_IN)/*.proto $(PROTO_IN)/*.proto
db_fuzzer: librocksdb db_fuzzer.cc db_fuzzer: db_fuzzer.cc
$(CC) $(CCFLAGS) -o db_fuzzer db_fuzzer.cc $(CFLAGS) $(LDFLAGS) $(CC) $(CCFLAGS) -o db_fuzzer db_fuzzer.cc $(CFLAGS) $(LDFLAGS)
db_map_fuzzer: librocksdb gen_proto db_map_fuzzer.cc proto/gen/db_operation.pb.cc db_map_fuzzer: gen_proto db_map_fuzzer.cc proto/gen/db_operation.pb.cc
$(CC) $(CCFLAGS) -o db_map_fuzzer db_map_fuzzer.cc proto/gen/db_operation.pb.cc $(CFLAGS) $(LDFLAGS) $(CC) $(CCFLAGS) -o db_map_fuzzer db_map_fuzzer.cc proto/gen/db_operation.pb.cc $(CFLAGS) $(LDFLAGS)
sst_file_writer_fuzzer: librocksdb gen_proto sst_file_writer_fuzzer.cc proto/gen/db_operation.pb.cc sst_file_writer_fuzzer: gen_proto sst_file_writer_fuzzer.cc proto/gen/db_operation.pb.cc
$(CC) $(CCFLAGS) -o sst_file_writer_fuzzer sst_file_writer_fuzzer.cc proto/gen/db_operation.pb.cc $(CFLAGS) $(LDFLAGS) $(CC) $(CCFLAGS) -o sst_file_writer_fuzzer sst_file_writer_fuzzer.cc proto/gen/db_operation.pb.cc $(CFLAGS) $(LDFLAGS)

@ -56,6 +56,8 @@ protobuf_mutator::libfuzzer::PostProcessorRegistration<DBOperations>
### Compile and link the fuzzer ### Compile and link the fuzzer
In the rocksdb root directory, compile rocksdb library by `make static_lib`.
Go to the `fuzz` directory, Go to the `fuzz` directory,
run `make sst_file_writer_fuzzer` to generate the fuzzer, run `make sst_file_writer_fuzzer` to generate the fuzzer,
it will compile rocksdb static library, generate protobuf, then compile and link `sst_file_writer_fuzzer`. it will compile rocksdb static library, generate protobuf, then compile and link `sst_file_writer_fuzzer`.

@ -12,28 +12,7 @@
#include "rocksdb/db.h" #include "rocksdb/db.h"
#include "rocksdb/file_system.h" #include "rocksdb/file_system.h"
#include "src/libfuzzer/libfuzzer_macro.h" #include "src/libfuzzer/libfuzzer_macro.h"
#include "util.h"
#define CHECK_OK(expression) \
do { \
auto status = (expression); \
if (!status.ok()) { \
std::cerr << status.ToString() << std::endl; \
abort(); \
} \
} while (0)
#define CHECK_EQ(a, b) \
if (a != b) { \
std::cerr << "(" << #a << "=" << a << ") != (" << #b << "=" << b << ")" \
<< std::endl; \
abort(); \
}
#define CHECK_TRUE(cond) \
if (!(cond)) { \
std::cerr << "\"" << #cond << "\" is false" << std::endl; \
abort(); \
}
protobuf_mutator::libfuzzer::PostProcessorRegistration<DBOperations> reg = { protobuf_mutator::libfuzzer::PostProcessorRegistration<DBOperations> reg = {
[](DBOperations* input, unsigned int /* seed */) { [](DBOperations* input, unsigned int /* seed */) {
@ -81,6 +60,9 @@ DEFINE_PROTO_FUZZER(DBOperations& input) {
kv[op.key()] = op.value(); kv[op.key()] = op.value();
break; break;
} }
case OpType::MERGE: {
break;
}
case OpType::DELETE: { case OpType::DELETE: {
CHECK_OK(db->Delete(rocksdb::WriteOptions(), op.key())); CHECK_OK(db->Delete(rocksdb::WriteOptions(), op.key()));
kv.erase(op.key()); kv.erase(op.key());

@ -10,6 +10,7 @@ syntax = "proto2";
enum OpType { enum OpType {
PUT = 0; PUT = 0;
MERGE = 1;
DELETE = 2; DELETE = 2;
DELETE_RANGE = 3; DELETE_RANGE = 3;
} }

@ -5,20 +5,39 @@
#include <algorithm> #include <algorithm>
#include <iostream> #include <iostream>
#include <memory>
#include <string>
#include "proto/gen/db_operation.pb.h" #include "proto/gen/db_operation.pb.h"
#include "rocksdb/file_system.h" #include "rocksdb/file_system.h"
#include "rocksdb/sst_file_reader.h"
#include "rocksdb/sst_file_writer.h" #include "rocksdb/sst_file_writer.h"
#include "src/libfuzzer/libfuzzer_macro.h" #include "src/libfuzzer/libfuzzer_macro.h"
#include "table/table_reader.h"
#include "util.h"
using namespace ROCKSDB_NAMESPACE;
// Keys in SST file writer operations must be unique and in ascending order. // Keys in SST file writer operations must be unique and in ascending order.
// For each DBOperation generated by the fuzzer, this function is called on // For each DBOperation generated by the fuzzer, this function is called on
// it to deduplicate and sort the keys in the DBOperations. // it to deduplicate and sort the keys in the DBOperations.
protobuf_mutator::libfuzzer::PostProcessorRegistration<DBOperations> reg = { protobuf_mutator::libfuzzer::PostProcessorRegistration<DBOperations> reg = {
[](DBOperations* input, unsigned int /* seed */) { [](DBOperations* input, unsigned int /* seed */) {
const rocksdb::Comparator* comparator = rocksdb::BytewiseComparator(); const Comparator* comparator = BytewiseComparator();
auto ops = input->mutable_operations(); auto ops = input->mutable_operations();
// Make sure begin <= end for DELETE_RANGE.
for (DBOperation& op : *ops) {
if (op.type() == OpType::DELETE_RANGE) {
auto begin = op.key();
auto end = op.value();
if (comparator->Compare(begin, end) > 0) {
std::swap(begin, end);
op.set_key(begin);
op.set_value(end);
}
}
}
std::sort(ops->begin(), ops->end(), std::sort(ops->begin(), ops->end(),
[&comparator](const DBOperation& a, const DBOperation& b) { [&comparator](const DBOperation& a, const DBOperation& b) {
return comparator->Compare(a.key(), b.key()) < 0; return comparator->Compare(a.key(), b.key()) < 0;
@ -32,15 +51,60 @@ protobuf_mutator::libfuzzer::PostProcessorRegistration<DBOperations> reg = {
ops->erase(last, ops->end()); ops->erase(last, ops->end());
}}; }};
#define CHECK_OK(status) \ TableReader* NewTableReader(const std::string& sst_file_path,
if (!status.ok()) { \ const Options& options,
std::cerr << status.ToString() << std::endl; \ const EnvOptions& env_options,
abort(); \ const ImmutableCFOptions& cf_ioptions) {
// This code block is similar to SstFileReader::Open.
uint64_t file_size = 0;
std::unique_ptr<RandomAccessFile> file;
std::unique_ptr<RandomAccessFileReader> file_reader;
std::unique_ptr<TableReader> table_reader;
Status s = options.env->GetFileSize(sst_file_path, &file_size);
if (s.ok()) {
s = options.env->NewRandomAccessFile(sst_file_path, &file, env_options);
}
if (s.ok()) {
file_reader.reset(new RandomAccessFileReader(
NewLegacyRandomAccessFileWrapper(file), sst_file_path));
} }
if (s.ok()) {
TableReaderOptions t_opt(cf_ioptions, /*prefix_extractor=*/nullptr,
env_options, cf_ioptions.internal_comparator);
t_opt.largest_seqno = kMaxSequenceNumber;
s = options.table_factory->NewTableReader(t_opt, std::move(file_reader),
file_size, &table_reader,
/*prefetch=*/false);
}
if (!s.ok()) {
std::cerr << "Failed to create TableReader for " << sst_file_path << ": "
<< s.ToString() << std::endl;
abort();
}
return table_reader.release();
}
ValueType ToValueType(OpType op_type) {
switch (op_type) {
case OpType::PUT:
return ValueType::kTypeValue;
case OpType::MERGE:
return ValueType::kTypeMerge;
case OpType::DELETE:
return ValueType::kTypeDeletion;
case OpType::DELETE_RANGE:
return ValueType::kTypeRangeDeletion;
default:
std::cerr << "Unknown operation type " << static_cast<int>(op_type)
<< std::endl;
abort();
}
}
// Fuzzes DB operations as input, let SstFileWriter generate a SST file // Fuzzes DB operations as input, let SstFileWriter generate a SST file
// according to the operations, then let SstFileReader read the generated SST // according to the operations, then let TableReader read and check all the
// file to check its checksum. // key-value pairs from the generated SST file.
DEFINE_PROTO_FUZZER(DBOperations& input) { DEFINE_PROTO_FUZZER(DBOperations& input) {
if (input.operations().empty()) { if (input.operations().empty()) {
return; return;
@ -48,53 +112,76 @@ DEFINE_PROTO_FUZZER(DBOperations& input) {
std::string sstfile; std::string sstfile;
{ {
auto fs = rocksdb::FileSystem::Default(); auto fs = FileSystem::Default();
std::string dir; std::string dir;
rocksdb::IOOptions opt; IOOptions opt;
rocksdb::IOStatus s = fs->GetTestDirectory(opt, &dir, nullptr); CHECK_OK(fs->GetTestDirectory(opt, &dir, nullptr));
CHECK_OK(s);
sstfile = dir + "/SstFileWriterFuzzer.sst"; sstfile = dir + "/SstFileWriterFuzzer.sst";
} }
Options options;
EnvOptions env_options(options);
ImmutableCFOptions cf_ioptions(options);
// Generate sst file. // Generate sst file.
rocksdb::Options options; SstFileWriter writer(env_options, options);
rocksdb::EnvOptions env_options; CHECK_OK(writer.Open(sstfile));
rocksdb::SstFileWriter writer(env_options, options);
rocksdb::Status s = writer.Open(sstfile);
CHECK_OK(s);
for (const DBOperation& op : input.operations()) { for (const DBOperation& op : input.operations()) {
switch (op.type()) { switch (op.type()) {
case OpType::PUT: case OpType::PUT: {
s = writer.Put(op.key(), op.value()); CHECK_OK(writer.Put(op.key(), op.value()));
CHECK_OK(s);
break; break;
case OpType::MERGE: }
s = writer.Merge(op.key(), op.value()); case OpType::MERGE: {
CHECK_OK(s); CHECK_OK(writer.Merge(op.key(), op.value()));
break; break;
case OpType::DELETE: }
s = writer.Delete(op.key()); case OpType::DELETE: {
CHECK_OK(s); CHECK_OK(writer.Delete(op.key()));
break; break;
case OpType::DELETE_RANGE: }
s = writer.DeleteRange(op.key(), op.value()); case OpType::DELETE_RANGE: {
CHECK_OK(s); CHECK_OK(writer.DeleteRange(op.key(), op.value()));
break; break;
default: }
std::cerr << "Unsupported operation" << static_cast<int>(op.type()); default: {
return; std::cerr << "Unsupported operation" << static_cast<int>(op.type())
<< std::endl;
abort();
}
}
}
ExternalSstFileInfo info;
CHECK_OK(writer.Finish(&info));
// Iterate and verify key-value pairs.
std::unique_ptr<TableReader> table_reader(
NewTableReader(sstfile, options, env_options, cf_ioptions));
ReadOptions roptions;
CHECK_OK(table_reader->VerifyChecksum(roptions,
TableReaderCaller::kUncategorized));
std::unique_ptr<InternalIterator> it(
table_reader->NewIterator(roptions, /*prefix_extractor=*/nullptr,
/*arena=*/nullptr, /*skip_filters=*/true,
TableReaderCaller::kUncategorized));
it->SeekToFirst();
for (const DBOperation& op : input.operations()) {
if (op.type() == OpType::DELETE_RANGE) {
// InternalIterator cannot iterate over DELETE_RANGE entries.
continue;
}
CHECK_TRUE(it->Valid());
ParsedInternalKey ikey;
CHECK_OK(ParseInternalKey(it->key(), &ikey, /*log_err_key=*/true));
CHECK_EQ(ikey.user_key.ToString(), op.key());
CHECK_EQ(ikey.sequence, 0);
CHECK_EQ(ikey.type, ToValueType(op.type()));
if (op.type() != OpType::DELETE) {
CHECK_EQ(op.value(), it->value().ToString());
} }
it->Next();
} }
rocksdb::ExternalSstFileInfo info; CHECK_TRUE(!it->Valid());
s = writer.Finish(&info);
CHECK_OK(s);
// Verify checksum.
rocksdb::SstFileReader reader(options);
s = reader.Open(sstfile);
CHECK_OK(s);
s = reader.VerifyChecksum();
CHECK_OK(s);
// Delete sst file. // Delete sst file.
remove(sstfile.c_str()); remove(sstfile.c_str());

@ -0,0 +1,23 @@
#pragma once
#define CHECK_OK(expression) \
do { \
auto status = (expression); \
if (!status.ok()) { \
std::cerr << status.ToString() << std::endl; \
abort(); \
} \
} while (0)
#define CHECK_EQ(a, b) \
if (a != b) { \
std::cerr << "(" << #a << "=" << a << ") != (" << #b << "=" << b << ")" \
<< std::endl; \
abort(); \
}
#define CHECK_TRUE(cond) \
if (!(cond)) { \
std::cerr << "\"" << #cond << "\" is false" << std::endl; \
abort(); \
}
Loading…
Cancel
Save