From 699411b2ca2ed1442106e4617cb09f6ef46baeaf Mon Sep 17 00:00:00 2001 From: Cheng Chang Date: Tue, 17 Nov 2020 12:55:27 -0800 Subject: [PATCH] Fuzzing RocksDB (#7685) Summary: This is the initial PR to support adding fuzz tests to RocksDB. It includes the necessary build infrastructure, and includes an example fuzzer. There is also a README serving as the tutorial for how to add more tests. Pull Request resolved: https://github.com/facebook/rocksdb/pull/7685 Test Plan: Manually build and run the fuzz test according to README. Reviewed By: pdillinger Differential Revision: D25013847 Pulled By: cheng-chang fbshipit-source-id: c91e3b337398d7f4d8f769fd5091cd080487b171 --- .gitignore | 3 + fuzz/Makefile | 38 ++++++++ fuzz/README.md | 158 +++++++++++++++++++++++++++++++++ fuzz/proto/db_operation.proto | 28 ++++++ fuzz/sst_file_writer_fuzzer.cc | 101 +++++++++++++++++++++ 5 files changed, 328 insertions(+) create mode 100644 fuzz/Makefile create mode 100644 fuzz/README.md create mode 100644 fuzz/proto/db_operation.proto create mode 100644 fuzz/sst_file_writer_fuzzer.cc diff --git a/.gitignore b/.gitignore index dc59bb7e6..737684274 100644 --- a/.gitignore +++ b/.gitignore @@ -90,3 +90,6 @@ buckifier/__pycache__ compile_commands.json clang-format-diff.py .py3/ + +fuzz/proto/gen/ +fuzz/crash-* diff --git a/fuzz/Makefile b/fuzz/Makefile new file mode 100644 index 000000000..838c78261 --- /dev/null +++ b/fuzz/Makefile @@ -0,0 +1,38 @@ +# Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +# This source code is licensed under both the GPLv2 (found in the +# COPYING file in the root directory) and Apache 2.0 License +# (found in the LICENSE.Apache file in the root directory). + +CWD = $(shell pwd) + +PROTOBUF_CFLAGS = `pkg-config --cflags protobuf` +PROTOBUF_LDFLAGS = `pkg-config --libs protobuf` + +PROTOBUF_MUTATOR_CFLAGS = `pkg-config --cflags libprotobuf-mutator` +PROTOBUF_MUTATOR_LDFLAGS = `pkg-config --libs libprotobuf-mutator` + +ROCKSDB_INCLUDE_DIR = $(CWD)/../include +ROCKSDB_LIB_DIR = $(CWD)/.. + +PROTO_IN = $(CWD)/proto +PROTO_OUT = $(CWD)/proto/gen + +CC = clang++ +CCFLAGS += -std=c++14 -Wall -fsanitize=address,fuzzer +CFLAGS += $(PROTOBUF_CFLAGS) $(PROTOBUF_MUTATOR_CFLAGS) -I$(PROTO_OUT) -I$(ROCKSDB_INCLUDE_DIR) +LDFLAGS += $(PROTOBUF_LDFLAGS) $(PROTOBUF_MUTATOR_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb -lz -lbz2 + +.PHONY: librocksdb gen_proto + +librocksdb: + cd $(CWD)/.. && make static_lib + +gen_proto: + mkdir -p $(PROTO_OUT) + protoc \ + --proto_path=$(PROTO_IN) \ + --cpp_out=$(PROTO_OUT) \ + $(PROTO_IN)/*.proto + +sst_file_writer_fuzzer: librocksdb gen_proto sst_file_writer_fuzzer.cc proto/gen/db_operation.pb.cc + $(CC) $(CCFLAGS) $(CFLAGS) $(LDFLAGS) -o sst_file_writer_fuzzer sst_file_writer_fuzzer.cc proto/gen/db_operation.pb.cc diff --git a/fuzz/README.md b/fuzz/README.md new file mode 100644 index 000000000..48158e039 --- /dev/null +++ b/fuzz/README.md @@ -0,0 +1,158 @@ +# Fuzzing RocksDB + +## Overview + +This directory contains [fuzz tests](https://en.wikipedia.org/wiki/Fuzzing) for RocksDB. +RocksDB testing infrastructure currently includes unit tests and [stress tests](https://github.com/facebook/rocksdb/wiki/Stress-test), +we hope fuzz testing can catch more bugs. + +## Prerequisite + +We use [LLVM libFuzzer](http://llvm.org/docs/LibFuzzer.html) as the fuzzying engine, +so make sure you have [clang](https://clang.llvm.org/get_started.html) as your compiler. + +Some tests rely on [structure aware fuzzing](https://github.com/google/fuzzing/blob/master/docs/structure-aware-fuzzing.md). +We use [protobuf](https://developers.google.com/protocol-buffers) to define structured input to the fuzzer, +and use [libprotobuf-mutator](https://github.com/google/libprotobuf-mutator) as the custom libFuzzer mutator. +So make sure you have protobuf and libprotobuf-mutator installed, and make sure `pkg-config` can find them. + +## Example + +This example shows you how to do structure aware fuzzing to `rocksdb::SstFileWriter`. + +After walking through the steps to create the fuzzer, we'll introduce a bug into `rocksdb::SstFileWriter::Put`, +then show that the fuzzer can catch the bug. + +### Design the test + +We want the fuzzing engine to automatically generate a list of database operations, +then we apply these operations to `SstFileWriter` in sequence, +finally, after the SST file is generated, we use `SstFileReader` to check the file's checksum. + +### Define input + +We define the database operations in protobuf, each operation has a type of operation and a key value pair, +see [proto/db_operation.proto](proto/db_operation.proto) for details. + +### Define tests with the input + +In [sst_file_writer_fuzzer.cc](sst_file_writer_fuzzer.cc), +we define the tests to be run on the generated input: + +``` +DEFINE_PROTO_FUZZER(DBOperations& input) { + // apply the operations to SstFileWriter and use SstFileReader to verify checksum. + // ... +} +``` + +`SstFileWriter` requires the keys of the operations to be unique and be in ascending order, +but the fuzzing engine generates the input randomly, so we need to process the generated input before +passing it to `DEFINE_PROTO_FUZZER`, this is accomplished by registering a post processor: + +``` +protobuf_mutator::libfuzzer::PostProcessorRegistration +``` + +### Compile and link the fuzzer + +Go to the `fuzz` directory, +run `make sst_file_writer_fuzzer` to generate the fuzzer, +it will compile rocksdb static library, generate protobuf, then compile and link `sst_file_writer_fuzzer`. + +### Introduce a bug + +Manually introduce a bug to `SstFileWriter::Put`: + +``` +diff --git a/table/sst_file_writer.cc b/table/sst_file_writer.cc +index ab1ee7c4e..c7da9ffa0 100644 +--- a/table/sst_file_writer.cc ++++ b/table/sst_file_writer.cc +@@ -277,6 +277,11 @@ Status SstFileWriter::Add(const Slice& user_key, const Slice& value) { + } + + Status SstFileWriter::Put(const Slice& user_key, const Slice& value) { ++ if (user_key.starts_with("!")) { ++ if (value.ends_with("!")) { ++ return Status::Corruption("bomb"); ++ } ++ } + return rep_->Add(user_key, value, ValueType::kTypeValue); + } +``` + +The bug is that for `Put`, if `user_key` starts with `!` and `value` ends with `!`, then corrupt. + +### Run fuzz testing to catch the bug + +Run the fuzzer by `time ./sst_file_writer_fuzzer`. + +Here is the output on my machine: + +``` +Corruption: bomb +==59680== ERROR: libFuzzer: deadly signal + #0 0x109487315 in __sanitizer_print_stack_trace+0x35 (libclang_rt.asan_osx_dynamic.dylib:x86_64+0x4d315) + #1 0x108d63f18 in fuzzer::PrintStackTrace() FuzzerUtil.cpp:205 + #2 0x108d47613 in fuzzer::Fuzzer::CrashCallback() FuzzerLoop.cpp:232 + #3 0x7fff6af535fc in _sigtramp+0x1c (libsystem_platform.dylib:x86_64+0x35fc) + #4 0x7ffee720f3ef () + #5 0x7fff6ae29807 in abort+0x77 (libsystem_c.dylib:x86_64+0x7f807) + #6 0x108cf1c4c in TestOneProtoInput(DBOperations&)+0x113c (sst_file_writer_fuzzer:x86_64+0x100302c4c) + #7 0x108cf09be in LLVMFuzzerTestOneInput+0x16e (sst_file_writer_fuzzer:x86_64+0x1003019be) + #8 0x108d48ce0 in fuzzer::Fuzzer::ExecuteCallback(unsigned char const*, unsigned long) FuzzerLoop.cpp:556 + #9 0x108d48425 in fuzzer::Fuzzer::RunOne(unsigned char const*, unsigned long, bool, fuzzer::InputInfo*, bool*) FuzzerLoop.cpp:470 + #10 0x108d4a626 in fuzzer::Fuzzer::MutateAndTestOne() FuzzerLoop.cpp:698 + #11 0x108d4b325 in fuzzer::Fuzzer::Loop(std::__1::vector >&) FuzzerLoop.cpp:830 + #12 0x108d37fcd in fuzzer::FuzzerDriver(int*, char***, int (*)(unsigned char const*, unsigned long)) FuzzerDriver.cpp:829 + #13 0x108d652b2 in main FuzzerMain.cpp:19 + #14 0x7fff6ad5acc8 in start+0x0 (libdyld.dylib:x86_64+0x1acc8) + +NOTE: libFuzzer has rudimentary signal handlers. + Combine libFuzzer with AddressSanitizer or similar for better crash reports. +SUMMARY: libFuzzer: deadly signal +MS: 7 Custom-CustomCrossOver-InsertByte-Custom-ChangeBit-Custom-CustomCrossOver-; base unit: 90863b4d83c3f994bba0a417d0c2ee3b68f9e795 +0x6f,0x70,0x65,0x72,0x61,0x74,0x69,0x6f,0x6e,0x73,0x20,0x7b,0xa,0x20,0x20,0x6b,0x65,0x79,0x3a,0x20,0x22,0x21,0x22,0xa,0x20,0x20,0x76,0x61,0x6c,0x75,0x65,0x3a,0x20,0x22,0x21,0x22,0xa,0x20,0x20,0x74,0x79,0x70,0x65,0x3a,0x20,0x50,0x55,0x54,0xa,0x7d,0xa,0x6f,0x70,0x65,0x72,0x61,0x74,0x69,0x6f,0x6e,0x73,0x20,0x7b,0xa,0x20,0x20,0x6b,0x65,0x79,0x3a,0x20,0x22,0x2b,0x22,0xa,0x20,0x20,0x74,0x79,0x70,0x65,0x3a,0x20,0x50,0x55,0x54,0xa,0x7d,0xa,0x6f,0x70,0x65,0x72,0x61,0x74,0x69,0x6f,0x6e,0x73,0x20,0x7b,0xa,0x20,0x20,0x6b,0x65,0x79,0x3a,0x20,0x22,0x2e,0x22,0xa,0x20,0x20,0x74,0x79,0x70,0x65,0x3a,0x20,0x50,0x55,0x54,0xa,0x7d,0xa,0x6f,0x70,0x65,0x72,0x61,0x74,0x69,0x6f,0x6e,0x73,0x20,0x7b,0xa,0x20,0x20,0x6b,0x65,0x79,0x3a,0x20,0x22,0x5c,0x32,0x35,0x33,0x22,0xa,0x20,0x20,0x74,0x79,0x70,0x65,0x3a,0x20,0x50,0x55,0x54,0xa,0x7d,0xa, +operations {\x0a key: \"!\"\x0a value: \"!\"\x0a type: PUT\x0a}\x0aoperations {\x0a key: \"+\"\x0a type: PUT\x0a}\x0aoperations {\x0a key: \".\"\x0a type: PUT\x0a}\x0aoperations {\x0a key: \"\\253\"\x0a type: PUT\x0a}\x0a +artifact_prefix='./'; Test unit written to ./crash-a1460be302d09b548e61787178d9edaa40aea467 +Base64: b3BlcmF0aW9ucyB7CiAga2V5OiAiISIKICB2YWx1ZTogIiEiCiAgdHlwZTogUFVUCn0Kb3BlcmF0aW9ucyB7CiAga2V5OiAiKyIKICB0eXBlOiBQVVQKfQpvcGVyYXRpb25zIHsKICBrZXk6ICIuIgogIHR5cGU6IFBVVAp9Cm9wZXJhdGlvbnMgewogIGtleTogIlwyNTMiCiAgdHlwZTogUFVUCn0K +./sst_file_writer_fuzzer 5.97s user 4.40s system 64% cpu 16.195 total +``` + +Within 6 seconds, it catches the bug. + +The input that triggers the bug is persisted in `./crash-a1460be302d09b548e61787178d9edaa40aea467`: + +``` +$ cat ./crash-a1460be302d09b548e61787178d9edaa40aea467 +operations { + key: "!" + value: "!" + type: PUT +} +operations { + key: "+" + type: PUT +} +operations { + key: "." + type: PUT +} +operations { + key: "\253" + type: PUT +} +``` + +### Reproduce the crash to debug + +The above crash can be reproduced by `./sst_file_writer_fuzzer ./crash-a1460be302d09b548e61787178d9edaa40aea467`, +so you can debug the crash. + +## Future Work + +According to [OSS-Fuzz](https://github.com/google/oss-fuzz), +`as of June 2020, OSS-Fuzz has found over 20,000 bugs in 300 open source projects.` + +RocksDB can join OSS-Fuzz together with other open source projects such as sqlite. diff --git a/fuzz/proto/db_operation.proto b/fuzz/proto/db_operation.proto new file mode 100644 index 000000000..20a55eaa5 --- /dev/null +++ b/fuzz/proto/db_operation.proto @@ -0,0 +1,28 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +// Defines database operations. +// Each operation is a key-value pair and an operation type. + +syntax = "proto2"; + +enum OpType { + PUT = 0; + MERGE = 1; + DELETE = 2; + DELETE_RANGE = 3; +} + +message DBOperation { + required string key = 1; + // value is ignored for DELETE. + // [key, value] is the range for DELETE_RANGE. + optional string value = 2; + required OpType type = 3; +} + +message DBOperations { + repeated DBOperation operations = 1; +} diff --git a/fuzz/sst_file_writer_fuzzer.cc b/fuzz/sst_file_writer_fuzzer.cc new file mode 100644 index 000000000..61721b815 --- /dev/null +++ b/fuzz/sst_file_writer_fuzzer.cc @@ -0,0 +1,101 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). + +#include +#include + +#include "proto/gen/db_operation.pb.h" +#include "rocksdb/file_system.h" +#include "rocksdb/sst_file_reader.h" +#include "rocksdb/sst_file_writer.h" +#include "src/libfuzzer/libfuzzer_macro.h" + +// Keys in SST file writer operations must be unique and in ascending order. +// For each DBOperation generated by the fuzzer, this function is called on +// it to deduplicate and sort the keys in the DBOperations. +protobuf_mutator::libfuzzer::PostProcessorRegistration reg = { + [](DBOperations* input, unsigned int /* seed */) { + const rocksdb::Comparator* comparator = rocksdb::BytewiseComparator(); + auto ops = input->mutable_operations(); + std::sort(ops->begin(), ops->end(), + [&comparator](const DBOperation& a, const DBOperation& b) { + return comparator->Compare(a.key(), b.key()) < 0; + }); + + auto last = std::unique( + ops->begin(), ops->end(), + [&comparator](const DBOperation& a, const DBOperation& b) { + return comparator->Compare(a.key(), b.key()) == 0; + }); + ops->erase(last, ops->end()); + }}; + +#define CHECK_OK(status) \ + if (!status.ok()) { \ + std::cerr << status.ToString() << std::endl; \ + abort(); \ + } + +// Fuzzes DB operations as input, let SstFileWriter generate a SST file +// according to the operations, then let SstFileReader read the generated SST +// file to check its checksum. +DEFINE_PROTO_FUZZER(DBOperations& input) { + if (input.operations().empty()) { + return; + } + + std::string sstfile; + { + auto fs = rocksdb::FileSystem::Default(); + std::string dir; + rocksdb::IOOptions opt; + rocksdb::IOStatus s = fs->GetTestDirectory(opt, &dir, nullptr); + CHECK_OK(s); + sstfile = dir + "/SstFileWriterFuzzer.sst"; + } + + // Generate sst file. + rocksdb::Options options; + rocksdb::EnvOptions env_options; + rocksdb::SstFileWriter writer(env_options, options); + rocksdb::Status s = writer.Open(sstfile); + CHECK_OK(s); + for (const DBOperation& op : input.operations()) { + switch (op.type()) { + case OpType::PUT: + s = writer.Put(op.key(), op.value()); + CHECK_OK(s); + break; + case OpType::MERGE: + s = writer.Merge(op.key(), op.value()); + CHECK_OK(s); + break; + case OpType::DELETE: + s = writer.Delete(op.key()); + CHECK_OK(s); + break; + case OpType::DELETE_RANGE: + s = writer.DeleteRange(op.key(), op.value()); + CHECK_OK(s); + break; + default: + std::cerr << "Unsupported operation" << static_cast(op.type()); + return; + } + } + rocksdb::ExternalSstFileInfo info; + s = writer.Finish(&info); + CHECK_OK(s); + + // Verify checksum. + rocksdb::SstFileReader reader(options); + s = reader.Open(sstfile); + CHECK_OK(s); + s = reader.VerifyChecksum(); + CHECK_OK(s); + + // Delete sst file. + remove(sstfile.c_str()); +}