Fix compilation errors and add fuzzers to CircleCI (#9420)

Summary:
This PR does the following:
- Fix compilation and linking errors when building fuzzer
- Add the above to CircleCI
- Update documentation

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9420

Test Plan: CI

Reviewed By: jay-zhuang

Differential Revision: D33849452

Pulled By: riversand963

fbshipit-source-id: 0794e5d04a3f53bfd2216fe2b3cd827ca2083ac3
main
Yanqin Jin 3 years ago committed by Facebook GitHub Bot
parent 980b9ff385
commit c58c5596e7
  1. 35
      .circleci/config.yml
  2. 5
      fuzz/.gitignore
  3. 14
      fuzz/Makefile
  4. 5
      fuzz/README.md
  5. 2
      fuzz/db_fuzzer.cc
  6. 34
      fuzz/sst_file_writer_fuzzer.cc

@ -136,6 +136,21 @@ commands:
command: | command: |
sudo apt-get update -y && sudo apt-get install -y libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev sudo apt-get update -y && sudo apt-get install -y libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
install-libprotobuf-mutator:
steps:
- run:
name: Install libprotobuf-mutator libs
command: |
git clone --single-branch --branch master --depth 1 git@github.com:google/libprotobuf-mutator.git ~/libprotobuf-mutator
cd ~/libprotobuf-mutator && mkdir build && cd build
cmake .. -GNinja -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang++-13 -DCMAKE_BUILD_TYPE=Release -DLIB_PROTO_MUTATOR_DOWNLOAD_PROTOBUF=ON
ninja && sudo ninja install
- run:
name: Setup environment variables
command: |
echo "export PKG_CONFIG_PATH=/usr/local/OFF/:~/libprotobuf-mutator/build/external.protobuf/lib/pkgconfig/" >> $BASH_ENV
echo "export PROTOC_BIN=~/libprotobuf-mutator/build/external.protobuf/bin/protoc" >> $BASH_ENV
executors: executors:
windows-2xlarge: windows-2xlarge:
machine: machine:
@ -732,6 +747,23 @@ jobs:
tools/check_format_compatible.sh tools/check_format_compatible.sh
- post-steps - post-steps
build-fuzzers:
machine:
image: ubuntu-2004:202010-01
resource_class: large
steps:
- pre-steps
- install-clang-13
- run: sudo apt-get update -y && sudo apt-get install -y cmake ninja-build binutils liblzma-dev libz-dev pkg-config autoconf libtool
- install-libprotobuf-mutator
- run:
name: "Build rocksdb lib"
command: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j4 static_lib
- run:
name: "Build fuzzers"
command: cd fuzz && make sst_file_writer_fuzzer db_fuzzer db_map_fuzzer
- post-steps
workflows: workflows:
version: 2 version: 2
build-linux: build-linux:
@ -838,6 +870,9 @@ workflows:
build-microbench: build-microbench:
jobs: jobs:
- build-linux-microbench - build-linux-microbench
build-fuzzers:
jobs:
- build-fuzzers
nightly: nightly:
triggers: triggers:
- schedule: - schedule:

5
fuzz/.gitignore vendored

@ -0,0 +1,5 @@
db_fuzzer
db_map_fuzzer
sst_file_writer_fuzzer
proto/gen/*

@ -20,10 +20,10 @@ PROTO_IN = $(ROOT_DIR)/fuzz/proto
PROTO_OUT = $(ROOT_DIR)/fuzz/proto/gen PROTO_OUT = $(ROOT_DIR)/fuzz/proto/gen
ifneq ($(FUZZ_ENV), ossfuzz) ifneq ($(FUZZ_ENV), ossfuzz)
CC = clang++ CC = $(CXX)
CCFLAGS += -Wall -fsanitize=address,fuzzer CCFLAGS += -Wall -fsanitize=address,fuzzer
CFLAGS += $(PLATFORM_CXXFLAGS) $(PROTOBUF_CFLAGS) $(PROTOBUF_MUTATOR_CFLAGS) -I$(PROTO_OUT) -I$(ROCKSDB_INCLUDE_DIR) -I$(ROCKSDB_LIB_DIR) CFLAGS += $(PLATFORM_CXXFLAGS) $(PROTOBUF_CFLAGS) $(PROTOBUF_MUTATOR_CFLAGS) -I$(PROTO_OUT) -I$(ROCKSDB_INCLUDE_DIR) -I$(ROCKSDB_LIB_DIR)
LDFLAGS += $(PLATFORM_LDFLAGS) $(PROTOBUF_LDFLAGS) $(PROTOBUF_MUTATOR_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb LDFLAGS += $(PLATFORM_LDFLAGS) $(PROTOBUF_MUTATOR_LDFLAGS) $(PROTOBUF_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb
else else
# OSS-Fuzz sets various environment flags that are used for compilation. # OSS-Fuzz sets various environment flags that are used for compilation.
# These environment flags depend on which type of sanitizer build is being # These environment flags depend on which type of sanitizer build is being
@ -42,15 +42,21 @@ CFLAGS += $(PROTOBUF_CFLAGS) $(PROTOBUF_MUTATOR_CFLAGS) -I$(PROTO_OUT) -I$(ROCKS
LDFLAGS += $(PLATFORM_LDFLAGS) $(LIB_FUZZING_ENGINE) $(PROTOBUF_MUTATOR_LDFLAGS) $(PROTOBUF_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb LDFLAGS += $(PLATFORM_LDFLAGS) $(LIB_FUZZING_ENGINE) $(PROTOBUF_MUTATOR_LDFLAGS) $(PROTOBUF_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb
endif endif
.PHONY: gen_proto .PHONY: gen_proto clean
# Set PROTOC_BIN when invoking `make` if a custom protoc is required.
PROTOC_BIN ?= protoc
gen_proto: gen_proto:
mkdir -p $(PROTO_OUT) mkdir -p $(PROTO_OUT)
protoc \ $(PROTOC_BIN) \
--proto_path=$(PROTO_IN) \ --proto_path=$(PROTO_IN) \
--cpp_out=$(PROTO_OUT) \ --cpp_out=$(PROTO_OUT) \
$(PROTO_IN)/*.proto $(PROTO_IN)/*.proto
clean:
rm -rf db_fuzzer db_map_fuzzer sst_file_writer_fuzzer $(PROTO_OUT)
db_fuzzer: db_fuzzer.cc db_fuzzer: db_fuzzer.cc
$(CC) $(CCFLAGS) -o db_fuzzer db_fuzzer.cc $(CFLAGS) $(LDFLAGS) $(CC) $(CCFLAGS) -o db_fuzzer db_fuzzer.cc $(CFLAGS) $(LDFLAGS)

@ -15,6 +15,11 @@ Some tests rely on [structure aware fuzzing](https://github.com/google/fuzzing/b
We use [protobuf](https://developers.google.com/protocol-buffers) to define structured input to the fuzzer, We use [protobuf](https://developers.google.com/protocol-buffers) to define structured input to the fuzzer,
and use [libprotobuf-mutator](https://github.com/google/libprotobuf-mutator) as the custom libFuzzer mutator. and use [libprotobuf-mutator](https://github.com/google/libprotobuf-mutator) as the custom libFuzzer mutator.
So make sure you have protobuf and libprotobuf-mutator installed, and make sure `pkg-config` can find them. So make sure you have protobuf and libprotobuf-mutator installed, and make sure `pkg-config` can find them.
On some systems, there are both protobuf2 and protobuf3 in the package management system,
make sure protobuf3 is installed.
If you do not want to install protobuf library yourself, you can rely on libprotobuf-mutator to download protobuf
for you. For details about installation, please refer to [libprotobuf-mutator README](https://github.com/google/libprotobuf-mutator#readme)
## Example ## Example

@ -153,6 +153,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
delete iter; delete iter;
break; break;
} }
case OP_COUNT:
break;
} }
} }

@ -12,9 +12,33 @@
#include "rocksdb/file_system.h" #include "rocksdb/file_system.h"
#include "rocksdb/sst_file_writer.h" #include "rocksdb/sst_file_writer.h"
#include "src/libfuzzer/libfuzzer_macro.h" #include "src/libfuzzer/libfuzzer_macro.h"
#include "table/table_builder.h"
#include "table/table_reader.h" #include "table/table_reader.h"
#include "util.h" #include "util.h"
using ROCKSDB_NAMESPACE::BytewiseComparator;
using ROCKSDB_NAMESPACE::Comparator;
using ROCKSDB_NAMESPACE::EnvOptions;
using ROCKSDB_NAMESPACE::ExternalSstFileInfo;
using ROCKSDB_NAMESPACE::FileOptions;
using ROCKSDB_NAMESPACE::FileSystem;
using ROCKSDB_NAMESPACE::ImmutableCFOptions;
using ROCKSDB_NAMESPACE::ImmutableOptions;
using ROCKSDB_NAMESPACE::InternalIterator;
using ROCKSDB_NAMESPACE::IOOptions;
using ROCKSDB_NAMESPACE::kMaxSequenceNumber;
using ROCKSDB_NAMESPACE::Options;
using ROCKSDB_NAMESPACE::ParsedInternalKey;
using ROCKSDB_NAMESPACE::ParseInternalKey;
using ROCKSDB_NAMESPACE::RandomAccessFileReader;
using ROCKSDB_NAMESPACE::ReadOptions;
using ROCKSDB_NAMESPACE::SstFileWriter;
using ROCKSDB_NAMESPACE::Status;
using ROCKSDB_NAMESPACE::TableReader;
using ROCKSDB_NAMESPACE::TableReaderCaller;
using ROCKSDB_NAMESPACE::TableReaderOptions;
using ROCKSDB_NAMESPACE::ValueType;
// Keys in SST file writer operations must be unique and in ascending order. // Keys in SST file writer operations must be unique and in ascending order.
// For each DBOperation generated by the fuzzer, this function is called on // For each DBOperation generated by the fuzzer, this function is called on
// it to deduplicate and sort the keys in the DBOperations. // it to deduplicate and sort the keys in the DBOperations.
@ -60,15 +84,15 @@ TableReader* NewTableReader(const std::string& sst_file_path,
std::unique_ptr<TableReader> table_reader; std::unique_ptr<TableReader> table_reader;
const auto& fs = options.env->GetFileSystem(); const auto& fs = options.env->GetFileSystem();
FileOptions fopts(env_options); FileOptions fopts(env_options);
Status s = options.env->GetFileSize(sst_file_path, fopts.io_options, Status s = options.env->GetFileSize(sst_file_path, &file_size);
&file_size, nullptr);
if (s.ok()) { if (s.ok()) {
s = RandomAccessFileReader::Create(fs, sst_file_path, fopts, &file_reader, s = RandomAccessFileReader::Create(fs, sst_file_path, fopts, &file_reader,
nullptr); nullptr);
} }
if (s.ok()) { if (s.ok()) {
TableReaderOptions t_opt(cf_ioptions, /*prefix_extractor=*/nullptr, ImmutableOptions iopts(options, cf_ioptions);
env_options, cf_ioptions.internal_comparator); TableReaderOptions t_opt(iopts, /*prefix_extractor=*/nullptr, env_options,
cf_ioptions.internal_comparator);
t_opt.largest_seqno = kMaxSequenceNumber; t_opt.largest_seqno = kMaxSequenceNumber;
s = options.table_factory->NewTableReader(t_opt, std::move(file_reader), s = options.table_factory->NewTableReader(t_opt, std::move(file_reader),
file_size, &table_reader, file_size, &table_reader,
@ -153,7 +177,7 @@ DEFINE_PROTO_FUZZER(DBOperations& input) {
// Iterate and verify key-value pairs. // Iterate and verify key-value pairs.
std::unique_ptr<TableReader> table_reader( std::unique_ptr<TableReader> table_reader(
NewTableReader(sstfile, options, env_options, cf_ioptions)); ::NewTableReader(sstfile, options, env_options, cf_ioptions));
ReadOptions roptions; ReadOptions roptions;
CHECK_OK(table_reader->VerifyChecksum(roptions, CHECK_OK(table_reader->VerifyChecksum(roptions,
TableReaderCaller::kUncategorized)); TableReaderCaller::kUncategorized));

Loading…
Cancel
Save