NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix compilation errors and add fuzzers to CircleCI (#9420)

Browse Source 
Summary:
This PR does the following:
- Fix compilation and linking errors when building fuzzer
- Add the above to CircleCI
- Update documentation

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9420

Test Plan: CI

Reviewed By: jay-zhuang

Differential Revision: D33849452

Pulled By: riversand963

fbshipit-source-id: 0794e5d04a3f53bfd2216fe2b3cd827ca2083ac3
main
Yanqin Jin 2 years ago committed by Facebook GitHub Bot
parent 980b9ff385
commit c58c5596e7
6 changed files with 86 additions and 9 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 35
      .circleci/config.yml
  2. 5
      fuzz/.gitignore
  3. 14
      fuzz/Makefile
  4. 5
      fuzz/README.md
  5. 2
      fuzz/db_fuzzer.cc
  6. 34
      fuzz/sst_file_writer_fuzzer.cc

35
.circleci/config.yml
Unescape View File

@ -136,6 +136,21 @@ commands:
command: |
sudo apt-get update -y && sudo apt-get install -y libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
install-libprotobuf-mutator:
steps:
- run:
name: Install libprotobuf-mutator libs
command: |
git clone --single-branch --branch master --depth 1 git@github.com:google/libprotobuf-mutator.git ~/libprotobuf-mutator
cd ~/libprotobuf-mutator && mkdir build && cd build
cmake .. -GNinja -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang++-13 -DCMAKE_BUILD_TYPE=Release -DLIB_PROTO_MUTATOR_DOWNLOAD_PROTOBUF=ON
ninja && sudo ninja install
- run:
name: Setup environment variables
command: |
echo "export PKG_CONFIG_PATH=/usr/local/OFF/:~/libprotobuf-mutator/build/external.protobuf/lib/pkgconfig/" >> $BASH_ENV
echo "export PROTOC_BIN=~/libprotobuf-mutator/build/external.protobuf/bin/protoc" >> $BASH_ENV
executors:
windows-2xlarge:
machine:
@ -732,6 +747,23 @@ jobs:
tools/check_format_compatible.sh
- post-steps
build-fuzzers:
machine:
image: ubuntu-2004:202010-01
resource_class: large
steps:
- pre-steps
- install-clang-13
- run: sudo apt-get update -y && sudo apt-get install -y cmake ninja-build binutils liblzma-dev libz-dev pkg-config autoconf libtool
- install-libprotobuf-mutator
- run:
name: "Build rocksdb lib"
command: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j4 static_lib
- run:
name: "Build fuzzers"
command: cd fuzz && make sst_file_writer_fuzzer db_fuzzer db_map_fuzzer
- post-steps
workflows:
version: 2
build-linux:
@ -838,6 +870,9 @@ workflows:
build-microbench:
jobs:
- build-linux-microbench
build-fuzzers:
jobs:
- build-fuzzers
nightly:
triggers:
- schedule:

5
fuzz/.gitignore vendored
Unescape View File

@ -0,0 +1,5 @@
db_fuzzer
db_map_fuzzer
sst_file_writer_fuzzer
proto/gen/*

14
fuzz/Makefile
Unescape View File

@ -20,10 +20,10 @@ PROTO_IN = $(ROOT_DIR)/fuzz/proto
PROTO_OUT = $(ROOT_DIR)/fuzz/proto/gen
ifneq ($(FUZZ_ENV), ossfuzz)
CC = clang++
CC = $(CXX)
CCFLAGS += -Wall -fsanitize=address,fuzzer
CFLAGS += $(PLATFORM_CXXFLAGS) $(PROTOBUF_CFLAGS) $(PROTOBUF_MUTATOR_CFLAGS) -I$(PROTO_OUT) -I$(ROCKSDB_INCLUDE_DIR) -I$(ROCKSDB_LIB_DIR)
LDFLAGS += $(PLATFORM_LDFLAGS) $(PROTOBUF_LDFLAGS) $(PROTOBUF_MUTATOR_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb
LDFLAGS += $(PLATFORM_LDFLAGS) $(PROTOBUF_MUTATOR_LDFLAGS) $(PROTOBUF_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb
else
# OSS-Fuzz sets various environment flags that are used for compilation.
# These environment flags depend on which type of sanitizer build is being
@ -42,15 +42,21 @@ CFLAGS += $(PROTOBUF_CFLAGS) $(PROTOBUF_MUTATOR_CFLAGS) -I$(PROTO_OUT) -I$(ROCKS
LDFLAGS += $(PLATFORM_LDFLAGS) $(LIB_FUZZING_ENGINE) $(PROTOBUF_MUTATOR_LDFLAGS) $(PROTOBUF_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb
endif
.PHONY: gen_proto
.PHONY: gen_proto clean
# Set PROTOC_BIN when invoking `make` if a custom protoc is required.
PROTOC_BIN ?= protoc
gen_proto:
mkdir -p $(PROTO_OUT)
protoc \
$(PROTOC_BIN) \
--proto_path=$(PROTO_IN) \
--cpp_out=$(PROTO_OUT) \
$(PROTO_IN)/*.proto
clean:
rm -rf db_fuzzer db_map_fuzzer sst_file_writer_fuzzer $(PROTO_OUT)
db_fuzzer: db_fuzzer.cc
$(CC) $(CCFLAGS) -o db_fuzzer db_fuzzer.cc $(CFLAGS) $(LDFLAGS)

5
fuzz/README.md
Unescape View File

@ -15,6 +15,11 @@ Some tests rely on [structure aware fuzzing](https://github.com/google/fuzzing/b
We use [protobuf](https://developers.google.com/protocol-buffers) to define structured input to the fuzzer,
and use [libprotobuf-mutator](https://github.com/google/libprotobuf-mutator) as the custom libFuzzer mutator.
So make sure you have protobuf and libprotobuf-mutator installed, and make sure `pkg-config` can find them.
On some systems, there are both protobuf2 and protobuf3 in the package management system,
make sure protobuf3 is installed.
If you do not want to install protobuf library yourself, you can rely on libprotobuf-mutator to download protobuf
for you. For details about installation, please refer to [libprotobuf-mutator README](https://github.com/google/libprotobuf-mutator#readme)
## Example

2
fuzz/db_fuzzer.cc
Unescape View File

@ -153,6 +153,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
delete iter;
break;
}
case OP_COUNT:
break;
}
}

34
fuzz/sst_file_writer_fuzzer.cc
Unescape View File

@ -12,9 +12,33 @@
#include "rocksdb/file_system.h"
#include "rocksdb/sst_file_writer.h"
#include "src/libfuzzer/libfuzzer_macro.h"
#include "table/table_builder.h"
#include "table/table_reader.h"
#include "util.h"
using ROCKSDB_NAMESPACE::BytewiseComparator;
using ROCKSDB_NAMESPACE::Comparator;
using ROCKSDB_NAMESPACE::EnvOptions;
using ROCKSDB_NAMESPACE::ExternalSstFileInfo;
using ROCKSDB_NAMESPACE::FileOptions;
using ROCKSDB_NAMESPACE::FileSystem;
using ROCKSDB_NAMESPACE::ImmutableCFOptions;
using ROCKSDB_NAMESPACE::ImmutableOptions;
using ROCKSDB_NAMESPACE::InternalIterator;
using ROCKSDB_NAMESPACE::IOOptions;
using ROCKSDB_NAMESPACE::kMaxSequenceNumber;
using ROCKSDB_NAMESPACE::Options;
using ROCKSDB_NAMESPACE::ParsedInternalKey;
using ROCKSDB_NAMESPACE::ParseInternalKey;
using ROCKSDB_NAMESPACE::RandomAccessFileReader;
using ROCKSDB_NAMESPACE::ReadOptions;
using ROCKSDB_NAMESPACE::SstFileWriter;
using ROCKSDB_NAMESPACE::Status;
using ROCKSDB_NAMESPACE::TableReader;
using ROCKSDB_NAMESPACE::TableReaderCaller;
using ROCKSDB_NAMESPACE::TableReaderOptions;
using ROCKSDB_NAMESPACE::ValueType;
// Keys in SST file writer operations must be unique and in ascending order.
// For each DBOperation generated by the fuzzer, this function is called on
// it to deduplicate and sort the keys in the DBOperations.
@ -60,15 +84,15 @@ TableReader* NewTableReader(const std::string& sst_file_path,
std::unique_ptr<TableReader> table_reader;
const auto& fs = options.env->GetFileSystem();
FileOptions fopts(env_options);
Status s = options.env->GetFileSize(sst_file_path, fopts.io_options,
&file_size, nullptr);
Status s = options.env->GetFileSize(sst_file_path, &file_size);
if (s.ok()) {
s = RandomAccessFileReader::Create(fs, sst_file_path, fopts, &file_reader,
nullptr);
}
if (s.ok()) {
TableReaderOptions t_opt(cf_ioptions, /*prefix_extractor=*/nullptr,
env_options, cf_ioptions.internal_comparator);
ImmutableOptions iopts(options, cf_ioptions);
TableReaderOptions t_opt(iopts, /*prefix_extractor=*/nullptr, env_options,
cf_ioptions.internal_comparator);
t_opt.largest_seqno = kMaxSequenceNumber;
s = options.table_factory->NewTableReader(t_opt, std::move(file_reader),
file_size, &table_reader,
@ -153,7 +177,7 @@ DEFINE_PROTO_FUZZER(DBOperations& input) {
// Iterate and verify key-value pairs.
std::unique_ptr<TableReader> table_reader(
NewTableReader(sstfile, options, env_options, cf_ioptions));
::NewTableReader(sstfile, options, env_options, cf_ioptions));
ReadOptions roptions;
CHECK_OK(table_reader->VerifyChecksum(roptions,
TableReaderCaller::kUncategorized));

Write Preview
Loading…
Cancel
Save