OSS-Fuzz integration and db_fuzzer (#7674)

Summary:
This PR adds a fuzzer to the project and infrastructure to integrate Rocksdb with OSS-Fuzz. OSS-Fuzz is a service run by Google that performs continuous fuzzing of important open source projects. The LevelDB project is also in being fuzzed by OSS-Fuzz (https://github.com/google/oss-fuzz/tree/master/projects/leveldb). Essentially, OSS-Fuzz will perform the fuzzing for you and email you bug reports, coverage reports etc. All we need is a set of email addresses that will receive this information.

For cross-referencing, the PR that adds the OSS-Fuzz logic is here: https://github.com/google/oss-fuzz/pull/4642

The `db_fuzzer` of the PR performs stateful fuzzing of Rocksdb by calling a sequence of Rockdb's APIs with random input in each fuzz iteration. Each fuzz iteration, thus, creates a new instance of Rocksdb and operates on this given instance. The goal is to test diverse states of Rocksdb and ensure no state lead to error conditions, e.g. memory corruption vulnerabilities.

The fuzzer is similar (although more complex) to the fuzzer that is currently being used to analyse Leveldb (https://github.com/google/oss-fuzz/blob/master/projects/leveldb/fuzz_db.cc)

Pull Request resolved: https://github.com/facebook/rocksdb/pull/7674

Reviewed By: pdillinger

Differential Revision: D25238536

Pulled By: cheng-chang

fbshipit-source-id: 610331c49a77eb68d3b1d7d5ef1b0ce230ac0630

fuzz/Makefile

@@ -17,10 +17,28 @@ ROCKSDB_LIB_DIR = $(CWD)/..
 PROTO_IN = $(CWD)/proto
 PROTO_OUT = $(CWD)/proto/gen
 
+ifneq ($(FUZZ_ENV), ossfuzz)
 CC = clang++
 CCFLAGS += -std=c++14 -Wall -fsanitize=address,fuzzer
 CFLAGS += $(PROTOBUF_CFLAGS) $(PROTOBUF_MUTATOR_CFLAGS) -I$(PROTO_OUT) -I$(ROCKSDB_INCLUDE_DIR)
 LDFLAGS += $(PROTOBUF_LDFLAGS) $(PROTOBUF_MUTATOR_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb -lz -lbz2
+else
+# OSS-Fuzz sets various environment flags that are used for compilation.
+# These environment flags depend on which type of sanitizer build is being
+# used, however, an ASan build would set the environment flags as follows:
+# CFLAGS="-O1 -fno-omit-frame-pointer -gline-tables-only \
+         -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address \
+         -fsanitize-address-use-after-scope -fsanitize=fuzzer-no-link"
+# CXXFLAGS="-O1 -fno-omit-frame-pointer -gline-tables-only \
+           -DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -fsanitize=address \
+           -fsanitize-address-use-after-scope -fsanitize=fuzzer-no-link \
+           -stdlib=libc++"
+# LIB_FUZZING_ENGINE="-fsanitize=fuzzer"
+CC = $(CXX)
+CCFLAGS = $(CXXFLAGS)
+CFLAGS += $(PROTOBUF_CFLAGS) $(PROTOBUF_MUTATOR_CFLAGS) -I$(PROTO_OUT) -I$(ROCKSDB_INCLUDE_DIR)
+LDFLAGS += $(LIB_FUZZING_ENGINE) $(PROTOBUF_MUTATOR_LDFLAGS) $(PROTOBUF_LDFLAGS) -L$(ROCKSDB_LIB_DIR) -lrocksdb -lz -lbz2
+endif
 
 .PHONY: librocksdb gen_proto
 
@@ -34,5 +52,8 @@ gen_proto:
 		--cpp_out=$(PROTO_OUT) \
 		$(PROTO_IN)/*.proto
 
+db_fuzzer: librocksdb db_fuzzer.cc
+	$(CC) $(CCFLAGS) -o db_fuzzer db_fuzzer.cc $(CFLAGS) $(LDFLAGS)
+
 sst_file_writer_fuzzer: librocksdb gen_proto sst_file_writer_fuzzer.cc proto/gen/db_operation.pb.cc
-	$(CC) $(CCFLAGS) $(CFLAGS) $(LDFLAGS) -o sst_file_writer_fuzzer sst_file_writer_fuzzer.cc proto/gen/db_operation.pb.cc
+	$(CC) $(CCFLAGS) -o sst_file_writer_fuzzer sst_file_writer_fuzzer.cc proto/gen/db_operation.pb.cc $(CFLAGS) $(LDFLAGS)

fuzz/db_fuzzer.cc

@@ -0,0 +1,159 @@
+#include <fuzzer/FuzzedDataProvider.h>
+
+#include "rocksdb/db.h"
+
+enum OperationType {
+  kPut,
+  kGet,
+  kDelete,
+  kGetProperty,
+  kIterator,
+  kSnapshot,
+  kOpenClose,
+  kColumn,
+  kCompactRange,
+  kSeekForPrev,
+  OP_COUNT
+};
+
+constexpr char db_path[] = "/tmp/testdb";
+
+// Fuzzes DB operations by doing interpretations on the data. Both the
+// sequence of API calls to be called on the DB as well as the arguments
+// to each of these APIs are interpreted by way of the data buffer.
+// The operations that the fuzzer supports are given by the OperationType
+// enum. The goal is to capture sanitizer bugs, so the code should be
+// compiled with a given sanitizer (ASan, UBSan, MSan).
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
+  rocksdb::DB* db;
+  rocksdb::Options options;
+  options.create_if_missing = true;
+  rocksdb::Status status = rocksdb::DB::Open(options, db_path, &db);
+  if (!status.ok()) {
+    return 0;
+  }
+  FuzzedDataProvider fuzzed_data(data, size);
+
+  // perform a sequence of calls on our db instance
+  int max_iter = static_cast<int>(data[0]);
+  for (int i = 0; i < max_iter && i < size; i++) {
+    OperationType op = static_cast<OperationType>(data[i] % OP_COUNT);
+
+    switch (op) {
+      case kPut: {
+        std::string key = fuzzed_data.ConsumeRandomLengthString();
+        std::string val = fuzzed_data.ConsumeRandomLengthString();
+        db->Put(rocksdb::WriteOptions(), key, val);
+        break;
+      }
+      case kGet: {
+        std::string key = fuzzed_data.ConsumeRandomLengthString();
+        std::string value;
+        db->Get(rocksdb::ReadOptions(), key, &value);
+        break;
+      }
+      case kDelete: {
+        std::string key = fuzzed_data.ConsumeRandomLengthString();
+        db->Delete(rocksdb::WriteOptions(), key);
+        break;
+      }
+      case kGetProperty: {
+        std::string prop;
+        std::string property_name = fuzzed_data.ConsumeRandomLengthString();
+        db->GetProperty(property_name, &prop);
+        break;
+      }
+      case kIterator: {
+        rocksdb::Iterator* it = db->NewIterator(rocksdb::ReadOptions());
+        for (it->SeekToFirst(); it->Valid(); it->Next()) {
+        }
+        delete it;
+        break;
+      }
+      case kSnapshot: {
+        rocksdb::ReadOptions snapshot_options;
+        snapshot_options.snapshot = db->GetSnapshot();
+        rocksdb::Iterator* it = db->NewIterator(snapshot_options);
+        db->ReleaseSnapshot(snapshot_options.snapshot);
+        delete it;
+        break;
+      }
+      case kOpenClose: {
+        db->Close();
+        delete db;
+        status = rocksdb::DB::Open(options, db_path, &db);
+        if (!status.ok()) {
+          rocksdb::DestroyDB(db_path, options);
+          return 0;
+        }
+
+        break;
+      }
+      case kColumn: {
+        rocksdb::ColumnFamilyHandle* cf;
+        rocksdb::Status s;
+        s = db->CreateColumnFamily(rocksdb::ColumnFamilyOptions(), "new_cf",
+                                   &cf);
+        s = db->DestroyColumnFamilyHandle(cf);
+        db->Close();
+        delete db;
+
+        // open DB with two column families
+        std::vector<rocksdb::ColumnFamilyDescriptor> column_families;
+        // have to open default column family
+        column_families.push_back(rocksdb::ColumnFamilyDescriptor(
+            rocksdb::kDefaultColumnFamilyName, rocksdb::ColumnFamilyOptions()));
+        // open the new one, too
+        column_families.push_back(rocksdb::ColumnFamilyDescriptor(
+            "new_cf", rocksdb::ColumnFamilyOptions()));
+        std::vector<rocksdb::ColumnFamilyHandle*> handles;
+        s = rocksdb::DB::Open(rocksdb::DBOptions(), db_path, column_families,
+                              &handles, &db);
+
+        if (s.ok()) {
+          std::string key1 = fuzzed_data.ConsumeRandomLengthString();
+          std::string val1 = fuzzed_data.ConsumeRandomLengthString();
+          std::string key2 = fuzzed_data.ConsumeRandomLengthString();
+          s = db->Put(rocksdb::WriteOptions(), handles[1], key1, val1);
+          std::string value;
+          s = db->Get(rocksdb::ReadOptions(), handles[1], key2, &value);
+          s = db->DropColumnFamily(handles[1]);
+          for (auto handle : handles) {
+            s = db->DestroyColumnFamilyHandle(handle);
+          }
+        } else {
+          status = rocksdb::DB::Open(options, db_path, &db);
+          if (!status.ok()) {
+            // At this point there is no saving to do. So we exit
+            rocksdb::DestroyDB(db_path, rocksdb::Options());
+            return 0;
+          }
+        }
+        break;
+      }
+      case kCompactRange: {
+        std::string slice_start = fuzzed_data.ConsumeRandomLengthString();
+        std::string slice_end = fuzzed_data.ConsumeRandomLengthString();
+
+        rocksdb::Slice begin(slice_start);
+        rocksdb::Slice end(slice_end);
+        rocksdb::CompactRangeOptions options;
+        rocksdb::Status s = db->CompactRange(options, &begin, &end);
+        break;
+      }
+      case kSeekForPrev: {
+        std::string key = fuzzed_data.ConsumeRandomLengthString();
+        auto iter = db->NewIterator(rocksdb::ReadOptions());
+        iter->SeekForPrev(key);
+        delete iter;
+        break;
+      }
+    }
+  }
+
+  // Cleanup DB
+  db->Close();
+  delete db;
+  rocksdb::DestroyDB(db_path, options);
+  return 0;
+}