Add more kill points

Summary:
Add kill points in:
1. after creating a file
2. before writing a manifest record
3. before syncing manifest
4. before creating a new current file
5. after creating a new current file

Test Plan: Run all current tests.

Reviewers: yhchiang, igor, anthony, IslamAbdelRahman, rven, kradhakrishnan

Reviewed By: kradhakrishnan

Subscribers: leveldb, dhruba

Differential Revision: https://reviews.facebook.net/D48855

db/builder.cc

@@ -76,7 +76,7 @@ Status BuildTable(
     unique_ptr<WritableFileWriter> file_writer;
     {
       unique_ptr<WritableFile> file;
-      s = env->NewWritableFile(fname, &file, env_options);
+      s = NewWritableFile(env, fname, &file, env_options);
       if (!s.ok()) {
         return s;
       }

db/compaction_job.cc

@@ -911,7 +911,7 @@ Status CompactionJob::OpenCompactionOutputFile(
   unique_ptr<WritableFile> writable_file;
   std::string fname = TableFileName(db_options_.db_paths, file_number,
                                     sub_compact->compaction->output_path_id());
-  Status s = env_->NewWritableFile(fname, &writable_file, env_options_);
+  Status s = NewWritableFile(env_, fname, &writable_file, env_options_);
   if (!s.ok()) {
     Log(InfoLogLevel::ERROR_LEVEL, db_options_.info_log,
         "[%s] [JOB %d] OpenCompactionOutputFiles for table #%" PRIu64

db/db_impl.cc

@@ -407,7 +407,7 @@ Status DBImpl::NewDB() {
   {
     unique_ptr<WritableFile> file;
     EnvOptions env_options = env_->OptimizeForManifestWrite(env_options_);
-    s = env_->NewWritableFile(manifest, &file, env_options);
+    s = NewWritableFile(env_, manifest, &file, env_options);
     if (!s.ok()) {
       return s;
     }
@@ -4074,9 +4074,9 @@ Status DBImpl::SwitchMemtable(ColumnFamilyData* cfd, WriteContext* context) {
     if (creating_new_log) {
       EnvOptions opt_env_opt =
           env_->OptimizeForLogWrite(env_options_, db_options_);
-      s = env_->NewWritableFile(
+      s = NewWritableFile(env_,
-          LogFileName(db_options_.wal_dir, new_log_number), &lfile,
+                          LogFileName(db_options_.wal_dir, new_log_number),
-          opt_env_opt);
+                          &lfile, opt_env_opt);
       if (s.ok()) {
         // Our final size should be less than write_buffer_size
         // (compression, etc) but err on the side of caution.
@@ -4692,9 +4692,9 @@ Status DB::Open(const DBOptions& db_options, const std::string& dbname,
     EnvOptions soptions(db_options);
     EnvOptions opt_env_options =
         impl->db_options_.env->OptimizeForLogWrite(soptions, impl->db_options_);
-    s = impl->db_options_.env->NewWritableFile(
+    s = NewWritableFile(impl->db_options_.env,
-        LogFileName(impl->db_options_.wal_dir, new_log_number), &lfile,
+                        LogFileName(impl->db_options_.wal_dir, new_log_number),
-        opt_env_options);
+                        &lfile, opt_env_options);
     if (s.ok()) {
       lfile->SetPreallocationBlockSize(1.1 * max_write_buffer_size);
       impl->logfile_number_ = new_log_number;

db/filename.cc

@@ -21,6 +21,7 @@
 #include "util/file_reader_writer.h"
 #include "util/logging.h"
 #include "util/stop_watch.h"
+#include "util/sync_point.h"
 
 namespace rocksdb {
 
@@ -322,7 +323,9 @@ Status SetCurrentFile(Env* env, const std::string& dbname,
   std::string tmp = TempFileName(dbname, descriptor_number);
   Status s = WriteStringToFile(env, contents.ToString() + "\n", tmp, true);
   if (s.ok()) {
+    TEST_KILL_RANDOM("SetCurrentFile:0", rocksdb_kill_odds * REDUCE_ODDS2);
     s = env->RenameFile(tmp, CurrentFileName(dbname));
+    TEST_KILL_RANDOM("SetCurrentFile:1", rocksdb_kill_odds * REDUCE_ODDS2);
   }
   if (s.ok()) {
     if (directory_to_fsync != nullptr) {
@@ -351,6 +354,7 @@ Status SetIdentityFile(Env* env, const std::string& dbname) {
 
 Status SyncManifest(Env* env, const DBOptions* db_options,
                     WritableFileWriter* file) {
+  TEST_KILL_RANDOM("SyncManifest:0", rocksdb_kill_odds * REDUCE_ODDS2);
   if (db_options->disableDataSync) {
     return Status::OK();
   } else {

db/version_set.cc

@@ -2071,8 +2071,8 @@ Status VersionSet::LogAndApply(ColumnFamilyData* column_family_data,
           "Creating manifest %" PRIu64 "\n", pending_manifest_file_number_);
       unique_ptr<WritableFile> descriptor_file;
       EnvOptions opt_env_opts = env_->OptimizeForManifestWrite(env_options_);
-      s = env_->NewWritableFile(
+      s = NewWritableFile(
-          DescriptorFileName(dbname_, pending_manifest_file_number_),
+          env_, DescriptorFileName(dbname_, pending_manifest_file_number_),
           &descriptor_file, opt_env_opts);
       if (s.ok()) {
         descriptor_file->SetPreallocationBlockSize(
@@ -2099,6 +2099,8 @@ Status VersionSet::LogAndApply(ColumnFamilyData* column_family_data,
               "Unable to Encode VersionEdit:" + e->DebugString(true));
           break;
         }
+        TEST_KILL_RANDOM("VersionSet::LogAndApply:BeforeAddRecord",
+                         rocksdb_kill_odds * REDUCE_ODDS2);
         s = descriptor_log_->AddRecord(record);
         if (!s.ok()) {
           break;

util/file_reader_writer.cc

@@ -149,6 +149,7 @@ Status WritableFileWriter::Close() {
   }
 
   writable_file_.reset();
+  TEST_KILL_RANDOM("WritableFileWriter::Close:1", rocksdb_kill_odds);
 
   return s;
 }
@@ -470,4 +471,12 @@ std::unique_ptr<RandomAccessFile> NewReadaheadRandomAccessFile(
   return result;
 }
 
+Status NewWritableFile(Env* env, const std::string& fname,
+                       unique_ptr<WritableFile>* result,
+                       const EnvOptions& options) {
+  Status s = env->NewWritableFile(fname, result, options);
+  TEST_KILL_RANDOM("NewWritableFile:0", rocksdb_kill_odds * REDUCE_ODDS2);
+  return s;
+}
+
 }  // namespace rocksdb

util/file_reader_writer.h

@@ -7,6 +7,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 #pragma once
+#include <string>
 #include "rocksdb/env.h"
 #include "util/aligned_buffer.h"
 #include "port/port.h"
@@ -36,8 +37,8 @@ class SequentialFileReader {
     return *this;
   }
 
-  SequentialFileReader(SequentialFileReader&) = delete;
+  SequentialFileReader(const SequentialFileReader&) = delete;
-  SequentialFileReader& operator=(SequentialFileReader&) = delete;
+  SequentialFileReader& operator=(const SequentialFileReader&) = delete;
 
   Status Read(size_t n, Slice* result, char* scratch);
 
@@ -163,4 +164,8 @@ class WritableFileWriter {
   size_t RequestToken(size_t bytes, bool align);
   Status SyncInternal(bool use_fsync);
 };
+
+extern Status NewWritableFile(Env* env, const std::string& fname,
+                              unique_ptr<WritableFile>* result,
+                              const EnvOptions& options);
 }  // namespace rocksdb