A mechanism to detect manifest file write errors and put db in readonly mode.

Summary:
If there is an error while writing an edit to the manifest file, the manifest
file is closed and reopened to check if the edit made it in. However, if the
re-opening of the manifest is unsuccessful and options.paranoid_checks is set
t true, then the db refuses to accept new puts, effectively putting the db
in readonly mode.

In a future diff, I would like to make the default value of paranoid_check
to true.

Test Plan: make check

Reviewers: sheki

Reviewed By: sheki

CC: leveldb

Differential Revision: https://reviews.facebook.net/D9201

db/db_impl.cc

@@ -1445,8 +1445,7 @@ Status DBImpl::InstallCompactionResults(CompactionState* compact) {
   // still exist in the current version and in the same original level.
   // This ensures that a concurrent compaction did not erroneously
   // pick the same files to compact.
-  if (options_.paranoid_checks &&
+  if (!versions_->VerifyCompactionFileConsistency(compact->compaction)) {
-      !versions_->VerifyCompactionFileConsistency(compact->compaction)) {
     Log(options_.info_log,  "Compaction %d@%d + %d@%d files aborted",
       compact->compaction->num_input_files(0),
       compact->compaction->level(),

db/version_set.cc

@@ -1038,7 +1038,8 @@ Status VersionSet::LogAndApply(VersionEdit* edit, port::Mutex* mu,
         if (ManifestContains(record)) {
           Log(options_->info_log,
               "MANIFEST contains log record despite error; advancing to new "
-              "version to prevent mismatch between in-memory and logged state");
+              "version to prevent mismatch between in-memory and logged state"
+              " If paranoid is set, then the db is now in readonly mode.");
           s = Status::OK();
         }
       }
@@ -1565,6 +1566,9 @@ bool VersionSet::ManifestContains(const std::string& record) const {
   Status s = env_->NewSequentialFile(fname, &file);
   if (!s.ok()) {
     Log(options_->info_log, "ManifestContains: %s\n", s.ToString().c_str());
+    Log(options_->info_log,
+        "ManifestContains: is unable to reopen the manifest file  %s",
+        fname.c_str());
     return false;
   }
   log::Reader reader(std::move(file), nullptr, true/*checksum*/, 0);
@@ -1762,6 +1766,7 @@ int64_t VersionSet::MaxGrandParentOverlapBytes(int level) {
 // verify that the files listed in this compaction are present
 // in the current version
 bool VersionSet::VerifyCompactionFileConsistency(Compaction* c) {
+#ifndef NDEBUG
   if (c->input_version_ != current_) {
     Log(options_->info_log, "VerifyCompactionFileConsistency version mismatch");
   }
@@ -1802,6 +1807,7 @@ bool VersionSet::VerifyCompactionFileConsistency(Compaction* c) {
       return false; // input files non existant in current version
     }
   }
+#endif
   return true;     // everything good
 }