disable the log_number check in Recover()

Summary:
There is a chance that an old MANIFEST is corrupted in 2.7 but just not noticed.
This check would fail them. Change it to log instead of returning a
Corruption status.

Test Plan: make

Reviewers: haobo, igor

Reviewed By: igor

CC: leveldb

Differential Revision: https://reviews.facebook.net/D16923

db/db_impl.cc

@@ -449,6 +449,7 @@ uint64_t DBImpl::TEST_Current_Manifest_FileNo() {
 
 Status DBImpl::NewDB() {
   VersionEdit new_db;
+  new_db.SetVersionNumber();
   new_db.SetComparatorName(user_comparator()->Name());
   new_db.SetLogNumber(0);
   new_db.SetNextFile(2);

db/version_edit.cc

@@ -28,16 +28,19 @@ enum Tag {
   kPrevLogNumber        = 9,
 
   // these are new formats divergent from open source leveldb
-  kNewFile2             = 100  // store smallest & largest seqno
+  kNewFile2             = 100,  // store smallest & largest seqno
+  kVersionNumber        = 101,  // manifest version number, available after 2.8
 };
 
 void VersionEdit::Clear() {
+  version_number_ = 0;
   comparator_.clear();
   max_level_ = 0;
   log_number_ = 0;
   prev_log_number_ = 0;
   last_sequence_ = 0;
   next_file_number_ = 0;
+  has_version_number_ = false;
   has_comparator_ = false;
   has_log_number_ = false;
   has_prev_log_number_ = false;
@@ -48,6 +51,10 @@ void VersionEdit::Clear() {
 }
 
 void VersionEdit::EncodeTo(std::string* dst) const {
+  if (has_version_number_) {
+    PutVarint32(dst, kVersionNumber);
+    PutVarint32(dst, version_number_);
+  }
   if (has_comparator_) {
     PutVarint32(dst, kComparator);
     PutLengthPrefixedSlice(dst, comparator_);
@@ -126,6 +133,14 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
 
   while (msg == nullptr && GetVarint32(&input, &tag)) {
     switch (tag) {
+      case kVersionNumber:
+        if (GetVarint32(&input, &version_number_)) {
+          has_version_number_ = true;
+        } else {
+          msg = "version number";
+        }
+        break;
+
       case kComparator:
         if (GetLengthPrefixedSlice(&input, &str)) {
           comparator_ = str.ToString();

db/version_edit.h

@@ -46,6 +46,10 @@ class VersionEdit {
 
   void Clear();
 
+  void SetVersionNumber() {
+    has_version_number_ = true;
+    version_number_ = kManifestVersion;
+  }
   void SetComparatorName(const Slice& name) {
     has_comparator_ = true;
     comparator_ = name.ToString();
@@ -110,11 +114,13 @@ class VersionEdit {
   bool GetLevel(Slice* input, int* level, const char** msg);
 
   int max_level_;
+  uint32_t version_number_;
   std::string comparator_;
   uint64_t log_number_;
   uint64_t prev_log_number_;
   uint64_t next_file_number_;
   SequenceNumber last_sequence_;
+  bool has_version_number_;
   bool has_comparator_;
   bool has_log_number_;
   bool has_prev_log_number_;
@@ -123,6 +129,10 @@ class VersionEdit {
 
   DeletedFileSet deleted_files_;
   std::vector<std::pair<int, FileMetaData> > new_files_;
+
+  enum {
+    kManifestVersion = 1
+  };
 };
 
 }  // namespace rocksdb

db/version_set.cc

@@ -1746,6 +1746,8 @@ Status VersionSet::Recover() {
     return s;
   }
 
+  bool have_version_number = false;
+  bool log_number_decrease = false;
   bool have_log_number = false;
   bool have_prev_log_number = false;
   bool have_next_file = false;
@@ -1786,17 +1788,21 @@ Status VersionSet::Recover() {
 
       builder.Apply(&edit);
 
+      if (edit.has_version_number_) {
+        have_version_number = true;
+      }
+
       // Only a flush's edit or a new snapshot can write log number during
       // LogAndApply. Since memtables are flushed and inserted into
       // manifest_writers_ queue in order, the log number in MANIFEST file
       // should be monotonically increasing.
       if (edit.has_log_number_) {
-        if (have_log_number && log_number > edit.log_number_) {
-          s = Status::Corruption("log number decreases");
-          break;
+        if (have_log_number && log_number >= edit.log_number_) {
+          log_number_decrease = true;
+        } else {
+          log_number = edit.log_number_;
+          have_log_number = true;
         }
-        log_number = edit.log_number_;
-        have_log_number = true;
       }
 
       if (edit.has_prev_log_number_) {
@@ -1814,6 +1820,20 @@ Status VersionSet::Recover() {
         have_last_sequence = true;
       }
     }
+
+    if (s.ok() && log_number_decrease) {
+      // Since release 2.8, version number is added into MANIFEST file.
+      // Prior release 2.8, a bug in LogAndApply() can cause log_number
+      // to be smaller than the one from previous edit. To ensure backward
+      // compatibility, only fail for MANIFEST genearated by release 2.8
+      // and after.
+      if (have_version_number) {
+        s = Status::Corruption("log number decreases");
+      } else {
+        Log(options_->info_log, "decreasing of log_number is detected "
+            "in MANIFEST\n");
+      }
+    }
   }
 
   if (s.ok()) {
@@ -2083,6 +2103,7 @@ Status VersionSet::WriteSnapshot(log::Writer* log) {
 
   // Save metadata
   VersionEdit edit;
+  edit.SetVersionNumber();
   edit.SetComparatorName(icmp_.user_comparator()->Name());
 
   // Save files