utilities/backupable: backup should limit the copy size of wal.

Summary:
Since the backup work as snapshot, we should only copy
 the bytes of the wal while we get the alive files.
Closes https://github.com/facebook/rocksdb/pull/1733

Differential Revision: D4373457

Pulled By: ajkr

fbshipit-source-id: 389318f

db/c.cc

@@ -448,7 +448,11 @@ rocksdb_backup_engine_t* rocksdb_backup_engine_open(
     const rocksdb_options_t* options, const char* path, char** errptr) {
   BackupEngine* be;
   if (SaveError(errptr, BackupEngine::Open(options->rep.env,
-                                           BackupableDBOptions(path), &be))) {
+                                           BackupableDBOptions(path,
+                                                               nullptr,
+                                                               true,
+                                                               options->rep.info_log.get()),
+                                           &be))) {
     return nullptr;
   }
   rocksdb_backup_engine_t* result = new rocksdb_backup_engine_t;

utilities/backupable/backupable_db.cc

@@ -781,36 +781,24 @@ Status BackupEngineImpl::CreateNewBackupWithMetadata(
         manifest_fname.size(), 0 /* size_limit */, false /* shared_checksum */,
         progress_callback, manifest_fname.substr(1) + "\n");
   }
-
-  // Pre-fetch sizes for WAL files
-  std::unordered_map<std::string, uint64_t> wal_path_to_size;
-  if (s.ok()) {
-    if (db->GetOptions().wal_dir != "") {
-      s = InsertPathnameToSizeBytes(db->GetOptions().wal_dir, db_env_,
-                                    &wal_path_to_size);
-    } else {
-      wal_path_to_size = std::move(data_path_to_size);
-    }
-  }
-
+  Log(options_.info_log, "begin add wal files for backup -- %" ROCKSDB_PRIszt,
+      live_wal_files.size());
   // Add a CopyOrCreateWorkItem to the channel for each WAL file
   for (size_t i = 0; s.ok() && i < live_wal_files.size(); ++i) {
-    auto wal_path_to_size_iter =
-        wal_path_to_size.find(live_wal_files[i]->PathName());
-    uint64_t size_bytes = wal_path_to_size_iter == wal_path_to_size.end()
-                              ? port::kMaxUint64
-                              : wal_path_to_size_iter->second;
+    uint64_t size_bytes = live_wal_files[i]->SizeFileBytes();
     if (live_wal_files[i]->Type() == kAliveLogFile) {
+      Log(options_.info_log, "add wal file for backup %s -- %" PRIu64,
+          live_wal_files[i]->PathName().c_str(), size_bytes);
       // we only care about live log files
       // copy the file into backup_dir/files/<new backup>/
       s = AddBackupFileWorkItem(live_dst_paths, backup_items_to_finish,
                                 new_backup_id, false, /* not shared */
                                 db->GetOptions().wal_dir,
                                 live_wal_files[i]->PathName(), rate_limiter,
-                                size_bytes);
+                                size_bytes, size_bytes);
     }
   }
-
+  Log(options_.info_log, "add files for backup done, wait finish.");
   Status item_status;
   for (auto& item : backup_items_to_finish) {
     item.result.wait();