BlobDB: GetLiveFiles and GetLiveFilesMetadata return relative path (#4326)

Summary:
`GetLiveFiles` and `GetLiveFilesMetadata` should return path relative to db path.

It is a separate issue when `path_relative` is false how can we return relative path. But `DBImpl::GetLiveFiles` don't handle it as well when there are multiple `db_paths`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4326

Differential Revision: D9545904

Pulled By: yiwu-arbug

fbshipit-source-id: 6762d879fcb561df2b612e6fdfb4a6b51db03f5d
main
Yi Wu 6 years ago committed by Facebook Github Bot
parent 1cf17ba53b
commit 462ed70d64
  1. 16
      include/rocksdb/db.h
  2. 7
      util/filename.cc
  3. 3
      util/filename.h
  4. 15
      utilities/blob_db/blob_db_impl_filesnapshot.cc
  5. 10
      utilities/blob_db/blob_db_test.cc

@ -950,14 +950,14 @@ class DB {
// GetLiveFiles followed by GetSortedWalFiles can generate a lossless backup
// Retrieve the list of all files in the database. The files are
// relative to the dbname and are not absolute paths. The valid size of the
// manifest file is returned in manifest_file_size. The manifest file is an
// ever growing file, but only the portion specified by manifest_file_size is
// valid for this snapshot.
// Setting flush_memtable to true does Flush before recording the live files.
// Setting flush_memtable to false is useful when we don't want to wait for
// flush which may have to wait for compaction to complete taking an
// indeterminate time.
// relative to the dbname and are not absolute paths. Despite being relative
// paths, the file names begin with "/". The valid size of the manifest file
// is returned in manifest_file_size. The manifest file is an ever growing
// file, but only the portion specified by manifest_file_size is valid for
// this snapshot. Setting flush_memtable to true does Flush before recording
// the live files. Setting flush_memtable to false is useful when we don't
// want to wait for flush which may have to wait for compaction to complete
// taking an indeterminate time.
//
// In case you have multiple column families, even if flush_memtable is true,
// you still need to call GetSortedWalFiles after GetLiveFiles to compensate

@ -80,6 +80,13 @@ std::string BlobFileName(const std::string& blobdirname, uint64_t number) {
return MakeFileName(blobdirname, number, kRocksDBBlobFileExt.c_str());
}
std::string BlobFileName(const std::string& dbname, const std::string& blob_dir,
uint64_t number) {
assert(number > 0);
return MakeFileName(dbname + "/" + blob_dir, number,
kRocksDBBlobFileExt.c_str());
}
std::string ArchivalDirectory(const std::string& dir) {
return dir + "/" + ARCHIVAL_DIR;
}

@ -49,6 +49,9 @@ extern std::string LogFileName(const std::string& dbname, uint64_t number);
extern std::string BlobFileName(const std::string& bdirname, uint64_t number);
extern std::string BlobFileName(const std::string& dbname,
const std::string& blob_dir, uint64_t number);
static const std::string ARCHIVAL_DIR = "archive";
extern std::string ArchivalDirectory(const std::string& dbname);

@ -7,6 +7,7 @@
#include "utilities/blob_db/blob_db_impl.h"
#include "util/filename.h"
#include "util/logging.h"
#include "util/mutexlock.h"
@ -63,6 +64,10 @@ Status BlobDBImpl::EnableFileDeletions(bool force) {
Status BlobDBImpl::GetLiveFiles(std::vector<std::string>& ret,
uint64_t* manifest_file_size,
bool flush_memtable) {
if (!bdb_options_.path_relative) {
return Status::NotSupported(
"Not able to get relative blob file path from absolute blob_dir.");
}
// Hold a lock in the beginning to avoid updates to base DB during the call
ReadLock rl(&mutex_);
Status s = db_->GetLiveFiles(ret, manifest_file_size, flush_memtable);
@ -72,12 +77,16 @@ Status BlobDBImpl::GetLiveFiles(std::vector<std::string>& ret,
ret.reserve(ret.size() + blob_files_.size());
for (auto bfile_pair : blob_files_) {
auto blob_file = bfile_pair.second;
ret.emplace_back(blob_file->PathName());
// Path should be relative to db_name, but begin with slash.
ret.emplace_back(
BlobFileName("", bdb_options_.blob_dir, blob_file->BlobFileNumber()));
}
return Status::OK();
}
void BlobDBImpl::GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata) {
// Path should be relative to db_name.
assert(bdb_options_.path_relative);
// Hold a lock in the beginning to avoid updates to base DB during the call
ReadLock rl(&mutex_);
db_->GetLiveFilesMetaData(metadata);
@ -85,7 +94,9 @@ void BlobDBImpl::GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata) {
auto blob_file = bfile_pair.second;
LiveFileMetaData filemetadata;
filemetadata.size = blob_file->GetFileSize();
filemetadata.name = blob_file->PathName();
// Path should be relative to db_name, but begin with slash.
filemetadata.name =
BlobFileName("", bdb_options_.blob_dir, blob_file->BlobFileNumber());
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(DefaultColumnFamily());
filemetadata.column_family_name = cfh->GetName();
metadata->emplace_back(filemetadata);

@ -834,6 +834,8 @@ TEST_F(BlobDBTest, ColumnFamilyNotSupported) {
TEST_F(BlobDBTest, GetLiveFilesMetaData) {
Random rnd(301);
BlobDBOptions bdb_options;
bdb_options.blob_dir = "blob_dir";
bdb_options.path_relative = true;
bdb_options.min_blob_size = 0;
bdb_options.disable_background_tasks = true;
Open(bdb_options);
@ -841,16 +843,16 @@ TEST_F(BlobDBTest, GetLiveFilesMetaData) {
for (size_t i = 0; i < 100; i++) {
PutRandom("key" + ToString(i), &rnd, &data);
}
auto *bdb_impl = static_cast<BlobDBImpl *>(blob_db_);
std::vector<LiveFileMetaData> metadata;
bdb_impl->GetLiveFilesMetaData(&metadata);
blob_db_->GetLiveFilesMetaData(&metadata);
ASSERT_EQ(1U, metadata.size());
std::string filename = dbname_ + "/blob_dir/000001.blob";
// Path should be relative to db_name, but begin with slash.
std::string filename = "/blob_dir/000001.blob";
ASSERT_EQ(filename, metadata[0].name);
ASSERT_EQ("default", metadata[0].column_family_name);
std::vector<std::string> livefile;
uint64_t mfs;
bdb_impl->GetLiveFiles(livefile, &mfs, false);
ASSERT_OK(blob_db_->GetLiveFiles(livefile, &mfs, false));
ASSERT_EQ(4U, livefile.size());
ASSERT_EQ(filename, livefile[3]);
VerifyDB(data);

Loading…
Cancel
Save