NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update all blob db TTL and timestamps to uint64_t

Browse Source 
Summary:
The current blob db implementation use mix of int32_t, uint32_t and uint64_t for TTL and expiration. Update all timestamps to uint64_t for consistency.
Closes https://github.com/facebook/rocksdb/pull/2683

Differential Revision: D5557103

Pulled By: yiwu-arbug

fbshipit-source-id: e4eab2691629a755e614e8cf1eed9c3a681d0c42
main
Yi Wu 8 years ago committed by Facebook Github Bot
parent 5883a1ae24
commit 92afe830f9
10 changed files with 112 additions and 114 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 14
      utilities/blob_db/blob_db.h
  2. 78
      utilities/blob_db/blob_db_impl.cc
  3. 15
      utilities/blob_db/blob_db_impl.h
  4. 10
      utilities/blob_db/blob_db_test.cc
  5. 14
      utilities/blob_db/blob_dump_tool.cc
  6. 4
      utilities/blob_db/blob_file.cc
  7. 30
      utilities/blob_db/blob_log_format.cc
  8. 33
      utilities/blob_db/blob_log_format.h
  9. 18
      utilities/blob_db/blob_log_writer.cc
  10. 4
      utilities/blob_db/blob_log_writer.h

14
utilities/blob_db/blob_db.h
Unescape View File

@ -50,7 +50,7 @@ struct BlobDBOptions {
// first bucket is 1471542000 - 1471542600 // first bucket is 1471542000 - 1471542600
// second bucket is 1471542600 - 1471543200 // second bucket is 1471542600 - 1471543200
// and so on // and so on
uint32_t ttl_range_secs = 3600; uint64_t ttl_range_secs = 3600;
// at what bytes will the blob files be synced to blob log. // at what bytes will the blob files be synced to blob log.
uint64_t bytes_per_sync = 0; uint64_t bytes_per_sync = 0;
@ -97,21 +97,21 @@ class BlobDB : public StackableDB {
virtual Status PutWithTTL(const WriteOptions& options, virtual Status PutWithTTL(const WriteOptions& options,
ColumnFamilyHandle* column_family, const Slice& key, ColumnFamilyHandle* column_family, const Slice& key,
const Slice& value, int32_t ttl) = 0; const Slice& value, uint64_t ttl) = 0;
virtual Status PutWithTTL(const WriteOptions& options, const Slice& key, virtual Status PutWithTTL(const WriteOptions& options, const Slice& key,
const Slice& value, int32_t ttl) { const Slice& value, uint64_t ttl) {
return PutWithTTL(options, DefaultColumnFamily(), key, value, ttl); return PutWithTTL(options, DefaultColumnFamily(), key, value, ttl);
} }
// Put with expiration. Key with expiration time equal to -1 // Put with expiration. Key with expiration time equal to
// means the key don't expire. // std::numeric_limits<uint64_t>::max() means the key don't expire.
virtual Status PutUntil(const WriteOptions& options, virtual Status PutUntil(const WriteOptions& options,
ColumnFamilyHandle* column_family, const Slice& key, ColumnFamilyHandle* column_family, const Slice& key,
const Slice& value, int32_t expiration) = 0; const Slice& value, uint64_t expiration) = 0;
virtual Status PutUntil(const WriteOptions& options, const Slice& key, virtual Status PutUntil(const WriteOptions& options, const Slice& key,
const Slice& value, int32_t expiration) { const Slice& value, uint64_t expiration) {
return PutUntil(options, DefaultColumnFamily(), key, value, expiration); return PutUntil(options, DefaultColumnFamily(), key, value, expiration);
} }

78
utilities/blob_db/blob_db_impl.cc
Unescape View File

@ -37,7 +37,7 @@
namespace { namespace {
int kBlockBasedTableVersionFormat = 2; int kBlockBasedTableVersionFormat = 2;
void extendTTL(rocksdb::blob_db::ttlrange_t* ttl_range, uint32_t ttl) { void extendTTL(rocksdb::blob_db::ttlrange_t* ttl_range, uint64_t ttl) {
ttl_range->first = std::min(ttl_range->first, ttl); ttl_range->first = std::min(ttl_range->first, ttl);
ttl_range->second = std::max(ttl_range->second, ttl); ttl_range->second = std::max(ttl_range->second, ttl);
} }
@ -489,9 +489,8 @@ Status BlobDBImpl::OpenAllFiles() {
ttl_range.first + (uint32_t)bdb_options_.ttl_range_secs); ttl_range.first + (uint32_t)bdb_options_.ttl_range_secs);
bfptr->set_ttl_range(ttl_range); bfptr->set_ttl_range(ttl_range);
std::time_t epoch_now = std::chrono::system_clock::to_time_t( uint64_t now = EpochNow();
std::chrono::system_clock::now()); if (ttl_range.second < now) {
if (ttl_range.second < epoch_now) {
Status fstatus = CreateWriterLocked(bfptr); Status fstatus = CreateWriterLocked(bfptr);
if (fstatus.ok()) fstatus = bfptr->WriteFooterAndCloseLocked(); if (fstatus.ok()) fstatus = bfptr->WriteFooterAndCloseLocked();
if (!fstatus.ok()) { if (!fstatus.ok()) {
@ -503,7 +502,7 @@ Status BlobDBImpl::OpenAllFiles() {
} else { } else {
ROCKS_LOG_ERROR(db_options_.info_log, ROCKS_LOG_ERROR(db_options_.info_log,
"Blob File Closed: %s now: %d ttl_range: (%d, %d)", "Blob File Closed: %s now: %d ttl_range: (%d, %d)",
bfpath.c_str(), epoch_now, ttl_range.first, bfpath.c_str(), now, ttl_range.first,
ttl_range.second); ttl_range.second);
} }
} else { } else {
@ -591,7 +590,7 @@ Status BlobDBImpl::CreateWriterLocked(const std::shared_ptr<BlobFile>& bfile) {
} }
std::shared_ptr<BlobFile> BlobDBImpl::FindBlobFileLocked( std::shared_ptr<BlobFile> BlobDBImpl::FindBlobFileLocked(
uint32_t expiration) const { uint64_t expiration) const {
if (open_blob_files_.empty()) return nullptr; if (open_blob_files_.empty()) return nullptr;
std::shared_ptr<BlobFile> tmp = std::make_shared<BlobFile>(); std::shared_ptr<BlobFile> tmp = std::make_shared<BlobFile>();
@ -684,7 +683,8 @@ std::shared_ptr<BlobFile> BlobDBImpl::SelectBlobFile() {
return bfile; return bfile;
} }
std::shared_ptr<BlobFile> BlobDBImpl::SelectBlobFileTTL(uint32_t expiration) { std::shared_ptr<BlobFile> BlobDBImpl::SelectBlobFileTTL(uint64_t expiration) {
assert(expiration != kNoExpiration);
uint64_t epoch_read = 0; uint64_t epoch_read = 0;
std::shared_ptr<BlobFile> bfile; std::shared_ptr<BlobFile> bfile;
{ {
@ -698,9 +698,9 @@ std::shared_ptr<BlobFile> BlobDBImpl::SelectBlobFileTTL(uint32_t expiration) {
return bfile; return bfile;
} }
uint32_t exp_low = uint64_t exp_low =
(expiration / bdb_options_.ttl_range_secs) * bdb_options_.ttl_range_secs; (expiration / bdb_options_.ttl_range_secs) * bdb_options_.ttl_range_secs;
uint32_t exp_high = exp_low + bdb_options_.ttl_range_secs; uint64_t exp_high = exp_low + bdb_options_.ttl_range_secs;
ttlrange_t ttl_guess = std::make_pair(exp_low, exp_high); ttlrange_t ttl_guess = std::make_pair(exp_low, exp_high);
bfile = NewBlobFile("SelectBlobFileTTL"); bfile = NewBlobFile("SelectBlobFileTTL");
@ -758,7 +758,7 @@ Status BlobDBImpl::Put(const WriteOptions& options,
const Slice& value) { const Slice& value) {
std::string new_value; std::string new_value;
Slice value_slice; Slice value_slice;
int32_t expiration = ExtractExpiration(key, value, &value_slice, &new_value); uint64_t expiration = ExtractExpiration(key, value, &value_slice, &new_value);
return PutUntil(options, column_family, key, value_slice, expiration); return PutUntil(options, column_family, key, value_slice, expiration);
} }
@ -808,11 +808,11 @@ Status BlobDBImpl::Write(const WriteOptions& opts, WriteBatch* updates) {
virtual Status PutCF(uint32_t column_family_id, const Slice& key, virtual Status PutCF(uint32_t column_family_id, const Slice& key,
const Slice& value_slice) override { const Slice& value_slice) override {
Slice value_unc; Slice value_unc;
int32_t expiration = uint64_t expiration =
impl_->ExtractExpiration(key, value_slice, &value_unc, &new_value_); impl_->ExtractExpiration(key, value_slice, &value_unc, &new_value_);
std::shared_ptr<BlobFile> bfile = std::shared_ptr<BlobFile> bfile =
(expiration != -1) (expiration != kNoExpiration)
? impl_->SelectBlobFileTTL(expiration) ? impl_->SelectBlobFileTTL(expiration)
: ((last_file_) ? last_file_ : impl_->SelectBlobFile()); : ((last_file_) ? last_file_ : impl_->SelectBlobFile());
if (last_file_ && last_file_ != bfile) { if (last_file_ && last_file_ != bfile) {
@ -840,8 +840,8 @@ Status BlobDBImpl::Write(const WriteOptions& opts, WriteBatch* updates) {
sequence_++; sequence_++;
} }
if (expiration != -1) { if (expiration != kNoExpiration) {
extendTTL(&(bfile->ttl_range_), (uint32_t)expiration); extendTTL(&(bfile->ttl_range_), expiration);
} }
if (!st.ok()) { if (!st.ok()) {
@ -935,9 +935,10 @@ Status BlobDBImpl::Write(const WriteOptions& opts, WriteBatch* updates) {
Status BlobDBImpl::PutWithTTL(const WriteOptions& options, Status BlobDBImpl::PutWithTTL(const WriteOptions& options,
ColumnFamilyHandle* column_family, ColumnFamilyHandle* column_family,
const Slice& key, const Slice& value, const Slice& key, const Slice& value,
int32_t ttl) { uint64_t ttl) {
return PutUntil(options, column_family, key, value, uint64_t now = EpochNow();
static_cast<int32_t>(EpochNow()) + ttl); assert(std::numeric_limits<uint64_t>::max() - now > ttl);
return PutUntil(options, column_family, key, value, now + ttl);
} }
Slice BlobDBImpl::GetCompressedSlice(const Slice& raw, Slice BlobDBImpl::GetCompressedSlice(const Slice& raw,
@ -952,15 +953,15 @@ Slice BlobDBImpl::GetCompressedSlice(const Slice& raw,
return *compression_output; return *compression_output;
} }
// TODO(yiwu): We should use uint64_t for expiration.
Status BlobDBImpl::PutUntil(const WriteOptions& options, Status BlobDBImpl::PutUntil(const WriteOptions& options,
ColumnFamilyHandle* column_family, const Slice& key, ColumnFamilyHandle* column_family, const Slice& key,
const Slice& value_unc, int32_t expiration) { const Slice& value_unc, uint64_t expiration) {
MutexLock l(&write_mutex_); MutexLock l(&write_mutex_);
UpdateWriteOptions(options); UpdateWriteOptions(options);
std::shared_ptr<BlobFile> bfile = std::shared_ptr<BlobFile> bfile = (expiration != kNoExpiration)
(expiration != -1) ? SelectBlobFileTTL(expiration) : SelectBlobFile(); ? SelectBlobFileTTL(expiration)
: SelectBlobFile();
if (!bfile) return Status::NotFound("Blob file not found"); if (!bfile) return Status::NotFound("Blob file not found");
@ -1020,29 +1021,27 @@ Status BlobDBImpl::PutUntil(const WriteOptions& options,
bfile->DumpState().c_str()); bfile->DumpState().c_str());
} }
if (expiration != -1) extendTTL(&(bfile->ttl_range_), (uint32_t)expiration); if (expiration != kNoExpiration) {
extendTTL(&(bfile->ttl_range_), expiration);
}
CloseIf(bfile); CloseIf(bfile);
return s; return s;
} }
// TODO(yiwu): We should return uint64_t after updating the rest of the code uint64_t BlobDBImpl::ExtractExpiration(const Slice& key, const Slice& value,
// to use uint64_t for expiration.
int32_t BlobDBImpl::ExtractExpiration(const Slice& key, const Slice& value,
Slice* value_slice, Slice* value_slice,
std::string* new_value) { std::string* new_value) {
uint64_t expiration = kNoExpiration; uint64_t expiration = kNoExpiration;
bool has_expiration = false;
bool value_changed = false; bool value_changed = false;
if (ttl_extractor_ != nullptr) { if (ttl_extractor_ != nullptr) {
bool has_ttl = ttl_extractor_->ExtractExpiration( has_expiration = ttl_extractor_->ExtractExpiration(
key, value, EpochNow(), &expiration, new_value, &value_changed); key, value, EpochNow(), &expiration, new_value, &value_changed);
if (!has_ttl) {
expiration = kNoExpiration;
}
} }
*value_slice = value_changed ? Slice(*new_value) : value; *value_slice = value_changed ? Slice(*new_value) : value;
return (expiration == kNoExpiration) ? -1 : static_cast<int32_t>(expiration); return has_expiration ? expiration : kNoExpiration;
} }
Status BlobDBImpl::AppendBlob(const std::shared_ptr<BlobFile>& bfile, Status BlobDBImpl::AppendBlob(const std::shared_ptr<BlobFile>& bfile,
@ -1847,11 +1846,11 @@ Status BlobDBImpl::GCFileAndUpdateLSM(const std::shared_ptr<BlobFile>& bfptr,
// Ideally we should hold the lock during the entire function, // Ideally we should hold the lock during the entire function,
// but under the asusmption that this is only called when a // but under the asusmption that this is only called when a
// file is Immutable, we can reduce the critical section // file is Immutable, we can reduce the critical section
bool BlobDBImpl::ShouldGCFile(std::shared_ptr<BlobFile> bfile, std::time_t tt, bool BlobDBImpl::ShouldGCFile(std::shared_ptr<BlobFile> bfile, uint64_t now,
uint64_t last_id, std::string* reason) { uint64_t last_id, std::string* reason) {
if (bfile->HasTTL()) { if (bfile->HasTTL()) {
ttlrange_t ttl_range = bfile->GetTTLRange(); ttlrange_t ttl_range = bfile->GetTTLRange();
if (tt > ttl_range.second) { if (now > ttl_range.second) {
*reason = "entire file ttl expired"; *reason = "entire file ttl expired";
return true; return true;
} }
@ -2057,8 +2056,7 @@ void BlobDBImpl::FilterSubsetOfFiles(
// 100.0 / 15.0 = 7 // 100.0 / 15.0 = 7
uint64_t next_epoch_increment = static_cast<uint64_t>( uint64_t next_epoch_increment = static_cast<uint64_t>(
std::ceil(100 / static_cast<double>(kGCFilePercentage))); std::ceil(100 / static_cast<double>(kGCFilePercentage)));
std::chrono::system_clock::time_point now = std::chrono::system_clock::now(); uint64_t now = EpochNow();
std::time_t tt = std::chrono::system_clock::to_time_t(now);
size_t files_processed = 0; size_t files_processed = 0;
for (auto bfile : blob_files) { for (auto bfile : blob_files) {
@ -2081,18 +2079,20 @@ void BlobDBImpl::FilterSubsetOfFiles(
if (bfile->Obsolete() || !bfile->Immutable()) continue; if (bfile->Obsolete() || !bfile->Immutable()) continue;
std::string reason; std::string reason;
bool shouldgc = ShouldGCFile(bfile, tt, last_id, &reason); bool shouldgc = ShouldGCFile(bfile, now, last_id, &reason);
if (!shouldgc) { if (!shouldgc) {
ROCKS_LOG_DEBUG(db_options_.info_log, ROCKS_LOG_DEBUG(db_options_.info_log,
"File has been skipped for GC ttl %s %d %d reason='%s'", "File has been skipped for GC ttl %s %" PRIu64 " %" PRIu64
bfile->PathName().c_str(), tt, " reason='%s'",
bfile->PathName().c_str(), now,
bfile->GetTTLRange().second, reason.c_str()); bfile->GetTTLRange().second, reason.c_str());
continue; continue;
} }
ROCKS_LOG_INFO(db_options_.info_log, ROCKS_LOG_INFO(db_options_.info_log,
"File has been chosen for GC ttl %s %d %d reason='%s'", "File has been chosen for GC ttl %s %" PRIu64 " %" PRIu64
bfile->PathName().c_str(), tt, bfile->GetTTLRange().second, " reason='%s'",
bfile->PathName().c_str(), now, bfile->GetTTLRange().second,
reason.c_str()); reason.c_str());
to_process->push_back(bfile); to_process->push_back(bfile);
} }

15
utilities/blob_db/blob_db_impl.h
Unescape View File

@ -202,9 +202,6 @@ class BlobDBImpl : public BlobDB {
// how often to schedule check seq files period // how often to schedule check seq files period
static constexpr uint32_t kCheckSeqFilesPeriodMillisecs = 10 * 1000; static constexpr uint32_t kCheckSeqFilesPeriodMillisecs = 10 * 1000;
static constexpr uint64_t kNoExpiration =
std::numeric_limits<uint64_t>::max();
using rocksdb::StackableDB::Put; using rocksdb::StackableDB::Put;
Status Put(const WriteOptions& options, ColumnFamilyHandle* column_family, Status Put(const WriteOptions& options, ColumnFamilyHandle* column_family,
const Slice& key, const Slice& value) override; const Slice& key, const Slice& value) override;
@ -238,12 +235,12 @@ class BlobDBImpl : public BlobDB {
using BlobDB::PutWithTTL; using BlobDB::PutWithTTL;
Status PutWithTTL(const WriteOptions& options, Status PutWithTTL(const WriteOptions& options,
ColumnFamilyHandle* column_family, const Slice& key, ColumnFamilyHandle* column_family, const Slice& key,
const Slice& value, int32_t ttl) override; const Slice& value, uint64_t ttl) override;
using BlobDB::PutUntil; using BlobDB::PutUntil;
Status PutUntil(const WriteOptions& options, Status PutUntil(const WriteOptions& options,
ColumnFamilyHandle* column_family, const Slice& key, ColumnFamilyHandle* column_family, const Slice& key,
const Slice& value_unc, int32_t expiration) override; const Slice& value_unc, uint64_t expiration) override;
Status LinkToBaseDB(DB* db) override; Status LinkToBaseDB(DB* db) override;
@ -290,7 +287,7 @@ class BlobDBImpl : public BlobDB {
// has expired or if threshold of the file has been evicted // has expired or if threshold of the file has been evicted
// tt - current time // tt - current time
// last_id - the id of the non-TTL file to evict // last_id - the id of the non-TTL file to evict
bool ShouldGCFile(std::shared_ptr<BlobFile> bfile, std::time_t tt, bool ShouldGCFile(std::shared_ptr<BlobFile> bfile, uint64_t now,
uint64_t last_id, std::string* reason); uint64_t last_id, std::string* reason);
// collect all the blob log files from the blob directory // collect all the blob log files from the blob directory
@ -299,7 +296,7 @@ class BlobDBImpl : public BlobDB {
// appends a task into timer queue to close the file // appends a task into timer queue to close the file
void CloseIf(const std::shared_ptr<BlobFile>& bfile); void CloseIf(const std::shared_ptr<BlobFile>& bfile);
int32_t ExtractExpiration(const Slice& key, const Slice& value, uint64_t ExtractExpiration(const Slice& key, const Slice& value,
Slice* value_slice, std::string* new_value); Slice* value_slice, std::string* new_value);
Status AppendBlob(const std::shared_ptr<BlobFile>& bfile, Status AppendBlob(const std::shared_ptr<BlobFile>& bfile,
@ -311,12 +308,12 @@ class BlobDBImpl : public BlobDB {
// find an existing blob log file based on the expiration unix epoch // find an existing blob log file based on the expiration unix epoch
// if such a file does not exist, return nullptr // if such a file does not exist, return nullptr
std::shared_ptr<BlobFile> SelectBlobFileTTL(uint32_t expiration); std::shared_ptr<BlobFile> SelectBlobFileTTL(uint64_t expiration);
// find an existing blob log file to append the value to // find an existing blob log file to append the value to
std::shared_ptr<BlobFile> SelectBlobFile(); std::shared_ptr<BlobFile> SelectBlobFile();
std::shared_ptr<BlobFile> FindBlobFileLocked(uint32_t expiration) const; std::shared_ptr<BlobFile> FindBlobFileLocked(uint64_t expiration) const;
void UpdateWriteOptions(const WriteOptions& options); void UpdateWriteOptions(const WriteOptions& options);

10
utilities/blob_db/blob_db_test.cc
Unescape View File

@ -63,7 +63,7 @@ class BlobDBTest : public testing::Test {
} }
} }
void PutRandomWithTTL(const std::string &key, int32_t ttl, Random *rnd, void PutRandomWithTTL(const std::string &key, uint64_t ttl, Random *rnd,
std::map<std::string, std::string> *data = nullptr) { std::map<std::string, std::string> *data = nullptr) {
int len = rnd->Next() % kMaxBlobSize + 1; int len = rnd->Next() % kMaxBlobSize + 1;
std::string value = test::RandomHumanReadableString(rnd, len); std::string value = test::RandomHumanReadableString(rnd, len);
@ -74,7 +74,7 @@ class BlobDBTest : public testing::Test {
} }
} }
void PutRandomUntil(const std::string &key, int32_t expiration, Random *rnd, void PutRandomUntil(const std::string &key, uint64_t expiration, Random *rnd,
std::map<std::string, std::string> *data = nullptr) { std::map<std::string, std::string> *data = nullptr) {
int len = rnd->Next() % kMaxBlobSize + 1; int len = rnd->Next() % kMaxBlobSize + 1;
std::string value = test::RandomHumanReadableString(rnd, len); std::string value = test::RandomHumanReadableString(rnd, len);
@ -136,7 +136,7 @@ class BlobDBTest : public testing::Test {
Random rnd(301); Random rnd(301);
for (size_t i = 0; i < 100000; i++) { for (size_t i = 0; i < 100000; i++) {
int32_t ttl = rnd.Next() % 86400; uint64_t ttl = rnd.Next() % 86400;
PutRandomWithTTL("key" + ToString(i % 500), ttl, &rnd, nullptr); PutRandomWithTTL("key" + ToString(i % 500), ttl, &rnd, nullptr);
} }
@ -175,7 +175,7 @@ TEST_F(BlobDBTest, PutWithTTL) {
std::map<std::string, std::string> data; std::map<std::string, std::string> data;
mock_env_->set_now_micros(50 * 1000000); mock_env_->set_now_micros(50 * 1000000);
for (size_t i = 0; i < 100; i++) { for (size_t i = 0; i < 100; i++) {
int32_t ttl = rnd.Next() % 100; uint64_t ttl = rnd.Next() % 100;
PutRandomWithTTL("key" + ToString(i), ttl, &rnd, PutRandomWithTTL("key" + ToString(i), ttl, &rnd,
(ttl < 50 ? nullptr : &data)); (ttl < 50 ? nullptr : &data));
} }
@ -204,7 +204,7 @@ TEST_F(BlobDBTest, PutUntil) {
std::map<std::string, std::string> data; std::map<std::string, std::string> data;
mock_env_->set_now_micros(50 * 1000000); mock_env_->set_now_micros(50 * 1000000);
for (size_t i = 0; i < 100; i++) { for (size_t i = 0; i < 100; i++) {
int32_t expiration = rnd.Next() % 100 + 50; uint64_t expiration = rnd.Next() % 100 + 50;
PutRandomUntil("key" + ToString(i), expiration, &rnd, PutRandomUntil("key" + ToString(i), expiration, &rnd,
(expiration < 100 ? nullptr : &data)); (expiration < 100 ? nullptr : &data));
} }

14
utilities/blob_db/blob_dump_tool.cc
Unescape View File

@ -102,8 +102,8 @@ Status BlobDumpTool::DumpBlobLogHeader(uint64_t* offset) {
return s; return s;
} }
fprintf(stdout, "Blob log header:\n"); fprintf(stdout, "Blob log header:\n");
fprintf(stdout, " Magic Number : %u\n", header.magic_number()); fprintf(stdout, " Magic Number : %" PRIu32 "\n", header.magic_number());
fprintf(stdout, " Version : %d\n", header.version()); fprintf(stdout, " Version : %" PRIu32 "\n", header.version());
CompressionType compression = header.compression(); CompressionType compression = header.compression();
std::string compression_str; std::string compression_str;
if (!GetStringFromCompressionType(&compression_str, compression).ok()) { if (!GetStringFromCompressionType(&compression_str, compression).ok()) {
@ -175,13 +175,13 @@ Status BlobDumpTool::DumpRecord(DisplayType show_key, DisplayType show_blob,
} }
uint32_t key_size = record.GetKeySize(); uint32_t key_size = record.GetKeySize();
uint64_t blob_size = record.GetBlobSize(); uint64_t blob_size = record.GetBlobSize();
fprintf(stdout, " key size : %d\n", key_size); fprintf(stdout, " key size : %" PRIu32 "\n", key_size);
fprintf(stdout, " blob size : %" PRIu64 "\n", record.GetBlobSize()); fprintf(stdout, " blob size : %" PRIu64 "\n", record.GetBlobSize());
fprintf(stdout, " TTL : %u\n", record.GetTTL()); fprintf(stdout, " TTL : %" PRIu64 "\n", record.GetTTL());
fprintf(stdout, " time : %" PRIu64 "\n", record.GetTimeVal()); fprintf(stdout, " time : %" PRIu64 "\n", record.GetTimeVal());
fprintf(stdout, " type : %d, %d\n", record.type(), record.subtype()); fprintf(stdout, " type : %d, %d\n", record.type(), record.subtype());
fprintf(stdout, " header CRC : %u\n", record.header_checksum()); fprintf(stdout, " header CRC : %" PRIu32 "\n", record.header_checksum());
fprintf(stdout, " CRC : %u\n", record.checksum()); fprintf(stdout, " CRC : %" PRIu32 "\n", record.checksum());
uint32_t header_crc = uint32_t header_crc =
crc32c::Extend(0, slice.data(), slice.size() - 2 * sizeof(uint32_t)); crc32c::Extend(0, slice.data(), slice.size() - 2 * sizeof(uint32_t));
*offset += BlobLogRecord::kHeaderSize; *offset += BlobLogRecord::kHeaderSize;
@ -213,7 +213,7 @@ Status BlobDumpTool::DumpRecord(DisplayType show_key, DisplayType show_blob,
if (!s.ok()) { if (!s.ok()) {
return s; return s;
} }
fprintf(stdout, " footer CRC : %u\n", record.footer_checksum()); fprintf(stdout, " footer CRC : %" PRIu32 "\n", record.footer_checksum());
fprintf(stdout, " sequence : %" PRIu64 "\n", record.GetSN()); fprintf(stdout, " sequence : %" PRIu64 "\n", record.GetSN());
*offset += key_size + blob_size + BlobLogRecord::kFooterSize; *offset += key_size + blob_size + BlobLogRecord::kFooterSize;
return s; return s;

4
utilities/blob_db/blob_file.cc
Unescape View File

@ -94,8 +94,8 @@ std::string BlobFile::DumpState() const {
"path: %s fn: %" PRIu64 " blob_count: %" PRIu64 " gc_epoch: %" PRIu64 "path: %s fn: %" PRIu64 " blob_count: %" PRIu64 " gc_epoch: %" PRIu64
" file_size: %" PRIu64 " deleted_count: %" PRIu64 " file_size: %" PRIu64 " deleted_count: %" PRIu64
" deleted_size: %" PRIu64 " deleted_size: %" PRIu64
" closed: %d can_be_deleted: %d ttl_range: (%d, %d)" " closed: %d can_be_deleted: %d ttl_range: (%" PRIu64 ", %" PRIu64
" sn_range: (%" PRIu64 " %" PRIu64 "), writer: %d reader: %d", ") sn_range: (%" PRIu64 " %" PRIu64 "), writer: %d reader: %d",
path_to_dir_.c_str(), file_number_, blob_count_.load(), path_to_dir_.c_str(), file_number_, blob_count_.load(),
gc_epoch_.load(), file_size_.load(), deleted_count_, deleted_size_, gc_epoch_.load(), file_size_.load(), deleted_count_, deleted_size_,
closed_.load(), can_be_deleted_.load(), ttl_range_.first, closed_.load(), can_be_deleted_.load(), ttl_range_.first,

30
utilities/blob_db/blob_log_format.cc
Unescape View File

@ -61,8 +61,8 @@ Status BlobLogFooter::DecodeFrom(const Slice& input) {
} }
ttlrange_t temp_ttl; ttlrange_t temp_ttl;
if (!GetFixed32(&slice, &temp_ttl.first) || if (!GetFixed64(&slice, &temp_ttl.first) ||
!GetFixed32(&slice, &temp_ttl.second)) { !GetFixed64(&slice, &temp_ttl.second)) {
return Status::Corruption("Invalid Blob Footer: ttl_range"); return Status::Corruption("Invalid Blob Footer: ttl_range");
} }
if (has_ttl) { if (has_ttl) {
@ -108,11 +108,11 @@ void BlobLogFooter::EncodeTo(std::string* dst) const {
bool has_ts = HasTimestamp(); bool has_ts = HasTimestamp();
if (has_ttl) { if (has_ttl) {
PutFixed32(dst, ttl_range_.get()->first); PutFixed64(dst, ttl_range_.get()->first);
PutFixed32(dst, ttl_range_.get()->second); PutFixed64(dst, ttl_range_.get()->second);
} else { } else {
PutFixed32(dst, 0); PutFixed64(dst, 0);
PutFixed32(dst, 0); PutFixed64(dst, 0);
} }
PutFixed64(dst, sn_range_.first); PutFixed64(dst, sn_range_.first);
PutFixed64(dst, sn_range_.second); PutFixed64(dst, sn_range_.second);
@ -149,11 +149,11 @@ void BlobLogHeader::EncodeTo(std::string* dst) const {
PutFixed32(dst, val); PutFixed32(dst, val);
if (has_ttl) { if (has_ttl) {
PutFixed32(dst, ttl_guess_.get()->first); PutFixed64(dst, ttl_guess_.get()->first);
PutFixed32(dst, ttl_guess_.get()->second); PutFixed64(dst, ttl_guess_.get()->second);
} else { } else {
PutFixed32(dst, 0); PutFixed64(dst, 0);
PutFixed32(dst, 0); PutFixed64(dst, 0);
} }
if (has_ts) { if (has_ts) {
@ -199,11 +199,13 @@ Status BlobLogHeader::DecodeFrom(const Slice& input) {
} }
ttlrange_t temp_ttl; ttlrange_t temp_ttl;
if (!GetFixed32(&slice, &temp_ttl.first) || if (!GetFixed64(&slice, &temp_ttl.first) ||
!GetFixed32(&slice, &temp_ttl.second)) { !GetFixed64(&slice, &temp_ttl.second)) {
return Status::Corruption("Invalid Blob Log Header: ttl"); return Status::Corruption("Invalid Blob Log Header: ttl");
} }
if (has_ttl) set_ttl_guess(temp_ttl); if (has_ttl) {
set_ttl_guess(temp_ttl);
}
tsrange_t temp_ts; tsrange_t temp_ts;
if (!GetFixed64(&slice, &temp_ts.first) || if (!GetFixed64(&slice, &temp_ts.first) ||
@ -265,7 +267,7 @@ Status BlobLogRecord::DecodeHeaderFrom(const Slice& hdrslice) {
if (!GetFixed64(&input, &blob_size_)) { if (!GetFixed64(&input, &blob_size_)) {
return Status::Corruption("Invalid Blob Record Header: blob_size"); return Status::Corruption("Invalid Blob Record Header: blob_size");
} }
if (!GetFixed32(&input, &ttl_val_)) { if (!GetFixed64(&input, &ttl_val_)) {
return Status::Corruption("Invalid Blob Record Header: ttl_val"); return Status::Corruption("Invalid Blob Record Header: ttl_val");
} }
if (!GetFixed64(&input, &time_val_)) { if (!GetFixed64(&input, &time_val_)) {

33
utilities/blob_db/blob_log_format.h
Unescape View File

@ -25,6 +25,8 @@ namespace blob_db {
class BlobFile; class BlobFile;
class BlobDBImpl; class BlobDBImpl;
constexpr uint64_t kNoExpiration = std::numeric_limits<uint64_t>::max();
enum RecordType : uint8_t { enum RecordType : uint8_t {
// Zero is reserved for preallocated files // Zero is reserved for preallocated files
kFullType = 0, kFullType = 0,
@ -46,9 +48,9 @@ extern const uint32_t kMagicNumber;
class Reader; class Reader;
typedef std::pair<uint32_t, uint32_t> ttlrange_t; using ttlrange_t = std::pair<uint64_t, uint64_t>;
typedef std::pair<uint64_t, uint64_t> tsrange_t; using tsrange_t = std::pair<uint64_t, uint64_t>;
typedef std::pair<rocksdb::SequenceNumber, rocksdb::SequenceNumber> snrange_t; using snrange_t = std::pair<rocksdb::SequenceNumber, rocksdb::SequenceNumber>;
class BlobLogHeader { class BlobLogHeader {
friend class BlobFile; friend class BlobFile;
@ -71,8 +73,8 @@ class BlobLogHeader {
void set_ts_guess(const tsrange_t& ts) { ts_guess_.reset(new tsrange_t(ts)); } void set_ts_guess(const tsrange_t& ts) { ts_guess_.reset(new tsrange_t(ts)); }
public: public:
// magic number + version + flags + ttl guess + timestamp range = 36 // magic number + version + flags + ttl guess + timestamp range = 44
static const size_t kHeaderSize = 4 + 4 + 4 + 4 * 2 + 8 * 2; static const size_t kHeaderSize = 4 + 4 + 4 + 8 * 2 + 8 * 2;
void EncodeTo(std::string* dst) const; void EncodeTo(std::string* dst) const;
@ -100,9 +102,9 @@ class BlobLogHeader {
return *ts_guess_; return *ts_guess_;
} }
bool HasTTL() const { return !!ttl_guess_; } bool HasTTL() const { return ttl_guess_ != nullptr; }
bool HasTimestamp() const { return !!ts_guess_; } bool HasTimestamp() const { return ts_guess_ != nullptr; }
BlobLogHeader& operator=(BlobLogHeader&& in) noexcept; BlobLogHeader& operator=(BlobLogHeader&& in) noexcept;
}; };
@ -128,11 +130,11 @@ class BlobLogFooter {
// footer size = 4 byte magic number // footer size = 4 byte magic number
// 8 bytes count // 8 bytes count
// 4, 4 - ttl range // 8, 8 - ttl range
// 8, 8 - sn range // 8, 8 - sn range
// 8, 8 - ts range // 8, 8 - ts range
// = 56 // = 64
static const size_t kFooterSize = 4 + 4 + 8 + (4 * 2) + (8 * 2) + (8 * 2); static const size_t kFooterSize = 4 + 4 + 8 + (8 * 2) + (8 * 2) + (8 * 2);
bool HasTTL() const { return !!ttl_range_; } bool HasTTL() const { return !!ttl_range_; }
@ -185,7 +187,7 @@ class BlobLogRecord {
uint32_t key_size_; uint32_t key_size_;
uint64_t blob_size_; uint64_t blob_size_;
uint64_t time_val_; uint64_t time_val_;
uint32_t ttl_val_; uint64_t ttl_val_;
SequenceNumber sn_; SequenceNumber sn_;
uint32_t footer_cksum_; uint32_t footer_cksum_;
char type_; char type_;
@ -209,11 +211,12 @@ class BlobLogRecord {
public: public:
// Header is // Header is
// Key Length ( 4 bytes ), // Key Length ( 4 bytes ),
// Blob Length ( 8 bytes), timestamp/ttl (8 bytes), // Blob Length ( 8 bytes),
// ttl (8 bytes), timestamp (8 bytes),
// type (1 byte), subtype (1 byte) // type (1 byte), subtype (1 byte)
// header checksum (4 bytes), blob checksum (4 bytes), // header checksum (4 bytes), blob checksum (4 bytes),
// = 34 // = 42
static const size_t kHeaderSize = 4 + 4 + 4 + 8 + 4 + 8 + 1 + 1; static const size_t kHeaderSize = 4 + 4 + 8 + 8 + 4 + 8 + 1 + 1;
static const size_t kFooterSize = 8 + 4; static const size_t kFooterSize = 8 + 4;
@ -234,7 +237,7 @@ class BlobLogRecord {
return ttl_val_ != std::numeric_limits<uint32_t>::max(); return ttl_val_ != std::numeric_limits<uint32_t>::max();
} }
uint32_t GetTTL() const { return ttl_val_; } uint64_t GetTTL() const { return ttl_val_; }
uint64_t GetTimeVal() const { return time_val_; } uint64_t GetTimeVal() const { return time_val_; }

18
utilities/blob_db/blob_log_writer.cc
Unescape View File

@ -8,7 +8,6 @@
#include "utilities/blob_db/blob_log_writer.h" #include "utilities/blob_db/blob_log_writer.h"
#include <cstdint> #include <cstdint>
#include <limits>
#include <string> #include <string>
#include "rocksdb/env.h" #include "rocksdb/env.h"
#include "util/coding.h" #include "util/coding.h"
@ -72,7 +71,7 @@ Status Writer::AppendFooter(const BlobLogFooter& footer) {
Status Writer::AddRecord(const Slice& key, const Slice& val, Status Writer::AddRecord(const Slice& key, const Slice& val,
uint64_t* key_offset, uint64_t* blob_offset, uint64_t* key_offset, uint64_t* blob_offset,
uint32_t ttl) { uint64_t ttl) {
assert(block_offset_ != 0); assert(block_offset_ != 0);
assert(last_elem_type_ == kEtFileHdr || last_elem_type_ == kEtFooter); assert(last_elem_type_ == kEtFileHdr || last_elem_type_ == kEtFooter);
@ -96,26 +95,23 @@ Status Writer::AddRecord(const Slice& key, const Slice& val,
} }
void Writer::ConstructBlobHeader(std::string* headerbuf, const Slice& key, void Writer::ConstructBlobHeader(std::string* headerbuf, const Slice& key,
const Slice& val, int32_t ttl, int64_t ts) { const Slice& val, uint64_t ttl, int64_t ts) {
headerbuf->reserve(BlobLogRecord::kHeaderSize); headerbuf->reserve(BlobLogRecord::kHeaderSize);
uint32_t key_size = static_cast<uint32_t>(key.size()); uint32_t key_size = static_cast<uint32_t>(key.size());
PutFixed32(headerbuf, key_size); PutFixed32(headerbuf, key_size);
PutFixed64(headerbuf, val.size()); PutFixed64(headerbuf, val.size());
uint32_t ttl_write = (ttl != -1) ? static_cast<uint32_t>(ttl) PutFixed64(headerbuf, ttl);
: std::numeric_limits<uint32_t>::max(); PutFixed64(headerbuf, ts);
PutFixed32(headerbuf, ttl_write);
uint64_t ts_write = (ts != -1) ? static_cast<uint64_t>(ts)
: std::numeric_limits<uint64_t>::max();
PutFixed64(headerbuf, ts_write);
RecordType t = kFullType; RecordType t = kFullType;
headerbuf->push_back(static_cast<char>(t)); headerbuf->push_back(static_cast<char>(t));
RecordSubType st = kRegularType; RecordSubType st = kRegularType;
if (ttl != -1) st = kTTLType; if (ttl != kNoExpiration) {
st = kTTLType;
}
headerbuf->push_back(static_cast<char>(st)); headerbuf->push_back(static_cast<char>(st));
uint32_t header_crc = 0; uint32_t header_crc = 0;

4
utilities/blob_db/blob_log_writer.h
Unescape View File

@ -41,13 +41,13 @@ class Writer {
~Writer(); ~Writer();
static void ConstructBlobHeader(std::string* headerbuf, const Slice& key, static void ConstructBlobHeader(std::string* headerbuf, const Slice& key,
const Slice& val, int32_t ttl, int64_t ts); const Slice& val, uint64_t ttl, int64_t ts);
Status AddRecord(const Slice& key, const Slice& val, uint64_t* key_offset, Status AddRecord(const Slice& key, const Slice& val, uint64_t* key_offset,
uint64_t* blob_offset); uint64_t* blob_offset);
Status AddRecord(const Slice& key, const Slice& val, uint64_t* key_offset, Status AddRecord(const Slice& key, const Slice& val, uint64_t* key_offset,
uint64_t* blob_offset, uint32_t ttl); uint64_t* blob_offset, uint64_t ttl);
Status EmitPhysicalRecord(const std::string& headerbuf, const Slice& key, Status EmitPhysicalRecord(const std::string& headerbuf, const Slice& key,
const Slice& val, uint64_t* key_offset, const Slice& val, uint64_t* key_offset,

Write Preview
Loading…
Cancel
Save