From 10ba50e9eb9a38ba3e4b8e6241b2e65f4150f2d2 Mon Sep 17 00:00:00 2001 From: Yi Wu Date: Fri, 13 Oct 2017 14:36:36 -0700 Subject: [PATCH] Blob DB: Move BlobFile definition to a separate file Summary: simply move BlobFile definition from blob_db_impl.h to blob_file.h. Closes https://github.com/facebook/rocksdb/pull/3002 Differential Revision: D6050143 Pulled By: yiwu-arbug fbshipit-source-id: a8fb6e094fe39bdeace6279569834bc65aa64a34 --- utilities/blob_db/blob_db_impl.h | 168 +-------------------------- utilities/blob_db/blob_file.cc | 10 +- utilities/blob_db/blob_file.h | 187 +++++++++++++++++++++++++++++++ 3 files changed, 194 insertions(+), 171 deletions(-) create mode 100644 utilities/blob_db/blob_file.h diff --git a/utilities/blob_db/blob_db_impl.h b/utilities/blob_db/blob_db_impl.h index a9e0c3846..6cb0d30a2 100644 --- a/utilities/blob_db/blob_db_impl.h +++ b/utilities/blob_db/blob_db_impl.h @@ -9,7 +9,6 @@ #include #include -#include #include #include #include @@ -24,11 +23,11 @@ #include "rocksdb/listener.h" #include "rocksdb/options.h" #include "rocksdb/wal_filter.h" -#include "util/file_reader_writer.h" #include "util/mpsc.h" #include "util/mutexlock.h" #include "util/timer_queue.h" #include "utilities/blob_db/blob_db.h" +#include "utilities/blob_db/blob_file.h" #include "utilities/blob_db/blob_log_format.h" #include "utilities/blob_db/blob_log_reader.h" #include "utilities/blob_db/blob_log_writer.h" @@ -533,171 +532,6 @@ class BlobDBImpl : public BlobDB { uint32_t debug_level_; }; -class BlobFile { - friend class BlobDBImpl; - friend struct blobf_compare_ttl; - - private: - // access to parent - const BlobDBImpl* parent_; - - // path to blob directory - std::string path_to_dir_; - - // the id of the file. - // the above 2 are created during file creation and never changed - // after that - uint64_t file_number_; - - // number of blobs in the file - std::atomic blob_count_; - - // the file will be selected for GC in this future epoch - std::atomic gc_epoch_; - - // size of the file - std::atomic file_size_; - - // number of blobs in this particular file which have been evicted - uint64_t deleted_count_; - - // size of deleted blobs (used by heuristic to select file for GC) - uint64_t deleted_size_; - - BlobLogHeader header_; - - // closed_ = true implies the file is no more mutable - // no more blobs will be appended and the footer has been written out - std::atomic closed_; - - // has a pass of garbage collection successfully finished on this file - // can_be_deleted_ still needs to do iterator/snapshot checks - std::atomic can_be_deleted_; - - // should this file been gc'd once to reconcile lost deletes/compactions - std::atomic gc_once_after_open_; - - // et - lt of the blobs - ttlrange_t ttl_range_; - - // et - lt of the timestamp of the KV pairs. - tsrange_t time_range_; - - // ESN - LSN of the blobs - snrange_t sn_range_; - - // Sequential/Append writer for blobs - std::shared_ptr log_writer_; - - // random access file reader for GET calls - std::shared_ptr ra_file_reader_; - - // This Read-Write mutex is per file specific and protects - // all the datastructures - mutable port::RWMutex mutex_; - - // time when the random access reader was last created. - std::atomic last_access_; - - // last time file was fsync'd/fdatasyncd - std::atomic last_fsync_; - - bool header_valid_; - - public: - BlobFile(); - - BlobFile(const BlobDBImpl* parent, const std::string& bdir, uint64_t fnum); - - ~BlobFile(); - - ColumnFamilyHandle* GetColumnFamily(DB* db); - - // Returns log file's pathname relative to the main db dir - // Eg. For a live-log-file = blob_dir/000003.blob - std::string PathName() const; - - // Primary identifier for blob file. - // once the file is created, this never changes - uint64_t BlobFileNumber() const { return file_number_; } - - // the following functions are atomic, and don't need - // read lock - uint64_t BlobCount() const { - return blob_count_.load(std::memory_order_acquire); - } - - std::string DumpState() const; - - // if the file has gone through GC and blobs have been relocated - bool Obsolete() const { return can_be_deleted_.load(); } - - // if the file is not taking any more appends. - bool Immutable() const { return closed_.load(); } - - // we will assume this is atomic - bool NeedsFsync(bool hard, uint64_t bytes_per_sync) const; - - uint64_t GetFileSize() const { - return file_size_.load(std::memory_order_acquire); - } - - // All Get functions which are not atomic, will need ReadLock on the mutex - tsrange_t GetTimeRange() const { - assert(HasTimestamp()); - return time_range_; - } - - ttlrange_t GetTTLRange() const { return ttl_range_; } - - snrange_t GetSNRange() const { return sn_range_; } - - bool HasTTL() const { - assert(header_valid_); - return header_.HasTTL(); - } - - bool HasTimestamp() const { - assert(header_valid_); - return header_.HasTimestamp(); - } - - std::shared_ptr GetWriter() const { return log_writer_; } - - void Fsync(); - - private: - std::shared_ptr OpenSequentialReader( - Env* env, const DBOptions& db_options, - const EnvOptions& env_options) const; - - Status ReadFooter(BlobLogFooter* footer); - - Status WriteFooterAndCloseLocked(); - - std::shared_ptr GetOrOpenRandomAccessReader( - Env* env, const EnvOptions& env_options, bool* fresh_open); - - void CloseRandomAccessLocked(); - - // this is used, when you are reading only the footer of a - // previously closed file - Status SetFromFooterLocked(const BlobLogFooter& footer); - - void set_time_range(const tsrange_t& tr) { time_range_ = tr; } - - void set_ttl_range(const ttlrange_t& ttl) { ttl_range_ = ttl; } - - void SetSNRange(const snrange_t& snr) { sn_range_ = snr; } - - // The following functions are atomic, and don't need locks - void SetFileSize(uint64_t fs) { file_size_ = fs; } - - void SetBlobCount(uint64_t bc) { blob_count_ = bc; } - - void SetCanBeDeleted() { can_be_deleted_ = true; } -}; - class BlobDBIterator : public Iterator { public: explicit BlobDBIterator(Iterator* iter, BlobDBImpl* impl, bool own_snapshot, diff --git a/utilities/blob_db/blob_file.cc b/utilities/blob_db/blob_file.cc index 8ba005dd6..dd98ed9d1 100644 --- a/utilities/blob_db/blob_file.cc +++ b/utilities/blob_db/blob_file.cc @@ -3,15 +3,15 @@ // COPYING file in the root directory) and Apache 2.0 License // (found in the LICENSE.Apache file in the root directory). #ifndef ROCKSDB_LITE +#include "utilities/blob_db/blob_file.h" #include -#include #include #include -#include "utilities/blob_db/blob_db_impl.h" #include "util/filename.h" #include "util/logging.h" +#include "utilities/blob_db/blob_db_impl.h" namespace rocksdb { @@ -193,8 +193,10 @@ void BlobFile::CloseRandomAccessLocked() { std::shared_ptr BlobFile::GetOrOpenRandomAccessReader( Env* env, const EnvOptions& env_options, bool* fresh_open) { *fresh_open = false; - last_access_ = - std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); + int64_t current_time = 0; + env->GetCurrentTime(¤t_time); + last_access_.store(current_time); + { ReadLock lockbfile_r(&mutex_); if (ra_file_reader_) return ra_file_reader_; diff --git a/utilities/blob_db/blob_file.h b/utilities/blob_db/blob_file.h new file mode 100644 index 000000000..d648ddee4 --- /dev/null +++ b/utilities/blob_db/blob_file.h @@ -0,0 +1,187 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found in the +// COPYING file in the root directory) and Apache 2.0 License +// (found in the LICENSE.Apache file in the root directory). +#pragma once +#ifndef ROCKSDB_LITE + +#include +#include + +#include "port/port.h" +#include "rocksdb/env.h" +#include "rocksdb/options.h" +#include "util/file_reader_writer.h" +#include "utilities/blob_db/blob_log_format.h" +#include "utilities/blob_db/blob_log_writer.h" + +namespace rocksdb { +namespace blob_db { + +class BlobFile { + friend class BlobDBImpl; + friend struct blobf_compare_ttl; + + private: + // access to parent + const BlobDBImpl* parent_; + + // path to blob directory + std::string path_to_dir_; + + // the id of the file. + // the above 2 are created during file creation and never changed + // after that + uint64_t file_number_; + + // number of blobs in the file + std::atomic blob_count_; + + // the file will be selected for GC in this future epoch + std::atomic gc_epoch_; + + // size of the file + std::atomic file_size_; + + // number of blobs in this particular file which have been evicted + uint64_t deleted_count_; + + // size of deleted blobs (used by heuristic to select file for GC) + uint64_t deleted_size_; + + BlobLogHeader header_; + + // closed_ = true implies the file is no more mutable + // no more blobs will be appended and the footer has been written out + std::atomic closed_; + + // has a pass of garbage collection successfully finished on this file + // can_be_deleted_ still needs to do iterator/snapshot checks + std::atomic can_be_deleted_; + + // should this file been gc'd once to reconcile lost deletes/compactions + std::atomic gc_once_after_open_; + + // et - lt of the blobs + ttlrange_t ttl_range_; + + // et - lt of the timestamp of the KV pairs. + tsrange_t time_range_; + + // ESN - LSN of the blobs + snrange_t sn_range_; + + // Sequential/Append writer for blobs + std::shared_ptr log_writer_; + + // random access file reader for GET calls + std::shared_ptr ra_file_reader_; + + // This Read-Write mutex is per file specific and protects + // all the datastructures + mutable port::RWMutex mutex_; + + // time when the random access reader was last created. + std::atomic last_access_; + + // last time file was fsync'd/fdatasyncd + std::atomic last_fsync_; + + bool header_valid_; + + public: + BlobFile(); + + BlobFile(const BlobDBImpl* parent, const std::string& bdir, uint64_t fnum); + + ~BlobFile(); + + ColumnFamilyHandle* GetColumnFamily(DB* db); + + // Returns log file's pathname relative to the main db dir + // Eg. For a live-log-file = blob_dir/000003.blob + std::string PathName() const; + + // Primary identifier for blob file. + // once the file is created, this never changes + uint64_t BlobFileNumber() const { return file_number_; } + + // the following functions are atomic, and don't need + // read lock + uint64_t BlobCount() const { + return blob_count_.load(std::memory_order_acquire); + } + + std::string DumpState() const; + + // if the file has gone through GC and blobs have been relocated + bool Obsolete() const { return can_be_deleted_.load(); } + + // if the file is not taking any more appends. + bool Immutable() const { return closed_.load(); } + + // we will assume this is atomic + bool NeedsFsync(bool hard, uint64_t bytes_per_sync) const; + + uint64_t GetFileSize() const { + return file_size_.load(std::memory_order_acquire); + } + + // All Get functions which are not atomic, will need ReadLock on the mutex + tsrange_t GetTimeRange() const { + assert(HasTimestamp()); + return time_range_; + } + + ttlrange_t GetTTLRange() const { return ttl_range_; } + + snrange_t GetSNRange() const { return sn_range_; } + + bool HasTTL() const { + assert(header_valid_); + return header_.HasTTL(); + } + + bool HasTimestamp() const { + assert(header_valid_); + return header_.HasTimestamp(); + } + + std::shared_ptr GetWriter() const { return log_writer_; } + + void Fsync(); + + private: + std::shared_ptr OpenSequentialReader( + Env* env, const DBOptions& db_options, + const EnvOptions& env_options) const; + + Status ReadFooter(BlobLogFooter* footer); + + Status WriteFooterAndCloseLocked(); + + std::shared_ptr GetOrOpenRandomAccessReader( + Env* env, const EnvOptions& env_options, bool* fresh_open); + + void CloseRandomAccessLocked(); + + // this is used, when you are reading only the footer of a + // previously closed file + Status SetFromFooterLocked(const BlobLogFooter& footer); + + void set_time_range(const tsrange_t& tr) { time_range_ = tr; } + + void set_ttl_range(const ttlrange_t& ttl) { ttl_range_ = ttl; } + + void SetSNRange(const snrange_t& snr) { sn_range_ = snr; } + + // The following functions are atomic, and don't need locks + void SetFileSize(uint64_t fs) { file_size_ = fs; } + + void SetBlobCount(uint64_t bc) { blob_count_ = bc; } + + void SetCanBeDeleted() { can_be_deleted_ = true; } +}; +} // namespace blob_db +} // namespace rocksdb +#endif // ROCKSDB_LITE