Summary: The final pull request for Blob Storage. Closes https://github.com/facebook/rocksdb/pull/2269 Differential Revision: D5033189 Pulled By: yiwu-arbug fbshipit-source-id: 6356b683ccd58cbf38a1dc55e2ea400feecd5d06main
Anirban Rahut
8 years ago
committed by
Facebook Github Bot
parent
492fc49a86
commit
d85ff4953c
@ -0,0 +1,158 @@ |
|||||||
|
// Portions Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
// Large parts of this file is borrowed from the public domain code below.
|
||||||
|
// from https://github.com/mstump/queues
|
||||||
|
|
||||||
|
// C++ implementation of Dmitry Vyukov's non-intrusive
|
||||||
|
// lock free unbound MPSC queue
|
||||||
|
// http://www.1024cores.net/home/
|
||||||
|
// lock-free-algorithms/queues/non-intrusive-mpsc-node-based-queue
|
||||||
|
|
||||||
|
// License from mstump/queues
|
||||||
|
// This is free and unencumbered software released into the public domain.
|
||||||
|
//
|
||||||
|
// Anyone is free to copy, modify, publish, use, compile, sell, or
|
||||||
|
// distribute this software, either in source code form or as a compiled
|
||||||
|
// binary, for any purpose, commercial or non-commercial, and by any
|
||||||
|
// means.
|
||||||
|
//
|
||||||
|
// In jurisdictions that recognize copyright laws, the author or authors
|
||||||
|
// of this software dedicate any and all copyright interest in the
|
||||||
|
// software to the public domain. We make this dedication for the benefit
|
||||||
|
// of the public at large and to the detriment of our heirs and
|
||||||
|
// successors. We intend this dedication to be an overt act of
|
||||||
|
// relinquishment in perpetuity of all present and future rights to this
|
||||||
|
// software under copyright law.
|
||||||
|
//
|
||||||
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||||
|
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||||
|
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||||
|
// IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||||
|
// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||||
|
// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||||
|
// OTHER DEALINGS IN THE SOFTWARE.
|
||||||
|
//
|
||||||
|
// For more information, please refer to <http://unlicense.org>
|
||||||
|
|
||||||
|
// License from http://www.1024cores.net/home/
|
||||||
|
// lock-free-algorithms/queues/non-intrusive-mpsc-node-based-queue
|
||||||
|
// Copyright (c) 2010-2011 Dmitry Vyukov. All rights reserved.
|
||||||
|
// Redistribution and use in source and binary forms, with or
|
||||||
|
// without modification, are permitted provided that the following
|
||||||
|
// conditions are met:
|
||||||
|
// 1. Redistributions of source code must retain the above copyright notice,
|
||||||
|
// this list of conditions and the following disclaimer.
|
||||||
|
//
|
||||||
|
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||||
|
// this list of conditions and the following disclaimer in the documentation
|
||||||
|
// and/or other materials provided with the distribution.
|
||||||
|
//
|
||||||
|
// THIS SOFTWARE IS PROVIDED BY DMITRY VYUKOV "AS IS" AND ANY EXPRESS OR
|
||||||
|
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
||||||
|
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
|
||||||
|
// EVENT SHALL DMITRY VYUKOV OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||||
|
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||||
|
// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
||||||
|
// USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||||
|
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||||
|
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||||
|
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||||
|
//
|
||||||
|
// The views and conclusions contained in the software and documentation
|
||||||
|
// are those of the authors and should not be interpreted as representing
|
||||||
|
// official policies, either expressed or implied, of Dmitry Vyukov.
|
||||||
|
//
|
||||||
|
|
||||||
|
#ifndef UTIL_MPSC_H_ |
||||||
|
#define UTIL_MPSC_H_ |
||||||
|
|
||||||
|
#include <atomic> |
||||||
|
#include <cassert> |
||||||
|
#include <type_traits> |
||||||
|
|
||||||
|
/**
|
||||||
|
* Multiple Producer Single Consumer Lockless Q |
||||||
|
*/ |
||||||
|
template <typename T> |
||||||
|
class mpsc_queue_t { |
||||||
|
public: |
||||||
|
struct buffer_node_t { |
||||||
|
T data; |
||||||
|
std::atomic<buffer_node_t*> next; |
||||||
|
}; |
||||||
|
|
||||||
|
mpsc_queue_t() { |
||||||
|
buffer_node_aligned_t* al_st = new buffer_node_aligned_t; |
||||||
|
buffer_node_t* node = new (al_st) buffer_node_t(); |
||||||
|
_head.store(node); |
||||||
|
_tail.store(node); |
||||||
|
|
||||||
|
node->next.store(nullptr, std::memory_order_relaxed); |
||||||
|
} |
||||||
|
|
||||||
|
~mpsc_queue_t() { |
||||||
|
T output; |
||||||
|
while (this->dequeue(&output)) { |
||||||
|
} |
||||||
|
buffer_node_t* front = _head.load(std::memory_order_relaxed); |
||||||
|
front->~buffer_node_t(); |
||||||
|
|
||||||
|
::operator delete(front); |
||||||
|
} |
||||||
|
|
||||||
|
void enqueue(const T& input) { |
||||||
|
buffer_node_aligned_t* al_st = new buffer_node_aligned_t; |
||||||
|
buffer_node_t* node = new (al_st) buffer_node_t(); |
||||||
|
|
||||||
|
node->data = input; |
||||||
|
node->next.store(nullptr, std::memory_order_relaxed); |
||||||
|
|
||||||
|
buffer_node_t* prev_head = _head.exchange(node, std::memory_order_acq_rel); |
||||||
|
prev_head->next.store(node, std::memory_order_release); |
||||||
|
} |
||||||
|
|
||||||
|
bool dequeue(T* output) { |
||||||
|
buffer_node_t* tail = _tail.load(std::memory_order_relaxed); |
||||||
|
buffer_node_t* next = tail->next.load(std::memory_order_acquire); |
||||||
|
|
||||||
|
if (next == nullptr) { |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
*output = next->data; |
||||||
|
_tail.store(next, std::memory_order_release); |
||||||
|
|
||||||
|
tail->~buffer_node_t(); |
||||||
|
|
||||||
|
::operator delete(tail); |
||||||
|
return true; |
||||||
|
} |
||||||
|
|
||||||
|
// you can only use pop_all if the queue is SPSC
|
||||||
|
buffer_node_t* pop_all() { |
||||||
|
// nobody else can move the tail pointer.
|
||||||
|
buffer_node_t* tptr = _tail.load(std::memory_order_relaxed); |
||||||
|
buffer_node_t* next = |
||||||
|
tptr->next.exchange(nullptr, std::memory_order_acquire); |
||||||
|
_head.exchange(tptr, std::memory_order_acquire); |
||||||
|
|
||||||
|
// there is a race condition here
|
||||||
|
return next; |
||||||
|
} |
||||||
|
|
||||||
|
private: |
||||||
|
typedef typename std::aligned_storage< |
||||||
|
sizeof(buffer_node_t), std::alignment_of<buffer_node_t>::value>::type |
||||||
|
buffer_node_aligned_t; |
||||||
|
|
||||||
|
std::atomic<buffer_node_t*> _head; |
||||||
|
std::atomic<buffer_node_t*> _tail; |
||||||
|
|
||||||
|
mpsc_queue_t(const mpsc_queue_t&) = delete; |
||||||
|
mpsc_queue_t& operator=(const mpsc_queue_t&) = delete; |
||||||
|
}; |
||||||
|
|
||||||
|
#endif // UTIL_MPSC_H_
|
||||||
@ -0,0 +1,217 @@ |
|||||||
|
// Portions Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
// Borrowed from
|
||||||
|
// http://www.crazygaze.com/blog/2016/03/24/portable-c-timer-queue/
|
||||||
|
// Timer Queue
|
||||||
|
//
|
||||||
|
// License
|
||||||
|
//
|
||||||
|
// The source code in this article is licensed under the CC0 license, so feel
|
||||||
|
// free to copy, modify, share, do whatever you want with it.
|
||||||
|
// No attribution is required, but Ill be happy if you do.
|
||||||
|
// CC0 license
|
||||||
|
|
||||||
|
// The person who associated a work with this deed has dedicated the work to the
|
||||||
|
// public domain by waiving all of his or her rights to the work worldwide
|
||||||
|
// under copyright law, including all related and neighboring rights, to the
|
||||||
|
// extent allowed by law. You can copy, modify, distribute and perform the
|
||||||
|
// work, even for commercial purposes, all without asking permission.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
#include <assert.h> |
||||||
|
#include <chrono> |
||||||
|
#include <condition_variable> |
||||||
|
#include <functional> |
||||||
|
#include <queue> |
||||||
|
#include <thread> |
||||||
|
#include <utility> |
||||||
|
#include <vector> |
||||||
|
|
||||||
|
// Allows execution of handlers at a specified time in the future
|
||||||
|
// Guarantees:
|
||||||
|
// - All handlers are executed ONCE, even if cancelled (aborted parameter will
|
||||||
|
// be set to true)
|
||||||
|
// - If TimerQueue is destroyed, it will cancel all handlers.
|
||||||
|
// - Handlers are ALWAYS executed in the Timer Queue worker thread.
|
||||||
|
// - Handlers execution order is NOT guaranteed
|
||||||
|
//
|
||||||
|
////////////////////////////////////////////////////////////////////////////////
|
||||||
|
// borrowed from
|
||||||
|
// http://www.crazygaze.com/blog/2016/03/24/portable-c-timer-queue/
|
||||||
|
class TimerQueue { |
||||||
|
public: |
||||||
|
TimerQueue() : m_th(&TimerQueue::run, this) {} |
||||||
|
|
||||||
|
~TimerQueue() { |
||||||
|
cancelAll(); |
||||||
|
// Abusing the timer queue to trigger the shutdown.
|
||||||
|
add(0, [this](bool) { |
||||||
|
m_finish = true; |
||||||
|
return std::make_pair(false, 0); |
||||||
|
}); |
||||||
|
m_th.join(); |
||||||
|
} |
||||||
|
|
||||||
|
// Adds a new timer
|
||||||
|
// \return
|
||||||
|
// Returns the ID of the new timer. You can use this ID to cancel the
|
||||||
|
// timer
|
||||||
|
uint64_t add(int64_t milliseconds, |
||||||
|
std::function<std::pair<bool, int64_t>(bool)> handler) { |
||||||
|
WorkItem item; |
||||||
|
Clock::time_point tp = Clock::now(); |
||||||
|
item.end = tp + std::chrono::milliseconds(milliseconds); |
||||||
|
item.period = milliseconds; |
||||||
|
item.handler = std::move(handler); |
||||||
|
|
||||||
|
std::unique_lock<std::mutex> lk(m_mtx); |
||||||
|
uint64_t id = ++m_idcounter; |
||||||
|
item.id = id; |
||||||
|
m_items.push(std::move(item)); |
||||||
|
|
||||||
|
// Something changed, so wake up timer thread
|
||||||
|
m_checkWork.notify_one(); |
||||||
|
return id; |
||||||
|
} |
||||||
|
|
||||||
|
// Cancels the specified timer
|
||||||
|
// \return
|
||||||
|
// 1 if the timer was cancelled.
|
||||||
|
// 0 if you were too late to cancel (or the timer ID was never valid to
|
||||||
|
// start with)
|
||||||
|
size_t cancel(uint64_t id) { |
||||||
|
// Instead of removing the item from the container (thus breaking the
|
||||||
|
// heap integrity), we set the item as having no handler, and put
|
||||||
|
// that handler on a new item at the top for immediate execution
|
||||||
|
// The timer thread will then ignore the original item, since it has no
|
||||||
|
// handler.
|
||||||
|
std::unique_lock<std::mutex> lk(m_mtx); |
||||||
|
for (auto&& item : m_items.getContainer()) { |
||||||
|
if (item.id == id && item.handler) { |
||||||
|
WorkItem newItem; |
||||||
|
// Zero time, so it stays at the top for immediate execution
|
||||||
|
newItem.end = Clock::time_point(); |
||||||
|
newItem.id = 0; // Means it is a canceled item
|
||||||
|
// Move the handler from item to newitem (thus clearing item)
|
||||||
|
newItem.handler = std::move(item.handler); |
||||||
|
m_items.push(std::move(newItem)); |
||||||
|
|
||||||
|
// Something changed, so wake up timer thread
|
||||||
|
m_checkWork.notify_one(); |
||||||
|
return 1; |
||||||
|
} |
||||||
|
} |
||||||
|
return 0; |
||||||
|
} |
||||||
|
|
||||||
|
// Cancels all timers
|
||||||
|
// \return
|
||||||
|
// The number of timers cancelled
|
||||||
|
size_t cancelAll() { |
||||||
|
// Setting all "end" to 0 (for immediate execution) is ok,
|
||||||
|
// since it maintains the heap integrity
|
||||||
|
std::unique_lock<std::mutex> lk(m_mtx); |
||||||
|
m_cancel = true; |
||||||
|
for (auto&& item : m_items.getContainer()) { |
||||||
|
if (item.id && item.handler) { |
||||||
|
item.end = Clock::time_point(); |
||||||
|
item.id = 0; |
||||||
|
} |
||||||
|
} |
||||||
|
auto ret = m_items.size(); |
||||||
|
|
||||||
|
m_checkWork.notify_one(); |
||||||
|
return ret; |
||||||
|
} |
||||||
|
|
||||||
|
private: |
||||||
|
using Clock = std::chrono::steady_clock; |
||||||
|
TimerQueue(const TimerQueue&) = delete; |
||||||
|
TimerQueue& operator=(const TimerQueue&) = delete; |
||||||
|
|
||||||
|
void run() { |
||||||
|
std::unique_lock<std::mutex> lk(m_mtx); |
||||||
|
while (!m_finish) { |
||||||
|
auto end = calcWaitTime_lock(); |
||||||
|
if (end.first) { |
||||||
|
// Timers found, so wait until it expires (or something else
|
||||||
|
// changes)
|
||||||
|
m_checkWork.wait_until(lk, end.second); |
||||||
|
} else { |
||||||
|
// No timers exist, so wait forever until something changes
|
||||||
|
m_checkWork.wait(lk); |
||||||
|
} |
||||||
|
|
||||||
|
// Check and execute as much work as possible, such as, all expired
|
||||||
|
// timers
|
||||||
|
checkWork(&lk); |
||||||
|
} |
||||||
|
|
||||||
|
// If we are shutting down, we should not have any items left,
|
||||||
|
// since the shutdown cancels all items
|
||||||
|
assert(m_items.size() == 0); |
||||||
|
} |
||||||
|
|
||||||
|
std::pair<bool, Clock::time_point> calcWaitTime_lock() { |
||||||
|
while (m_items.size()) { |
||||||
|
if (m_items.top().handler) { |
||||||
|
// Item present, so return the new wait time
|
||||||
|
return std::make_pair(true, m_items.top().end); |
||||||
|
} else { |
||||||
|
// Discard empty handlers (they were cancelled)
|
||||||
|
m_items.pop(); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
// No items found, so return no wait time (causes the thread to wait
|
||||||
|
// indefinitely)
|
||||||
|
return std::make_pair(false, Clock::time_point()); |
||||||
|
} |
||||||
|
|
||||||
|
void checkWork(std::unique_lock<std::mutex>* lk) { |
||||||
|
while (m_items.size() && m_items.top().end <= Clock::now()) { |
||||||
|
WorkItem item(m_items.top()); |
||||||
|
m_items.pop(); |
||||||
|
|
||||||
|
if (item.handler) { |
||||||
|
(*lk).unlock(); |
||||||
|
auto reschedule_pair = item.handler(item.id == 0); |
||||||
|
(*lk).lock(); |
||||||
|
if (!m_cancel && reschedule_pair.first) { |
||||||
|
int64_t new_period = (reschedule_pair.second == -1) |
||||||
|
? item.period |
||||||
|
: reschedule_pair.second; |
||||||
|
|
||||||
|
item.period = new_period; |
||||||
|
item.end = Clock::now() + std::chrono::milliseconds(new_period); |
||||||
|
m_items.push(std::move(item)); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
bool m_finish = false; |
||||||
|
bool m_cancel = false; |
||||||
|
uint64_t m_idcounter = 0; |
||||||
|
std::condition_variable m_checkWork; |
||||||
|
|
||||||
|
struct WorkItem { |
||||||
|
Clock::time_point end; |
||||||
|
int64_t period; |
||||||
|
uint64_t id; // id==0 means it was cancelled
|
||||||
|
std::function<std::pair<bool, int64_t>(bool)> handler; |
||||||
|
bool operator>(const WorkItem& other) const { return end > other.end; } |
||||||
|
}; |
||||||
|
|
||||||
|
std::mutex m_mtx; |
||||||
|
// Inheriting from priority_queue, so we can access the internal container
|
||||||
|
class Queue : public std::priority_queue<WorkItem, std::vector<WorkItem>, |
||||||
|
std::greater<WorkItem>> { |
||||||
|
public: |
||||||
|
std::vector<WorkItem>& getContainer() { return this->c; } |
||||||
|
} m_items; |
||||||
|
std::thread m_th; |
||||||
|
}; |
||||||
@ -0,0 +1,72 @@ |
|||||||
|
// Portions Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
// borrowed from
|
||||||
|
// http://www.crazygaze.com/blog/2016/03/24/portable-c-timer-queue/
|
||||||
|
// Timer Queue
|
||||||
|
//
|
||||||
|
// License
|
||||||
|
//
|
||||||
|
// The source code in this article is licensed under the CC0 license, so feel
|
||||||
|
// free
|
||||||
|
// to copy, modify, share, do whatever you want with it.
|
||||||
|
// No attribution is required, but Ill be happy if you do.
|
||||||
|
// CC0 license
|
||||||
|
|
||||||
|
// The person who associated a work with this deed has dedicated the work to the
|
||||||
|
// public domain by waiving all of his or her rights to the work worldwide
|
||||||
|
// under copyright law, including all related and neighboring rights, to the
|
||||||
|
// extent allowed by law. You can copy, modify, distribute and perform the
|
||||||
|
// work, even for
|
||||||
|
// commercial purposes, all without asking permission. See Other Information
|
||||||
|
// below.
|
||||||
|
//
|
||||||
|
|
||||||
|
#include "util/timer_queue.h" |
||||||
|
#include <future> |
||||||
|
|
||||||
|
namespace Timing { |
||||||
|
|
||||||
|
using Clock = std::chrono::high_resolution_clock; |
||||||
|
double now() { |
||||||
|
static auto start = Clock::now(); |
||||||
|
return std::chrono::duration<double, std::milli>(Clock::now() - start) |
||||||
|
.count(); |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace Timing
|
||||||
|
|
||||||
|
int main() { |
||||||
|
TimerQueue q; |
||||||
|
|
||||||
|
double tnow = Timing::now(); |
||||||
|
|
||||||
|
q.add(10000, [tnow](bool aborted) mutable { |
||||||
|
printf("T 1: %d, Elapsed %4.2fms\n", aborted, Timing::now() - tnow); |
||||||
|
return std::make_pair(false, 0); |
||||||
|
}); |
||||||
|
q.add(10001, [tnow](bool aborted) mutable { |
||||||
|
printf("T 2: %d, Elapsed %4.2fms\n", aborted, Timing::now() - tnow); |
||||||
|
return std::make_pair(false, 0); |
||||||
|
}); |
||||||
|
|
||||||
|
q.add(1000, [tnow](bool aborted) mutable { |
||||||
|
printf("T 3: %d, Elapsed %4.2fms\n", aborted, Timing::now() - tnow); |
||||||
|
return std::make_pair(!aborted, 1000); |
||||||
|
}); |
||||||
|
|
||||||
|
auto id = q.add(2000, [tnow](bool aborted) mutable { |
||||||
|
printf("T 4: %d, Elapsed %4.2fms\n", aborted, Timing::now() - tnow); |
||||||
|
return std::make_pair(!aborted, 2000); |
||||||
|
}); |
||||||
|
|
||||||
|
(void)id; |
||||||
|
// auto ret = q.cancel(id);
|
||||||
|
// assert(ret == 1);
|
||||||
|
// q.cancelAll();
|
||||||
|
|
||||||
|
return 0; |
||||||
|
} |
||||||
|
//////////////////////////////////////////
|
||||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,657 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
|
||||||
|
#include <atomic> |
||||||
|
#include <condition_variable> |
||||||
|
#include <ctime> |
||||||
|
#include <list> |
||||||
|
#include <memory> |
||||||
|
#include <set> |
||||||
|
#include <string> |
||||||
|
#include <thread> |
||||||
|
#include <utility> |
||||||
|
#include <vector> |
||||||
|
|
||||||
|
#include "rocksdb/compaction_filter.h" |
||||||
|
#include "rocksdb/db.h" |
||||||
|
#include "rocksdb/listener.h" |
||||||
|
#include "rocksdb/options.h" |
||||||
|
#include "rocksdb/wal_filter.h" |
||||||
|
#include "util/file_reader_writer.h" |
||||||
|
#include "util/mpsc.h" |
||||||
|
#include "util/mutexlock.h" |
||||||
|
#include "util/timer_queue.h" |
||||||
|
#include "utilities/blob_db/blob_db.h" |
||||||
|
#include "utilities/blob_db/blob_db_options_impl.h" |
||||||
|
#include "utilities/blob_db/blob_log_format.h" |
||||||
|
#include "utilities/blob_db/blob_log_reader.h" |
||||||
|
#include "utilities/blob_db/blob_log_writer.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class DBImpl; |
||||||
|
class ColumnFamilyHandle; |
||||||
|
class ColumnFamilyData; |
||||||
|
class OptimisticTransactionDBImpl; |
||||||
|
struct FlushJobInfo; |
||||||
|
|
||||||
|
namespace blob_db { |
||||||
|
|
||||||
|
class BlobFile; |
||||||
|
class BlobDBImpl; |
||||||
|
struct GCStats; |
||||||
|
|
||||||
|
class BlobDBFlushBeginListener : public EventListener { |
||||||
|
public: |
||||||
|
explicit BlobDBFlushBeginListener() : impl_(nullptr) {} |
||||||
|
|
||||||
|
void OnFlushBegin(DB* db, const FlushJobInfo& info) override; |
||||||
|
|
||||||
|
void SetImplPtr(BlobDBImpl* p) { impl_ = p; } |
||||||
|
|
||||||
|
protected: |
||||||
|
BlobDBImpl* impl_; |
||||||
|
}; |
||||||
|
|
||||||
|
// this implements the callback from the WAL which ensures that the
|
||||||
|
// blob record is present in the blob log. If fsync/fdatasync in not
|
||||||
|
// happening on every write, there is the probability that keys in the
|
||||||
|
// blob log can lag the keys in blobs
|
||||||
|
class BlobReconcileWalFilter : public WalFilter { |
||||||
|
public: |
||||||
|
virtual WalFilter::WalProcessingOption LogRecordFound( |
||||||
|
unsigned long long log_number, const std::string& log_file_name, |
||||||
|
const WriteBatch& batch, WriteBatch* new_batch, |
||||||
|
bool* batch_changed) override; |
||||||
|
|
||||||
|
virtual const char* Name() const override { return "BlobDBWalReconciler"; } |
||||||
|
|
||||||
|
void SetImplPtr(BlobDBImpl* p) { impl_ = p; } |
||||||
|
|
||||||
|
protected: |
||||||
|
BlobDBImpl* impl_; |
||||||
|
}; |
||||||
|
|
||||||
|
class EvictAllVersionsCompactionListener : public EventListener { |
||||||
|
public: |
||||||
|
class InternalListener : public CompactionEventListener { |
||||||
|
friend class BlobDBImpl; |
||||||
|
|
||||||
|
public: |
||||||
|
virtual void OnCompaction(int level, const Slice& key, |
||||||
|
CompactionListenerValueType value_type, |
||||||
|
const Slice& existing_value, |
||||||
|
const SequenceNumber& sn, bool is_new) override; |
||||||
|
|
||||||
|
void SetImplPtr(BlobDBImpl* p) { impl_ = p; } |
||||||
|
|
||||||
|
private: |
||||||
|
BlobDBImpl* impl_; |
||||||
|
}; |
||||||
|
|
||||||
|
explicit EvictAllVersionsCompactionListener() |
||||||
|
: internal_listener_(new InternalListener()) {} |
||||||
|
|
||||||
|
virtual CompactionEventListener* GetCompactionEventListener() override { |
||||||
|
return internal_listener_.get(); |
||||||
|
} |
||||||
|
|
||||||
|
void SetImplPtr(BlobDBImpl* p) { internal_listener_->SetImplPtr(p); } |
||||||
|
|
||||||
|
private: |
||||||
|
std::unique_ptr<InternalListener> internal_listener_; |
||||||
|
}; |
||||||
|
|
||||||
|
#if 0 |
||||||
|
class EvictAllVersionsFilterFactory : public CompactionFilterFactory { |
||||||
|
private: |
||||||
|
BlobDBImpl* impl_; |
||||||
|
|
||||||
|
public: |
||||||
|
EvictAllVersionsFilterFactory() : impl_(nullptr) {} |
||||||
|
|
||||||
|
void SetImplPtr(BlobDBImpl* p) { impl_ = p; } |
||||||
|
|
||||||
|
virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter( |
||||||
|
const CompactionFilter::Context& context) override; |
||||||
|
|
||||||
|
virtual const char* Name() const override { |
||||||
|
return "EvictAllVersionsFilterFactory"; |
||||||
|
} |
||||||
|
}; |
||||||
|
#endif |
||||||
|
|
||||||
|
// Comparator to sort "TTL" aware Blob files based on the lower value of
|
||||||
|
// TTL range.
|
||||||
|
struct blobf_compare_ttl { |
||||||
|
bool operator()(const std::shared_ptr<BlobFile>& lhs, |
||||||
|
const std::shared_ptr<BlobFile>& rhs) const; |
||||||
|
}; |
||||||
|
|
||||||
|
/**
|
||||||
|
* The implementation class for BlobDB. This manages the value |
||||||
|
* part in TTL aware sequentially written files. These files are |
||||||
|
* Garbage Collected. |
||||||
|
*/ |
||||||
|
class BlobDBImpl : public BlobDB { |
||||||
|
friend class BlobDBFlushBeginListener; |
||||||
|
friend class EvictAllVersionsCompactionListener; |
||||||
|
friend class BlobDB; |
||||||
|
friend class BlobFile; |
||||||
|
friend class BlobDBIterator; |
||||||
|
|
||||||
|
public: |
||||||
|
using rocksdb::StackableDB::Put; |
||||||
|
Status Put(const WriteOptions& options, ColumnFamilyHandle* column_family, |
||||||
|
const Slice& key, const Slice& value) override; |
||||||
|
|
||||||
|
using rocksdb::StackableDB::Delete; |
||||||
|
Status Delete(const WriteOptions& options, ColumnFamilyHandle* column_family, |
||||||
|
const Slice& key) override; |
||||||
|
|
||||||
|
using rocksdb::StackableDB::SingleDelete; |
||||||
|
virtual Status SingleDelete(const WriteOptions& wopts, |
||||||
|
ColumnFamilyHandle* column_family, |
||||||
|
const Slice& key) override; |
||||||
|
|
||||||
|
using rocksdb::StackableDB::Get; |
||||||
|
Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family, |
||||||
|
const Slice& key, std::string* value) override; |
||||||
|
|
||||||
|
using rocksdb::StackableDB::NewIterator; |
||||||
|
virtual Iterator* NewIterator(const ReadOptions& opts, |
||||||
|
ColumnFamilyHandle* column_family) override; |
||||||
|
|
||||||
|
using rocksdb::StackableDB::MultiGet; |
||||||
|
virtual std::vector<Status> MultiGet( |
||||||
|
const ReadOptions& options, |
||||||
|
const std::vector<ColumnFamilyHandle*>& column_family, |
||||||
|
const std::vector<Slice>& keys, |
||||||
|
std::vector<std::string>* values) override; |
||||||
|
|
||||||
|
virtual Status Write(const WriteOptions& opts, WriteBatch* updates) override; |
||||||
|
|
||||||
|
using BlobDB::PutWithTTL; |
||||||
|
Status PutWithTTL(const WriteOptions& options, |
||||||
|
ColumnFamilyHandle* column_family, const Slice& key, |
||||||
|
const Slice& value, int32_t ttl) override; |
||||||
|
|
||||||
|
using BlobDB::PutUntil; |
||||||
|
Status PutUntil(const WriteOptions& options, |
||||||
|
ColumnFamilyHandle* column_family, const Slice& key, |
||||||
|
const Slice& value_unc, int32_t expiration) override; |
||||||
|
|
||||||
|
Status LinkToBaseDB(DB* db) override; |
||||||
|
|
||||||
|
BlobDBImpl(DB* db, const BlobDBOptions& bdb_options); |
||||||
|
|
||||||
|
BlobDBImpl(const std::string& dbname, const BlobDBOptions& bdb_options, |
||||||
|
const DBOptions& db_options); |
||||||
|
|
||||||
|
~BlobDBImpl(); |
||||||
|
|
||||||
|
private: |
||||||
|
static bool ExtractTTLFromBlob(const Slice& value, Slice* newval, |
||||||
|
int32_t* ttl_val); |
||||||
|
|
||||||
|
Status OpenPhase1(); |
||||||
|
|
||||||
|
Status CommonGet(const ColumnFamilyData* cfd, const Slice& key, |
||||||
|
const std::string& index_entry, std::string* value); |
||||||
|
|
||||||
|
// Just before flush starts acting on memtable files,
|
||||||
|
// this handler is called.
|
||||||
|
void OnFlushBeginHandler(DB* db, const FlushJobInfo& info); |
||||||
|
|
||||||
|
// timer queue callback to close a file by appending a footer
|
||||||
|
// removes file from open files list
|
||||||
|
std::pair<bool, int64_t> CloseSeqWrite(std::shared_ptr<BlobFile> bfile, |
||||||
|
bool aborted); |
||||||
|
|
||||||
|
// is this file ready for Garbage collection. if the TTL of the file
|
||||||
|
// has expired or if threshold of the file has been evicted
|
||||||
|
// tt - current time
|
||||||
|
// last_id - the id of the non-TTL file to evict
|
||||||
|
bool ShouldGCFile(std::shared_ptr<BlobFile> bfile, std::time_t tt, |
||||||
|
uint64_t last_id, std::string* reason); |
||||||
|
|
||||||
|
// collect all the blob log files from the blob directory
|
||||||
|
Status GetAllLogFiles(std::set<std::pair<uint64_t, std::string>>* file_nums); |
||||||
|
|
||||||
|
// appends a task into timer queue to close the file
|
||||||
|
void CloseIf(const std::shared_ptr<BlobFile>& bfile); |
||||||
|
|
||||||
|
Status AppendBlob(const std::shared_ptr<BlobFile>& bfile, |
||||||
|
const std::string& headerbuf, const Slice& key, |
||||||
|
const Slice& value, std::string* index_entry); |
||||||
|
|
||||||
|
Status AppendSN(const std::shared_ptr<BlobFile>& bfile, |
||||||
|
const SequenceNumber& sn); |
||||||
|
|
||||||
|
// find an existing blob log file based on the expiration unix epoch
|
||||||
|
// if such a file does not exist, return nullptr
|
||||||
|
std::shared_ptr<BlobFile> SelectBlobFileTTL(uint32_t expiration); |
||||||
|
|
||||||
|
// find an existing blob log file to append the value to
|
||||||
|
std::shared_ptr<BlobFile> SelectBlobFile(); |
||||||
|
|
||||||
|
std::shared_ptr<BlobFile> FindBlobFileLocked(uint32_t expiration) const; |
||||||
|
|
||||||
|
void UpdateWriteOptions(const WriteOptions& options); |
||||||
|
|
||||||
|
void Shutdown(); |
||||||
|
|
||||||
|
// periodic sanity check. Bunch of checks
|
||||||
|
std::pair<bool, int64_t> SanityCheck(bool aborted); |
||||||
|
|
||||||
|
// delete files which have been garbage collected and marked
|
||||||
|
// obsolete. Check whether any snapshots exist which refer to
|
||||||
|
// the same
|
||||||
|
std::pair<bool, int64_t> DeleteObsFiles(bool aborted); |
||||||
|
|
||||||
|
// Major task to garbage collect expired and deleted blobs
|
||||||
|
std::pair<bool, int64_t> RunGC(bool aborted); |
||||||
|
|
||||||
|
// asynchronous task to fsync/fdatasync the open blob files
|
||||||
|
std::pair<bool, int64_t> FsyncFiles(bool aborted); |
||||||
|
|
||||||
|
// periodically check if open blob files and their TTL's has expired
|
||||||
|
// if expired, close the sequential writer and make the file immutable
|
||||||
|
std::pair<bool, int64_t> CheckSeqFiles(bool aborted); |
||||||
|
|
||||||
|
// if the number of open files, approaches ULIMIT's this
|
||||||
|
// task will close random readers, which are kept around for
|
||||||
|
// efficiency
|
||||||
|
std::pair<bool, int64_t> ReclaimOpenFiles(bool aborted); |
||||||
|
|
||||||
|
// periodically print write amplification statistics
|
||||||
|
std::pair<bool, int64_t> WaStats(bool aborted); |
||||||
|
|
||||||
|
// background task to do book-keeping of deleted keys
|
||||||
|
std::pair<bool, int64_t> EvictDeletions(bool aborted); |
||||||
|
|
||||||
|
std::pair<bool, int64_t> EvictCompacted(bool aborted); |
||||||
|
|
||||||
|
bool CallbackEvictsImpl(std::shared_ptr<BlobFile> bfile); |
||||||
|
|
||||||
|
std::pair<bool, int64_t> RemoveTimerQ(TimerQueue* tq, bool aborted); |
||||||
|
|
||||||
|
std::pair<bool, int64_t> CallbackEvicts(TimerQueue* tq, |
||||||
|
std::shared_ptr<BlobFile> bfile, |
||||||
|
bool aborted); |
||||||
|
|
||||||
|
// Adds the background tasks to the timer queue
|
||||||
|
void StartBackgroundTasks(); |
||||||
|
|
||||||
|
// add a new Blob File
|
||||||
|
std::shared_ptr<BlobFile> NewBlobFile(const std::string& reason); |
||||||
|
|
||||||
|
Status OpenAllFiles(); |
||||||
|
|
||||||
|
// hold write mutex on file and call
|
||||||
|
// creates a Random Access reader for GET call
|
||||||
|
std::shared_ptr<RandomAccessFileReader> GetOrOpenRandomAccessReader( |
||||||
|
const std::shared_ptr<BlobFile>& bfile, Env* env, |
||||||
|
const EnvOptions& env_options); |
||||||
|
|
||||||
|
// hold write mutex on file and call.
|
||||||
|
// Close the above Random Access reader
|
||||||
|
void CloseRandomAccessLocked(const std::shared_ptr<BlobFile>& bfile); |
||||||
|
|
||||||
|
// hold write mutex on file and call
|
||||||
|
// creates a sequential (append) writer for this blobfile
|
||||||
|
Status CreateWriterLocked(const std::shared_ptr<BlobFile>& bfile); |
||||||
|
|
||||||
|
// returns a Writer object for the file. If writer is not
|
||||||
|
// already present, creates one. Needs Write Mutex to be held
|
||||||
|
std::shared_ptr<Writer> CheckOrCreateWriterLocked( |
||||||
|
const std::shared_ptr<BlobFile>& bfile); |
||||||
|
|
||||||
|
// Iterate through keys and values on Blob and write into
|
||||||
|
// separate file the remaining blobs and delete/update pointers
|
||||||
|
// in LSM atomically
|
||||||
|
Status GCFileAndUpdateLSM(const std::shared_ptr<BlobFile>& bfptr, |
||||||
|
GCStats* gcstats); |
||||||
|
|
||||||
|
// checks if there is no snapshot which is referencing the
|
||||||
|
// blobs
|
||||||
|
bool FileDeleteOk_SnapshotCheckLocked(const std::shared_ptr<BlobFile>& bfile); |
||||||
|
|
||||||
|
bool MarkBlobDeleted(const Slice& key, const Slice& lsmValue); |
||||||
|
|
||||||
|
bool FindFileAndEvictABlob(uint64_t file_number, uint64_t key_size, |
||||||
|
uint64_t blob_offset, uint64_t blob_size); |
||||||
|
|
||||||
|
void CopyBlobFiles(std::vector<std::shared_ptr<BlobFile>>* bfiles_copy, |
||||||
|
uint64_t* last_id); |
||||||
|
|
||||||
|
void FilterSubsetOfFiles( |
||||||
|
const std::vector<std::shared_ptr<BlobFile>>& blob_files, |
||||||
|
std::vector<std::shared_ptr<BlobFile>>* to_process, uint64_t epoch, |
||||||
|
uint64_t last_id, size_t files_to_collect); |
||||||
|
|
||||||
|
private: |
||||||
|
// the base DB
|
||||||
|
DBImpl* db_impl_; |
||||||
|
|
||||||
|
Env* myenv_; |
||||||
|
|
||||||
|
// Optimistic Transaction DB used during Garbage collection
|
||||||
|
// for atomicity
|
||||||
|
std::unique_ptr<OptimisticTransactionDBImpl> opt_db_; |
||||||
|
|
||||||
|
// a boolean to capture whether write_options has been set
|
||||||
|
std::atomic<bool> wo_set_; |
||||||
|
WriteOptions write_options_; |
||||||
|
|
||||||
|
// the options that govern the behavior of Blob Storage
|
||||||
|
BlobDBOptionsImpl bdb_options_; |
||||||
|
DBOptions db_options_; |
||||||
|
EnvOptions env_options_; |
||||||
|
|
||||||
|
// name of the database directory
|
||||||
|
std::string dbname_; |
||||||
|
|
||||||
|
// by default this is "blob_dir" under dbname_
|
||||||
|
// but can be configured
|
||||||
|
std::string blob_dir_; |
||||||
|
|
||||||
|
// pointer to directory
|
||||||
|
std::unique_ptr<Directory> dir_ent_; |
||||||
|
|
||||||
|
std::atomic<bool> dir_change_; |
||||||
|
|
||||||
|
// Read Write Mutex, which protects all the data structures
|
||||||
|
// HEAVILY TRAFFICKED
|
||||||
|
port::RWMutex mutex_; |
||||||
|
|
||||||
|
// counter for blob file number
|
||||||
|
std::atomic<uint64_t> next_file_number_; |
||||||
|
|
||||||
|
// entire metadata of all the BLOB files memory
|
||||||
|
std::unordered_map<uint64_t, std::shared_ptr<BlobFile>> blob_files_; |
||||||
|
|
||||||
|
// epoch or version of the open files.
|
||||||
|
std::atomic<uint64_t> epoch_of_; |
||||||
|
|
||||||
|
// typically we keep 4 open blob files (simple i.e. no TTL)
|
||||||
|
std::vector<std::shared_ptr<BlobFile>> open_simple_files_; |
||||||
|
|
||||||
|
// all the blob files which are currently being appended to based
|
||||||
|
// on variety of incoming TTL's
|
||||||
|
std::multiset<std::shared_ptr<BlobFile>, blobf_compare_ttl> open_blob_files_; |
||||||
|
|
||||||
|
// packet of information to put in lockess delete(s) queue
|
||||||
|
struct delete_packet_t { |
||||||
|
ColumnFamilyHandle* cfh_; |
||||||
|
std::string key_; |
||||||
|
SequenceNumber dsn_; |
||||||
|
}; |
||||||
|
|
||||||
|
struct override_packet_t { |
||||||
|
uint64_t file_number_; |
||||||
|
uint64_t key_size_; |
||||||
|
uint64_t blob_offset_; |
||||||
|
uint64_t blob_size_; |
||||||
|
SequenceNumber dsn_; |
||||||
|
}; |
||||||
|
|
||||||
|
// LOCKLESS multiple producer single consumer queue to quickly append
|
||||||
|
// deletes without taking lock. Can rapidly grow in size!!
|
||||||
|
// deletes happen in LSM, but minor book-keeping needs to happen on
|
||||||
|
// BLOB side (for triggering eviction)
|
||||||
|
mpsc_queue_t<delete_packet_t> delete_keys_q_; |
||||||
|
|
||||||
|
// LOCKLESS multiple producer single consumer queue for values
|
||||||
|
// that are being compacted
|
||||||
|
mpsc_queue_t<override_packet_t> override_vals_q_; |
||||||
|
|
||||||
|
// atomic bool to represent shutdown
|
||||||
|
std::atomic<bool> shutdown_; |
||||||
|
|
||||||
|
// timer based queue to execute tasks
|
||||||
|
TimerQueue tqueue_; |
||||||
|
|
||||||
|
// timer queues to call eviction callbacks.
|
||||||
|
std::vector<std::shared_ptr<TimerQueue>> cb_threads_; |
||||||
|
|
||||||
|
// only accessed in GC thread, hence not atomic. The epoch of the
|
||||||
|
// GC task. Each execution is one epoch. Helps us in allocating
|
||||||
|
// files to one execution
|
||||||
|
uint64_t current_epoch_; |
||||||
|
|
||||||
|
// number of files opened for random access/GET
|
||||||
|
// counter is used to monitor and close excess RA files.
|
||||||
|
std::atomic<uint32_t> open_file_count_; |
||||||
|
|
||||||
|
// should hold mutex to modify
|
||||||
|
// STATISTICS for WA of Blob Files due to GC
|
||||||
|
// collect by default 24 hourly periods
|
||||||
|
std::list<uint64_t> all_periods_write_; |
||||||
|
std::list<uint64_t> all_periods_ampl_; |
||||||
|
|
||||||
|
std::atomic<uint64_t> last_period_write_; |
||||||
|
std::atomic<uint64_t> last_period_ampl_; |
||||||
|
|
||||||
|
uint64_t total_periods_write_; |
||||||
|
uint64_t total_periods_ampl_; |
||||||
|
|
||||||
|
// total size of all blob files at a given time
|
||||||
|
std::atomic<uint64_t> total_blob_space_; |
||||||
|
std::list<std::shared_ptr<BlobFile>> obsolete_files_; |
||||||
|
bool open_p1_done_; |
||||||
|
|
||||||
|
uint32_t debug_level_; |
||||||
|
}; |
||||||
|
|
||||||
|
class BlobFile { |
||||||
|
friend class BlobDBImpl; |
||||||
|
friend struct blobf_compare_ttl; |
||||||
|
|
||||||
|
private: |
||||||
|
// access to parent
|
||||||
|
const BlobDBImpl* parent_; |
||||||
|
|
||||||
|
// path to blob directory
|
||||||
|
std::string path_to_dir_; |
||||||
|
|
||||||
|
// the id of the file.
|
||||||
|
// the above 2 are created during file creation and never changed
|
||||||
|
// after that
|
||||||
|
uint64_t file_number_; |
||||||
|
|
||||||
|
// number of blobs in the file
|
||||||
|
std::atomic<uint64_t> blob_count_; |
||||||
|
|
||||||
|
// the file will be selected for GC in this future epoch
|
||||||
|
std::atomic<int64_t> gc_epoch_; |
||||||
|
|
||||||
|
// size of the file
|
||||||
|
std::atomic<uint64_t> file_size_; |
||||||
|
|
||||||
|
// number of blobs in this particular file which have been evicted
|
||||||
|
uint64_t deleted_count_; |
||||||
|
|
||||||
|
// size of deleted blobs (used by heuristic to select file for GC)
|
||||||
|
uint64_t deleted_size_; |
||||||
|
|
||||||
|
BlobLogHeader header_; |
||||||
|
|
||||||
|
// closed_ = true implies the file is no more mutable
|
||||||
|
// no more blobs will be appended and the footer has been written out
|
||||||
|
std::atomic<bool> closed_; |
||||||
|
|
||||||
|
// has a pass of garbage collection successfully finished on this file
|
||||||
|
// can_be_deleted_ still needs to do iterator/snapshot checks
|
||||||
|
std::atomic<bool> can_be_deleted_; |
||||||
|
|
||||||
|
// should this file been gc'd once to reconcile lost deletes/compactions
|
||||||
|
std::atomic<bool> gc_once_after_open_; |
||||||
|
|
||||||
|
// et - lt of the blobs
|
||||||
|
ttlrange_t ttl_range_; |
||||||
|
|
||||||
|
// et - lt of the timestamp of the KV pairs.
|
||||||
|
tsrange_t time_range_; |
||||||
|
|
||||||
|
// ESN - LSN of the blobs
|
||||||
|
snrange_t sn_range_; |
||||||
|
|
||||||
|
// Sequential/Append writer for blobs
|
||||||
|
std::shared_ptr<Writer> log_writer_; |
||||||
|
|
||||||
|
// random access file reader for GET calls
|
||||||
|
std::shared_ptr<RandomAccessFileReader> ra_file_reader_; |
||||||
|
|
||||||
|
// This Read-Write mutex is per file specific and protects
|
||||||
|
// all the datastructures
|
||||||
|
port::RWMutex mutex_; |
||||||
|
|
||||||
|
// time when the random access reader was last created.
|
||||||
|
std::atomic<std::time_t> last_access_; |
||||||
|
|
||||||
|
// last time file was fsync'd/fdatasyncd
|
||||||
|
std::atomic<uint64_t> last_fsync_; |
||||||
|
|
||||||
|
bool header_valid_; |
||||||
|
|
||||||
|
public: |
||||||
|
BlobFile(); |
||||||
|
|
||||||
|
BlobFile(const BlobDBImpl* parent, const std::string& bdir, uint64_t fnum); |
||||||
|
|
||||||
|
~BlobFile(); |
||||||
|
|
||||||
|
ColumnFamilyHandle* GetColumnFamily(DB* db); |
||||||
|
|
||||||
|
// Returns log file's pathname relative to the main db dir
|
||||||
|
// Eg. For a live-log-file = blob_dir/000003.blob
|
||||||
|
std::string PathName() const; |
||||||
|
|
||||||
|
// Primary identifier for blob file.
|
||||||
|
// once the file is created, this never changes
|
||||||
|
uint64_t BlobFileNumber() const { return file_number_; } |
||||||
|
|
||||||
|
// the following functions are atomic, and don't need
|
||||||
|
// read lock
|
||||||
|
uint64_t BlobCount() const { |
||||||
|
return blob_count_.load(std::memory_order_acquire); |
||||||
|
} |
||||||
|
|
||||||
|
std::string DumpState() const; |
||||||
|
|
||||||
|
// if the file has gone through GC and blobs have been relocated
|
||||||
|
bool Obsolete() const { return can_be_deleted_.load(); } |
||||||
|
|
||||||
|
// if the file is not taking any more appends.
|
||||||
|
bool Immutable() const { return closed_.load(); } |
||||||
|
|
||||||
|
// we will assume this is atomic
|
||||||
|
bool NeedsFsync(bool hard, uint64_t bytes_per_sync) const; |
||||||
|
|
||||||
|
uint64_t GetFileSize() const { |
||||||
|
return file_size_.load(std::memory_order_acquire); |
||||||
|
} |
||||||
|
|
||||||
|
// All Get functions which are not atomic, will need ReadLock on the mutex
|
||||||
|
tsrange_t GetTimeRange() const { |
||||||
|
assert(HasTimestamp()); |
||||||
|
return time_range_; |
||||||
|
} |
||||||
|
|
||||||
|
ttlrange_t GetTTLRange() const { return ttl_range_; } |
||||||
|
|
||||||
|
snrange_t GetSNRange() const { return sn_range_; } |
||||||
|
|
||||||
|
bool HasTTL() const { |
||||||
|
assert(header_valid_); |
||||||
|
return header_.HasTTL(); |
||||||
|
} |
||||||
|
|
||||||
|
bool HasTimestamp() const { |
||||||
|
assert(header_valid_); |
||||||
|
return header_.HasTimestamp(); |
||||||
|
} |
||||||
|
|
||||||
|
std::shared_ptr<Writer> GetWriter() const { return log_writer_; } |
||||||
|
|
||||||
|
void Fsync(); |
||||||
|
|
||||||
|
private: |
||||||
|
std::shared_ptr<Reader> OpenSequentialReader( |
||||||
|
Env* env, const DBOptions& db_options, |
||||||
|
const EnvOptions& env_options) const; |
||||||
|
|
||||||
|
Status ReadFooter(BlobLogFooter* footer); |
||||||
|
|
||||||
|
Status WriteFooterAndCloseLocked(); |
||||||
|
|
||||||
|
std::shared_ptr<RandomAccessFileReader> GetOrOpenRandomAccessReader( |
||||||
|
Env* env, const EnvOptions& env_options, bool* fresh_open); |
||||||
|
|
||||||
|
void CloseRandomAccessLocked(); |
||||||
|
|
||||||
|
// this is used, when you are reading only the footer of a
|
||||||
|
// previously closed file
|
||||||
|
Status SetFromFooterLocked(const BlobLogFooter& footer); |
||||||
|
|
||||||
|
void set_time_range(const tsrange_t& tr) { time_range_ = tr; } |
||||||
|
|
||||||
|
void set_ttl_range(const ttlrange_t& ttl) { ttl_range_ = ttl; } |
||||||
|
|
||||||
|
void SetSNRange(const snrange_t& snr) { sn_range_ = snr; } |
||||||
|
|
||||||
|
// The following functions are atomic, and don't need locks
|
||||||
|
void SetFileSize(uint64_t fs) { file_size_ = fs; } |
||||||
|
|
||||||
|
void SetBlobCount(uint64_t bc) { blob_count_ = bc; } |
||||||
|
|
||||||
|
void SetCanBeDeleted() { can_be_deleted_ = true; } |
||||||
|
}; |
||||||
|
|
||||||
|
class BlobDBIterator : public Iterator { |
||||||
|
public: |
||||||
|
explicit BlobDBIterator(Iterator* iter, ColumnFamilyHandle* column_family, |
||||||
|
BlobDBImpl* impl) |
||||||
|
: iter_(iter), cfh_(column_family), db_impl_(impl) { |
||||||
|
assert(iter_); |
||||||
|
} |
||||||
|
|
||||||
|
~BlobDBIterator() { delete iter_; } |
||||||
|
|
||||||
|
bool Valid() const override { return iter_->Valid(); } |
||||||
|
|
||||||
|
void SeekToFirst() override { iter_->SeekToFirst(); } |
||||||
|
|
||||||
|
void SeekToLast() override { iter_->SeekToLast(); } |
||||||
|
|
||||||
|
void Seek(const Slice& target) override { iter_->Seek(target); } |
||||||
|
|
||||||
|
void SeekForPrev(const Slice& target) override { iter_->SeekForPrev(target); } |
||||||
|
|
||||||
|
void Next() override { iter_->Next(); } |
||||||
|
|
||||||
|
void Prev() override { iter_->Prev(); } |
||||||
|
|
||||||
|
Slice key() const override { return iter_->key(); } |
||||||
|
|
||||||
|
Slice value() const override; |
||||||
|
|
||||||
|
Status status() const override { return iter_->status(); } |
||||||
|
|
||||||
|
private: |
||||||
|
Iterator* iter_; |
||||||
|
ColumnFamilyHandle* cfh_; |
||||||
|
BlobDBImpl* db_impl_; |
||||||
|
mutable std::string vpart_; |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace blob_db
|
||||||
|
} // namespace rocksdb
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
@ -0,0 +1,66 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
|
||||||
|
#include "utilities/blob_db/blob_db_options_impl.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
namespace blob_db { |
||||||
|
|
||||||
|
BlobDBOptionsImpl::BlobDBOptionsImpl(const BlobDBOptions& in) |
||||||
|
: BlobDBOptions(in), |
||||||
|
deletion_check_period_millisecs(2 * 1000), |
||||||
|
gc_file_pct(20), |
||||||
|
gc_check_period_millisecs(60 * 1000), |
||||||
|
sanity_check_period_millisecs(20 * 60 * 1000), |
||||||
|
open_files_trigger(100), |
||||||
|
wa_num_stats_periods(24), |
||||||
|
wa_stats_period_millisecs(3600 * 1000), |
||||||
|
partial_expiration_gc_range_secs(4 * 3600), |
||||||
|
partial_expiration_pct(75), |
||||||
|
fsync_files_period_millisecs(10 * 1000), |
||||||
|
reclaim_of_period_millisecs(1 * 1000), |
||||||
|
delete_obsf_period_millisecs(10 * 1000), |
||||||
|
check_seqf_period_millisecs(10 * 1000) {} |
||||||
|
|
||||||
|
BlobDBOptionsImpl::BlobDBOptionsImpl() |
||||||
|
: deletion_check_period_millisecs(2 * 1000), |
||||||
|
gc_file_pct(20), |
||||||
|
gc_check_period_millisecs(60 * 1000), |
||||||
|
sanity_check_period_millisecs(20 * 60 * 1000), |
||||||
|
open_files_trigger(100), |
||||||
|
wa_num_stats_periods(24), |
||||||
|
wa_stats_period_millisecs(3600 * 1000), |
||||||
|
partial_expiration_gc_range_secs(4 * 3600), |
||||||
|
partial_expiration_pct(75), |
||||||
|
fsync_files_period_millisecs(10 * 1000), |
||||||
|
reclaim_of_period_millisecs(1 * 1000), |
||||||
|
delete_obsf_period_millisecs(10 * 1000), |
||||||
|
check_seqf_period_millisecs(10 * 1000) {} |
||||||
|
|
||||||
|
BlobDBOptionsImpl& BlobDBOptionsImpl::operator=(const BlobDBOptionsImpl& in) { |
||||||
|
BlobDBOptions::operator=(in); |
||||||
|
if (this != &in) { |
||||||
|
deletion_check_period_millisecs = in.deletion_check_period_millisecs; |
||||||
|
gc_file_pct = in.gc_file_pct; |
||||||
|
gc_check_period_millisecs = in.gc_check_period_millisecs; |
||||||
|
sanity_check_period_millisecs = in.sanity_check_period_millisecs; |
||||||
|
open_files_trigger = in.open_files_trigger; |
||||||
|
wa_num_stats_periods = in.wa_num_stats_periods; |
||||||
|
wa_stats_period_millisecs = in.wa_stats_period_millisecs; |
||||||
|
partial_expiration_gc_range_secs = in.partial_expiration_gc_range_secs; |
||||||
|
partial_expiration_pct = in.partial_expiration_pct; |
||||||
|
fsync_files_period_millisecs = in.fsync_files_period_millisecs; |
||||||
|
reclaim_of_period_millisecs = in.reclaim_of_period_millisecs; |
||||||
|
delete_obsf_period_millisecs = in.delete_obsf_period_millisecs; |
||||||
|
check_seqf_period_millisecs = in.check_seqf_period_millisecs; |
||||||
|
} |
||||||
|
return *this; |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace blob_db
|
||||||
|
} // namespace rocksdb
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
@ -0,0 +1,73 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
|
||||||
|
#include "utilities/blob_db/blob_db.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
namespace blob_db { |
||||||
|
|
||||||
|
struct BlobDBOptionsImpl : public BlobDBOptions { |
||||||
|
// deletions check period
|
||||||
|
uint32_t deletion_check_period_millisecs; |
||||||
|
|
||||||
|
// gc percentage each check period
|
||||||
|
uint32_t gc_file_pct; |
||||||
|
|
||||||
|
// gc period
|
||||||
|
uint32_t gc_check_period_millisecs; |
||||||
|
|
||||||
|
// sanity check task
|
||||||
|
uint32_t sanity_check_period_millisecs; |
||||||
|
|
||||||
|
// how many random access open files can we tolerate
|
||||||
|
uint32_t open_files_trigger; |
||||||
|
|
||||||
|
// how many periods of stats do we keep.
|
||||||
|
uint32_t wa_num_stats_periods; |
||||||
|
|
||||||
|
// what is the length of any period
|
||||||
|
uint32_t wa_stats_period_millisecs; |
||||||
|
|
||||||
|
// we will garbage collect blob files in
|
||||||
|
// which entire files have expired. However if the
|
||||||
|
// ttl_range of files is very large say a day, we
|
||||||
|
// would have to wait for the entire day, before we
|
||||||
|
// recover most of the space.
|
||||||
|
uint32_t partial_expiration_gc_range_secs; |
||||||
|
|
||||||
|
// this should be based on allowed Write Amplification
|
||||||
|
// if 50% of the space of a blob file has been deleted/expired,
|
||||||
|
uint32_t partial_expiration_pct; |
||||||
|
|
||||||
|
// how often should we schedule a job to fsync open files
|
||||||
|
uint32_t fsync_files_period_millisecs; |
||||||
|
|
||||||
|
// how often to schedule reclaim open files.
|
||||||
|
uint32_t reclaim_of_period_millisecs; |
||||||
|
|
||||||
|
// how often to schedule delete obs files periods
|
||||||
|
uint32_t delete_obsf_period_millisecs; |
||||||
|
|
||||||
|
// how often to schedule check seq files period
|
||||||
|
uint32_t check_seqf_period_millisecs; |
||||||
|
|
||||||
|
// default constructor
|
||||||
|
BlobDBOptionsImpl(); |
||||||
|
|
||||||
|
explicit BlobDBOptionsImpl(const BlobDBOptions& in); |
||||||
|
|
||||||
|
BlobDBOptionsImpl& operator=(const BlobDBOptionsImpl& in); |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace blob_db
|
||||||
|
|
||||||
|
} // namespace rocksdb
|
||||||
|
|
||||||
|
#endif // endif ROCKSDB
|
||||||
@ -0,0 +1,225 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
|
||||||
|
#include <chrono> |
||||||
|
#include <cinttypes> |
||||||
|
#include <memory> |
||||||
|
#include "utilities/blob_db/blob_db_impl.h" |
||||||
|
|
||||||
|
#include "util/filename.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
namespace blob_db { |
||||||
|
|
||||||
|
BlobFile::BlobFile() |
||||||
|
: parent_(nullptr), |
||||||
|
file_number_(0), |
||||||
|
blob_count_(0), |
||||||
|
gc_epoch_(-1), |
||||||
|
file_size_(0), |
||||||
|
deleted_count_(0), |
||||||
|
deleted_size_(0), |
||||||
|
closed_(false), |
||||||
|
can_be_deleted_(false), |
||||||
|
gc_once_after_open_(false), |
||||||
|
ttl_range_(std::make_pair(0, 0)), |
||||||
|
time_range_(std::make_pair(0, 0)), |
||||||
|
sn_range_(std::make_pair(0, 0)), |
||||||
|
last_access_(-1), |
||||||
|
last_fsync_(0), |
||||||
|
header_valid_(false) {} |
||||||
|
|
||||||
|
BlobFile::BlobFile(const BlobDBImpl* p, const std::string& bdir, uint64_t fn) |
||||||
|
: parent_(p), |
||||||
|
path_to_dir_(bdir), |
||||||
|
file_number_(fn), |
||||||
|
blob_count_(0), |
||||||
|
gc_epoch_(-1), |
||||||
|
file_size_(0), |
||||||
|
deleted_count_(0), |
||||||
|
deleted_size_(0), |
||||||
|
closed_(false), |
||||||
|
can_be_deleted_(false), |
||||||
|
gc_once_after_open_(false), |
||||||
|
ttl_range_(std::make_pair(0, 0)), |
||||||
|
time_range_(std::make_pair(0, 0)), |
||||||
|
sn_range_(std::make_pair(0, 0)), |
||||||
|
last_access_(-1), |
||||||
|
last_fsync_(0), |
||||||
|
header_valid_(false) {} |
||||||
|
|
||||||
|
BlobFile::~BlobFile() { |
||||||
|
if (can_be_deleted_) { |
||||||
|
std::string pn(PathName()); |
||||||
|
Status s = Env::Default()->DeleteFile(PathName()); |
||||||
|
if (!s.ok()) { |
||||||
|
// Log(InfoLogLevel::INFO_LEVEL, db_options_.info_log,
|
||||||
|
// "File could not be deleted %s", pn.c_str());
|
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
std::string BlobFile::PathName() const { |
||||||
|
return BlobFileName(path_to_dir_, file_number_); |
||||||
|
} |
||||||
|
|
||||||
|
std::shared_ptr<Reader> BlobFile::OpenSequentialReader( |
||||||
|
Env* env, const DBOptions& db_options, |
||||||
|
const EnvOptions& env_options) const { |
||||||
|
std::unique_ptr<SequentialFile> sfile; |
||||||
|
Status s = env->NewSequentialFile(PathName(), &sfile, env_options); |
||||||
|
if (!s.ok()) { |
||||||
|
// report something here.
|
||||||
|
return nullptr; |
||||||
|
} |
||||||
|
|
||||||
|
std::unique_ptr<SequentialFileReader> sfile_reader; |
||||||
|
sfile_reader.reset(new SequentialFileReader(std::move(sfile))); |
||||||
|
|
||||||
|
std::shared_ptr<Reader> log_reader = |
||||||
|
std::make_shared<Reader>(db_options.info_log, std::move(sfile_reader)); |
||||||
|
|
||||||
|
return log_reader; |
||||||
|
} |
||||||
|
|
||||||
|
std::string BlobFile::DumpState() const { |
||||||
|
char str[1000]; |
||||||
|
std::snprintf(str, sizeof(str), |
||||||
|
"path: %s fn: %" PRIu64 " blob_count: %" PRIu64 |
||||||
|
" gc_epoch: %" PRIu64 " file_size: %" PRIu64 |
||||||
|
" deleted_count: %" PRIu64 " deleted_size: %" PRIu64 |
||||||
|
" closed: %d can_be_deleted: %d ttl_range: (%d, %d)" |
||||||
|
" sn_range: (%" PRIu64 " %" PRIu64 "), writer: %d reader: %d", |
||||||
|
path_to_dir_.c_str(), file_number_, blob_count_.load(), |
||||||
|
gc_epoch_.load(), file_size_.load(), deleted_count_, |
||||||
|
deleted_size_, closed_.load(), can_be_deleted_.load(), |
||||||
|
ttl_range_.first, ttl_range_.second, sn_range_.first, |
||||||
|
sn_range_.second, (!!log_writer_), (!!ra_file_reader_)); |
||||||
|
return str; |
||||||
|
} |
||||||
|
|
||||||
|
bool BlobFile::NeedsFsync(bool hard, uint64_t bytes_per_sync) const { |
||||||
|
assert(last_fsync_ <= file_size_); |
||||||
|
return (hard) ? file_size_ > last_fsync_ |
||||||
|
: (file_size_ - last_fsync_) >= bytes_per_sync; |
||||||
|
} |
||||||
|
|
||||||
|
Status BlobFile::WriteFooterAndCloseLocked() { |
||||||
|
Log(InfoLogLevel::INFO_LEVEL, parent_->db_options_.info_log, |
||||||
|
"File is being closed after footer %s", PathName().c_str()); |
||||||
|
|
||||||
|
BlobLogFooter footer; |
||||||
|
footer.blob_count_ = blob_count_; |
||||||
|
if (HasTTL()) footer.set_ttl_range(ttl_range_); |
||||||
|
|
||||||
|
footer.sn_range_ = sn_range_; |
||||||
|
if (HasTimestamp()) footer.set_time_range(time_range_); |
||||||
|
|
||||||
|
// this will close the file and reset the Writable File Pointer.
|
||||||
|
Status s = log_writer_->AppendFooter(footer); |
||||||
|
if (s.ok()) { |
||||||
|
closed_ = true; |
||||||
|
file_size_ += BlobLogFooter::kFooterSize; |
||||||
|
} else { |
||||||
|
Log(InfoLogLevel::ERROR_LEVEL, parent_->db_options_.info_log, |
||||||
|
"Failure to read Header for blob-file %s", PathName().c_str()); |
||||||
|
} |
||||||
|
// delete the sequential writer
|
||||||
|
log_writer_.reset(); |
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
Status BlobFile::ReadFooter(BlobLogFooter* bf) { |
||||||
|
if (file_size_ < (BlobLogHeader::kHeaderSize + BlobLogFooter::kFooterSize)) { |
||||||
|
return Status::IOError("File does not have footer", PathName()); |
||||||
|
} |
||||||
|
|
||||||
|
uint64_t footer_offset = file_size_ - BlobLogFooter::kFooterSize; |
||||||
|
// assume that ra_file_reader_ is valid before we enter this
|
||||||
|
assert(ra_file_reader_); |
||||||
|
|
||||||
|
Slice result; |
||||||
|
char scratch[BlobLogFooter::kFooterSize + 10]; |
||||||
|
Status s = ra_file_reader_->Read(footer_offset, BlobLogFooter::kFooterSize, |
||||||
|
&result, scratch); |
||||||
|
if (!s.ok()) return s; |
||||||
|
if (result.size() != BlobLogFooter::kFooterSize) { |
||||||
|
// should not happen
|
||||||
|
return Status::IOError("EOF reached before footer"); |
||||||
|
} |
||||||
|
|
||||||
|
s = bf->DecodeFrom(&result); |
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
Status BlobFile::SetFromFooterLocked(const BlobLogFooter& footer) { |
||||||
|
if (footer.HasTTL() != header_.HasTTL()) { |
||||||
|
return Status::Corruption("has_ttl mismatch"); |
||||||
|
} |
||||||
|
if (footer.HasTimestamp() != header_.HasTimestamp()) { |
||||||
|
return Status::Corruption("has_ts mismatch"); |
||||||
|
} |
||||||
|
|
||||||
|
// assume that file has been fully fsync'd
|
||||||
|
last_fsync_.store(file_size_); |
||||||
|
blob_count_ = footer.GetBlobCount(); |
||||||
|
ttl_range_ = footer.GetTTLRange(); |
||||||
|
time_range_ = footer.GetTimeRange(); |
||||||
|
sn_range_ = footer.GetSNRange(); |
||||||
|
closed_ = true; |
||||||
|
|
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
void BlobFile::Fsync() { |
||||||
|
if (log_writer_.get()) { |
||||||
|
log_writer_->Sync(); |
||||||
|
last_fsync_.store(file_size_.load()); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
void BlobFile::CloseRandomAccessLocked() { |
||||||
|
ra_file_reader_.reset(); |
||||||
|
last_access_ = -1; |
||||||
|
} |
||||||
|
|
||||||
|
std::shared_ptr<RandomAccessFileReader> BlobFile::GetOrOpenRandomAccessReader( |
||||||
|
Env* env, const EnvOptions& env_options, bool* fresh_open) { |
||||||
|
*fresh_open = false; |
||||||
|
last_access_ = |
||||||
|
std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); |
||||||
|
{ |
||||||
|
ReadLock lockbfile_r(&mutex_); |
||||||
|
if (ra_file_reader_) return ra_file_reader_; |
||||||
|
} |
||||||
|
|
||||||
|
WriteLock lockbfile_w(&mutex_); |
||||||
|
if (ra_file_reader_) return ra_file_reader_; |
||||||
|
|
||||||
|
std::unique_ptr<RandomAccessFile> rfile; |
||||||
|
Status s = env->NewRandomAccessFile(PathName(), &rfile, env_options); |
||||||
|
if (!s.ok()) { |
||||||
|
Log(InfoLogLevel::ERROR_LEVEL, parent_->db_options_.info_log, |
||||||
|
"Failed to open blob file for random-read: %s status: '%s'" |
||||||
|
" exists: '%s'", |
||||||
|
PathName().c_str(), s.ToString().c_str(), |
||||||
|
env->FileExists(PathName()).ToString().c_str()); |
||||||
|
return nullptr; |
||||||
|
} |
||||||
|
|
||||||
|
ra_file_reader_ = std::make_shared<RandomAccessFileReader>(std::move(rfile)); |
||||||
|
*fresh_open = true; |
||||||
|
return ra_file_reader_; |
||||||
|
} |
||||||
|
|
||||||
|
ColumnFamilyHandle* BlobFile::GetColumnFamily(DB* db) { |
||||||
|
return db->DefaultColumnFamily(); |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace blob_db
|
||||||
|
} // namespace rocksdb
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
@ -0,0 +1,313 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
|
||||||
|
#include "utilities/blob_db/blob_log_format.h" |
||||||
|
#include "util/coding.h" |
||||||
|
#include "util/crc32c.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
namespace blob_db { |
||||||
|
|
||||||
|
const uint32_t kMagicNumber = 2395959; |
||||||
|
const uint32_t kVersion1 = 1; |
||||||
|
const size_t kBlockSize = 32768; |
||||||
|
|
||||||
|
BlobLogHeader::BlobLogHeader() |
||||||
|
: magic_number_(kMagicNumber), compression_(kNoCompression) {} |
||||||
|
|
||||||
|
BlobLogHeader& BlobLogHeader::operator=(BlobLogHeader&& in) noexcept { |
||||||
|
if (this != &in) { |
||||||
|
magic_number_ = in.magic_number_; |
||||||
|
version_ = in.version_; |
||||||
|
ttl_guess_ = std::move(in.ttl_guess_); |
||||||
|
ts_guess_ = std::move(in.ts_guess_); |
||||||
|
compression_ = in.compression_; |
||||||
|
} |
||||||
|
return *this; |
||||||
|
} |
||||||
|
|
||||||
|
BlobLogFooter::BlobLogFooter() : magic_number_(kMagicNumber), blob_count_(0) {} |
||||||
|
|
||||||
|
Status BlobLogFooter::DecodeFrom(Slice* input) { |
||||||
|
uint32_t val; |
||||||
|
if (!GetFixed32(input, &val)) { |
||||||
|
return Status::Corruption("Invalid Blob Footer: flags"); |
||||||
|
} |
||||||
|
|
||||||
|
bool has_ttl = false; |
||||||
|
bool has_ts = false; |
||||||
|
val >>= 8; |
||||||
|
RecordSubType st = static_cast<RecordSubType>(val); |
||||||
|
switch (st) { |
||||||
|
case kRegularType: |
||||||
|
break; |
||||||
|
case kTTLType: |
||||||
|
has_ttl = true; |
||||||
|
break; |
||||||
|
case kTimestampType: |
||||||
|
has_ts = true; |
||||||
|
break; |
||||||
|
default: |
||||||
|
return Status::Corruption("Invalid Blob Footer: flags_val"); |
||||||
|
} |
||||||
|
|
||||||
|
if (!GetFixed64(input, &blob_count_)) { |
||||||
|
return Status::Corruption("Invalid Blob Footer: blob_count"); |
||||||
|
} |
||||||
|
|
||||||
|
ttlrange_t temp_ttl; |
||||||
|
if (!GetFixed32(input, &temp_ttl.first) || |
||||||
|
!GetFixed32(input, &temp_ttl.second)) { |
||||||
|
return Status::Corruption("Invalid Blob Footer: ttl_range"); |
||||||
|
} |
||||||
|
if (has_ttl) { |
||||||
|
printf("has ttl\n"); |
||||||
|
ttl_range_.reset(new ttlrange_t(temp_ttl)); |
||||||
|
} |
||||||
|
|
||||||
|
if (!GetFixed64(input, &sn_range_.first) || |
||||||
|
!GetFixed64(input, &sn_range_.second)) { |
||||||
|
return Status::Corruption("Invalid Blob Footer: sn_range"); |
||||||
|
} |
||||||
|
|
||||||
|
tsrange_t temp_ts; |
||||||
|
if (!GetFixed64(input, &temp_ts.first) || |
||||||
|
!GetFixed64(input, &temp_ts.second)) { |
||||||
|
return Status::Corruption("Invalid Blob Footer: ts_range"); |
||||||
|
} |
||||||
|
if (has_ts) ts_range_.reset(new tsrange_t(temp_ts)); |
||||||
|
|
||||||
|
if (!GetFixed32(input, &magic_number_) || magic_number_ != kMagicNumber) { |
||||||
|
return Status::Corruption("Invalid Blob Footer: magic"); |
||||||
|
} |
||||||
|
|
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
void BlobLogFooter::EncodeTo(std::string* dst) const { |
||||||
|
dst->reserve(kFooterSize); |
||||||
|
|
||||||
|
RecordType rt = kFullType; |
||||||
|
RecordSubType st = kRegularType; |
||||||
|
if (HasTTL()) { |
||||||
|
st = kTTLType; |
||||||
|
} else if (HasTimestamp()) { |
||||||
|
st = kTimestampType; |
||||||
|
} |
||||||
|
uint32_t val = static_cast<uint32_t>(rt) | (static_cast<uint32_t>(st) << 8); |
||||||
|
PutFixed32(dst, val); |
||||||
|
|
||||||
|
PutFixed64(dst, blob_count_); |
||||||
|
bool has_ttl = HasTTL(); |
||||||
|
bool has_ts = HasTimestamp(); |
||||||
|
|
||||||
|
if (has_ttl) { |
||||||
|
PutFixed32(dst, ttl_range_.get()->first); |
||||||
|
PutFixed32(dst, ttl_range_.get()->second); |
||||||
|
} else { |
||||||
|
PutFixed32(dst, 0); |
||||||
|
PutFixed32(dst, 0); |
||||||
|
} |
||||||
|
PutFixed64(dst, sn_range_.first); |
||||||
|
PutFixed64(dst, sn_range_.second); |
||||||
|
|
||||||
|
if (has_ts) { |
||||||
|
PutFixed64(dst, ts_range_.get()->first); |
||||||
|
PutFixed64(dst, ts_range_.get()->second); |
||||||
|
} else { |
||||||
|
PutFixed64(dst, 0); |
||||||
|
PutFixed64(dst, 0); |
||||||
|
} |
||||||
|
|
||||||
|
PutFixed32(dst, magic_number_); |
||||||
|
} |
||||||
|
|
||||||
|
void BlobLogHeader::EncodeTo(std::string* dst) const { |
||||||
|
dst->reserve(kHeaderSize); |
||||||
|
|
||||||
|
PutFixed32(dst, magic_number_); |
||||||
|
|
||||||
|
PutFixed32(dst, version_); |
||||||
|
|
||||||
|
RecordSubType st = kRegularType; |
||||||
|
bool has_ttl = HasTTL(); |
||||||
|
bool has_ts = HasTimestamp(); |
||||||
|
|
||||||
|
if (has_ttl) { |
||||||
|
st = kTTLType; |
||||||
|
} else if (has_ts) { |
||||||
|
st = kTimestampType; |
||||||
|
} |
||||||
|
uint32_t val = |
||||||
|
static_cast<uint32_t>(st) | (static_cast<uint32_t>(compression_) << 8); |
||||||
|
PutFixed32(dst, val); |
||||||
|
|
||||||
|
if (has_ttl) { |
||||||
|
PutFixed32(dst, ttl_guess_.get()->first); |
||||||
|
PutFixed32(dst, ttl_guess_.get()->second); |
||||||
|
} else { |
||||||
|
PutFixed32(dst, 0); |
||||||
|
PutFixed32(dst, 0); |
||||||
|
} |
||||||
|
|
||||||
|
if (has_ts) { |
||||||
|
PutFixed64(dst, ts_guess_.get()->first); |
||||||
|
PutFixed64(dst, ts_guess_.get()->second); |
||||||
|
} else { |
||||||
|
PutFixed64(dst, 0); |
||||||
|
PutFixed64(dst, 0); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
Status BlobLogHeader::DecodeFrom(Slice* input) { |
||||||
|
if (!GetFixed32(input, &magic_number_) || magic_number_ != kMagicNumber) { |
||||||
|
return Status::Corruption("Invalid Blob Log Header: magic"); |
||||||
|
} |
||||||
|
|
||||||
|
// as of today, we only support 1 version
|
||||||
|
if (!GetFixed32(input, &version_) || version_ != kVersion1) { |
||||||
|
return Status::Corruption("Invalid Blob Log Header: version"); |
||||||
|
} |
||||||
|
|
||||||
|
uint32_t val; |
||||||
|
if (!GetFixed32(input, &val)) { |
||||||
|
return Status::Corruption("Invalid Blob Log Header: subtype"); |
||||||
|
} |
||||||
|
|
||||||
|
bool has_ttl = false; |
||||||
|
bool has_ts = false; |
||||||
|
RecordSubType st = static_cast<RecordSubType>(val & 0xff); |
||||||
|
compression_ = static_cast<CompressionType>((val >> 8) & 0xff); |
||||||
|
switch (st) { |
||||||
|
case kRegularType: |
||||||
|
break; |
||||||
|
case kTTLType: |
||||||
|
has_ttl = true; |
||||||
|
break; |
||||||
|
case kTimestampType: |
||||||
|
has_ts = true; |
||||||
|
break; |
||||||
|
default: |
||||||
|
return Status::Corruption("Invalid Blob Log Header: subtype_2"); |
||||||
|
} |
||||||
|
|
||||||
|
ttlrange_t temp_ttl; |
||||||
|
if (!GetFixed32(input, &temp_ttl.first) || |
||||||
|
!GetFixed32(input, &temp_ttl.second)) { |
||||||
|
return Status::Corruption("Invalid Blob Log Header: ttl"); |
||||||
|
} |
||||||
|
if (has_ttl) set_ttl_guess(temp_ttl); |
||||||
|
|
||||||
|
tsrange_t temp_ts; |
||||||
|
if (!GetFixed64(input, &temp_ts.first) || |
||||||
|
!GetFixed64(input, &temp_ts.second)) { |
||||||
|
return Status::Corruption("Invalid Blob Log Header: timestamp"); |
||||||
|
} |
||||||
|
if (has_ts) set_ts_guess(temp_ts); |
||||||
|
|
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
BlobLogRecord::BlobLogRecord() |
||||||
|
: checksum_(0), |
||||||
|
header_cksum_(0), |
||||||
|
key_size_(0), |
||||||
|
blob_size_(0), |
||||||
|
time_val_(0), |
||||||
|
ttl_val_(0), |
||||||
|
sn_(0), |
||||||
|
type_(0), |
||||||
|
subtype_(0) {} |
||||||
|
|
||||||
|
BlobLogRecord::~BlobLogRecord() {} |
||||||
|
|
||||||
|
void BlobLogRecord::ResizeKeyBuffer(size_t kbs) { |
||||||
|
if (kbs > key_buffer_.size()) { |
||||||
|
key_buffer_.resize(kbs); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
void BlobLogRecord::ResizeBlobBuffer(size_t bbs) { |
||||||
|
if (bbs > blob_buffer_.size()) { |
||||||
|
blob_buffer_.resize(bbs); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
void BlobLogRecord::Clear() { |
||||||
|
checksum_ = 0; |
||||||
|
header_cksum_ = 0; |
||||||
|
key_size_ = 0; |
||||||
|
blob_size_ = 0; |
||||||
|
time_val_ = 0; |
||||||
|
ttl_val_ = 0; |
||||||
|
sn_ = 0; |
||||||
|
type_ = subtype_ = 0; |
||||||
|
key_.clear(); |
||||||
|
blob_.clear(); |
||||||
|
} |
||||||
|
|
||||||
|
Status BlobLogRecord::DecodeHeaderFrom(const Slice& hdrslice) { |
||||||
|
Slice input = hdrslice; |
||||||
|
if (input.size() < kHeaderSize) { |
||||||
|
return Status::Corruption("Invalid Blob Record Header: size"); |
||||||
|
} |
||||||
|
|
||||||
|
if (!GetFixed32(&input, &key_size_)) { |
||||||
|
return Status::Corruption("Invalid Blob Record Header: key_size"); |
||||||
|
} |
||||||
|
if (!GetFixed64(&input, &blob_size_)) { |
||||||
|
return Status::Corruption("Invalid Blob Record Header: blob_size"); |
||||||
|
} |
||||||
|
if (!GetFixed32(&input, &ttl_val_)) { |
||||||
|
return Status::Corruption("Invalid Blob Record Header: ttl_val"); |
||||||
|
} |
||||||
|
if (!GetFixed64(&input, &time_val_)) { |
||||||
|
return Status::Corruption("Invalid Blob Record Header: time_val"); |
||||||
|
} |
||||||
|
|
||||||
|
type_ = *(input.data()); |
||||||
|
input.remove_prefix(1); |
||||||
|
subtype_ = *(input.data()); |
||||||
|
input.remove_prefix(1); |
||||||
|
|
||||||
|
if (!GetFixed32(&input, &header_cksum_)) { |
||||||
|
return Status::Corruption("Invalid Blob Record Header: header_cksum"); |
||||||
|
} |
||||||
|
if (!GetFixed32(&input, &checksum_)) { |
||||||
|
return Status::Corruption("Invalid Blob Record Header: checksum"); |
||||||
|
} |
||||||
|
|
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
Status BlobLogRecord::DecodeFooterFrom(const Slice& footerslice) { |
||||||
|
Slice input = footerslice; |
||||||
|
if (input.size() < kFooterSize) { |
||||||
|
return Status::Corruption("Invalid Blob Record Footer: size"); |
||||||
|
} |
||||||
|
|
||||||
|
uint32_t f_crc = crc32c::Extend(0, input.data(), 8); |
||||||
|
f_crc = crc32c::Mask(f_crc); |
||||||
|
|
||||||
|
if (!GetFixed64(&input, &sn_)) { |
||||||
|
return Status::Corruption("Invalid Blob Record Footer: sn"); |
||||||
|
} |
||||||
|
|
||||||
|
if (!GetFixed32(&input, &footer_cksum_)) { |
||||||
|
return Status::Corruption("Invalid Blob Record Footer: cksum"); |
||||||
|
} |
||||||
|
|
||||||
|
if (f_crc != footer_cksum_) { |
||||||
|
return Status::Corruption("Record Checksum mismatch: footer_cksum"); |
||||||
|
} |
||||||
|
|
||||||
|
return Status::OK(); |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace blob_db
|
||||||
|
} // namespace rocksdb
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
@ -0,0 +1,226 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
// Log format information shared by reader and writer.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
|
||||||
|
#include <cstddef> |
||||||
|
#include <cstdint> |
||||||
|
#include <memory> |
||||||
|
#include <string> |
||||||
|
#include <utility> |
||||||
|
#include "rocksdb/options.h" |
||||||
|
#include "rocksdb/status.h" |
||||||
|
#include "rocksdb/types.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
namespace blob_db { |
||||||
|
class BlobFile; |
||||||
|
class BlobDBImpl; |
||||||
|
|
||||||
|
enum RecordType : uint8_t { |
||||||
|
// Zero is reserved for preallocated files
|
||||||
|
kFullType = 0, |
||||||
|
|
||||||
|
// For fragments
|
||||||
|
kFirstType = 1, |
||||||
|
kMiddleType = 2, |
||||||
|
kLastType = 3, |
||||||
|
kMaxRecordType = kLastType |
||||||
|
}; |
||||||
|
|
||||||
|
enum RecordSubType : uint8_t { |
||||||
|
kRegularType = 0, |
||||||
|
kTTLType = 1, |
||||||
|
kTimestampType = 2, |
||||||
|
}; |
||||||
|
|
||||||
|
extern const uint32_t kMagicNumber; |
||||||
|
|
||||||
|
class Reader; |
||||||
|
|
||||||
|
typedef std::pair<uint32_t, uint32_t> ttlrange_t; |
||||||
|
typedef std::pair<uint64_t, uint64_t> tsrange_t; |
||||||
|
typedef std::pair<rocksdb::SequenceNumber, rocksdb::SequenceNumber> snrange_t; |
||||||
|
|
||||||
|
class BlobLogHeader { |
||||||
|
friend class BlobFile; |
||||||
|
friend class BlobDBImpl; |
||||||
|
|
||||||
|
private: |
||||||
|
uint32_t magic_number_ = 0; |
||||||
|
uint32_t version_ = 1; |
||||||
|
CompressionType compression_; |
||||||
|
std::unique_ptr<ttlrange_t> ttl_guess_; |
||||||
|
std::unique_ptr<tsrange_t> ts_guess_; |
||||||
|
|
||||||
|
private: |
||||||
|
void set_ttl_guess(const ttlrange_t& ttl) { |
||||||
|
ttl_guess_.reset(new ttlrange_t(ttl)); |
||||||
|
} |
||||||
|
|
||||||
|
void set_version(uint32_t v) { version_ = v; } |
||||||
|
|
||||||
|
void set_ts_guess(const tsrange_t& ts) { ts_guess_.reset(new tsrange_t(ts)); } |
||||||
|
|
||||||
|
public: |
||||||
|
// magic number + version + flags + ttl guess + timestamp range
|
||||||
|
static const size_t kHeaderSize = 4 + 4 + 4 + 4 * 2 + 8 * 2; |
||||||
|
// 32
|
||||||
|
|
||||||
|
void EncodeTo(std::string* dst) const; |
||||||
|
|
||||||
|
Status DecodeFrom(Slice* input); |
||||||
|
|
||||||
|
BlobLogHeader(); |
||||||
|
|
||||||
|
bool HasTTL() const { return !!ttl_guess_; } |
||||||
|
|
||||||
|
bool HasTimestamp() const { return !!ts_guess_; } |
||||||
|
|
||||||
|
BlobLogHeader& operator=(BlobLogHeader&& in) noexcept; |
||||||
|
}; |
||||||
|
|
||||||
|
// Footer encapsulates the fixed information stored at the tail
|
||||||
|
// end of every blob log file.
|
||||||
|
class BlobLogFooter { |
||||||
|
friend class BlobFile; |
||||||
|
|
||||||
|
public: |
||||||
|
// Use this constructor when you plan to write out the footer using
|
||||||
|
// EncodeTo(). Never use this constructor with DecodeFrom().
|
||||||
|
BlobLogFooter(); |
||||||
|
|
||||||
|
uint64_t magic_number() const { return magic_number_; } |
||||||
|
|
||||||
|
void EncodeTo(std::string* dst) const; |
||||||
|
|
||||||
|
Status DecodeFrom(Slice* input); |
||||||
|
|
||||||
|
// convert this object to a human readable form
|
||||||
|
std::string ToString() const; |
||||||
|
|
||||||
|
// footer size = 4 byte magic number
|
||||||
|
// 8 bytes count
|
||||||
|
// 4, 4 - ttl range
|
||||||
|
// 8, 8 - sn range
|
||||||
|
// 8, 8 - ts range
|
||||||
|
// = 56
|
||||||
|
static const size_t kFooterSize = 4 + 4 + 8 + (4 * 2) + (8 * 2) + (8 * 2); |
||||||
|
|
||||||
|
bool HasTTL() const { return !!ttl_range_; } |
||||||
|
|
||||||
|
bool HasTimestamp() const { return !!ts_range_; } |
||||||
|
|
||||||
|
uint64_t GetBlobCount() const { return blob_count_; } |
||||||
|
|
||||||
|
ttlrange_t GetTTLRange() const { |
||||||
|
if (ttl_range_) { |
||||||
|
*ttl_range_; |
||||||
|
} |
||||||
|
return {0, 0}; |
||||||
|
} |
||||||
|
|
||||||
|
tsrange_t GetTimeRange() const { |
||||||
|
if (ts_range_) { |
||||||
|
return *ts_range_; |
||||||
|
} |
||||||
|
return {0, 0}; |
||||||
|
} |
||||||
|
|
||||||
|
const snrange_t& GetSNRange() const { return sn_range_; } |
||||||
|
|
||||||
|
private: |
||||||
|
uint32_t magic_number_ = 0; |
||||||
|
uint64_t blob_count_ = 0; |
||||||
|
|
||||||
|
std::unique_ptr<ttlrange_t> ttl_range_; |
||||||
|
std::unique_ptr<tsrange_t> ts_range_; |
||||||
|
snrange_t sn_range_; |
||||||
|
|
||||||
|
private: |
||||||
|
void set_ttl_range(const ttlrange_t& ttl) { |
||||||
|
ttl_range_.reset(new ttlrange_t(ttl)); |
||||||
|
} |
||||||
|
void set_time_range(const tsrange_t& ts) { |
||||||
|
ts_range_.reset(new tsrange_t(ts)); |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
extern const size_t kBlockSize; |
||||||
|
|
||||||
|
class BlobLogRecord { |
||||||
|
friend class Reader; |
||||||
|
|
||||||
|
private: |
||||||
|
// this might not be set.
|
||||||
|
uint32_t checksum_; |
||||||
|
uint32_t header_cksum_; |
||||||
|
uint32_t key_size_; |
||||||
|
uint64_t blob_size_; |
||||||
|
uint64_t time_val_; |
||||||
|
uint32_t ttl_val_; |
||||||
|
SequenceNumber sn_; |
||||||
|
uint32_t footer_cksum_; |
||||||
|
char type_; |
||||||
|
char subtype_; |
||||||
|
Slice key_; |
||||||
|
Slice blob_; |
||||||
|
std::string key_buffer_; |
||||||
|
std::string blob_buffer_; |
||||||
|
|
||||||
|
private: |
||||||
|
void Clear(); |
||||||
|
|
||||||
|
char* GetKeyBuffer() { return &(key_buffer_[0]); } |
||||||
|
|
||||||
|
char* GetBlobBuffer() { return &(blob_buffer_[0]); } |
||||||
|
|
||||||
|
void ResizeKeyBuffer(size_t kbs); |
||||||
|
|
||||||
|
void ResizeBlobBuffer(size_t bbs); |
||||||
|
|
||||||
|
public: |
||||||
|
// Header is
|
||||||
|
// Key Length ( 4 bytes ),
|
||||||
|
// Blob Length ( 8 bytes), timestamp/ttl (8 bytes),
|
||||||
|
// type (1 byte), subtype (1 byte)
|
||||||
|
// header checksum (4 bytes), blob checksum (4 bytes),
|
||||||
|
// = 34
|
||||||
|
static const size_t kHeaderSize = 4 + 4 + 4 + 8 + 4 + 8 + 1 + 1; |
||||||
|
|
||||||
|
static const size_t kFooterSize = 8 + 4; |
||||||
|
|
||||||
|
public: |
||||||
|
BlobLogRecord(); |
||||||
|
|
||||||
|
~BlobLogRecord(); |
||||||
|
|
||||||
|
const Slice& Key() const { return key_; } |
||||||
|
|
||||||
|
const Slice& Blob() const { return blob_; } |
||||||
|
|
||||||
|
uint32_t GetKeySize() const { return key_size_; } |
||||||
|
|
||||||
|
uint64_t GetBlobSize() const { return blob_size_; } |
||||||
|
|
||||||
|
uint32_t GetTTL() const { return ttl_val_; } |
||||||
|
|
||||||
|
uint64_t GetTimeVal() const { return time_val_; } |
||||||
|
|
||||||
|
SequenceNumber GetSN() const { return sn_; } |
||||||
|
|
||||||
|
Status DecodeHeaderFrom(const Slice& hdrslice); |
||||||
|
|
||||||
|
Status DecodeFooterFrom(const Slice& footerslice); |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace blob_db
|
||||||
|
} // namespace rocksdb
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
@ -0,0 +1,163 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
|
||||||
|
#include "utilities/blob_db/blob_log_reader.h" |
||||||
|
|
||||||
|
#include <cstdio> |
||||||
|
#include "rocksdb/env.h" |
||||||
|
#include "util/coding.h" |
||||||
|
#include "util/crc32c.h" |
||||||
|
#include "util/file_reader_writer.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
namespace blob_db { |
||||||
|
|
||||||
|
Reader::Reader(std::shared_ptr<Logger> info_log, |
||||||
|
unique_ptr<SequentialFileReader>&& _file) |
||||||
|
: info_log_(info_log), file_(std::move(_file)), buffer_(), next_byte_(0) { |
||||||
|
backing_store_.resize(kBlockSize); |
||||||
|
} |
||||||
|
|
||||||
|
Reader::~Reader() {} |
||||||
|
|
||||||
|
Status Reader::ReadHeader(BlobLogHeader* header) { |
||||||
|
assert(file_.get() != nullptr); |
||||||
|
assert(next_byte_ == 0); |
||||||
|
Status status = |
||||||
|
file_->Read(BlobLogHeader::kHeaderSize, &buffer_, GetReadBuffer()); |
||||||
|
next_byte_ += buffer_.size(); |
||||||
|
if (!status.ok()) return status; |
||||||
|
|
||||||
|
if (buffer_.size() != BlobLogHeader::kHeaderSize) { |
||||||
|
return Status::IOError("EOF reached before file header"); |
||||||
|
} |
||||||
|
|
||||||
|
status = header->DecodeFrom(&buffer_); |
||||||
|
return status; |
||||||
|
} |
||||||
|
|
||||||
|
Status Reader::ReadRecord(BlobLogRecord* record, ReadLevel level, |
||||||
|
WALRecoveryMode wal_recovery_mode) { |
||||||
|
record->Clear(); |
||||||
|
buffer_.clear(); |
||||||
|
backing_store_[0] = '\0'; |
||||||
|
|
||||||
|
Status status = |
||||||
|
file_->Read(BlobLogRecord::kHeaderSize, &buffer_, GetReadBuffer()); |
||||||
|
next_byte_ += buffer_.size(); |
||||||
|
if (!status.ok()) return status; |
||||||
|
if (buffer_.size() != BlobLogRecord::kHeaderSize) { |
||||||
|
return Status::IOError("EOF reached before record header"); |
||||||
|
} |
||||||
|
|
||||||
|
status = record->DecodeHeaderFrom(buffer_); |
||||||
|
if (!status.ok()) return status; |
||||||
|
|
||||||
|
uint32_t header_crc = 0; |
||||||
|
uint32_t blob_crc = 0; |
||||||
|
size_t crc_data_size = BlobLogRecord::kHeaderSize - 2 * sizeof(uint32_t); |
||||||
|
header_crc = crc32c::Extend(header_crc, buffer_.data(), crc_data_size); |
||||||
|
|
||||||
|
uint64_t kb_size = record->GetKeySize() + record->GetBlobSize(); |
||||||
|
switch (level) { |
||||||
|
case kReadHdrFooter: |
||||||
|
file_->Skip(kb_size); |
||||||
|
next_byte_ += kb_size; |
||||||
|
status = |
||||||
|
file_->Read(BlobLogRecord::kFooterSize, &buffer_, GetReadBuffer()); |
||||||
|
next_byte_ += buffer_.size(); |
||||||
|
if (!status.ok()) return status; |
||||||
|
if (buffer_.size() != BlobLogRecord::kFooterSize) { |
||||||
|
return Status::IOError("EOF reached before record footer"); |
||||||
|
} |
||||||
|
|
||||||
|
status = record->DecodeFooterFrom(buffer_); |
||||||
|
return status; |
||||||
|
|
||||||
|
case kReadHdrKeyFooter: |
||||||
|
record->ResizeKeyBuffer(record->GetKeySize()); |
||||||
|
status = file_->Read(record->GetKeySize(), &record->key_, |
||||||
|
record->GetKeyBuffer()); |
||||||
|
next_byte_ += record->key_.size(); |
||||||
|
if (!status.ok()) return status; |
||||||
|
if (record->key_.size() != record->GetKeySize()) { |
||||||
|
return Status::IOError("EOF reached before key read"); |
||||||
|
} |
||||||
|
|
||||||
|
header_crc = |
||||||
|
crc32c::Extend(header_crc, record->key_.data(), record->GetKeySize()); |
||||||
|
header_crc = crc32c::Mask(header_crc); |
||||||
|
if (header_crc != record->header_cksum_) { |
||||||
|
return Status::Corruption("Record Checksum mismatch: header_cksum"); |
||||||
|
} |
||||||
|
|
||||||
|
file_->Skip(record->GetBlobSize()); |
||||||
|
next_byte_ += record->GetBlobSize(); |
||||||
|
|
||||||
|
status = |
||||||
|
file_->Read(BlobLogRecord::kFooterSize, &buffer_, GetReadBuffer()); |
||||||
|
next_byte_ += buffer_.size(); |
||||||
|
if (!status.ok()) return status; |
||||||
|
if (buffer_.size() != BlobLogRecord::kFooterSize) { |
||||||
|
return Status::IOError("EOF reached during footer read"); |
||||||
|
} |
||||||
|
|
||||||
|
status = record->DecodeFooterFrom(buffer_); |
||||||
|
return status; |
||||||
|
|
||||||
|
case kReadHdrKeyBlobFooter: |
||||||
|
record->ResizeKeyBuffer(record->GetKeySize()); |
||||||
|
status = file_->Read(record->GetKeySize(), &record->key_, |
||||||
|
record->GetKeyBuffer()); |
||||||
|
next_byte_ += record->key_.size(); |
||||||
|
if (!status.ok()) return status; |
||||||
|
if (record->key_.size() != record->GetKeySize()) { |
||||||
|
return Status::IOError("EOF reached before key read"); |
||||||
|
} |
||||||
|
|
||||||
|
header_crc = |
||||||
|
crc32c::Extend(header_crc, record->key_.data(), record->GetKeySize()); |
||||||
|
header_crc = crc32c::Mask(header_crc); |
||||||
|
if (header_crc != record->header_cksum_) { |
||||||
|
return Status::Corruption("Record Checksum mismatch: header_cksum"); |
||||||
|
} |
||||||
|
|
||||||
|
record->ResizeBlobBuffer(record->GetBlobSize()); |
||||||
|
status = file_->Read(record->GetBlobSize(), &record->blob_, |
||||||
|
record->GetBlobBuffer()); |
||||||
|
next_byte_ += record->blob_.size(); |
||||||
|
if (!status.ok()) return status; |
||||||
|
if (record->blob_.size() != record->GetBlobSize()) { |
||||||
|
return Status::IOError("EOF reached during blob read"); |
||||||
|
} |
||||||
|
|
||||||
|
blob_crc = |
||||||
|
crc32c::Extend(blob_crc, record->blob_.data(), record->blob_.size()); |
||||||
|
blob_crc = crc32c::Mask(blob_crc); |
||||||
|
if (blob_crc != record->checksum_) { |
||||||
|
return Status::Corruption("Blob Checksum mismatch"); |
||||||
|
} |
||||||
|
|
||||||
|
status = |
||||||
|
file_->Read(BlobLogRecord::kFooterSize, &buffer_, GetReadBuffer()); |
||||||
|
next_byte_ += buffer_.size(); |
||||||
|
if (!status.ok()) return status; |
||||||
|
if (buffer_.size() != BlobLogRecord::kFooterSize) { |
||||||
|
return Status::IOError("EOF reached during blob footer read"); |
||||||
|
} |
||||||
|
|
||||||
|
status = record->DecodeFooterFrom(buffer_); |
||||||
|
return status; |
||||||
|
default: |
||||||
|
assert(0); |
||||||
|
return status; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace blob_db
|
||||||
|
} // namespace rocksdb
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
@ -0,0 +1,93 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
|
||||||
|
#include <cstdint> |
||||||
|
#include <memory> |
||||||
|
#include <string> |
||||||
|
|
||||||
|
#include "rocksdb/options.h" |
||||||
|
#include "rocksdb/slice.h" |
||||||
|
#include "rocksdb/status.h" |
||||||
|
#include "utilities/blob_db/blob_log_format.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class SequentialFileReader; |
||||||
|
class Logger; |
||||||
|
|
||||||
|
namespace blob_db { |
||||||
|
|
||||||
|
/**
|
||||||
|
* Reader is a general purpose log stream reader implementation. The actual job |
||||||
|
* of reading from the device is implemented by the SequentialFile interface. |
||||||
|
* |
||||||
|
* Please see Writer for details on the file and record layout. |
||||||
|
*/ |
||||||
|
class Reader { |
||||||
|
public: |
||||||
|
enum ReadLevel { |
||||||
|
kReadHdrFooter, |
||||||
|
kReadHdrKeyFooter, |
||||||
|
kReadHdrKeyBlobFooter, |
||||||
|
}; |
||||||
|
|
||||||
|
// Create a reader that will return log records from "*file".
|
||||||
|
// "*file" must remain live while this Reader is in use.
|
||||||
|
//
|
||||||
|
// If "reporter" is non-nullptr, it is notified whenever some data is
|
||||||
|
// dropped due to a detected corruption. "*reporter" must remain
|
||||||
|
// live while this Reader is in use.
|
||||||
|
//
|
||||||
|
// If "checksum" is true, verify checksums if available.
|
||||||
|
//
|
||||||
|
// The Reader will start reading at the first record located at physical
|
||||||
|
// position >= initial_offset within the file.
|
||||||
|
Reader(std::shared_ptr<Logger> info_log, |
||||||
|
std::unique_ptr<SequentialFileReader>&& file); |
||||||
|
|
||||||
|
~Reader(); |
||||||
|
|
||||||
|
Status ReadHeader(BlobLogHeader* header); |
||||||
|
|
||||||
|
// Read the next record into *record. Returns true if read
|
||||||
|
// successfully, false if we hit end of the input. May use
|
||||||
|
// "*scratch" as temporary storage. The contents filled in *record
|
||||||
|
// will only be valid until the next mutating operation on this
|
||||||
|
// reader or the next mutation to *scratch.
|
||||||
|
Status ReadRecord(BlobLogRecord* record, ReadLevel level = kReadHdrFooter, |
||||||
|
WALRecoveryMode wal_recovery_mode = |
||||||
|
WALRecoveryMode::kTolerateCorruptedTailRecords); |
||||||
|
|
||||||
|
SequentialFileReader* file() { return file_.get(); } |
||||||
|
|
||||||
|
void ResetNextByte() { next_byte_ = 0; } |
||||||
|
|
||||||
|
uint64_t GetNextByte() const { return next_byte_; } |
||||||
|
|
||||||
|
private: |
||||||
|
char* GetReadBuffer() { return &(backing_store_[0]); } |
||||||
|
|
||||||
|
private: |
||||||
|
std::shared_ptr<Logger> info_log_; |
||||||
|
const std::unique_ptr<SequentialFileReader> file_; |
||||||
|
|
||||||
|
std::string backing_store_; |
||||||
|
Slice buffer_; |
||||||
|
|
||||||
|
// which byte to read next. For asserting proper usage
|
||||||
|
uint64_t next_byte_; |
||||||
|
|
||||||
|
// No copying allowed
|
||||||
|
Reader(const Reader&) = delete; |
||||||
|
Reader& operator=(const Reader&) = delete; |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace blob_db
|
||||||
|
} // namespace rocksdb
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
@ -0,0 +1,172 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
|
||||||
|
#include "utilities/blob_db/blob_log_writer.h" |
||||||
|
|
||||||
|
#include <cstdint> |
||||||
|
#include <limits> |
||||||
|
#include <string> |
||||||
|
#include "rocksdb/env.h" |
||||||
|
#include "util/coding.h" |
||||||
|
#include "util/crc32c.h" |
||||||
|
#include "util/file_reader_writer.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
namespace blob_db { |
||||||
|
|
||||||
|
Writer::Writer(unique_ptr<WritableFileWriter>&& dest, uint64_t log_number, |
||||||
|
uint64_t bpsync, bool use_fs, uint64_t boffset) |
||||||
|
: dest_(std::move(dest)), |
||||||
|
log_number_(log_number), |
||||||
|
block_offset_(boffset), |
||||||
|
bytes_per_sync_(bpsync), |
||||||
|
next_sync_offset_(0), |
||||||
|
use_fsync_(use_fs), |
||||||
|
last_elem_type_(kEtNone) { |
||||||
|
for (int i = 0; i <= kMaxRecordType; i++) { |
||||||
|
char t = static_cast<char>(i); |
||||||
|
type_crc_[i] = crc32c::Value(&t, 1); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
Writer::~Writer() {} |
||||||
|
|
||||||
|
void Writer::Sync() { dest_->Sync(use_fsync_); } |
||||||
|
|
||||||
|
Status Writer::WriteHeader(const BlobLogHeader& header) { |
||||||
|
assert(block_offset_ == 0); |
||||||
|
assert(last_elem_type_ == kEtNone); |
||||||
|
std::string str; |
||||||
|
header.EncodeTo(&str); |
||||||
|
|
||||||
|
Status s = dest_->Append(Slice(str)); |
||||||
|
if (s.ok()) { |
||||||
|
block_offset_ += str.size(); |
||||||
|
s = dest_->Flush(); |
||||||
|
} |
||||||
|
last_elem_type_ = kEtFileHdr; |
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
Status Writer::AppendFooter(const BlobLogFooter& footer) { |
||||||
|
assert(block_offset_ != 0); |
||||||
|
assert(last_elem_type_ == kEtFileHdr || last_elem_type_ == kEtFooter); |
||||||
|
|
||||||
|
std::string str; |
||||||
|
footer.EncodeTo(&str); |
||||||
|
|
||||||
|
Status s = dest_->Append(Slice(str)); |
||||||
|
if (s.ok()) { |
||||||
|
block_offset_ += str.size(); |
||||||
|
s = dest_->Close(); |
||||||
|
dest_.reset(); |
||||||
|
} |
||||||
|
|
||||||
|
last_elem_type_ = kEtFileFooter; |
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
Status Writer::AddRecord(const Slice& key, const Slice& val, |
||||||
|
uint64_t* key_offset, uint64_t* blob_offset, |
||||||
|
uint32_t ttl) { |
||||||
|
assert(block_offset_ != 0); |
||||||
|
assert(last_elem_type_ == kEtFileHdr || last_elem_type_ == kEtFooter); |
||||||
|
|
||||||
|
std::string buf; |
||||||
|
ConstructBlobHeader(&buf, key, val, ttl, -1); |
||||||
|
|
||||||
|
Status s = EmitPhysicalRecord(buf, key, val, key_offset, blob_offset); |
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
Status Writer::AddRecord(const Slice& key, const Slice& val, |
||||||
|
uint64_t* key_offset, uint64_t* blob_offset) { |
||||||
|
assert(block_offset_ != 0); |
||||||
|
assert(last_elem_type_ == kEtFileHdr || last_elem_type_ == kEtFooter); |
||||||
|
|
||||||
|
std::string buf; |
||||||
|
ConstructBlobHeader(&buf, key, val, -1, -1); |
||||||
|
|
||||||
|
Status s = EmitPhysicalRecord(buf, key, val, key_offset, blob_offset); |
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
void Writer::ConstructBlobHeader(std::string* headerbuf, const Slice& key, |
||||||
|
const Slice& val, int32_t ttl, int64_t ts) { |
||||||
|
headerbuf->reserve(BlobLogRecord::kHeaderSize); |
||||||
|
|
||||||
|
uint32_t key_size = static_cast<uint32_t>(key.size()); |
||||||
|
PutFixed32(headerbuf, key_size); |
||||||
|
PutFixed64(headerbuf, val.size()); |
||||||
|
|
||||||
|
uint32_t ttl_write = (ttl != -1) ? static_cast<uint32_t>(ttl) |
||||||
|
: std::numeric_limits<uint32_t>::max(); |
||||||
|
PutFixed32(headerbuf, ttl_write); |
||||||
|
|
||||||
|
uint64_t ts_write = (ts != -1) ? static_cast<uint64_t>(ts) |
||||||
|
: std::numeric_limits<uint64_t>::max(); |
||||||
|
PutFixed64(headerbuf, ts_write); |
||||||
|
|
||||||
|
RecordType t = kFullType; |
||||||
|
headerbuf->push_back(static_cast<char>(t)); |
||||||
|
|
||||||
|
RecordSubType st = kRegularType; |
||||||
|
if (ttl != -1) st = kTTLType; |
||||||
|
headerbuf->push_back(static_cast<char>(st)); |
||||||
|
|
||||||
|
uint32_t header_crc = 0; |
||||||
|
header_crc = |
||||||
|
crc32c::Extend(header_crc, headerbuf->c_str(), headerbuf->size()); |
||||||
|
header_crc = crc32c::Extend(header_crc, key.data(), key.size()); |
||||||
|
header_crc = crc32c::Mask(header_crc); |
||||||
|
PutFixed32(headerbuf, header_crc); |
||||||
|
|
||||||
|
uint32_t crc = 0; |
||||||
|
// Compute the crc of the record type and the payload.
|
||||||
|
crc = crc32c::Extend(crc, val.data(), val.size()); |
||||||
|
crc = crc32c::Mask(crc); // Adjust for storage
|
||||||
|
PutFixed32(headerbuf, crc); |
||||||
|
} |
||||||
|
|
||||||
|
Status Writer::EmitPhysicalRecord(const std::string& headerbuf, |
||||||
|
const Slice& key, const Slice& val, |
||||||
|
uint64_t* key_offset, uint64_t* blob_offset) { |
||||||
|
Status s = dest_->Append(Slice(headerbuf)); |
||||||
|
if (s.ok()) { |
||||||
|
s = dest_->Append(key); |
||||||
|
if (s.ok()) s = dest_->Append(val); |
||||||
|
} |
||||||
|
|
||||||
|
*key_offset = block_offset_ + BlobLogRecord::kHeaderSize; |
||||||
|
*blob_offset = *key_offset + key.size(); |
||||||
|
block_offset_ = *blob_offset + val.size(); |
||||||
|
last_elem_type_ = kEtRecord; |
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
Status Writer::AddRecordFooter(const SequenceNumber& seq) { |
||||||
|
assert(last_elem_type_ == kEtRecord); |
||||||
|
|
||||||
|
std::string buf; |
||||||
|
PutFixed64(&buf, seq); |
||||||
|
|
||||||
|
uint32_t footer_crc = crc32c::Extend(0, buf.c_str(), buf.size()); |
||||||
|
footer_crc = crc32c::Mask(footer_crc); |
||||||
|
PutFixed32(&buf, footer_crc); |
||||||
|
|
||||||
|
Status s = dest_->Append(Slice(buf)); |
||||||
|
block_offset_ += BlobLogRecord::kFooterSize; |
||||||
|
|
||||||
|
if (s.ok()) dest_->Flush(); |
||||||
|
|
||||||
|
last_elem_type_ = kEtFooter; |
||||||
|
return s; |
||||||
|
} |
||||||
|
|
||||||
|
} // namespace blob_db
|
||||||
|
} // namespace rocksdb
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
@ -0,0 +1,98 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#ifndef ROCKSDB_LITE |
||||||
|
|
||||||
|
#include <cstdint> |
||||||
|
#include <memory> |
||||||
|
#include <string> |
||||||
|
|
||||||
|
#include "rocksdb/slice.h" |
||||||
|
#include "rocksdb/status.h" |
||||||
|
#include "rocksdb/types.h" |
||||||
|
#include "utilities/blob_db/blob_log_format.h" |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
|
||||||
|
class WritableFileWriter; |
||||||
|
|
||||||
|
namespace blob_db { |
||||||
|
|
||||||
|
/**
|
||||||
|
* Writer is the blob log stream writer. It provides an append-only |
||||||
|
* abstraction for writing blob data. |
||||||
|
* |
||||||
|
* |
||||||
|
* Look at blob_db_format.h to see the details of the record formats. |
||||||
|
*/ |
||||||
|
|
||||||
|
class Writer { |
||||||
|
public: |
||||||
|
// Create a writer that will append data to "*dest".
|
||||||
|
// "*dest" must be initially empty.
|
||||||
|
// "*dest" must remain live while this Writer is in use.
|
||||||
|
explicit Writer(std::unique_ptr<WritableFileWriter>&& dest, |
||||||
|
uint64_t log_number, uint64_t bpsync, bool use_fsync, |
||||||
|
uint64_t boffset = 0); |
||||||
|
~Writer(); |
||||||
|
|
||||||
|
static void ConstructBlobHeader(std::string* headerbuf, const Slice& key, |
||||||
|
const Slice& val, int32_t ttl, int64_t ts); |
||||||
|
|
||||||
|
Status AddRecord(const Slice& key, const Slice& val, uint64_t* key_offset, |
||||||
|
uint64_t* blob_offset); |
||||||
|
|
||||||
|
Status AddRecord(const Slice& key, const Slice& val, uint64_t* key_offset, |
||||||
|
uint64_t* blob_offset, uint32_t ttl); |
||||||
|
|
||||||
|
Status EmitPhysicalRecord(const std::string& headerbuf, const Slice& key, |
||||||
|
const Slice& val, uint64_t* key_offset, |
||||||
|
uint64_t* blob_offset); |
||||||
|
|
||||||
|
Status AddRecordFooter(const SequenceNumber& sn); |
||||||
|
|
||||||
|
Status AppendFooter(const BlobLogFooter& footer); |
||||||
|
|
||||||
|
Status WriteHeader(const BlobLogHeader& header); |
||||||
|
|
||||||
|
WritableFileWriter* file() { return dest_.get(); } |
||||||
|
|
||||||
|
const WritableFileWriter* file() const { return dest_.get(); } |
||||||
|
|
||||||
|
uint64_t get_log_number() const { return log_number_; } |
||||||
|
|
||||||
|
bool ShouldSync() const { return block_offset_ > next_sync_offset_; } |
||||||
|
|
||||||
|
void Sync(); |
||||||
|
|
||||||
|
void ResetSyncPointer() { next_sync_offset_ += bytes_per_sync_; } |
||||||
|
|
||||||
|
private: |
||||||
|
std::unique_ptr<WritableFileWriter> dest_; |
||||||
|
uint64_t log_number_; |
||||||
|
uint64_t block_offset_; // Current offset in block
|
||||||
|
uint64_t bytes_per_sync_; |
||||||
|
uint64_t next_sync_offset_; |
||||||
|
bool use_fsync_; |
||||||
|
|
||||||
|
// crc32c values for all supported record types. These are
|
||||||
|
// pre-computed to reduce the overhead of computing the crc of the
|
||||||
|
// record type stored in the header.
|
||||||
|
uint32_t type_crc_[kMaxRecordType + 1]; |
||||||
|
|
||||||
|
// No copying allowed
|
||||||
|
Writer(const Writer&) = delete; |
||||||
|
Writer& operator=(const Writer&) = delete; |
||||||
|
|
||||||
|
public: |
||||||
|
enum ElemType { kEtNone, kEtFileHdr, kEtRecord, kEtFooter, kEtFileFooter }; |
||||||
|
ElemType last_elem_type_; |
||||||
|
}; |
||||||
|
|
||||||
|
} // namespace blob_db
|
||||||
|
} // namespace rocksdb
|
||||||
|
#endif // ROCKSDB_LITE
|
||||||
Loading…
Reference in new issue