|
|
|
|
@ -3,16 +3,18 @@ |
|
|
|
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
|
|
|
|
#ifndef ROCKSDB_LITE |
|
|
|
|
|
|
|
|
|
#include "utilities/ttl/db_ttl.h" |
|
|
|
|
#include "utilities/ttl/db_ttl_impl.h" |
|
|
|
|
|
|
|
|
|
#include "utilities/db_ttl.h" |
|
|
|
|
#include "db/filename.h" |
|
|
|
|
#include "db/write_batch_internal.h" |
|
|
|
|
#include "util/coding.h" |
|
|
|
|
#include "include/rocksdb/env.h" |
|
|
|
|
#include "include/rocksdb/iterator.h" |
|
|
|
|
#include "rocksdb/env.h" |
|
|
|
|
#include "rocksdb/iterator.h" |
|
|
|
|
|
|
|
|
|
namespace rocksdb { |
|
|
|
|
|
|
|
|
|
void DBWithTTL::SanitizeOptions(int32_t ttl, ColumnFamilyOptions* options) { |
|
|
|
|
void DBWithTTLImpl::SanitizeOptions(int32_t ttl, ColumnFamilyOptions* options) { |
|
|
|
|
if (options->compaction_filter) { |
|
|
|
|
options->compaction_filter = |
|
|
|
|
new TtlCompactionFilter(ttl, options->compaction_filter); |
|
|
|
|
@ -28,19 +30,25 @@ void DBWithTTL::SanitizeOptions(int32_t ttl, ColumnFamilyOptions* options) { |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// Open the db inside DBWithTTL because options needs pointer to its ttl
|
|
|
|
|
DBWithTTL::DBWithTTL(DB* db) : StackableDB(db) {} |
|
|
|
|
// Open the db inside DBWithTTLImpl because options needs pointer to its ttl
|
|
|
|
|
DBWithTTLImpl::DBWithTTLImpl(DB* db) : DBWithTTL(db) {} |
|
|
|
|
|
|
|
|
|
DBWithTTLImpl::~DBWithTTLImpl() { delete GetOptions().compaction_filter; } |
|
|
|
|
|
|
|
|
|
DBWithTTL::~DBWithTTL() { |
|
|
|
|
delete GetOptions().compaction_filter; |
|
|
|
|
Status UtilityDB::OpenTtlDB(const Options& options, const std::string& dbname, |
|
|
|
|
StackableDB** dbptr, int32_t ttl, bool read_only) { |
|
|
|
|
DBWithTTL* db; |
|
|
|
|
Status s = DBWithTTL::Open(options, dbname, &db, ttl, read_only); |
|
|
|
|
if (s.ok()) { |
|
|
|
|
*dbptr = db; |
|
|
|
|
} else { |
|
|
|
|
*dbptr = nullptr; |
|
|
|
|
} |
|
|
|
|
return s; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
Status UtilityDB::OpenTtlDB( |
|
|
|
|
const Options& options, |
|
|
|
|
const std::string& dbname, |
|
|
|
|
StackableDB** dbptr, |
|
|
|
|
int32_t ttl, |
|
|
|
|
bool read_only) { |
|
|
|
|
Status DBWithTTL::Open(const Options& options, const std::string& dbname, |
|
|
|
|
DBWithTTL** dbptr, int32_t ttl, bool read_only) { |
|
|
|
|
|
|
|
|
|
DBOptions db_options(options); |
|
|
|
|
ColumnFamilyOptions cf_options(options); |
|
|
|
|
@ -48,8 +56,8 @@ Status UtilityDB::OpenTtlDB( |
|
|
|
|
column_families.push_back( |
|
|
|
|
ColumnFamilyDescriptor(kDefaultColumnFamilyName, cf_options)); |
|
|
|
|
std::vector<ColumnFamilyHandle*> handles; |
|
|
|
|
Status s = UtilityDB::OpenTtlDB(db_options, dbname, column_families, &handles, |
|
|
|
|
dbptr, {ttl}, read_only); |
|
|
|
|
Status s = DBWithTTL::Open(db_options, dbname, column_families, &handles, |
|
|
|
|
dbptr, {ttl}, read_only); |
|
|
|
|
if (s.ok()) { |
|
|
|
|
assert(handles.size() == 1); |
|
|
|
|
// i can delete the handle since DBImpl is always holding a reference to
|
|
|
|
|
@ -59,10 +67,10 @@ Status UtilityDB::OpenTtlDB( |
|
|
|
|
return s; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
Status UtilityDB::OpenTtlDB( |
|
|
|
|
Status DBWithTTL::Open( |
|
|
|
|
const DBOptions& db_options, const std::string& dbname, |
|
|
|
|
const std::vector<ColumnFamilyDescriptor>& column_families, |
|
|
|
|
std::vector<ColumnFamilyHandle*>* handles, StackableDB** dbptr, |
|
|
|
|
std::vector<ColumnFamilyHandle*>* handles, DBWithTTL** dbptr, |
|
|
|
|
std::vector<int32_t> ttls, bool read_only) { |
|
|
|
|
|
|
|
|
|
if (ttls.size() != column_families.size()) { |
|
|
|
|
@ -73,7 +81,8 @@ Status UtilityDB::OpenTtlDB( |
|
|
|
|
std::vector<ColumnFamilyDescriptor> column_families_sanitized = |
|
|
|
|
column_families; |
|
|
|
|
for (size_t i = 0; i < column_families_sanitized.size(); ++i) { |
|
|
|
|
DBWithTTL::SanitizeOptions(ttls[i], &column_families_sanitized[i].options); |
|
|
|
|
DBWithTTLImpl::SanitizeOptions(ttls[i], |
|
|
|
|
&column_families_sanitized[i].options); |
|
|
|
|
} |
|
|
|
|
DB* db; |
|
|
|
|
|
|
|
|
|
@ -85,66 +94,81 @@ Status UtilityDB::OpenTtlDB( |
|
|
|
|
st = DB::Open(db_options, dbname, column_families_sanitized, handles, &db); |
|
|
|
|
} |
|
|
|
|
if (st.ok()) { |
|
|
|
|
*dbptr = new DBWithTTL(db); |
|
|
|
|
*dbptr = new DBWithTTLImpl(db); |
|
|
|
|
} else { |
|
|
|
|
*dbptr = nullptr; |
|
|
|
|
} |
|
|
|
|
return st; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
Status DBWithTTLImpl::CreateColumnFamilyWithTtl( |
|
|
|
|
const ColumnFamilyOptions& options, const std::string& column_family_name, |
|
|
|
|
ColumnFamilyHandle** handle, int ttl) { |
|
|
|
|
ColumnFamilyOptions sanitized_options = options; |
|
|
|
|
DBWithTTLImpl::SanitizeOptions(ttl, &sanitized_options); |
|
|
|
|
|
|
|
|
|
return DBWithTTL::CreateColumnFamily(sanitized_options, column_family_name, |
|
|
|
|
handle); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
Status DBWithTTLImpl::CreateColumnFamily(const ColumnFamilyOptions& options, |
|
|
|
|
const std::string& column_family_name, |
|
|
|
|
ColumnFamilyHandle** handle) { |
|
|
|
|
return CreateColumnFamilyWithTtl(options, column_family_name, handle, 0); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// Gives back the current time
|
|
|
|
|
Status DBWithTTL::GetCurrentTime(int64_t& curtime) { |
|
|
|
|
return Env::Default()->GetCurrentTime(&curtime); |
|
|
|
|
Status DBWithTTLImpl::GetCurrentTime(int64_t* curtime) { |
|
|
|
|
return Env::Default()->GetCurrentTime(curtime); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// Appends the current timestamp to the string.
|
|
|
|
|
// Returns false if could not get the current_time, true if append succeeds
|
|
|
|
|
Status DBWithTTL::AppendTS(const Slice& val, std::string& val_with_ts) { |
|
|
|
|
val_with_ts.reserve(kTSLength + val.size()); |
|
|
|
|
Status DBWithTTLImpl::AppendTS(const Slice& val, std::string* val_with_ts) { |
|
|
|
|
val_with_ts->reserve(kTSLength + val.size()); |
|
|
|
|
char ts_string[kTSLength]; |
|
|
|
|
int64_t curtime; |
|
|
|
|
Status st = GetCurrentTime(curtime); |
|
|
|
|
Status st = GetCurrentTime(&curtime); |
|
|
|
|
if (!st.ok()) { |
|
|
|
|
return st; |
|
|
|
|
} |
|
|
|
|
EncodeFixed32(ts_string, (int32_t)curtime); |
|
|
|
|
val_with_ts.append(val.data(), val.size()); |
|
|
|
|
val_with_ts.append(ts_string, kTSLength); |
|
|
|
|
val_with_ts->append(val.data(), val.size()); |
|
|
|
|
val_with_ts->append(ts_string, kTSLength); |
|
|
|
|
return st; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// Returns corruption if the length of the string is lesser than timestamp, or
|
|
|
|
|
// timestamp refers to a time lesser than ttl-feature release time
|
|
|
|
|
Status DBWithTTL::SanityCheckTimestamp(const Slice& str) { |
|
|
|
|
Status DBWithTTLImpl::SanityCheckTimestamp(const Slice& str) { |
|
|
|
|
if (str.size() < kTSLength) { |
|
|
|
|
return Status::Corruption("Error: value's length less than timestamp's\n"); |
|
|
|
|
} |
|
|
|
|
// Checks that TS is not lesser than kMinTimestamp
|
|
|
|
|
// Gaurds against corruption & normal database opened incorrectly in ttl mode
|
|
|
|
|
int32_t timestamp_value = |
|
|
|
|
DecodeFixed32(str.data() + str.size() - kTSLength); |
|
|
|
|
if (timestamp_value < kMinTimestamp){ |
|
|
|
|
int32_t timestamp_value = DecodeFixed32(str.data() + str.size() - kTSLength); |
|
|
|
|
if (timestamp_value < kMinTimestamp) { |
|
|
|
|
return Status::Corruption("Error: Timestamp < ttl feature release time!\n"); |
|
|
|
|
} |
|
|
|
|
return Status::OK(); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// Checks if the string is stale or not according to TTl provided
|
|
|
|
|
bool DBWithTTL::IsStale(const Slice& value, int32_t ttl) { |
|
|
|
|
if (ttl <= 0) { // Data is fresh if TTL is non-positive
|
|
|
|
|
bool DBWithTTLImpl::IsStale(const Slice& value, int32_t ttl) { |
|
|
|
|
if (ttl <= 0) { // Data is fresh if TTL is non-positive
|
|
|
|
|
return false; |
|
|
|
|
} |
|
|
|
|
int64_t curtime; |
|
|
|
|
if (!GetCurrentTime(curtime).ok()) { |
|
|
|
|
return false; // Treat the data as fresh if could not get current time
|
|
|
|
|
if (!GetCurrentTime(&curtime).ok()) { |
|
|
|
|
return false; // Treat the data as fresh if could not get current time
|
|
|
|
|
} |
|
|
|
|
int32_t timestamp_value = |
|
|
|
|
DecodeFixed32(value.data() + value.size() - kTSLength); |
|
|
|
|
DecodeFixed32(value.data() + value.size() - kTSLength); |
|
|
|
|
return (timestamp_value + ttl) < curtime; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// Strips the TS from the end of the string
|
|
|
|
|
Status DBWithTTL::StripTS(std::string* str) { |
|
|
|
|
Status DBWithTTLImpl::StripTS(std::string* str) { |
|
|
|
|
Status st; |
|
|
|
|
if (str->length() < kTSLength) { |
|
|
|
|
return Status::Corruption("Bad timestamp in key-value"); |
|
|
|
|
@ -154,17 +178,17 @@ Status DBWithTTL::StripTS(std::string* str) { |
|
|
|
|
return st; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
Status DBWithTTL::Put(const WriteOptions& options, |
|
|
|
|
ColumnFamilyHandle* column_family, const Slice& key, |
|
|
|
|
const Slice& val) { |
|
|
|
|
Status DBWithTTLImpl::Put(const WriteOptions& options, |
|
|
|
|
ColumnFamilyHandle* column_family, const Slice& key, |
|
|
|
|
const Slice& val) { |
|
|
|
|
WriteBatch batch; |
|
|
|
|
batch.Put(column_family, key, val); |
|
|
|
|
return Write(options, &batch); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
Status DBWithTTL::Get(const ReadOptions& options, |
|
|
|
|
ColumnFamilyHandle* column_family, const Slice& key, |
|
|
|
|
std::string* value) { |
|
|
|
|
Status DBWithTTLImpl::Get(const ReadOptions& options, |
|
|
|
|
ColumnFamilyHandle* column_family, const Slice& key, |
|
|
|
|
std::string* value) { |
|
|
|
|
Status st = db_->Get(options, column_family, key, value); |
|
|
|
|
if (!st.ok()) { |
|
|
|
|
return st; |
|
|
|
|
@ -176,18 +200,18 @@ Status DBWithTTL::Get(const ReadOptions& options, |
|
|
|
|
return StripTS(value); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
std::vector<Status> DBWithTTL::MultiGet( |
|
|
|
|
std::vector<Status> DBWithTTLImpl::MultiGet( |
|
|
|
|
const ReadOptions& options, |
|
|
|
|
const std::vector<ColumnFamilyHandle*>& column_family, |
|
|
|
|
const std::vector<Slice>& keys, std::vector<std::string>* values) { |
|
|
|
|
return std::vector<Status>(keys.size(), |
|
|
|
|
Status::NotSupported("MultiGet not\
|
|
|
|
|
supported with TTL")); |
|
|
|
|
return std::vector<Status>( |
|
|
|
|
keys.size(), Status::NotSupported("MultiGet not supported with TTL")); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
bool DBWithTTL::KeyMayExist(const ReadOptions& options, |
|
|
|
|
ColumnFamilyHandle* column_family, const Slice& key, |
|
|
|
|
std::string* value, bool* value_found) { |
|
|
|
|
bool DBWithTTLImpl::KeyMayExist(const ReadOptions& options, |
|
|
|
|
ColumnFamilyHandle* column_family, |
|
|
|
|
const Slice& key, std::string* value, |
|
|
|
|
bool* value_found) { |
|
|
|
|
bool ret = db_->KeyMayExist(options, column_family, key, value, value_found); |
|
|
|
|
if (ret && value != nullptr && value_found != nullptr && *value_found) { |
|
|
|
|
if (!SanityCheckTimestamp(*value).ok() || !StripTS(value).ok()) { |
|
|
|
|
@ -197,15 +221,15 @@ bool DBWithTTL::KeyMayExist(const ReadOptions& options, |
|
|
|
|
return ret; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
Status DBWithTTL::Merge(const WriteOptions& options, |
|
|
|
|
ColumnFamilyHandle* column_family, const Slice& key, |
|
|
|
|
const Slice& value) { |
|
|
|
|
Status DBWithTTLImpl::Merge(const WriteOptions& options, |
|
|
|
|
ColumnFamilyHandle* column_family, const Slice& key, |
|
|
|
|
const Slice& value) { |
|
|
|
|
WriteBatch batch; |
|
|
|
|
batch.Merge(column_family, key, value); |
|
|
|
|
return Write(options, &batch); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
Status DBWithTTL::Write(const WriteOptions& opts, WriteBatch* updates) { |
|
|
|
|
Status DBWithTTLImpl::Write(const WriteOptions& opts, WriteBatch* updates) { |
|
|
|
|
class Handler : public WriteBatch::Handler { |
|
|
|
|
public: |
|
|
|
|
WriteBatch updates_ttl; |
|
|
|
|
@ -213,7 +237,7 @@ Status DBWithTTL::Write(const WriteOptions& opts, WriteBatch* updates) { |
|
|
|
|
virtual Status PutCF(uint32_t column_family_id, const Slice& key, |
|
|
|
|
const Slice& value) { |
|
|
|
|
std::string value_with_ts; |
|
|
|
|
Status st = AppendTS(value, value_with_ts); |
|
|
|
|
Status st = AppendTS(value, &value_with_ts); |
|
|
|
|
if (!st.ok()) { |
|
|
|
|
batch_rewrite_status = st; |
|
|
|
|
} else { |
|
|
|
|
@ -225,7 +249,7 @@ Status DBWithTTL::Write(const WriteOptions& opts, WriteBatch* updates) { |
|
|
|
|
virtual Status MergeCF(uint32_t column_family_id, const Slice& key, |
|
|
|
|
const Slice& value) { |
|
|
|
|
std::string value_with_ts; |
|
|
|
|
Status st = AppendTS(value, value_with_ts); |
|
|
|
|
Status st = AppendTS(value, &value_with_ts); |
|
|
|
|
if (!st.ok()) { |
|
|
|
|
batch_rewrite_status = st; |
|
|
|
|
} else { |
|
|
|
|
@ -238,9 +262,7 @@ Status DBWithTTL::Write(const WriteOptions& opts, WriteBatch* updates) { |
|
|
|
|
WriteBatchInternal::Delete(&updates_ttl, column_family_id, key); |
|
|
|
|
return Status::OK(); |
|
|
|
|
} |
|
|
|
|
virtual void LogData(const Slice& blob) { |
|
|
|
|
updates_ttl.PutLogData(blob); |
|
|
|
|
} |
|
|
|
|
virtual void LogData(const Slice& blob) { updates_ttl.PutLogData(blob); } |
|
|
|
|
}; |
|
|
|
|
Handler handler; |
|
|
|
|
updates->Iterate(&handler); |
|
|
|
|
@ -251,8 +273,8 @@ Status DBWithTTL::Write(const WriteOptions& opts, WriteBatch* updates) { |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
Iterator* DBWithTTL::NewIterator(const ReadOptions& opts, |
|
|
|
|
ColumnFamilyHandle* column_family) { |
|
|
|
|
Iterator* DBWithTTLImpl::NewIterator(const ReadOptions& opts, |
|
|
|
|
ColumnFamilyHandle* column_family) { |
|
|
|
|
return new TtlIterator(db_->NewIterator(opts, column_family)); |
|
|
|
|
} |
|
|
|
|
|
Ankit Gupta
11 years ago