NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Simplify bucketing logic in ldb-ttl

Browse Source 
Summary: [start_time, end_time) is waht I'm following for the buckets and the whole time-range. Also cleaned up some code in db_ttl.* Not correcting the spacing/indenting convention for util/ldb_cmd.cc in this diff.

Test Plan: python ldb_test.py, make ttl_test, Run mcrocksdb-backup tool, Run the ldb tool on 2 mcrocksdb production backups form sigmafio033.prn1

Reviewers: vamsi, haobo

Reviewed By: vamsi

Differential Revision: https://reviews.facebook.net/D11433
main
Mayank Agarwal 12 years ago
parent 61f1baaedf
commit b858da709a
3 changed files with 35 additions and 59 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 61
      util/ldb_cmd.cc
  2. 6
      utilities/ttl/db_ttl.cc
  3. 27
      utilities/ttl/db_ttl.h

61
util/ldb_cmd.cc
Unescape View File

@ -538,20 +538,16 @@ string ReadableTime(int unixtime) {
// This function only called when it's the sane case of >1 buckets in time-range // This function only called when it's the sane case of >1 buckets in time-range
// Also called only when timekv falls between ttl_start and ttl_end provided // Also called only when timekv falls between ttl_start and ttl_end provided
void IncBucketCounts(uint64_t* bucket_counts, int ttl_start, int time_range, void IncBucketCounts(vector<uint64_t>& bucket_counts, int ttl_start,
int bucket_size, int timekv, int num_buckets) { int time_range, int bucket_size, int timekv, int num_buckets) {
if (time_range <= 0 || timekv < ttl_start || timekv > (ttl_start + time_range) assert(time_range > 0 && timekv >= ttl_start && bucket_size > 0 &&
|| bucket_size <= 0 || num_buckets < 2) { timekv < (ttl_start + time_range) && num_buckets > 1);
fprintf(stderr, "Error: bucketizing\n"); int bucket = (timekv - ttl_start) / bucket_size;
return; bucket_counts[bucket]++;
}
int bucket = (timekv - ttl_start);
bucket = (bucket == 0) ? 1 : ceil(bucket / (double)bucket_size);
bucket_counts[bucket - 1]++;
} }
void PrintBucketCounts(uint64_t* bucket_counts, int ttl_start, int ttl_end, void PrintBucketCounts(const vector<uint64_t>& bucket_counts, int ttl_start,
int bucket_size, int num_buckets) { int ttl_end, int bucket_size, int num_buckets) {
int time_point = ttl_start; int time_point = ttl_start;
for(int i = 0; i < num_buckets - 1; i++, time_point += bucket_size) { for(int i = 0; i < num_buckets - 1; i++, time_point += bucket_size) {
fprintf(stdout, "Keys in range %s to %s : %lu\n", fprintf(stdout, "Keys in range %s to %s : %lu\n",
@ -629,8 +625,8 @@ void DBDumperCommand::Help(string& ret) {
ret.append(" [--" + ARG_COUNT_ONLY + "]"); ret.append(" [--" + ARG_COUNT_ONLY + "]");
ret.append(" [--" + ARG_STATS + "]"); ret.append(" [--" + ARG_STATS + "]");
ret.append(" [--" + ARG_TTL_BUCKET + "=<N>]"); ret.append(" [--" + ARG_TTL_BUCKET + "=<N>]");
ret.append(" [--" + ARG_TTL_START + "=<N>]"); ret.append(" [--" + ARG_TTL_START + "=<N>:- is inclusive]");
ret.append(" [--" + ARG_TTL_END + "=<N>]"); ret.append(" [--" + ARG_TTL_END + "=<N>:- is exclusive]");
ret.append("\n"); ret.append("\n");
} }
@ -682,10 +678,9 @@ void DBDumperCommand::DoCommand() {
bucket_size = time_range; // Will have just 1 bucket by default bucket_size = time_range; // Will have just 1 bucket by default
} }
// At this point, bucket_size=0 => time_range=0 // At this point, bucket_size=0 => time_range=0
uint64_t num_buckets = uint64_t num_buckets = (bucket_size >= time_range) ? 1 :
bucket_size >= time_range ? 1 : ceil((double)time_range/bucket_size); ((time_range + bucket_size - 1) / bucket_size);
unique_ptr<uint64_t[]> bucket_counts(new uint64_t[num_buckets]); vector<uint64_t> bucket_counts(num_buckets, 0);
fill(bucket_counts.get(), bucket_counts.get() + num_buckets, 0);
if (is_db_ttl_ && !count_only_ && timestamp_) { if (is_db_ttl_ && !count_only_ && timestamp_) {
fprintf(stdout, "Dumping key-values from %s to %s\n", fprintf(stdout, "Dumping key-values from %s to %s\n",
ReadableTime(ttl_start).c_str(), ReadableTime(ttl_end).c_str()); ReadableTime(ttl_start).c_str(), ReadableTime(ttl_end).c_str());
@ -693,7 +688,6 @@ void DBDumperCommand::DoCommand() {
for (; iter->Valid(); iter->Next()) { for (; iter->Valid(); iter->Next()) {
int rawtime = 0; int rawtime = 0;
string value;
// If end marker was specified, we stop before it // If end marker was specified, we stop before it
if (!null_to_ && (iter->key().ToString() >= to_)) if (!null_to_ && (iter->key().ToString() >= to_))
break; break;
@ -702,20 +696,16 @@ void DBDumperCommand::DoCommand() {
break; break;
if (is_db_ttl_) { if (is_db_ttl_) {
TtlIterator* it_ttl = (TtlIterator*)iter; TtlIterator* it_ttl = (TtlIterator*)iter;
struct ValueAndTimestamp val_ts = it_ttl->ValueWithTimestamp(); rawtime = it_ttl->timestamp();
value = val_ts.value.ToString(); if (rawtime < ttl_start || rawtime >= ttl_end) {
rawtime = val_ts.timestamp;
if (rawtime < ttl_start || rawtime > ttl_end) {
continue; continue;
} }
} else {
value = iter->value().ToString();
} }
if (max_keys > 0) { if (max_keys > 0) {
--max_keys; --max_keys;
} }
if (is_db_ttl_ && num_buckets > 1) { if (is_db_ttl_ && num_buckets > 1) {
IncBucketCounts(bucket_counts.get(), ttl_start, time_range, bucket_size, IncBucketCounts(bucket_counts, ttl_start, time_range, bucket_size,
rawtime, num_buckets); rawtime, num_buckets);
} }
++count; ++count;
@ -724,12 +714,13 @@ void DBDumperCommand::DoCommand() {
fprintf(stdout, "%s ", ReadableTime(rawtime).c_str()); fprintf(stdout, "%s ", ReadableTime(rawtime).c_str());
} }
string str = PrintKeyValue(iter->key().ToString(), string str = PrintKeyValue(iter->key().ToString(),
value, is_key_hex_, is_value_hex_); iter->value().ToString(), is_key_hex_,
is_value_hex_);
fprintf(stdout, "%s\n", str.c_str()); fprintf(stdout, "%s\n", str.c_str());
} }
} }
if (num_buckets > 1 && is_db_ttl_) { if (num_buckets > 1 && is_db_ttl_) {
PrintBucketCounts(bucket_counts.get(), ttl_start, ttl_end, bucket_size, PrintBucketCounts(bucket_counts, ttl_start, ttl_end, bucket_size,
num_buckets); num_buckets);
} else { } else {
fprintf(stdout, "Keys in range: %lld\n", (long long) count); fprintf(stdout, "Keys in range: %lld\n", (long long) count);
@ -1189,8 +1180,8 @@ void ScanCommand::Help(string& ret) {
ret.append(" [--" + ARG_TTL + "]"); ret.append(" [--" + ARG_TTL + "]");
ret.append(" [--" + ARG_TIMESTAMP + "]"); ret.append(" [--" + ARG_TIMESTAMP + "]");
ret.append(" [--" + ARG_MAX_KEYS + "=<N>q] "); ret.append(" [--" + ARG_MAX_KEYS + "=<N>q] ");
ret.append(" [--" + ARG_TTL_START + "=<N>]"); ret.append(" [--" + ARG_TTL_START + "=<N>:- is inclusive]");
ret.append(" [--" + ARG_TTL_END + "=<N>]"); ret.append(" [--" + ARG_TTL_END + "=<N>:- is exclusive]");
ret.append("\n"); ret.append("\n");
} }
@ -1224,21 +1215,17 @@ void ScanCommand::DoCommand() {
it->Valid() && (!end_key_specified_ || it->key().ToString() < end_key_); it->Valid() && (!end_key_specified_ || it->key().ToString() < end_key_);
it->Next()) { it->Next()) {
string key = it->key().ToString(); string key = it->key().ToString();
string value;
if (is_db_ttl_) { if (is_db_ttl_) {
TtlIterator* it_ttl = (TtlIterator*)it; TtlIterator* it_ttl = (TtlIterator*)it;
struct ValueAndTimestamp val_ts = it_ttl->ValueWithTimestamp(); int rawtime = it_ttl->timestamp();
int rawtime = val_ts.timestamp; if (rawtime < ttl_start || rawtime >= ttl_end) {
value = val_ts.value.ToString();
if (rawtime < ttl_start || rawtime > ttl_end) {
continue; continue;
} }
if (timestamp_) { if (timestamp_) {
fprintf(stdout, "%s ", ReadableTime(rawtime).c_str()); fprintf(stdout, "%s ", ReadableTime(rawtime).c_str());
} }
} else {
value = it->value().ToString();
} }
string value = it->value().ToString();
fprintf(stdout, "%s : %s\n", fprintf(stdout, "%s : %s\n",
(is_key_hex_ ? StringToHex(key) : key).c_str(), (is_key_hex_ ? StringToHex(key) : key).c_str(),
(is_value_hex_ ? StringToHex(value) : value).c_str() (is_value_hex_ ? StringToHex(value) : value).c_str()

6
utilities/ttl/db_ttl.cc
Unescape View File

@ -84,8 +84,8 @@ Status DBWithTTL::AppendTS(const Slice& val, std::string& val_with_ts) {
// Returns corruption if the length of the string is lesser than timestamp, or // Returns corruption if the length of the string is lesser than timestamp, or
// timestamp refers to a time lesser than ttl-feature release time // timestamp refers to a time lesser than ttl-feature release time
Status DBWithTTL::SanityCheckTimestamp(const std::string& str) { Status DBWithTTL::SanityCheckTimestamp(const Slice& str) {
if (str.length() < (unsigned)kTSLength) { if (str.size() < (unsigned)kTSLength) {
return Status::Corruption("Error: value's length less than timestamp's\n"); return Status::Corruption("Error: value's length less than timestamp's\n");
} }
// Checks that TS is not lesser than kMinTimestamp // Checks that TS is not lesser than kMinTimestamp
@ -200,7 +200,7 @@ Status DBWithTTL::Write(const WriteOptions& opts, WriteBatch* updates) {
} }
Iterator* DBWithTTL::NewIterator(const ReadOptions& opts) { Iterator* DBWithTTL::NewIterator(const ReadOptions& opts) {
return new TtlIterator(db_->NewIterator(opts), kTSLength); return new TtlIterator(db_->NewIterator(opts));
} }
const Snapshot* DBWithTTL::GetSnapshot() { const Snapshot* DBWithTTL::GetSnapshot() {

27
utilities/ttl/db_ttl.h
Unescape View File

@ -89,7 +89,7 @@ class DBWithTTL : public DB, CompactionFilter {
static Status AppendTS(const Slice& val, std::string& val_with_ts); static Status AppendTS(const Slice& val, std::string& val_with_ts);
static Status SanityCheckTimestamp(const std::string& str); static Status SanityCheckTimestamp(const Slice& str);
static Status StripTS(std::string* str); static Status StripTS(std::string* str);
@ -106,17 +106,11 @@ class DBWithTTL : public DB, CompactionFilter {
int32_t ttl_; int32_t ttl_;
}; };
struct ValueAndTimestamp {
Slice value;
int32_t timestamp;
};
class TtlIterator : public Iterator { class TtlIterator : public Iterator {
public: public:
TtlIterator(Iterator* iter, int32_t ts_len) explicit TtlIterator(Iterator* iter)
: iter_(iter), : iter_(iter) {
ts_len_(ts_len) {
assert(iter_); assert(iter_);
} }
@ -152,20 +146,16 @@ class TtlIterator : public Iterator {
return iter_->key(); return iter_->key();
} }
struct ValueAndTimestamp ValueWithTimestamp() const { int32_t timestamp() const {
assert(DBWithTTL::SanityCheckTimestamp(iter_->value().ToString()).ok()); return DecodeFixed32(
struct ValueAndTimestamp val_ts;
val_ts.timestamp = DecodeFixed32(
iter_->value().data() + iter_->value().size() - DBWithTTL::kTSLength); iter_->value().data() + iter_->value().size() - DBWithTTL::kTSLength);
val_ts.value = iter_->value();
val_ts.value.size_ -= ts_len_;
return val_ts;
} }
Slice value() const { Slice value() const {
assert(DBWithTTL::SanityCheckTimestamp(iter_->value().ToString()).ok()); //TODO: handle timestamp corruption like in general iterator semantics
assert(DBWithTTL::SanityCheckTimestamp(iter_->value()).ok());
Slice trimmed_value = iter_->value(); Slice trimmed_value = iter_->value();
trimmed_value.size_ -= ts_len_; trimmed_value.size_ -= DBWithTTL::kTSLength;
return trimmed_value; return trimmed_value;
} }
@ -175,7 +165,6 @@ class TtlIterator : public Iterator {
private: private:
Iterator* iter_; Iterator* iter_;
int32_t ts_len_;
}; };
} }

Write Preview
Loading…
Cancel
Save