From 8a48410f09d7933369d5bcfaddeaa733bd1d04d0 Mon Sep 17 00:00:00 2001 From: Mayank Agarwal Date: Mon, 13 May 2013 19:11:56 -0700 Subject: [PATCH] Enhance the ldb tool to support ttl databases Summary: ldb works with raw data from the database and needs to be aware of ttl-database to work with it meaningfully. '-ttl' option now tells it that. Also added onto the ldb_test.py test. This option may be specified alongwith put, get, scan or dump. There is no support to provide a ttl-value and it uses default forever because there is no use-case for this currently. Test Plan: make ldb_test; python tools/ldb_test.py Reviewers: dhruba, sheki, haobo, vamsi Reviewed By: sheki CC: leveldb Differential Revision: https://reviews.facebook.net/D10797 --- tools/ldb_test.py | 35 ++++++++++++++++++++++++++++++----- util/ldb_cmd.cc | 30 +++++++++++++++++------------- util/ldb_cmd.h | 18 +++++++++++++++--- util/ldb_tool.cc | 3 +++ 4 files changed, 65 insertions(+), 21 deletions(-) diff --git a/tools/ldb_test.py b/tools/ldb_test.py index 52b73488d..a76e9c9ab 100644 --- a/tools/ldb_test.py +++ b/tools/ldb_test.py @@ -43,7 +43,7 @@ class LDBTestCase(unittest.TestCase): def dbParam(self, dbName): return "--db=%s" % os.path.join(self.TMP_DIR, dbName) - def assertRunOKFull(self, params, expectedOutput): + def assertRunOKFull(self, params, expectedOutput, unexpected=False): """ All command-line params must be specified. Allows full flexibility in testing; for example: missing db param. @@ -52,7 +52,10 @@ class LDBTestCase(unittest.TestCase): output = my_check_output("./ldb %s |grep -v \"Created bg thread\"" % params, shell=True) - self.assertEquals(output.strip(), expectedOutput.strip()); + if not unexpected: + self.assertEqual(output.strip(), expectedOutput.strip()) + else: + self.assertNotEqual(output.strip(), expectedOutput.strip()) def assertRunFAILFull(self, params): """ @@ -70,13 +73,13 @@ class LDBTestCase(unittest.TestCase): "Exception should have been raised for command with params: %s" % params) - def assertRunOK(self, params, expectedOutput): + def assertRunOK(self, params, expectedOutput, unexpected=False): """ Uses the default test db. """ self.assertRunOKFull("%s %s" % (self.dbParam(self.DB_NAME), params), - expectedOutput) + expectedOutput, unexpected) def assertRunFAIL(self, params): """ @@ -123,7 +126,7 @@ class LDBTestCase(unittest.TestCase): self.assertRunOK("scan", "x2 : y2\nx3 : y3") self.assertRunOK("delete NonExistentKey", "OK") - # It is wierd that GET and SCAN raise exception for + # It is weird that GET and SCAN raise exception for # non-existent key, while delete does not def dumpDb(self, params, dumpFile): @@ -142,6 +145,16 @@ class LDBTestCase(unittest.TestCase): self.assertRunFAIL("batchput k1") self.assertRunFAIL("batchput k1 v1 k2") + def testInvalidCmdLines(self): + print "Running testInvalidCmdLines..." + # db not specified + self.assertRunFAILFull("put 0x6133 0x6233 --hex --create_if_missing") + # No param called he + self.assertRunFAIL("put 0x6133 0x6233 --he --create_if_missing") + # max_keys is not applicable for put + self.assertRunFAIL("put 0x6133 0x6233 --max_keys=1 --create_if_missing") + # hex has invalid boolean value + def testHexPutGet(self): print "Running testHexPutGet..." self.assertRunOK("put a1 b1 --create_if_missing", "OK") @@ -171,6 +184,18 @@ class LDBTestCase(unittest.TestCase): self.assertRunOK("delete --hex 0x6133", "OK") self.assertRunOK("scan", "a1 : b1\na2 : b2\na4 : b4") + def testTtlPutGet(self): + print "Running testTtlPutGet..." + self.assertRunOK("put a1 b1 --ttl --create_if_missing", "OK") + self.assertRunOK("scan ", "a1 : b1", True) + self.assertRunOK("dump --ttl ", "a1 ==> b1", True) + self.assertRunOK("scan --hex --ttl", "0x6131 : 0x6231") + self.assertRunOK("get a1", "b1", True) + self.assertRunOK("get --ttl a1", "b1") + self.assertRunOK("put a3 b3 --create_if_missing", "OK") + # fails because timstamp's length is greater than value's + self.assertRunFAIL("get --ttl a3") + def testInvalidCmdLines(self): print "Running testInvalidCmdLines..." # db not specified diff --git a/util/ldb_cmd.cc b/util/ldb_cmd.cc index b52a23075..b4e3beec5 100644 --- a/util/ldb_cmd.cc +++ b/util/ldb_cmd.cc @@ -33,6 +33,7 @@ const string LDBCommand::ARG_DB = "db"; const string LDBCommand::ARG_HEX = "hex"; const string LDBCommand::ARG_KEY_HEX = "key_hex"; const string LDBCommand::ARG_VALUE_HEX = "value_hex"; +const string LDBCommand::ARG_TTL = "ttl"; const string LDBCommand::ARG_FROM = "from"; const string LDBCommand::ARG_TO = "to"; const string LDBCommand::ARG_MAX_KEYS = "max_keys"; @@ -538,9 +539,9 @@ const string DBDumperCommand::ARG_STATS = "stats"; DBDumperCommand::DBDumperCommand(const vector& params, const map& options, const vector& flags) : LDBCommand(options, flags, true, - BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, - ARG_FROM, ARG_TO, ARG_MAX_KEYS, - ARG_COUNT_ONLY, ARG_STATS})), + BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, + ARG_VALUE_HEX, ARG_FROM, ARG_TO, + ARG_MAX_KEYS, ARG_COUNT_ONLY, ARG_STATS})), null_from_(true), null_to_(true), max_keys_(-1), @@ -568,7 +569,7 @@ DBDumperCommand::DBDumperCommand(const vector& params, " has an invalid value"); } catch(const out_of_range&) { exec_state_ = LDBCommandExecuteResult::FAILED(ARG_MAX_KEYS + - " has a valuei out-of-range"); + " has a value out-of-range"); } } @@ -636,8 +637,8 @@ void DBDumperCommand::DoCommand() { ++count; if (!count_only_) { string str = PrintKeyValue(iter->key().ToString(), - iter->value().ToString(), - is_key_hex_, is_value_hex_); + iter->value().ToString(), + is_key_hex_, is_value_hex_); fprintf(stdout, "%s\n", str.c_str()); } } @@ -908,8 +909,9 @@ void WALDumperCommand::DoCommand() { GetCommand::GetCommand(const vector& params, const map& options, const vector& flags) : - LDBCommand(options, flags, true, - BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) { + LDBCommand(options, flags, true, BuildCmdLineOptions({ARG_TTL, ARG_HEX, + ARG_KEY_HEX, + ARG_VALUE_HEX})) { if (params.size() != 1) { exec_state_ = LDBCommandExecuteResult::FAILED( @@ -996,7 +998,7 @@ void ApproxSizeCommand::DoCommand() { BatchPutCommand::BatchPutCommand(const vector& params, const map& options, const vector& flags) : LDBCommand(options, flags, false, - BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, + BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_CREATE_IF_MISSING})) { if (params.size() < 2) { @@ -1048,8 +1050,9 @@ Options BatchPutCommand::PrepareOptionsForOpenDB() { ScanCommand::ScanCommand(const vector& params, const map& options, const vector& flags) : LDBCommand(options, flags, true, - BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, - ARG_FROM, ARG_TO, ARG_MAX_KEYS})), + BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, + ARG_VALUE_HEX, ARG_FROM, ARG_TO, + ARG_MAX_KEYS})), start_key_specified_(false), end_key_specified_(false), max_keys_scanned_(-1) { @@ -1158,7 +1161,7 @@ void DeleteCommand::DoCommand() { PutCommand::PutCommand(const vector& params, const map& options, const vector& flags) : LDBCommand(options, flags, false, - BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, + BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX, ARG_CREATE_IF_MISSING})) { if (params.size() != 2) { @@ -1209,7 +1212,8 @@ const char* DBQuerierCommand::DELETE_CMD = "delete"; DBQuerierCommand::DBQuerierCommand(const vector& params, const map& options, const vector& flags) : LDBCommand(options, flags, false, - BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) { + BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, + ARG_VALUE_HEX})) { } diff --git a/util/ldb_cmd.h b/util/ldb_cmd.h index 77ba58128..2899ce4ce 100644 --- a/util/ldb_cmd.h +++ b/util/ldb_cmd.h @@ -12,14 +12,14 @@ #include #include -#include "leveldb/db.h" +#include "db/version_set.h" #include "leveldb/env.h" #include "leveldb/options.h" #include "leveldb/iterator.h" #include "leveldb/slice.h" -#include "db/version_set.h" #include "util/logging.h" #include "util/ldb_cmd_execute_result.h" +#include "utilities/utility_db.h" using std::string; using std::map; @@ -36,6 +36,7 @@ public: static const string ARG_HEX; static const string ARG_KEY_HEX; static const string ARG_VALUE_HEX; + static const string ARG_TTL; static const string ARG_FROM; static const string ARG_TO; static const string ARG_MAX_KEYS; @@ -157,6 +158,9 @@ protected: /** If true, the value is input/output as hex in get/put/scan/delete etc. */ bool is_value_hex_; + /** If true, the value is treated as timestamp suffixed */ + bool is_db_ttl_; + /** * Map of options passed on the command-line. */ @@ -179,6 +183,7 @@ protected: is_read_only_(is_read_only), is_key_hex_(false), is_value_hex_(false), + is_db_ttl_(false), option_map_(options), flags_(flags), valid_cmd_line_options_(valid_cmd_line_options) { @@ -190,6 +195,7 @@ protected: is_key_hex_ = IsKeyHex(options, flags); is_value_hex_ = IsValueHex(options, flags); + is_db_ttl_ = IsFlagPresent(flags, ARG_TTL); } void OpenDB() { @@ -199,7 +205,13 @@ protected: } // Open the DB. Status st; - if (is_read_only_) { + if (is_db_ttl_) { + if (is_read_only_) { + st = UtilityDB::OpenTtlDB(opt, db_path_, &db_, 0, true); + } else { + st = UtilityDB::OpenTtlDB(opt, db_path_, &db_); + } + } else if (is_read_only_) { st = DB::OpenForReadOnly(opt, db_path_, &db_); } else { st = DB::Open(opt, db_path_, &db_); diff --git a/util/ldb_tool.cc b/util/ldb_tool.cc index 2d46bce2e..3f4cfe950 100644 --- a/util/ldb_tool.cc +++ b/util/ldb_tool.cc @@ -18,6 +18,9 @@ public: ret.append("\n"); ret.append("The following optional parameters control if keys/values are " "input/output as hex or as plain strings:\n"); + ret.append(" --" + LDBCommand::ARG_TTL + + " with 'put','get','scan','dump','query','batchput'" + " : DB supports ttl and value is internally timestamp-suffixed\n"); ret.append(" --" + LDBCommand::ARG_KEY_HEX + " : Keys are input/output as hex\n"); ret.append(" --" + LDBCommand::ARG_VALUE_HEX +