Enhance the ldb tool to support ttl databases

Summary: ldb works with raw data from the database and needs to be aware of ttl-database to work with it meaningfully. '-ttl' option now tells it that. Also added onto the ldb_test.py test. This option may be specified alongwith put, get, scan or dump. There is no support to provide a ttl-value and it uses default forever because there is no use-case for this currently.

Test Plan: make ldb_test; python tools/ldb_test.py

Reviewers: dhruba, sheki, haobo, vamsi

Reviewed By: sheki

CC: leveldb

Differential Revision: https://reviews.facebook.net/D10797

tools/ldb_test.py

@@ -43,7 +43,7 @@ class LDBTestCase(unittest.TestCase):
     def dbParam(self, dbName):
         return "--db=%s" % os.path.join(self.TMP_DIR, dbName)
 
-    def assertRunOKFull(self, params, expectedOutput):
+    def assertRunOKFull(self, params, expectedOutput, unexpected=False):
         """
         All command-line params must be specified.
         Allows full flexibility in testing; for example: missing db param.
@@ -52,7 +52,10 @@ class LDBTestCase(unittest.TestCase):
 
         output = my_check_output("./ldb %s |grep -v \"Created bg thread\"" %
                             params, shell=True)
-        self.assertEquals(output.strip(), expectedOutput.strip());
+        if not unexpected:
+            self.assertEqual(output.strip(), expectedOutput.strip())
+        else:
+            self.assertNotEqual(output.strip(), expectedOutput.strip())
 
     def assertRunFAILFull(self, params):
         """
@@ -70,13 +73,13 @@ class LDBTestCase(unittest.TestCase):
             "Exception should have been raised for command with params: %s" %
             params)
 
-    def assertRunOK(self, params, expectedOutput):
+    def assertRunOK(self, params, expectedOutput, unexpected=False):
         """
         Uses the default test db.
 
         """
         self.assertRunOKFull("%s %s" % (self.dbParam(self.DB_NAME), params),
-                expectedOutput)
+                             expectedOutput, unexpected)
 
     def assertRunFAIL(self, params):
         """
@@ -123,7 +126,7 @@ class LDBTestCase(unittest.TestCase):
         self.assertRunOK("scan", "x2 : y2\nx3 : y3")
 
         self.assertRunOK("delete NonExistentKey", "OK")
-        # It is wierd that GET and SCAN raise exception for
+        # It is weird that GET and SCAN raise exception for
         # non-existent key, while delete does not
 
     def dumpDb(self, params, dumpFile):
@@ -142,6 +145,16 @@ class LDBTestCase(unittest.TestCase):
         self.assertRunFAIL("batchput k1")
         self.assertRunFAIL("batchput k1 v1 k2")
 
+    def testInvalidCmdLines(self):
+        print "Running testInvalidCmdLines..."
+        # db not specified
+        self.assertRunFAILFull("put 0x6133 0x6233 --hex --create_if_missing")
+        # No param called he
+        self.assertRunFAIL("put 0x6133 0x6233 --he --create_if_missing")
+        # max_keys is not applicable for put
+        self.assertRunFAIL("put 0x6133 0x6233 --max_keys=1 --create_if_missing")
+        # hex has invalid boolean value
+
     def testHexPutGet(self):
         print "Running testHexPutGet..."
         self.assertRunOK("put a1 b1 --create_if_missing", "OK")
@@ -171,6 +184,18 @@ class LDBTestCase(unittest.TestCase):
         self.assertRunOK("delete --hex 0x6133", "OK")
         self.assertRunOK("scan", "a1 : b1\na2 : b2\na4 : b4")
 
+    def testTtlPutGet(self):
+        print "Running testTtlPutGet..."
+        self.assertRunOK("put a1 b1 --ttl --create_if_missing", "OK")
+        self.assertRunOK("scan ", "a1 : b1", True)
+        self.assertRunOK("dump --ttl ", "a1 ==> b1", True)
+        self.assertRunOK("scan --hex --ttl", "0x6131 : 0x6231")
+        self.assertRunOK("get a1", "b1", True)
+        self.assertRunOK("get --ttl a1", "b1")
+        self.assertRunOK("put a3 b3 --create_if_missing", "OK")
+        # fails because timstamp's length is greater than value's
+        self.assertRunFAIL("get --ttl a3")
+
     def testInvalidCmdLines(self):
         print "Running testInvalidCmdLines..."
         # db not specified

util/ldb_cmd.cc

@@ -33,6 +33,7 @@ const string LDBCommand::ARG_DB = "db";
 const string LDBCommand::ARG_HEX = "hex";
 const string LDBCommand::ARG_KEY_HEX = "key_hex";
 const string LDBCommand::ARG_VALUE_HEX = "value_hex";
+const string LDBCommand::ARG_TTL = "ttl";
 const string LDBCommand::ARG_FROM = "from";
 const string LDBCommand::ARG_TO = "to";
 const string LDBCommand::ARG_MAX_KEYS = "max_keys";
@@ -538,9 +539,9 @@ const string DBDumperCommand::ARG_STATS = "stats";
 DBDumperCommand::DBDumperCommand(const vector<string>& params,
       const map<string, string>& options, const vector<string>& flags) :
     LDBCommand(options, flags, true,
-               BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX,
+               BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX,
-                                    ARG_FROM, ARG_TO, ARG_MAX_KEYS,
+                                    ARG_VALUE_HEX, ARG_FROM, ARG_TO,
-                                    ARG_COUNT_ONLY, ARG_STATS})),
+                                    ARG_MAX_KEYS, ARG_COUNT_ONLY, ARG_STATS})),
     null_from_(true),
     null_to_(true),
     max_keys_(-1),
@@ -568,7 +569,7 @@ DBDumperCommand::DBDumperCommand(const vector<string>& params,
                         " has an invalid value");
     } catch(const out_of_range&) {
       exec_state_ = LDBCommandExecuteResult::FAILED(ARG_MAX_KEYS +
-                        " has a valuei out-of-range");
+                        " has a value out-of-range");
     }
   }
 
@@ -636,8 +637,8 @@ void DBDumperCommand::DoCommand() {
     ++count;
     if (!count_only_) {
       string str = PrintKeyValue(iter->key().ToString(),
-                                      iter->value().ToString(),
+                                 iter->value().ToString(),
-                                      is_key_hex_, is_value_hex_);
+                                 is_key_hex_, is_value_hex_);
       fprintf(stdout, "%s\n", str.c_str());
     }
   }
@@ -908,8 +909,9 @@ void WALDumperCommand::DoCommand() {
 
 GetCommand::GetCommand(const vector<string>& params,
       const map<string, string>& options, const vector<string>& flags) :
-  LDBCommand(options, flags, true,
+  LDBCommand(options, flags, true, BuildCmdLineOptions({ARG_TTL, ARG_HEX,
-             BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
+                                                        ARG_KEY_HEX,
+                                                        ARG_VALUE_HEX})) {
 
   if (params.size() != 1) {
     exec_state_ = LDBCommandExecuteResult::FAILED(
@@ -996,7 +998,7 @@ void ApproxSizeCommand::DoCommand() {
 BatchPutCommand::BatchPutCommand(const vector<string>& params,
       const map<string, string>& options, const vector<string>& flags) :
   LDBCommand(options, flags, false,
-             BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX,
+             BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX,
                                   ARG_CREATE_IF_MISSING})) {
 
   if (params.size() < 2) {
@@ -1048,8 +1050,9 @@ Options BatchPutCommand::PrepareOptionsForOpenDB() {
 ScanCommand::ScanCommand(const vector<string>& params,
       const map<string, string>& options, const vector<string>& flags) :
     LDBCommand(options, flags, true,
-               BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX,
+               BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX,
-                                    ARG_FROM, ARG_TO, ARG_MAX_KEYS})),
+                                    ARG_VALUE_HEX, ARG_FROM, ARG_TO,
+                                    ARG_MAX_KEYS})),
     start_key_specified_(false),
     end_key_specified_(false),
     max_keys_scanned_(-1) {
@@ -1158,7 +1161,7 @@ void DeleteCommand::DoCommand() {
 PutCommand::PutCommand(const vector<string>& params,
       const map<string, string>& options, const vector<string>& flags) :
   LDBCommand(options, flags, false,
-             BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX,
+             BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX,
                                   ARG_CREATE_IF_MISSING})) {
 
   if (params.size() != 2) {
@@ -1209,7 +1212,8 @@ const char* DBQuerierCommand::DELETE_CMD = "delete";
 DBQuerierCommand::DBQuerierCommand(const vector<string>& params,
     const map<string, string>& options, const vector<string>& flags) :
   LDBCommand(options, flags, false,
-             BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
+             BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX,
+                                  ARG_VALUE_HEX})) {
 
 }
 

util/ldb_cmd.h

@@ -12,14 +12,14 @@
 #include <algorithm>
 #include <stdio.h>
 
-#include "leveldb/db.h"
+#include "db/version_set.h"
 #include "leveldb/env.h"
 #include "leveldb/options.h"
 #include "leveldb/iterator.h"
 #include "leveldb/slice.h"
-#include "db/version_set.h"
 #include "util/logging.h"
 #include "util/ldb_cmd_execute_result.h"
+#include "utilities/utility_db.h"
 
 using std::string;
 using std::map;
@@ -36,6 +36,7 @@ public:
   static const string ARG_HEX;
   static const string ARG_KEY_HEX;
   static const string ARG_VALUE_HEX;
+  static const string ARG_TTL;
   static const string ARG_FROM;
   static const string ARG_TO;
   static const string ARG_MAX_KEYS;
@@ -157,6 +158,9 @@ protected:
   /** If true, the value is input/output as hex in get/put/scan/delete etc. */
   bool is_value_hex_;
 
+  /** If true, the value is treated as timestamp suffixed */
+  bool is_db_ttl_;
+
   /**
    * Map of options passed on the command-line.
    */
@@ -179,6 +183,7 @@ protected:
       is_read_only_(is_read_only),
       is_key_hex_(false),
       is_value_hex_(false),
+      is_db_ttl_(false),
       option_map_(options),
       flags_(flags),
       valid_cmd_line_options_(valid_cmd_line_options) {
@@ -190,6 +195,7 @@ protected:
 
     is_key_hex_ = IsKeyHex(options, flags);
     is_value_hex_ = IsValueHex(options, flags);
+    is_db_ttl_ = IsFlagPresent(flags, ARG_TTL);
   }
 
   void OpenDB() {
@@ -199,7 +205,13 @@ protected:
     }
     // Open the DB.
     Status st;
-    if (is_read_only_) {
+    if (is_db_ttl_) {
+      if (is_read_only_) {
+        st = UtilityDB::OpenTtlDB(opt, db_path_, &db_, 0, true);
+      } else {
+        st = UtilityDB::OpenTtlDB(opt, db_path_, &db_);
+      }
+    } else if (is_read_only_) {
       st = DB::OpenForReadOnly(opt, db_path_, &db_);
     } else {
       st = DB::Open(opt, db_path_, &db_);

util/ldb_tool.cc

@@ -18,6 +18,9 @@ public:
     ret.append("\n");
     ret.append("The following optional parameters control if keys/values are "
         "input/output as hex or as plain strings:\n");
+    ret.append("  --" + LDBCommand::ARG_TTL +
+        " with 'put','get','scan','dump','query','batchput'"
+        " : DB supports ttl and value is internally timestamp-suffixed\n");
     ret.append("  --" + LDBCommand::ARG_KEY_HEX +
         " : Keys are input/output as hex\n");
     ret.append("  --" + LDBCommand::ARG_VALUE_HEX +