Enhanced ldb to support data access commands

Summary: Added put/get/scan/batchput/delete/approxsize

Test Plan: Added pyunit script to test the newly added commands

Reviewers: chip, leveldb

Reviewed By: chip

CC: zshao, emayanke

Differential Revision: https://reviews.facebook.net/D7947

Makefile

@@ -104,9 +104,12 @@ release:
 	make clean
 	OPT=-DNDEBUG make -j32
 
-check: all $(PROGRAMS) $(TESTS) $(TOOLS)
+check: all $(PROGRAMS) $(TESTS) $(TOOLS) ldb_tests
 	for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done
 
+ldb_tests: all $(PROGRAMS) $(TOOLS)
+	python tools/ldb_test.py
+
 clean:
 	-rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) $(SHARED) $(MEMENVLIBRARY) $(THRIFTSERVER) */*.o */*/*.o ios-x86/*/*.o ios-arm/*/*.o build_config.mk
 	-rm -rf ios-x86/* ios-arm/*

tools/ldb.cc

@@ -10,36 +10,53 @@ class LDBCommandRunner {
 public:
 
   static void PrintHelp(const char* exec_name) {
-    std::string ret;
-    ret.append("--- compact ----:\n");
-    ret.append(exec_name);
-    ret.append(" compact ");
-    Compactor::Help(ret);
-
-    ret.append("\n--- dump ----:\n");
-    ret.append(exec_name);
-    ret.append(" dump ");
-    DBDumper::Help(ret);
-
-    ret.append("\n--- load ----:\n");
-    ret.append(exec_name);
-    ret.append(" load ");
-    DBLoader::Help(ret);
-
-    ret.append("\n--- query ----:\n");
-    ret.append(exec_name);
-    ret.append(" query ");
-    DBQuerier::Help(ret);
-
-    ret.append("\n---reduce_levels ----:\n");
-    ret.append(exec_name);
-    ret.append(" reduce_levels ");
-    ReduceDBLevels::Help(ret);
-
-    ret.append("\n---dump_wal----:\n");
-    ret.append(exec_name);
-    ret.append(" dump_wal ");
-    WALDumper::Help(ret);
+    string ret;
+
+    ret.append("ldb - LevelDB Tool");
+    ret.append("\n\n");
+    ret.append("All commands MUST specify --" + LDBCommand::ARG_DB +
+        "=<full_path_to_db_directory>\n");
+    ret.append("\n");
+    ret.append("The following optional parameters control if keys/values are "
+        "input/output as hex or as plain strings:\n");
+    ret.append("  --" + LDBCommand::ARG_KEY_HEX +
+        " : Keys are input/output as hex\n");
+    ret.append("  --" + LDBCommand::ARG_VALUE_HEX +
+        " : Values are input/output as hex\n");
+    ret.append("  --" + LDBCommand::ARG_HEX +
+        " : Both keys and values are input/output as hex\n");
+    ret.append("\n");
+
+    ret.append("The following optional parameters control the database "
+        "internals:\n");
+    ret.append("  --" + LDBCommand::ARG_BLOOM_BITS + "=<int,e.g.:14>\n");
+    ret.append("  --" + LDBCommand::ARG_COMPRESSION_TYPE +
+        "=<no|snappy|zlib|bzip2>\n");
+    ret.append("  --" + LDBCommand::ARG_BLOCK_SIZE +
+        "=<block_size_in_bytes>\n");
+    ret.append("  --" + LDBCommand::ARG_AUTO_COMPACTION + "=<true|false>\n");
+    ret.append("  --" + LDBCommand::ARG_WRITE_BUFFER_SIZE +
+        "=<int,e.g.:4194304>\n");
+    ret.append("  --" + LDBCommand::ARG_FILE_SIZE + "=<int,e.g.:2097152>\n");
+
+    ret.append("\n\n");
+    ret.append("Data Access Commands:\n");
+    PutCommand::Help(ret);
+    GetCommand::Help(ret);
+    BatchPutCommand::Help(ret);
+    ScanCommand::Help(ret);
+    DeleteCommand::Help(ret);
+    DBQuerierCommand::Help(ret);
+    ApproxSizeCommand::Help(ret);
+
+    ret.append("\n\n");
+    ret.append("Admin Commands:\n");
+    WALDumperCommand::Help(ret);
+    CompactorCommand::Help(ret);
+    ReduceDBLevelsCommand::Help(ret);
+    DBDumperCommand::Help(ret);
+    DBLoaderCommand::Help(ret);
+
     fprintf(stderr, "%s\n", ret.c_str());
   }
 
@@ -48,46 +65,26 @@ public:
       PrintHelp(argv[0]);
       exit(1);
     }
-    const char* cmd = argv[1];
-    std::string db_name;
-    std::vector<std::string> args;
-    for (int i = 2; i < argc; i++) {
-      if (strncmp(argv[i], "--db=", strlen("--db=")) == 0) {
-        db_name = argv[i] + strlen("--db=");
-      } else {
-        args.push_back(argv[i]);
-      }
-    }
 
-    LDBCommand* cmdObj = NULL;
-    if (strcmp(cmd, "compact") == 0) {
-      // run compactor
-      cmdObj = new Compactor(db_name, args);
-    } else if (strcmp(cmd, "dump") == 0) {
-      // run dump
-      cmdObj = new DBDumper(db_name, args);
-    } else if (strcmp(cmd, "load") == 0) {
-      // run loader
-      cmdObj = new DBLoader(db_name, args);
-    } else if (strcmp(cmd, "query") == 0) {
-      // run querier
-      cmdObj = new DBQuerier(db_name, args);
-    } else if (strcmp(cmd, "reduce_levels") == 0) {
-      // reduce db levels
-      cmdObj = new ReduceDBLevels(db_name, args);
-    } else if (strcmp(cmd, "dump_wal") == 0) {
-      cmdObj = new WALDumper(args);
-    } else {
-      fprintf(stderr, "Unknown command: %s\n", cmd);
+    LDBCommand* cmdObj = LDBCommand::InitFromCmdLineArgs(argc, argv);
+    if (cmdObj == NULL) {
+      fprintf(stderr, "Unknown command\n");
       PrintHelp(argv[0]);
       exit(1);
     }
 
+    if (!cmdObj->ValidateCmdLineOptions()) {
+      exit(1);
+    }
+
     cmdObj->Run();
     LDBCommandExecuteResult ret = cmdObj->GetExecuteState();
     fprintf(stderr, "%s\n", ret.ToString().c_str());
     delete cmdObj;
+
+    exit(ret.IsFailed());
   }
+
 };
 
 }

tools/ldb_test.py

@@ -0,0 +1,308 @@
+import os
+import os.path
+import shutil
+import subprocess
+import time
+import unittest
+import tempfile
+
+def my_check_output(*popenargs, **kwargs):
+    """
+    If we had python 2.7, we should simply use subprocess.check_output.
+    This is a stop-gap solution for python 2.6
+    """
+    if 'stdout' in kwargs:
+        raise ValueError('stdout argument not allowed, it will be overridden.')
+    process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
+    output, unused_err = process.communicate()
+    retcode = process.poll()
+    if retcode:
+        cmd = kwargs.get("args")
+        if cmd is None:
+            cmd = popenargs[0]
+        raise Exception("Exit code is not 0.  It is %d.  Command: %s" %
+                (retcode, cmd))
+    return output
+
+
+class LDBTestCase(unittest.TestCase):
+    def setUp(self):
+        self.TMP_DIR  = tempfile.mkdtemp(prefix="ldb_test_")
+        self.DB_NAME = "testdb"
+
+    def tearDown(self):
+        assert(self.TMP_DIR.strip() != "/"
+                and self.TMP_DIR.strip() != "/tmp"
+                and self.TMP_DIR.strip() != "/tmp/") #Just some paranoia
+
+        shutil.rmtree(self.TMP_DIR)
+
+    def dbParam(self, dbName):
+        return "--db=%s" % os.path.join(self.TMP_DIR, dbName)
+
+    def assertRunOKFull(self, params, expectedOutput):
+        """
+        All command-line params must be specified.
+        Allows full flexibility in testing; for example: missing db param.
+
+        """
+
+        output = my_check_output("./ldb %s |grep -v \"Created bg thread\"" %
+                            params, shell=True)
+        self.assertEquals(output.strip(), expectedOutput.strip());
+
+    def assertRunFAILFull(self, params):
+        """
+        All command-line params must be specified.
+        Allows full flexibility in testing; for example: missing db param.
+
+        """
+        try:
+            my_check_output("./ldb %s |grep -v \"Created bg thread\"" % params,
+                            shell=True)
+        except Exception, e:
+            return
+        self.fail(
+            "Exception should have been raised for command with params: %s" %
+            params)
+
+    def assertRunOK(self, params, expectedOutput):
+        """
+        Uses the default test db.
+
+        """
+        self.assertRunOKFull("%s %s" % (self.dbParam(self.DB_NAME), params),
+                expectedOutput)
+
+    def assertRunFAIL(self, params):
+        """
+        Uses the default test db.
+        """
+        self.assertRunFAILFull("%s %s" % (self.dbParam(self.DB_NAME), params))
+
+    def testSimpleStringPutGet(self):
+        self.assertRunFAIL("put x1 y1")
+        self.assertRunOK("put --create_if_missing x1 y1", "OK")
+        self.assertRunOK("get x1", "y1")
+        self.assertRunFAIL("get x2")
+
+        self.assertRunOK("put x2 y2", "OK")
+        self.assertRunOK("get x1", "y1")
+        self.assertRunOK("get x2", "y2")
+        self.assertRunFAIL("get x3")
+
+        self.assertRunOK("scan --from=x1 --to=z", "x1 : y1\nx2 : y2")
+        self.assertRunOK("put x3 y3", "OK")
+
+        self.assertRunOK("scan --from=x1 --to=z", "x1 : y1\nx2 : y2\nx3 : y3")
+        self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3")
+        self.assertRunOK("scan --from=x", "x1 : y1\nx2 : y2\nx3 : y3")
+
+        self.assertRunOK("scan --to=x2", "x1 : y1")
+        self.assertRunOK("scan --from=x1 --to=z --max_keys=1", "x1 : y1")
+        self.assertRunOK("scan --from=x1 --to=z --max_keys=2",
+                "x1 : y1\nx2 : y2")
+
+        self.assertRunOK("scan --from=x1 --to=z --max_keys=3",
+                "x1 : y1\nx2 : y2\nx3 : y3")
+        self.assertRunOK("scan --from=x1 --to=z --max_keys=4",
+                "x1 : y1\nx2 : y2\nx3 : y3")
+        self.assertRunOK("scan --from=x1 --to=x2", "x1 : y1")
+        self.assertRunOK("scan --from=x2 --to=x4", "x2 : y2\nx3 : y3")
+        self.assertRunFAIL("scan --from=x4 --to=z") # No results => FAIL
+        self.assertRunFAIL("scan --from=x1 --to=z --max_keys=foo")
+
+        self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3")
+
+        self.assertRunOK("delete x1", "OK")
+        self.assertRunOK("scan", "x2 : y2\nx3 : y3")
+
+        self.assertRunOK("delete NonExistentKey", "OK")
+        # It is wierd that GET and SCAN raise exception for
+        # non-existent key, while delete does not
+
+    def dumpDb(self, params, dumpFile):
+        return 0 == os.system("./ldb dump %s > %s" % (params, dumpFile))
+
+    def loadDb(self, params, dumpFile):
+        return 0 == os.system("cat %s | ./ldb load %s" % (dumpFile, params))
+
+    def testStringBatchPut(self):
+        self.assertRunOK("batchput x1 y1 --create_if_missing", "OK")
+        self.assertRunOK("scan", "x1 : y1")
+        self.assertRunOK("batchput x2 y2 x3 y3 \"x4 abc\" \"y4 xyz\"", "OK")
+        self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 abc : y4 xyz")
+        self.assertRunFAIL("batchput")
+        self.assertRunFAIL("batchput k1")
+        self.assertRunFAIL("batchput k1 v1 k2")
+
+
+    def testHexPutGet(self):
+        self.assertRunOK("put a1 b1 --create_if_missing", "OK")
+        self.assertRunOK("scan", "a1 : b1")
+        self.assertRunOK("scan --hex", "0x6131 : 0x6231")
+        self.assertRunFAIL("put --hex 6132 6232")
+        self.assertRunOK("put --hex 0x6132 0x6232", "OK")
+        self.assertRunOK("scan --hex", "0x6131 : 0x6231\n0x6132 : 0x6232")
+        self.assertRunOK("scan", "a1 : b1\na2 : b2")
+        self.assertRunOK("get a1", "b1")
+        self.assertRunOK("get --hex 0x6131", "0x6231")
+        self.assertRunOK("get a2", "b2")
+        self.assertRunOK("get --hex 0x6132", "0x6232")
+        self.assertRunOK("get --key_hex 0x6132", "b2")
+        self.assertRunOK("get --key_hex --value_hex 0x6132", "0x6232")
+        self.assertRunOK("get --value_hex a2", "0x6232")
+        self.assertRunOK("scan --key_hex --value_hex",
+                "0x6131 : 0x6231\n0x6132 : 0x6232")
+        self.assertRunOK("scan --hex --from=0x6131 --to=0x6133",
+                "0x6131 : 0x6231\n0x6132 : 0x6232")
+        self.assertRunOK("scan --hex --from=0x6131 --to=0x6132",
+                "0x6131 : 0x6231")
+        self.assertRunOK("scan --key_hex", "0x6131 : b1\n0x6132 : b2")
+        self.assertRunOK("scan --value_hex", "a1 : 0x6231\na2 : 0x6232")
+        self.assertRunOK("batchput --hex 0x6133 0x6233 0x6134 0x6234", "OK")
+        self.assertRunOK("scan", "a1 : b1\na2 : b2\na3 : b3\na4 : b4")
+        self.assertRunOK("delete --hex 0x6133", "OK")
+        self.assertRunOK("scan", "a1 : b1\na2 : b2\na4 : b4")
+
+
+    def testInvalidCmdLines(self):
+        # db not specified
+        self.assertRunFAILFull("put 0x6133 0x6233 --hex --create_if_missing")
+        # No param called he
+        self.assertRunFAIL("put 0x6133 0x6233 --he --create_if_missing")
+        # max_keys is not applicable for put
+        self.assertRunFAIL("put 0x6133 0x6233 --max_keys=1 --create_if_missing")
+        # hex has invalid boolean value
+        self.assertRunFAIL("put 0x6133 0x6233 --hex=Boo --create_if_missing")
+
+
+    def testDumpLoad(self):
+        self.assertRunOK("batchput --create_if_missing x1 y1 x2 y2 x3 y3 x4 y4",
+                "OK")
+        self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
+        origDbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
+
+        # Dump and load without any additional params specified
+        dumpFilePath = os.path.join(self.TMP_DIR, "dump1")
+        loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump1")
+        self.assertTrue(self.dumpDb("--db=%s" % origDbPath, dumpFilePath))
+        self.assertTrue(self.loadDb(
+            "--db=%s --create_if_missing" % loadedDbPath, dumpFilePath))
+        self.assertRunOKFull("scan --db=%s" % loadedDbPath,
+                "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
+
+        # Dump and load in hex
+        dumpFilePath = os.path.join(self.TMP_DIR, "dump2")
+        loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump2")
+        self.assertTrue(self.dumpDb("--db=%s --hex" % origDbPath, dumpFilePath))
+        self.assertTrue(self.loadDb(
+            "--db=%s --hex --create_if_missing" % loadedDbPath, dumpFilePath))
+        self.assertRunOKFull("scan --db=%s" % loadedDbPath,
+                "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
+
+        # Dump only a portion of the key range
+        dumpFilePath = os.path.join(self.TMP_DIR, "dump3")
+        loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump3")
+        self.assertTrue(self.dumpDb(
+            "--db=%s --from=x1 --to=x3" % origDbPath, dumpFilePath))
+        self.assertTrue(self.loadDb(
+            "--db=%s --create_if_missing" % loadedDbPath, dumpFilePath))
+        self.assertRunOKFull("scan --db=%s" % loadedDbPath, "x1 : y1\nx2 : y2")
+
+        # Dump upto max_keys rows
+        dumpFilePath = os.path.join(self.TMP_DIR, "dump4")
+        loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump4")
+        self.assertTrue(self.dumpDb(
+            "--db=%s --max_keys=3" % origDbPath, dumpFilePath))
+        self.assertTrue(self.loadDb(
+            "--db=%s --create_if_missing" % loadedDbPath, dumpFilePath))
+        self.assertRunOKFull("scan --db=%s" % loadedDbPath,
+                "x1 : y1\nx2 : y2\nx3 : y3")
+
+        # Load into an existing db, create_if_missing is not specified
+        self.assertTrue(self.dumpDb("--db=%s" % origDbPath, dumpFilePath))
+        self.assertTrue(self.loadDb("--db=%s" % loadedDbPath, dumpFilePath))
+        self.assertRunOKFull("scan --db=%s" % loadedDbPath,
+                "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
+
+        # Dump and load with WAL disabled
+        dumpFilePath = os.path.join(self.TMP_DIR, "dump5")
+        loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump5")
+        self.assertTrue(self.dumpDb("--db=%s" % origDbPath, dumpFilePath))
+        self.assertTrue(self.loadDb(
+            "--db=%s --disable_wal --create_if_missing" % loadedDbPath,
+            dumpFilePath))
+        self.assertRunOKFull("scan --db=%s" % loadedDbPath,
+                "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
+
+        # Dump and load with lots of extra params specified
+        extraParams = " ".join(["--bloom_bits=14", "--compression_type=bzip2",
+                                "--block_size=1024", "--auto_compaction=true",
+                                "--write_buffer_size=4194304",
+                                "--file_size=2097152"])
+        dumpFilePath = os.path.join(self.TMP_DIR, "dump6")
+        loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump6")
+        self.assertTrue(self.dumpDb(
+            "--db=%s %s" % (origDbPath, extraParams), dumpFilePath))
+        self.assertTrue(self.loadDb(
+            "--db=%s %s --create_if_missing" % (loadedDbPath, extraParams),
+            dumpFilePath))
+        self.assertRunOKFull("scan --db=%s" % loadedDbPath,
+                "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
+
+        # Dump with count_only
+        dumpFilePath = os.path.join(self.TMP_DIR, "dump7")
+        loadedDbPath = os.path.join(self.TMP_DIR, "loaded_from_dump7")
+        self.assertTrue(self.dumpDb(
+            "--db=%s --count_only" % origDbPath, dumpFilePath))
+        self.assertTrue(self.loadDb(
+            "--db=%s --create_if_missing" % loadedDbPath, dumpFilePath))
+        # DB should have atleast one value for scan to work
+        self.assertRunOKFull("put --db=%s k1 v1" % loadedDbPath, "OK")
+        self.assertRunOKFull("scan --db=%s" % loadedDbPath, "k1 : v1")
+
+        # Dump command fails because of typo in params
+        dumpFilePath = os.path.join(self.TMP_DIR, "dump8")
+        self.assertFalse(self.dumpDb(
+            "--db=%s --create_if_missin" % origDbPath, dumpFilePath))
+
+
+    def testMiscAdminTask(self):
+        # These tests need to be improved; for example with asserts about
+        # whether compaction or level reduction actually took place.
+        self.assertRunOK("batchput --create_if_missing x1 y1 x2 y2 x3 y3 x4 y4",
+                "OK")
+        self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
+        origDbPath = os.path.join(self.TMP_DIR, self.DB_NAME)
+
+        self.assertTrue(0 == os.system("./ldb compact --db=%s" % origDbPath))
+        self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
+
+        self.assertTrue(0 == os.system(
+            "./ldb reduce_levels --db=%s --new_levels=2" % origDbPath))
+        self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
+
+        self.assertTrue(0 == os.system(
+            "./ldb reduce_levels --db=%s --new_levels=3" % origDbPath))
+        self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
+
+        self.assertTrue(0 == os.system(
+            "./ldb compact --db=%s --from=x1 --to=x3" % origDbPath))
+        self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
+
+        self.assertTrue(0 == os.system(
+            "./ldb compact --db=%s --hex --from=0x6131 --to=0x6134" %
+            origDbPath))
+        self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
+
+        #TODO(dilip): Not sure what should be passed to WAL.Currently corrupted.
+        self.assertTrue(0 == os.system(
+            "./ldb dump_wal --db=%s --walfile=%s --header" % (
+                origDbPath, os.path.join(origDbPath, "LOG"))))
+        self.assertRunOK("scan", "x1 : y1\nx2 : y2\nx3 : y3\nx4 : y4")
+
+
+if __name__ == "__main__":
+    unittest.main()
+

tools/reduce_levels_test.cc

@@ -84,11 +84,13 @@ Status ReduceLevelTest::OpenDB(bool create_if_missing, int num_levels,
 }
 
 bool ReduceLevelTest::ReduceLevels(int target_level) {
-  std::vector<std::string> args = leveldb::ReduceDBLevels::PrepareArgs(
-      target_level, false);
-  ReduceDBLevels level_reducer(dbname_, args);
-  level_reducer.Run();
-  return level_reducer.GetExecuteState().IsSucceed();
+  std::vector<std::string> args = leveldb::ReduceDBLevelsCommand::PrepareArgs(
+      dbname_, target_level, false);
+  LDBCommand* level_reducer = LDBCommand::InitFromCmdLineArgs(args);
+  level_reducer->Run();
+  bool is_succeed = level_reducer->GetExecuteState().IsSucceed();
+  delete level_reducer;
+  return is_succeed;
 }
 
 TEST(ReduceLevelTest, Last_Level) {

util/ldb_cmd.h

@@ -2,8 +2,8 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-#ifndef LEVELDB_UTIL_LDB_H_
-#define LEVELDB_UTIL_LDB_H_
+#ifndef LEVELDB_UTIL_LDB_CMD_H_
+#define LEVELDB_UTIL_LDB_CMD_H_
 
 #include <string>
 #include <iostream>
@@ -12,6 +12,9 @@
 #include <algorithm>
 #include <stdio.h>
 
+#include <boost/algorithm/string/case_conv.hpp>
+#include <boost/algorithm/string/predicate.hpp>
+
 #include "leveldb/db.h"
 #include "leveldb/env.h"
 #include "leveldb/options.h"
@@ -19,90 +22,37 @@
 #include "leveldb/slice.h"
 #include "db/version_set.h"
 #include "util/logging.h"
+#include "util/ldb_cmd_execute_result.h"
 
-namespace leveldb {
-
-class LDBCommandExecuteResult {
-public:
-  enum State {
-    EXEC_NOT_STARTED = 0, EXEC_SUCCEED = 1, EXEC_FAILED = 2,
-  };
-
-  LDBCommandExecuteResult() {
-    state_ = EXEC_NOT_STARTED;
-    message_ = "";
-  }
-
-  LDBCommandExecuteResult(State state, std::string& msg) {
-    state_ = state;
-    message_ = msg;
-  }
-
-  std::string ToString() {
-    std::string ret;
-    switch (state_) {
-    case EXEC_SUCCEED:
-      break;
-    case EXEC_FAILED:
-      ret.append("Failed: ");
-      break;
-    case EXEC_NOT_STARTED:
-      ret.append("Not started: ");
-    }
-    if (!message_.empty()) {
-      ret.append(message_);
-    }
-    return ret;
-  }
-
-  void Reset() {
-    state_ = EXEC_NOT_STARTED;
-    message_ = "";
-  }
-
-  bool IsSucceed() {
-    return state_ == EXEC_SUCCEED;
-  }
-
-  bool IsNotStarted() {
-    return state_ == EXEC_NOT_STARTED;
-  }
-
-  bool IsFailed() {
-    return state_ == EXEC_FAILED;
-  }
-
-  static LDBCommandExecuteResult SUCCEED(std::string msg) {
-    return LDBCommandExecuteResult(EXEC_SUCCEED, msg);
-  }
+using std::string;
+using std::map;
+using std::vector;
+using std::ostringstream;
 
-  static LDBCommandExecuteResult FAILED(std::string msg) {
-    return LDBCommandExecuteResult(EXEC_FAILED, msg);
-  }
-
-private:
-  State state_;
-  std::string message_;
-
-  bool operator==(const LDBCommandExecuteResult&);
-  bool operator!=(const LDBCommandExecuteResult&);
-};
+namespace leveldb {
 
 class LDBCommand {
 public:
 
-  /* Constructor */
-  LDBCommand(std::string& db_name, std::vector<std::string>& args) :
-    db_path_(db_name),
-    db_(NULL) {
-    parse_open_args(args);
-  }
-
-  LDBCommand(std::vector<std::string>& args) :
-    db_path_(""),
-    db_(NULL) {
-    parse_open_args(args);
-  }
+  // Command-line arguments
+  static const string ARG_DB;
+  static const string ARG_HEX;
+  static const string ARG_KEY_HEX;
+  static const string ARG_VALUE_HEX;
+  static const string ARG_FROM;
+  static const string ARG_TO;
+  static const string ARG_MAX_KEYS;
+  static const string ARG_BLOOM_BITS;
+  static const string ARG_COMPRESSION_TYPE;
+  static const string ARG_BLOCK_SIZE;
+  static const string ARG_AUTO_COMPACTION;
+  static const string ARG_WRITE_BUFFER_SIZE;
+  static const string ARG_FILE_SIZE;
+  static const string ARG_CREATE_IF_MISSING;
+
+  static LDBCommand* InitFromCmdLineArgs(const vector<string>& args);
+  static LDBCommand* InitFromCmdLineArgs(int argc, char** argv);
+  bool ValidateCmdLineOptions();
 
   virtual leveldb::Options PrepareOptionsForOpenDB();
 
@@ -117,23 +67,6 @@ public:
     }
   }
 
-  /* Print the help message */
-  static void Help(std::string& ret) {
-    ret.append("--db=DB_PATH [");
-    ret.append(LDBCommand::BLOOM_ARG);
-    ret.append("<int,e.g.:14>] [");
-    ret.append(LDBCommand::COMPRESSION_TYPE_ARG);
-    ret.append("<no|snappy|zlib|bzip2>] [");
-    ret.append(LDBCommand::BLOCK_SIZE);
-    ret.append("<block_size_in_bytes>] [");
-    ret.append(LDBCommand::AUTO_COMPACTION);
-    ret.append("<true|false>] [");
-    ret.append(LDBCommand::WRITE_BUFFER_SIZE_ARG);
-    ret.append("<int,e.g.:4194304>] [");
-    ret.append(LDBCommand::FILE_SIZE_ARG);
-    ret.append("<int,e.g.:2097152>] ");
-  }
-
   /* Run the command, and return the execute result. */
   void Run() {
     if (!exec_state_.IsNotStarted()) {
@@ -146,7 +79,7 @@ public:
         return;
       }
     }
-    
+
     DoCommand();
     if (exec_state_.IsNotStarted()) {
       exec_state_ = LDBCommandExecuteResult::SUCCEED("");
@@ -167,9 +100,15 @@ public:
     exec_state_.Reset();
   }
 
-  static std::string HexToString(const std::string& str) {
-    std::string parsed;
-    for (unsigned int i = 0; i < str.length();) {
+  static string HexToString(const string& str) {
+    string parsed;
+    if (!boost::starts_with(str, "0x")) {
+      fprintf(stderr, "Invalid hex input %s.  Must start with 0x\n",
+              str.c_str());
+      throw "Invalid hex input";
+    }
+
+    for (unsigned int i = 2; i < str.length();) {
       int c;
       sscanf(str.c_str() + i, "%2X", &c);
       parsed.push_back(c);
@@ -178,8 +117,8 @@ public:
     return parsed;
   }
 
-  static std::string StringToHex(const std::string& str) {
-    std::string result;
+  static string StringToHex(const string& str) {
+    string result = "0x";
     char buf[10];
     for (size_t i = 0; i < str.length(); i++) {
       snprintf(buf, 10, "%02X", (unsigned char)str[i]);
@@ -189,43 +128,74 @@ public:
   }
 
   static const char* DELIM;
-  static bool ParseKeyValue(const std::string& line,
-                              std::string* key,
-                              std::string* value,
-                              bool hex) {
-    size_t pos = line.find(DELIM);
-    if (pos != std::string::npos) {
-      (*key) = line.substr(0, pos);
-      (*value) = line.substr(pos + strlen(DELIM));
-      if (hex) {
-        (*key) = HexToString(*key);
-        (*value) = HexToString(*value);
-      }
-      return true;
-    } else {
-      return false;
+
+protected:
+
+  LDBCommandExecuteResult exec_state_;
+  std::string db_path_;
+  leveldb::DB* db_;
+
+  /**
+   * true implies that this command can work if the db is opened in read-only
+   * mode.
+   */
+  bool is_read_only_;
+
+  /** If true, the key is input/output as hex in get/put/scan/delete etc. */
+  bool is_key_hex_;
+
+  /** If true, the value is input/output as hex in get/put/scan/delete etc. */
+  bool is_value_hex_;
+
+  /**
+   * Map of options passed on the command-line.
+   */
+  const map<string, string> options_;
+
+  /**
+   * Flags passed on the command-line.
+   */
+  const vector<string> flags_;
+
+  /** List of command-line options valid for this command */
+  const vector<string> valid_cmd_line_options_;
+
+  bool ParseKeyValue(const string& line, string* key, string* value,
+                      bool is_key_hex, bool is_value_hex);
+
+  LDBCommand(const map<string, string>& options, const vector<string>& flags,
+             bool is_read_only, const vector<string>& valid_cmd_line_options) :
+      db_(NULL),
+      is_read_only_(is_read_only),
+      is_key_hex_(false),
+      is_value_hex_(false),
+      options_(options),
+      flags_(flags),
+      valid_cmd_line_options_(valid_cmd_line_options) {
+
+    map<string, string>::const_iterator itr = options.find(ARG_DB);
+    if (itr != options.end()) {
+      db_path_ = itr->second;
     }
-  }
 
-  static std::string PrintKeyValue(const std::string& key,
-                                   const std::string& value,
-                                   bool hex) {
-    std::string result;
-    result.append(hex ? StringToHex(key) : key);
-    result.append(DELIM);
-    result.append(hex ? StringToHex(value) : value);
-    return result;
+    is_key_hex_ = IsKeyHex(options, flags);
+    is_value_hex_ = IsValueHex(options, flags);
   }
 
-protected:
-
   void OpenDB() {
     leveldb::Options opt = PrepareOptionsForOpenDB();
     if (!exec_state_.IsNotStarted()) {
       return;
     }
     // Open the DB.
-    leveldb::Status st = leveldb::DB::Open(opt, db_path_, &db_);
+    leveldb::Status st;
+    if (is_read_only_) {
+      //st = leveldb::DB::OpenForReadOnly(opt, db_path_, &db_);
+      // Could not get this to work
+      st = leveldb::DB::Open(opt, db_path_, &db_);
+    } else {
+      st = leveldb::DB::Open(opt, db_path_, &db_);
+    }
     if (!st.ok()) {
       std::string msg = st.ToString();
       exec_state_ = LDBCommandExecuteResult::FAILED(msg);
@@ -239,109 +209,181 @@ protected:
     }
   }
 
-  static const char* FROM_ARG;
-  static const char* END_ARG;
-  static const char* HEX_ARG;
-  LDBCommandExecuteResult exec_state_;
-  std::string db_path_;
-  leveldb::DB* db_;
+  static string PrintKeyValue(const string& key, const string& value,
+        bool is_key_hex, bool is_value_hex) {
+    string result;
+    result.append(is_key_hex ? StringToHex(key) : key);
+    result.append(DELIM);
+    result.append(is_value_hex ? StringToHex(value) : value);
+    return result;
+  }
+
+  static string PrintKeyValue(const string& key, const string& value,
+        bool is_hex) {
+    return PrintKeyValue(key, value, is_hex, is_hex);
+  }
+
+  /**
+   * Return true if the specified flag is present in the specified flags vector
+   */
+  static bool IsFlagPresent(const vector<string>& flags, const string& flag) {
+    return (std::find(flags.begin(), flags.end(), flag) != flags.end());
+  }
+
+  static string HelpRangeCmdArgs() {
+    ostringstream str_stream;
+    str_stream << " ";
+    str_stream << "[--" << ARG_FROM << "] ";
+    str_stream << "[--" << ARG_TO << "] ";
+    return str_stream.str();
+  }
+
+  /**
+   * A helper function that returns a list of command line options
+   * used by this command.  It includes the common options and the ones
+   * passed in.
+   */
+  vector<string> BuildCmdLineOptions(vector<string> options) {
+    vector<string> ret = {ARG_DB, ARG_BLOOM_BITS, ARG_BLOCK_SIZE,
+                          ARG_AUTO_COMPACTION, ARG_COMPRESSION_TYPE,
+                          ARG_WRITE_BUFFER_SIZE, ARG_FILE_SIZE};
+    ret.insert(ret.end(), options.begin(), options.end());
+    return ret;
+  }
+
+  bool ParseIntOption(const map<string, string>& options, string option,
+      int& value, LDBCommandExecuteResult& exec_state);
 
 private:
 
-  static const char* BLOOM_ARG;
-  static const char* COMPRESSION_TYPE_ARG;
-  static const char* BLOCK_SIZE;
-  static const char* AUTO_COMPACTION;
-  static const char* WRITE_BUFFER_SIZE_ARG;
-  static const char* FILE_SIZE_ARG;
-  std::vector<std::string> open_args_;
-  void parse_open_args(std::vector<std::string>& args);
+  /**
+   * Interpret command line options and flags to determine if the key
+   * should be input/output in hex.
+   */
+  bool IsKeyHex(const map<string, string>& options,
+      const vector<string>& flags) {
+    return (IsFlagPresent(flags, ARG_HEX) ||
+        IsFlagPresent(flags, ARG_KEY_HEX) ||
+        ParseBooleanOption(options, ARG_HEX, false) ||
+        ParseBooleanOption(options, ARG_KEY_HEX, false));
+  }
+
+  /**
+   * Interpret command line options and flags to determine if the value
+   * should be input/output in hex.
+   */
+  bool IsValueHex(const map<string, string>& options,
+      const vector<string>& flags) {
+    return (IsFlagPresent(flags, ARG_HEX) ||
+          IsFlagPresent(flags, ARG_VALUE_HEX) ||
+          ParseBooleanOption(options, ARG_HEX, false) ||
+          ParseBooleanOption(options, ARG_VALUE_HEX, false));
+  }
+
+  /**
+   * Returns the value of the specified option as a boolean.
+   * default_val is used if the option is not found in options.
+   * Throws an exception if the value of the option is not
+   * "true" or "false" (case insensitive).
+   */
+  bool ParseBooleanOption(const map<string, string>& options,
+      const string& option, bool default_val) {
+
+    map<string, string>::const_iterator itr = options.find(option);
+    if (itr != options.end()) {
+      string option_val = itr->second;
+      return StringToBool(itr->second);
+    }
+    return default_val;
+  }
+
+  /**
+   * Converts val to a boolean.
+   * val must be either true or false (case insensitive).
+   * Otherwise an exception is thrown.
+   */
+  bool StringToBool(string val) {
+    boost::algorithm::to_lower(val);
+    if (val == "true") {
+      return true;
+    } else if (val == "false") {
+      return false;
+    } else {
+      throw "Invalid value for boolean argument";
+    }
+  }
+
 };
 
-class Compactor: public LDBCommand {
+class CompactorCommand: public LDBCommand {
 public:
-  Compactor(std::string& db_name, std::vector<std::string>& args);
+  static string Name() { return "compact"; }
 
-  virtual ~Compactor() {}
+  CompactorCommand(const vector<string>& params,
+      const map<string, string>& options, const vector<string>& flags);
 
-  static void Help(std::string& ret);
+  static void Help(string& ret);
 
   virtual void DoCommand();
 
 private:
   bool null_from_;
-  std::string from_;
+  string from_;
   bool null_to_;
-  std::string to_;
-  bool hex_;
+  string to_;
 };
 
-class DBDumper: public LDBCommand {
+class DBDumperCommand: public LDBCommand {
 public:
-  DBDumper(std::string& db_name, std::vector<std::string>& args);
-  virtual ~DBDumper() {}
-  static void Help(std::string& ret);
+  static string Name() { return "dump"; }
+
+  DBDumperCommand(const vector<string>& params,
+      const map<string, string>& options, const vector<string>& flags);
+
+  static void Help(string& ret);
+
   virtual void DoCommand();
+
 private:
   bool null_from_;
-  std::string from_;
+  string from_;
   bool null_to_;
-  std::string to_;
+  string to_;
   int max_keys_;
   bool count_only_;
   bool print_stats_;
-  bool hex_;
-  bool hex_output_;
 
-  static const char* MAX_KEYS_ARG;
-  static const char* COUNT_ONLY_ARG;
-  static const char* STATS_ARG;
-  static const char* HEX_OUTPUT_ARG;
+  static const string ARG_COUNT_ONLY;
+  static const string ARG_STATS;
 };
 
-class DBLoader: public LDBCommand {
+class DBLoaderCommand: public LDBCommand {
 public:
-  DBLoader(std::string& db_name, std::vector<std::string>& args);
-  virtual ~DBLoader() {}
-  static void Help(std::string& ret);
+  static string Name() { return "load"; }
+
+  DBLoaderCommand(string& db_name, vector<string>& args);
+
+  DBLoaderCommand(const vector<string>& params,
+      const map<string, string>& options, const vector<string>& flags);
+
+  static void Help(string& ret);
   virtual void DoCommand();
 
   virtual leveldb::Options PrepareOptionsForOpenDB();
 
 private:
-  bool hex_input_;
   bool create_if_missing_;
   bool disable_wal_;
 
-  static const char* HEX_INPUT_ARG;
-  static const char* CREATE_IF_MISSING_ARG;
-  static const char* DISABLE_WAL_ARG;
+  static const string ARG_DISABLE_WAL;
 };
 
-class DBQuerier: public LDBCommand {
+class ReduceDBLevelsCommand : public LDBCommand {
 public:
-  DBQuerier(std::string& db_name, std::vector<std::string>& args);
-  virtual ~DBQuerier() {}
-  static void Help(std::string& ret);
-  virtual void DoCommand();
-
-private:
-  bool hex_;
-
-  static const char* HEX_ARG;
-
-  static const char* HELP_CMD;
-  static const char* GET_CMD;
-  static const char* PUT_CMD;
-  static const char* DELETE_CMD;
-};
+  static string Name() { return "reduce_levels"; }
 
-class ReduceDBLevels : public LDBCommand {
-public:
-
-  ReduceDBLevels (std::string& db_name, std::vector<std::string>& args);
-
-  ~ReduceDBLevels() {}
+  ReduceDBLevelsCommand(const vector<string>& params,
+      const map<string, string>& options, const vector<string>& flags);
 
   virtual leveldb::Options PrepareOptionsForOpenDB();
 
@@ -351,8 +393,9 @@ public:
     return true;
   }
 
-  static void Help(std::string& msg);
-  static std::vector<std::string> PrepareArgs(int new_levels,
+  static void Help(string& msg);
+
+  static vector<string> PrepareArgs(const string& db_path, int new_levels,
       bool print_old_level = false);
 
 private:
@@ -360,30 +403,159 @@ private:
   int new_levels_;
   bool print_old_levels_;
 
-  static const char* NEW_LEVLES_ARG;
-  static const char* PRINT_OLD_LEVELS_ARG;
+  static const string ARG_NEW_LEVELS;
+  static const string ARG_PRINT_OLD_LEVELS;
 
   Status GetOldNumOfLevels(leveldb::Options& opt, int* levels);
 };
 
-class WALDumper : public LDBCommand {
+class WALDumperCommand : public LDBCommand {
 public:
+  static string Name() { return "dump_wal"; }
 
-  WALDumper (std::vector<std::string>& args);
-
-  ~WALDumper() {}
+  WALDumperCommand(const vector<string>& params,
+      const map<string, string>& options, const vector<string>& flags);
 
   virtual bool  NoDBOpen() {
     return true;
   }
 
-  static void Help(std::string& ret);
+  static void Help(string& ret);
   virtual void DoCommand();
+
 private:
   bool print_header_;
-  std::string wal_file_;
+  string wal_file_;
+
+  static const string ARG_WAL_FILE;
+  static const string ARG_PRINT_HEADER;
+};
+
+
+class GetCommand : public LDBCommand {
+public:
+  static string Name() { return "get"; }
+
+  GetCommand(const vector<string>& params, const map<string, string>& options,
+      const vector<string>& flags);
+
+  virtual void DoCommand();
+
+  static void Help(string& ret);
+
+private:
+  string key_;
+};
+
+class ApproxSizeCommand : public LDBCommand {
+public:
+  static string Name() { return "approxsize"; }
+
+  ApproxSizeCommand(const vector<string>& params,
+      const map<string, string>& options, const vector<string>& flags);
+
+  virtual void DoCommand();
+
+  static void Help(string& ret);
+
+private:
+  string start_key_;
+  string end_key_;
+};
+
+class BatchPutCommand : public LDBCommand {
+public:
+  static string Name() { return "batchput"; }
+
+  BatchPutCommand(const vector<string>& params,
+      const map<string, string>& options, const vector<string>& flags);
+
+  virtual void DoCommand();
+
+  static void Help(string& ret);
+
+  virtual leveldb::Options PrepareOptionsForOpenDB();
+
+private:
+  /**
+   * The key-values to be inserted.
+   */
+  vector<std::pair<string, string>> key_values_;
+};
+
+class ScanCommand : public LDBCommand {
+public:
+  static string Name() { return "scan"; }
 
-  static const char* WAL_FILE_ARG;
+  ScanCommand(const vector<string>& params, const map<string, string>& options,
+      const vector<string>& flags);
+
+  virtual void DoCommand();
+
+  static void Help(string& ret);
+
+private:
+  string start_key_;
+  string end_key_;
+  bool start_key_specified_;
+  bool end_key_specified_;
+  int max_keys_scanned_;
 };
+
+class DeleteCommand : public LDBCommand {
+public:
+  static string Name() { return "delete"; }
+
+  DeleteCommand(const vector<string>& params,
+      const map<string, string>& options, const vector<string>& flags);
+
+  virtual void DoCommand();
+
+  static void Help(string& ret);
+
+private:
+  string key_;
+};
+
+class PutCommand : public LDBCommand {
+public:
+  static string Name() { return "put"; }
+
+  PutCommand(const vector<string>& params, const map<string, string>& options,
+      const vector<string>& flags);
+
+  virtual void DoCommand();
+
+  static void Help(string& ret);
+
+  virtual leveldb::Options PrepareOptionsForOpenDB();
+
+private:
+  string key_;
+  string value_;
+};
+
+/**
+ * Command that starts up a REPL shell that allows
+ * get/put/delete.
+ */
+class DBQuerierCommand: public LDBCommand {
+public:
+  static string Name() { return "query"; }
+
+  DBQuerierCommand(const vector<string>& params,
+      const map<string, string>& options, const vector<string>& flags);
+
+  static void Help(string& ret);
+
+  virtual void DoCommand();
+
+private:
+  static const char* HELP_CMD;
+  static const char* GET_CMD;
+  static const char* PUT_CMD;
+  static const char* DELETE_CMD;
+};
+
 }
 #endif

util/ldb_cmd_execute_result.h

@@ -0,0 +1,74 @@
+#ifndef LEVELDB_UTIL_LDB_CMD_EXECUTE_RESULT_H_
+#define LEVELDB_UTIL_LDB_CMD_EXECUTE_RESULT_H_
+
+namespace leveldb {
+
+class LDBCommandExecuteResult {
+public:
+  enum State {
+    EXEC_NOT_STARTED = 0, EXEC_SUCCEED = 1, EXEC_FAILED = 2,
+  };
+
+  LDBCommandExecuteResult() {
+    state_ = EXEC_NOT_STARTED;
+    message_ = "";
+  }
+
+  LDBCommandExecuteResult(State state, std::string& msg) {
+    state_ = state;
+    message_ = msg;
+  }
+
+  std::string ToString() {
+    std::string ret;
+    switch (state_) {
+    case EXEC_SUCCEED:
+      break;
+    case EXEC_FAILED:
+      ret.append("Failed: ");
+      break;
+    case EXEC_NOT_STARTED:
+      ret.append("Not started: ");
+    }
+    if (!message_.empty()) {
+      ret.append(message_);
+    }
+    return ret;
+  }
+
+  void Reset() {
+    state_ = EXEC_NOT_STARTED;
+    message_ = "";
+  }
+
+  bool IsSucceed() {
+    return state_ == EXEC_SUCCEED;
+  }
+
+  bool IsNotStarted() {
+    return state_ == EXEC_NOT_STARTED;
+  }
+
+  bool IsFailed() {
+    return state_ == EXEC_FAILED;
+  }
+
+  static LDBCommandExecuteResult SUCCEED(std::string msg) {
+    return LDBCommandExecuteResult(EXEC_SUCCEED, msg);
+  }
+
+  static LDBCommandExecuteResult FAILED(std::string msg) {
+    return LDBCommandExecuteResult(EXEC_FAILED, msg);
+  }
+
+private:
+  State state_;
+  std::string message_;
+
+  bool operator==(const LDBCommandExecuteResult&);
+  bool operator!=(const LDBCommandExecuteResult&);
+};
+
+}
+
+#endif