Rules Advisor: some fixes to support fetching stats from ODS (#4223)

Summary:
This PR includes fixes for some bugs that I encountered while testing the Optimizer with ODS stats support.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4223

Differential Revision: D9140786

Pulled By: poojam23

fbshipit-source-id: 045cb3f27d075c2042040ac2d561938349419516

tools/advisor/advisor/config_optimizer_example.py

@@ -78,16 +78,16 @@ if __name__ == '__main__':
     )
     # these are options that are column-family agnostic and are not yet
     # supported by the Rocksdb Options file: eg. bloom_bits=2
-    parser.add_argument(
-        '--base_db_path', required=True, type=str,
-        help='path for the Rocksdb database'
-    )
     parser.add_argument(
         '--misc_options', nargs='*',
         help='whitespace-separated list of options that are not supported ' +
         'by the Rocksdb OPTIONS file, given in the ' +
         '<option_name>=<option_value> format eg. "bloom_bits=2 ' +
         'rate_limiter_bytes_per_sec=128000000"')
+    parser.add_argument(
+        '--base_db_path', required=True, type=str,
+        help='path for the Rocksdb database'
+    )
     parser.add_argument(
         '--rules_spec', required=True, type=str,
         help='path of the file containing the expert-specified Rules'

tools/advisor/advisor/db_bench_runner.py

@@ -1,3 +1,8 @@
+# Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+#  This source code is licensed under both the GPLv2 (found in the
+#  COPYING file in the root directory) and Apache 2.0 License
+#  (found in the LICENSE.Apache file in the root directory).
+
 from advisor.bench_runner import BenchmarkRunner
 from advisor.db_log_parser import DataSource, DatabaseLogs, NO_COL_FAMILY
 from advisor.db_stats_fetcher import (
@@ -196,8 +201,10 @@ class DBBenchRunner(BenchmarkRunner):
         self._setup_db_before_experiment(db_options, db_path)
         # get the command to run the experiment
         command = self._build_experiment_command(db_options, db_path)
+        experiment_start_time = int(time.time())
         # run experiment
         self._run_command(command)
+        experiment_end_time = int(time.time())
         # parse the db_bench experiment output
         parsed_output = self._parse_output(get_perf_context=True)
 
@@ -224,10 +231,15 @@ class DBBenchRunner(BenchmarkRunner):
         }
         # Create the ODS STATS object
         if self.ods_args:
+            key_prefix = ''
+            if 'key_prefix' in self.ods_args:
+                key_prefix = self.ods_args['key_prefix']
             data_sources[DataSource.Type.TIME_SERIES].append(OdsStatsFetcher(
                 self.ods_args['client_script'],
                 self.ods_args['entity'],
-                self.ods_args['key_prefix']
+                experiment_start_time,
+                experiment_end_time,
+                key_prefix
             ))
         # return the experiment's data-sources and throughput
         return data_sources, parsed_output[self.THROUGHPUT]

tools/advisor/advisor/db_stats_fetcher.py

@@ -174,7 +174,6 @@ class OdsStatsFetcher(TimeSeriesData):
     OUTPUT_FILE = 'temp/stats_out.tmp'
     ERROR_FILE = 'temp/stats_err.tmp'
     RAPIDO_COMMAND = "%s --entity=%s --key=%s --tstart=%s --tend=%s --showtime"
-    ODS_COMMAND = '%s %s %s'  # client, entities, keys
 
     # static methods
     @staticmethod
@@ -191,16 +190,23 @@ class OdsStatsFetcher(TimeSeriesData):
         second = float(pair[1].strip())
         return [first, second]
 
-    def __init__(self, client, entities, key_prefix=None):
+    @staticmethod
+    def _get_ods_cli_stime(start_time):
+        diff = int(time.time() - int(start_time))
+        stime = str(diff) + '_s'
+        return stime
+
+    def __init__(
+        self, client, entities, start_time, end_time, key_prefix=None
+    ):
         super().__init__()
         self.client = client
         self.entities = entities
+        self.start_time = start_time
+        self.end_time = end_time
         self.key_prefix = key_prefix
         self.stats_freq_sec = 60
         self.duration_sec = 60
-        # Fetch last 3 hours data by default
-        self.end_time = int(time.time())
-        self.start_time = self.end_time - (3 * 60 * 60)
 
     def execute_script(self, command):
         print('executing...')
@@ -265,9 +271,10 @@ class OdsStatsFetcher(TimeSeriesData):
             # Parse output and populate the 'keys_ts' map
             self.parse_rapido_output()
         elif re.search('ods', self.client, re.IGNORECASE):
-            command = self.ODS_COMMAND % (
-                self.client,
-                self._get_string_in_quotes(self.entities),
+            command = (
+                self.client + ' ' +
+                '--stime=' + self._get_ods_cli_stime(self.start_time) + ' ' +
+                self._get_string_in_quotes(self.entities) + ' ' +
                 self._get_string_in_quotes(','.join(statistics))
             )
             # Run the tool and fetch the time-series data
@@ -304,17 +311,26 @@ class OdsStatsFetcher(TimeSeriesData):
             transform_desc = transform_desc + ",%)"
         else:
             transform_desc = transform_desc + ")"
-
-        command = self.RAPIDO_COMMAND + " --transform=%s --url=%s"
-        command = command % (
-            self.client,
-            self._get_string_in_quotes(','.join(entities)),
-            self._get_string_in_quotes(','.join(keys)),
-            self._get_string_in_quotes(self.start_time),
-            self._get_string_in_quotes(self.end_time),
-            self._get_string_in_quotes(transform_desc),
-            self._get_string_in_quotes(display)
-        )
+        if re.search('rapido', self.client, re.IGNORECASE):
+            command = self.RAPIDO_COMMAND + " --transform=%s --url=%s"
+            command = command % (
+                self.client,
+                self._get_string_in_quotes(','.join(entities)),
+                self._get_string_in_quotes(','.join(keys)),
+                self._get_string_in_quotes(self.start_time),
+                self._get_string_in_quotes(self.end_time),
+                self._get_string_in_quotes(transform_desc),
+                self._get_string_in_quotes(display)
+            )
+        elif re.search('ods', self.client, re.IGNORECASE):
+            command = (
+                self.client + ' ' +
+                '--stime=' + self._get_ods_cli_stime(self.start_time) + ' ' +
+                '--fburlonly ' +
+                self._get_string_in_quotes(entities) + ' ' +
+                self._get_string_in_quotes(','.join(keys)) + ' ' +
+                self._get_string_in_quotes(transform_desc)
+            )
         self.execute_script(command)
         url = ""
         with open(self.OUTPUT_FILE, 'r') as fp: