Rules Advisor: some fixes to support fetching stats from ODS (#4223)

Summary:
This PR includes fixes for some bugs that I encountered while testing the Optimizer with ODS stats support.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4223

Differential Revision: D9140786

Pulled By: poojam23

fbshipit-source-id: 045cb3f27d075c2042040ac2d561938349419516
main
Pooja Malik 6 years ago committed by Facebook Github Bot
parent 892a156267
commit 9dbf39399e
  1. 8
      tools/advisor/advisor/config_optimizer_example.py
  2. 14
      tools/advisor/advisor/db_bench_runner.py
  3. 34
      tools/advisor/advisor/db_stats_fetcher.py

@ -78,16 +78,16 @@ if __name__ == '__main__':
)
# these are options that are column-family agnostic and are not yet
# supported by the Rocksdb Options file: eg. bloom_bits=2
parser.add_argument(
'--base_db_path', required=True, type=str,
help='path for the Rocksdb database'
)
parser.add_argument(
'--misc_options', nargs='*',
help='whitespace-separated list of options that are not supported ' +
'by the Rocksdb OPTIONS file, given in the ' +
'<option_name>=<option_value> format eg. "bloom_bits=2 ' +
'rate_limiter_bytes_per_sec=128000000"')
parser.add_argument(
'--base_db_path', required=True, type=str,
help='path for the Rocksdb database'
)
parser.add_argument(
'--rules_spec', required=True, type=str,
help='path of the file containing the expert-specified Rules'

@ -1,3 +1,8 @@
# Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
# This source code is licensed under both the GPLv2 (found in the
# COPYING file in the root directory) and Apache 2.0 License
# (found in the LICENSE.Apache file in the root directory).
from advisor.bench_runner import BenchmarkRunner
from advisor.db_log_parser import DataSource, DatabaseLogs, NO_COL_FAMILY
from advisor.db_stats_fetcher import (
@ -196,8 +201,10 @@ class DBBenchRunner(BenchmarkRunner):
self._setup_db_before_experiment(db_options, db_path)
# get the command to run the experiment
command = self._build_experiment_command(db_options, db_path)
experiment_start_time = int(time.time())
# run experiment
self._run_command(command)
experiment_end_time = int(time.time())
# parse the db_bench experiment output
parsed_output = self._parse_output(get_perf_context=True)
@ -224,10 +231,15 @@ class DBBenchRunner(BenchmarkRunner):
}
# Create the ODS STATS object
if self.ods_args:
key_prefix = ''
if 'key_prefix' in self.ods_args:
key_prefix = self.ods_args['key_prefix']
data_sources[DataSource.Type.TIME_SERIES].append(OdsStatsFetcher(
self.ods_args['client_script'],
self.ods_args['entity'],
self.ods_args['key_prefix']
experiment_start_time,
experiment_end_time,
key_prefix
))
# return the experiment's data-sources and throughput
return data_sources, parsed_output[self.THROUGHPUT]

@ -174,7 +174,6 @@ class OdsStatsFetcher(TimeSeriesData):
OUTPUT_FILE = 'temp/stats_out.tmp'
ERROR_FILE = 'temp/stats_err.tmp'
RAPIDO_COMMAND = "%s --entity=%s --key=%s --tstart=%s --tend=%s --showtime"
ODS_COMMAND = '%s %s %s' # client, entities, keys
# static methods
@staticmethod
@ -191,16 +190,23 @@ class OdsStatsFetcher(TimeSeriesData):
second = float(pair[1].strip())
return [first, second]
def __init__(self, client, entities, key_prefix=None):
@staticmethod
def _get_ods_cli_stime(start_time):
diff = int(time.time() - int(start_time))
stime = str(diff) + '_s'
return stime
def __init__(
self, client, entities, start_time, end_time, key_prefix=None
):
super().__init__()
self.client = client
self.entities = entities
self.start_time = start_time
self.end_time = end_time
self.key_prefix = key_prefix
self.stats_freq_sec = 60
self.duration_sec = 60
# Fetch last 3 hours data by default
self.end_time = int(time.time())
self.start_time = self.end_time - (3 * 60 * 60)
def execute_script(self, command):
print('executing...')
@ -265,9 +271,10 @@ class OdsStatsFetcher(TimeSeriesData):
# Parse output and populate the 'keys_ts' map
self.parse_rapido_output()
elif re.search('ods', self.client, re.IGNORECASE):
command = self.ODS_COMMAND % (
self.client,
self._get_string_in_quotes(self.entities),
command = (
self.client + ' ' +
'--stime=' + self._get_ods_cli_stime(self.start_time) + ' ' +
self._get_string_in_quotes(self.entities) + ' ' +
self._get_string_in_quotes(','.join(statistics))
)
# Run the tool and fetch the time-series data
@ -304,7 +311,7 @@ class OdsStatsFetcher(TimeSeriesData):
transform_desc = transform_desc + ",%)"
else:
transform_desc = transform_desc + ")"
if re.search('rapido', self.client, re.IGNORECASE):
command = self.RAPIDO_COMMAND + " --transform=%s --url=%s"
command = command % (
self.client,
@ -315,6 +322,15 @@ class OdsStatsFetcher(TimeSeriesData):
self._get_string_in_quotes(transform_desc),
self._get_string_in_quotes(display)
)
elif re.search('ods', self.client, re.IGNORECASE):
command = (
self.client + ' ' +
'--stime=' + self._get_ods_cli_stime(self.start_time) + ' ' +
'--fburlonly ' +
self._get_string_in_quotes(entities) + ' ' +
self._get_string_in_quotes(','.join(keys)) + ' ' +
self._get_string_in_quotes(transform_desc)
)
self.execute_script(command)
url = ""
with open(self.OUTPUT_FILE, 'r') as fp:

Loading…
Cancel
Save