NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
7580 Commits
4 Branches
0 Tags
169 MiB
 
 
 
 
 
 
Tag: Branch: Tree: a2de8e52bb
Branches Tags
${ item.name }
Create tag ${ searchTerm }
Create branch ${ searchTerm }
from 'a2de8e52bb'
${ noResults }
rocksdb/tools/advisor/advisor/db_stats_fetcher.py

338 lines
14 KiB
Raw Blame History

# Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
# This source code is licensed under both the GPLv2 (found in the
# COPYING file in the root directory) and Apache 2.0 License
# (found in the LICENSE.Apache file in the root directory).
from advisor.db_log_parser import Log
from advisor.db_timeseries_parser import TimeSeriesData, NO_ENTITY
import copy
import glob
import re
import subprocess
import time
class LogStatsParser(TimeSeriesData):
STATS = 'STATISTICS:'
@staticmethod
def parse_log_line_for_stats(log_line):
# Example stat line (from LOG file):
# "rocksdb.db.get.micros P50 : 8.4 P95 : 21.8 P99 : 33.9 P100 : 92.0\n"
token_list = log_line.strip().split()
# token_list = ['rocksdb.db.get.micros', 'P50', ':', '8.4', 'P95', ':',
# '21.8', 'P99', ':', '33.9', 'P100', ':', '92.0']
stat_prefix = token_list[0] + '.' # 'rocksdb.db.get.micros.'
stat_values = [
token
for token in token_list[1:]
if token != ':'
]
# stat_values = ['P50', '8.4', 'P95', '21.8', 'P99', '33.9', 'P100',
# '92.0']
stat_dict = {}
for ix, metric in enumerate(stat_values):
if ix % 2 == 0:
stat_name = stat_prefix + metric
stat_name = stat_name.lower() # Note: case insensitive names
else:
stat_dict[stat_name] = float(metric)
# stat_dict = {'rocksdb.db.get.micros.p50': 8.4,
# 'rocksdb.db.get.micros.p95': 21.8, 'rocksdb.db.get.micros.p99': 33.9,
# 'rocksdb.db.get.micros.p100': 92.0}
return stat_dict
def __init__(self, logs_path_prefix, stats_freq_sec):
super().__init__()
self.logs_file_prefix = logs_path_prefix
self.stats_freq_sec = stats_freq_sec
self.duration_sec = 60
def get_keys_from_conditions(self, conditions):
# Note: case insensitive stat names
reqd_stats = []
for cond in conditions:
for key in cond.keys:
key = key.lower()
# some keys are prepended with '[]' for OdsStatsFetcher to
# replace this with the appropriate key_prefix, remove these
# characters here since the LogStatsParser does not need
# a prefix
if key.startswith('[]'):
reqd_stats.append(key[2:])
else:
reqd_stats.append(key)
return reqd_stats
def add_to_timeseries(self, log, reqd_stats):
# this method takes in the Log object that contains the Rocksdb stats
# and a list of required stats, then it parses the stats line by line
# to fetch required stats and add them to the keys_ts object
# Example: reqd_stats = ['rocksdb.block.cache.hit.count',
# 'rocksdb.db.get.micros.p99']
# Let log.get_message() returns following string:
# "[WARN] [db/db_impl.cc:485] STATISTICS:\n
# rocksdb.block.cache.miss COUNT : 1459\n
# rocksdb.block.cache.hit COUNT : 37\n
# ...
# rocksdb.db.get.micros P50 : 15.6 P95 : 39.7 P99 : 62.6 P100 : 148.0\n
# ..."
new_lines = log.get_message().split('\n')
# let log_ts = 1532518219
log_ts = log.get_timestamp()
# example updates to keys_ts:
# keys_ts[NO_ENTITY]['rocksdb.db.get.micros.p99'][1532518219] = 62.6
# keys_ts[NO_ENTITY]['rocksdb.block.cache.hit.count'][1532518219] = 37
for line in new_lines[1:]: # new_lines[0] does not contain any stats
stats_on_line = self.parse_log_line_for_stats(line)
for stat in stats_on_line:
if stat in reqd_stats:
if stat not in self.keys_ts[NO_ENTITY]:
self.keys_ts[NO_ENTITY][stat] = {}
self.keys_ts[NO_ENTITY][stat][log_ts] = stats_on_line[stat]
def fetch_timeseries(self, reqd_stats):
# this method parses the Rocksdb LOG file and generates timeseries for
# each of the statistic in the list reqd_stats
self.keys_ts = {NO_ENTITY: {}}
for file_name in glob.glob(self.logs_file_prefix + '*'):
# TODO(poojam23): find a way to distinguish between 'old' log files
# from current and previous experiments, present in the same
# directory
if re.search('old', file_name, re.IGNORECASE):
continue
with open(file_name, 'r') as db_logs:
new_log = None
for line in db_logs:
if Log.is_new_log(line):
if (
new_log and
re.search(self.STATS, new_log.get_message())
):
self.add_to_timeseries(new_log, reqd_stats)
new_log = Log(line, column_families=[])
else:
# To account for logs split into multiple lines
new_log.append_message(line)
# Check for the last log in the file.
if new_log and re.search(self.STATS, new_log.get_message()):
self.add_to_timeseries(new_log, reqd_stats)
class DatabasePerfContext(TimeSeriesData):
# TODO(poojam23): check if any benchrunner provides PerfContext sampled at
# regular intervals
def __init__(self, perf_context_ts, stats_freq_sec, cumulative):
'''
perf_context_ts is expected to be in the following format:
Dict[metric, Dict[timestamp, value]], where for
each (metric, timestamp) pair, the value is database-wide (i.e.
summed over all the threads involved)
if stats_freq_sec == 0, per-metric only one value is reported
'''
super().__init__()
self.stats_freq_sec = stats_freq_sec
self.keys_ts = {NO_ENTITY: perf_context_ts}
if cumulative:
self.unaccumulate_metrics()
def unaccumulate_metrics(self):
# if the perf context metrics provided are cumulative in nature, this
# method can be used to convert them to a disjoint format
epoch_ts = copy.deepcopy(self.keys_ts)
for stat in self.keys_ts[NO_ENTITY]:
timeseries = sorted(
list(self.keys_ts[NO_ENTITY][stat].keys()), reverse=True
)
if len(timeseries) < 2:
continue
for ix, ts in enumerate(timeseries[:-1]):
epoch_ts[NO_ENTITY][stat][ts] = (
epoch_ts[NO_ENTITY][stat][ts] -
epoch_ts[NO_ENTITY][stat][timeseries[ix+1]]
)
if epoch_ts[NO_ENTITY][stat][ts] < 0:
raise ValueError('DBPerfContext: really cumulative?')
# drop the smallest timestamp in the timeseries for this metric
epoch_ts[NO_ENTITY][stat].pop(timeseries[-1])
self.keys_ts = epoch_ts
def get_keys_from_conditions(self, conditions):
reqd_stats = []
for cond in conditions:
reqd_stats.extend([key.lower() for key in cond.keys])
return reqd_stats
def fetch_timeseries(self, statistics):
# this method is redundant for DatabasePerfContext because the __init__
# does the job of populating 'keys_ts'
pass
class OdsStatsFetcher(TimeSeriesData):
# class constants
OUTPUT_FILE = 'temp/stats_out.tmp'
ERROR_FILE = 'temp/stats_err.tmp'
RAPIDO_COMMAND = "%s --entity=%s --key=%s --tstart=%s --tend=%s --showtime"
# static methods
@staticmethod
def _get_string_in_quotes(value):
return '"' + str(value) + '"'
@staticmethod
def _get_time_value_pair(pair_string):
# example pair_string: '[1532544591, 97.3653601828]'
pair_string = pair_string.replace('[', '')
pair_string = pair_string.replace(']', '')
pair = pair_string.split(',')
first = int(pair[0].strip())
second = float(pair[1].strip())
return [first, second]
@staticmethod
def _get_ods_cli_stime(start_time):
diff = int(time.time() - int(start_time))
stime = str(diff) + '_s'
return stime
def __init__(
self, client, entities, start_time, end_time, key_prefix=None
):
super().__init__()
self.client = client
self.entities = entities
self.start_time = start_time
self.end_time = end_time
self.key_prefix = key_prefix
self.stats_freq_sec = 60
self.duration_sec = 60
def execute_script(self, command):
print('executing...')
print(command)
out_file = open(self.OUTPUT_FILE, "w+")
err_file = open(self.ERROR_FILE, "w+")
subprocess.call(command, shell=True, stdout=out_file, stderr=err_file)
out_file.close()
err_file.close()
def parse_rapido_output(self):
# Output looks like the following:
# <entity_name>\t<key_name>\t[[ts, value], [ts, value], ...]
# ts = timestamp; value = value of key_name in entity_name at time ts
self.keys_ts = {}
with open(self.OUTPUT_FILE, 'r') as fp:
for line in fp:
token_list = line.strip().split('\t')
entity = token_list[0]
key = token_list[1]
if entity not in self.keys_ts:
self.keys_ts[entity] = {}
if key not in self.keys_ts[entity]:
self.keys_ts[entity][key] = {}
list_of_lists = [
self._get_time_value_pair(pair_string)
for pair_string in token_list[2].split('],')
]
value = {pair[0]: pair[1] for pair in list_of_lists}
self.keys_ts[entity][key] = value
def parse_ods_output(self):
# Output looks like the following:
# <entity_name>\t<key_name>\t<timestamp>\t<value>
# there is one line per (entity_name, key_name, timestamp)
self.keys_ts = {}
with open(self.OUTPUT_FILE, 'r') as fp:
for line in fp:
token_list = line.split()
entity = token_list[0]
if entity not in self.keys_ts:
self.keys_ts[entity] = {}
key = token_list[1]
if key not in self.keys_ts[entity]:
self.keys_ts[entity][key] = {}
self.keys_ts[entity][key][token_list[2]] = token_list[3]
def fetch_timeseries(self, statistics):
# this method fetches the timeseries of required stats from the ODS
# service and populates the 'keys_ts' object appropriately
print('OdsStatsFetcher: fetching ' + str(statistics))
if re.search('rapido', self.client, re.IGNORECASE):
command = self.RAPIDO_COMMAND % (
self.client,
self._get_string_in_quotes(self.entities),
self._get_string_in_quotes(','.join(statistics)),
self._get_string_in_quotes(self.start_time),
self._get_string_in_quotes(self.end_time)
)
# Run the tool and fetch the time-series data
self.execute_script(command)
# Parse output and populate the 'keys_ts' map
self.parse_rapido_output()
elif re.search('ods', self.client, re.IGNORECASE):
command = (
self.client + ' ' +
'--stime=' + self._get_ods_cli_stime(self.start_time) + ' ' +
self._get_string_in_quotes(self.entities) + ' ' +
self._get_string_in_quotes(','.join(statistics))
)
# Run the tool and fetch the time-series data
self.execute_script(command)
# Parse output and populate the 'keys_ts' map
self.parse_ods_output()
def get_keys_from_conditions(self, conditions):
reqd_stats = []
for cond in conditions:
for key in cond.keys:
use_prefix = False
if key.startswith('[]'):
use_prefix = True
key = key[2:]
# TODO(poojam23): this is very hacky and needs to be improved
if key.startswith("rocksdb"):
key += ".60"
if use_prefix:
if not self.key_prefix:
print('Warning: OdsStatsFetcher might need key prefix')
print('for the key: ' + key)
else:
key = self.key_prefix + "." + key
reqd_stats.append(key)
return reqd_stats
def fetch_rate_url(self, entities, keys, window_len, percent, display):
# type: (List[str], List[str], str, str, bool) -> str
transform_desc = (
"rate(" + str(window_len) + ",duration=" + str(self.duration_sec)
)
if percent:
transform_desc = transform_desc + ",%)"
else:
transform_desc = transform_desc + ")"
if re.search('rapido', self.client, re.IGNORECASE):
command = self.RAPIDO_COMMAND + " --transform=%s --url=%s"
command = command % (
self.client,
self._get_string_in_quotes(','.join(entities)),
self._get_string_in_quotes(','.join(keys)),
self._get_string_in_quotes(self.start_time),
self._get_string_in_quotes(self.end_time),
self._get_string_in_quotes(transform_desc),
self._get_string_in_quotes(display)
)
elif re.search('ods', self.client, re.IGNORECASE):
command = (
self.client + ' ' +
'--stime=' + self._get_ods_cli_stime(self.start_time) + ' ' +
'--fburlonly ' +
self._get_string_in_quotes(entities) + ' ' +
self._get_string_in_quotes(','.join(keys)) + ' ' +
self._get_string_in_quotes(transform_desc)
)
self.execute_script(command)
url = ""
with open(self.OUTPUT_FILE, 'r') as fp:
url = fp.readline()
return url