# Copyright (c) 2011-present, Facebook, Inc. All rights reserved. # This source code is licensed under both the GPLv2 (found in the # COPYING file in the root directory) and Apache 2.0 License # (found in the LICENSE.Apache file in the root directory). import copy from advisor.db_log_parser import DataSource, NO_COL_FAMILY from advisor.ini_parser import IniParser import os class OptionsSpecParser(IniParser): @staticmethod def is_new_option(line): return '=' in line @staticmethod def get_section_type(line): ''' Example section header: [TableOptions/BlockBasedTable "default"] Here ConfigurationOptimizer returned would be 'TableOptions.BlockBasedTable' ''' section_path = line.strip()[1:-1].split()[0] section_type = '.'.join(section_path.split('/')) return section_type @staticmethod def get_section_name(line): # example: get_section_name('[CFOptions "default"]') token_list = line.strip()[1:-1].split('"') # token_list = ['CFOptions', 'default', ''] if len(token_list) < 3: return None return token_list[1] # return 'default' @staticmethod def get_section_str(section_type, section_name): # Example: # Case 1: get_section_str('DBOptions', NO_COL_FAMILY) # Case 2: get_section_str('TableOptions.BlockBasedTable', 'default') section_type = '/'.join(section_type.strip().split('.')) # Case 1: section_type = 'DBOptions' # Case 2: section_type = 'TableOptions/BlockBasedTable' section_str = '[' + section_type if section_name == NO_COL_FAMILY: # Case 1: '[DBOptions]' return (section_str + ']') else: # Case 2: '[TableOptions/BlockBasedTable "default"]' return section_str + ' "' + section_name + '"]' @staticmethod def get_option_str(key, values): option_str = key + '=' # get_option_str('db_log_dir', None), returns 'db_log_dir=' if values: # example: # get_option_str('max_bytes_for_level_multiplier_additional', # [1,1,1,1,1,1,1]), returned string: # 'max_bytes_for_level_multiplier_additional=1:1:1:1:1:1:1' if isinstance(values, list): for value in values: option_str += (str(value) + ':') option_str = option_str[:-1] else: # example: get_option_str('write_buffer_size', 1048576) # returned string: 'write_buffer_size=1048576' option_str += str(values) return option_str class DatabaseOptions(DataSource): @staticmethod def is_misc_option(option_name): # these are miscellaneous options that are not yet supported by the # Rocksdb options file, hence they are not prefixed with any section # name return '.' not in option_name @staticmethod def get_options_diff(opt_old, opt_new): # type: Dict[option, Dict[col_fam, value]] X 2 -> # Dict[option, Dict[col_fam, Tuple(old_value, new_value)]] # note: diff should contain a tuple of values only if they are # different from each other options_union = set(opt_old.keys()).union(set(opt_new.keys())) diff = {} for opt in options_union: diff[opt] = {} # if option in options_union, then it must be in one of the configs if opt not in opt_old: for col_fam in opt_new[opt]: diff[opt][col_fam] = (None, opt_new[opt][col_fam]) elif opt not in opt_new: for col_fam in opt_old[opt]: diff[opt][col_fam] = (opt_old[opt][col_fam], None) else: for col_fam in opt_old[opt]: if col_fam in opt_new[opt]: if opt_old[opt][col_fam] != opt_new[opt][col_fam]: diff[opt][col_fam] = ( opt_old[opt][col_fam], opt_new[opt][col_fam] ) else: diff[opt][col_fam] = (opt_old[opt][col_fam], None) for col_fam in opt_new[opt]: if col_fam in opt_old[opt]: if opt_old[opt][col_fam] != opt_new[opt][col_fam]: diff[opt][col_fam] = ( opt_old[opt][col_fam], opt_new[opt][col_fam] ) else: diff[opt][col_fam] = (None, opt_new[opt][col_fam]) if not diff[opt]: diff.pop(opt) return diff def __init__(self, rocksdb_options, misc_options=None): super().__init__(DataSource.Type.DB_OPTIONS) # The options are stored in the following data structure: # Dict[section_type, Dict[section_name, Dict[option_name, value]]] self.options_dict = None self.column_families = None # Load the options from the given file to a dictionary. self.load_from_source(rocksdb_options) # Setup the miscellaneous options expected to be List[str], where each # element in the List has the format "=" # These options are the ones that are not yet supported by the Rocksdb # OPTIONS file, so they are provided separately self.setup_misc_options(misc_options) def setup_misc_options(self, misc_options): self.misc_options = {} if misc_options: for option_pair_str in misc_options: option_name = option_pair_str.split('=')[0].strip() option_value = option_pair_str.split('=')[1].strip() self.misc_options[option_name] = option_value def load_from_source(self, options_path): self.options_dict = {} with open(options_path, 'r') as db_options: for line in db_options: line = OptionsSpecParser.remove_trailing_comment(line) if not line: continue if OptionsSpecParser.is_section_header(line): curr_sec_type = ( OptionsSpecParser.get_section_type(line) ) curr_sec_name = OptionsSpecParser.get_section_name(line) if curr_sec_type not in self.options_dict: self.options_dict[curr_sec_type] = {} if not curr_sec_name: curr_sec_name = NO_COL_FAMILY self.options_dict[curr_sec_type][curr_sec_name] = {} # example: if the line read from the Rocksdb OPTIONS file # is [CFOptions "default"], then the section type is # CFOptions and 'default' is the name of a column family # that for this database, so it's added to the list of # column families stored in this object if curr_sec_type == 'CFOptions': if not self.column_families: self.column_families = [] self.column_families.append(curr_sec_name) elif OptionsSpecParser.is_new_option(line): key, value = OptionsSpecParser.get_key_value_pair(line) self.options_dict[curr_sec_type][curr_sec_name][key] = ( value ) else: error = 'Not able to parse line in Options file.' OptionsSpecParser.exit_with_parse_error(line, error) def get_misc_options(self): # these are options that are not yet supported by the Rocksdb OPTIONS # file, hence they are provided and stored separately return self.misc_options def get_column_families(self): return self.column_families def get_all_options(self): # This method returns all the options that are stored in this object as # a: Dict[.: Dict[col_fam, option_value]] all_options = [] # Example: in the section header '[CFOptions "default"]' read from the # OPTIONS file, sec_type='CFOptions' for sec_type in self.options_dict: for col_fam in self.options_dict[sec_type]: for opt_name in self.options_dict[sec_type][col_fam]: option = sec_type + '.' + opt_name all_options.append(option) all_options.extend(list(self.misc_options.keys())) return self.get_options(all_options) def get_options(self, reqd_options): # type: List[str] -> Dict[str, Dict[str, Any]] # List[option] -> Dict[option, Dict[col_fam, value]] reqd_options_dict = {} for option in reqd_options: if DatabaseOptions.is_misc_option(option): # the option is not prefixed by '.' because it is # not yet supported by the Rocksdb OPTIONS file; so it has to # be fetched from the misc_options dictionary if option not in self.misc_options: continue if option not in reqd_options_dict: reqd_options_dict[option] = {} reqd_options_dict[option][NO_COL_FAMILY] = ( self.misc_options[option] ) else: # Example: option = 'TableOptions.BlockBasedTable.block_align' # then, sec_type = 'TableOptions.BlockBasedTable' sec_type = '.'.join(option.split('.')[:-1]) # opt_name = 'block_align' opt_name = option.split('.')[-1] if sec_type not in self.options_dict: continue for col_fam in self.options_dict[sec_type]: if opt_name in self.options_dict[sec_type][col_fam]: if option not in reqd_options_dict: reqd_options_dict[option] = {} reqd_options_dict[option][col_fam] = ( self.options_dict[sec_type][col_fam][opt_name] ) return reqd_options_dict def update_options(self, options): # An example 'options' object looks like: # {'DBOptions.max_background_jobs': {NO_COL_FAMILY: 2}, # 'CFOptions.write_buffer_size': {'default': 1048576, 'cf_A': 128000}, # 'bloom_bits': {NO_COL_FAMILY: 4}} for option in options: if DatabaseOptions.is_misc_option(option): # this is a misc_option i.e. an option that is not yet # supported by the Rocksdb OPTIONS file, so it is not prefixed # by '.' and must be stored in the separate # misc_options dictionary if NO_COL_FAMILY not in options[option]: print( 'WARNING(DatabaseOptions.update_options): not ' + 'updating option ' + option + ' because it is in ' + 'misc_option format but its scope is not ' + NO_COL_FAMILY + '. Check format of option.' ) continue self.misc_options[option] = options[option][NO_COL_FAMILY] else: sec_name = '.'.join(option.split('.')[:-1]) opt_name = option.split('.')[-1] if sec_name not in self.options_dict: self.options_dict[sec_name] = {} for col_fam in options[option]: # if the option is not already present in the dictionary, # it will be inserted, else it will be updated to the new # value if col_fam not in self.options_dict[sec_name]: self.options_dict[sec_name][col_fam] = {} self.options_dict[sec_name][col_fam][opt_name] = ( copy.deepcopy(options[option][col_fam]) ) def generate_options_config(self, nonce): # this method generates a Rocksdb OPTIONS file in the INI format from # the options stored in self.options_dict this_path = os.path.abspath(os.path.dirname(__file__)) file_name = '../temp/OPTIONS_' + str(nonce) + '.tmp' file_path = os.path.join(this_path, file_name) with open(file_path, 'w') as fp: for section in self.options_dict: for col_fam in self.options_dict[section]: fp.write( OptionsSpecParser.get_section_str(section, col_fam) + '\n' ) for option in self.options_dict[section][col_fam]: values = self.options_dict[section][col_fam][option] fp.write( OptionsSpecParser.get_option_str(option, values) + '\n' ) fp.write('\n') return file_path def check_and_trigger_conditions(self, conditions): for cond in conditions: reqd_options_dict = self.get_options(cond.options) # This contains the indices of options that are specific to some # column family and are not database-wide options. incomplete_option_ix = [] options = [] missing_reqd_option = False for ix, option in enumerate(cond.options): if option not in reqd_options_dict: print( 'WARNING(DatabaseOptions.check_and_trigger): ' + 'skipping condition ' + cond.name + ' because it ' 'requires option ' + option + ' but this option is' + ' not available' ) missing_reqd_option = True break # required option is absent if NO_COL_FAMILY in reqd_options_dict[option]: options.append(reqd_options_dict[option][NO_COL_FAMILY]) else: options.append(None) incomplete_option_ix.append(ix) if missing_reqd_option: continue # if all the options are database-wide options if not incomplete_option_ix: try: if eval(cond.eval_expr): cond.set_trigger({NO_COL_FAMILY: options}) except Exception as e: print( 'WARNING(DatabaseOptions) check_and_trigger:' + str(e) ) continue # for all the options that are not database-wide, we look for their # values specific to column families col_fam_options_dict = {} for col_fam in self.column_families: present = True for ix in incomplete_option_ix: option = cond.options[ix] if col_fam not in reqd_options_dict[option]: present = False break options[ix] = reqd_options_dict[option][col_fam] if present: try: if eval(cond.eval_expr): col_fam_options_dict[col_fam] = ( copy.deepcopy(options) ) except Exception as e: print( 'WARNING(DatabaseOptions) check_and_trigger: ' + str(e) ) # Trigger for an OptionCondition object is of the form: # Dict[col_fam_name: List[option_value]] # where col_fam_name is the name of a column family for which # 'eval_expr' evaluated to True and List[option_value] is the list # of values of the options specified in the condition's 'options' # field if col_fam_options_dict: cond.set_trigger(col_fam_options_dict)