# Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
# This source code is licensed under both the GPLv2 (found in the
# COPYING file in the root directory) and Apache 2.0 License
# (found in the LICENSE.Apache file in the root directory).
from advisor.db_log_parser import NO_COL_FAMILY
from advisor.db_options_parser import DatabaseOptions
from advisor.rule_parser import Condition, OptionCondition
import os
import unittest
class TestDatabaseOptions(unittest.TestCase):
def setUp(self):
self.this_path = os.path.abspath(os.path.dirname(__file__))
self.og_options = os.path.join(
self.this_path, 'input_files/OPTIONS-000005'
)
misc_options = [
'bloom_bits = 4', 'rate_limiter_bytes_per_sec = 1024000'
]
# create the options object
self.db_options = DatabaseOptions(self.og_options, misc_options)
# perform clean-up before running tests
self.generated_options = os.path.join(
self.this_path, '../temp/OPTIONS_testing.tmp'
)
if os.path.isfile(self.generated_options):
os.remove(self.generated_options)
def test_get_options_diff(self):
old_opt = {
'DBOptions.stats_dump_freq_sec': {NO_COL_FAMILY: '20'},
'CFOptions.write_buffer_size': {
'default': '1024000',
'col_fam_A': '128000',
'col_fam_B': '128000000'
},
'DBOptions.use_fsync': {NO_COL_FAMILY: 'true'},
'DBOptions.max_log_file_size': {NO_COL_FAMILY: '128000000'}
}
new_opt = {
'bloom_bits': {NO_COL_FAMILY: '4'},
'CFOptions.write_buffer_size': {
'default': '128000000',
'col_fam_A': '128000',
'col_fam_C': '128000000'
},
'DBOptions.use_fsync': {NO_COL_FAMILY: 'true'},
'DBOptions.max_log_file_size': {NO_COL_FAMILY: '0'}
}
diff = DatabaseOptions.get_options_diff(old_opt, new_opt)
expected_diff = {
'DBOptions.stats_dump_freq_sec': {NO_COL_FAMILY: ('20', None)},
'bloom_bits': {NO_COL_FAMILY: (None, '4')},
'CFOptions.write_buffer_size': {
'default': ('1024000', '128000000'),
'col_fam_B': ('128000000', None),
'col_fam_C': (None, '128000000')
},
'DBOptions.max_log_file_size': {NO_COL_FAMILY: ('128000000', '0')}
}
self.assertDictEqual(diff, expected_diff)
def test_is_misc_option(self):
self.assertTrue(DatabaseOptions.is_misc_option('bloom_bits'))
self.assertFalse(
DatabaseOptions.is_misc_option('DBOptions.stats_dump_freq_sec')
)
def test_set_up(self):
options = self.db_options.get_all_options()
self.assertEqual(22, len(options.keys()))
expected_misc_options = {
'bloom_bits': '4', 'rate_limiter_bytes_per_sec': '1024000'
}
self.assertDictEqual(
expected_misc_options, self.db_options.get_misc_options()
)
self.assertListEqual(
['default', 'col_fam_A'], self.db_options.get_column_families()
)
def test_get_options(self):
opt_to_get = [
'DBOptions.manual_wal_flush', 'DBOptions.db_write_buffer_size',
'bloom_bits', 'CFOptions.compaction_filter_factory',
'CFOptions.num_levels', 'rate_limiter_bytes_per_sec',
'TableOptions.BlockBasedTable.block_align', 'random_option'
]
options = self.db_options.get_options(opt_to_get)
expected_options = {
'DBOptions.manual_wal_flush': {NO_COL_FAMILY: 'false'},
'DBOptions.db_write_buffer_size': {NO_COL_FAMILY: '0'},
'bloom_bits': {NO_COL_FAMILY: '4'},
'CFOptions.compaction_filter_factory': {
'default': 'nullptr', 'col_fam_A': 'nullptr'
},
'CFOptions.num_levels': {'default': '7', 'col_fam_A': '5'},
'rate_limiter_bytes_per_sec': {NO_COL_FAMILY: '1024000'},
'TableOptions.BlockBasedTable.block_align': {
'default': 'false', 'col_fam_A': 'true'
}
}
self.assertDictEqual(expected_options, options)
def test_update_options(self):
# add new, update old, set old
# before updating
expected_old_opts = {
'DBOptions.db_log_dir': {NO_COL_FAMILY: None},
'DBOptions.manual_wal_flush': {NO_COL_FAMILY: 'false'},
'bloom_bits': {NO_COL_FAMILY: '4'},
'CFOptions.num_levels': {'default': '7', 'col_fam_A': '5'},
'TableOptions.BlockBasedTable.block_restart_interval': {
'col_fam_A': '16'
}
}
get_opts = list(expected_old_opts.keys())
options = self.db_options.get_options(get_opts)
self.assertEqual(expected_old_opts, options)
# after updating options
update_opts = {
'DBOptions.db_log_dir': {NO_COL_FAMILY: '/dev/shm'},
'DBOptions.manual_wal_flush': {NO_COL_FAMILY: 'true'},
'bloom_bits': {NO_COL_FAMILY: '2'},
'CFOptions.num_levels': {'col_fam_A': '7'},
'TableOptions.BlockBasedTable.block_restart_interval': {
'default': '32'
},
'random_misc_option': {NO_COL_FAMILY: 'something'}
}
self.db_options.update_options(update_opts)
update_opts['CFOptions.num_levels']['default'] = '7'
update_opts['TableOptions.BlockBasedTable.block_restart_interval'] = {
'default': '32', 'col_fam_A': '16'
}
get_opts.append('random_misc_option')
options = self.db_options.get_options(get_opts)
self.assertDictEqual(update_opts, options)
expected_misc_options = {
'bloom_bits': '2',
'rate_limiter_bytes_per_sec': '1024000',
'random_misc_option': 'something'
}
self.assertDictEqual(
expected_misc_options, self.db_options.get_misc_options()
)
def test_generate_options_config(self):
# make sure file does not exist from before
self.assertFalse(os.path.isfile(self.generated_options))
self.db_options.generate_options_config('testing')
self.assertTrue(os.path.isfile(self.generated_options))
def test_check_and_trigger_conditions(self):
# options only from CFOptions
# setup the OptionCondition objects to check and trigger
update_dict = {
'CFOptions.level0_file_num_compaction_trigger': {'col_fam_A': '4'},
'CFOptions.max_bytes_for_level_base': {'col_fam_A': '10'}
}
self.db_options.update_options(update_dict)
cond1 = Condition('opt-cond-1')
cond1 = OptionCondition.create(cond1)
cond1.set_parameter(
'options', [
'CFOptions.level0_file_num_compaction_trigger',
'TableOptions.BlockBasedTable.block_restart_interval',
'CFOptions.max_bytes_for_level_base'
]
)
cond1.set_parameter(
'evaluate',
'int(options[0])*int(options[1])-int(options[2])>=0'
)
# only DBOptions
cond2 = Condition('opt-cond-2')
cond2 = OptionCondition.create(cond2)
cond2.set_parameter(
'options', [
'DBOptions.db_write_buffer_size',
'bloom_bits',
'rate_limiter_bytes_per_sec'
]
)
cond2.set_parameter(
'evaluate',
'(int(options[2]) * int(options[1]) * int(options[0]))==0'
)
# mix of CFOptions and DBOptions
cond3 = Condition('opt-cond-3')
cond3 = OptionCondition.create(cond3)
cond3.set_parameter(
'options', [
'DBOptions.db_write_buffer_size', # 0
'CFOptions.num_levels', # 5, 7
'bloom_bits' # 4
]
)
cond3.set_parameter(
'evaluate', 'int(options[2])*int(options[0])+int(options[1])>6'
)
self.db_options.check_and_trigger_conditions([cond1, cond2, cond3])
cond1_trigger = {'col_fam_A': ['4', '16', '10']}
self.assertDictEqual(cond1_trigger, cond1.get_trigger())
cond2_trigger = {NO_COL_FAMILY: ['0', '4', '1024000']}
self.assertDictEqual(cond2_trigger, cond2.get_trigger())
cond3_trigger = {'default': ['0', '7', '4']}
self.assertDictEqual(cond3_trigger, cond3.get_trigger())
if __name__ == '__main__':
unittest.main()