rocksdb/tools/advisor/advisor/config_optimizer_example.py

# Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
#  This source code is licensed under both the GPLv2 (found in the
#  COPYING file in the root directory) and Apache 2.0 License
#  (found in the LICENSE.Apache file in the root directory).

import argparse

from advisor.db_config_optimizer import ConfigOptimizer
from advisor.db_log_parser import NO_COL_FAMILY
from advisor.db_options_parser import DatabaseOptions
from advisor.rule_parser import RulesSpec


CONFIG_OPT_NUM_ITER = 10


def main(args):
    # initialise the RulesSpec parser
    rule_spec_parser = RulesSpec(args.rules_spec)
    # initialise the benchmark runner
    bench_runner_module = __import__(
        args.benchrunner_module, fromlist=[args.benchrunner_class]
    )
    bench_runner_class = getattr(bench_runner_module, args.benchrunner_class)
    ods_args = {}
    if args.ods_client and args.ods_entity:
        ods_args["client_script"] = args.ods_client
        ods_args["entity"] = args.ods_entity
        if args.ods_key_prefix:
            ods_args["key_prefix"] = args.ods_key_prefix
    db_bench_runner = bench_runner_class(args.benchrunner_pos_args, ods_args)
    # initialise the database configuration
    db_options = DatabaseOptions(args.rocksdb_options, args.misc_options)
    # set the frequency at which stats are dumped in the LOG file and the
    # location of the LOG file.
    db_log_dump_settings = {
        "DBOptions.stats_dump_period_sec": {NO_COL_FAMILY: args.stats_dump_period_sec}
    }
    db_options.update_options(db_log_dump_settings)
    # initialise the configuration optimizer
    config_optimizer = ConfigOptimizer(
        db_bench_runner, db_options, rule_spec_parser, args.base_db_path
    )
    # run the optimiser to improve the database configuration for given
    # benchmarks, with the help of expert-specified rules
    final_db_options = config_optimizer.run()
    # generate the final rocksdb options file
    print(
        "Final configuration in: " + final_db_options.generate_options_config("final")
    )
    print("Final miscellaneous options: " + repr(final_db_options.get_misc_options()))


if __name__ == "__main__":
    """
    An example run of this tool from the command-line would look like:
    python3 -m advisor.config_optimizer_example
    --base_db_path=/tmp/rocksdbtest-155919/dbbench
    --rocksdb_options=temp/OPTIONS_boot.tmp --misc_options bloom_bits=2
    --rules_spec=advisor/rules.ini --stats_dump_period_sec=20
    --benchrunner_module=advisor.db_bench_runner
    --benchrunner_class=DBBenchRunner --benchrunner_pos_args ./../../db_bench
    readwhilewriting use_existing_db=true duration=90
    """
    parser = argparse.ArgumentParser(
        description="This script is used for\
        searching for a better database configuration"
    )
    parser.add_argument(
        "--rocksdb_options",
        required=True,
        type=str,
        help="path of the starting Rocksdb OPTIONS file",
    )
    # these are options that are column-family agnostic and are not yet
    # supported by the Rocksdb Options file: eg. bloom_bits=2
    parser.add_argument(
        "--misc_options",
        nargs="*",
        help="whitespace-separated list of options that are not supported "
        + "by the Rocksdb OPTIONS file, given in the "
        + '<option_name>=<option_value> format eg. "bloom_bits=2 '
        + 'rate_limiter_bytes_per_sec=128000000"',
    )
    parser.add_argument(
        "--base_db_path", required=True, type=str, help="path for the Rocksdb database"
    )
    parser.add_argument(
        "--rules_spec",
        required=True,
        type=str,
        help="path of the file containing the expert-specified Rules",
    )
    parser.add_argument(
        "--stats_dump_period_sec",
        required=True,
        type=int,
        help="the frequency (in seconds) at which STATISTICS are printed to "
        + "the Rocksdb LOG file",
    )
    # ODS arguments
    parser.add_argument("--ods_client", type=str, help="the ODS client binary")
    parser.add_argument(
        "--ods_entity",
        type=str,
        help="the servers for which the ODS stats need to be fetched",
    )
    parser.add_argument(
        "--ods_key_prefix",
        type=str,
        help="the prefix that needs to be attached to the keys of time "
        + "series to be fetched from ODS",
    )
    # benchrunner_module example: advisor.db_benchmark_client
    parser.add_argument(
        "--benchrunner_module",
        required=True,
        type=str,
        help="the module containing the BenchmarkRunner class to be used by "
        + "the Optimizer, example: advisor.db_bench_runner",
    )
    # benchrunner_class example: DBBenchRunner
    parser.add_argument(
        "--benchrunner_class",
        required=True,
        type=str,
        help="the name of the BenchmarkRunner class to be used by the "
        + "Optimizer, should be present in the module provided in the "
        + "benchrunner_module argument, example: DBBenchRunner",
    )
    parser.add_argument(
        "--benchrunner_pos_args",
        nargs="*",
        help="whitespace-separated positional arguments that are passed on "
        + "to the constructor of the BenchmarkRunner class provided in the "
        + 'benchrunner_class argument, example: "use_existing_db=true '
        + 'duration=900"',
    )
    args = parser.parse_args()
    main(args)