|
|
|
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
|
|
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
|
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
|
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
|
|
|
|
|
|
|
|
#include "options/cf_options.h"
|
|
|
|
|
|
|
|
|
|
#include <cassert>
|
|
|
|
|
#include <cinttypes>
|
|
|
|
|
#include <limits>
|
|
|
|
|
#include <string>
|
|
|
|
|
|
|
|
|
|
#include "logging/logging.h"
|
|
|
|
|
#include "options/configurable_helper.h"
|
|
|
|
|
#include "options/db_options.h"
|
|
|
|
|
#include "options/options_helper.h"
|
|
|
|
|
#include "options/options_parser.h"
|
|
|
|
|
#include "port/port.h"
|
|
|
|
|
#include "rocksdb/compaction_filter.h"
|
|
|
|
|
#include "rocksdb/concurrent_task_limiter.h"
|
|
|
|
|
#include "rocksdb/configurable.h"
|
|
|
|
|
#include "rocksdb/convenience.h"
|
|
|
|
|
#include "rocksdb/env.h"
|
Introduce a new storage specific Env API (#5761)
Summary:
The current Env API encompasses both storage/file operations, as well as OS related operations. Most of the APIs return a Status, which does not have enough metadata about an error, such as whether its retry-able or not, scope (i.e fault domain) of the error etc., that may be required in order to properly handle a storage error. The file APIs also do not provide enough control over the IO SLA, such as timeout, prioritization, hinting about placement and redundancy etc.
This PR separates out the file/storage APIs from Env into a new FileSystem class. The APIs are updated to return an IOStatus with metadata about the error, as well as to take an IOOptions structure as input in order to allow more control over the IO.
The user can set both ```options.env``` and ```options.file_system``` to specify that RocksDB should use the former for OS related operations and the latter for storage operations. Internally, a ```CompositeEnvWrapper``` has been introduced that inherits from ```Env``` and redirects individual methods to either an ```Env``` implementation or the ```FileSystem``` as appropriate. When options are sanitized during ```DB::Open```, ```options.env``` is replaced with a newly allocated ```CompositeEnvWrapper``` instance if both env and file_system have been specified. This way, the rest of the RocksDB code can continue to function as before.
This PR also ports PosixEnv to the new API by splitting it into two - PosixEnv and PosixFileSystem. PosixEnv is defined as a sub-class of CompositeEnvWrapper, and threading/time functions are overridden with Posix specific implementations in order to avoid an extra level of indirection.
The ```CompositeEnvWrapper``` translates ```IOStatus``` return code to ```Status```, and sets the severity to ```kSoftError``` if the io_status is retryable. The error handling code in RocksDB can then recover the DB automatically.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5761
Differential Revision: D18868376
Pulled By: anand1976
fbshipit-source-id: 39efe18a162ea746fabac6360ff529baba48486f
5 years ago
|
|
|
#include "rocksdb/file_system.h"
|
|
|
|
|
#include "rocksdb/merge_operator.h"
|
|
|
|
|
#include "rocksdb/options.h"
|
|
|
|
|
#include "rocksdb/table.h"
|
|
|
|
|
#include "rocksdb/utilities/object_registry.h"
|
|
|
|
|
#include "rocksdb/utilities/options_type.h"
|
|
|
|
|
#include "util/cast_util.h"
|
|
|
|
|
|
|
|
|
|
namespace ROCKSDB_NAMESPACE {
|
|
|
|
|
// offset_of is used to get the offset of a class data member
|
|
|
|
|
// ex: offset_of(&ColumnFamilyOptions::num_levels)
|
|
|
|
|
// This call will return the offset of num_levels in ColumnFamilyOptions class
|
|
|
|
|
//
|
|
|
|
|
// This is the same as offsetof() but allow us to work with non standard-layout
|
|
|
|
|
// classes and structures
|
|
|
|
|
// refs:
|
|
|
|
|
// http://en.cppreference.com/w/cpp/concept/StandardLayoutType
|
|
|
|
|
// https://gist.github.com/graphitemaster/494f21190bb2c63c5516
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
|
static ImmutableCFOptions dummy_cf_options;
|
|
|
|
|
template <typename T1>
|
|
|
|
|
int offset_of(T1 ImmutableCFOptions::*member) {
|
|
|
|
|
return int(size_t(&(dummy_cf_options.*member)) - size_t(&dummy_cf_options));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static Status ParseCompressionOptions(const std::string& value,
|
|
|
|
|
const std::string& name,
|
|
|
|
|
CompressionOptions& compression_opts) {
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
4 years ago
|
|
|
const char kDelimiter = ':';
|
|
|
|
|
std::istringstream field_stream(value);
|
|
|
|
|
std::string field;
|
|
|
|
|
|
|
|
|
|
if (!std::getline(field_stream, field, kDelimiter)) {
|
|
|
|
|
return Status::InvalidArgument("unable to parse the specified CF option " +
|
|
|
|
|
name);
|
|
|
|
|
}
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
4 years ago
|
|
|
compression_opts.window_bits = ParseInt(field);
|
|
|
|
|
|
|
|
|
|
if (!std::getline(field_stream, field, kDelimiter)) {
|
|
|
|
|
return Status::InvalidArgument("unable to parse the specified CF option " +
|
|
|
|
|
name);
|
|
|
|
|
}
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
4 years ago
|
|
|
compression_opts.level = ParseInt(field);
|
|
|
|
|
|
|
|
|
|
if (!std::getline(field_stream, field, kDelimiter)) {
|
|
|
|
|
return Status::InvalidArgument("unable to parse the specified CF option " +
|
|
|
|
|
name);
|
|
|
|
|
}
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
4 years ago
|
|
|
compression_opts.strategy = ParseInt(field);
|
|
|
|
|
|
|
|
|
|
// max_dict_bytes is optional for backwards compatibility
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
4 years ago
|
|
|
if (!field_stream.eof()) {
|
|
|
|
|
if (!std::getline(field_stream, field, kDelimiter)) {
|
|
|
|
|
return Status::InvalidArgument(
|
|
|
|
|
"unable to parse the specified CF option " + name);
|
|
|
|
|
}
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
4 years ago
|
|
|
compression_opts.max_dict_bytes = ParseInt(field);
|
|
|
|
|
}
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
4 years ago
|
|
|
|
|
|
|
|
// zstd_max_train_bytes is optional for backwards compatibility
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
4 years ago
|
|
|
if (!field_stream.eof()) {
|
|
|
|
|
if (!std::getline(field_stream, field, kDelimiter)) {
|
|
|
|
|
return Status::InvalidArgument(
|
|
|
|
|
"unable to parse the specified CF option " + name);
|
|
|
|
|
}
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
4 years ago
|
|
|
compression_opts.zstd_max_train_bytes = ParseInt(field);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// parallel_threads is optional for backwards compatibility
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
4 years ago
|
|
|
if (!field_stream.eof()) {
|
|
|
|
|
if (!std::getline(field_stream, field, kDelimiter)) {
|
|
|
|
|
return Status::InvalidArgument(
|
|
|
|
|
"unable to parse the specified CF option " + name);
|
|
|
|
|
}
|
|
|
|
|
// Since parallel_threads comes before enabled but was added optionally
|
|
|
|
|
// later, we need to check if this is the final token (meaning it is the
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
4 years ago
|
|
|
// enabled bit), or if there are more tokens (meaning this one is
|
|
|
|
|
// parallel_threads).
|
|
|
|
|
if (!field_stream.eof()) {
|
|
|
|
|
compression_opts.parallel_threads = ParseInt(field);
|
|
|
|
|
} else {
|
|
|
|
|
// parallel_threads is not serialized with this format, but enabled is
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
4 years ago
|
|
|
compression_opts.enabled = ParseBoolean("", field);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// enabled is optional for backwards compatibility
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
4 years ago
|
|
|
if (!field_stream.eof()) {
|
|
|
|
|
if (!std::getline(field_stream, field, kDelimiter)) {
|
|
|
|
|
return Status::InvalidArgument(
|
|
|
|
|
"unable to parse the specified CF option " + name);
|
|
|
|
|
}
|
|
|
|
|
compression_opts.enabled = ParseBoolean("", field);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// max_dict_buffer_bytes is optional for backwards compatibility
|
|
|
|
|
if (!field_stream.eof()) {
|
|
|
|
|
if (!std::getline(field_stream, field, kDelimiter)) {
|
|
|
|
|
return Status::InvalidArgument(
|
|
|
|
|
"unable to parse the specified CF option " + name);
|
|
|
|
|
}
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
4 years ago
|
|
|
compression_opts.max_dict_buffer_bytes = ParseUint64(field);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!field_stream.eof()) {
|
|
|
|
|
return Status::InvalidArgument("unable to parse the specified CF option " +
|
|
|
|
|
name);
|
|
|
|
|
}
|
|
|
|
|
return Status::OK();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const std::string kOptNameBMCompOpts = "bottommost_compression_opts";
|
|
|
|
|
const std::string kOptNameCompOpts = "compression_opts";
|
|
|
|
|
|
|
|
|
|
// OptionTypeInfo map for CompressionOptions
|
|
|
|
|
static std::unordered_map<std::string, OptionTypeInfo>
|
|
|
|
|
compression_options_type_info = {
|
|
|
|
|
{"window_bits",
|
|
|
|
|
{offsetof(struct CompressionOptions, window_bits), OptionType::kInt,
|
|
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
|
|
|
|
|
{"level",
|
|
|
|
|
{offsetof(struct CompressionOptions, level), OptionType::kInt,
|
|
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
|
|
|
|
|
{"strategy",
|
|
|
|
|
{offsetof(struct CompressionOptions, strategy), OptionType::kInt,
|
|
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
|
|
|
|
|
{"max_dict_bytes",
|
|
|
|
|
{offsetof(struct CompressionOptions, max_dict_bytes), OptionType::kInt,
|
|
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
|
|
|
|
|
{"zstd_max_train_bytes",
|
|
|
|
|
{offsetof(struct CompressionOptions, zstd_max_train_bytes),
|
|
|
|
|
OptionType::kUInt32T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"parallel_threads",
|
|
|
|
|
{offsetof(struct CompressionOptions, parallel_threads),
|
|
|
|
|
OptionType::kUInt32T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"enabled",
|
|
|
|
|
{offsetof(struct CompressionOptions, enabled), OptionType::kBoolean,
|
|
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
|
Limit buffering for collecting samples for compression dictionary (#7970)
Summary:
For dictionary compression, we need to collect some representative samples of the data to be compressed, which we use to either generate or train (when `CompressionOptions::zstd_max_train_bytes > 0`) a dictionary. Previously, the strategy was to buffer all the data blocks during flush, and up to the target file size during compaction. That strategy allowed us to randomly pick samples from as wide a range as possible that'd be guaranteed to land in a single output file.
However, some users try to make huge files in memory-constrained environments, where this strategy can cause OOM. This PR introduces an option, `CompressionOptions::max_dict_buffer_bytes`, that limits how much data blocks are buffered before we switch to unbuffered mode (which means creating the per-SST dictionary, writing out the buffered data, and compressing/writing new blocks as soon as they are built). It is not strict as we currently buffer more than just data blocks -- also keys are buffered. But it does make a step towards giving users predictable memory usage.
Related changes include:
- Changed sampling for dictionary compression to select unique data blocks when there is limited availability of data blocks
- Made use of `BlockBuilder::SwapAndReset()` to save an allocation+memcpy when buffering data blocks for building a dictionary
- Changed `ParseBoolean()` to accept an input containing characters after the boolean. This is necessary since, with this PR, a value for `CompressionOptions::enabled` is no longer necessarily the final component in the `CompressionOptions` string.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/7970
Test Plan:
- updated `CompressionOptions` unit tests to verify limit is respected (to the extent expected in the current implementation) in various scenarios of flush/compaction to bottommost/non-bottommost level
- looked at jemalloc heap profiles right before and after switching to unbuffered mode during flush/compaction. Verified memory usage in buffering is proportional to the limit set.
Reviewed By: pdillinger
Differential Revision: D26467994
Pulled By: ajkr
fbshipit-source-id: 3da4ef9fba59974e4ef40e40c01611002c861465
4 years ago
|
|
|
{"max_dict_buffer_bytes",
|
|
|
|
|
{offsetof(struct CompressionOptions, max_dict_buffer_bytes),
|
|
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static std::unordered_map<std::string, OptionTypeInfo>
|
|
|
|
|
fifo_compaction_options_type_info = {
|
|
|
|
|
{"max_table_files_size",
|
|
|
|
|
{offsetof(struct CompactionOptionsFIFO, max_table_files_size),
|
|
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"age_for_warm",
|
|
|
|
|
{offsetof(struct CompactionOptionsFIFO, age_for_warm),
|
|
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"ttl",
|
|
|
|
|
{0, OptionType::kUInt64T, OptionVerificationType::kDeprecated,
|
|
|
|
|
OptionTypeFlags::kNone}},
|
|
|
|
|
{"allow_compaction",
|
|
|
|
|
{offsetof(struct CompactionOptionsFIFO, allow_compaction),
|
|
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static std::unordered_map<std::string, OptionTypeInfo>
|
|
|
|
|
universal_compaction_options_type_info = {
|
|
|
|
|
{"size_ratio",
|
|
|
|
|
{offsetof(class CompactionOptionsUniversal, size_ratio),
|
|
|
|
|
OptionType::kUInt, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"min_merge_width",
|
|
|
|
|
{offsetof(class CompactionOptionsUniversal, min_merge_width),
|
|
|
|
|
OptionType::kUInt, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"max_merge_width",
|
|
|
|
|
{offsetof(class CompactionOptionsUniversal, max_merge_width),
|
|
|
|
|
OptionType::kUInt, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"max_size_amplification_percent",
|
|
|
|
|
{offsetof(class CompactionOptionsUniversal,
|
|
|
|
|
max_size_amplification_percent),
|
|
|
|
|
OptionType::kUInt, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"compression_size_percent",
|
|
|
|
|
{offsetof(class CompactionOptionsUniversal, compression_size_percent),
|
|
|
|
|
OptionType::kInt, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"stop_style",
|
|
|
|
|
{offsetof(class CompactionOptionsUniversal, stop_style),
|
|
|
|
|
OptionType::kCompactionStopStyle, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
Incremental Space Amp Compactions in Universal Style (#8655)
Summary:
This commit introduces incremental compaction in univeral style for space amplification. This follows the first improvement mentioned in https://rocksdb.org/blog/2021/04/12/universal-improvements.html . The implemention simply picks up files about size of max_compaction_bytes to compact and execute if the penalty is not too big. More optimizations can be done in the future, e.g. prioritizing between this compaction and other types. But for now, the feature is supposed to be functional and can often reduce frequency of full compactions, although it can introduce penalty.
In order to add cut files more efficiently so that more files from upper levels can be included, SST file cutting threshold (for current file + overlapping parent level files) is set to 1.5X of target file size. A 2MB target file size will generate files like this: https://gist.github.com/siying/29d2676fba417404f3c95e6c013c7de8 Number of files indeed increases but it is not out of control.
Two set of write benchmarks are run:
1. For ingestion rate limited scenario, we can see full compaction is mostly eliminated: https://gist.github.com/siying/959bc1186066906831cf4c808d6e0a19 . The write amp increased from 7.7 to 9.4, as expected. After applying file cutting, the number is improved to 8.9. In another benchmark, the write amp is even better with the incremental approach: https://gist.github.com/siying/d1c16c286d7c59c4d7bba718ca198163
2. For ingestion rate unlimited scenario, incremental compaction turns out to be too expensive most of the time and is not executed, as expected.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8655
Test Plan: Add unit tests to the functionality.
Reviewed By: ajkr
Differential Revision: D31787034
fbshipit-source-id: ce813e63b15a61d5a56e97bf8902a1b28e011beb
3 years ago
|
|
|
{"incremental",
|
|
|
|
|
{offsetof(class CompactionOptionsUniversal, incremental),
|
|
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"allow_trivial_move",
|
|
|
|
|
{offsetof(class CompactionOptionsUniversal, allow_trivial_move),
|
|
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}}};
|
|
|
|
|
|
|
|
|
|
static std::unordered_map<std::string, OptionTypeInfo>
|
|
|
|
|
cf_mutable_options_type_info = {
|
|
|
|
|
{"report_bg_io_stats",
|
|
|
|
|
{offsetof(struct MutableCFOptions, report_bg_io_stats),
|
|
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"disable_auto_compactions",
|
|
|
|
|
{offsetof(struct MutableCFOptions, disable_auto_compactions),
|
|
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"filter_deletes",
|
|
|
|
|
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"check_flush_compaction_key_order",
|
|
|
|
|
{offsetof(struct MutableCFOptions, check_flush_compaction_key_order),
|
|
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"paranoid_file_checks",
|
|
|
|
|
{offsetof(struct MutableCFOptions, paranoid_file_checks),
|
|
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"verify_checksums_in_compaction",
|
|
|
|
|
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"soft_pending_compaction_bytes_limit",
|
|
|
|
|
{offsetof(struct MutableCFOptions,
|
|
|
|
|
soft_pending_compaction_bytes_limit),
|
|
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"hard_pending_compaction_bytes_limit",
|
|
|
|
|
{offsetof(struct MutableCFOptions,
|
|
|
|
|
hard_pending_compaction_bytes_limit),
|
|
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"hard_rate_limit",
|
|
|
|
|
{0, OptionType::kDouble, OptionVerificationType::kDeprecated,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"soft_rate_limit",
|
|
|
|
|
{0, OptionType::kDouble, OptionVerificationType::kDeprecated,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"max_compaction_bytes",
|
|
|
|
|
{offsetof(struct MutableCFOptions, max_compaction_bytes),
|
|
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"expanded_compaction_factor",
|
|
|
|
|
{0, OptionType::kInt, OptionVerificationType::kDeprecated,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"level0_file_num_compaction_trigger",
|
|
|
|
|
{offsetof(struct MutableCFOptions, level0_file_num_compaction_trigger),
|
|
|
|
|
OptionType::kInt, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"level0_slowdown_writes_trigger",
|
|
|
|
|
{offsetof(struct MutableCFOptions, level0_slowdown_writes_trigger),
|
|
|
|
|
OptionType::kInt, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"level0_stop_writes_trigger",
|
|
|
|
|
{offsetof(struct MutableCFOptions, level0_stop_writes_trigger),
|
|
|
|
|
OptionType::kInt, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"max_grandparent_overlap_factor",
|
|
|
|
|
{0, OptionType::kInt, OptionVerificationType::kDeprecated,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"max_write_buffer_number",
|
|
|
|
|
{offsetof(struct MutableCFOptions, max_write_buffer_number),
|
|
|
|
|
OptionType::kInt, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"source_compaction_factor",
|
|
|
|
|
{0, OptionType::kInt, OptionVerificationType::kDeprecated,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"target_file_size_multiplier",
|
|
|
|
|
{offsetof(struct MutableCFOptions, target_file_size_multiplier),
|
|
|
|
|
OptionType::kInt, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"arena_block_size",
|
|
|
|
|
{offsetof(struct MutableCFOptions, arena_block_size),
|
|
|
|
|
OptionType::kSizeT, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"inplace_update_num_locks",
|
|
|
|
|
{offsetof(struct MutableCFOptions, inplace_update_num_locks),
|
|
|
|
|
OptionType::kSizeT, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"max_successive_merges",
|
|
|
|
|
{offsetof(struct MutableCFOptions, max_successive_merges),
|
|
|
|
|
OptionType::kSizeT, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"memtable_huge_page_size",
|
|
|
|
|
{offsetof(struct MutableCFOptions, memtable_huge_page_size),
|
|
|
|
|
OptionType::kSizeT, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"memtable_prefix_bloom_huge_page_tlb_size",
|
|
|
|
|
{0, OptionType::kSizeT, OptionVerificationType::kDeprecated,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"write_buffer_size",
|
|
|
|
|
{offsetof(struct MutableCFOptions, write_buffer_size),
|
|
|
|
|
OptionType::kSizeT, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"memtable_prefix_bloom_bits",
|
|
|
|
|
{0, OptionType::kUInt32T, OptionVerificationType::kDeprecated,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"memtable_prefix_bloom_size_ratio",
|
|
|
|
|
{offsetof(struct MutableCFOptions, memtable_prefix_bloom_size_ratio),
|
|
|
|
|
OptionType::kDouble, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"memtable_prefix_bloom_probes",
|
|
|
|
|
{0, OptionType::kUInt32T, OptionVerificationType::kDeprecated,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"memtable_whole_key_filtering",
|
|
|
|
|
{offsetof(struct MutableCFOptions, memtable_whole_key_filtering),
|
|
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"min_partial_merge_operands",
|
|
|
|
|
{0, OptionType::kUInt32T, OptionVerificationType::kDeprecated,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"max_bytes_for_level_base",
|
|
|
|
|
{offsetof(struct MutableCFOptions, max_bytes_for_level_base),
|
|
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"snap_refresh_nanos",
|
|
|
|
|
{0, OptionType::kUInt64T, OptionVerificationType::kDeprecated,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"max_bytes_for_level_multiplier",
|
|
|
|
|
{offsetof(struct MutableCFOptions, max_bytes_for_level_multiplier),
|
|
|
|
|
OptionType::kDouble, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"max_bytes_for_level_multiplier_additional",
|
|
|
|
|
OptionTypeInfo::Vector<int>(
|
|
|
|
|
offsetof(struct MutableCFOptions,
|
|
|
|
|
max_bytes_for_level_multiplier_additional),
|
|
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
|
|
|
|
|
{0, OptionType::kInt})},
|
|
|
|
|
{"max_sequential_skip_in_iterations",
|
|
|
|
|
{offsetof(struct MutableCFOptions, max_sequential_skip_in_iterations),
|
|
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"target_file_size_base",
|
|
|
|
|
{offsetof(struct MutableCFOptions, target_file_size_base),
|
|
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"compression",
|
|
|
|
|
{offsetof(struct MutableCFOptions, compression),
|
|
|
|
|
OptionType::kCompressionType, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"prefix_extractor",
|
|
|
|
|
OptionTypeInfo::AsCustomSharedPtr<const SliceTransform>(
|
|
|
|
|
offsetof(struct MutableCFOptions, prefix_extractor),
|
|
|
|
|
OptionVerificationType::kByNameAllowNull,
|
|
|
|
|
(OptionTypeFlags::kMutable | OptionTypeFlags::kAllowNull))},
|
|
|
|
|
{"compaction_options_fifo",
|
|
|
|
|
OptionTypeInfo::Struct(
|
|
|
|
|
"compaction_options_fifo", &fifo_compaction_options_type_info,
|
|
|
|
|
offsetof(struct MutableCFOptions, compaction_options_fifo),
|
|
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kMutable,
|
|
|
|
|
[](const ConfigOptions& opts, const std::string& name,
|
|
|
|
|
const std::string& value, void* addr) {
|
|
|
|
|
// This is to handle backward compatibility, where
|
|
|
|
|
// compaction_options_fifo could be assigned a single scalar
|
|
|
|
|
// value, say, like "23", which would be assigned to
|
|
|
|
|
// max_table_files_size.
|
|
|
|
|
if (name == "compaction_options_fifo" &&
|
|
|
|
|
value.find("=") == std::string::npos) {
|
|
|
|
|
// Old format. Parse just a single uint64_t value.
|
|
|
|
|
auto options = static_cast<CompactionOptionsFIFO*>(addr);
|
|
|
|
|
options->max_table_files_size = ParseUint64(value);
|
|
|
|
|
return Status::OK();
|
|
|
|
|
} else {
|
|
|
|
|
return OptionTypeInfo::ParseStruct(
|
|
|
|
|
opts, "compaction_options_fifo",
|
|
|
|
|
&fifo_compaction_options_type_info, name, value, addr);
|
|
|
|
|
}
|
|
|
|
|
})},
|
|
|
|
|
{"compaction_options_universal",
|
|
|
|
|
OptionTypeInfo::Struct(
|
|
|
|
|
"compaction_options_universal",
|
|
|
|
|
&universal_compaction_options_type_info,
|
|
|
|
|
offsetof(struct MutableCFOptions, compaction_options_universal),
|
|
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kMutable)},
|
|
|
|
|
{"ttl",
|
|
|
|
|
{offsetof(struct MutableCFOptions, ttl), OptionType::kUInt64T,
|
|
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
|
|
|
|
|
{"periodic_compaction_seconds",
|
|
|
|
|
{offsetof(struct MutableCFOptions, periodic_compaction_seconds),
|
|
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"enable_blob_files",
|
|
|
|
|
{offsetof(struct MutableCFOptions, enable_blob_files),
|
|
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"min_blob_size",
|
|
|
|
|
{offsetof(struct MutableCFOptions, min_blob_size),
|
|
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"blob_file_size",
|
|
|
|
|
{offsetof(struct MutableCFOptions, blob_file_size),
|
|
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"blob_compression_type",
|
|
|
|
|
{offsetof(struct MutableCFOptions, blob_compression_type),
|
|
|
|
|
OptionType::kCompressionType, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"enable_blob_garbage_collection",
|
|
|
|
|
{offsetof(struct MutableCFOptions, enable_blob_garbage_collection),
|
|
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"blob_garbage_collection_age_cutoff",
|
|
|
|
|
{offsetof(struct MutableCFOptions, blob_garbage_collection_age_cutoff),
|
|
|
|
|
OptionType::kDouble, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
Make it possible to force the garbage collection of the oldest blob files (#8994)
Summary:
The current BlobDB garbage collection logic works by relocating the valid
blobs from the oldest blob files as they are encountered during compaction,
and cleaning up blob files once they contain nothing but garbage. However,
with sufficiently skewed workloads, it is theoretically possible to end up in a
situation when few or no compactions get scheduled for the SST files that contain
references to the oldest blob files, which can lead to increased space amp due
to the lack of GC.
In order to efficiently handle such workloads, the patch adds a new BlobDB
configuration option called `blob_garbage_collection_force_threshold`,
which signals to BlobDB to schedule targeted compactions for the SST files
that keep alive the oldest batch of blob files if the overall ratio of garbage in
the given blob files meets the threshold *and* all the given blob files are
eligible for GC based on `blob_garbage_collection_age_cutoff`. (For example,
if the new option is set to 0.9, targeted compactions will get scheduled if the
sum of garbage bytes meets or exceeds 90% of the sum of total bytes in the
oldest blob files, assuming all affected blob files are below the age-based cutoff.)
The net result of these targeted compactions is that the valid blobs in the oldest
blob files are relocated and the oldest blob files themselves cleaned up (since
*all* SST files that rely on them get compacted away).
These targeted compactions are similar to periodic compactions in the sense
that they force certain SST files that otherwise would not get picked up to undergo
compaction and also in the sense that instead of merging files from multiple levels,
they target a single file. (Note: such compactions might still include neighboring files
from the same level due to the need of having a "clean cut" boundary but they never
include any files from any other level.)
This functionality is currently only supported with the leveled compaction style
and is inactive by default (since the default value is set to 1.0, i.e. 100%).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8994
Test Plan: Ran `make check` and tested using `db_bench` and the stress/crash tests.
Reviewed By: riversand963
Differential Revision: D31489850
Pulled By: ltamasi
fbshipit-source-id: 44057d511726a0e2a03c5d9313d7511b3f0c4eab
3 years ago
|
|
|
{"blob_garbage_collection_force_threshold",
|
|
|
|
|
{offsetof(struct MutableCFOptions,
|
|
|
|
|
blob_garbage_collection_force_threshold),
|
|
|
|
|
OptionType::kDouble, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"blob_compaction_readahead_size",
|
|
|
|
|
{offsetof(struct MutableCFOptions, blob_compaction_readahead_size),
|
|
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"sample_for_compression",
|
|
|
|
|
{offsetof(struct MutableCFOptions, sample_for_compression),
|
|
|
|
|
OptionType::kUInt64T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{"bottommost_compression",
|
|
|
|
|
{offsetof(struct MutableCFOptions, bottommost_compression),
|
|
|
|
|
OptionType::kCompressionType, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kMutable}},
|
|
|
|
|
{kOptNameCompOpts,
|
|
|
|
|
OptionTypeInfo::Struct(
|
|
|
|
|
kOptNameCompOpts, &compression_options_type_info,
|
|
|
|
|
offsetof(struct MutableCFOptions, compression_opts),
|
|
|
|
|
OptionVerificationType::kNormal,
|
|
|
|
|
(OptionTypeFlags::kMutable | OptionTypeFlags::kCompareNever),
|
|
|
|
|
[](const ConfigOptions& opts, const std::string& name,
|
|
|
|
|
const std::string& value, void* addr) {
|
|
|
|
|
// This is to handle backward compatibility, where
|
|
|
|
|
// compression_options was a ":" separated list.
|
|
|
|
|
if (name == kOptNameCompOpts &&
|
|
|
|
|
value.find("=") == std::string::npos) {
|
|
|
|
|
auto* compression = static_cast<CompressionOptions*>(addr);
|
|
|
|
|
return ParseCompressionOptions(value, name, *compression);
|
|
|
|
|
} else {
|
|
|
|
|
return OptionTypeInfo::ParseStruct(
|
|
|
|
|
opts, kOptNameCompOpts, &compression_options_type_info,
|
|
|
|
|
name, value, addr);
|
|
|
|
|
}
|
|
|
|
|
})},
|
|
|
|
|
{kOptNameBMCompOpts,
|
|
|
|
|
OptionTypeInfo::Struct(
|
|
|
|
|
kOptNameBMCompOpts, &compression_options_type_info,
|
|
|
|
|
offsetof(struct MutableCFOptions, bottommost_compression_opts),
|
|
|
|
|
OptionVerificationType::kNormal,
|
|
|
|
|
(OptionTypeFlags::kMutable | OptionTypeFlags::kCompareNever),
|
|
|
|
|
[](const ConfigOptions& opts, const std::string& name,
|
|
|
|
|
const std::string& value, void* addr) {
|
|
|
|
|
// This is to handle backward compatibility, where
|
|
|
|
|
// compression_options was a ":" separated list.
|
|
|
|
|
if (name == kOptNameBMCompOpts &&
|
|
|
|
|
value.find("=") == std::string::npos) {
|
|
|
|
|
auto* compression = static_cast<CompressionOptions*>(addr);
|
|
|
|
|
return ParseCompressionOptions(value, name, *compression);
|
|
|
|
|
} else {
|
|
|
|
|
return OptionTypeInfo::ParseStruct(
|
|
|
|
|
opts, kOptNameBMCompOpts, &compression_options_type_info,
|
|
|
|
|
name, value, addr);
|
|
|
|
|
}
|
|
|
|
|
})},
|
|
|
|
|
// End special case properties
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static std::unordered_map<std::string, OptionTypeInfo>
|
|
|
|
|
cf_immutable_options_type_info = {
|
|
|
|
|
/* not yet supported
|
|
|
|
|
CompressionOptions compression_opts;
|
|
|
|
|
TablePropertiesCollectorFactories table_properties_collector_factories;
|
|
|
|
|
using TablePropertiesCollectorFactories =
|
|
|
|
|
std::vector<std::shared_ptr<TablePropertiesCollectorFactory>>;
|
|
|
|
|
UpdateStatus (*inplace_callback)(char* existing_value,
|
|
|
|
|
uint34_t* existing_value_size,
|
|
|
|
|
Slice delta_value,
|
|
|
|
|
std::string* merged_value);
|
|
|
|
|
std::vector<DbPath> cf_paths;
|
|
|
|
|
*/
|
|
|
|
|
{"compaction_measure_io_stats",
|
|
|
|
|
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
|
|
|
|
|
OptionTypeFlags::kNone}},
|
|
|
|
|
{"inplace_update_support",
|
|
|
|
|
{offset_of(&ImmutableCFOptions::inplace_update_support),
|
|
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kNone}},
|
|
|
|
|
{"level_compaction_dynamic_level_bytes",
|
|
|
|
|
{offset_of(&ImmutableCFOptions::level_compaction_dynamic_level_bytes),
|
|
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kNone}},
|
|
|
|
|
{"optimize_filters_for_hits",
|
|
|
|
|
{offset_of(&ImmutableCFOptions::optimize_filters_for_hits),
|
|
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kNone}},
|
|
|
|
|
{"force_consistency_checks",
|
|
|
|
|
{offset_of(&ImmutableCFOptions::force_consistency_checks),
|
|
|
|
|
OptionType::kBoolean, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kNone}},
|
|
|
|
|
{"purge_redundant_kvs_while_flush",
|
|
|
|
|
{offset_of(&ImmutableCFOptions::purge_redundant_kvs_while_flush),
|
|
|
|
|
OptionType::kBoolean, OptionVerificationType::kDeprecated,
|
|
|
|
|
OptionTypeFlags::kNone}},
|
|
|
|
|
{"max_mem_compaction_level",
|
|
|
|
|
{0, OptionType::kInt, OptionVerificationType::kDeprecated,
|
|
|
|
|
OptionTypeFlags::kNone}},
|
|
|
|
|
{"max_write_buffer_number_to_maintain",
|
|
|
|
|
{offset_of(&ImmutableCFOptions::max_write_buffer_number_to_maintain),
|
|
|
|
|
OptionType::kInt, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kNone, 0}},
|
|
|
|
|
{"max_write_buffer_size_to_maintain",
|
|
|
|
|
{offset_of(&ImmutableCFOptions::max_write_buffer_size_to_maintain),
|
|
|
|
|
OptionType::kInt64T, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kNone}},
|
|
|
|
|
{"min_write_buffer_number_to_merge",
|
|
|
|
|
{offset_of(&ImmutableCFOptions::min_write_buffer_number_to_merge),
|
|
|
|
|
OptionType::kInt, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kNone, 0}},
|
|
|
|
|
{"num_levels",
|
|
|
|
|
{offset_of(&ImmutableCFOptions::num_levels), OptionType::kInt,
|
|
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
|
|
|
|
|
{"bloom_locality",
|
|
|
|
|
{offset_of(&ImmutableCFOptions::bloom_locality), OptionType::kUInt32T,
|
|
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kNone}},
|
|
|
|
|
{"rate_limit_delay_max_milliseconds",
|
|
|
|
|
{0, OptionType::kUInt, OptionVerificationType::kDeprecated,
|
|
|
|
|
OptionTypeFlags::kNone}},
|
|
|
|
|
{"compression_per_level",
|
|
|
|
|
OptionTypeInfo::Vector<CompressionType>(
|
|
|
|
|
offset_of(&ImmutableCFOptions::compression_per_level),
|
|
|
|
|
OptionVerificationType::kNormal, OptionTypeFlags::kNone,
|
|
|
|
|
{0, OptionType::kCompressionType})},
|
|
|
|
|
{"comparator",
|
|
|
|
|
OptionTypeInfo::AsCustomRawPtr<const Comparator>(
|
|
|
|
|
offset_of(&ImmutableCFOptions::user_comparator),
|
|
|
|
|
OptionVerificationType::kByName, OptionTypeFlags::kCompareLoose,
|
|
|
|
|
// Serializes a Comparator
|
|
|
|
|
[](const ConfigOptions& opts, const std::string&, const void* addr,
|
|
|
|
|
std::string* value) {
|
|
|
|
|
// it's a const pointer of const Comparator*
|
|
|
|
|
const auto* ptr = static_cast<const Comparator* const*>(addr);
|
|
|
|
|
|
|
|
|
|
// Since the user-specified comparator will be wrapped by
|
|
|
|
|
// InternalKeyComparator, we should persist the user-specified
|
|
|
|
|
// one instead of InternalKeyComparator.
|
|
|
|
|
if (*ptr == nullptr) {
|
|
|
|
|
*value = kNullptrString;
|
|
|
|
|
} else if (opts.mutable_options_only) {
|
|
|
|
|
*value = "";
|
|
|
|
|
} else {
|
|
|
|
|
const Comparator* root_comp = (*ptr)->GetRootComparator();
|
|
|
|
|
if (root_comp == nullptr) {
|
|
|
|
|
root_comp = (*ptr);
|
|
|
|
|
}
|
|
|
|
|
*value = root_comp->ToString(opts);
|
|
|
|
|
}
|
|
|
|
|
return Status::OK();
|
|
|
|
|
},
|
|
|
|
|
/* Use the default match function*/ nullptr)},
|
|
|
|
|
{"memtable_insert_with_hint_prefix_extractor",
|
|
|
|
|
OptionTypeInfo::AsCustomSharedPtr<const SliceTransform>(
|
|
|
|
|
offset_of(&ImmutableCFOptions::
|
|
|
|
|
memtable_insert_with_hint_prefix_extractor),
|
|
|
|
|
OptionVerificationType::kByNameAllowNull, OptionTypeFlags::kNone)},
|
|
|
|
|
{"memtable_factory",
|
|
|
|
|
{offset_of(&ImmutableCFOptions::memtable_factory),
|
|
|
|
|
OptionType::kCustomizable, OptionVerificationType::kByName,
|
|
|
|
|
OptionTypeFlags::kShared,
|
|
|
|
|
[](const ConfigOptions& opts, const std::string&,
|
|
|
|
|
const std::string& value, void* addr) {
|
|
|
|
|
std::unique_ptr<MemTableRepFactory> factory;
|
|
|
|
|
auto* shared =
|
|
|
|
|
static_cast<std::shared_ptr<MemTableRepFactory>*>(addr);
|
|
|
|
|
Status s =
|
|
|
|
|
MemTableRepFactory::CreateFromString(opts, value, &factory);
|
|
|
|
|
if (s.ok()) {
|
|
|
|
|
shared->reset(factory.release());
|
|
|
|
|
}
|
|
|
|
|
return s;
|
|
|
|
|
}}},
|
|
|
|
|
{"memtable",
|
|
|
|
|
{offset_of(&ImmutableCFOptions::memtable_factory),
|
|
|
|
|
OptionType::kCustomizable, OptionVerificationType::kAlias,
|
|
|
|
|
OptionTypeFlags::kShared,
|
|
|
|
|
[](const ConfigOptions& opts, const std::string&,
|
|
|
|
|
const std::string& value, void* addr) {
|
|
|
|
|
std::unique_ptr<MemTableRepFactory> factory;
|
|
|
|
|
auto* shared =
|
|
|
|
|
static_cast<std::shared_ptr<MemTableRepFactory>*>(addr);
|
|
|
|
|
Status s =
|
|
|
|
|
MemTableRepFactory::CreateFromString(opts, value, &factory);
|
|
|
|
|
if (s.ok()) {
|
|
|
|
|
shared->reset(factory.release());
|
|
|
|
|
}
|
|
|
|
|
return s;
|
|
|
|
|
}}},
|
|
|
|
|
{"table_factory", OptionTypeInfo::AsCustomSharedPtr<TableFactory>(
|
|
|
|
|
offset_of(&ImmutableCFOptions::table_factory),
|
|
|
|
|
OptionVerificationType::kByName,
|
|
|
|
|
(OptionTypeFlags::kCompareLoose |
|
|
|
|
|
OptionTypeFlags::kStringNameOnly |
|
|
|
|
|
OptionTypeFlags::kDontPrepare))},
|
|
|
|
|
{"block_based_table_factory",
|
|
|
|
|
{offset_of(&ImmutableCFOptions::table_factory),
|
|
|
|
|
OptionType::kCustomizable, OptionVerificationType::kAlias,
|
|
|
|
|
OptionTypeFlags::kShared | OptionTypeFlags::kCompareLoose,
|
|
|
|
|
// Parses the input value and creates a BlockBasedTableFactory
|
|
|
|
|
[](const ConfigOptions& opts, const std::string& name,
|
|
|
|
|
const std::string& value, void* addr) {
|
|
|
|
|
BlockBasedTableOptions* old_opts = nullptr;
|
|
|
|
|
auto table_factory =
|
|
|
|
|
static_cast<std::shared_ptr<TableFactory>*>(addr);
|
|
|
|
|
if (table_factory->get() != nullptr) {
|
|
|
|
|
old_opts =
|
|
|
|
|
table_factory->get()->GetOptions<BlockBasedTableOptions>();
|
|
|
|
|
}
|
|
|
|
|
if (name == "block_based_table_factory") {
|
|
|
|
|
std::unique_ptr<TableFactory> new_factory;
|
|
|
|
|
if (old_opts != nullptr) {
|
|
|
|
|
new_factory.reset(NewBlockBasedTableFactory(*old_opts));
|
|
|
|
|
} else {
|
|
|
|
|
new_factory.reset(NewBlockBasedTableFactory());
|
|
|
|
|
}
|
|
|
|
|
Status s = new_factory->ConfigureFromString(opts, value);
|
|
|
|
|
if (s.ok()) {
|
|
|
|
|
table_factory->reset(new_factory.release());
|
|
|
|
|
}
|
|
|
|
|
return s;
|
|
|
|
|
} else if (old_opts != nullptr) {
|
|
|
|
|
return table_factory->get()->ConfigureOption(opts, name, value);
|
|
|
|
|
} else {
|
|
|
|
|
return Status::NotFound("Mismatched table option: ", name);
|
|
|
|
|
}
|
|
|
|
|
}}},
|
|
|
|
|
{"plain_table_factory",
|
|
|
|
|
{offset_of(&ImmutableCFOptions::table_factory),
|
|
|
|
|
OptionType::kCustomizable, OptionVerificationType::kAlias,
|
|
|
|
|
OptionTypeFlags::kShared | OptionTypeFlags::kCompareLoose,
|
|
|
|
|
// Parses the input value and creates a PlainTableFactory
|
|
|
|
|
[](const ConfigOptions& opts, const std::string& name,
|
|
|
|
|
const std::string& value, void* addr) {
|
|
|
|
|
PlainTableOptions* old_opts = nullptr;
|
|
|
|
|
auto table_factory =
|
|
|
|
|
static_cast<std::shared_ptr<TableFactory>*>(addr);
|
|
|
|
|
if (table_factory->get() != nullptr) {
|
|
|
|
|
old_opts = table_factory->get()->GetOptions<PlainTableOptions>();
|
|
|
|
|
}
|
|
|
|
|
if (name == "plain_table_factory") {
|
|
|
|
|
std::unique_ptr<TableFactory> new_factory;
|
|
|
|
|
if (old_opts != nullptr) {
|
|
|
|
|
new_factory.reset(NewPlainTableFactory(*old_opts));
|
|
|
|
|
} else {
|
|
|
|
|
new_factory.reset(NewPlainTableFactory());
|
|
|
|
|
}
|
|
|
|
|
Status s = new_factory->ConfigureFromString(opts, value);
|
|
|
|
|
if (s.ok()) {
|
|
|
|
|
table_factory->reset(new_factory.release());
|
|
|
|
|
}
|
|
|
|
|
return s;
|
|
|
|
|
} else if (old_opts != nullptr) {
|
|
|
|
|
return table_factory->get()->ConfigureOption(opts, name, value);
|
|
|
|
|
} else {
|
|
|
|
|
return Status::NotFound("Mismatched table option: ", name);
|
|
|
|
|
}
|
|
|
|
|
}}},
|
|
|
|
|
{"table_properties_collectors",
|
|
|
|
|
OptionTypeInfo::Vector<
|
|
|
|
|
std::shared_ptr<TablePropertiesCollectorFactory>>(
|
|
|
|
|
offset_of(
|
|
|
|
|
&ImmutableCFOptions::table_properties_collector_factories),
|
|
|
|
|
OptionVerificationType::kByName, OptionTypeFlags::kNone,
|
|
|
|
|
OptionTypeInfo::AsCustomSharedPtr<TablePropertiesCollectorFactory>(
|
|
|
|
|
0, OptionVerificationType::kByName, OptionTypeFlags::kNone))},
|
|
|
|
|
{"compaction_filter",
|
|
|
|
|
OptionTypeInfo::AsCustomRawPtr<const CompactionFilter>(
|
|
|
|
|
offset_of(&ImmutableCFOptions::compaction_filter),
|
|
|
|
|
OptionVerificationType::kByName, OptionTypeFlags::kAllowNull)},
|
|
|
|
|
{"compaction_filter_factory",
|
|
|
|
|
OptionTypeInfo::AsCustomSharedPtr<CompactionFilterFactory>(
|
|
|
|
|
offset_of(&ImmutableCFOptions::compaction_filter_factory),
|
|
|
|
|
OptionVerificationType::kByName, OptionTypeFlags::kAllowNull)},
|
|
|
|
|
{"merge_operator",
|
|
|
|
|
OptionTypeInfo::AsCustomSharedPtr<MergeOperator>(
|
|
|
|
|
offset_of(&ImmutableCFOptions::merge_operator),
|
|
|
|
|
OptionVerificationType::kByNameAllowFromNull,
|
|
|
|
|
OptionTypeFlags::kCompareLoose | OptionTypeFlags::kAllowNull)},
|
|
|
|
|
{"compaction_style",
|
|
|
|
|
{offset_of(&ImmutableCFOptions::compaction_style),
|
|
|
|
|
OptionType::kCompactionStyle, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kNone}},
|
|
|
|
|
{"compaction_pri",
|
|
|
|
|
{offset_of(&ImmutableCFOptions::compaction_pri),
|
|
|
|
|
OptionType::kCompactionPri, OptionVerificationType::kNormal,
|
|
|
|
|
OptionTypeFlags::kNone}},
|
|
|
|
|
{"sst_partitioner_factory",
|
|
|
|
|
OptionTypeInfo::AsCustomSharedPtr<SstPartitionerFactory>(
|
|
|
|
|
offset_of(&ImmutableCFOptions::sst_partitioner_factory),
|
|
|
|
|
OptionVerificationType::kByName, OptionTypeFlags::kAllowNull)},
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
const std::string OptionsHelper::kCFOptionsName = "ColumnFamilyOptions";
|
|
|
|
|
|
|
|
|
|
class ConfigurableMutableCFOptions : public Configurable {
|
|
|
|
|
public:
|
|
|
|
|
explicit ConfigurableMutableCFOptions(const MutableCFOptions& mcf) {
|
|
|
|
|
mutable_ = mcf;
|
|
|
|
|
RegisterOptions(&mutable_, &cf_mutable_options_type_info);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
protected:
|
|
|
|
|
MutableCFOptions mutable_;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
class ConfigurableCFOptions : public ConfigurableMutableCFOptions {
|
|
|
|
|
public:
|
|
|
|
|
ConfigurableCFOptions(const ColumnFamilyOptions& opts,
|
|
|
|
|
const std::unordered_map<std::string, std::string>* map)
|
|
|
|
|
: ConfigurableMutableCFOptions(MutableCFOptions(opts)),
|
|
|
|
|
immutable_(opts),
|
|
|
|
|
cf_options_(opts),
|
|
|
|
|
opt_map_(map) {
|
|
|
|
|
RegisterOptions(&immutable_, &cf_immutable_options_type_info);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
protected:
|
|
|
|
|
Status ConfigureOptions(
|
|
|
|
|
const ConfigOptions& config_options,
|
|
|
|
|
const std::unordered_map<std::string, std::string>& opts_map,
|
|
|
|
|
std::unordered_map<std::string, std::string>* unused) override {
|
|
|
|
|
Status s = Configurable::ConfigureOptions(config_options, opts_map, unused);
|
|
|
|
|
if (s.ok()) {
|
|
|
|
|
UpdateColumnFamilyOptions(mutable_, &cf_options_);
|
|
|
|
|
UpdateColumnFamilyOptions(immutable_, &cf_options_);
|
|
|
|
|
s = PrepareOptions(config_options);
|
|
|
|
|
}
|
|
|
|
|
return s;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
virtual const void* GetOptionsPtr(const std::string& name) const override {
|
|
|
|
|
if (name == OptionsHelper::kCFOptionsName) {
|
|
|
|
|
return &cf_options_;
|
|
|
|
|
} else {
|
|
|
|
|
return ConfigurableMutableCFOptions::GetOptionsPtr(name);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
bool OptionsAreEqual(const ConfigOptions& config_options,
|
|
|
|
|
const OptionTypeInfo& opt_info,
|
|
|
|
|
const std::string& opt_name, const void* const this_ptr,
|
|
|
|
|
const void* const that_ptr,
|
|
|
|
|
std::string* mismatch) const override {
|
|
|
|
|
bool equals = opt_info.AreEqual(config_options, opt_name, this_ptr,
|
|
|
|
|
that_ptr, mismatch);
|
|
|
|
|
if (!equals && opt_info.IsByName()) {
|
|
|
|
|
if (opt_map_ == nullptr) {
|
|
|
|
|
equals = true;
|
|
|
|
|
} else {
|
|
|
|
|
const auto& iter = opt_map_->find(opt_name);
|
|
|
|
|
if (iter == opt_map_->end()) {
|
|
|
|
|
equals = true;
|
|
|
|
|
} else {
|
|
|
|
|
equals = opt_info.AreEqualByName(config_options, opt_name, this_ptr,
|
|
|
|
|
iter->second);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (equals) { // False alarm, clear mismatch
|
|
|
|
|
*mismatch = "";
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (equals && opt_info.IsConfigurable() && opt_map_ != nullptr) {
|
|
|
|
|
const auto* this_config = opt_info.AsRawPointer<Configurable>(this_ptr);
|
|
|
|
|
if (this_config == nullptr) {
|
|
|
|
|
const auto& iter = opt_map_->find(opt_name);
|
|
|
|
|
// If the name exists in the map and is not empty/null,
|
|
|
|
|
// then the this_config should be set.
|
|
|
|
|
if (iter != opt_map_->end() && !iter->second.empty() &&
|
|
|
|
|
iter->second != kNullptrString) {
|
|
|
|
|
*mismatch = opt_name;
|
|
|
|
|
equals = false;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return equals;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private:
|
|
|
|
|
ImmutableCFOptions immutable_;
|
|
|
|
|
ColumnFamilyOptions cf_options_;
|
|
|
|
|
const std::unordered_map<std::string, std::string>* opt_map_;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
std::unique_ptr<Configurable> CFOptionsAsConfigurable(
|
|
|
|
|
const MutableCFOptions& opts) {
|
|
|
|
|
std::unique_ptr<Configurable> ptr(new ConfigurableMutableCFOptions(opts));
|
|
|
|
|
return ptr;
|
|
|
|
|
}
|
|
|
|
|
std::unique_ptr<Configurable> CFOptionsAsConfigurable(
|
|
|
|
|
const ColumnFamilyOptions& opts,
|
|
|
|
|
const std::unordered_map<std::string, std::string>* opt_map) {
|
|
|
|
|
std::unique_ptr<Configurable> ptr(new ConfigurableCFOptions(opts, opt_map));
|
|
|
|
|
return ptr;
|
|
|
|
|
}
|
|
|
|
|
#endif // ROCKSDB_LITE
|
|
|
|
|
|
|
|
|
|
ImmutableCFOptions::ImmutableCFOptions() : ImmutableCFOptions(Options()) {}
|
|
|
|
|
|
|
|
|
|
ImmutableCFOptions::ImmutableCFOptions(const ColumnFamilyOptions& cf_options)
|
|
|
|
|
: compaction_style(cf_options.compaction_style),
|
|
|
|
|
compaction_pri(cf_options.compaction_pri),
|
|
|
|
|
user_comparator(cf_options.comparator),
|
|
|
|
|
internal_comparator(InternalKeyComparator(cf_options.comparator)),
|
|
|
|
|
merge_operator(cf_options.merge_operator),
|
|
|
|
|
compaction_filter(cf_options.compaction_filter),
|
|
|
|
|
compaction_filter_factory(cf_options.compaction_filter_factory),
|
|
|
|
|
min_write_buffer_number_to_merge(
|
|
|
|
|
cf_options.min_write_buffer_number_to_merge),
|
|
|
|
|
max_write_buffer_number_to_maintain(
|
|
|
|
|
cf_options.max_write_buffer_number_to_maintain),
|
Refactor trimming logic for immutable memtables (#5022)
Summary:
MyRocks currently sets `max_write_buffer_number_to_maintain` in order to maintain enough history for transaction conflict checking. The effectiveness of this approach depends on the size of memtables. When memtables are small, it may not keep enough history; when memtables are large, this may consume too much memory.
We are proposing a new way to configure memtable list history: by limiting the memory usage of immutable memtables. The new option is `max_write_buffer_size_to_maintain` and it will take precedence over the old `max_write_buffer_number_to_maintain` if they are both set to non-zero values. The new option accounts for the total memory usage of flushed immutable memtables and mutable memtable. When the total usage exceeds the limit, RocksDB may start dropping immutable memtables (which is also called trimming history), starting from the oldest one.
The semantics of the old option actually works both as an upper bound and lower bound. History trimming will start if number of immutable memtables exceeds the limit, but it will never go below (limit-1) due to history trimming.
In order the mimic the behavior with the new option, history trimming will stop if dropping the next immutable memtable causes the total memory usage go below the size limit. For example, assuming the size limit is set to 64MB, and there are 3 immutable memtables with sizes of 20, 30, 30. Although the total memory usage is 80MB > 64MB, dropping the oldest memtable will reduce the memory usage to 60MB < 64MB, so in this case no memtable will be dropped.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5022
Differential Revision: D14394062
Pulled By: miasantreble
fbshipit-source-id: 60457a509c6af89d0993f988c9b5c2aa9e45f5c5
5 years ago
|
|
|
max_write_buffer_size_to_maintain(
|
|
|
|
|
cf_options.max_write_buffer_size_to_maintain),
|
|
|
|
|
inplace_update_support(cf_options.inplace_update_support),
|
|
|
|
|
inplace_callback(cf_options.inplace_callback),
|
|
|
|
|
memtable_factory(cf_options.memtable_factory),
|
|
|
|
|
table_factory(cf_options.table_factory),
|
|
|
|
|
table_properties_collector_factories(
|
|
|
|
|
cf_options.table_properties_collector_factories),
|
|
|
|
|
bloom_locality(cf_options.bloom_locality),
|
|
|
|
|
purge_redundant_kvs_while_flush(
|
|
|
|
|
cf_options.purge_redundant_kvs_while_flush),
|
|
|
|
|
compression_per_level(cf_options.compression_per_level),
|
|
|
|
|
level_compaction_dynamic_level_bytes(
|
|
|
|
|
cf_options.level_compaction_dynamic_level_bytes),
|
|
|
|
|
num_levels(cf_options.num_levels),
|
|
|
|
|
optimize_filters_for_hits(cf_options.optimize_filters_for_hits),
|
|
|
|
|
force_consistency_checks(cf_options.force_consistency_checks),
|
|
|
|
|
memtable_insert_with_hint_prefix_extractor(
|
|
|
|
|
cf_options.memtable_insert_with_hint_prefix_extractor),
|
Concurrent task limiter for compaction thread control (#4332)
Summary:
The PR is targeting to resolve the issue of:
https://github.com/facebook/rocksdb/issues/3972#issue-330771918
We have a rocksdb created with leveled-compaction with multiple column families (CFs), some of CFs are using HDD to store big and less frequently accessed data and others are using SSD.
When there are continuously write traffics going on to all CFs, the compaction thread pool is mostly occupied by those slow HDD compactions, which blocks fully utilize SSD bandwidth.
Since atomic write and transaction is needed across CFs, so splitting it to multiple rocksdb instance is not an option for us.
With the compaction thread control, we got 30%+ HDD write throughput gain, and also a lot smooth SSD write since less write stall happening.
ConcurrentTaskLimiter can be shared with multi-CFs across rocksdb instances, so the feature does not only work for multi-CFs scenarios, but also for multi-rocksdbs scenarios, who need disk IO resource control per tenant.
The usage is straight forward:
e.g.:
//
// Enable compaction thread limiter thru ColumnFamilyOptions
//
std::shared_ptr<ConcurrentTaskLimiter> ctl(NewConcurrentTaskLimiter("foo_limiter", 4));
Options options;
ColumnFamilyOptions cf_opt(options);
cf_opt.compaction_thread_limiter = ctl;
...
//
// Compaction thread limiter can be tuned or disabled on-the-fly
//
ctl->SetMaxOutstandingTask(12); // enlarge to 12 tasks
...
ctl->ResetMaxOutstandingTask(); // disable (bypass) thread limiter
ctl->SetMaxOutstandingTask(-1); // Same as above
...
ctl->SetMaxOutstandingTask(0); // full throttle (0 task)
//
// Sharing compaction thread limiter among CFs (to resolve multiple storage perf issue)
//
std::shared_ptr<ConcurrentTaskLimiter> ctl_ssd(NewConcurrentTaskLimiter("ssd_limiter", 8));
std::shared_ptr<ConcurrentTaskLimiter> ctl_hdd(NewConcurrentTaskLimiter("hdd_limiter", 4));
Options options;
ColumnFamilyOptions cf_opt_ssd1(options);
ColumnFamilyOptions cf_opt_ssd2(options);
ColumnFamilyOptions cf_opt_hdd1(options);
ColumnFamilyOptions cf_opt_hdd2(options);
ColumnFamilyOptions cf_opt_hdd3(options);
// SSD CFs
cf_opt_ssd1.compaction_thread_limiter = ctl_ssd;
cf_opt_ssd2.compaction_thread_limiter = ctl_ssd;
// HDD CFs
cf_opt_hdd1.compaction_thread_limiter = ctl_hdd;
cf_opt_hdd2.compaction_thread_limiter = ctl_hdd;
cf_opt_hdd3.compaction_thread_limiter = ctl_hdd;
...
//
// The limiter is disabled by default (or set to nullptr explicitly)
//
Options options;
ColumnFamilyOptions cf_opt(options);
cf_opt.compaction_thread_limiter = nullptr;
Pull Request resolved: https://github.com/facebook/rocksdb/pull/4332
Differential Revision: D13226590
Pulled By: siying
fbshipit-source-id: 14307aec55b8bd59c8223d04aa6db3c03d1b0c1d
6 years ago
|
|
|
cf_paths(cf_options.cf_paths),
|
|
|
|
|
compaction_thread_limiter(cf_options.compaction_thread_limiter),
|
|
|
|
|
sst_partitioner_factory(cf_options.sst_partitioner_factory) {}
|
|
|
|
|
|
|
|
|
|
ImmutableOptions::ImmutableOptions() : ImmutableOptions(Options()) {}
|
|
|
|
|
|
|
|
|
|
ImmutableOptions::ImmutableOptions(const Options& options)
|
|
|
|
|
: ImmutableOptions(options, options) {}
|
|
|
|
|
|
|
|
|
|
ImmutableOptions::ImmutableOptions(const DBOptions& db_options,
|
|
|
|
|
const ColumnFamilyOptions& cf_options)
|
|
|
|
|
: ImmutableDBOptions(db_options), ImmutableCFOptions(cf_options) {}
|
|
|
|
|
|
|
|
|
|
ImmutableOptions::ImmutableOptions(const DBOptions& db_options,
|
|
|
|
|
const ImmutableCFOptions& cf_options)
|
|
|
|
|
: ImmutableDBOptions(db_options), ImmutableCFOptions(cf_options) {}
|
|
|
|
|
|
|
|
|
|
ImmutableOptions::ImmutableOptions(const ImmutableDBOptions& db_options,
|
|
|
|
|
const ColumnFamilyOptions& cf_options)
|
|
|
|
|
: ImmutableDBOptions(db_options), ImmutableCFOptions(cf_options) {}
|
|
|
|
|
|
|
|
|
|
ImmutableOptions::ImmutableOptions(const ImmutableDBOptions& db_options,
|
|
|
|
|
const ImmutableCFOptions& cf_options)
|
|
|
|
|
: ImmutableDBOptions(db_options), ImmutableCFOptions(cf_options) {}
|
|
|
|
|
|
|
|
|
|
// Multiple two operands. If they overflow, return op1.
|
|
|
|
|
uint64_t MultiplyCheckOverflow(uint64_t op1, double op2) {
|
|
|
|
|
if (op1 == 0 || op2 <= 0) {
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
if (port::kMaxUint64 / op1 < op2) {
|
|
|
|
|
return op1;
|
|
|
|
|
}
|
|
|
|
|
return static_cast<uint64_t>(op1 * op2);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// when level_compaction_dynamic_level_bytes is true and leveled compaction
|
|
|
|
|
// is used, the base level is not always L1, so precomupted max_file_size can
|
|
|
|
|
// no longer be used. Recompute file_size_for_level from base level.
|
|
|
|
|
uint64_t MaxFileSizeForLevel(const MutableCFOptions& cf_options,
|
|
|
|
|
int level, CompactionStyle compaction_style, int base_level,
|
|
|
|
|
bool level_compaction_dynamic_level_bytes) {
|
|
|
|
|
if (!level_compaction_dynamic_level_bytes || level < base_level ||
|
|
|
|
|
compaction_style != kCompactionStyleLevel) {
|
|
|
|
|
assert(level >= 0);
|
|
|
|
|
assert(level < (int)cf_options.max_file_size.size());
|
|
|
|
|
return cf_options.max_file_size[level];
|
|
|
|
|
} else {
|
|
|
|
|
assert(level >= 0 && base_level >= 0);
|
|
|
|
|
assert(level - base_level < (int)cf_options.max_file_size.size());
|
|
|
|
|
return cf_options.max_file_size[level - base_level];
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
size_t MaxFileSizeForL0MetaPin(const MutableCFOptions& cf_options) {
|
|
|
|
|
// We do not want to pin meta-blocks that almost certainly came from intra-L0
|
|
|
|
|
// or a former larger `write_buffer_size` value to avoid surprising users with
|
|
|
|
|
// pinned memory usage. We use a factor of 1.5 to account for overhead
|
|
|
|
|
// introduced during flush in most cases.
|
|
|
|
|
if (port::kMaxSizet / 3 < cf_options.write_buffer_size / 2) {
|
|
|
|
|
return port::kMaxSizet;
|
|
|
|
|
}
|
|
|
|
|
return cf_options.write_buffer_size / 2 * 3;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void MutableCFOptions::RefreshDerivedOptions(int num_levels,
|
|
|
|
|
CompactionStyle compaction_style) {
|
|
|
|
|
max_file_size.resize(num_levels);
|
|
|
|
|
for (int i = 0; i < num_levels; ++i) {
|
|
|
|
|
if (i == 0 && compaction_style == kCompactionStyleUniversal) {
|
|
|
|
|
max_file_size[i] = ULLONG_MAX;
|
|
|
|
|
} else if (i > 1) {
|
|
|
|
|
max_file_size[i] = MultiplyCheckOverflow(max_file_size[i - 1],
|
|
|
|
|
target_file_size_multiplier);
|
|
|
|
|
} else {
|
|
|
|
|
max_file_size[i] = target_file_size_base;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void MutableCFOptions::Dump(Logger* log) const {
|
|
|
|
|
// Memtable related options
|
|
|
|
|
ROCKS_LOG_INFO(log,
|
|
|
|
|
" write_buffer_size: %" ROCKSDB_PRIszt,
|
|
|
|
|
write_buffer_size);
|
|
|
|
|
ROCKS_LOG_INFO(log, " max_write_buffer_number: %d",
|
|
|
|
|
max_write_buffer_number);
|
|
|
|
|
ROCKS_LOG_INFO(log,
|
|
|
|
|
" arena_block_size: %" ROCKSDB_PRIszt,
|
|
|
|
|
arena_block_size);
|
|
|
|
|
ROCKS_LOG_INFO(log, " memtable_prefix_bloom_ratio: %f",
|
|
|
|
|
memtable_prefix_bloom_size_ratio);
|
|
|
|
|
ROCKS_LOG_INFO(log, " memtable_whole_key_filtering: %d",
|
|
|
|
|
memtable_whole_key_filtering);
|
|
|
|
|
ROCKS_LOG_INFO(log,
|
|
|
|
|
" memtable_huge_page_size: %" ROCKSDB_PRIszt,
|
|
|
|
|
memtable_huge_page_size);
|
|
|
|
|
ROCKS_LOG_INFO(log,
|
|
|
|
|
" max_successive_merges: %" ROCKSDB_PRIszt,
|
|
|
|
|
max_successive_merges);
|
|
|
|
|
ROCKS_LOG_INFO(log,
|
|
|
|
|
" inplace_update_num_locks: %" ROCKSDB_PRIszt,
|
|
|
|
|
inplace_update_num_locks);
|
|
|
|
|
ROCKS_LOG_INFO(log, " prefix_extractor: %s",
|
|
|
|
|
prefix_extractor == nullptr
|
|
|
|
|
? "nullptr"
|
|
|
|
|
: prefix_extractor->GetId().c_str());
|
|
|
|
|
ROCKS_LOG_INFO(log, " disable_auto_compactions: %d",
|
|
|
|
|
disable_auto_compactions);
|
|
|
|
|
ROCKS_LOG_INFO(log, " soft_pending_compaction_bytes_limit: %" PRIu64,
|
|
|
|
|
soft_pending_compaction_bytes_limit);
|
|
|
|
|
ROCKS_LOG_INFO(log, " hard_pending_compaction_bytes_limit: %" PRIu64,
|
|
|
|
|
hard_pending_compaction_bytes_limit);
|
|
|
|
|
ROCKS_LOG_INFO(log, " level0_file_num_compaction_trigger: %d",
|
|
|
|
|
level0_file_num_compaction_trigger);
|
|
|
|
|
ROCKS_LOG_INFO(log, " level0_slowdown_writes_trigger: %d",
|
|
|
|
|
level0_slowdown_writes_trigger);
|
|
|
|
|
ROCKS_LOG_INFO(log, " level0_stop_writes_trigger: %d",
|
|
|
|
|
level0_stop_writes_trigger);
|
|
|
|
|
ROCKS_LOG_INFO(log, " max_compaction_bytes: %" PRIu64,
|
|
|
|
|
max_compaction_bytes);
|
|
|
|
|
ROCKS_LOG_INFO(log, " target_file_size_base: %" PRIu64,
|
|
|
|
|
target_file_size_base);
|
|
|
|
|
ROCKS_LOG_INFO(log, " target_file_size_multiplier: %d",
|
|
|
|
|
target_file_size_multiplier);
|
|
|
|
|
ROCKS_LOG_INFO(log, " max_bytes_for_level_base: %" PRIu64,
|
|
|
|
|
max_bytes_for_level_base);
|
|
|
|
|
ROCKS_LOG_INFO(log, " max_bytes_for_level_multiplier: %f",
|
|
|
|
|
max_bytes_for_level_multiplier);
|
|
|
|
|
ROCKS_LOG_INFO(log, " ttl: %" PRIu64,
|
|
|
|
|
ttl);
|
Periodic Compactions (#5166)
Summary:
Introducing Periodic Compactions.
This feature allows all the files in a CF to be periodically compacted. It could help in catching any corruptions that could creep into the DB proactively as every file is constantly getting re-compacted. And also, of course, it helps to cleanup data older than certain threshold.
- Introduced a new option `periodic_compaction_time` to control how long a file can live without being compacted in a CF.
- This works across all levels.
- The files are put in the same level after going through the compaction. (Related files in the same level are picked up as `ExpandInputstoCleanCut` is used).
- Compaction filters, if any, are invoked as usual.
- A new table property, `file_creation_time`, is introduced to implement this feature. This property is set to the time at which the SST file was created (and that time is given by the underlying Env/OS).
This feature can be enabled on its own, or in conjunction with `ttl`. It is possible to set a different time threshold for the bottom level when used in conjunction with ttl. Since `ttl` works only on 0 to last but one levels, you could set `ttl` to, say, 1 day, and `periodic_compaction_time` to, say, 7 days. Since `ttl < periodic_compaction_time` all files in last but one levels keep getting picked up based on ttl, and almost never based on periodic_compaction_time. The files in the bottom level get picked up for compaction based on `periodic_compaction_time`.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/5166
Differential Revision: D14884441
Pulled By: sagar0
fbshipit-source-id: 408426cbacb409c06386a98632dcf90bfa1bda47
6 years ago
|
|
|
ROCKS_LOG_INFO(log, " periodic_compaction_seconds: %" PRIu64,
|
|
|
|
|
periodic_compaction_seconds);
|
|
|
|
|
std::string result;
|
|
|
|
|
char buf[10];
|
|
|
|
|
for (const auto m : max_bytes_for_level_multiplier_additional) {
|
|
|
|
|
snprintf(buf, sizeof(buf), "%d, ", m);
|
|
|
|
|
result += buf;
|
|
|
|
|
}
|
|
|
|
|
if (result.size() >= 2) {
|
|
|
|
|
result.resize(result.size() - 2);
|
|
|
|
|
} else {
|
|
|
|
|
result = "";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
ROCKS_LOG_INFO(log, "max_bytes_for_level_multiplier_additional: %s",
|
|
|
|
|
result.c_str());
|
|
|
|
|
ROCKS_LOG_INFO(log, " max_sequential_skip_in_iterations: %" PRIu64,
|
|
|
|
|
max_sequential_skip_in_iterations);
|
|
|
|
|
ROCKS_LOG_INFO(log, " check_flush_compaction_key_order: %d",
|
|
|
|
|
check_flush_compaction_key_order);
|
|
|
|
|
ROCKS_LOG_INFO(log, " paranoid_file_checks: %d",
|
|
|
|
|
paranoid_file_checks);
|
|
|
|
|
ROCKS_LOG_INFO(log, " report_bg_io_stats: %d",
|
|
|
|
|
report_bg_io_stats);
|
|
|
|
|
ROCKS_LOG_INFO(log, " compression: %d",
|
|
|
|
|
static_cast<int>(compression));
|
|
|
|
|
|
|
|
|
|
// Universal Compaction Options
|
|
|
|
|
ROCKS_LOG_INFO(log, "compaction_options_universal.size_ratio : %d",
|
|
|
|
|
compaction_options_universal.size_ratio);
|
|
|
|
|
ROCKS_LOG_INFO(log, "compaction_options_universal.min_merge_width : %d",
|
|
|
|
|
compaction_options_universal.min_merge_width);
|
|
|
|
|
ROCKS_LOG_INFO(log, "compaction_options_universal.max_merge_width : %d",
|
|
|
|
|
compaction_options_universal.max_merge_width);
|
|
|
|
|
ROCKS_LOG_INFO(
|
|
|
|
|
log, "compaction_options_universal.max_size_amplification_percent : %d",
|
|
|
|
|
compaction_options_universal.max_size_amplification_percent);
|
|
|
|
|
ROCKS_LOG_INFO(log,
|
|
|
|
|
"compaction_options_universal.compression_size_percent : %d",
|
|
|
|
|
compaction_options_universal.compression_size_percent);
|
|
|
|
|
ROCKS_LOG_INFO(log, "compaction_options_universal.stop_style : %d",
|
|
|
|
|
compaction_options_universal.stop_style);
|
|
|
|
|
ROCKS_LOG_INFO(
|
|
|
|
|
log, "compaction_options_universal.allow_trivial_move : %d",
|
|
|
|
|
static_cast<int>(compaction_options_universal.allow_trivial_move));
|
Incremental Space Amp Compactions in Universal Style (#8655)
Summary:
This commit introduces incremental compaction in univeral style for space amplification. This follows the first improvement mentioned in https://rocksdb.org/blog/2021/04/12/universal-improvements.html . The implemention simply picks up files about size of max_compaction_bytes to compact and execute if the penalty is not too big. More optimizations can be done in the future, e.g. prioritizing between this compaction and other types. But for now, the feature is supposed to be functional and can often reduce frequency of full compactions, although it can introduce penalty.
In order to add cut files more efficiently so that more files from upper levels can be included, SST file cutting threshold (for current file + overlapping parent level files) is set to 1.5X of target file size. A 2MB target file size will generate files like this: https://gist.github.com/siying/29d2676fba417404f3c95e6c013c7de8 Number of files indeed increases but it is not out of control.
Two set of write benchmarks are run:
1. For ingestion rate limited scenario, we can see full compaction is mostly eliminated: https://gist.github.com/siying/959bc1186066906831cf4c808d6e0a19 . The write amp increased from 7.7 to 9.4, as expected. After applying file cutting, the number is improved to 8.9. In another benchmark, the write amp is even better with the incremental approach: https://gist.github.com/siying/d1c16c286d7c59c4d7bba718ca198163
2. For ingestion rate unlimited scenario, incremental compaction turns out to be too expensive most of the time and is not executed, as expected.
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8655
Test Plan: Add unit tests to the functionality.
Reviewed By: ajkr
Differential Revision: D31787034
fbshipit-source-id: ce813e63b15a61d5a56e97bf8902a1b28e011beb
3 years ago
|
|
|
ROCKS_LOG_INFO(log, "compaction_options_universal.incremental : %d",
|
|
|
|
|
static_cast<int>(compaction_options_universal.incremental));
|
|
|
|
|
|
|
|
|
|
// FIFO Compaction Options
|
|
|
|
|
ROCKS_LOG_INFO(log, "compaction_options_fifo.max_table_files_size : %" PRIu64,
|
|
|
|
|
compaction_options_fifo.max_table_files_size);
|
|
|
|
|
ROCKS_LOG_INFO(log, "compaction_options_fifo.allow_compaction : %d",
|
|
|
|
|
compaction_options_fifo.allow_compaction);
|
|
|
|
|
|
|
|
|
|
// Blob file related options
|
|
|
|
|
ROCKS_LOG_INFO(log, " enable_blob_files: %s",
|
|
|
|
|
enable_blob_files ? "true" : "false");
|
|
|
|
|
ROCKS_LOG_INFO(log, " min_blob_size: %" PRIu64,
|
|
|
|
|
min_blob_size);
|
|
|
|
|
ROCKS_LOG_INFO(log, " blob_file_size: %" PRIu64,
|
|
|
|
|
blob_file_size);
|
|
|
|
|
ROCKS_LOG_INFO(log, " blob_compression_type: %s",
|
|
|
|
|
CompressionTypeToString(blob_compression_type).c_str());
|
|
|
|
|
ROCKS_LOG_INFO(log, " enable_blob_garbage_collection: %s",
|
|
|
|
|
enable_blob_garbage_collection ? "true" : "false");
|
|
|
|
|
ROCKS_LOG_INFO(log, " blob_garbage_collection_age_cutoff: %f",
|
|
|
|
|
blob_garbage_collection_age_cutoff);
|
Make it possible to force the garbage collection of the oldest blob files (#8994)
Summary:
The current BlobDB garbage collection logic works by relocating the valid
blobs from the oldest blob files as they are encountered during compaction,
and cleaning up blob files once they contain nothing but garbage. However,
with sufficiently skewed workloads, it is theoretically possible to end up in a
situation when few or no compactions get scheduled for the SST files that contain
references to the oldest blob files, which can lead to increased space amp due
to the lack of GC.
In order to efficiently handle such workloads, the patch adds a new BlobDB
configuration option called `blob_garbage_collection_force_threshold`,
which signals to BlobDB to schedule targeted compactions for the SST files
that keep alive the oldest batch of blob files if the overall ratio of garbage in
the given blob files meets the threshold *and* all the given blob files are
eligible for GC based on `blob_garbage_collection_age_cutoff`. (For example,
if the new option is set to 0.9, targeted compactions will get scheduled if the
sum of garbage bytes meets or exceeds 90% of the sum of total bytes in the
oldest blob files, assuming all affected blob files are below the age-based cutoff.)
The net result of these targeted compactions is that the valid blobs in the oldest
blob files are relocated and the oldest blob files themselves cleaned up (since
*all* SST files that rely on them get compacted away).
These targeted compactions are similar to periodic compactions in the sense
that they force certain SST files that otherwise would not get picked up to undergo
compaction and also in the sense that instead of merging files from multiple levels,
they target a single file. (Note: such compactions might still include neighboring files
from the same level due to the need of having a "clean cut" boundary but they never
include any files from any other level.)
This functionality is currently only supported with the leveled compaction style
and is inactive by default (since the default value is set to 1.0, i.e. 100%).
Pull Request resolved: https://github.com/facebook/rocksdb/pull/8994
Test Plan: Ran `make check` and tested using `db_bench` and the stress/crash tests.
Reviewed By: riversand963
Differential Revision: D31489850
Pulled By: ltamasi
fbshipit-source-id: 44057d511726a0e2a03c5d9313d7511b3f0c4eab
3 years ago
|
|
|
ROCKS_LOG_INFO(log, " blob_garbage_collection_force_threshold: %f",
|
|
|
|
|
blob_garbage_collection_force_threshold);
|
|
|
|
|
ROCKS_LOG_INFO(log, " blob_compaction_readahead_size: %" PRIu64,
|
|
|
|
|
blob_compaction_readahead_size);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
MutableCFOptions::MutableCFOptions(const Options& options)
|
|
|
|
|
: MutableCFOptions(ColumnFamilyOptions(options)) {}
|
|
|
|
|
|
|
|
|
|
#ifndef ROCKSDB_LITE
|
|
|
|
|
Status GetMutableOptionsFromStrings(
|
|
|
|
|
const MutableCFOptions& base_options,
|
|
|
|
|
const std::unordered_map<std::string, std::string>& options_map,
|
|
|
|
|
Logger* /*info_log*/, MutableCFOptions* new_options) {
|
|
|
|
|
assert(new_options);
|
|
|
|
|
*new_options = base_options;
|
|
|
|
|
ConfigOptions config_options;
|
|
|
|
|
Status s = OptionTypeInfo::ParseType(
|
|
|
|
|
config_options, options_map, cf_mutable_options_type_info, new_options);
|
|
|
|
|
if (!s.ok()) {
|
|
|
|
|
*new_options = base_options;
|
|
|
|
|
}
|
|
|
|
|
return s;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
Status GetStringFromMutableCFOptions(const ConfigOptions& config_options,
|
|
|
|
|
const MutableCFOptions& mutable_opts,
|
|
|
|
|
std::string* opt_string) {
|
|
|
|
|
assert(opt_string);
|
|
|
|
|
opt_string->clear();
|
|
|
|
|
return OptionTypeInfo::SerializeType(
|
|
|
|
|
config_options, cf_mutable_options_type_info, &mutable_opts, opt_string);
|
|
|
|
|
}
|
|
|
|
|
#endif // ROCKSDB_LITE
|
|
|
|
|
} // namespace ROCKSDB_NAMESPACE
|
