From 29a9161f34fff5d5fb76e47175baf2f755c97e45 Mon Sep 17 00:00:00 2001 From: Lei Jin Date: Tue, 4 Nov 2014 16:23:45 -0800 Subject: [PATCH] Note dynamic options in options.h Summary: as title Test Plan: n/a Reviewers: igor, yhchiang, rven, sdong Reviewed By: sdong Subscribers: dhruba, leveldb Differential Revision: https://reviews.facebook.net/D28287 --- HISTORY.md | 1 + include/rocksdb/options.h | 69 +++++++++++++++++++++++++++++++++++---- 2 files changed, 63 insertions(+), 7 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 7c0b5a9b8..9a7cd7810 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -3,6 +3,7 @@ ## Unreleased ### Public API changes +* Introduce SetOptions() API to allow adjusting a subset of options dynamically online * Introduce 4 new convenient functions for converting Options from string: GetColumnFamilyOptionsFromMap(), GetColumnFamilyOptionsFromString(), GetDBOptionsFromMap(), GetDBOptionsFromString() * Remove WriteBatchWithIndex.Delete() overloads using SliceParts * When opening a DB, if options.max_background_compactions is larger than the existing low pri pool of options.env, it will enlarge it. Similarly, options.max_background_flushes is larger than the existing high pri pool of options.env, it will enlarge it. diff --git a/include/rocksdb/options.h b/include/rocksdb/options.h index d9a82fd5a..b3ce77255 100644 --- a/include/rocksdb/options.h +++ b/include/rocksdb/options.h @@ -191,13 +191,18 @@ struct ColumnFamilyOptions { // the next time the database is opened. // // Default: 4MB + // + // Dynamically changeable through SetOptions() API size_t write_buffer_size; // The maximum number of write buffers that are built up in memory. // The default and the minimum number is 2, so that when 1 write buffer // is being flushed to storage, new writes can continue to the other // write buffer. + // // Default: 2 + // + // Dynamically changeable through SetOptions() API int max_write_buffer_number; // The minimum number of write buffers that will be merged together @@ -260,14 +265,20 @@ struct ColumnFamilyOptions { // level-0 compaction will not be triggered by number of files at all. // // Default: 4 + // + // Dynamically changeable through SetOptions() API int level0_file_num_compaction_trigger; // Soft limit on number of level-0 files. We start slowing down writes at this // point. A value <0 means that no writing slow down will be triggered by // number of files in level-0. + // + // Dynamically changeable through SetOptions() API int level0_slowdown_writes_trigger; // Maximum number of level-0 files. We stop writes at this point. + // + // Dynamically changeable through SetOptions() API int level0_stop_writes_trigger; // Maximum level to which a new compacted memtable is pushed if it @@ -276,6 +287,8 @@ struct ColumnFamilyOptions { // expensive manifest file operations. We do not push all the way to // the largest level since that can generate a lot of wasted disk // space if the same key space is being repeatedly overwritten. + // + // Dynamically changeable through SetOptions() API int max_mem_compaction_level; // Target file size for compaction. @@ -286,11 +299,15 @@ struct ColumnFamilyOptions { // target_file_size_multiplier is 10, then each file on level-1 will // be 2MB, and each file on level 2 will be 20MB, // and each file on level-3 will be 200MB. - - // by default target_file_size_base is 2MB. + // + // Default: 2MB. + // + // Dynamically changeable through SetOptions() API uint64_t target_file_size_base; - // by default target_file_size_multiplier is 1, which means + // By default target_file_size_multiplier is 1, which means // by default files in different levels will have similar size. + // + // Dynamically changeable through SetOptions() API int target_file_size_multiplier; // Control maximum total data size for a level. @@ -301,22 +318,31 @@ struct ColumnFamilyOptions { // max_bytes_for_level_multiplier is 10, total data size for level-1 // will be 20MB, total file size for level-2 will be 200MB, // and total file size for level-3 will be 2GB. - - // by default 'max_bytes_for_level_base' is 10MB. + // + // Default: 10MB. + // + // Dynamically changeable through SetOptions() API uint64_t max_bytes_for_level_base; - // by default 'max_bytes_for_level_base' is 10. + // Default: 10. + // + // Dynamically changeable through SetOptions() API int max_bytes_for_level_multiplier; // Different max-size multipliers for different levels. // These are multiplied by max_bytes_for_level_multiplier to arrive // at the max-size of each level. + // // Default: 1 + // + // Dynamically changeable through SetOptions() API std::vector max_bytes_for_level_multiplier_additional; // Maximum number of bytes in all compacted files. We avoid expanding // the lower level file set of a compaction if it would make the // total compaction cover more than // (expanded_compaction_factor * targetFileSizeLevel()) many bytes. + // + // Dynamically changeable through SetOptions() API int expanded_compaction_factor; // Maximum number of bytes in all source files to be compacted in a @@ -326,22 +352,32 @@ struct ColumnFamilyOptions { // (source_compaction_factor * targetFileSizeLevel()) many bytes. // Default:1, i.e. pick maxfilesize amount of data as the source of // a compaction. + // + // Dynamically changeable through SetOptions() API int source_compaction_factor; // Control maximum bytes of overlaps in grandparent (i.e., level+2) before we // stop building a single file in a level->level+1 compaction. + // + // Dynamically changeable through SetOptions() API int max_grandparent_overlap_factor; // Puts are delayed 0-1 ms when any level has a compaction score that exceeds // soft_rate_limit. This is ignored when == 0.0. // CONSTRAINT: soft_rate_limit <= hard_rate_limit. If this constraint does not // hold, RocksDB will set soft_rate_limit = hard_rate_limit + // // Default: 0 (disabled) + // + // Dynamically changeable through SetOptions() API double soft_rate_limit; // Puts are delayed 1ms at a time when any level has a compaction score that // exceeds hard_rate_limit. This is ignored when <= 1.0. + // // Default: 0 (disabled) + // + // Dynamically changeable through SetOptions() API double hard_rate_limit; // DEPRECATED -- this options is no longer used @@ -360,10 +396,14 @@ struct ColumnFamilyOptions { // conforms to the restrictions. // // Default: 0 + // + // Dynamically changeable through SetOptions() API size_t arena_block_size; // Disable automatic compactions. Manual compactions can still // be issued on this column family + // + // Dynamically changeable through SetOptions() API bool disable_auto_compactions; // Purge duplicate/deleted keys when a memtable is flushed to storage. @@ -388,14 +428,20 @@ struct ColumnFamilyOptions { // If KeyMayExist returns false, i.e. the key definitely does not exist, then // the delete is a noop. KeyMayExist only incurs in-memory look up. // This optimization avoids writing the delete to storage when appropriate. + // // Default: false + // + // Dynamically changeable through SetOptions() API bool filter_deletes; // An iteration->Next() sequentially skips over keys with the same // user-key unless this option is set. This number specifies the number // of keys (with the same userkey) that will be sequentially // skipped before a reseek is issued. + // // Default: 8 + // + // Dynamically changeable through SetOptions() API uint64_t max_sequential_skip_in_iterations; // This is a factory that provides MemTableRep objects. @@ -444,6 +490,8 @@ struct ColumnFamilyOptions { // Number of locks used for inplace update // Default: 10000, if inplace_update_support = true, else 0. + // + // Dynamically changeable through SetOptions() API size_t inplace_update_num_locks; // existing_value - pointer to previous value (from both memtable and sst). @@ -490,9 +538,13 @@ struct ColumnFamilyOptions { // if prefix_extractor is set and bloom_bits is not 0, create prefix bloom // for memtable + // + // Dynamically changeable through SetOptions() API uint32_t memtable_prefix_bloom_bits; // number of hash probes per key + // + // Dynamically changeable through SetOptions() API uint32_t memtable_prefix_bloom_probes; // Page size for huge page TLB for bloom in memtable. If <=0, not allocate @@ -500,7 +552,8 @@ struct ColumnFamilyOptions { // Need to reserve huge pages for it to be allocated. For example: // sysctl -w vm.nr_hugepages=20 // See linux doc Documentation/vm/hugetlbpage.txt - + // + // Dynamically changeable through SetOptions() API size_t memtable_prefix_bloom_huge_page_tlb_size; // Control locality of bloom filter probes to improve cache miss rate. @@ -520,6 +573,8 @@ struct ColumnFamilyOptions { // operations in the memtable. // // Default: 0 (disabled) + // + // Dynamically changeable through SetOptions() API size_t max_successive_merges; // The number of partial merge operands to accumulate before partial