From c0e0331d0cb1f652c541d35cebd66de00df26992 Mon Sep 17 00:00:00 2001 From: Karl Hobley Date: Wed, 26 Oct 2016 13:53:30 +0100 Subject: [PATCH] Add docstrings from RocksDB comments All the docstrings were taken from https://github.com/facebook/rocksdb/blob/869ae5d7868d7d48dd55b7be02a23581101ed69a/include/rocksdb/options.h --- src/rocksdb_options.rs | 217 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 216 insertions(+), 1 deletion(-) diff --git a/src/rocksdb_options.rs b/src/rocksdb_options.rs index 5f0c67e..1120a7b 100644 --- a/src/rocksdb_options.rs +++ b/src/rocksdb_options.rs @@ -109,6 +109,11 @@ impl Default for BlockBasedOptions { } impl Options { + /// By default, RocksDB uses only one background thread for flush and + /// compaction. Calling this function will set it up such that total of + /// `total_threads` is used. Good value for `total_threads` is the number of + /// cores. You almost definitely want to call this function if your system is + /// bottlenecked by RocksDB. pub fn increase_parallelism(&mut self, parallelism: i32) { unsafe { rocksdb_ffi::rocksdb_options_increase_parallelism(self.inner, @@ -124,6 +129,9 @@ impl Options { } } + /// If true, the database will be created if it is missing. + /// + /// Default: false pub fn create_if_missing(&mut self, create_if_missing: bool) { unsafe { rocksdb_ffi::rocksdb_options_set_create_if_missing( @@ -131,12 +139,24 @@ impl Options { } } + /// Sets the compression algorithm that will be used for the bottommost level that + /// contain files. If level-compaction is used, this option will only affect + /// levels after base level. + /// + /// Default: DBCompressionType::None pub fn compression(&mut self, t: DBCompressionType) { unsafe { rocksdb_ffi::rocksdb_options_set_compression(self.inner, t); } } + /// Different levels can have different compression policies. There + /// are cases where most lower levels would like to use quick compression + /// algorithms while the higher levels (which have more data) use + /// compression algorithms that have better compression but could + /// be slower. This array, if non-empty, should have an entry for + /// each level of the database; these override the value specified in + /// the previous field 'compression'. pub fn compression_per_level(&mut self, level_types: &[DBCompressionType]) { unsafe { rocksdb_ffi::rocksdb_options_set_compression_per_level(self.inner, @@ -165,6 +185,12 @@ impl Options { } } + /// Sets the comparator used to define the order of keys in the table. + /// Default: a comparator that uses lexicographic byte-wise ordering + /// + /// The client must ensure that the comparator supplied here has the + //// same name and orders keys *exactly* the same as the comparator + /// provided to previous open calls on the same DB. pub fn add_comparator(&mut self, name: &str, compare_fn: fn(&[u8], &[u8]) -> i32) { @@ -183,7 +209,6 @@ impl Options { } } - pub fn set_block_cache_size_mb(&mut self, cache_size: u64) { unsafe { rocksdb_ffi::rocksdb_options_optimize_for_point_lookup(self.inner, @@ -191,12 +216,24 @@ impl Options { } } + /// Sets the number of open files that can be used by the DB. You may need to + /// increase this if your database has a large working set. Value -1 means + /// files opened are always kept open. You can estimate number of files based + /// on target_file_size_base and target_file_size_multiplier for level-based + /// compaction. For universal-style compaction, you can usually set it to -1. + /// + /// Default: -1 pub fn set_max_open_files(&mut self, nfiles: c_int) { unsafe { rocksdb_ffi::rocksdb_options_set_max_open_files(self.inner, nfiles); } } + /// If true, then every store to stable storage will issue a fsync. + /// If false, then every store to stable storage will issue a fdatasync. + /// This parameter should be set to true while storing data to + /// filesystem like ext3 that can lose files after a reboot. + /// Default: false pub fn set_use_fsync(&mut self, useit: bool) { unsafe { if useit { @@ -207,6 +244,19 @@ impl Options { } } + /// Allows OS to incrementally sync files to disk while they are being + /// written, asynchronously, in the background. This operation can be used + /// to smooth out write I/Os over time. Users shouldn't rely on it for + /// persistency guarantee. + /// Issue one request for every bytes_per_sync written. 0 turns it off. + /// + /// Default: 0 + /// + /// You may consider using rate_limiter to regulate write rate to device. + /// When rate limiter is enabled, it automatically enables bytes_per_sync + /// to 1MB. + /// + /// This option applies to table files pub fn set_bytes_per_sync(&mut self, nbytes: u64) { unsafe { rocksdb_ffi::rocksdb_options_set_bytes_per_sync(self.inner, nbytes); @@ -223,6 +273,24 @@ impl Options { } } + /// Hints to the OS that it should not buffer disk I/O. Enabling this + /// parameter may improve performance but increases pressure on the + /// system cache. + /// + /// The exact behavior of this parameter is platform dependent. + /// + /// On POSIX systems, after RocksDB reads data from disk it will + /// mark the pages as "unneeded". The operating system may - or may not + /// - evict these pages from memory, reducing pressure on the system + /// cache. If the disk block is requested again this can result in + /// additional disk I/O. + /// + /// On WINDOWS system, files will be opened in "unbuffered I/O" mode + /// which means that data read from the disk will not be cached or + /// bufferized. The hardware buffer of the devices may however still + /// be used. Memory mapped files are not impacted by this parameter. + /// + /// Default: true pub fn allow_os_buffer(&mut self, is_allow: bool) { unsafe { rocksdb_ffi::rocksdb_options_set_allow_os_buffer(self.inner, @@ -230,6 +298,7 @@ impl Options { } } + /// Sets the number of shards used for table cache. pub fn set_table_cache_num_shard_bits(&mut self, nbits: c_int) { unsafe { rocksdb_ffi::rocksdb_options_set_table_cache_numshardbits(self.inner, @@ -237,6 +306,15 @@ impl Options { } } + /// Sets the minimum number of write buffers that will be merged together + /// before writing to storage. If set to 1, then + /// all write buffers are flushed to L0 as individual files and this increases + /// read amplification because a get request has to check in all of these + /// files. Also, an in-memory merge may result in writing lesser + /// data to storage if there are duplicate records in each of these + /// individual write buffers. + /// + /// Default: 1 pub fn set_min_write_buffer_number(&mut self, nbuf: c_int) { unsafe { rocksdb_ffi::rocksdb_options_set_min_write_buffer_number_to_merge( @@ -244,6 +322,30 @@ impl Options { } } + /// Sets the total maximum number of write buffers to maintain in memory including + /// copies of buffers that have already been flushed. Unlike + /// max_write_buffer_number, this parameter does not affect flushing. + /// This controls the minimum amount of write history that will be available + /// in memory for conflict checking when Transactions are used. + /// + /// When using an OptimisticTransactionDB: + /// If this value is too low, some transactions may fail at commit time due + /// to not being able to determine whether there were any write conflicts. + /// + /// When using a TransactionDB: + /// If Transaction::SetSnapshot is used, TransactionDB will read either + /// in-memory write buffers or SST files to do write-conflict checking. + /// Increasing this value can reduce the number of reads to SST files + /// done for conflict detection. + /// + /// Setting this value to 0 will cause write buffers to be freed immediately + /// after they are flushed. + /// If this value is set to -1, 'max_write_buffer_number' will be used. + /// + /// Default: + /// If using a TransactionDB/OptimisticTransactionDB, the default value will + /// be set to the value of 'max_write_buffer_number' if it is not explicitly + /// set by the user. Otherwise, the default is 0. pub fn set_max_write_buffer_number(&mut self, nbuf: c_int) { unsafe { rocksdb_ffi::rocksdb_options_set_max_write_buffer_number(self.inner, @@ -251,6 +353,22 @@ impl Options { } } + /// Sets the amount of data to build up in memory (backed by an unsorted log + /// on disk) before converting to a sorted on-disk file. + /// + /// Larger values increase performance, especially during bulk loads. + /// Up to max_write_buffer_number write buffers may be held in memory + /// at the same time, + /// so you may wish to adjust this parameter to control memory usage. + /// Also, a larger write buffer will result in a longer recovery time + /// the next time the database is opened. + /// + /// Note that write_buffer_size is enforced per column family. + /// See db_write_buffer_size for sharing memory across column families. + /// + /// Default: 64MB + /// + /// Dynamically changeable through SetOptions() API pub fn set_write_buffer_size(&mut self, size: usize) { unsafe { rocksdb_ffi::rocksdb_options_set_write_buffer_size(self.inner, @@ -258,24 +376,52 @@ impl Options { } } + /// Control maximum total data size for a level. + /// max_bytes_for_level_base is the max total for level-1. + /// Maximum number of bytes for level L can be calculated as + /// (max_bytes_for_level_base) * (max_bytes_for_level_multiplier ^ (L-1)) + /// For example, if max_bytes_for_level_base is 200MB, and if + /// max_bytes_for_level_multiplier is 10, total data size for level-1 + /// will be 200MB, total file size for level-2 will be 2GB, + /// and total file size for level-3 will be 20GB. + /// + /// Default: 256MB. + /// + /// Dynamically changeable through SetOptions() API pub fn set_max_bytes_for_level_base(&mut self, size: u64) { unsafe { rocksdb_ffi::rocksdb_options_set_max_bytes_for_level_base(self.inner, size); } } + /// Default: 10 pub fn set_max_bytes_for_level_multiplier(&mut self, mul: i32) { unsafe { rocksdb_ffi::rocksdb_options_set_max_bytes_for_level_multiplier(self.inner, mul); } } + /// The manifest file is rolled over on reaching this limit. + /// The older manifest file be deleted. + /// The default value is MAX_INT so that roll-over does not take place. pub fn set_max_manifest_file_size(&mut self, size: usize) { unsafe { rocksdb_ffi::rocksdb_options_set_max_manifest_file_size(self.inner, size); } } + /// Sets the target file size for compaction. + /// target_file_size_base is per-file size for level-1. + /// Target file size for level L can be calculated by + /// target_file_size_base * (target_file_size_multiplier ^ (L-1)) + /// For example, if target_file_size_base is 2MB and + /// target_file_size_multiplier is 10, then each file on level-1 will + /// be 2MB, and each file on level 2 will be 20MB, + /// and each file on level-3 will be 200MB. + /// + /// Default: 64MB. + /// + /// Dynamically changeable through SetOptions() API pub fn set_target_file_size_base(&mut self, size: u64) { unsafe { rocksdb_ffi::rocksdb_options_set_target_file_size_base(self.inner, @@ -283,6 +429,15 @@ impl Options { } } + /// Sets the minimum number of write buffers that will be merged together + /// before writing to storage. If set to 1, then + /// all write buffers are flushed to L0 as individual files and this increases + /// read amplification because a get request has to check in all of these + /// files. Also, an in-memory merge may result in writing lesser + /// data to storage if there are duplicate records in each of these + /// individual write buffers. + /// + /// Default: 1 pub fn set_min_write_buffer_number_to_merge(&mut self, to_merge: c_int) { unsafe { rocksdb_ffi::rocksdb_options_set_min_write_buffer_number_to_merge( @@ -290,6 +445,12 @@ impl Options { } } + /// Sets the number of files to trigger level-0 compaction. A value <0 means that + /// level-0 compaction will not be triggered by number of files at all. + /// + /// Default: 4 + /// + /// Dynamically changeable through SetOptions() API pub fn set_level_zero_file_num_compaction_trigger(&mut self, n: c_int) { unsafe { rocksdb_ffi::rocksdb_options_set_level0_file_num_compaction_trigger( @@ -297,6 +458,11 @@ impl Options { } } + /// Sets the soft limit on number of level-0 files. We start slowing down writes at this + /// point. A value <0 means that no writing slow down will be triggered by + /// number of files in level-0. + /// + /// Dynamically changeable through SetOptions() API pub fn set_level_zero_slowdown_writes_trigger(&mut self, n: c_int) { unsafe { rocksdb_ffi::rocksdb_options_set_level0_slowdown_writes_trigger( @@ -304,6 +470,9 @@ impl Options { } } + /// Sets the maximum number of level-0 files. We stop writes at this point. + /// + /// Dynamically changeable through SetOptions() API pub fn set_level_zero_stop_writes_trigger(&mut self, n: c_int) { unsafe { rocksdb_ffi::rocksdb_options_set_level0_stop_writes_trigger( @@ -311,6 +480,9 @@ impl Options { } } + /// Sets the compaction style. + /// + /// Default: DBCompactionStyle::Level pub fn set_compaction_style(&mut self, style: rocksdb_ffi::DBCompactionStyle) { unsafe { @@ -319,6 +491,19 @@ impl Options { } } + + /// Sets the maximum number of concurrent background compaction jobs, submitted to + /// the default LOW priority thread pool. + /// We first try to schedule compactions based on + /// `base_background_compactions`. If the compaction cannot catch up , we + /// will increase number of compaction threads up to + /// `max_background_compactions`. + /// + /// If you're increasing this, also consider increasing number of threads in + /// LOW priority thread pool. For more information, see + /// Env::SetBackgroundThreads + /// + /// Default: 1 pub fn set_max_background_compactions(&mut self, n: c_int) { unsafe { rocksdb_ffi::rocksdb_options_set_max_background_compactions( @@ -326,6 +511,22 @@ impl Options { } } + /// Sets the maximum number of concurrent background memtable flush jobs, submitted to + /// the HIGH priority thread pool. + /// + /// By default, all background jobs (major compaction and memtable flush) go + /// to the LOW priority pool. If this option is set to a positive number, + /// memtable flush jobs will be submitted to the HIGH priority pool. + /// It is important when the same Env is shared by multiple db instances. + /// Without a separate pool, long running major compaction jobs could + /// potentially block memtable flush jobs of other db instances, leading to + /// unnecessary Put stalls. + /// + /// If you're increasing this, also consider increasing number of threads in + /// HIGH priority thread pool. For more information, see + /// Env::SetBackgroundThreads + /// + /// Default: 1 pub fn set_max_background_flushes(&mut self, n: c_int) { unsafe { rocksdb_ffi::rocksdb_options_set_max_background_flushes(self.inner, @@ -339,6 +540,10 @@ impl Options { } } + /// Disables automatic compactions. Manual compactions can still + /// be issued on this column family + /// + /// Dynamically changeable through SetOptions() API pub fn set_disable_auto_compactions(&mut self, disable: bool) { let c_bool = if disable { 1 @@ -357,6 +562,9 @@ impl Options { } } + /// Measure IO stats in compactions and flushes, if true. + /// + /// Default: false pub fn set_report_bg_io_stats(&mut self, enable: bool) { unsafe { if enable { @@ -367,6 +575,9 @@ impl Options { } } + /// Recovery mode to control the consistency while replaying WAL + /// + /// Default: DBRecoveryMode::PointInTime pub fn set_wal_recovery_mode(&mut self, mode: DBRecoveryMode) { unsafe { rocksdb_ffi::rocksdb_options_set_wal_recovery_mode(self.inner, mode); @@ -395,6 +606,9 @@ impl Options { } } + /// If not zero, dump rocksdb.stats to LOG every stats_dump_period_sec + /// + /// Default: 600 (10 min) pub fn set_stats_dump_period_sec(&mut self, period: usize) { unsafe { rocksdb_ffi::rocksdb_options_set_stats_dump_period_sec(self.inner, @@ -402,6 +616,7 @@ impl Options { } } + /// Sets the number of levels for this database pub fn set_num_levels(&mut self, n: c_int) { unsafe { rocksdb_ffi::rocksdb_options_set_num_levels(self.inner, n);