Remove deprecated API AdvancedColumnFamilyOptions::soft_rate_limit/hard_rate_limit (#9452)

Summary:
**Context/Summary:**
AdvancedColumnFamilyOptions::soft_rate_limit/hard_rate_limit have been marked as deprecated and it's time to actually remove the code.
- Keep `soft_rate_limit`/`hard_rate_limit` in `cf_mutable_options_type_info` to prevent throwing `InvalidArgument` in `GetColumnFamilyOptionsFromMap` when reading an option file still with these options (e.g, old option file generated from RocksDB before the deprecation)
- Keep `soft_rate_limit`/`hard_rate_limit` in under `OptionsOldApiTest.GetOptionsFromMapTest` to test the case mentioned above.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/9452

Test Plan: Rely on my eyeball and CI

Reviewed By: ajkr

Differential Revision: D33804938

Pulled By: hx235

fbshipit-source-id: 133d49f7ec5238d7efceeb0a3122a5792a2b9945

HISTORY.md

@@ -6,6 +6,8 @@
 * Remove deprecated API DB::AddFile from main repo.
 * Remove deprecated API ObjectLibrary::Register() and the (now obsolete) Regex public API. Use ObjectLibrary::AddFactory() with PatternEntry instead.
 * Remove deprecated option DBOption::table_cache_remove_scan_count_limit.
+* Remove deprecated API AdvancedColumnFamilyOptions::soft_rate_limit.
+* Remove deprecated API AdvancedColumnFamilyOptions::hard_rate_limit.
 
 ## 6.29.0 (01/21/2022)
 Note: The next release will be major release 7.0. See https://github.com/facebook/rocksdb/issues/9390 for more info.

build_tools/regression_build_test.sh

@@ -258,7 +258,6 @@ common_in_mem_args="--db=/dev/shm/rocksdb \
     --value_size=100 \
     --compression_type=none \
     --compression_ratio=1 \
-    --hard_rate_limit=2 \
     --write_buffer_size=134217728 \
     --max_write_buffer_number=4 \
     --level0_file_num_compaction_trigger=8 \

db/c.cc

@@ -3276,22 +3276,6 @@ size_t rocksdb_options_get_recycle_log_file_num(rocksdb_options_t* opt) {
   return opt->rep.recycle_log_file_num;
 }
 
-void rocksdb_options_set_soft_rate_limit(rocksdb_options_t* opt, double v) {
-  opt->rep.soft_rate_limit = v;
-}
-
-double rocksdb_options_get_soft_rate_limit(rocksdb_options_t* opt) {
-  return opt->rep.soft_rate_limit;
-}
-
-void rocksdb_options_set_hard_rate_limit(rocksdb_options_t* opt, double v) {
-  opt->rep.hard_rate_limit = v;
-}
-
-double rocksdb_options_get_hard_rate_limit(rocksdb_options_t* opt) {
-  return opt->rep.hard_rate_limit;
-}
-
 void rocksdb_options_set_soft_pending_compaction_bytes_limit(rocksdb_options_t* opt, size_t v) {
   opt->rep.soft_pending_compaction_bytes_limit = v;
 }

db/c_test.c

@@ -1631,12 +1631,6 @@ int main(int argc, char** argv) {
     rocksdb_options_set_recycle_log_file_num(o, 9);
     CheckCondition(9 == rocksdb_options_get_recycle_log_file_num(o));
 
-    rocksdb_options_set_soft_rate_limit(o, 2.0);
-    CheckCondition(2.0 == rocksdb_options_get_soft_rate_limit(o));
-
-    rocksdb_options_set_hard_rate_limit(o, 4.0);
-    CheckCondition(4.0 == rocksdb_options_get_hard_rate_limit(o));
-
     rocksdb_options_set_soft_pending_compaction_bytes_limit(o, 10);
     CheckCondition(10 ==
                    rocksdb_options_get_soft_pending_compaction_bytes_limit(o));
@@ -1857,8 +1851,6 @@ int main(int argc, char** argv) {
     CheckCondition(7 == rocksdb_options_get_log_file_time_to_roll(copy));
     CheckCondition(8 == rocksdb_options_get_keep_log_file_num(copy));
     CheckCondition(9 == rocksdb_options_get_recycle_log_file_num(copy));
-    CheckCondition(2.0 == rocksdb_options_get_soft_rate_limit(copy));
-    CheckCondition(4.0 == rocksdb_options_get_hard_rate_limit(copy));
     CheckCondition(
         10 == rocksdb_options_get_soft_pending_compaction_bytes_limit(copy));
     CheckCondition(
@@ -2083,14 +2075,6 @@ int main(int argc, char** argv) {
     CheckCondition(19 == rocksdb_options_get_recycle_log_file_num(copy));
     CheckCondition(9 == rocksdb_options_get_recycle_log_file_num(o));
 
-    rocksdb_options_set_soft_rate_limit(copy, 4.0);
-    CheckCondition(4.0 == rocksdb_options_get_soft_rate_limit(copy));
-    CheckCondition(2.0 == rocksdb_options_get_soft_rate_limit(o));
-
-    rocksdb_options_set_hard_rate_limit(copy, 2.0);
-    CheckCondition(2.0 == rocksdb_options_get_hard_rate_limit(copy));
-    CheckCondition(4.0 == rocksdb_options_get_hard_rate_limit(o));
-
     rocksdb_options_set_soft_pending_compaction_bytes_limit(copy, 110);
     CheckCondition(
         110 == rocksdb_options_get_soft_pending_compaction_bytes_limit(copy));

db/db_dynamic_level_test.cc

@@ -69,7 +69,6 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase) {
       options.level_compaction_dynamic_level_bytes = true;
       options.max_bytes_for_level_base = 10240;
       options.max_bytes_for_level_multiplier = 4;
-      options.soft_rate_limit = 1.1;
       options.max_background_compactions = max_background_compactions;
       options.num_levels = 5;
 
@@ -370,7 +369,6 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBaseInc) {
   options.level_compaction_dynamic_level_bytes = true;
   options.max_bytes_for_level_base = 10240;
   options.max_bytes_for_level_multiplier = 4;
-  options.soft_rate_limit = 1.1;
   options.max_background_compactions = 2;
   options.num_levels = 5;
   options.max_compaction_bytes = 100000000;
@@ -422,7 +420,6 @@ TEST_F(DBTestDynamicLevel, DISABLED_MigrateToDynamicLevelMaxBytesBase) {
   options.level_compaction_dynamic_level_bytes = false;
   options.max_bytes_for_level_base = 10240;
   options.max_bytes_for_level_multiplier = 4;
-  options.soft_rate_limit = 1.1;
   options.num_levels = 8;
 
   DestroyAndReopen(options);

db/db_test.cc

@@ -6530,6 +6530,10 @@ TEST_F(DBTest, SoftLimit) {
   ASSERT_OK(dbfull()->TEST_WaitForCompact());
 
   // Now there is one L1 file but doesn't trigger soft_rate_limit
+  //
+  // TODO: soft_rate_limit is depreciated. If this test
+  // relies on soft_rate_limit, then we need to change the test.
+  //
   // The L1 file size is around 30KB.
   ASSERT_EQ(NumTableFilesAtLevel(1), 1);
   ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());

db/db_test_util.cc

@@ -456,7 +456,6 @@ Options DBTestBase::GetOptions(
       options.max_manifest_file_size = 50;  // 50 bytes
       break;
     case kPerfOptions:
-      options.soft_rate_limit = 2.0;
       options.delayed_write_rate = 8 * 1024 * 1024;
       options.report_bg_io_stats = true;
       // TODO(3.13) -- test more options

db_stress_tool/db_stress_test_base.cc

@@ -202,8 +202,6 @@ bool StressTest::BuildOptionsTable() {
       {"inplace_update_num_locks", {"100", "200", "300"}},
       // TODO(ljin): enable test for this option
       // {"disable_auto_compactions", {"100", "200", "300"}},
-      {"soft_rate_limit", {"0", "0.5", "0.9"}},
-      {"hard_rate_limit", {"0", "1.1", "2.0"}},
       {"level0_file_num_compaction_trigger",
        {
            ToString(options_.level0_file_num_compaction_trigger),
@@ -572,12 +570,9 @@ Status StressTest::SetOptions(ThreadState* thread) {
   std::string name =
       options_index_[thread->rand.Next() % options_index_.size()];
   int value_idx = thread->rand.Next() % options_table_[name].size();
-  if (name == "soft_rate_limit" || name == "hard_rate_limit") {
+  if (name == "level0_file_num_compaction_trigger" ||
-    opts["soft_rate_limit"] = options_table_["soft_rate_limit"][value_idx];
+      name == "level0_slowdown_writes_trigger" ||
-    opts["hard_rate_limit"] = options_table_["hard_rate_limit"][value_idx];
+      name == "level0_stop_writes_trigger") {
-  } else if (name == "level0_file_num_compaction_trigger" ||
-             name == "level0_slowdown_writes_trigger" ||
-             name == "level0_stop_writes_trigger") {
     opts["level0_file_num_compaction_trigger"] =
         options_table_["level0_file_num_compaction_trigger"][value_idx];
     opts["level0_slowdown_writes_trigger"] =

examples/rocksdb_option_file_example.ini

@@ -86,7 +86,6 @@
   table_factory=BlockBasedTable
   comparator=leveldb.BytewiseComparator
   max_sequential_skip_in_iterations=8
-  soft_rate_limit=0.000000
   max_bytes_for_level_base=1073741824
   memtable_prefix_bloom_probes=6
   memtable_prefix_bloom_bits=0

include/rocksdb/advanced_options.h

@@ -916,18 +916,6 @@ struct AdvancedColumnFamilyOptions {
   // This does not do anything anymore.
   int max_mem_compaction_level;
 
-  // NOT SUPPORTED ANYMORE -- this options is no longer used
-  // Puts are delayed to options.delayed_write_rate when any level has a
-  // compaction score that exceeds soft_rate_limit. This is ignored when == 0.0.
-  //
-  // Default: 0 (disabled)
-  //
-  // Dynamically changeable through SetOptions() API
-  double soft_rate_limit = 0.0;
-
-  // NOT SUPPORTED ANYMORE -- this options is no longer used
-  double hard_rate_limit = 0.0;
-
   // NOT SUPPORTED ANYMORE -- this options is no longer used
   unsigned int rate_limit_delay_max_milliseconds = 100;
 };

include/rocksdb/c.h

@@ -1194,14 +1194,6 @@ extern ROCKSDB_LIBRARY_API void rocksdb_options_set_recycle_log_file_num(
     rocksdb_options_t*, size_t);
 extern ROCKSDB_LIBRARY_API size_t
 rocksdb_options_get_recycle_log_file_num(rocksdb_options_t*);
-extern ROCKSDB_LIBRARY_API void rocksdb_options_set_soft_rate_limit(
-    rocksdb_options_t*, double);
-extern ROCKSDB_LIBRARY_API double rocksdb_options_get_soft_rate_limit(
-    rocksdb_options_t*);
-extern ROCKSDB_LIBRARY_API void rocksdb_options_set_hard_rate_limit(
-    rocksdb_options_t*, double);
-extern ROCKSDB_LIBRARY_API double rocksdb_options_get_hard_rate_limit(
-    rocksdb_options_t*);
 extern ROCKSDB_LIBRARY_API void rocksdb_options_set_soft_pending_compaction_bytes_limit(
     rocksdb_options_t* opt, size_t v);
 extern ROCKSDB_LIBRARY_API size_t

include/rocksdb/convenience.h

@@ -141,8 +141,10 @@ struct ConfigOptions {
 //   Doubles / Floating Points are converted directly from string.  Note that
 //   currently we do not support units.
 //   [Example]:
-//   - {"hard_rate_limit", "2.1"} in GetColumnFamilyOptionsFromMap, or
+//   - {"memtable_prefix_bloom_size_ratio", "0.1"} in
-//   - "hard_rate_limit=2.1" in GetColumnFamilyOptionsFromString.
+//   GetColumnFamilyOptionsFromMap, or
+//   - "memtable_prefix_bloom_size_ratio=0.1" in
+//   GetColumnFamilyOptionsFromString.
 // * Array / Vectors:
 //   An array is specified by a list of values, where ':' is used as
 //   the delimiter to separate each value.

java/benchmark/src/main/java/org/rocksdb/benchmark/DbBenchmark.java

@@ -1350,18 +1350,6 @@ public class DbBenchmark {
         return Integer.parseInt(value);
       }
     },
-    soft_rate_limit(0.0d,"") {
-      @Override public Object parseValue(String value) {
-        return Double.parseDouble(value);
-      }
-    },
-    hard_rate_limit(0.0d,"When not equal to 0 this make threads\n" +
-        "\tsleep at each stats reporting interval until the compaction\n" +
-        "\tscore for all levels is less than or equal to this value.") {
-      @Override public Object parseValue(String value) {
-        return Double.parseDouble(value);
-      }
-    },
     rate_limit_delay_max_milliseconds(1000,
         "When hard_rate_limit is set then this is the max time a put will\n" +
         "\tbe stalled.") {

java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java

@@ -87,9 +87,7 @@ public class MutableColumnFamilyOptions
 
   public enum CompactionOption implements MutableColumnFamilyOptionKey {
     disable_auto_compactions(ValueType.BOOLEAN),
-    @Deprecated soft_rate_limit(ValueType.DOUBLE),
     soft_pending_compaction_bytes_limit(ValueType.LONG),
-    @Deprecated hard_rate_limit(ValueType.DOUBLE),
     hard_pending_compaction_bytes_limit(ValueType.LONG),
     level0_file_num_compaction_trigger(ValueType.INT),
     level0_slowdown_writes_trigger(ValueType.INT),

options/options_settable_test.cc

@@ -441,8 +441,6 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
   // GetColumnFamilyOptionsFromString():
   options->rate_limit_delay_max_milliseconds = 33;
   options->compaction_options_universal = CompactionOptionsUniversal();
-  options->hard_rate_limit = 0;
-  options->soft_rate_limit = 0;
   options->num_levels = 42;  // Initialize options for MutableCF
   options->max_mem_compaction_level = 0;
   options->compaction_filter = nullptr;
@@ -487,7 +485,6 @@ TEST_F(OptionsSettableTest, ColumnFamilyOptionsAllFieldsSettable) {
       "level0_slowdown_writes_trigger=22;"
       "level0_file_num_compaction_trigger=14;"
       "compaction_filter=urxcqstuwnCompactionFilter;"
-      "soft_rate_limit=530.615385;"
       "soft_pending_compaction_bytes_limit=0;"
       "max_write_buffer_number_to_maintain=84;"
       "max_write_buffer_size_to_maintain=2147483648;"

options/options_test.cc

@@ -80,8 +80,6 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
       {"max_bytes_for_level_multiplier", "15.0"},
       {"max_bytes_for_level_multiplier_additional", "16:17:18"},
       {"max_compaction_bytes", "21"},
-      {"soft_rate_limit", "1.1"},
-      {"hard_rate_limit", "2.1"},
       {"hard_pending_compaction_bytes_limit", "211"},
       {"arena_block_size", "22"},
       {"disable_auto_compactions", "true"},

test_util/testutil.cc

@@ -398,8 +398,6 @@ void RandomInitCFOptions(ColumnFamilyOptions* cf_opt, DBOptions& db_options,
   cf_opt->enable_blob_garbage_collection = rnd->Uniform(2);
 
   // double options
-  cf_opt->hard_rate_limit = static_cast<double>(rnd->Uniform(10000)) / 13;
-  cf_opt->soft_rate_limit = static_cast<double>(rnd->Uniform(10000)) / 13;
   cf_opt->memtable_prefix_bloom_size_ratio =
       static_cast<double>(rnd->Uniform(10000)) / 20000.0;
   cf_opt->blob_garbage_collection_age_cutoff = rnd->Uniform(10000) / 10000.0;

tools/benchmark.sh

@@ -140,7 +140,6 @@ const_params="
   --pin_l0_filter_and_index_blocks_in_cache=1 \
   --benchmark_write_rate_limit=$(( 1024 * 1024 * $mb_written_per_sec )) \
   \
-  --hard_rate_limit=3 \
   --rate_limit_delay_max_milliseconds=1000000 \
   --write_buffer_size=$((128 * M)) \
   --target_file_size_base=$((128 * M)) \

tools/db_bench_tool.cc

@@ -1203,19 +1203,6 @@ DEFINE_int32(thread_status_per_interval, 0,
 DEFINE_int32(perf_level, ROCKSDB_NAMESPACE::PerfLevel::kDisable,
              "Level of perf collection");
 
-static bool ValidateRateLimit(const char* flagname, double value) {
-  const double EPSILON = 1e-10;
-  if ( value < -EPSILON ) {
-    fprintf(stderr, "Invalid value for --%s: %12.6f, must be >= 0.0\n",
-            flagname, value);
-    return false;
-  }
-  return true;
-}
-DEFINE_double(soft_rate_limit, 0.0, "DEPRECATED");
-
-DEFINE_double(hard_rate_limit, 0.0, "DEPRECATED");
-
 DEFINE_uint64(soft_pending_compaction_bytes_limit, 64ull * 1024 * 1024 * 1024,
               "Slowdown writes if pending compaction bytes exceed this number");
 
@@ -1534,12 +1521,6 @@ DEFINE_string(secondary_cache_uri, "",
 static class std::shared_ptr<ROCKSDB_NAMESPACE::SecondaryCache> secondary_cache;
 #endif  // ROCKSDB_LITE
 
-static const bool FLAGS_soft_rate_limit_dummy __attribute__((__unused__)) =
-    RegisterFlagValidator(&FLAGS_soft_rate_limit, &ValidateRateLimit);
-
-static const bool FLAGS_hard_rate_limit_dummy __attribute__((__unused__)) =
-    RegisterFlagValidator(&FLAGS_hard_rate_limit, &ValidateRateLimit);
-
 static const bool FLAGS_prefix_size_dummy __attribute__((__unused__)) =
     RegisterFlagValidator(&FLAGS_prefix_size, &ValidatePrefixSize);
 
@@ -4290,8 +4271,6 @@ class Benchmark {
         options.compression_per_level[i] = FLAGS_compression_type_e;
       }
     }
-    options.soft_rate_limit = FLAGS_soft_rate_limit;
-    options.hard_rate_limit = FLAGS_hard_rate_limit;
     options.soft_pending_compaction_bytes_limit =
         FLAGS_soft_pending_compaction_bytes_limit;
     options.hard_pending_compaction_bytes_limit =

tools/db_bench_tool_test.cc

@@ -252,7 +252,6 @@ const std::string options_file_content = R"OPTIONS_FILE(
   level0_slowdown_writes_trigger=50
   level0_file_num_compaction_trigger=10
   expanded_compaction_factor=25
-  soft_rate_limit=0.000000
   max_write_buffer_number_to_maintain=0
   max_write_buffer_size_to_maintain=0
   verify_checksums_in_compaction=true