Change max_bytes_for_level_multiplier to double

Summary: Closes https://github.com/facebook/rocksdb/pull/1427

Differential Revision: D4094732

Pulled By: yiwu-arbug

fbshipit-source-id: b9b79e9

HISTORY.md

@@ -1,4 +1,8 @@
 # Rocksdb Change Log
+## Unreleased
+### Public API Change
+* Options::max_bytes_for_level_multiplier is now a double along with all getters and setters.
+
 ## 4.13.0 (10/18/2016)
 ### Public API Change
 * DB::GetOptions() reflect dynamic changed options (i.e. through DB::SetOptions()) and return copy of options instead of reference.

db/c.cc

@@ -1517,8 +1517,8 @@ void rocksdb_options_set_max_bytes_for_level_base(
   opt->rep.max_bytes_for_level_base = n;
 }
 
-void rocksdb_options_set_max_bytes_for_level_multiplier(
+void rocksdb_options_set_max_bytes_for_level_multiplier(rocksdb_options_t* opt,
-    rocksdb_options_t* opt, int n) {
+                                                        double n) {
   opt->rep.max_bytes_for_level_multiplier = n;
 }
 

db/compaction_picker.cc

@@ -1184,7 +1184,8 @@ uint32_t LevelCompactionPicker::GetPathId(
         return p;
       } else {
         current_path_size -= level_size;
-        level_size *= mutable_cf_options.max_bytes_for_level_multiplier;
+        level_size = static_cast<uint64_t>(
+            level_size * mutable_cf_options.max_bytes_for_level_multiplier);
         cur_level++;
         continue;
       }

db/db_compaction_test.cc

@@ -1259,7 +1259,8 @@ TEST_F(DBCompactionTest, ManualPartialFill) {
   uint64_t target_size = 4 * options.max_bytes_for_level_base;
   for (int32_t i = 1; i < options.num_levels; i++) {
     ASSERT_LE(SizeAtLevel(i), target_size);
-    target_size *= options.max_bytes_for_level_multiplier;
+    target_size = static_cast<uint64_t>(target_size *
+                                        options.max_bytes_for_level_multiplier);
   }
 
   TEST_SYNC_POINT("DBCompaction::PartialFill:2");
@@ -1336,7 +1337,8 @@ TEST_F(DBCompactionTest, DeleteFileRange) {
   uint64_t target_size = 4 * options.max_bytes_for_level_base;
   for (int32_t i = 1; i < options.num_levels; i++) {
     ASSERT_LE(SizeAtLevel(i), target_size);
-    target_size *= options.max_bytes_for_level_multiplier;
+    target_size = static_cast<uint64_t>(target_size *
+                                        options.max_bytes_for_level_multiplier);
   }
 
   size_t old_num_files = CountFiles();

db/version_set.cc

@@ -1980,14 +1980,15 @@ void VersionStorageInfo::CalculateBaseBytes(const ImmutableCFOptions& ioptions,
       base_level_ = num_levels_ - 1;
     } else {
       uint64_t base_bytes_max = options.max_bytes_for_level_base;
-      uint64_t base_bytes_min =
+      uint64_t base_bytes_min = static_cast<uint64_t>(
-          base_bytes_max / options.max_bytes_for_level_multiplier;
+          base_bytes_max / options.max_bytes_for_level_multiplier);
 
       // Try whether we can make last level's target size to be max_level_size
       uint64_t cur_level_size = max_level_size;
       for (int i = num_levels_ - 2; i >= first_non_empty_level; i--) {
         // Round up after dividing
-        cur_level_size /= options.max_bytes_for_level_multiplier;
+        cur_level_size = static_cast<uint64_t>(
+            cur_level_size / options.max_bytes_for_level_multiplier);
       }
 
       // Calculate base level and its size.
@@ -2006,8 +2007,8 @@ void VersionStorageInfo::CalculateBaseBytes(const ImmutableCFOptions& ioptions,
         base_level_ = first_non_empty_level;
         while (base_level_ > 1 && cur_level_size > base_bytes_max) {
           --base_level_;
-          cur_level_size =
+          cur_level_size = static_cast<uint64_t>(
-              cur_level_size / options.max_bytes_for_level_multiplier;
+              cur_level_size / options.max_bytes_for_level_multiplier);
         }
         if (cur_level_size > base_bytes_max) {
           // Even L1 will be too large

include/rocksdb/c.h

@@ -561,7 +561,7 @@ extern ROCKSDB_LIBRARY_API void rocksdb_options_set_target_file_size_multiplier(
 extern ROCKSDB_LIBRARY_API void rocksdb_options_set_max_bytes_for_level_base(
     rocksdb_options_t*, uint64_t);
 extern ROCKSDB_LIBRARY_API void
-rocksdb_options_set_max_bytes_for_level_multiplier(rocksdb_options_t*, int);
+rocksdb_options_set_max_bytes_for_level_multiplier(rocksdb_options_t*, double);
 extern ROCKSDB_LIBRARY_API void
 rocksdb_options_set_max_bytes_for_level_multiplier_additional(
     rocksdb_options_t*, int* level_values, size_t num_levels);

include/rocksdb/options.h

@@ -527,7 +527,7 @@ struct ColumnFamilyOptions {
   // Default: 10.
   //
   // Dynamically changeable through SetOptions() API
-  int max_bytes_for_level_multiplier;
+  double max_bytes_for_level_multiplier;
 
   // Different max-size multipliers for different levels.
   // These are multiplied by max_bytes_for_level_multiplier to arrive

java/benchmark/src/main/java/org/rocksdb/benchmark/DbBenchmark.java

@@ -574,12 +574,10 @@ public class DbBenchmark {
         (Integer)flags_.get(Flag.num_levels));
     options.setTargetFileSizeBase(
         (Integer)flags_.get(Flag.target_file_size_base));
-    options.setTargetFileSizeMultiplier(
+    options.setTargetFileSizeMultiplier((Double) flags_.get(Flag.target_file_size_multiplier));
-        (Integer)flags_.get(Flag.target_file_size_multiplier));
     options.setMaxBytesForLevelBase(
         (Integer)flags_.get(Flag.max_bytes_for_level_base));
-    options.setMaxBytesForLevelMultiplier(
+    options.setMaxBytesForLevelMultiplier((Double) flags_.get(Flag.max_bytes_for_level_multiplier));
-        (Integer)flags_.get(Flag.max_bytes_for_level_multiplier));
     options.setLevelZeroStopWritesTrigger(
         (Integer)flags_.get(Flag.level0_stop_writes_trigger));
     options.setLevelZeroSlowdownWritesTrigger(
@@ -1268,7 +1266,7 @@ public class DbBenchmark {
     max_bytes_for_level_multiplier(10,
         "A multiplier to compute max bytes for level-N (N >= 2)") {
       @Override public Object parseValue(String value) {
-        return Integer.parseInt(value);
+        return Double.parseDouble(value);
       }
     },
     level0_stop_writes_trigger(12,"Number of files in level-0\n" +

java/rocksjni/options.cc

@@ -1522,10 +1522,11 @@ void Java_org_rocksdb_Options_setLevelCompactionDynamicLevelBytes(
 /*
  * Class:     org_rocksdb_Options
  * Method:    maxBytesForLevelMultiplier
- * Signature: (J)I
+ * Signature: (J)D
  */
-jint Java_org_rocksdb_Options_maxBytesForLevelMultiplier(
+jdouble Java_org_rocksdb_Options_maxBytesForLevelMultiplier(JNIEnv* env,
-    JNIEnv* env, jobject jobj, jlong jhandle) {
+                                                            jobject jobj,
+                                                            jlong jhandle) {
   return reinterpret_cast<rocksdb::Options*>(
       jhandle)->max_bytes_for_level_multiplier;
 }
@@ -1533,14 +1534,13 @@ jint Java_org_rocksdb_Options_maxBytesForLevelMultiplier(
 /*
  * Class:     org_rocksdb_Options
  * Method:    setMaxBytesForLevelMultiplier
- * Signature: (JI)V
+ * Signature: (JD)V
  */
 void Java_org_rocksdb_Options_setMaxBytesForLevelMultiplier(
     JNIEnv* env, jobject jobj, jlong jhandle,
-    jint jmax_bytes_for_level_multiplier) {
+    jdouble jmax_bytes_for_level_multiplier) {
-  reinterpret_cast<rocksdb::Options*>(
+  reinterpret_cast<rocksdb::Options*>(jhandle)->max_bytes_for_level_multiplier =
-      jhandle)->max_bytes_for_level_multiplier =
+      static_cast<double>(jmax_bytes_for_level_multiplier);
-          static_cast<int>(jmax_bytes_for_level_multiplier);
 }
 
 /*
@@ -2823,9 +2823,9 @@ void Java_org_rocksdb_ColumnFamilyOptions_setLevelCompactionDynamicLevelBytes(
 /*
  * Class:     org_rocksdb_ColumnFamilyOptions
  * Method:    maxBytesForLevelMultiplier
- * Signature: (J)I
+ * Signature: (J)D
  */
-jint Java_org_rocksdb_ColumnFamilyOptions_maxBytesForLevelMultiplier(
+jdouble Java_org_rocksdb_ColumnFamilyOptions_maxBytesForLevelMultiplier(
     JNIEnv* env, jobject jobj, jlong jhandle) {
   return reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
       jhandle)->max_bytes_for_level_multiplier;
@@ -2834,14 +2834,14 @@ jint Java_org_rocksdb_ColumnFamilyOptions_maxBytesForLevelMultiplier(
 /*
  * Class:     org_rocksdb_ColumnFamilyOptions
  * Method:    setMaxBytesForLevelMultiplier
- * Signature: (JI)V
+ * Signature: (JD)V
  */
 void Java_org_rocksdb_ColumnFamilyOptions_setMaxBytesForLevelMultiplier(
     JNIEnv* env, jobject jobj, jlong jhandle,
-    jint jmax_bytes_for_level_multiplier) {
+    jdouble jmax_bytes_for_level_multiplier) {
-  reinterpret_cast<rocksdb::ColumnFamilyOptions*>(
+  reinterpret_cast<rocksdb::ColumnFamilyOptions*>(jhandle)
-      jhandle)->max_bytes_for_level_multiplier =
+      ->max_bytes_for_level_multiplier =
-          static_cast<int>(jmax_bytes_for_level_multiplier);
+      static_cast<double>(jmax_bytes_for_level_multiplier);
 }
 
 /*

java/src/main/java/org/rocksdb/ColumnFamilyOptions.java

@@ -352,14 +352,13 @@ public class ColumnFamilyOptions extends RocksObject
   }
 
   @Override
-  public ColumnFamilyOptions setMaxBytesForLevelMultiplier(
+  public ColumnFamilyOptions setMaxBytesForLevelMultiplier(final double multiplier) {
-      final int multiplier) {
     setMaxBytesForLevelMultiplier(nativeHandle_, multiplier);
     return this;
   }
 
   @Override
-  public int maxBytesForLevelMultiplier() {
+  public double maxBytesForLevelMultiplier() {
     return maxBytesForLevelMultiplier(nativeHandle_);
   }
 
@@ -775,9 +774,8 @@ public class ColumnFamilyOptions extends RocksObject
       long handle, boolean enableLevelCompactionDynamicLevelBytes);
   private native boolean levelCompactionDynamicLevelBytes(
       long handle);
-  private native void setMaxBytesForLevelMultiplier(
+  private native void setMaxBytesForLevelMultiplier(long handle, double multiplier);
-      long handle, int multiplier);
+  private native double maxBytesForLevelMultiplier(long handle);
-  private native int maxBytesForLevelMultiplier(long handle);
   private native void setMaxCompactionBytes(long handle, long maxCompactionBytes);
   private native long maxCompactionBytes(long handle);
   private native void setSoftRateLimit(

java/src/main/java/org/rocksdb/ColumnFamilyOptionsInterface.java

@@ -456,7 +456,7 @@ public interface ColumnFamilyOptionsInterface {
    *     files and the total size of level-L files for all L.
    * @return the reference to the current option.
    */
-  Object setMaxBytesForLevelMultiplier(int multiplier);
+  Object setMaxBytesForLevelMultiplier(double multiplier);
 
   /**
    * The ratio between the total size of level-(L+1) files and the total
@@ -466,7 +466,7 @@ public interface ColumnFamilyOptionsInterface {
    * @return the ratio between the total size of level-(L+1) files and
    *     the total size of level-L files for all L.
    */
-  int maxBytesForLevelMultiplier();
+  double maxBytesForLevelMultiplier();
 
   /**
    * Maximum size of each compaction (not guarantee)

java/src/main/java/org/rocksdb/MutableColumnFamilyOptions.java

@@ -843,14 +843,13 @@ public class MutableColumnFamilyOptions {
 
     @Override
     public MutableColumnFamilyOptionsBuilder setMaxBytesForLevelMultiplier(
-        final int maxBytesForLevelMultiplier) {
+        final double maxBytesForLevelMultiplier) {
-      return setInt(CompactionOption.max_bytes_for_level_multiplier,
+      return setDouble(CompactionOption.max_bytes_for_level_multiplier, maxBytesForLevelMultiplier);
-          maxBytesForLevelMultiplier);
     }
 
     @Override
-    public int maxBytesForLevelMultiplier() {
+    public double maxBytesForLevelMultiplier() {
-      return getInt(CompactionOption.max_bytes_for_level_multiplier);
+      return getDouble(CompactionOption.max_bytes_for_level_multiplier);
     }
 
     @Override

java/src/main/java/org/rocksdb/MutableColumnFamilyOptionsInterface.java

@@ -474,7 +474,7 @@ public interface MutableColumnFamilyOptionsInterface {
    * @param maxBytesForLevelBase maximum bytes for level base.
    *
    * @return the reference to the current option.
-   * @see #setMaxBytesForLevelMultiplier(int)
+   * @see #setMaxBytesForLevelMultiplier(double)
    */
   MutableColumnFamilyOptionsInterface setMaxBytesForLevelBase(
       long maxBytesForLevelBase);
@@ -505,8 +505,7 @@ public interface MutableColumnFamilyOptionsInterface {
    * @return the reference to the current option.
    * @see #setMaxBytesForLevelBase(long)
    */
-  MutableColumnFamilyOptionsInterface setMaxBytesForLevelMultiplier(
+  MutableColumnFamilyOptionsInterface setMaxBytesForLevelMultiplier(double multiplier);
-      int multiplier);
 
   /**
    * The ratio between the total size of level-(L+1) files and the total
@@ -517,7 +516,7 @@ public interface MutableColumnFamilyOptionsInterface {
    *     the total size of level-L files for all L.
    * @see #maxBytesForLevelBase()
    */
-  int maxBytesForLevelMultiplier();
+  double maxBytesForLevelMultiplier();
 
   /**
    * Different max-size multipliers for different levels.

java/src/main/java/org/rocksdb/Options.java

@@ -904,12 +904,12 @@ public class Options extends RocksObject
   }
 
   @Override
-  public int maxBytesForLevelMultiplier() {
+  public double maxBytesForLevelMultiplier() {
     return maxBytesForLevelMultiplier(nativeHandle_);
   }
 
   @Override
-  public Options setMaxBytesForLevelMultiplier(final int multiplier) {
+  public Options setMaxBytesForLevelMultiplier(final double multiplier) {
     setMaxBytesForLevelMultiplier(nativeHandle_, multiplier);
     return this;
   }
@@ -1382,9 +1382,8 @@ public class Options extends RocksObject
       long handle, boolean enableLevelCompactionDynamicLevelBytes);
   private native boolean levelCompactionDynamicLevelBytes(
       long handle);
-  private native void setMaxBytesForLevelMultiplier(
+  private native void setMaxBytesForLevelMultiplier(long handle, double multiplier);
-      long handle, int multiplier);
+  private native double maxBytesForLevelMultiplier(long handle);
-  private native int maxBytesForLevelMultiplier(long handle);
   private native void setMaxCompactionBytes(long handle, long maxCompactionBytes);
   private native long maxCompactionBytes(long handle);
   private native void setSoftRateLimit(

java/src/test/java/org/rocksdb/ColumnFamilyOptionsTest.java

@@ -172,9 +172,9 @@ public class ColumnFamilyOptionsTest {
   @Test
   public void maxBytesForLevelMultiplier() {
     try (final ColumnFamilyOptions opt = new ColumnFamilyOptions()) {
-      final int intValue = rand.nextInt();
+      final double doubleValue = rand.nextDouble();
-      opt.setMaxBytesForLevelMultiplier(intValue);
+      opt.setMaxBytesForLevelMultiplier(doubleValue);
-      assertThat(opt.maxBytesForLevelMultiplier()).isEqualTo(intValue);
+      assertThat(opt.maxBytesForLevelMultiplier()).isEqualTo(doubleValue);
     }
   }
 

java/src/test/java/org/rocksdb/OptionsTest.java

@@ -135,9 +135,9 @@ public class OptionsTest {
   @Test
   public void maxBytesForLevelMultiplier() {
     try (final Options opt = new Options()) {
-      final int intValue = rand.nextInt();
+      final double doubleValue = rand.nextDouble();
-      opt.setMaxBytesForLevelMultiplier(intValue);
+      opt.setMaxBytesForLevelMultiplier(doubleValue);
-      assertThat(opt.maxBytesForLevelMultiplier()).isEqualTo(intValue);
+      assertThat(opt.maxBytesForLevelMultiplier()).isEqualTo(doubleValue);
     }
   }
 

tools/db_bench_tool.cc

@@ -482,8 +482,8 @@ DEFINE_uint64(max_bytes_for_level_base,  10 * 1048576, "Max bytes for level-1");
 DEFINE_bool(level_compaction_dynamic_level_bytes, false,
             "Whether level size base is dynamic");
 
-DEFINE_int32(max_bytes_for_level_multiplier, 10,
+DEFINE_double(max_bytes_for_level_multiplier, 10,
-             "A multiplier to compute max bytes for level-N (N >= 2)");
+              "A multiplier to compute max bytes for level-N (N >= 2)");
 
 static std::vector<int> FLAGS_max_bytes_for_level_multiplier_additional_v;
 DEFINE_string(max_bytes_for_level_multiplier_additional, "",

tools/db_stress.cc

@@ -303,8 +303,8 @@ DEFINE_int32(target_file_size_multiplier, 1,
 
 DEFINE_uint64(max_bytes_for_level_base, 256 * KB, "Max bytes for level-1");
 
-DEFINE_int32(max_bytes_for_level_multiplier, 2,
+DEFINE_double(max_bytes_for_level_multiplier, 2,
-             "A multiplier to compute max bytes for level-N (N >= 2)");
+              "A multiplier to compute max bytes for level-N (N >= 2)");
 
 // Temporarily disable this to allows it to detect new bugs
 DEFINE_int32(compact_files_one_in, 0,

util/cf_options.cc

@@ -75,18 +75,14 @@ ImmutableCFOptions::ImmutableCFOptions(const ImmutableDBOptions& db_options,
       max_subcompactions(db_options.max_subcompactions) {}
 
 // Multiple two operands. If they overflow, return op1.
-uint64_t MultiplyCheckOverflow(uint64_t op1, int op2) {
+uint64_t MultiplyCheckOverflow(uint64_t op1, double op2) {
-  if (op1 == 0) {
+  if (op1 == 0 || op2 <= 0) {
     return 0;
   }
-  if (op2 <= 0) {
+  if (port::kMaxUint64 / op1 < op2) {
     return op1;
   }
-  uint64_t casted_op2 = (uint64_t) op2;
+  return static_cast<uint64_t>(op1 * op2);
-  if (std::numeric_limits<uint64_t>::max() / op1 < casted_op2) {
-    return op1;
-  }
-  return op1 * casted_op2;
 }
 
 void MutableCFOptions::RefreshDerivedOptions(int num_levels,
@@ -146,7 +142,7 @@ void MutableCFOptions::Dump(Logger* log) const {
       target_file_size_multiplier);
   Log(log, "                 max_bytes_for_level_base: %" PRIu64,
       max_bytes_for_level_base);
-  Log(log, "           max_bytes_for_level_multiplier: %d",
+  Log(log, "           max_bytes_for_level_multiplier: %f",
       max_bytes_for_level_multiplier);
   std::string result;
   char buf[10];

util/cf_options.h

@@ -220,7 +220,7 @@ struct MutableCFOptions {
   uint64_t target_file_size_base;
   int target_file_size_multiplier;
   uint64_t max_bytes_for_level_base;
-  int max_bytes_for_level_multiplier;
+  double max_bytes_for_level_multiplier;
   std::vector<int> max_bytes_for_level_multiplier_additional;
   bool verify_checksums_in_compaction;
 
@@ -236,6 +236,6 @@ struct MutableCFOptions {
   std::vector<uint64_t> max_file_size;
 };
 
-uint64_t MultiplyCheckOverflow(uint64_t op1, int op2);
+uint64_t MultiplyCheckOverflow(uint64_t op1, double op2);
 
 }  // namespace rocksdb

util/options.cc

@@ -489,8 +489,8 @@ void ColumnFamilyOptions::Dump(Logger* log) const {
         max_bytes_for_level_base);
     Header(log, "Options.level_compaction_dynamic_level_bytes: %d",
         level_compaction_dynamic_level_bytes);
-    Header(log, "         Options.max_bytes_for_level_multiplier: %d",
+    Header(log, "         Options.max_bytes_for_level_multiplier: %f",
-        max_bytes_for_level_multiplier);
+           max_bytes_for_level_multiplier);
     for (size_t i = 0; i < max_bytes_for_level_multiplier_additional.size();
          i++) {
       Header(log,

util/options_helper.h

@@ -490,7 +490,7 @@ static std::unordered_map<std::string, OptionTypeInfo> cf_options_type_info = {
       offsetof(struct MutableCFOptions, max_bytes_for_level_base)}},
     {"max_bytes_for_level_multiplier",
      {offsetof(struct ColumnFamilyOptions, max_bytes_for_level_multiplier),
-      OptionType::kInt, OptionVerificationType::kNormal, true,
+      OptionType::kDouble, OptionVerificationType::kNormal, true,
       offsetof(struct MutableCFOptions, max_bytes_for_level_multiplier)}},
     {"max_bytes_for_level_multiplier_additional",
      {offsetof(struct ColumnFamilyOptions,

util/options_test.cc

@@ -68,7 +68,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
       {"target_file_size_multiplier", "13"},
       {"max_bytes_for_level_base", "14"},
       {"level_compaction_dynamic_level_bytes", "true"},
-      {"max_bytes_for_level_multiplier", "15"},
+      {"max_bytes_for_level_multiplier", "15.0"},
       {"max_bytes_for_level_multiplier_additional", "16:17:18"},
       {"max_compaction_bytes", "21"},
       {"soft_rate_limit", "1.1"},
@@ -164,7 +164,7 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
   ASSERT_EQ(new_cf_opt.target_file_size_multiplier, 13);
   ASSERT_EQ(new_cf_opt.max_bytes_for_level_base, 14U);
   ASSERT_EQ(new_cf_opt.level_compaction_dynamic_level_bytes, true);
-  ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier, 15);
+  ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier, 15.0);
   ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional.size(), 3U);
   ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[0], 16);
   ASSERT_EQ(new_cf_opt.max_bytes_for_level_multiplier_additional[1], 17);