Expose new configration variables via the thrift api

Summary: Expose new configration variables via the thrift api. when compiing for fbcode, always build thrift server

Test Plan: none.

Reviewers: heyongqiang

Reviewed By: heyongqiang

Differential Revision: https://reviews.facebook.net/D4689

fbcode.sh

@@ -8,6 +8,9 @@ TOOLCHAIN_REV=d28c90311ca14f9f0b2bb720f4e34b285513d4f4
 TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native"
 TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.6.2-glibc-2.13"
 
+# always build thrift server
+export USE_THRIFT=1
+
 # location of libhdfs libraries
 if test "$USE_HDFS"; then
   JAVA_HOME="/usr/local/jdk-6u22-64"

thrift/README

@@ -21,5 +21,5 @@ You can run the leveldb server unit tests by
 You can regenerate the thrift cpp files by doing the following
 
 cd ./thrift
-thrift --gen cpp if/leveldb.thrift
+bin/thrift --gen cpp if/leveldb.thrift
 

thrift/gen-cpp/leveldb_types.cpp

@@ -422,6 +422,76 @@ void reflectionInitializer_6731746507948871532(::apache::thrift::reflection::Sch
     f.name = "compression";
     dt.fields[7] = f;
   }
+  {
+    ::apache::thrift::reflection::StructField f;
+    f.isRequired = true;
+    f.type = 5U;
+    f.name = "num_levels";
+    dt.fields[8] = f;
+  }
+  {
+    ::apache::thrift::reflection::StructField f;
+    f.isRequired = true;
+    f.type = 5U;
+    f.name = "level0_file_num_compaction_trigger";
+    dt.fields[9] = f;
+  }
+  {
+    ::apache::thrift::reflection::StructField f;
+    f.isRequired = true;
+    f.type = 5U;
+    f.name = "level0_slowdown_writes_trigger";
+    dt.fields[10] = f;
+  }
+  {
+    ::apache::thrift::reflection::StructField f;
+    f.isRequired = true;
+    f.type = 5U;
+    f.name = "level0_stop_writes_trigger";
+    dt.fields[11] = f;
+  }
+  {
+    ::apache::thrift::reflection::StructField f;
+    f.isRequired = true;
+    f.type = 5U;
+    f.name = "target_file_size_base";
+    dt.fields[12] = f;
+  }
+  {
+    ::apache::thrift::reflection::StructField f;
+    f.isRequired = true;
+    f.type = 5U;
+    f.name = "target_file_size_multiplier";
+    dt.fields[13] = f;
+  }
+  {
+    ::apache::thrift::reflection::StructField f;
+    f.isRequired = true;
+    f.type = 5U;
+    f.name = "max_bytes_for_level_base";
+    dt.fields[14] = f;
+  }
+  {
+    ::apache::thrift::reflection::StructField f;
+    f.isRequired = true;
+    f.type = 5U;
+    f.name = "max_bytes_for_level_multiplier";
+    dt.fields[15] = f;
+  }
+  {
+    ::apache::thrift::reflection::StructField f;
+    f.isRequired = true;
+    f.type = 5U;
+    f.name = "max_grandparent_overlap_factor";
+    dt.fields[16] = f;
+  }
+  {
+    ::apache::thrift::reflection::StructField f;
+    f.isRequired = true;
+    f.type = 2U;
+    f.name = "disableDataSync";
+    dt.fields[17] = f;
+  }
   schema.dataTypes[id] = dt;
   schema.names[dt.name] = id;
 }
@@ -509,6 +579,86 @@ uint32_t DBOptions::read(apache::thrift::protocol::TProtocol* iprot) {
           xfer += iprot->skip(ftype);
         }
         break;
+      case 8:
+        if (ftype == apache::thrift::protocol::T_I32) {
+          xfer += iprot->readI32(this->num_levels);
+          this->__isset.num_levels = true;
+        } else {
+          xfer += iprot->skip(ftype);
+        }
+        break;
+      case 9:
+        if (ftype == apache::thrift::protocol::T_I32) {
+          xfer += iprot->readI32(this->level0_file_num_compaction_trigger);
+          this->__isset.level0_file_num_compaction_trigger = true;
+        } else {
+          xfer += iprot->skip(ftype);
+        }
+        break;
+      case 10:
+        if (ftype == apache::thrift::protocol::T_I32) {
+          xfer += iprot->readI32(this->level0_slowdown_writes_trigger);
+          this->__isset.level0_slowdown_writes_trigger = true;
+        } else {
+          xfer += iprot->skip(ftype);
+        }
+        break;
+      case 11:
+        if (ftype == apache::thrift::protocol::T_I32) {
+          xfer += iprot->readI32(this->level0_stop_writes_trigger);
+          this->__isset.level0_stop_writes_trigger = true;
+        } else {
+          xfer += iprot->skip(ftype);
+        }
+        break;
+      case 12:
+        if (ftype == apache::thrift::protocol::T_I32) {
+          xfer += iprot->readI32(this->target_file_size_base);
+          this->__isset.target_file_size_base = true;
+        } else {
+          xfer += iprot->skip(ftype);
+        }
+        break;
+      case 13:
+        if (ftype == apache::thrift::protocol::T_I32) {
+          xfer += iprot->readI32(this->target_file_size_multiplier);
+          this->__isset.target_file_size_multiplier = true;
+        } else {
+          xfer += iprot->skip(ftype);
+        }
+        break;
+      case 14:
+        if (ftype == apache::thrift::protocol::T_I32) {
+          xfer += iprot->readI32(this->max_bytes_for_level_base);
+          this->__isset.max_bytes_for_level_base = true;
+        } else {
+          xfer += iprot->skip(ftype);
+        }
+        break;
+      case 15:
+        if (ftype == apache::thrift::protocol::T_I32) {
+          xfer += iprot->readI32(this->max_bytes_for_level_multiplier);
+          this->__isset.max_bytes_for_level_multiplier = true;
+        } else {
+          xfer += iprot->skip(ftype);
+        }
+        break;
+      case 16:
+        if (ftype == apache::thrift::protocol::T_I32) {
+          xfer += iprot->readI32(this->max_grandparent_overlap_factor);
+          this->__isset.max_grandparent_overlap_factor = true;
+        } else {
+          xfer += iprot->skip(ftype);
+        }
+        break;
+      case 17:
+        if (ftype == apache::thrift::protocol::T_BOOL) {
+          xfer += iprot->readBool(this->disableDataSync);
+          this->__isset.disableDataSync = true;
+        } else {
+          xfer += iprot->skip(ftype);
+        }
+        break;
       default:
         xfer += iprot->skip(ftype);
         break;
@@ -545,6 +695,36 @@ uint32_t DBOptions::write(apache::thrift::protocol::TProtocol* oprot) const {
   xfer += oprot->writeFieldBegin("compression", apache::thrift::protocol::T_I32, 7);
   xfer += oprot->writeI32((int32_t)this->compression);
   xfer += oprot->writeFieldEnd();
+  xfer += oprot->writeFieldBegin("num_levels", apache::thrift::protocol::T_I32, 8);
+  xfer += oprot->writeI32(this->num_levels);
+  xfer += oprot->writeFieldEnd();
+  xfer += oprot->writeFieldBegin("level0_file_num_compaction_trigger", apache::thrift::protocol::T_I32, 9);
+  xfer += oprot->writeI32(this->level0_file_num_compaction_trigger);
+  xfer += oprot->writeFieldEnd();
+  xfer += oprot->writeFieldBegin("level0_slowdown_writes_trigger", apache::thrift::protocol::T_I32, 10);
+  xfer += oprot->writeI32(this->level0_slowdown_writes_trigger);
+  xfer += oprot->writeFieldEnd();
+  xfer += oprot->writeFieldBegin("level0_stop_writes_trigger", apache::thrift::protocol::T_I32, 11);
+  xfer += oprot->writeI32(this->level0_stop_writes_trigger);
+  xfer += oprot->writeFieldEnd();
+  xfer += oprot->writeFieldBegin("target_file_size_base", apache::thrift::protocol::T_I32, 12);
+  xfer += oprot->writeI32(this->target_file_size_base);
+  xfer += oprot->writeFieldEnd();
+  xfer += oprot->writeFieldBegin("target_file_size_multiplier", apache::thrift::protocol::T_I32, 13);
+  xfer += oprot->writeI32(this->target_file_size_multiplier);
+  xfer += oprot->writeFieldEnd();
+  xfer += oprot->writeFieldBegin("max_bytes_for_level_base", apache::thrift::protocol::T_I32, 14);
+  xfer += oprot->writeI32(this->max_bytes_for_level_base);
+  xfer += oprot->writeFieldEnd();
+  xfer += oprot->writeFieldBegin("max_bytes_for_level_multiplier", apache::thrift::protocol::T_I32, 15);
+  xfer += oprot->writeI32(this->max_bytes_for_level_multiplier);
+  xfer += oprot->writeFieldEnd();
+  xfer += oprot->writeFieldBegin("max_grandparent_overlap_factor", apache::thrift::protocol::T_I32, 16);
+  xfer += oprot->writeI32(this->max_grandparent_overlap_factor);
+  xfer += oprot->writeFieldEnd();
+  xfer += oprot->writeFieldBegin("disableDataSync", apache::thrift::protocol::T_BOOL, 17);
+  xfer += oprot->writeBool(this->disableDataSync);
+  xfer += oprot->writeFieldEnd();
   xfer += oprot->writeFieldStop();
   xfer += oprot->writeStructEnd();
   return xfer;
@@ -561,6 +741,16 @@ void swap(DBOptions &a, DBOptions &b) {
   swap(a.block_size, b.block_size);
   swap(a.block_restart_interval, b.block_restart_interval);
   swap(a.compression, b.compression);
+  swap(a.num_levels, b.num_levels);
+  swap(a.level0_file_num_compaction_trigger, b.level0_file_num_compaction_trigger);
+  swap(a.level0_slowdown_writes_trigger, b.level0_slowdown_writes_trigger);
+  swap(a.level0_stop_writes_trigger, b.level0_stop_writes_trigger);
+  swap(a.target_file_size_base, b.target_file_size_base);
+  swap(a.target_file_size_multiplier, b.target_file_size_multiplier);
+  swap(a.max_bytes_for_level_base, b.max_bytes_for_level_base);
+  swap(a.max_bytes_for_level_multiplier, b.max_bytes_for_level_multiplier);
+  swap(a.max_grandparent_overlap_factor, b.max_grandparent_overlap_factor);
+  swap(a.disableDataSync, b.disableDataSync);
   swap(a.__isset, b.__isset);
 }
 
@@ -579,6 +769,13 @@ void reflectionInitializer_8830325115029814540(::apache::thrift::reflection::Sch
     f.name = "sync";
     dt.fields[1] = f;
   }
+  {
+    ::apache::thrift::reflection::StructField f;
+    f.isRequired = true;
+    f.type = 2U;
+    f.name = "disableWAL";
+    dt.fields[2] = f;
+  }
   schema.dataTypes[id] = dt;
   schema.names[dt.name] = id;
 }
@@ -616,6 +813,14 @@ uint32_t WriteOptions::read(apache::thrift::protocol::TProtocol* iprot) {
           xfer += iprot->skip(ftype);
         }
         break;
+      case 2:
+        if (ftype == apache::thrift::protocol::T_BOOL) {
+          xfer += iprot->readBool(this->disableWAL);
+          this->__isset.disableWAL = true;
+        } else {
+          xfer += iprot->skip(ftype);
+        }
+        break;
       default:
         xfer += iprot->skip(ftype);
         break;
@@ -634,6 +839,9 @@ uint32_t WriteOptions::write(apache::thrift::protocol::TProtocol* oprot) const {
   xfer += oprot->writeFieldBegin("sync", apache::thrift::protocol::T_BOOL, 1);
   xfer += oprot->writeBool(this->sync);
   xfer += oprot->writeFieldEnd();
+  xfer += oprot->writeFieldBegin("disableWAL", apache::thrift::protocol::T_BOOL, 2);
+  xfer += oprot->writeBool(this->disableWAL);
+  xfer += oprot->writeFieldEnd();
   xfer += oprot->writeFieldStop();
   xfer += oprot->writeStructEnd();
   return xfer;
@@ -644,6 +852,7 @@ void swap(WriteOptions &a, WriteOptions &b) {
   (void)a;
   (void)b;
   swap(a.sync, b.sync);
+  swap(a.disableWAL, b.disableWAL);
   swap(a.__isset, b.__isset);
 }
 

thrift/gen-cpp/leveldb_types.h

@@ -238,7 +238,7 @@ class DBOptions {
 
   static const uint64_t _reflection_id = 6731746507948871532U;
   static void _reflection_register(::apache::thrift::reflection::Schema&);
-  DBOptions() : create_if_missing(0), error_if_exists(0), write_buffer_size(0), max_open_files(0), block_size(0), block_restart_interval(0), compression(static_cast<CompressionType>(0)) {
+  DBOptions() : create_if_missing(0), error_if_exists(0), write_buffer_size(0), max_open_files(0), block_size(0), block_restart_interval(0), compression(static_cast<CompressionType>(0)), num_levels(0), level0_file_num_compaction_trigger(0), level0_slowdown_writes_trigger(0), level0_stop_writes_trigger(0), target_file_size_base(0), target_file_size_multiplier(0), max_bytes_for_level_base(0), max_bytes_for_level_multiplier(0), max_grandparent_overlap_factor(0), disableDataSync(0) {
   }
 
   DBOptions(const DBOptions&) = default;
@@ -254,6 +254,16 @@ class DBOptions {
     block_size = 0;
     block_restart_interval = 0;
     compression = static_cast<CompressionType>(0);
+    num_levels = 0;
+    level0_file_num_compaction_trigger = 0;
+    level0_slowdown_writes_trigger = 0;
+    level0_stop_writes_trigger = 0;
+    target_file_size_base = 0;
+    target_file_size_multiplier = 0;
+    max_bytes_for_level_base = 0;
+    max_bytes_for_level_multiplier = 0;
+    max_grandparent_overlap_factor = 0;
+    disableDataSync = 0;
     __isset.__clear();
   }
 
@@ -266,6 +276,16 @@ class DBOptions {
   int32_t block_size;
   int32_t block_restart_interval;
   CompressionType compression;
+  int32_t num_levels;
+  int32_t level0_file_num_compaction_trigger;
+  int32_t level0_slowdown_writes_trigger;
+  int32_t level0_stop_writes_trigger;
+  int32_t target_file_size_base;
+  int32_t target_file_size_multiplier;
+  int32_t max_bytes_for_level_base;
+  int32_t max_bytes_for_level_multiplier;
+  int32_t max_grandparent_overlap_factor;
+  bool disableDataSync;
 
   struct __isset {
     __isset() { __clear(); } 
@@ -277,6 +297,16 @@ class DBOptions {
       block_size = false;
       block_restart_interval = false;
       compression = false;
+      num_levels = false;
+      level0_file_num_compaction_trigger = false;
+      level0_slowdown_writes_trigger = false;
+      level0_stop_writes_trigger = false;
+      target_file_size_base = false;
+      target_file_size_multiplier = false;
+      max_bytes_for_level_base = false;
+      max_bytes_for_level_multiplier = false;
+      max_grandparent_overlap_factor = false;
+      disableDataSync = false;
     }
     bool create_if_missing;
     bool error_if_exists;
@@ -285,6 +315,16 @@ class DBOptions {
     bool block_size;
     bool block_restart_interval;
     bool compression;
+    bool num_levels;
+    bool level0_file_num_compaction_trigger;
+    bool level0_slowdown_writes_trigger;
+    bool level0_stop_writes_trigger;
+    bool target_file_size_base;
+    bool target_file_size_multiplier;
+    bool max_bytes_for_level_base;
+    bool max_bytes_for_level_multiplier;
+    bool max_grandparent_overlap_factor;
+    bool disableDataSync;
   } __isset;
 
   bool operator == (const DBOptions & rhs) const
@@ -303,6 +343,26 @@ class DBOptions {
       return false;
     if (!(this->compression == rhs.compression))
       return false;
+    if (!(this->num_levels == rhs.num_levels))
+      return false;
+    if (!(this->level0_file_num_compaction_trigger == rhs.level0_file_num_compaction_trigger))
+      return false;
+    if (!(this->level0_slowdown_writes_trigger == rhs.level0_slowdown_writes_trigger))
+      return false;
+    if (!(this->level0_stop_writes_trigger == rhs.level0_stop_writes_trigger))
+      return false;
+    if (!(this->target_file_size_base == rhs.target_file_size_base))
+      return false;
+    if (!(this->target_file_size_multiplier == rhs.target_file_size_multiplier))
+      return false;
+    if (!(this->max_bytes_for_level_base == rhs.max_bytes_for_level_base))
+      return false;
+    if (!(this->max_bytes_for_level_multiplier == rhs.max_bytes_for_level_multiplier))
+      return false;
+    if (!(this->max_grandparent_overlap_factor == rhs.max_grandparent_overlap_factor))
+      return false;
+    if (!(this->disableDataSync == rhs.disableDataSync))
+      return false;
     return true;
   }
   bool operator != (const DBOptions &rhs) const {
@@ -324,7 +384,7 @@ class WriteOptions {
 
   static const uint64_t _reflection_id = 8830325115029814540U;
   static void _reflection_register(::apache::thrift::reflection::Schema&);
-  WriteOptions() : sync(0) {
+  WriteOptions() : sync(0), disableWAL(0) {
   }
 
   WriteOptions(const WriteOptions&) = default;
@@ -334,25 +394,31 @@ class WriteOptions {
 
   void __clear() {
     sync = 0;
+    disableWAL = 0;
     __isset.__clear();
   }
 
   virtual ~WriteOptions() throw() {}
 
   bool sync;
+  bool disableWAL;
 
   struct __isset {
     __isset() { __clear(); } 
     void __clear() {
       sync = false;
+      disableWAL = false;
     }
     bool sync;
+    bool disableWAL;
   } __isset;
 
   bool operator == (const WriteOptions & rhs) const
   {
     if (!(this->sync == rhs.sync))
       return false;
+    if (!(this->disableWAL == rhs.disableWAL))
+      return false;
     return true;
   }
   bool operator != (const WriteOptions &rhs) const {

thrift/if/leveldb.thrift

@@ -47,12 +47,23 @@ struct DBOptions {
   4:i32 max_open_files;
   5:i32 block_size;
   6:i32 block_restart_interval;
-  7:CompressionType compression
+  7:CompressionType compression,
+  8:i32 num_levels,
+  9:i32 level0_file_num_compaction_trigger,
+  10:i32 level0_slowdown_writes_trigger,
+  11:i32 level0_stop_writes_trigger,
+  12:i32 target_file_size_base,
+  13:i32 target_file_size_multiplier,
+  14:i32 max_bytes_for_level_base,
+  15:i32 max_bytes_for_level_multiplier,
+  16:i32 max_grandparent_overlap_factor,
+  17:bool disableDataSync
 }
 
 // Options for writing
 struct WriteOptions {
-  1:bool sync
+  1:bool sync,
+  2:bool disableWAL
 }
 
 struct Snapshot {

thrift/server_utils.cpp

@@ -75,6 +75,26 @@ class DBHandler : virtual public DBIf {
     } else if (dboptions.compression == kSnappyCompression) {
       options.compression = leveldb::kSnappyCompression;
     }
+    if (dboptions.num_levels > 0)
+      options.num_levels = dboptions.num_levels;
+    if (dboptions.level0_file_num_compaction_trigger > 0) 
+      options.level0_file_num_compaction_trigger = dboptions.level0_file_num_compaction_trigger;
+    if (dboptions.level0_slowdown_writes_trigger > 0)
+      options.level0_slowdown_writes_trigger = dboptions.level0_slowdown_writes_trigger;
+    if (dboptions.level0_stop_writes_trigger) 
+      options.level0_stop_writes_trigger = dboptions.level0_stop_writes_trigger;
+    if (dboptions.target_file_size_base > 0) 
+      options.target_file_size_base = dboptions.target_file_size_base;
+    if (dboptions.target_file_size_multiplier > 0)
+      options.target_file_size_multiplier = dboptions.target_file_size_multiplier;
+    if (dboptions.max_bytes_for_level_base) 
+      options.max_bytes_for_level_base = dboptions.max_bytes_for_level_base;
+    if (dboptions.max_bytes_for_level_multiplier)
+      options.max_bytes_for_level_multiplier = dboptions.max_bytes_for_level_multiplier;
+    if (dboptions.max_grandparent_overlap_factor)
+      options.max_grandparent_overlap_factor = dboptions.max_grandparent_overlap_factor;
+    if (dboptions.disableDataSync) 
+      options.disableDataSync = dboptions.disableDataSync;
     openHandles->add(options, dbname, dbdir);
     _return.dbname = dbname;
   }
@@ -91,6 +111,7 @@ class DBHandler : virtual public DBIf {
     const WriteOptions& options) {
     leveldb::WriteOptions woptions;
     woptions.sync = options.sync;
+    woptions.disableWAL = options.disableWAL;
     leveldb::Slice key, value;
     key.data_ = kv.key.data.data();
     key.size_ = kv.key.size;
@@ -111,6 +132,7 @@ class DBHandler : virtual public DBIf {
     const WriteOptions& options) {
     leveldb::WriteOptions woptions;
     woptions.sync = options.sync;
+    woptions.disableWAL = options.disableWAL;
     leveldb::Slice key;
     key.data_ = kv.data.data();
     key.size_ = kv.size;
@@ -130,6 +152,7 @@ class DBHandler : virtual public DBIf {
     leveldb::WriteOptions woptions;
     leveldb::WriteBatch lbatch;
     woptions.sync = options.sync;
+    woptions.disableWAL = options.disableWAL;
     leveldb::Slice key, value;
     for (unsigned int i = 0; i < batch.size(); i++) {
       kv one = batch[i];