NextGraph
/
rocksdb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Expose new configration variables via the thrift api

Browse Source 
Summary: Expose new configration variables via the thrift api. when compiing for fbcode, always build thrift server

Test Plan: none.

Reviewers: heyongqiang

Reviewed By: heyongqiang

Differential Revision: https://reviews.facebook.net/D4689
main
Dhruba Borthakur 13 years ago
parent 7c0b5ec54a
commit b56ff5ea38
7 changed files with 317 additions and 5 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 3
      fbcode.sh
  2. 2
      thrift/README
  3. BIN
      thrift/bin/thrift
  4. 209
      thrift/gen-cpp/leveldb_types.cpp
  5. 70
      thrift/gen-cpp/leveldb_types.h
  6. 15
      thrift/if/leveldb.thrift
  7. 23
      thrift/server_utils.cpp

3
fbcode.sh
Unescape View File

@ -8,6 +8,9 @@ TOOLCHAIN_REV=d28c90311ca14f9f0b2bb720f4e34b285513d4f4
TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native" TOOLCHAIN_EXECUTABLES="/mnt/gvfs/third-party/$TOOLCHAIN_REV/centos5.2-native"
TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.6.2-glibc-2.13" TOOLCHAIN_LIB_BASE="/mnt/gvfs/third-party/$TOOLCHAIN_REV/gcc-4.6.2-glibc-2.13"
# always build thrift server
export USE_THRIFT=1
# location of libhdfs libraries # location of libhdfs libraries
if test "$USE_HDFS"; then if test "$USE_HDFS"; then
JAVA_HOME="/usr/local/jdk-6u22-64" JAVA_HOME="/usr/local/jdk-6u22-64"

2
thrift/README
Unescape View File

@ -21,5 +21,5 @@ You can run the leveldb server unit tests by
You can regenerate the thrift cpp files by doing the following You can regenerate the thrift cpp files by doing the following
cd ./thrift cd ./thrift
thrift --gen cpp if/leveldb.thrift bin/thrift --gen cpp if/leveldb.thrift

BIN
thrift/bin/thrift
View File

Binary file not shown.

209
thrift/gen-cpp/leveldb_types.cpp
Unescape View File

@ -422,6 +422,76 @@ void reflectionInitializer_6731746507948871532(::apache::thrift::reflection::Sch
f.name = "compression"; f.name = "compression";
dt.fields[7] = f; dt.fields[7] = f;
} }
{
::apache::thrift::reflection::StructField f;
f.isRequired = true;
f.type = 5U;
f.name = "num_levels";
dt.fields[8] = f;
}
{
::apache::thrift::reflection::StructField f;
f.isRequired = true;
f.type = 5U;
f.name = "level0_file_num_compaction_trigger";
dt.fields[9] = f;
}
{
::apache::thrift::reflection::StructField f;
f.isRequired = true;
f.type = 5U;
f.name = "level0_slowdown_writes_trigger";
dt.fields[10] = f;
}
{
::apache::thrift::reflection::StructField f;
f.isRequired = true;
f.type = 5U;
f.name = "level0_stop_writes_trigger";
dt.fields[11] = f;
}
{
::apache::thrift::reflection::StructField f;
f.isRequired = true;
f.type = 5U;
f.name = "target_file_size_base";
dt.fields[12] = f;
}
{
::apache::thrift::reflection::StructField f;
f.isRequired = true;
f.type = 5U;
f.name = "target_file_size_multiplier";
dt.fields[13] = f;
}
{
::apache::thrift::reflection::StructField f;
f.isRequired = true;
f.type = 5U;
f.name = "max_bytes_for_level_base";
dt.fields[14] = f;
}
{
::apache::thrift::reflection::StructField f;
f.isRequired = true;
f.type = 5U;
f.name = "max_bytes_for_level_multiplier";
dt.fields[15] = f;
}
{
::apache::thrift::reflection::StructField f;
f.isRequired = true;
f.type = 5U;
f.name = "max_grandparent_overlap_factor";
dt.fields[16] = f;
}
{
::apache::thrift::reflection::StructField f;
f.isRequired = true;
f.type = 2U;
f.name = "disableDataSync";
dt.fields[17] = f;
}
schema.dataTypes[id] = dt; schema.dataTypes[id] = dt;
schema.names[dt.name] = id; schema.names[dt.name] = id;
} }
@ -509,6 +579,86 @@ uint32_t DBOptions::read(apache::thrift::protocol::TProtocol* iprot) {
xfer += iprot->skip(ftype); xfer += iprot->skip(ftype);
} }
break; break;
case 8:
if (ftype == apache::thrift::protocol::T_I32) {
xfer += iprot->readI32(this->num_levels);
this->__isset.num_levels = true;
} else {
xfer += iprot->skip(ftype);
}
break;
case 9:
if (ftype == apache::thrift::protocol::T_I32) {
xfer += iprot->readI32(this->level0_file_num_compaction_trigger);
this->__isset.level0_file_num_compaction_trigger = true;
} else {
xfer += iprot->skip(ftype);
}
break;
case 10:
if (ftype == apache::thrift::protocol::T_I32) {
xfer += iprot->readI32(this->level0_slowdown_writes_trigger);
this->__isset.level0_slowdown_writes_trigger = true;
} else {
xfer += iprot->skip(ftype);
}
break;
case 11:
if (ftype == apache::thrift::protocol::T_I32) {
xfer += iprot->readI32(this->level0_stop_writes_trigger);
this->__isset.level0_stop_writes_trigger = true;
} else {
xfer += iprot->skip(ftype);
}
break;
case 12:
if (ftype == apache::thrift::protocol::T_I32) {
xfer += iprot->readI32(this->target_file_size_base);
this->__isset.target_file_size_base = true;
} else {
xfer += iprot->skip(ftype);
}
break;
case 13:
if (ftype == apache::thrift::protocol::T_I32) {
xfer += iprot->readI32(this->target_file_size_multiplier);
this->__isset.target_file_size_multiplier = true;
} else {
xfer += iprot->skip(ftype);
}
break;
case 14:
if (ftype == apache::thrift::protocol::T_I32) {
xfer += iprot->readI32(this->max_bytes_for_level_base);
this->__isset.max_bytes_for_level_base = true;
} else {
xfer += iprot->skip(ftype);
}
break;
case 15:
if (ftype == apache::thrift::protocol::T_I32) {
xfer += iprot->readI32(this->max_bytes_for_level_multiplier);
this->__isset.max_bytes_for_level_multiplier = true;
} else {
xfer += iprot->skip(ftype);
}
break;
case 16:
if (ftype == apache::thrift::protocol::T_I32) {
xfer += iprot->readI32(this->max_grandparent_overlap_factor);
this->__isset.max_grandparent_overlap_factor = true;
} else {
xfer += iprot->skip(ftype);
}
break;
case 17:
if (ftype == apache::thrift::protocol::T_BOOL) {
xfer += iprot->readBool(this->disableDataSync);
this->__isset.disableDataSync = true;
} else {
xfer += iprot->skip(ftype);
}
break;
default: default:
xfer += iprot->skip(ftype); xfer += iprot->skip(ftype);
break; break;
@ -545,6 +695,36 @@ uint32_t DBOptions::write(apache::thrift::protocol::TProtocol* oprot) const {
xfer += oprot->writeFieldBegin("compression", apache::thrift::protocol::T_I32, 7); xfer += oprot->writeFieldBegin("compression", apache::thrift::protocol::T_I32, 7);
xfer += oprot->writeI32((int32_t)this->compression); xfer += oprot->writeI32((int32_t)this->compression);
xfer += oprot->writeFieldEnd(); xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("num_levels", apache::thrift::protocol::T_I32, 8);
xfer += oprot->writeI32(this->num_levels);
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("level0_file_num_compaction_trigger", apache::thrift::protocol::T_I32, 9);
xfer += oprot->writeI32(this->level0_file_num_compaction_trigger);
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("level0_slowdown_writes_trigger", apache::thrift::protocol::T_I32, 10);
xfer += oprot->writeI32(this->level0_slowdown_writes_trigger);
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("level0_stop_writes_trigger", apache::thrift::protocol::T_I32, 11);
xfer += oprot->writeI32(this->level0_stop_writes_trigger);
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("target_file_size_base", apache::thrift::protocol::T_I32, 12);
xfer += oprot->writeI32(this->target_file_size_base);
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("target_file_size_multiplier", apache::thrift::protocol::T_I32, 13);
xfer += oprot->writeI32(this->target_file_size_multiplier);
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("max_bytes_for_level_base", apache::thrift::protocol::T_I32, 14);
xfer += oprot->writeI32(this->max_bytes_for_level_base);
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("max_bytes_for_level_multiplier", apache::thrift::protocol::T_I32, 15);
xfer += oprot->writeI32(this->max_bytes_for_level_multiplier);
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("max_grandparent_overlap_factor", apache::thrift::protocol::T_I32, 16);
xfer += oprot->writeI32(this->max_grandparent_overlap_factor);
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("disableDataSync", apache::thrift::protocol::T_BOOL, 17);
xfer += oprot->writeBool(this->disableDataSync);
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldStop(); xfer += oprot->writeFieldStop();
xfer += oprot->writeStructEnd(); xfer += oprot->writeStructEnd();
return xfer; return xfer;
@ -561,6 +741,16 @@ void swap(DBOptions &a, DBOptions &b) {
swap(a.block_size, b.block_size); swap(a.block_size, b.block_size);
swap(a.block_restart_interval, b.block_restart_interval); swap(a.block_restart_interval, b.block_restart_interval);
swap(a.compression, b.compression); swap(a.compression, b.compression);
swap(a.num_levels, b.num_levels);
swap(a.level0_file_num_compaction_trigger, b.level0_file_num_compaction_trigger);
swap(a.level0_slowdown_writes_trigger, b.level0_slowdown_writes_trigger);
swap(a.level0_stop_writes_trigger, b.level0_stop_writes_trigger);
swap(a.target_file_size_base, b.target_file_size_base);
swap(a.target_file_size_multiplier, b.target_file_size_multiplier);
swap(a.max_bytes_for_level_base, b.max_bytes_for_level_base);
swap(a.max_bytes_for_level_multiplier, b.max_bytes_for_level_multiplier);
swap(a.max_grandparent_overlap_factor, b.max_grandparent_overlap_factor);
swap(a.disableDataSync, b.disableDataSync);
swap(a.__isset, b.__isset); swap(a.__isset, b.__isset);
} }
@ -579,6 +769,13 @@ void reflectionInitializer_8830325115029814540(::apache::thrift::reflection::Sch
f.name = "sync"; f.name = "sync";
dt.fields[1] = f; dt.fields[1] = f;
} }
{
::apache::thrift::reflection::StructField f;
f.isRequired = true;
f.type = 2U;
f.name = "disableWAL";
dt.fields[2] = f;
}
schema.dataTypes[id] = dt; schema.dataTypes[id] = dt;
schema.names[dt.name] = id; schema.names[dt.name] = id;
} }
@ -616,6 +813,14 @@ uint32_t WriteOptions::read(apache::thrift::protocol::TProtocol* iprot) {
xfer += iprot->skip(ftype); xfer += iprot->skip(ftype);
} }
break; break;
case 2:
if (ftype == apache::thrift::protocol::T_BOOL) {
xfer += iprot->readBool(this->disableWAL);
this->__isset.disableWAL = true;
} else {
xfer += iprot->skip(ftype);
}
break;
default: default:
xfer += iprot->skip(ftype); xfer += iprot->skip(ftype);
break; break;
@ -634,6 +839,9 @@ uint32_t WriteOptions::write(apache::thrift::protocol::TProtocol* oprot) const {
xfer += oprot->writeFieldBegin("sync", apache::thrift::protocol::T_BOOL, 1); xfer += oprot->writeFieldBegin("sync", apache::thrift::protocol::T_BOOL, 1);
xfer += oprot->writeBool(this->sync); xfer += oprot->writeBool(this->sync);
xfer += oprot->writeFieldEnd(); xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("disableWAL", apache::thrift::protocol::T_BOOL, 2);
xfer += oprot->writeBool(this->disableWAL);
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldStop(); xfer += oprot->writeFieldStop();
xfer += oprot->writeStructEnd(); xfer += oprot->writeStructEnd();
return xfer; return xfer;
@ -644,6 +852,7 @@ void swap(WriteOptions &a, WriteOptions &b) {
(void)a; (void)a;
(void)b; (void)b;
swap(a.sync, b.sync); swap(a.sync, b.sync);
swap(a.disableWAL, b.disableWAL);
swap(a.__isset, b.__isset); swap(a.__isset, b.__isset);
} }

70
thrift/gen-cpp/leveldb_types.h
Unescape View File

@ -238,7 +238,7 @@ class DBOptions {
static const uint64_t _reflection_id = 6731746507948871532U; static const uint64_t _reflection_id = 6731746507948871532U;
static void _reflection_register(::apache::thrift::reflection::Schema&); static void _reflection_register(::apache::thrift::reflection::Schema&);
DBOptions() : create_if_missing(0), error_if_exists(0), write_buffer_size(0), max_open_files(0), block_size(0), block_restart_interval(0), compression(static_cast<CompressionType>(0)) { DBOptions() : create_if_missing(0), error_if_exists(0), write_buffer_size(0), max_open_files(0), block_size(0), block_restart_interval(0), compression(static_cast<CompressionType>(0)), num_levels(0), level0_file_num_compaction_trigger(0), level0_slowdown_writes_trigger(0), level0_stop_writes_trigger(0), target_file_size_base(0), target_file_size_multiplier(0), max_bytes_for_level_base(0), max_bytes_for_level_multiplier(0), max_grandparent_overlap_factor(0), disableDataSync(0) {
} }
DBOptions(const DBOptions&) = default; DBOptions(const DBOptions&) = default;
@ -254,6 +254,16 @@ class DBOptions {
block_size = 0; block_size = 0;
block_restart_interval = 0; block_restart_interval = 0;
compression = static_cast<CompressionType>(0); compression = static_cast<CompressionType>(0);
num_levels = 0;
level0_file_num_compaction_trigger = 0;
level0_slowdown_writes_trigger = 0;
level0_stop_writes_trigger = 0;
target_file_size_base = 0;
target_file_size_multiplier = 0;
max_bytes_for_level_base = 0;
max_bytes_for_level_multiplier = 0;
max_grandparent_overlap_factor = 0;
disableDataSync = 0;
__isset.__clear(); __isset.__clear();
} }
@ -266,6 +276,16 @@ class DBOptions {
int32_t block_size; int32_t block_size;
int32_t block_restart_interval; int32_t block_restart_interval;
CompressionType compression; CompressionType compression;
int32_t num_levels;
int32_t level0_file_num_compaction_trigger;
int32_t level0_slowdown_writes_trigger;
int32_t level0_stop_writes_trigger;
int32_t target_file_size_base;
int32_t target_file_size_multiplier;
int32_t max_bytes_for_level_base;
int32_t max_bytes_for_level_multiplier;
int32_t max_grandparent_overlap_factor;
bool disableDataSync;
struct __isset { struct __isset {
__isset() { __clear(); } __isset() { __clear(); }
@ -277,6 +297,16 @@ class DBOptions {
block_size = false; block_size = false;
block_restart_interval = false; block_restart_interval = false;
compression = false; compression = false;
num_levels = false;
level0_file_num_compaction_trigger = false;
level0_slowdown_writes_trigger = false;
level0_stop_writes_trigger = false;
target_file_size_base = false;
target_file_size_multiplier = false;
max_bytes_for_level_base = false;
max_bytes_for_level_multiplier = false;
max_grandparent_overlap_factor = false;
disableDataSync = false;
} }
bool create_if_missing; bool create_if_missing;
bool error_if_exists; bool error_if_exists;
@ -285,6 +315,16 @@ class DBOptions {
bool block_size; bool block_size;
bool block_restart_interval; bool block_restart_interval;
bool compression; bool compression;
bool num_levels;
bool level0_file_num_compaction_trigger;
bool level0_slowdown_writes_trigger;
bool level0_stop_writes_trigger;
bool target_file_size_base;
bool target_file_size_multiplier;
bool max_bytes_for_level_base;
bool max_bytes_for_level_multiplier;
bool max_grandparent_overlap_factor;
bool disableDataSync;
} __isset; } __isset;
bool operator == (const DBOptions & rhs) const bool operator == (const DBOptions & rhs) const
@ -303,6 +343,26 @@ class DBOptions {
return false; return false;
if (!(this->compression == rhs.compression)) if (!(this->compression == rhs.compression))
return false; return false;
if (!(this->num_levels == rhs.num_levels))
return false;
if (!(this->level0_file_num_compaction_trigger == rhs.level0_file_num_compaction_trigger))
return false;
if (!(this->level0_slowdown_writes_trigger == rhs.level0_slowdown_writes_trigger))
return false;
if (!(this->level0_stop_writes_trigger == rhs.level0_stop_writes_trigger))
return false;
if (!(this->target_file_size_base == rhs.target_file_size_base))
return false;
if (!(this->target_file_size_multiplier == rhs.target_file_size_multiplier))
return false;
if (!(this->max_bytes_for_level_base == rhs.max_bytes_for_level_base))
return false;
if (!(this->max_bytes_for_level_multiplier == rhs.max_bytes_for_level_multiplier))
return false;
if (!(this->max_grandparent_overlap_factor == rhs.max_grandparent_overlap_factor))
return false;
if (!(this->disableDataSync == rhs.disableDataSync))
return false;
return true; return true;
} }
bool operator != (const DBOptions &rhs) const { bool operator != (const DBOptions &rhs) const {
@ -324,7 +384,7 @@ class WriteOptions {
static const uint64_t _reflection_id = 8830325115029814540U; static const uint64_t _reflection_id = 8830325115029814540U;
static void _reflection_register(::apache::thrift::reflection::Schema&); static void _reflection_register(::apache::thrift::reflection::Schema&);
WriteOptions() : sync(0) { WriteOptions() : sync(0), disableWAL(0) {
} }
WriteOptions(const WriteOptions&) = default; WriteOptions(const WriteOptions&) = default;
@ -334,25 +394,31 @@ class WriteOptions {
void __clear() { void __clear() {
sync = 0; sync = 0;
disableWAL = 0;
__isset.__clear(); __isset.__clear();
} }
virtual ~WriteOptions() throw() {} virtual ~WriteOptions() throw() {}
bool sync; bool sync;
bool disableWAL;
struct __isset { struct __isset {
__isset() { __clear(); } __isset() { __clear(); }
void __clear() { void __clear() {
sync = false; sync = false;
disableWAL = false;
} }
bool sync; bool sync;
bool disableWAL;
} __isset; } __isset;
bool operator == (const WriteOptions & rhs) const bool operator == (const WriteOptions & rhs) const
{ {
if (!(this->sync == rhs.sync)) if (!(this->sync == rhs.sync))
return false; return false;
if (!(this->disableWAL == rhs.disableWAL))
return false;
return true; return true;
} }
bool operator != (const WriteOptions &rhs) const { bool operator != (const WriteOptions &rhs) const {

15
thrift/if/leveldb.thrift
Unescape View File

@ -47,12 +47,23 @@ struct DBOptions {
4:i32 max_open_files; 4:i32 max_open_files;
5:i32 block_size; 5:i32 block_size;
6:i32 block_restart_interval; 6:i32 block_restart_interval;
7:CompressionType compression 7:CompressionType compression,
8:i32 num_levels,
9:i32 level0_file_num_compaction_trigger,
10:i32 level0_slowdown_writes_trigger,
11:i32 level0_stop_writes_trigger,
12:i32 target_file_size_base,
13:i32 target_file_size_multiplier,
14:i32 max_bytes_for_level_base,
15:i32 max_bytes_for_level_multiplier,
16:i32 max_grandparent_overlap_factor,
17:bool disableDataSync
} }
// Options for writing // Options for writing
struct WriteOptions { struct WriteOptions {
1:bool sync 1:bool sync,
2:bool disableWAL
} }
struct Snapshot { struct Snapshot {

23
thrift/server_utils.cpp
Unescape View File

@ -75,6 +75,26 @@ class DBHandler : virtual public DBIf {
} else if (dboptions.compression == kSnappyCompression) { } else if (dboptions.compression == kSnappyCompression) {
options.compression = leveldb::kSnappyCompression; options.compression = leveldb::kSnappyCompression;
} }
if (dboptions.num_levels > 0)
options.num_levels = dboptions.num_levels;
if (dboptions.level0_file_num_compaction_trigger > 0)
options.level0_file_num_compaction_trigger = dboptions.level0_file_num_compaction_trigger;
if (dboptions.level0_slowdown_writes_trigger > 0)
options.level0_slowdown_writes_trigger = dboptions.level0_slowdown_writes_trigger;
if (dboptions.level0_stop_writes_trigger)
options.level0_stop_writes_trigger = dboptions.level0_stop_writes_trigger;
if (dboptions.target_file_size_base > 0)
options.target_file_size_base = dboptions.target_file_size_base;
if (dboptions.target_file_size_multiplier > 0)
options.target_file_size_multiplier = dboptions.target_file_size_multiplier;
if (dboptions.max_bytes_for_level_base)
options.max_bytes_for_level_base = dboptions.max_bytes_for_level_base;
if (dboptions.max_bytes_for_level_multiplier)
options.max_bytes_for_level_multiplier = dboptions.max_bytes_for_level_multiplier;
if (dboptions.max_grandparent_overlap_factor)
options.max_grandparent_overlap_factor = dboptions.max_grandparent_overlap_factor;
if (dboptions.disableDataSync)
options.disableDataSync = dboptions.disableDataSync;
openHandles->add(options, dbname, dbdir); openHandles->add(options, dbname, dbdir);
_return.dbname = dbname; _return.dbname = dbname;
} }
@ -91,6 +111,7 @@ class DBHandler : virtual public DBIf {
const WriteOptions& options) { const WriteOptions& options) {
leveldb::WriteOptions woptions; leveldb::WriteOptions woptions;
woptions.sync = options.sync; woptions.sync = options.sync;
woptions.disableWAL = options.disableWAL;
leveldb::Slice key, value; leveldb::Slice key, value;
key.data_ = kv.key.data.data(); key.data_ = kv.key.data.data();
key.size_ = kv.key.size; key.size_ = kv.key.size;
@ -111,6 +132,7 @@ class DBHandler : virtual public DBIf {
const WriteOptions& options) { const WriteOptions& options) {
leveldb::WriteOptions woptions; leveldb::WriteOptions woptions;
woptions.sync = options.sync; woptions.sync = options.sync;
woptions.disableWAL = options.disableWAL;
leveldb::Slice key; leveldb::Slice key;
key.data_ = kv.data.data(); key.data_ = kv.data.data();
key.size_ = kv.size; key.size_ = kv.size;
@ -130,6 +152,7 @@ class DBHandler : virtual public DBIf {
leveldb::WriteOptions woptions; leveldb::WriteOptions woptions;
leveldb::WriteBatch lbatch; leveldb::WriteBatch lbatch;
woptions.sync = options.sync; woptions.sync = options.sync;
woptions.disableWAL = options.disableWAL;
leveldb::Slice key, value; leveldb::Slice key, value;
for (unsigned int i = 0; i < batch.size(); i++) { for (unsigned int i = 0; i < batch.size(); i++) {
kv one = batch[i]; kv one = batch[i];

Write Preview
Loading…
Cancel
Save