Tyler Neely
10 years ago
parent
7a43bc64fd
commit
534266f09a
@ -0,0 +1,202 @@ |
||||
/* |
||||
Copyright 2014 Tyler Neely |
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); |
||||
you may not use this file except in compliance with the License. |
||||
You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
*/ |
||||
#![allow(unstable)] |
||||
|
||||
extern crate libc; |
||||
use self::libc::{c_char, c_int, c_void, size_t}; |
||||
use std::ffi::CString; |
||||
use std::mem; |
||||
use std::ptr; |
||||
use std::slice; |
||||
|
||||
use rocksdb_options::{RocksDBOptions}; |
||||
use rocksdb::{RocksDB, RocksDBResult, RocksDBVector}; |
||||
|
||||
pub struct MergeOperatorCallback { |
||||
pub name: CString, |
||||
pub merge_fn: fn (&[u8], Option<&[u8]>, &mut MergeOperands) -> Vec<u8>, |
||||
} |
||||
|
||||
pub extern "C" fn destructor_callback(raw_cb: *mut c_void) { |
||||
// turn this back into a local variable so rust will reclaim it
|
||||
let _: Box<MergeOperatorCallback> = unsafe {mem::transmute(raw_cb)}; |
||||
|
||||
} |
||||
|
||||
pub extern "C" fn name_callback(raw_cb: *mut c_void) -> *const c_char { |
||||
unsafe { |
||||
let cb: &mut MergeOperatorCallback = |
||||
&mut *(raw_cb as *mut MergeOperatorCallback); |
||||
let ptr = cb.name.as_ptr(); |
||||
ptr as *const c_char |
||||
} |
||||
} |
||||
|
||||
pub extern "C" fn full_merge_callback( |
||||
raw_cb: *mut c_void, raw_key: *const c_char, key_len: size_t, |
||||
existing_value: *const c_char, existing_value_len: size_t, |
||||
operands_list: *const *const c_char, operands_list_len: *const size_t, |
||||
num_operands: c_int, |
||||
success: *mut u8, new_value_length: *mut size_t) -> *const c_char { |
||||
unsafe { |
||||
let cb: &mut MergeOperatorCallback = |
||||
&mut *(raw_cb as *mut MergeOperatorCallback); |
||||
let operands = |
||||
&mut MergeOperands::new(operands_list, |
||||
operands_list_len, |
||||
num_operands); |
||||
let key: &[u8] = mem::transmute(slice::from_raw_buf(&raw_key, key_len as usize)); |
||||
let oldval: &[u8] = mem::transmute(slice::from_raw_buf(&existing_value, |
||||
existing_value_len as usize)); |
||||
let mut result = |
||||
(cb.merge_fn)(key, Some(oldval), operands); |
||||
result.shrink_to_fit(); |
||||
//TODO(tan) investigate zero-copy techniques to improve performance
|
||||
let buf = libc::malloc(result.len() as size_t); |
||||
assert!(!buf.is_null()); |
||||
*new_value_length = result.len() as size_t; |
||||
*success = 1 as u8; |
||||
ptr::copy_memory(&mut *buf, result.as_ptr() |
||||
as *const c_void, result.len()); |
||||
buf as *const c_char |
||||
} |
||||
} |
||||
|
||||
pub extern "C" fn partial_merge_callback( |
||||
raw_cb: *mut c_void, raw_key: *const c_char, key_len: size_t, |
||||
operands_list: *const *const c_char, operands_list_len: *const size_t, |
||||
num_operands: c_int, |
||||
success: *mut u8, new_value_length: *mut size_t) -> *const c_char { |
||||
unsafe { |
||||
let cb: &mut MergeOperatorCallback = |
||||
&mut *(raw_cb as *mut MergeOperatorCallback); |
||||
let operands = &mut MergeOperands::new(operands_list, |
||||
operands_list_len, |
||||
num_operands); |
||||
let key: &[u8] = mem::transmute(slice::from_raw_buf(&raw_key, key_len as usize)); |
||||
let mut result = (cb.merge_fn)(key, None, operands); |
||||
result.shrink_to_fit(); |
||||
//TODO(tan) investigate zero-copy techniques to improve performance
|
||||
let buf = libc::malloc(result.len() as size_t); |
||||
assert!(!buf.is_null()); |
||||
*new_value_length = 1 as size_t; |
||||
*success = 1 as u8; |
||||
ptr::copy_memory(&mut *buf, result.as_ptr() |
||||
as *const c_void, result.len()); |
||||
buf as *const c_char |
||||
} |
||||
} |
||||
|
||||
|
||||
pub struct MergeOperands<'a> { |
||||
operands_list: *const *const c_char, |
||||
operands_list_len: *const size_t, |
||||
num_operands: usize, |
||||
cursor: usize, |
||||
} |
||||
|
||||
impl <'a> MergeOperands<'a> { |
||||
fn new(operands_list: *const *const c_char, |
||||
operands_list_len: *const size_t, |
||||
num_operands: c_int) -> MergeOperands<'a> { |
||||
assert!(num_operands >= 0); |
||||
MergeOperands { |
||||
operands_list: operands_list, |
||||
operands_list_len: operands_list_len, |
||||
num_operands: num_operands as usize, |
||||
cursor: 0, |
||||
} |
||||
} |
||||
} |
||||
|
||||
impl<'a> Iterator for &'a mut MergeOperands<'a> { |
||||
type Item = &'a [u8]; |
||||
fn next(&mut self) -> Option<&'a [u8]> { |
||||
use std::raw::Slice; |
||||
match self.cursor == self.num_operands { |
||||
true => None, |
||||
false => { |
||||
unsafe { |
||||
let base = self.operands_list as usize; |
||||
let base_len = self.operands_list_len as usize; |
||||
let spacing = mem::size_of::<*const *const u8>(); |
||||
let spacing_len = mem::size_of::<*const size_t>(); |
||||
let len_ptr = (base_len + (spacing_len * self.cursor)) |
||||
as *const size_t; |
||||
let len = *len_ptr as usize; |
||||
let ptr = base + (spacing * self.cursor); |
||||
self.cursor += 1; |
||||
Some(mem::transmute(Slice{data:*(ptr as *const *const u8) |
||||
as *const u8, len: len})) |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
fn size_hint(&self) -> (usize, Option<usize>) { |
||||
let remaining = self.num_operands - self.cursor; |
||||
(remaining, Some(remaining)) |
||||
} |
||||
} |
||||
|
||||
fn test_provided_merge(new_key: &[u8], existing_val: Option<&[u8]>, |
||||
mut operands: &mut MergeOperands) -> Vec<u8> { |
||||
let nops = operands.size_hint().0; |
||||
let mut result: Vec<u8> = Vec::with_capacity(nops); |
||||
match existing_val { |
||||
Some(v) => result.push_all(v), |
||||
None => (), |
||||
} |
||||
for op in operands { |
||||
result.push_all(op); |
||||
} |
||||
result |
||||
} |
||||
|
||||
#[allow(dead_code)] |
||||
#[test] |
||||
fn mergetest() { |
||||
let path = "_rust_rocksdb_mergetest"; |
||||
let opts = RocksDBOptions::new(); |
||||
opts.create_if_missing(true); |
||||
opts.add_merge_operator("test operator", test_provided_merge); |
||||
let db = RocksDB::open(opts, path).unwrap(); |
||||
let p = db.put(b"k1", b"a"); |
||||
assert!(p.is_ok()); |
||||
db.merge(b"k1", b"b"); |
||||
db.merge(b"k1", b"c"); |
||||
db.merge(b"k1", b"d"); |
||||
db.merge(b"k1", b"efg"); |
||||
let m = db.merge(b"k1", b"h"); |
||||
assert!(m.is_ok()); |
||||
db.get(b"k1").map( |value| { |
||||
match value.to_utf8() { |
||||
Some(v) => |
||||
println!("retrieved utf8 value: {}", v), |
||||
None => |
||||
println!("did not read valid utf-8 out of the db"), |
||||
} |
||||
}).on_absent( || { println!("value not present!") }) |
||||
.on_error( |e| { println!("error reading value")}); //: {", e) });
|
||||
|
||||
assert!(m.is_ok()); |
||||
let r: RocksDBResult<RocksDBVector, &str> = db.get(b"k1"); |
||||
assert!(r.unwrap().to_utf8().unwrap() == "abcdefgh"); |
||||
assert!(db.delete(b"k1").is_ok()); |
||||
assert!(db.get(b"k1").is_none()); |
||||
db.close(); |
||||
assert!(RocksDB::destroy(opts, path).is_ok()); |
||||
} |
||||
@ -0,0 +1,275 @@ |
||||
/* |
||||
Copyright 2014 Tyler Neely |
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); |
||||
you may not use this file except in compliance with the License. |
||||
You may obtain a copy of the License at |
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software |
||||
distributed under the License is distributed on an "AS IS" BASIS, |
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
See the License for the specific language governing permissions and |
||||
limitations under the License. |
||||
*/ |
||||
#![allow(unstable)] |
||||
|
||||
extern crate libc; |
||||
use self::libc::{c_int, size_t}; |
||||
use std::ffi::CString; |
||||
use std::mem; |
||||
|
||||
use rocksdb_ffi; |
||||
use merge_operator::{MergeOperatorCallback, MergeOperands, destructor_callback, full_merge_callback, |
||||
partial_merge_callback, name_callback}; |
||||
|
||||
pub struct RocksDBOptions { |
||||
pub inner: rocksdb_ffi::RocksDBOptions, |
||||
block_options: rocksdb_ffi::RocksDBBlockBasedTableOptions, |
||||
} |
||||
|
||||
impl Copy for RocksDBOptions {} |
||||
|
||||
impl RocksDBOptions { |
||||
pub fn new() -> RocksDBOptions { |
||||
unsafe { |
||||
let opts = rocksdb_ffi::rocksdb_options_create(); |
||||
let rocksdb_ffi::RocksDBOptions(opt_ptr) = opts; |
||||
if opt_ptr.is_null() { |
||||
panic!("Could not create rocksdb options".to_string()); |
||||
} |
||||
let block_opts = rocksdb_ffi::rocksdb_block_based_options_create(); |
||||
|
||||
RocksDBOptions{ |
||||
inner: opts, |
||||
block_options: block_opts, |
||||
} |
||||
} |
||||
} |
||||
|
||||
pub fn increase_parallelism(&self, parallelism: i32) { |
||||
unsafe { |
||||
rocksdb_ffi::rocksdb_options_increase_parallelism( |
||||
self.inner, parallelism); |
||||
} |
||||
} |
||||
|
||||
pub fn optimize_level_style_compaction(&self, |
||||
memtable_memory_budget: i32) { |
||||
unsafe { |
||||
rocksdb_ffi::rocksdb_options_optimize_level_style_compaction( |
||||
self.inner, memtable_memory_budget); |
||||
} |
||||
} |
||||
|
||||
pub fn create_if_missing(&self, create_if_missing: bool) { |
||||
unsafe { |
||||
rocksdb_ffi::rocksdb_options_set_create_if_missing( |
||||
self.inner, create_if_missing); |
||||
} |
||||
} |
||||
|
||||
pub fn add_merge_operator<'a>( &self, name: &str, |
||||
merge_fn: fn (&[u8], Option<&[u8]>, &mut MergeOperands) -> Vec<u8>) { |
||||
let cb = Box::new(MergeOperatorCallback { |
||||
name: CString::from_slice(name.as_bytes()), |
||||
merge_fn: merge_fn, |
||||
}); |
||||
|
||||
unsafe { |
||||
let mo = rocksdb_ffi::rocksdb_mergeoperator_create( |
||||
mem::transmute(cb), |
||||
destructor_callback, |
||||
full_merge_callback, |
||||
partial_merge_callback, |
||||
None, |
||||
name_callback); |
||||
rocksdb_ffi::rocksdb_options_set_merge_operator(self.inner, mo); |
||||
} |
||||
} |
||||
|
||||
pub fn set_block_size(&self, size: u64) { |
||||
unsafe { |
||||
rocksdb_ffi::rocksdb_block_based_options_set_block_size( |
||||
self.block_options, size); |
||||
rocksdb_ffi::rocksdb_options_set_block_based_table_factory( |
||||
self.inner, |
||||
self.block_options); |
||||
} |
||||
} |
||||
|
||||
pub fn set_block_cache_size_mb(&self, cache_size: u64) { |
||||
unsafe { |
||||
rocksdb_ffi::rocksdb_options_optimize_for_point_lookup( |
||||
self.inner, cache_size); |
||||
} |
||||
} |
||||
|
||||
pub fn set_filter(&self, filter: rocksdb_ffi::RocksDBFilterPolicy) { |
||||
unsafe { |
||||
rocksdb_ffi::rocksdb_block_based_options_set_filter_policy( |
||||
self.block_options, filter); |
||||
rocksdb_ffi::rocksdb_options_set_block_based_table_factory( |
||||
self.inner, |
||||
self.block_options); |
||||
} |
||||
} |
||||
|
||||
pub fn set_cache(&self, cache: rocksdb_ffi::RocksDBCache) { |
||||
unsafe { |
||||
rocksdb_ffi::rocksdb_block_based_options_set_block_cache( |
||||
self.block_options, cache); |
||||
rocksdb_ffi::rocksdb_options_set_block_based_table_factory( |
||||
self.inner, |
||||
self.block_options); |
||||
} |
||||
} |
||||
|
||||
pub fn set_cache_compressed(&self, cache: rocksdb_ffi::RocksDBCache) { |
||||
unsafe { |
||||
rocksdb_ffi::rocksdb_block_based_options_set_block_cache_compressed( |
||||
self.block_options, cache); |
||||
rocksdb_ffi::rocksdb_options_set_block_based_table_factory( |
||||
self.inner, |
||||
self.block_options); |
||||
} |
||||
} |
||||
|
||||
pub fn set_max_open_files(&self, nfiles: c_int) { |
||||
unsafe { |
||||
rocksdb_ffi::rocksdb_options_set_max_open_files(self.inner, nfiles); |
||||
} |
||||
} |
||||
|
||||
pub fn set_use_fsync(&self, useit: bool) { |
||||
unsafe { |
||||
match useit { |
||||
true => |
||||
rocksdb_ffi::rocksdb_options_set_use_fsync(self.inner, 1), |
||||
false => |
||||
rocksdb_ffi::rocksdb_options_set_use_fsync(self.inner, 0), |
||||
} |
||||
} |
||||
} |
||||
|
||||
pub fn set_bytes_per_sync(&self, nbytes: u64) { |
||||
unsafe { |
||||
rocksdb_ffi::rocksdb_options_set_bytes_per_sync( |
||||
self.inner, nbytes); |
||||
} |
||||
} |
||||
|
||||
pub fn set_disable_data_sync(&self, disable: bool) { |
||||
unsafe { |
||||
match disable { |
||||
true => |
||||
rocksdb_ffi::rocksdb_options_set_disable_data_sync( |
||||
self.inner, 1), |
||||
false => |
||||
rocksdb_ffi::rocksdb_options_set_disable_data_sync( |
||||
self.inner, 0), |
||||
} |
||||
} |
||||
} |
||||
|
||||
pub fn set_table_cache_num_shard_bits(&self, nbits: c_int) { |
||||
unsafe { |
||||
rocksdb_ffi::rocksdb_options_set_table_cache_numshardbits( |
||||
self.inner, nbits); |
||||
} |
||||
} |
||||
|
||||
pub fn set_min_write_buffer_number(&self, nbuf: c_int) { |
||||
unsafe { |
||||
rocksdb_ffi::rocksdb_options_set_min_write_buffer_number_to_merge( |
||||
self.inner, nbuf); |
||||
} |
||||
} |
||||
|
||||
pub fn set_max_write_buffer_number(&self, nbuf: c_int) { |
||||
unsafe { |
||||
rocksdb_ffi::rocksdb_options_set_max_write_buffer_number( |
||||
self.inner, nbuf); |
||||
} |
||||
} |
||||
|
||||
pub fn set_write_buffer_size(&self, size: size_t) { |
||||
unsafe { |
||||
rocksdb_ffi::rocksdb_options_set_write_buffer_size( |
||||
self.inner, size); |
||||
} |
||||
} |
||||
|
||||
pub fn set_target_file_size_base(&self, size: u64) { |
||||
unsafe { |
||||
rocksdb_ffi::rocksdb_options_set_target_file_size_base( |
||||
self.inner, size); |
||||
} |
||||
} |
||||
|
||||
pub fn set_min_write_buffer_number_to_merge(&self, to_merge: c_int) { |
||||
unsafe { |
||||
rocksdb_ffi::rocksdb_options_set_min_write_buffer_number_to_merge( |
||||
self.inner, to_merge); |
||||
} |
||||
} |
||||
|
||||
pub fn set_level_zero_slowdown_writes_trigger(&self, n: c_int) { |
||||
unsafe { |
||||
rocksdb_ffi::rocksdb_options_set_level0_slowdown_writes_trigger( |
||||
self.inner, n); |
||||
} |
||||
} |
||||
|
||||
pub fn set_level_zero_stop_writes_trigger(&self, n: c_int) { |
||||
unsafe { |
||||
rocksdb_ffi::rocksdb_options_set_level0_stop_writes_trigger( |
||||
self.inner, n); |
||||
} |
||||
} |
||||
|
||||
pub fn set_compaction_style(&self, style: |
||||
rocksdb_ffi::RocksDBCompactionStyle) { |
||||
unsafe { |
||||
rocksdb_ffi::rocksdb_options_set_compaction_style( |
||||
self.inner, style); |
||||
} |
||||
} |
||||
|
||||
pub fn set_max_background_compactions(&self, n: c_int) { |
||||
unsafe { |
||||
rocksdb_ffi::rocksdb_options_set_max_background_compactions( |
||||
self.inner, n); |
||||
} |
||||
} |
||||
|
||||
pub fn set_max_background_flushes(&self, n: c_int) { |
||||
unsafe { |
||||
rocksdb_ffi::rocksdb_options_set_max_background_flushes( |
||||
self.inner, n); |
||||
} |
||||
} |
||||
|
||||
pub fn set_filter_deletes(&self, filter: bool) { |
||||
unsafe { |
||||
rocksdb_ffi::rocksdb_options_set_filter_deletes( |
||||
self.inner, filter); |
||||
} |
||||
} |
||||
|
||||
pub fn set_disable_auto_compactions(&self, disable: bool) { |
||||
unsafe { |
||||
match disable { |
||||
true => |
||||
rocksdb_ffi::rocksdb_options_set_disable_auto_compactions( |
||||
self.inner, 1), |
||||
false => |
||||
rocksdb_ffi::rocksdb_options_set_disable_auto_compactions( |
||||
self.inner, 0), |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
|
||||
Loading…
Reference in new issue