RocksDB: avoid not needed copies when retrieving string values

pull/10/head
Tpt 6 years ago
parent 0356de0e40
commit 25ea1df0ae
  1. 6
      lib/src/sparql/eval.rs
  2. 2
      lib/src/store/memory.rs
  3. 13
      lib/src/store/numeric_encoder.rs
  4. 38
      lib/src/store/rocksdb.rs

@ -775,7 +775,7 @@ impl<S: EncodedQuadsStore> SimpleEvaluator<S> {
fn to_simple_string(&self, term: EncodedTerm) -> Option<String> { fn to_simple_string(&self, term: EncodedTerm) -> Option<String> {
if let EncodedTerm::SimpleLiteral { value_id } = term { if let EncodedTerm::SimpleLiteral { value_id } = term {
self.store.get_str(value_id).ok() Some(self.store.get_str(value_id).ok()?.into())
} else { } else {
None None
} }
@ -793,7 +793,9 @@ impl<S: EncodedQuadsStore> SimpleEvaluator<S> {
match term { match term {
EncodedTerm::SimpleLiteral { value_id } EncodedTerm::SimpleLiteral { value_id }
| EncodedTerm::StringLiteral { value_id } | EncodedTerm::StringLiteral { value_id }
| EncodedTerm::LangStringLiteral { value_id, .. } => self.store.get_str(value_id).ok(), | EncodedTerm::LangStringLiteral { value_id, .. } => {
Some(self.store.get_str(value_id).ok()?.into())
}
_ => None, _ => None,
} }
} }

@ -68,6 +68,8 @@ struct MemoryGraphIndexes {
} }
impl StringStore for MemoryStore { impl StringStore for MemoryStore {
type StringType = String;
fn insert_str(&self, value: &str) -> Result<u64> { fn insert_str(&self, value: &str) -> Result<u64> {
self.string_store.insert_str(value) self.string_store.insert_str(value)
} }

@ -11,6 +11,7 @@ use rust_decimal::Decimal;
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::io::Read; use std::io::Read;
use std::io::Write; use std::io::Write;
use std::ops::Deref;
use std::str; use std::str;
use std::str::FromStr; use std::str::FromStr;
use std::sync::RwLock; use std::sync::RwLock;
@ -30,8 +31,10 @@ const XSD_DATE_ID: u64 = 9;
const XSD_TIME_ID: u64 = 10; const XSD_TIME_ID: u64 = 10;
pub trait StringStore { pub trait StringStore {
type StringType: Deref<Target = str> + ToString + Into<String>;
fn insert_str(&self, value: &str) -> Result<u64>; fn insert_str(&self, value: &str) -> Result<u64>;
fn get_str(&self, id: u64) -> Result<String>; fn get_str(&self, id: u64) -> Result<Self::StringType>;
fn get_url(&self, id: u64) -> Result<Url>; fn get_url(&self, id: u64) -> Result<Url>;
/// Should be called when the bytes store is created /// Should be called when the bytes store is created
@ -58,11 +61,13 @@ pub trait StringStore {
} }
impl<'a, S: StringStore> StringStore for &'a S { impl<'a, S: StringStore> StringStore for &'a S {
type StringType = S::StringType;
fn insert_str(&self, value: &str) -> Result<u64> { fn insert_str(&self, value: &str) -> Result<u64> {
(*self).insert_str(value) (*self).insert_str(value)
} }
fn get_str(&self, id: u64) -> Result<String> { fn get_str(&self, id: u64) -> Result<S::StringType> {
(*self).get_str(id) (*self).get_str(id)
} }
@ -88,6 +93,8 @@ impl Default for MemoryStringStore {
} }
impl StringStore for MemoryStringStore { impl StringStore for MemoryStringStore {
type StringType = String;
fn insert_str(&self, value: &str) -> Result<u64> { fn insert_str(&self, value: &str) -> Result<u64> {
let mut id2str = self.id2str.write().map_err(MutexPoisonError::from)?; let mut id2str = self.id2str.write().map_err(MutexPoisonError::from)?;
let mut str2id = self.str2id.write().map_err(MutexPoisonError::from)?; let mut str2id = self.str2id.write().map_err(MutexPoisonError::from)?;
@ -728,7 +735,7 @@ impl<S: StringStore> Encoder<S> {
) )
.into()), .into()),
EncodedTerm::StringLiteral { value_id } => { EncodedTerm::StringLiteral { value_id } => {
Ok(Literal::from(self.string_store.get_str(value_id)?).into()) Ok(Literal::from(self.string_store.get_str(value_id)?.into()).into())
} }
EncodedTerm::BooleanLiteral(value) => Ok(Literal::from(value).into()), EncodedTerm::BooleanLiteral(value) => Ok(Literal::from(value).into()),
EncodedTerm::FloatLiteral(value) => Ok(Literal::from(*value).into()), EncodedTerm::FloatLiteral(value) => Ok(Literal::from(*value).into()),

@ -10,6 +10,7 @@ use rocksdb::ColumnFamily;
use rocksdb::DBCompactionStyle; use rocksdb::DBCompactionStyle;
use rocksdb::DBCompressionType; use rocksdb::DBCompressionType;
use rocksdb::DBRawIterator; use rocksdb::DBRawIterator;
use rocksdb::DBVector;
use rocksdb::Options; use rocksdb::Options;
use rocksdb::WriteBatch; use rocksdb::WriteBatch;
use rocksdb::DB; use rocksdb::DB;
@ -90,6 +91,8 @@ impl RocksDbStore {
} }
impl StringStore for RocksDbStore { impl StringStore for RocksDbStore {
type StringType = RocksString;
fn insert_str(&self, value: &str) -> Result<u64> { fn insert_str(&self, value: &str) -> Result<u64> {
let value = value.as_bytes(); let value = value.as_bytes();
Ok(if let Some(id) = self.db.get_cf(*self.str2id_cf, value)? { Ok(if let Some(id) = self.db.get_cf(*self.str2id_cf, value)? {
@ -109,22 +112,17 @@ impl StringStore for RocksDbStore {
}) })
} }
fn get_str(&self, id: u64) -> Result<String> { fn get_str(&self, id: u64) -> Result<RocksString> {
let value = self.db.get_cf(*self.id2str_cf, &to_bytes(id))?; let value = self.db.get_cf(*self.id2str_cf, &to_bytes(id))?;
if let Some(value) = value { if let Some(value) = value {
Ok(str::from_utf8(&value)?.to_owned()) Ok(RocksString { vec: value })
} else { } else {
Err(format_err!("value not found in the dictionary")) Err(format_err!("value not found in the dictionary"))
} }
} }
fn get_url(&self, id: u64) -> Result<Url> { fn get_url(&self, id: u64) -> Result<Url> {
let value = self.db.get_cf(*self.id2str_cf, &to_bytes(id))?; Ok(Url::from_str(&self.get_str(id)?)?)
if let Some(value) = value {
Ok(Url::from_str(str::from_utf8(&value)?)?)
} else {
Err(format_err!("value not found in the dictionary"))
}
} }
} }
@ -553,6 +551,30 @@ fn to_bytes(int: u64) -> [u8; 8] {
buf buf
} }
pub struct RocksString {
vec: DBVector,
}
impl Deref for RocksString {
type Target = str;
fn deref(&self) -> &str {
unsafe { str::from_utf8_unchecked(&self.vec) }
}
}
impl ToString for RocksString {
fn to_string(&self) -> String {
self.deref().to_owned()
}
}
impl From<RocksString> for String {
fn from(val: RocksString) -> String {
val.deref().to_owned()
}
}
// TODO: very bad but I believe it is fine // TODO: very bad but I believe it is fine
#[derive(Clone, Copy)] #[derive(Clone, Copy)]
struct SendColumnFamily(ColumnFamily); struct SendColumnFamily(ColumnFamily);

Loading…
Cancel
Save