NextGraph
/
oxigraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Activity

Use Little Endian to store data

Browse Source 
It's the most common endianness
pull/10/head
Tpt 6 years ago
parent 467921944b
commit fc9a987dd7
2 changed files with 34 additions and 34 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 60
      lib/src/store/numeric_encoder.rs
  2. 8
      lib/src/store/rocksdb.rs

60
lib/src/store/numeric_encoder.rs
Unescape View File

@ -1,4 +1,4 @@
use byteorder::{NetworkEndian, ReadBytesExt, WriteBytesExt}; use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use chrono::DateTime; use chrono::DateTime;
use chrono::FixedOffset; use chrono::FixedOffset;
use chrono::NaiveDateTime; use chrono::NaiveDateTime;
@ -270,7 +270,7 @@ impl<R: Read> TermReader for R {
match self.read_u8()? { match self.read_u8()? {
TYPE_DEFAULT_GRAPH_ID => Ok(EncodedTerm::DefaultGraph {}), TYPE_DEFAULT_GRAPH_ID => Ok(EncodedTerm::DefaultGraph {}),
TYPE_NAMED_NODE_ID => Ok(EncodedTerm::NamedNode { TYPE_NAMED_NODE_ID => Ok(EncodedTerm::NamedNode {
iri_id: self.read_u64::<NetworkEndian>()?, iri_id: self.read_u64::<LittleEndian>()?,
}), }),
TYPE_BLANK_NODE_ID => { TYPE_BLANK_NODE_ID => {
let mut uuid_buffer = [0 as u8; 16]; let mut uuid_buffer = [0 as u8; 16];
@ -278,29 +278,29 @@ impl<R: Read> TermReader for R {
Ok(EncodedTerm::BlankNode(Uuid::from_bytes(uuid_buffer))) Ok(EncodedTerm::BlankNode(Uuid::from_bytes(uuid_buffer)))
} }
TYPE_SIMPLE_LITERAL_ID => Ok(EncodedTerm::SimpleLiteral { TYPE_SIMPLE_LITERAL_ID => Ok(EncodedTerm::SimpleLiteral {
value_id: self.read_u64::<NetworkEndian>()?, value_id: self.read_u64::<LittleEndian>()?,
}), }),
TYPE_LANG_STRING_LITERAL_ID => Ok(EncodedTerm::LangStringLiteral { TYPE_LANG_STRING_LITERAL_ID => Ok(EncodedTerm::LangStringLiteral {
language_id: self.read_u64::<NetworkEndian>()?, language_id: self.read_u64::<LittleEndian>()?,
value_id: self.read_u64::<NetworkEndian>()?, value_id: self.read_u64::<LittleEndian>()?,
}), }),
TYPE_TYPED_LITERAL_ID => Ok(EncodedTerm::TypedLiteral { TYPE_TYPED_LITERAL_ID => Ok(EncodedTerm::TypedLiteral {
datatype_id: self.read_u64::<NetworkEndian>()?, datatype_id: self.read_u64::<LittleEndian>()?,
value_id: self.read_u64::<NetworkEndian>()?, value_id: self.read_u64::<LittleEndian>()?,
}), }),
TYPE_STRING_LITERAL => Ok(EncodedTerm::StringLiteral { TYPE_STRING_LITERAL => Ok(EncodedTerm::StringLiteral {
value_id: self.read_u64::<NetworkEndian>()?, value_id: self.read_u64::<LittleEndian>()?,
}), }),
TYPE_BOOLEAN_LITERAL_TRUE => Ok(EncodedTerm::BooleanLiteral(true)), TYPE_BOOLEAN_LITERAL_TRUE => Ok(EncodedTerm::BooleanLiteral(true)),
TYPE_BOOLEAN_LITERAL_FALSE => Ok(EncodedTerm::BooleanLiteral(false)), TYPE_BOOLEAN_LITERAL_FALSE => Ok(EncodedTerm::BooleanLiteral(false)),
TYPE_FLOAT_LITERAL => Ok(EncodedTerm::FloatLiteral(OrderedFloat( TYPE_FLOAT_LITERAL => Ok(EncodedTerm::FloatLiteral(OrderedFloat(
self.read_f32::<NetworkEndian>()?, self.read_f32::<LittleEndian>()?,
))), ))),
TYPE_DOUBLE_LITERAL => Ok(EncodedTerm::DoubleLiteral(OrderedFloat( TYPE_DOUBLE_LITERAL => Ok(EncodedTerm::DoubleLiteral(OrderedFloat(
self.read_f64::<NetworkEndian>()?, self.read_f64::<LittleEndian>()?,
))), ))),
TYPE_INTEGER_LITERAL => Ok(EncodedTerm::IntegerLiteral( TYPE_INTEGER_LITERAL => Ok(EncodedTerm::IntegerLiteral(
self.read_i128::<NetworkEndian>()?, self.read_i128::<LittleEndian>()?,
)), )),
TYPE_DECIMAL_LITERAL => { TYPE_DECIMAL_LITERAL => {
let mut buffer = [0 as u8; 16]; let mut buffer = [0 as u8; 16];
@ -309,16 +309,16 @@ impl<R: Read> TermReader for R {
} }
TYPE_DATE_TIME_LITERAL => Ok(EncodedTerm::DateTime(DateTime::from_utc( TYPE_DATE_TIME_LITERAL => Ok(EncodedTerm::DateTime(DateTime::from_utc(
NaiveDateTime::from_timestamp_opt( NaiveDateTime::from_timestamp_opt(
self.read_i64::<NetworkEndian>()?, self.read_i64::<LittleEndian>()?,
self.read_u32::<NetworkEndian>()?, self.read_u32::<LittleEndian>()?,
).ok_or("Invalid date time serialization")?, ).ok_or("Invalid date time serialization")?,
FixedOffset::east_opt(self.read_i32::<NetworkEndian>()?) FixedOffset::east_opt(self.read_i32::<LittleEndian>()?)
.ok_or("Invalid timezone offset")?, .ok_or("Invalid timezone offset")?,
))), ))),
TYPE_NAIVE_DATE_TIME_LITERAL => Ok(EncodedTerm::NaiveDateTime( TYPE_NAIVE_DATE_TIME_LITERAL => Ok(EncodedTerm::NaiveDateTime(
NaiveDateTime::from_timestamp_opt( NaiveDateTime::from_timestamp_opt(
self.read_i64::<NetworkEndian>()?, self.read_i64::<LittleEndian>()?,
self.read_u32::<NetworkEndian>()?, self.read_u32::<LittleEndian>()?,
).ok_or("Invalid date time serialization")?, ).ok_or("Invalid date time serialization")?,
)), )),
_ => Err("the term buffer has an invalid type id".into()), _ => Err("the term buffer has an invalid type id".into()),
@ -377,38 +377,38 @@ impl<R: Write> TermWriter for R {
self.write_u8(term.type_id())?; self.write_u8(term.type_id())?;
match term { match term {
EncodedTerm::DefaultGraph {} => {} EncodedTerm::DefaultGraph {} => {}
EncodedTerm::NamedNode { iri_id } => self.write_u64::<NetworkEndian>(iri_id)?, EncodedTerm::NamedNode { iri_id } => self.write_u64::<LittleEndian>(iri_id)?,
EncodedTerm::BlankNode(id) => self.write_all(id.as_bytes())?, EncodedTerm::BlankNode(id) => self.write_all(id.as_bytes())?,
EncodedTerm::SimpleLiteral { value_id } | EncodedTerm::StringLiteral { value_id } => { EncodedTerm::SimpleLiteral { value_id } | EncodedTerm::StringLiteral { value_id } => {
self.write_u64::<NetworkEndian>(value_id)?; self.write_u64::<LittleEndian>(value_id)?;
} }
EncodedTerm::LangStringLiteral { EncodedTerm::LangStringLiteral {
value_id, value_id,
language_id, language_id,
} => { } => {
self.write_u64::<NetworkEndian>(language_id)?; self.write_u64::<LittleEndian>(language_id)?;
self.write_u64::<NetworkEndian>(value_id)?; self.write_u64::<LittleEndian>(value_id)?;
} }
EncodedTerm::TypedLiteral { EncodedTerm::TypedLiteral {
value_id, value_id,
datatype_id, datatype_id,
} => { } => {
self.write_u64::<NetworkEndian>(datatype_id)?; self.write_u64::<LittleEndian>(datatype_id)?;
self.write_u64::<NetworkEndian>(value_id)?; self.write_u64::<LittleEndian>(value_id)?;
} }
EncodedTerm::BooleanLiteral(_) => {} EncodedTerm::BooleanLiteral(_) => {}
EncodedTerm::FloatLiteral(value) => self.write_f32::<NetworkEndian>(*value)?, EncodedTerm::FloatLiteral(value) => self.write_f32::<LittleEndian>(*value)?,
EncodedTerm::DoubleLiteral(value) => self.write_f64::<NetworkEndian>(*value)?, EncodedTerm::DoubleLiteral(value) => self.write_f64::<LittleEndian>(*value)?,
EncodedTerm::IntegerLiteral(value) => self.write_i128::<NetworkEndian>(value)?, EncodedTerm::IntegerLiteral(value) => self.write_i128::<LittleEndian>(value)?,
EncodedTerm::DecimalLiteral(value) => self.write_all(&value.serialize())?, EncodedTerm::DecimalLiteral(value) => self.write_all(&value.serialize())?,
EncodedTerm::DateTime(value) => { EncodedTerm::DateTime(value) => {
self.write_i64::<NetworkEndian>(value.timestamp())?; self.write_i64::<LittleEndian>(value.timestamp())?;
self.write_u32::<NetworkEndian>(value.timestamp_subsec_nanos())?; self.write_u32::<LittleEndian>(value.timestamp_subsec_nanos())?;
self.write_i32::<NetworkEndian>(value.timezone().local_minus_utc())?; self.write_i32::<LittleEndian>(value.timezone().local_minus_utc())?;
} }
EncodedTerm::NaiveDateTime(value) => { EncodedTerm::NaiveDateTime(value) => {
self.write_i64::<NetworkEndian>(value.timestamp())?; self.write_i64::<LittleEndian>(value.timestamp())?;
self.write_u32::<NetworkEndian>(value.timestamp_subsec_nanos())?; self.write_u32::<LittleEndian>(value.timestamp_subsec_nanos())?;
} }
} }
Ok(()) Ok(())

8
lib/src/store/rocksdb.rs
Unescape View File

@ -1,5 +1,5 @@
use byteorder::ByteOrder; use byteorder::ByteOrder;
use byteorder::NetworkEndian; use byteorder::LittleEndian;
use rocksdb::ColumnFamily; use rocksdb::ColumnFamily;
use rocksdb::DBRawIterator; use rocksdb::DBRawIterator;
use rocksdb::DBVector; use rocksdb::DBVector;
@ -84,7 +84,7 @@ impl BytesStore for RocksDbStore {
fn insert_bytes(&self, value: &[u8]) -> Result<u64> { fn insert_bytes(&self, value: &[u8]) -> Result<u64> {
Ok(if let Some(id) = self.db.get_cf(self.str2id_cf, value)? { Ok(if let Some(id) = self.db.get_cf(self.str2id_cf, value)? {
NetworkEndian::read_u64(&id) LittleEndian::read_u64(&id)
} else { } else {
let id = self.str_id_counter.lock()?.get_and_increment(&self.db)? as u64; let id = self.str_id_counter.lock()?.get_and_increment(&self.db)? as u64;
let id_bytes = to_bytes(id); let id_bytes = to_bytes(id);
@ -341,7 +341,7 @@ impl RocksDBCounter {
fn get_and_increment(&self, db: &DB) -> Result<u64> { fn get_and_increment(&self, db: &DB) -> Result<u64> {
let value = db let value = db
.get(self.name.as_bytes())? .get(self.name.as_bytes())?
.map_or(0, |b| NetworkEndian::read_u64(&b)); .map_or(0, |b| LittleEndian::read_u64(&b));
db.put(self.name.as_bytes(), &to_bytes(value + 1))?; db.put(self.name.as_bytes(), &to_bytes(value + 1))?;
Ok(value) Ok(value)
} }
@ -513,6 +513,6 @@ impl<I: Iterator<Item = Result<EncodedQuad>>> Iterator for InGraphQuadsIterator<
fn to_bytes(int: u64) -> [u8; 8] { fn to_bytes(int: u64) -> [u8; 8] {
let mut buf = [0 as u8; 8]; let mut buf = [0 as u8; 8];
NetworkEndian::write_u64(&mut buf, int); LittleEndian::write_u64(&mut buf, int);
buf buf
} }

Write Preview
Loading…
Cancel
Save