use byteorder::{NetworkEndian, ReadBytesExt, WriteBytesExt}; use errors::*; use model::*; use std::io::Cursor; use std::io::Read; use std::io::Write; use std::mem::size_of; use std::ops::Deref; use std::str; use std::str::FromStr; use url::Url; use uuid::Uuid; pub trait BytesStore { type BytesOutput: Deref; fn put(&self, value: &[u8]) -> Result; fn get(&self, id: u64) -> Result>; } const TYPE_NAMED_NODE_ID: u8 = 1; const TYPE_BLANK_NODE_ID: u8 = 2; const TYPE_LANG_STRING_LITERAL_ID: u8 = 3; const TYPE_TYPED_LITERAL_ID: u8 = 4; #[derive(Eq, PartialEq, Ord, PartialOrd, Debug, Clone, Hash)] pub enum EncodedTerm { NamedNode { iri_id: u64 }, BlankNode(Uuid), LangStringLiteral { value_id: u64, language_id: u64 }, TypedLiteral { value_id: u64, datatype_id: u64 }, } impl EncodedTerm { pub fn read(reader: &mut impl Read) -> Result { let type_id = reader.read_u8()?; match type_id { TYPE_NAMED_NODE_ID => Ok(EncodedTerm::NamedNode { iri_id: reader.read_u64::()?, }), TYPE_BLANK_NODE_ID => { let mut uuid_buffer = [0 as u8; 16]; reader.read_exact(&mut uuid_buffer)?; Ok(EncodedTerm::BlankNode(Uuid::from_bytes(&uuid_buffer)?)) } TYPE_LANG_STRING_LITERAL_ID => Ok(EncodedTerm::LangStringLiteral { language_id: reader.read_u64::()?, value_id: reader.read_u64::()?, }), TYPE_TYPED_LITERAL_ID => Ok(EncodedTerm::TypedLiteral { datatype_id: reader.read_u64::()?, value_id: reader.read_u64::()?, }), _ => Err("the term buffer has an invalid type id".into()), } } pub fn encoding_size(&self) -> usize { Self::type_length(self.type_id()).unwrap() //It is not possible to fail here } fn type_id(&self) -> u8 { match self { EncodedTerm::NamedNode { .. } => TYPE_NAMED_NODE_ID, EncodedTerm::BlankNode(_) => TYPE_BLANK_NODE_ID, EncodedTerm::LangStringLiteral { .. } => TYPE_LANG_STRING_LITERAL_ID, EncodedTerm::TypedLiteral { .. } => TYPE_TYPED_LITERAL_ID, } } fn type_length(type_id: u8) -> Result { //TODO: useful match type_id { TYPE_NAMED_NODE_ID => Ok(1 + size_of::()), TYPE_BLANK_NODE_ID => Ok(17), //TODO: guess TYPE_LANG_STRING_LITERAL_ID => Ok(1 + 2 * size_of::()), TYPE_TYPED_LITERAL_ID => Ok(1 + 2 * size_of::()), _ => Err(format!("{} is not a known type id", type_id).into()), } } pub fn write(&self, writer: &mut impl Write) -> Result<()> { writer.write_u8(self.type_id())?; match self { EncodedTerm::NamedNode { iri_id } => writer.write_u64::(*iri_id)?, EncodedTerm::BlankNode(id) => writer.write_all(id.as_bytes())?, EncodedTerm::LangStringLiteral { value_id, language_id, } => { writer.write_u64::(*language_id)?; writer.write_u64::(*value_id)?; } EncodedTerm::TypedLiteral { value_id, datatype_id, } => { writer.write_u64::(*datatype_id)?; writer.write_u64::(*value_id)?; } } Ok(()) } } #[derive(Eq, PartialEq, Ord, PartialOrd, Debug, Clone, Hash)] pub struct EncodedQuad { pub subject: EncodedTerm, pub predicate: EncodedTerm, pub object: EncodedTerm, pub graph_name: Option, } impl EncodedQuad { pub fn new_from_spog_buffer(buffer: &[u8]) -> Result { let mut cursor = Cursor::new(buffer); let subject = EncodedTerm::read(&mut cursor)?; let predicate = EncodedTerm::read(&mut cursor)?; let object = EncodedTerm::read(&mut cursor)?; let graph_name = if cursor.position() < buffer.len() as u64 { Some(EncodedTerm::read(&mut cursor)?) } else { None }; Ok(Self { subject, predicate, object, graph_name, }) } pub fn new_from_posg_buffer(buffer: &[u8]) -> Result { let mut cursor = Cursor::new(buffer); let predicate = EncodedTerm::read(&mut cursor)?; let object = EncodedTerm::read(&mut cursor)?; let subject = EncodedTerm::read(&mut cursor)?; let graph_name = if cursor.position() < buffer.len() as u64 { Some(EncodedTerm::read(&mut cursor)?) } else { None }; Ok(Self { subject, predicate, object, graph_name, }) } pub fn new_from_ospg_buffer(buffer: &[u8]) -> Result { let mut cursor = Cursor::new(buffer); let object = EncodedTerm::read(&mut cursor)?; let subject = EncodedTerm::read(&mut cursor)?; let predicate = EncodedTerm::read(&mut cursor)?; let graph_name = if cursor.position() < buffer.len() as u64 { Some(EncodedTerm::read(&mut cursor)?) } else { None }; Ok(Self { subject, predicate, object, graph_name, }) } pub fn spog(&self) -> Result> { let mut spog = Vec::with_capacity(self.encoding_size()); self.subject.write(&mut spog)?; self.predicate.write(&mut spog)?; self.object.write(&mut spog)?; if let Some(ref graph_name) = self.graph_name { graph_name.write(&mut spog)?; } Ok(spog) } pub fn posg(&self) -> Result> { let mut posg = Vec::with_capacity(self.encoding_size()); self.predicate.write(&mut posg)?; self.object.write(&mut posg)?; self.subject.write(&mut posg)?; if let Some(ref graph_name) = self.graph_name { graph_name.write(&mut posg)?; } Ok(posg) } pub fn ospg(&self) -> Result> { let mut ospg = Vec::with_capacity(self.encoding_size()); self.object.write(&mut ospg)?; self.subject.write(&mut ospg)?; self.predicate.write(&mut ospg)?; if let Some(ref graph_name) = self.graph_name { graph_name.write(&mut ospg)?; } Ok(ospg) } fn encoding_size(&self) -> usize { self.subject.encoding_size() + self.predicate.encoding_size() + self.object.encoding_size() + match self.graph_name { Some(ref graph_name) => graph_name.encoding_size(), None => 0, } } } pub struct Encoder { string_store: S, } impl Encoder { pub fn new(string_store: S) -> Self { Self { string_store } } pub fn encode_named_node(&self, named_node: &NamedNode) -> Result { Ok(EncodedTerm::NamedNode { iri_id: self.encode_str_value(named_node.as_str())?, }) } pub fn encode_blank_node(&self, blank_node: &BlankNode) -> Result { Ok(EncodedTerm::BlankNode(blank_node.deref().clone())) } pub fn encode_literal(&self, literal: &Literal) -> Result { if let Some(language) = literal.language() { Ok(EncodedTerm::LangStringLiteral { value_id: self.encode_str_value(&literal.value())?, language_id: self.encode_str_value(language)?, }) } else { Ok(EncodedTerm::TypedLiteral { value_id: self.encode_str_value(&literal.value())?, datatype_id: self.encode_str_value(literal.datatype().as_ref())?, }) } } pub fn encode_named_or_blank_node(&self, term: &NamedOrBlankNode) -> Result { match term { NamedOrBlankNode::NamedNode(named_node) => self.encode_named_node(named_node), NamedOrBlankNode::BlankNode(blank_node) => self.encode_blank_node(blank_node), } } pub fn encode_term(&self, term: &Term) -> Result { match term { Term::NamedNode(named_node) => self.encode_named_node(named_node), Term::BlankNode(blank_node) => self.encode_blank_node(blank_node), Term::Literal(literal) => self.encode_literal(literal), } } pub fn encode_quad(&self, quad: &Quad) -> Result { Ok(EncodedQuad { subject: self.encode_named_or_blank_node(quad.subject())?, predicate: self.encode_named_node(quad.predicate())?, object: self.encode_term(quad.object())?, graph_name: match quad.graph_name() { Some(graph_name) => Some(self.encode_named_or_blank_node(&graph_name)?), None => None, }, }) } pub fn decode_term(&self, encoded: &EncodedTerm) -> Result { match encoded { EncodedTerm::NamedNode { iri_id } => { Ok(NamedNode::from(self.decode_url_value(*iri_id)?).into()) } EncodedTerm::BlankNode(id) => Ok(BlankNode::from(*id).into()), EncodedTerm::LangStringLiteral { value_id, language_id, } => Ok(Literal::new_language_tagged_literal( self.decode_str_value(*value_id)?, self.decode_str_value(*language_id)?, ).into()), EncodedTerm::TypedLiteral { value_id, datatype_id, } => Ok(Literal::new_typed_literal( self.decode_str_value(*value_id)?, NamedNode::from(self.decode_url_value(*datatype_id)?), ).into()), } } pub fn decode_named_or_blank_node(&self, encoded: &EncodedTerm) -> Result { match self.decode_term(encoded)? { Term::NamedNode(named_node) => Ok(named_node.into()), Term::BlankNode(blank_node) => Ok(blank_node.into()), Term::Literal(_) => Err("A literal has ben found instead of a named node".into()), } } pub fn decode_named_node(&self, encoded: &EncodedTerm) -> Result { match self.decode_term(encoded)? { Term::NamedNode(named_node) => Ok(named_node), Term::BlankNode(_) => Err("A blank node has been found instead of a named node".into()), Term::Literal(_) => Err("A literal has ben found instead of a named node".into()), } } pub fn decode_quad(&self, encoded: &EncodedQuad) -> Result { Ok(Quad::new( self.decode_named_or_blank_node(&encoded.subject)?, self.decode_named_node(&encoded.predicate)?, self.decode_term(&encoded.object)?, match encoded.graph_name { Some(ref graph_name) => Some(self.decode_named_or_blank_node(&graph_name)?), None => None, }, )) } fn encode_str_value(&self, text: &str) -> Result { self.string_store.put(text.as_bytes()) } fn decode_url_value(&self, id: u64) -> Result { let bytes = self.decode_value(id)?; Ok(Url::from_str(str::from_utf8(&bytes)?)?) } fn decode_str_value(&self, id: u64) -> Result { let bytes = self.decode_value(id)?; Ok(str::from_utf8(&bytes)?.to_owned()) } fn decode_value(&self, id: u64) -> Result { self.string_store .get(id)? .ok_or("value not found in the dictionary".into()) } } impl Default for Encoder { fn default() -> Self { Self { string_store: S::default(), } } } mod test { use std::cell::RefCell; use std::collections::BTreeMap; use store::numeric_encoder::*; #[derive(Default)] struct MemoryBytesStore { id2str: RefCell>>, str2id: RefCell, u64>>, } impl BytesStore for MemoryBytesStore { type BytesOutput = Vec; fn put(&self, value: &[u8]) -> Result { let mut str2id = self.str2id.borrow_mut(); let mut id2str = self.id2str.borrow_mut(); let id = str2id.entry(value.to_vec()).or_insert_with(|| { let id = id2str.len() as u64; id2str.insert(id, value.to_vec()); id }); Ok(*id) } fn get(&self, id: u64) -> Result>> { Ok(self.id2str.borrow().get(&id).map(|s| s.to_owned())) } } #[test] fn test_encoding() { use model::*; let encoder: Encoder = Encoder::default(); let terms: Vec = vec![ NamedNode::from_str("http://foo.com").unwrap().into(), NamedNode::from_str("http://bar.com").unwrap().into(), NamedNode::from_str("http://foo.com").unwrap().into(), BlankNode::default().into(), Literal::from(true).into(), Literal::from(1.2).into(), Literal::from("foo").into(), Literal::new_language_tagged_literal("foo", "fr").into(), ]; for term in terms { let encoded = encoder.encode_term(&term).unwrap(); assert_eq!(term, encoder.decode_term(&encoded).unwrap()) } } }