NextGraph
/
oxigraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Activity

Uses Rio RDF XML parser

Browse Source
pull/10/head
Tpt 5 years ago
parent 74dadf5f21
commit 4dee8a9aa2
4 changed files with 21 additions and 611 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 5
      lib/Cargo.toml
  2. 608
      lib/src/rio/xml.rs
  3. 15
      lib/tests/rdf_test_cases.rs
  4. 2
      lib/tests/sparql_test_cases.rs

5
lib/Cargo.toml
Unescape View File

@ -29,8 +29,9 @@ rust_decimal = "1"
chrono = "0.4" chrono = "0.4"
failure = "0.1" failure = "0.1"
regex = "1" regex = "1"
rio_api = "0.1" rio_api = "0.2"
rio_turtle = "0.1" rio_turtle = "0.2"
rio_xml = "0.2"
[build-dependencies] [build-dependencies]
peg = "0.5" peg = "0.5"

608
lib/src/rio/xml.rs
Unescape View File

@ -1,604 +1,22 @@
//! Implementation of [RDF XML](https://www.w3.org/TR/rdf-syntax-grammar/) syntax //! Implementation of [RDF XML](https://www.w3.org/TR/rdf-syntax-grammar/) syntax
use crate::model::vocab::rdf; use crate::model::Triple;
use crate::model::*; use crate::rio::rio::convert_triple;
use crate::Result; use crate::Result;
use failure::format_err; use rio_api::parser::TripleParser;
use lazy_static::lazy_static; use rio_xml::RdfXmlParser;
use quick_xml::events::BytesEnd;
use quick_xml::events::BytesStart;
use quick_xml::events::BytesText;
use quick_xml::events::Event;
use quick_xml::Reader;
use std::collections::BTreeMap; use std::collections::BTreeMap;
use std::io::BufRead; use std::io::BufRead;
use std::str::FromStr;
use url::Url; use url::Url;
/// Reads a [RDF XML](https://www.w3.org/TR/rdf-syntax-grammar/) file from a Rust `Read` and returns an iterator on the read `Triple`s /// Reads a [RDF XML](https://www.w3.org/TR/rdf-syntax-grammar/) file from a Rust `BufRead` and returns an iterator of the read `Triple`s
/// pub fn read_rdf_xml<R: BufRead>(
/// Warning: The `rdf:parseType="Literal"` and `rdf:parseType="Collection"` options are not supported yet reader: R,
pub fn read_rdf_xml( base_url: Option<Url>,
source: impl BufRead, ) -> Result<impl Iterator<Item = Result<Triple>>> {
base_uri: impl Into<Option<Url>>, let mut bnode_map = BTreeMap::default();
) -> impl Iterator<Item = Result<Triple>> { Ok(
let mut reader = Reader::from_reader(source); RdfXmlParser::new(reader, base_url.as_ref().map_or("", |url| url.as_str()))?
reader.expand_empty_elements(true); .into_iter(move |t| convert_triple(t, &mut bnode_map)),
reader.trim_text(true);
RdfXmlIterator {
reader,
namespace_buffer: Vec::default(),
state: vec![RdfXmlState::Doc {
base_uri: base_uri.into(),
}],
object: None,
bnodes_map: BTreeMap::default(),
triples_cache: Vec::default(),
li_counter: Vec::default(),
}
}
lazy_static! {
static ref RDF_ABOUT: Url =
Url::from_str("http://www.w3.org/1999/02/22-rdf-syntax-ns#about").unwrap();
static ref RDF_DATATYPE: Url =
Url::from_str("http://www.w3.org/1999/02/22-rdf-syntax-ns#datatype").unwrap();
static ref RDF_DESCRIPTION: NamedNode =
NamedNode::from_str("http://www.w3.org/1999/02/22-rdf-syntax-ns#Description").unwrap();
static ref RDF_ID: Url =
Url::from_str("http://www.w3.org/1999/02/22-rdf-syntax-ns#ID").unwrap();
static ref RDF_LI: Url =
Url::from_str("http://www.w3.org/1999/02/22-rdf-syntax-ns#li").unwrap();
static ref RDF_NODE_ID: Url =
Url::from_str("http://www.w3.org/1999/02/22-rdf-syntax-ns#nodeID").unwrap();
static ref RDF_PARSE_TYPE: Url =
Url::from_str("http://www.w3.org/1999/02/22-rdf-syntax-ns#parseType").unwrap();
static ref RDF_RDF: Url =
Url::from_str("http://www.w3.org/1999/02/22-rdf-syntax-ns#RDF").unwrap();
static ref RDF_RESOURCE: Url =
Url::from_str("http://www.w3.org/1999/02/22-rdf-syntax-ns#resource").unwrap();
}
struct RdfXmlIterator<R: BufRead> {
reader: Reader<R>,
namespace_buffer: Vec<u8>,
state: Vec<RdfXmlState>,
object: Option<NodeOrText>,
bnodes_map: BTreeMap<Vec<u8>, BlankNode>,
triples_cache: Vec<Triple>,
li_counter: Vec<usize>,
}
#[derive(Clone, Debug)]
enum NodeOrText {
Node(NamedOrBlankNode),
Text(String),
}
enum RdfXmlState {
Doc {
base_uri: Option<Url>,
},
RDF {
base_uri: Option<Url>,
language: Option<LanguageTag>,
},
NodeElt {
base_uri: Option<Url>,
language: Option<LanguageTag>,
subject: NamedOrBlankNode,
},
PropertyElt {
//Resource, Literal or Empty property element
uri: Url,
base_uri: Option<Url>,
language: Option<LanguageTag>,
subject: NamedOrBlankNode,
object: Option<NamedOrBlankNode>,
id_attr: Option<NamedNode>,
datatype_attr: Option<NamedNode>,
},
ParseTypeCollectionPropertyElt {
uri: NamedNode,
base_uri: Option<Url>,
language: Option<LanguageTag>,
subject: NamedOrBlankNode,
id_attr: Option<NamedNode>,
},
//TODO: ParseTypeOtherProperty and ParseTypeLiteralProperty
}
impl RdfXmlState {
fn base_uri(&self) -> &Option<Url> {
match self {
RdfXmlState::Doc { base_uri, .. } => base_uri,
RdfXmlState::RDF { base_uri, .. } => base_uri,
RdfXmlState::NodeElt { base_uri, .. } => base_uri,
RdfXmlState::PropertyElt { base_uri, .. } => base_uri,
RdfXmlState::ParseTypeCollectionPropertyElt { base_uri, .. } => base_uri,
}
}
fn language(&self) -> Option<&LanguageTag> {
match self {
RdfXmlState::Doc { .. } => None,
RdfXmlState::RDF { language, .. } => language.as_ref(),
RdfXmlState::NodeElt { language, .. } => language.as_ref(),
RdfXmlState::PropertyElt { language, .. } => language.as_ref(),
RdfXmlState::ParseTypeCollectionPropertyElt { language, .. } => language.as_ref(),
}
}
}
impl<R: BufRead> Iterator for RdfXmlIterator<R> {
type Item = Result<Triple>;
fn next(&mut self) -> Option<Result<Triple>> {
let mut buffer = Vec::default();
loop {
//Finish the stack
if let Some(triple) = self.triples_cache.pop() {
return Some(Ok(triple));
}
//Read more XML
match self
.reader
.read_namespaced_event(&mut buffer, &mut self.namespace_buffer)
{
Ok((_, event)) => match event {
Event::Start(event) => {
if let Err(error) = self.parse_start_event(&event) {
return Some(Err(error));
}
}
Event::Text(event) => {
if let Err(error) = self.parse_text_event(&event) {
return Some(Err(error));
}
}
Event::End(event) => {
if let Err(error) = self.parse_end_event(&event) {
return Some(Err(error));
}
}
Event::Eof => return None,
_ => (),
},
Err(error) => return Some(Err(error.into())),
}
}
}
}
impl<R: BufRead> RdfXmlIterator<R> {
fn parse_start_event(&mut self, event: &BytesStart<'_>) -> Result<()> {
#[derive(PartialEq, Eq)]
enum RdfXmlParseType {
Default,
Collection,
Literal,
Resource,
Other,
}
#[derive(PartialEq, Eq)]
enum RdfXmlNextProduction {
RDF,
NodeElt,
PropertyElt { subject: NamedOrBlankNode },
}
let uri = self.resolve_tag_name(event.name())?;
//We read attributes
let mut language = None;
let mut base_uri = None;
if let Some(current_state) = self.state.last() {
language = current_state.language().cloned();
base_uri = current_state.base_uri().clone();
}
let mut id_attr = None;
let mut node_id_attr = None;
let mut about_attr = None;
let mut property_attrs = Vec::default();
let mut resource_attr = None;
let mut datatype_attr = None;
let mut parse_type = RdfXmlParseType::Default;
let mut type_attr = None;
for attribute in event.attributes() {
let attribute = attribute?;
match attribute.key {
b"xml:lang" => {
language = Some(LanguageTag::parse(
&attribute.unescape_and_decode_value(&self.reader)?,
)?);
}
b"xml:base" => {
base_uri = Some(self.resolve_uri(&attribute.unescaped_value()?, &None)?)
}
key if !key.starts_with(b"xml") => {
let attribute_url = self.resolve_attribute_name(key)?;
if attribute_url == *RDF_ID {
let mut id = Vec::with_capacity(attribute.value.len() + 1);
id.push(b'#');
id.extend_from_slice(attribute.unescaped_value()?.as_ref());
id_attr = Some(id);
} else if attribute_url == *RDF_NODE_ID {
node_id_attr = Some(
self.bnodes_map
.entry(attribute.unescaped_value()?.to_vec())
.or_insert_with(BlankNode::default)
.clone(),
);
} else if attribute_url == *RDF_ABOUT {
about_attr = Some(attribute.unescaped_value()?.to_vec());
} else if attribute_url == *RDF_RESOURCE {
resource_attr = Some(attribute.unescaped_value()?.to_vec());
} else if attribute_url == *RDF_DATATYPE {
datatype_attr = Some(attribute.unescaped_value()?.to_vec());
} else if attribute_url == *RDF_PARSE_TYPE {
parse_type = match attribute.value.as_ref() {
b"Collection" => RdfXmlParseType::Collection,
b"Literal" => RdfXmlParseType::Literal,
b"Resource" => RdfXmlParseType::Resource,
_ => RdfXmlParseType::Other,
};
} else if attribute_url == *rdf::TYPE.as_url() {
type_attr = Some(attribute.unescaped_value()?.to_vec());
} else {
property_attrs.push((
NamedNode::from(attribute_url),
attribute.unescape_and_decode_value(&self.reader)?,
));
}
}
_ => (), //We do not fail for unknown tags in the XML namespace
}
}
//Parsing with the base URI
let id_attr = match id_attr {
Some(uri) => Some(NamedNode::from(self.resolve_uri(&uri, &base_uri)?)),
None => None,
};
let about_attr = match about_attr {
Some(uri) => Some(NamedNode::from(self.resolve_uri(&uri, &base_uri)?)),
None => None,
};
let resource_attr = match resource_attr {
Some(uri) => Some(NamedNode::from(self.resolve_uri(&uri, &base_uri)?)),
None => None,
};
let datatype_attr = match datatype_attr {
Some(uri) => Some(NamedNode::from(self.resolve_uri(&uri, &base_uri)?)),
None => None,
};
let type_attr = match type_attr {
Some(uri) => Some(NamedNode::from(self.resolve_uri(&uri, &base_uri)?)),
None => None,
};
let next_production = match self.state.last() {
Some(RdfXmlState::Doc { .. }) => RdfXmlNextProduction::RDF,
Some(RdfXmlState::RDF { .. }) => RdfXmlNextProduction::NodeElt,
Some(RdfXmlState::NodeElt { subject, .. }) => RdfXmlNextProduction::PropertyElt {
subject: subject.clone(),
},
Some(RdfXmlState::PropertyElt { .. }) => RdfXmlNextProduction::NodeElt {},
Some(RdfXmlState::ParseTypeCollectionPropertyElt { .. }) => {
RdfXmlNextProduction::NodeElt {}
}
None => {
return Err(format_err!(
"No state in the stack: the XML is not balanced"
));
}
};
let new_state = match next_production {
RdfXmlNextProduction::RDF => {
if uri == *RDF_RDF {
RdfXmlState::RDF { base_uri, language }
} else {
self.build_node_elt(
NamedNode::from(uri),
base_uri,
language,
id_attr,
node_id_attr,
about_attr,
type_attr,
property_attrs,
) )
} }
}
RdfXmlNextProduction::NodeElt => self.build_node_elt(
NamedNode::from(uri),
base_uri,
language,
id_attr,
node_id_attr,
about_attr,
type_attr,
property_attrs,
),
RdfXmlNextProduction::PropertyElt { subject } => match parse_type {
RdfXmlParseType::Default => {
if resource_attr.is_some()
|| node_id_attr.is_some()
|| !property_attrs.is_empty()
{
let object: NamedOrBlankNode = match resource_attr {
Some(resource_attr) => resource_attr.into(),
None => match node_id_attr {
Some(node_id_attr) => node_id_attr.into(),
None => BlankNode::default().into(),
},
};
self.emit_property_attrs(&object, property_attrs, &language);
if let Some(type_attr) = type_attr {
self.triples_cache.push(Triple::new(
object.clone(),
rdf::TYPE.clone(),
type_attr,
));
}
RdfXmlState::PropertyElt {
uri,
base_uri,
language,
subject,
object: Some(object),
id_attr,
datatype_attr,
}
} else {
RdfXmlState::PropertyElt {
uri,
base_uri,
language,
subject,
object: None,
id_attr,
datatype_attr,
}
}
}
RdfXmlParseType::Literal => {
return Err(format_err!(
"rdf:parseType=\"Literal\" is not supported yet"
));
}
RdfXmlParseType::Resource => self.build_parse_type_resource_property_elt(
NamedNode::from(uri),
base_uri,
language,
subject,
id_attr,
),
RdfXmlParseType::Collection => {
return Err(format_err!(
"rdf:parseType=\"Collection\" is not supported yet"
));
}
RdfXmlParseType::Other => {
return Err(format_err!("Arbitrary rdf:parseType are not supported yet"));
}
},
};
self.state.push(new_state);
Ok(())
}
fn parse_end_event(&mut self, _event: &BytesEnd<'_>) -> Result<()> {
if let Some(current_state) = self.state.pop() {
self.end_state(current_state)?;
}
Ok(())
}
fn parse_text_event(&mut self, event: &BytesText<'_>) -> Result<()> {
self.object = Some(NodeOrText::Text(event.unescape_and_decode(&self.reader)?));
Ok(())
}
fn resolve_tag_name(&self, qname: &[u8]) -> Result<Url> {
let (namespace, local_name) = self.reader.event_namespace(qname, &self.namespace_buffer);
self.resolve_ns_name(namespace, local_name)
}
fn resolve_attribute_name(&self, qname: &[u8]) -> Result<Url> {
let (namespace, local_name) = self
.reader
.attribute_namespace(qname, &self.namespace_buffer);
self.resolve_ns_name(namespace, local_name)
}
fn resolve_ns_name(&self, namespace: Option<&[u8]>, local_name: &[u8]) -> Result<Url> {
Ok(Url::parse(
&(match namespace {
Some(namespace) => self.reader.decode(namespace) + self.reader.decode(local_name),
None => self.reader.decode(local_name),
}),
)?)
}
fn resolve_uri(&self, uri: &[u8], base: &Option<Url>) -> Result<Url> {
Ok(Url::options()
.base_url(base.as_ref())
.parse(&self.reader.decode(uri))?)
}
fn build_node_elt(
&mut self,
uri: NamedNode,
base_uri: Option<Url>,
language: Option<LanguageTag>,
id_attr: Option<NamedNode>,
node_id_attr: Option<BlankNode>,
about_attr: Option<NamedNode>,
type_attr: Option<NamedNode>,
property_attrs: Vec<(NamedNode, String)>,
) -> RdfXmlState {
self.object = None; //We reset object return: we are in a list of elements
let subject = match id_attr {
Some(id_attr) => id_attr.into(),
None => match about_attr {
Some(about_attr) => about_attr.into(),
None => node_id_attr.unwrap_or_else(BlankNode::default).into(),
},
};
self.emit_property_attrs(&subject, property_attrs, &language);
if let Some(type_attr) = type_attr {
self.triples_cache
.push(Triple::new(subject.clone(), rdf::TYPE.clone(), type_attr));
}
if uri != *RDF_DESCRIPTION {
self.triples_cache
.push(Triple::new(subject.clone(), rdf::TYPE.clone(), uri));
}
self.li_counter.push(0);
RdfXmlState::NodeElt {
base_uri,
language,
subject: subject.clone(),
}
}
fn build_parse_type_resource_property_elt(
&mut self,
uri: NamedNode,
base_uri: Option<Url>,
language: Option<LanguageTag>,
subject: NamedOrBlankNode,
id_attr: Option<NamedNode>,
) -> RdfXmlState {
let object = BlankNode::default();
let triple = Triple::new(subject, uri, object.clone());
if let Some(id_attr) = id_attr {
self.reify(&triple, id_attr.into());
}
self.triples_cache.push(triple);
self.li_counter.push(0);
RdfXmlState::NodeElt {
base_uri,
language,
subject: object.into(),
}
}
fn end_state(&mut self, state: RdfXmlState) -> Result<()> {
match state {
RdfXmlState::PropertyElt {
uri,
language,
subject,
id_attr,
datatype_attr,
object,
..
} => {
let predicate = if uri == *RDF_LI {
if let Some(li_counter) = self.li_counter.last_mut() {
*li_counter += 1;
NamedNode::from_str(&format!(
"http://www.w3.org/1999/02/22-rdf-syntax-ns#_{}",
li_counter
))?
} else {
NamedNode::from(uri)
}
} else {
NamedNode::from(uri)
};
let object: Term = match object {
Some(object) => object.into(),
None => match self.object.clone() {
Some(NodeOrText::Node(node)) => node.into(),
Some(NodeOrText::Text(text)) => {
self.new_literal(text, language, datatype_attr).into()
}
None => self
.new_literal(String::default(), language, datatype_attr)
.into(),
},
};
self.object = None; //We have used self.object
let triple = Triple::new(subject, predicate, object);
if let Some(id_attr) = id_attr {
self.reify(&triple, id_attr.into());
}
self.triples_cache.push(triple);
}
RdfXmlState::NodeElt { subject, .. } => {
self.object = Some(NodeOrText::Node(subject));
self.li_counter.pop();
}
_ => (),
}
Ok(())
}
fn new_literal(
&self,
text: String,
language: Option<LanguageTag>,
datatype: Option<NamedNode>,
) -> Literal {
if let Some(datatype) = datatype {
Literal::new_typed_literal(text, datatype)
} else if let Some(language) = language {
Literal::new_language_tagged_literal(text, language)
} else {
Literal::new_simple_literal(text)
}
}
fn reify(&mut self, triple: &Triple, statement_id: NamedOrBlankNode) {
self.triples_cache.push(Triple::new(
statement_id.clone(),
rdf::OBJECT.clone(),
triple.object().clone(),
));
self.triples_cache.push(Triple::new(
statement_id.clone(),
rdf::PREDICATE.clone(),
triple.predicate().clone(),
));
self.triples_cache.push(Triple::new(
statement_id.clone(),
rdf::SUBJECT.clone(),
triple.subject().clone(),
));
self.triples_cache.push(Triple::new(
statement_id,
rdf::TYPE.clone(),
rdf::STATEMENT.clone(),
));
}
fn emit_property_attrs(
&mut self,
subject: &NamedOrBlankNode,
literal_attributes: Vec<(NamedNode, String)>,
language: &Option<LanguageTag>,
) {
for (literal_predicate, literal_value) in literal_attributes {
self.triples_cache.push(Triple::new(
subject.clone(),
literal_predicate,
if let Some(language) = language {
Literal::new_language_tagged_literal(literal_value, language.clone())
} else {
Literal::new_simple_literal(literal_value)
},
));
}
}
}

15
lib/tests/rdf_test_cases.rs
Unescape View File

@ -95,25 +95,16 @@ fn ntriples_w3c_testsuite() {
#[test] #[test]
fn rdf_xml_w3c_testsuite() -> Result<()> { fn rdf_xml_w3c_testsuite() -> Result<()> {
let manifest_url = Url::parse("http://www.w3.org/2013/RDFXMLTests/manifest.ttl")?; let manifest_url = Url::parse("http://www.w3.org/2013/RDFXMLTests/manifest.ttl")?;
//TODO: make blacklist pass
let test_blacklist = vec![
NamedNode::new(manifest_url.join("#xml-canon-test001")?),
NamedNode::new(manifest_url.join("#rdfms-seq-representation-test001")?),
NamedNode::new(manifest_url.join("#rdf-containers-syntax-vs-schema-test004")?),
];
for test_result in TestManifest::new(manifest_url) { for test_result in TestManifest::new(manifest_url) {
let test = test_result?; let test = test_result?;
if test_blacklist.contains(&test.id) {
continue;
}
if test.kind == "TestXMLNegativeSyntax" { if test.kind == "TestXMLNegativeSyntax" {
/*TODO assert!( assert!(
load_rdf_xml(test.action.clone()).is_err(), load_rdf_xml(test.action.clone()).is_err(),
"Failure on {}", "Failure on {}",
test test
);*/ );
} else if test.kind == "TestXMLEval" { } else if test.kind == "TestXMLEval" {
match load_rdf_xml(test.action.clone()) { match load_rdf_xml(test.action.clone()) {
Ok(action_graph) => match load_ntriples(test.result.clone().unwrap()) { Ok(action_graph) => match load_ntriples(test.result.clone().unwrap()) {
@ -150,7 +141,7 @@ fn load_ntriples(url: Url) -> Result<MemoryGraph> {
} }
fn load_rdf_xml(url: Url) -> Result<MemoryGraph> { fn load_rdf_xml(url: Url) -> Result<MemoryGraph> {
read_rdf_xml(read_file(&url)?, Some(url)).collect() read_rdf_xml(read_file(&url)?, Some(url))?.collect()
} }
fn to_relative_path(url: &Url) -> Result<String> { fn to_relative_path(url: &Url) -> Result<String> {

2
lib/tests/sparql_test_cases.rs
Unescape View File

@ -243,7 +243,7 @@ fn load_graph(url: Url) -> Result<MemoryGraph> {
if url.as_str().ends_with(".ttl") { if url.as_str().ends_with(".ttl") {
read_turtle(read_file(&url)?, Some(url))?.collect() read_turtle(read_file(&url)?, Some(url))?.collect()
} else if url.as_str().ends_with(".rdf") { } else if url.as_str().ends_with(".rdf") {
read_rdf_xml(read_file(&url)?, Some(url)).collect() read_rdf_xml(read_file(&url)?, Some(url))?.collect()
} else { } else {
Err(format_err!("Serialization type not found for {}", url)) Err(format_err!("Serialization type not found for {}", url))
} }

Write Preview
Loading…
Cancel
Save