Adopt new I/O API for parsing

pull/592/head
Tpt 1 year ago committed by Thomas Tanon
parent 217abaf7ee
commit f183196859
  1. 8
      fuzz/fuzz_targets/sparql_eval.rs
  2. 46
      js/src/store.rs
  3. 2
      js/test/store.mjs
  4. 18
      lib/benches/store.rs
  5. 134
      lib/src/io/error.rs
  6. 4
      lib/src/io/format.rs
  7. 3
      lib/src/io/mod.rs
  8. 7
      lib/src/io/read.rs
  9. 1
      lib/src/lib.rs
  10. 4
      lib/src/sparql/model.rs
  11. 10
      lib/src/storage/error.rs
  12. 162
      lib/src/store.rs
  13. 20
      lib/tests/store.rs
  14. 68
      python/src/io.rs
  15. 76
      python/src/store.rs
  16. 35
      python/tests/test_io.py
  17. 144
      server/src/main.rs
  18. 30
      testsuite/src/files.rs
  19. 4
      testsuite/src/manifest.rs
  20. 97
      testsuite/src/parser_evaluator.rs
  21. 16
      testsuite/src/report.rs
  22. 25
      testsuite/src/sparql_evaluator.rs

@ -2,7 +2,7 @@
use lazy_static::lazy_static; use lazy_static::lazy_static;
use libfuzzer_sys::fuzz_target; use libfuzzer_sys::fuzz_target;
use oxigraph::io::DatasetFormat; use oxigraph::io::RdfFormat;
use oxigraph::sparql::{Query, QueryOptions, QueryResults, QuerySolutionIter}; use oxigraph::sparql::{Query, QueryOptions, QueryResults, QuerySolutionIter};
use oxigraph::store::Store; use oxigraph::store::Store;
@ -10,11 +10,7 @@ lazy_static! {
static ref STORE: Store = { static ref STORE: Store = {
let store = Store::new().unwrap(); let store = Store::new().unwrap();
store store
.load_dataset( .load_dataset(sparql_smith::DATA_TRIG.as_bytes(), RdfFormat::TriG, None)
sparql_smith::DATA_TRIG.as_bytes(),
DatasetFormat::TriG,
None,
)
.unwrap(); .unwrap();
store store
}; };

@ -4,7 +4,7 @@ use crate::format_err;
use crate::model::*; use crate::model::*;
use crate::utils::to_err; use crate::utils::to_err;
use js_sys::{Array, Map}; use js_sys::{Array, Map};
use oxigraph::io::{DatasetFormat, GraphFormat, RdfFormat}; use oxigraph::io::RdfFormat;
use oxigraph::model::*; use oxigraph::model::*;
use oxigraph::sparql::QueryResults; use oxigraph::sparql::QueryResults;
use oxigraph::store::Store; use oxigraph::store::Store;
@ -148,6 +148,9 @@ impl JsStore {
base_iri: &JsValue, base_iri: &JsValue,
to_graph_name: &JsValue, to_graph_name: &JsValue,
) -> Result<(), JsValue> { ) -> Result<(), JsValue> {
let Some(format) = RdfFormat::from_media_type(mime_type) else {
return Err(format_err!("Not supported MIME type: {mime_type}"));
};
let base_iri = if base_iri.is_null() || base_iri.is_undefined() { let base_iri = if base_iri.is_null() || base_iri.is_undefined() {
None None
} else if base_iri.is_string() { } else if base_iri.is_string() {
@ -160,49 +163,28 @@ impl JsStore {
)); ));
}; };
let to_graph_name = if let Some(to_graph_name) = FROM_JS.with(|c| c.to_optional_term(to_graph_name))? {
if let Some(graph_name) = FROM_JS.with(|c| c.to_optional_term(to_graph_name))? { self.store.load_graph(
Some(graph_name.try_into()?)
} else {
None
};
if let Some(graph_format) = GraphFormat::from_media_type(mime_type) {
self.store
.load_graph(
data.as_bytes(), data.as_bytes(),
graph_format, format,
&to_graph_name.unwrap_or(GraphName::DefaultGraph), GraphName::try_from(to_graph_name)?,
base_iri.as_deref(), base_iri.as_deref(),
) )
.map_err(to_err)
} else if let Some(dataset_format) = DatasetFormat::from_media_type(mime_type) {
if to_graph_name.is_some() {
return Err(format_err!(
"The target graph name parameter is not available for dataset formats"
));
}
self.store
.load_dataset(data.as_bytes(), dataset_format, base_iri.as_deref())
.map_err(to_err)
} else { } else {
Err(format_err!("Not supported MIME type: {mime_type}")) self.store
.load_dataset(data.as_bytes(), format, base_iri.as_deref())
} }
.map_err(to_err)
} }
pub fn dump(&self, mime_type: &str, from_graph_name: &JsValue) -> Result<String, JsValue> { pub fn dump(&self, mime_type: &str, from_graph_name: &JsValue) -> Result<String, JsValue> {
let Some(format) = RdfFormat::from_media_type(mime_type) else { let Some(format) = RdfFormat::from_media_type(mime_type) else {
return Err(format_err!("Not supported MIME type: {mime_type}")); return Err(format_err!("Not supported MIME type: {mime_type}"));
}; };
let from_graph_name =
if let Some(graph_name) = FROM_JS.with(|c| c.to_optional_term(from_graph_name))? {
Some(GraphName::try_from(graph_name)?)
} else {
None
};
let mut buffer = Vec::new(); let mut buffer = Vec::new();
if let Some(from_graph_name) = &from_graph_name { if let Some(from_graph_name) = FROM_JS.with(|c| c.to_optional_term(from_graph_name))? {
self.store.dump_graph(&mut buffer, format, from_graph_name) self.store
.dump_graph(&mut buffer, format, &GraphName::try_from(from_graph_name)?)
} else { } else {
self.store.dump_dataset(&mut buffer, format) self.store.dump_dataset(&mut buffer, format)
} }

@ -186,7 +186,7 @@ describe("Store", function () {
it("dump default graph content", function () { it("dump default graph content", function () {
const store = new Store([dataModel.quad(ex, ex, ex, ex)]); const store = new Store([dataModel.quad(ex, ex, ex, ex)]);
assert.strictEqual("", store.dump("application/n-triples")); assert.strictEqual("", store.dump("application/n-triples", dataModel.defaultGraph()));
}); });
}); });
}); });

@ -1,7 +1,7 @@
use criterion::{criterion_group, criterion_main, Criterion, Throughput}; use criterion::{criterion_group, criterion_main, Criterion, Throughput};
use oxhttp::model::{Method, Request, Status}; use oxhttp::model::{Method, Request, Status};
use oxigraph::io::GraphFormat; use oxigraph::io::RdfFormat;
use oxigraph::model::GraphNameRef; use oxigraph::model::{GraphName, GraphNameRef};
use oxigraph::sparql::{Query, QueryResults, Update}; use oxigraph::sparql::{Query, QueryResults, Update};
use oxigraph::store::Store; use oxigraph::store::Store;
use rand::random; use rand::random;
@ -63,12 +63,7 @@ fn store_load(c: &mut Criterion) {
fn do_load(store: &Store, data: &[u8]) { fn do_load(store: &Store, data: &[u8]) {
store store
.load_graph( .load_graph(data, RdfFormat::NTriples, GraphName::DefaultGraph, None)
data,
GraphFormat::NTriples,
GraphNameRef::DefaultGraph,
None,
)
.unwrap(); .unwrap();
store.optimize().unwrap(); store.optimize().unwrap();
} }
@ -76,12 +71,7 @@ fn do_load(store: &Store, data: &[u8]) {
fn do_bulk_load(store: &Store, data: &[u8]) { fn do_bulk_load(store: &Store, data: &[u8]) {
store store
.bulk_loader() .bulk_loader()
.load_graph( .load_graph(data, RdfFormat::NTriples, GraphNameRef::DefaultGraph, None)
data,
GraphFormat::NTriples,
GraphNameRef::DefaultGraph,
None,
)
.unwrap(); .unwrap();
store.optimize().unwrap(); store.optimize().unwrap();
} }

@ -1,134 +0,0 @@
use oxiri::IriParseError;
use std::error::Error;
use std::{fmt, io};
/// Error returned during RDF format parsing.
#[derive(Debug)]
pub enum ParseError {
/// I/O error during parsing (file not found...).
Io(io::Error),
/// An error in the file syntax.
Syntax(SyntaxError),
}
impl ParseError {
#[inline]
pub(crate) fn invalid_base_iri(iri: &str, error: IriParseError) -> Self {
Self::Syntax(SyntaxError {
inner: SyntaxErrorKind::InvalidBaseIri {
iri: iri.to_owned(),
error,
},
})
}
}
impl fmt::Display for ParseError {
#[inline]
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Io(e) => e.fmt(f),
Self::Syntax(e) => e.fmt(f),
}
}
}
impl Error for ParseError {
#[inline]
fn source(&self) -> Option<&(dyn Error + 'static)> {
match self {
Self::Io(e) => Some(e),
Self::Syntax(e) => Some(e),
}
}
}
impl From<oxrdfio::SyntaxError> for SyntaxError {
#[inline]
fn from(error: oxrdfio::SyntaxError) -> Self {
SyntaxError {
inner: SyntaxErrorKind::IO(error),
}
}
}
impl From<oxrdfio::ParseError> for ParseError {
#[inline]
fn from(error: oxrdfio::ParseError) -> Self {
match error {
oxrdfio::ParseError::Syntax(e) => Self::Syntax(e.into()),
oxrdfio::ParseError::Io(e) => Self::Io(e),
}
}
}
impl From<io::Error> for ParseError {
#[inline]
fn from(error: io::Error) -> Self {
Self::Io(error)
}
}
impl From<SyntaxError> for ParseError {
#[inline]
fn from(error: SyntaxError) -> Self {
Self::Syntax(error)
}
}
impl From<ParseError> for io::Error {
#[inline]
fn from(error: ParseError) -> Self {
match error {
ParseError::Io(error) => error,
ParseError::Syntax(error) => error.into(),
}
}
}
/// An error in the syntax of the parsed file.
#[derive(Debug)]
pub struct SyntaxError {
inner: SyntaxErrorKind,
}
#[derive(Debug)]
enum SyntaxErrorKind {
IO(oxrdfio::SyntaxError),
InvalidBaseIri { iri: String, error: IriParseError },
}
impl fmt::Display for SyntaxError {
#[inline]
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match &self.inner {
SyntaxErrorKind::IO(e) => e.fmt(f),
SyntaxErrorKind::InvalidBaseIri { iri, error } => {
write!(f, "Invalid base IRI '{iri}': {error}")
}
}
}
}
impl Error for SyntaxError {
#[inline]
fn source(&self) -> Option<&(dyn Error + 'static)> {
match &self.inner {
SyntaxErrorKind::IO(e) => Some(e),
SyntaxErrorKind::InvalidBaseIri { .. } => None,
}
}
}
impl From<SyntaxError> for io::Error {
#[inline]
fn from(error: SyntaxError) -> Self {
match error.inner {
SyntaxErrorKind::IO(error) => error.into(),
SyntaxErrorKind::InvalidBaseIri { iri, error } => Self::new(
io::ErrorKind::InvalidInput,
format!("Invalid IRI '{iri}': {error}"),
),
}
}
}

@ -1,3 +1,5 @@
#![allow(deprecated)]
use oxrdfio::RdfFormat; use oxrdfio::RdfFormat;
/// [RDF graph](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-graph) serialization formats. /// [RDF graph](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-graph) serialization formats.
@ -5,6 +7,7 @@ use oxrdfio::RdfFormat;
/// This enumeration is non exhaustive. New formats like JSON-LD will be added in the future. /// This enumeration is non exhaustive. New formats like JSON-LD will be added in the future.
#[derive(Eq, PartialEq, Debug, Clone, Copy, Hash)] #[derive(Eq, PartialEq, Debug, Clone, Copy, Hash)]
#[non_exhaustive] #[non_exhaustive]
#[deprecated(note = "Use RdfFormat instead")]
pub enum GraphFormat { pub enum GraphFormat {
/// [N-Triples](https://www.w3.org/TR/n-triples/) /// [N-Triples](https://www.w3.org/TR/n-triples/)
NTriples, NTriples,
@ -119,6 +122,7 @@ impl From<GraphFormat> for RdfFormat {
/// This enumeration is non exhaustive. New formats like JSON-LD will be added in the future. /// This enumeration is non exhaustive. New formats like JSON-LD will be added in the future.
#[derive(Eq, PartialEq, Debug, Clone, Copy, Hash)] #[derive(Eq, PartialEq, Debug, Clone, Copy, Hash)]
#[non_exhaustive] #[non_exhaustive]
#[deprecated(note = "Use RdfFormat instead")]
pub enum DatasetFormat { pub enum DatasetFormat {
/// [N-Quads](https://www.w3.org/TR/n-quads/) /// [N-Quads](https://www.w3.org/TR/n-quads/)
NQuads, NQuads,

@ -1,11 +1,12 @@
//! Utilities to read and write RDF graphs and datasets. //! Utilities to read and write RDF graphs and datasets.
mod error;
mod format; mod format;
pub mod read; pub mod read;
pub mod write; pub mod write;
#[allow(deprecated)]
pub use self::format::{DatasetFormat, GraphFormat}; pub use self::format::{DatasetFormat, GraphFormat};
#[allow(deprecated)]
pub use self::read::{DatasetParser, GraphParser}; pub use self::read::{DatasetParser, GraphParser};
#[allow(deprecated)] #[allow(deprecated)]
pub use self::write::{DatasetSerializer, GraphSerializer}; pub use self::write::{DatasetSerializer, GraphSerializer};

@ -1,10 +1,11 @@
#![allow(deprecated)]
//! Utilities to read RDF graphs and datasets. //! Utilities to read RDF graphs and datasets.
pub use crate::io::error::{ParseError, SyntaxError};
use crate::io::{DatasetFormat, GraphFormat}; use crate::io::{DatasetFormat, GraphFormat};
use crate::model::*; use crate::model::*;
use oxiri::IriParseError; use oxiri::IriParseError;
use oxrdfio::{FromReadQuadReader, RdfParser}; use oxrdfio::{FromReadQuadReader, ParseError, RdfParser};
use std::io::Read; use std::io::Read;
/// Parsers for RDF graph serialization formats. /// Parsers for RDF graph serialization formats.
@ -26,6 +27,7 @@ use std::io::Read;
/// assert_eq!(triples[0].subject.to_string(), "<http://example.com/s>"); /// assert_eq!(triples[0].subject.to_string(), "<http://example.com/s>");
/// # std::io::Result::Ok(()) /// # std::io::Result::Ok(())
/// ``` /// ```
#[deprecated(note = "Use RdfParser instead")]
pub struct GraphParser { pub struct GraphParser {
inner: RdfParser, inner: RdfParser,
} }
@ -116,6 +118,7 @@ impl<R: Read> Iterator for TripleReader<R> {
/// assert_eq!(quads[0].subject.to_string(), "<http://example.com/s>"); /// assert_eq!(quads[0].subject.to_string(), "<http://example.com/s>");
/// # std::io::Result::Ok(()) /// # std::io::Result::Ok(())
/// ``` /// ```
#[deprecated(note = "Use RdfParser instead")]
pub struct DatasetParser { pub struct DatasetParser {
inner: RdfParser, inner: RdfParser,
} }

@ -1,5 +1,6 @@
#![doc = include_str!("../README.md")] #![doc = include_str!("../README.md")]
#![doc(test(attr(deny(warnings))))] #![doc(test(attr(deny(warnings))))]
#![doc(test(attr(allow(deprecated))))]
#![cfg_attr(docsrs, feature(doc_auto_cfg))] #![cfg_attr(docsrs, feature(doc_auto_cfg))]
#![doc(html_favicon_url = "https://raw.githubusercontent.com/oxigraph/oxigraph/main/logo.svg")] #![doc(html_favicon_url = "https://raw.githubusercontent.com/oxigraph/oxigraph/main/logo.svg")]
#![doc(html_logo_url = "https://raw.githubusercontent.com/oxigraph/oxigraph/main/logo.svg")] #![doc(html_logo_url = "https://raw.githubusercontent.com/oxigraph/oxigraph/main/logo.svg")]

@ -95,13 +95,13 @@ impl QueryResults {
/// ///
/// ``` /// ```
/// use oxigraph::store::Store; /// use oxigraph::store::Store;
/// use oxigraph::io::{RdfFormat, GraphFormat}; /// use oxigraph::io::RdfFormat;
/// use oxigraph::model::*; /// use oxigraph::model::*;
/// ///
/// let graph = "<http://example.com> <http://example.com> <http://example.com> .\n"; /// let graph = "<http://example.com> <http://example.com> <http://example.com> .\n";
/// ///
/// let store = Store::new()?; /// let store = Store::new()?;
/// store.load_graph(graph.as_bytes(), GraphFormat::NTriples, GraphNameRef::DefaultGraph, None)?; /// store.load_graph(graph.as_bytes(), RdfFormat::NTriples, GraphName::DefaultGraph, None)?;
/// ///
/// let mut results = Vec::new(); /// let mut results = Vec::new();
/// store.query("CONSTRUCT WHERE { ?s ?p ?o }")?.write_graph(&mut results, RdfFormat::NTriples)?; /// store.query("CONSTRUCT WHERE { ?s ?p ?o }")?.write_graph(&mut results, RdfFormat::NTriples)?;

@ -1,4 +1,5 @@
use crate::io::{read::ParseError, RdfFormat}; use crate::io::{ParseError, RdfFormat};
use oxiri::IriParseError;
use std::error::Error; use std::error::Error;
use std::fmt; use std::fmt;
use std::io; use std::io;
@ -126,6 +127,8 @@ pub enum LoaderError {
Parsing(ParseError), Parsing(ParseError),
/// An error raised during the insertion in the store. /// An error raised during the insertion in the store.
Storage(StorageError), Storage(StorageError),
/// The base IRI is invalid.
InvalidBaseIri { iri: String, error: IriParseError },
} }
impl fmt::Display for LoaderError { impl fmt::Display for LoaderError {
@ -134,6 +137,7 @@ impl fmt::Display for LoaderError {
match self { match self {
Self::Parsing(e) => e.fmt(f), Self::Parsing(e) => e.fmt(f),
Self::Storage(e) => e.fmt(f), Self::Storage(e) => e.fmt(f),
Self::InvalidBaseIri { iri, error } => write!(f, "Invalid base IRI '{iri}': {error}"),
} }
} }
} }
@ -144,6 +148,7 @@ impl Error for LoaderError {
match self { match self {
Self::Parsing(e) => Some(e), Self::Parsing(e) => Some(e),
Self::Storage(e) => Some(e), Self::Storage(e) => Some(e),
Self::InvalidBaseIri { error, .. } => Some(error),
} }
} }
} }
@ -168,6 +173,9 @@ impl From<LoaderError> for io::Error {
match error { match error {
LoaderError::Storage(error) => error.into(), LoaderError::Storage(error) => error.into(),
LoaderError::Parsing(error) => error.into(), LoaderError::Parsing(error) => error.into(),
LoaderError::InvalidBaseIri { .. } => {
io::Error::new(io::ErrorKind::InvalidInput, error.to_string())
}
} }
} }
} }

@ -23,8 +23,9 @@
//! }; //! };
//! # Result::<_, Box<dyn std::error::Error>>::Ok(()) //! # Result::<_, Box<dyn std::error::Error>>::Ok(())
//! ``` //! ```
use crate::io::read::ParseError; #[cfg(not(target_family = "wasm"))]
use crate::io::{DatasetFormat, DatasetParser, GraphFormat, GraphParser, RdfFormat, RdfSerializer}; use crate::io::ParseError;
use crate::io::{RdfFormat, RdfParser, RdfSerializer};
use crate::model::*; use crate::model::*;
use crate::sparql::{ use crate::sparql::{
evaluate_query, evaluate_update, EvaluationError, Query, QueryExplanation, QueryOptions, evaluate_query, evaluate_update, EvaluationError, Query, QueryExplanation, QueryOptions,
@ -451,38 +452,43 @@ impl Store {
/// Usage example: /// Usage example:
/// ``` /// ```
/// use oxigraph::store::Store; /// use oxigraph::store::Store;
/// use oxigraph::io::GraphFormat; /// use oxigraph::io::RdfFormat;
/// use oxigraph::model::*; /// use oxigraph::model::*;
/// ///
/// let store = Store::new()?; /// let store = Store::new()?;
/// ///
/// // insertion /// // insertion
/// let file = b"<http://example.com> <http://example.com> <http://example.com> ."; /// let file = b"<http://example.com> <http://example.com> <http://example.com> .";
/// store.load_graph(file.as_ref(), GraphFormat::NTriples, GraphNameRef::DefaultGraph, None)?; /// store.load_graph(file.as_ref(), RdfFormat::NTriples, GraphName::DefaultGraph, None)?;
/// ///
/// // we inspect the store contents /// // we inspect the store contents
/// let ex = NamedNodeRef::new("http://example.com")?; /// let ex = NamedNodeRef::new("http://example.com")?;
/// assert!(store.contains(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?); /// assert!(store.contains(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?);
/// # Result::<_, Box<dyn std::error::Error>>::Ok(()) /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
/// ``` /// ```
pub fn load_graph<'a>( pub fn load_graph(
&self, &self,
reader: impl Read, read: impl Read,
format: GraphFormat, format: impl Into<RdfFormat>,
to_graph_name: impl Into<GraphNameRef<'a>>, to_graph_name: impl Into<GraphName>,
base_iri: Option<&str>, base_iri: Option<&str>,
) -> Result<(), LoaderError> { ) -> Result<(), LoaderError> {
let mut parser = GraphParser::from_format(format); let mut parser = RdfParser::from_format(format.into())
.without_named_graphs()
.with_default_graph(to_graph_name)
.rename_blank_nodes();
if let Some(base_iri) = base_iri { if let Some(base_iri) = base_iri {
parser = parser parser = parser
.with_base_iri(base_iri) .with_base_iri(base_iri)
.map_err(|e| ParseError::invalid_base_iri(base_iri, e))?; .map_err(|e| LoaderError::InvalidBaseIri {
iri: base_iri.into(),
error: e,
})?;
} }
let quads = parser.read_triples(reader).collect::<Result<Vec<_>, _>>()?; let quads = parser.parse_read(read).collect::<Result<Vec<_>, _>>()?;
let to_graph_name = to_graph_name.into();
self.storage.transaction(move |mut t| { self.storage.transaction(move |mut t| {
for quad in &quads { for quad in &quads {
t.insert(quad.as_ref().in_graph(to_graph_name))?; t.insert(quad.as_ref())?;
} }
Ok(()) Ok(())
}) })
@ -495,14 +501,14 @@ impl Store {
/// Usage example: /// Usage example:
/// ``` /// ```
/// use oxigraph::store::Store; /// use oxigraph::store::Store;
/// use oxigraph::io::DatasetFormat; /// use oxigraph::io::RdfFormat;
/// use oxigraph::model::*; /// use oxigraph::model::*;
/// ///
/// let store = Store::new()?; /// let store = Store::new()?;
/// ///
/// // insertion /// // insertion
/// let file = b"<http://example.com> <http://example.com> <http://example.com> <http://example.com> ."; /// let file = b"<http://example.com> <http://example.com> <http://example.com> <http://example.com> .";
/// store.load_dataset(file.as_ref(), DatasetFormat::NQuads, None)?; /// store.load_dataset(file.as_ref(), RdfFormat::NQuads, None)?;
/// ///
/// // we inspect the store contents /// // we inspect the store contents
/// let ex = NamedNodeRef::new("http://example.com")?; /// let ex = NamedNodeRef::new("http://example.com")?;
@ -511,20 +517,23 @@ impl Store {
/// ``` /// ```
pub fn load_dataset( pub fn load_dataset(
&self, &self,
reader: impl Read, read: impl Read,
format: DatasetFormat, format: impl Into<RdfFormat>,
base_iri: Option<&str>, base_iri: Option<&str>,
) -> Result<(), LoaderError> { ) -> Result<(), LoaderError> {
let mut parser = DatasetParser::from_format(format); let mut parser = RdfParser::from_format(format.into()).rename_blank_nodes();
if let Some(base_iri) = base_iri { if let Some(base_iri) = base_iri {
parser = parser parser = parser
.with_base_iri(base_iri) .with_base_iri(base_iri)
.map_err(|e| ParseError::invalid_base_iri(base_iri, e))?; .map_err(|e| LoaderError::InvalidBaseIri {
iri: base_iri.into(),
error: e,
})?;
} }
let quads = parser.read_quads(reader).collect::<Result<Vec<_>, _>>()?; let quads = parser.parse_read(read).collect::<Result<Vec<_>, _>>()?;
self.storage.transaction(move |mut t| { self.storage.transaction(move |mut t| {
for quad in &quads { for quad in &quads {
t.insert(quad.into())?; t.insert(quad.as_ref())?;
} }
Ok(()) Ok(())
}) })
@ -595,16 +604,16 @@ impl Store {
/// Usage example: /// Usage example:
/// ``` /// ```
/// use oxigraph::store::Store; /// use oxigraph::store::Store;
/// use oxigraph::io::GraphFormat; /// use oxigraph::io::RdfFormat;
/// use oxigraph::model::GraphNameRef; /// use oxigraph::model::*;
/// ///
/// let file = "<http://example.com> <http://example.com> <http://example.com> .\n".as_bytes(); /// let file = "<http://example.com> <http://example.com> <http://example.com> .\n".as_bytes();
/// ///
/// let store = Store::new()?; /// let store = Store::new()?;
/// store.load_graph(file, GraphFormat::NTriples, GraphNameRef::DefaultGraph, None)?; /// store.load_graph(file, RdfFormat::NTriples, GraphName::DefaultGraph, None)?;
/// ///
/// let mut buffer = Vec::new(); /// let mut buffer = Vec::new();
/// store.dump_graph(&mut buffer, GraphFormat::NTriples, GraphNameRef::DefaultGraph)?; /// store.dump_graph(&mut buffer, RdfFormat::NTriples, GraphNameRef::DefaultGraph)?;
/// assert_eq!(file, buffer.as_slice()); /// assert_eq!(file, buffer.as_slice());
/// # std::io::Result::Ok(()) /// # std::io::Result::Ok(())
/// ``` /// ```
@ -626,15 +635,15 @@ impl Store {
/// ///
/// ``` /// ```
/// use oxigraph::store::Store; /// use oxigraph::store::Store;
/// use oxigraph::io::DatasetFormat; /// use oxigraph::io::RdfFormat;
/// ///
/// let file = "<http://example.com> <http://example.com> <http://example.com> <http://example.com> .\n".as_bytes(); /// let file = "<http://example.com> <http://example.com> <http://example.com> <http://example.com> .\n".as_bytes();
/// ///
/// let store = Store::new()?; /// let store = Store::new()?;
/// store.load_dataset(file, DatasetFormat::NQuads, None)?; /// store.load_dataset(file, RdfFormat::NQuads, None)?;
/// ///
/// let mut buffer = Vec::new(); /// let mut buffer = Vec::new();
/// store.dump_dataset(&mut buffer, DatasetFormat::NQuads)?; /// store.dump_dataset(&mut buffer, RdfFormat::NQuads)?;
/// assert_eq!(file, buffer.as_slice()); /// assert_eq!(file, buffer.as_slice());
/// # std::io::Result::Ok(()) /// # std::io::Result::Ok(())
/// ``` /// ```
@ -841,14 +850,14 @@ impl Store {
/// Usage example: /// Usage example:
/// ``` /// ```
/// use oxigraph::store::Store; /// use oxigraph::store::Store;
/// use oxigraph::io::DatasetFormat; /// use oxigraph::io::RdfFormat;
/// use oxigraph::model::*; /// use oxigraph::model::*;
/// ///
/// let store = Store::new()?; /// let store = Store::new()?;
/// ///
/// // quads file insertion /// // quads file insertion
/// let file = b"<http://example.com> <http://example.com> <http://example.com> <http://example.com> ."; /// let file = b"<http://example.com> <http://example.com> <http://example.com> <http://example.com> .";
/// store.bulk_loader().load_dataset(file.as_ref(), DatasetFormat::NQuads, None)?; /// store.bulk_loader().load_dataset(file.as_ref(), RdfFormat::NQuads, None)?;
/// ///
/// // we inspect the store contents /// // we inspect the store contents
/// let ex = NamedNodeRef::new("http://example.com")?; /// let ex = NamedNodeRef::new("http://example.com")?;
@ -1061,7 +1070,7 @@ impl<'a> Transaction<'a> {
/// Usage example: /// Usage example:
/// ``` /// ```
/// use oxigraph::store::Store; /// use oxigraph::store::Store;
/// use oxigraph::io::GraphFormat; /// use oxigraph::io::RdfFormat;
/// use oxigraph::model::*; /// use oxigraph::model::*;
/// ///
/// let store = Store::new()?; /// let store = Store::new()?;
@ -1069,7 +1078,7 @@ impl<'a> Transaction<'a> {
/// // insertion /// // insertion
/// let file = b"<http://example.com> <http://example.com> <http://example.com> ."; /// let file = b"<http://example.com> <http://example.com> <http://example.com> .";
/// store.transaction(|mut transaction| { /// store.transaction(|mut transaction| {
/// transaction.load_graph(file.as_ref(), GraphFormat::NTriples, GraphNameRef::DefaultGraph, None) /// transaction.load_graph(file.as_ref(), RdfFormat::NTriples, GraphName::DefaultGraph, None)
/// })?; /// })?;
/// ///
/// // we inspect the store contents /// // we inspect the store contents
@ -1077,23 +1086,27 @@ impl<'a> Transaction<'a> {
/// assert!(store.contains(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?); /// assert!(store.contains(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?);
/// # Result::<_,oxigraph::store::LoaderError>::Ok(()) /// # Result::<_,oxigraph::store::LoaderError>::Ok(())
/// ``` /// ```
pub fn load_graph<'b>( pub fn load_graph(
&mut self, &mut self,
reader: impl Read, read: impl Read,
format: GraphFormat, format: impl Into<RdfFormat>,
to_graph_name: impl Into<GraphNameRef<'b>>, to_graph_name: impl Into<GraphName>,
base_iri: Option<&str>, base_iri: Option<&str>,
) -> Result<(), LoaderError> { ) -> Result<(), LoaderError> {
let mut parser = GraphParser::from_format(format); let mut parser = RdfParser::from_format(format.into())
.without_named_graphs()
.with_default_graph(to_graph_name)
.rename_blank_nodes();
if let Some(base_iri) = base_iri { if let Some(base_iri) = base_iri {
parser = parser parser = parser
.with_base_iri(base_iri) .with_base_iri(base_iri)
.map_err(|e| ParseError::invalid_base_iri(base_iri, e))?; .map_err(|e| LoaderError::InvalidBaseIri {
iri: base_iri.into(),
error: e,
})?;
} }
let to_graph_name = to_graph_name.into(); for quad in parser.parse_read(read) {
for triple in parser.read_triples(reader) { self.writer.insert(quad?.as_ref())?;
self.writer
.insert(triple?.as_ref().in_graph(to_graph_name))?;
} }
Ok(()) Ok(())
} }
@ -1103,7 +1116,7 @@ impl<'a> Transaction<'a> {
/// Usage example: /// Usage example:
/// ``` /// ```
/// use oxigraph::store::Store; /// use oxigraph::store::Store;
/// use oxigraph::io::DatasetFormat; /// use oxigraph::io::RdfFormat;
/// use oxigraph::model::*; /// use oxigraph::model::*;
/// ///
/// let store = Store::new()?; /// let store = Store::new()?;
@ -1111,7 +1124,7 @@ impl<'a> Transaction<'a> {
/// // insertion /// // insertion
/// let file = b"<http://example.com> <http://example.com> <http://example.com> <http://example.com> ."; /// let file = b"<http://example.com> <http://example.com> <http://example.com> <http://example.com> .";
/// store.transaction(|mut transaction| { /// store.transaction(|mut transaction| {
/// transaction.load_dataset(file.as_ref(), DatasetFormat::NQuads, None) /// transaction.load_dataset(file.as_ref(), RdfFormat::NQuads, None)
/// })?; /// })?;
/// ///
/// // we inspect the store contents /// // we inspect the store contents
@ -1121,17 +1134,20 @@ impl<'a> Transaction<'a> {
/// ``` /// ```
pub fn load_dataset( pub fn load_dataset(
&mut self, &mut self,
reader: impl Read, read: impl Read,
format: DatasetFormat, format: impl Into<RdfFormat>,
base_iri: Option<&str>, base_iri: Option<&str>,
) -> Result<(), LoaderError> { ) -> Result<(), LoaderError> {
let mut parser = DatasetParser::from_format(format); let mut parser = RdfParser::from_format(format.into()).rename_blank_nodes();
if let Some(base_iri) = base_iri { if let Some(base_iri) = base_iri {
parser = parser parser = parser
.with_base_iri(base_iri) .with_base_iri(base_iri)
.map_err(|e| ParseError::invalid_base_iri(base_iri, e))?; .map_err(|e| LoaderError::InvalidBaseIri {
iri: base_iri.into(),
error: e,
})?;
} }
for quad in parser.read_quads(reader) { for quad in parser.parse_read(read) {
self.writer.insert(quad?.as_ref())?; self.writer.insert(quad?.as_ref())?;
} }
Ok(()) Ok(())
@ -1365,14 +1381,14 @@ impl Iterator for GraphNameIter {
/// Usage example with loading a dataset: /// Usage example with loading a dataset:
/// ``` /// ```
/// use oxigraph::store::Store; /// use oxigraph::store::Store;
/// use oxigraph::io::DatasetFormat; /// use oxigraph::io::RdfFormat;
/// use oxigraph::model::*; /// use oxigraph::model::*;
/// ///
/// let store = Store::new()?; /// let store = Store::new()?;
/// ///
/// // quads file insertion /// // quads file insertion
/// let file = b"<http://example.com> <http://example.com> <http://example.com> <http://example.com> ."; /// let file = b"<http://example.com> <http://example.com> <http://example.com> <http://example.com> .";
/// store.bulk_loader().load_dataset(file.as_ref(), DatasetFormat::NQuads, None)?; /// store.bulk_loader().load_dataset(file.as_ref(), RdfFormat::NQuads, None)?;
/// ///
/// // we inspect the store contents /// // we inspect the store contents
/// let ex = NamedNodeRef::new("http://example.com")?; /// let ex = NamedNodeRef::new("http://example.com")?;
@ -1448,14 +1464,14 @@ impl BulkLoader {
/// Usage example: /// Usage example:
/// ``` /// ```
/// use oxigraph::store::Store; /// use oxigraph::store::Store;
/// use oxigraph::io::DatasetFormat; /// use oxigraph::io::RdfFormat;
/// use oxigraph::model::*; /// use oxigraph::model::*;
/// ///
/// let store = Store::new()?; /// let store = Store::new()?;
/// ///
/// // insertion /// // insertion
/// let file = b"<http://example.com> <http://example.com> <http://example.com> <http://example.com> ."; /// let file = b"<http://example.com> <http://example.com> <http://example.com> <http://example.com> .";
/// store.bulk_loader().load_dataset(file.as_ref(), DatasetFormat::NQuads, None)?; /// store.bulk_loader().load_dataset(file.as_ref(), RdfFormat::NQuads, None)?;
/// ///
/// // we inspect the store contents /// // we inspect the store contents
/// let ex = NamedNodeRef::new("http://example.com")?; /// let ex = NamedNodeRef::new("http://example.com")?;
@ -1464,17 +1480,20 @@ impl BulkLoader {
/// ``` /// ```
pub fn load_dataset( pub fn load_dataset(
&self, &self,
reader: impl Read, read: impl Read,
format: DatasetFormat, format: impl Into<RdfFormat>,
base_iri: Option<&str>, base_iri: Option<&str>,
) -> Result<(), LoaderError> { ) -> Result<(), LoaderError> {
let mut parser = DatasetParser::from_format(format); let mut parser = RdfParser::from_format(format.into()).rename_blank_nodes();
if let Some(base_iri) = base_iri { if let Some(base_iri) = base_iri {
parser = parser parser = parser
.with_base_iri(base_iri) .with_base_iri(base_iri)
.map_err(|e| ParseError::invalid_base_iri(base_iri, e))?; .map_err(|e| LoaderError::InvalidBaseIri {
iri: base_iri.into(),
error: e,
})?;
} }
self.load_ok_quads(parser.read_quads(reader).filter_map(|r| match r { self.load_ok_quads(parser.parse_read(read).filter_map(|r| match r {
Ok(q) => Some(Ok(q)), Ok(q) => Some(Ok(q)),
Err(e) => { Err(e) => {
if let Some(callback) = &self.on_parse_error { if let Some(callback) = &self.on_parse_error {
@ -1503,36 +1522,41 @@ impl BulkLoader {
/// Usage example: /// Usage example:
/// ``` /// ```
/// use oxigraph::store::Store; /// use oxigraph::store::Store;
/// use oxigraph::io::GraphFormat; /// use oxigraph::io::RdfFormat;
/// use oxigraph::model::*; /// use oxigraph::model::*;
/// ///
/// let store = Store::new()?; /// let store = Store::new()?;
/// ///
/// // insertion /// // insertion
/// let file = b"<http://example.com> <http://example.com> <http://example.com> ."; /// let file = b"<http://example.com> <http://example.com> <http://example.com> .";
/// store.bulk_loader().load_graph(file.as_ref(), GraphFormat::NTriples, GraphNameRef::DefaultGraph, None)?; /// store.bulk_loader().load_graph(file.as_ref(), RdfFormat::NTriples, GraphName::DefaultGraph, None)?;
/// ///
/// // we inspect the store contents /// // we inspect the store contents
/// let ex = NamedNodeRef::new("http://example.com")?; /// let ex = NamedNodeRef::new("http://example.com")?;
/// assert!(store.contains(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?); /// assert!(store.contains(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?);
/// # Result::<_, Box<dyn std::error::Error>>::Ok(()) /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
/// ``` /// ```
pub fn load_graph<'a>( pub fn load_graph(
&self, &self,
reader: impl Read, read: impl Read,
format: GraphFormat, format: impl Into<RdfFormat>,
to_graph_name: impl Into<GraphNameRef<'a>>, to_graph_name: impl Into<GraphName>,
base_iri: Option<&str>, base_iri: Option<&str>,
) -> Result<(), LoaderError> { ) -> Result<(), LoaderError> {
let mut parser = GraphParser::from_format(format); let mut parser = RdfParser::from_format(format.into())
.without_named_graphs()
.with_default_graph(to_graph_name)
.rename_blank_nodes();
if let Some(base_iri) = base_iri { if let Some(base_iri) = base_iri {
parser = parser parser = parser
.with_base_iri(base_iri) .with_base_iri(base_iri)
.map_err(|e| ParseError::invalid_base_iri(base_iri, e))?; .map_err(|e| LoaderError::InvalidBaseIri {
iri: base_iri.into(),
error: e,
})?;
} }
let to_graph_name = to_graph_name.into(); self.load_ok_quads(parser.parse_read(read).filter_map(|r| match r {
self.load_ok_quads(parser.read_triples(reader).filter_map(|r| match r { Ok(q) => Some(Ok(q)),
Ok(q) => Some(Ok(q.in_graph(to_graph_name.into_owned()))),
Err(e) => { Err(e) => {
if let Some(callback) = &self.on_parse_error { if let Some(callback) = &self.on_parse_error {
if let Err(e) = callback(e) { if let Err(e) = callback(e) {

@ -1,4 +1,4 @@
use oxigraph::io::{DatasetFormat, GraphFormat, RdfFormat}; use oxigraph::io::RdfFormat;
use oxigraph::model::vocab::{rdf, xsd}; use oxigraph::model::vocab::{rdf, xsd};
use oxigraph::model::*; use oxigraph::model::*;
use oxigraph::store::Store; use oxigraph::store::Store;
@ -109,7 +109,7 @@ fn test_load_graph() -> Result<(), Box<dyn Error>> {
let store = Store::new()?; let store = Store::new()?;
store.load_graph( store.load_graph(
DATA.as_bytes(), DATA.as_bytes(),
GraphFormat::Turtle, RdfFormat::Turtle,
GraphNameRef::DefaultGraph, GraphNameRef::DefaultGraph,
None, None,
)?; )?;
@ -126,8 +126,8 @@ fn test_bulk_load_graph() -> Result<(), Box<dyn Error>> {
let store = Store::new()?; let store = Store::new()?;
store.bulk_loader().load_graph( store.bulk_loader().load_graph(
DATA.as_bytes(), DATA.as_bytes(),
GraphFormat::Turtle, RdfFormat::Turtle,
GraphNameRef::DefaultGraph, GraphName::DefaultGraph,
None, None,
)?; )?;
for q in quads(GraphNameRef::DefaultGraph) { for q in quads(GraphNameRef::DefaultGraph) {
@ -143,8 +143,8 @@ fn test_bulk_load_graph_lenient() -> Result<(), Box<dyn Error>> {
let store = Store::new()?; let store = Store::new()?;
store.bulk_loader().on_parse_error(|_| Ok(())).load_graph( store.bulk_loader().on_parse_error(|_| Ok(())).load_graph(
b"<http://example.com> <http://example.com> <http://example.com##> .\n<http://example.com> <http://example.com> <http://example.com> .".as_slice(), b"<http://example.com> <http://example.com> <http://example.com##> .\n<http://example.com> <http://example.com> <http://example.com> .".as_slice(),
GraphFormat::NTriples, RdfFormat::NTriples,
GraphNameRef::DefaultGraph, GraphName::DefaultGraph,
None, None,
)?; )?;
assert_eq!(store.len()?, 1); assert_eq!(store.len()?, 1);
@ -161,7 +161,7 @@ fn test_bulk_load_graph_lenient() -> Result<(), Box<dyn Error>> {
#[test] #[test]
fn test_load_dataset() -> Result<(), Box<dyn Error>> { fn test_load_dataset() -> Result<(), Box<dyn Error>> {
let store = Store::new()?; let store = Store::new()?;
store.load_dataset(GRAPH_DATA.as_bytes(), DatasetFormat::TriG, None)?; store.load_dataset(GRAPH_DATA.as_bytes(), RdfFormat::TriG, None)?;
for q in quads(NamedNodeRef::new_unchecked( for q in quads(NamedNodeRef::new_unchecked(
"http://www.wikidata.org/wiki/Special:EntityData/Q90", "http://www.wikidata.org/wiki/Special:EntityData/Q90",
)) { )) {
@ -177,7 +177,7 @@ fn test_bulk_load_dataset() -> Result<(), Box<dyn Error>> {
let store = Store::new()?; let store = Store::new()?;
store store
.bulk_loader() .bulk_loader()
.load_dataset(GRAPH_DATA.as_bytes(), DatasetFormat::TriG, None)?; .load_dataset(GRAPH_DATA.as_bytes(), RdfFormat::TriG, None)?;
let graph_name = let graph_name =
NamedNodeRef::new_unchecked("http://www.wikidata.org/wiki/Special:EntityData/Q90"); NamedNodeRef::new_unchecked("http://www.wikidata.org/wiki/Special:EntityData/Q90");
for q in quads(graph_name) { for q in quads(graph_name) {
@ -194,8 +194,8 @@ fn test_load_graph_generates_new_blank_nodes() -> Result<(), Box<dyn Error>> {
for _ in 0..2 { for _ in 0..2 {
store.load_graph( store.load_graph(
"_:a <http://example.com/p> <http://example.com/p> .".as_bytes(), "_:a <http://example.com/p> <http://example.com/p> .".as_bytes(),
GraphFormat::NTriples, RdfFormat::NTriples,
GraphNameRef::DefaultGraph, GraphName::DefaultGraph,
None, None,
)?; )?;
} }

@ -1,10 +1,7 @@
#![allow(clippy::needless_option_as_deref)] #![allow(clippy::needless_option_as_deref)]
use crate::model::{PyQuad, PyTriple}; use crate::model::{PyQuad, PyTriple};
use oxigraph::io::read::{ParseError, QuadReader, TripleReader}; use oxigraph::io::{FromReadQuadReader, ParseError, RdfFormat, RdfParser, RdfSerializer};
use oxigraph::io::{
DatasetFormat, DatasetParser, GraphFormat, GraphParser, RdfFormat, RdfSerializer,
};
use oxigraph::model::QuadRef; use oxigraph::model::QuadRef;
use pyo3::exceptions::{PyIOError, PySyntaxError, PyValueError}; use pyo3::exceptions::{PyIOError, PySyntaxError, PyValueError};
use pyo3::prelude::*; use pyo3::prelude::*;
@ -41,54 +38,54 @@ pub fn add_to_module(module: &PyModule) -> PyResult<()> {
/// :type mime_type: str /// :type mime_type: str
/// :param base_iri: the base IRI used to resolve the relative IRIs in the file or :py:const:`None` if relative IRI resolution should not be done. /// :param base_iri: the base IRI used to resolve the relative IRIs in the file or :py:const:`None` if relative IRI resolution should not be done.
/// :type base_iri: str or None, optional /// :type base_iri: str or None, optional
/// :param without_named_graphs: Sets that the parser must fail if parsing a named graph.
/// :type without_named_graphs: bool, optional
/// :param rename_blank_nodes: Renames the blank nodes ids from the ones set in the serialization to random ids. This allows to avoid id conflicts when merging graphs together.
/// :type rename_blank_nodes: bool, optional
/// :return: an iterator of RDF triples or quads depending on the format. /// :return: an iterator of RDF triples or quads depending on the format.
/// :rtype: iterator(Triple) or iterator(Quad) /// :rtype: iterator(Quad)
/// :raises ValueError: if the MIME type is not supported. /// :raises ValueError: if the MIME type is not supported.
/// :raises SyntaxError: if the provided data is invalid. /// :raises SyntaxError: if the provided data is invalid.
/// ///
/// >>> input = io.BytesIO(b'<foo> <p> "1" .') /// >>> input = io.BytesIO(b'<foo> <p> "1" .')
/// >>> list(parse(input, "text/turtle", base_iri="http://example.com/")) /// >>> list(parse(input, "text/turtle", base_iri="http://example.com/"))
/// [<Triple subject=<NamedNode value=http://example.com/foo> predicate=<NamedNode value=http://example.com/p> object=<Literal value=1 datatype=<NamedNode value=http://www.w3.org/2001/XMLSchema#string>>>] /// [<Quad subject=<NamedNode value=http://example.com/foo> predicate=<NamedNode value=http://example.com/p> object=<Literal value=1 datatype=<NamedNode value=http://www.w3.org/2001/XMLSchema#string>> graph_name=<DefaultGraph>>]
#[pyfunction] #[pyfunction]
#[pyo3(signature = (input, mime_type, *, base_iri = None))] #[pyo3(signature = (input, mime_type, *, base_iri = None, without_named_graphs = false, rename_blank_nodes = false))]
pub fn parse( pub fn parse(
input: PyObject, input: PyObject,
mime_type: &str, mime_type: &str,
base_iri: Option<&str>, base_iri: Option<&str>,
without_named_graphs: bool,
rename_blank_nodes: bool,
py: Python<'_>, py: Python<'_>,
) -> PyResult<PyObject> { ) -> PyResult<PyObject> {
let Some(format) = RdfFormat::from_media_type(mime_type) else {
return Err(PyValueError::new_err(format!(
"Not supported MIME type: {mime_type}"
)));
};
let input = if let Ok(path) = input.extract::<PathBuf>(py) { let input = if let Ok(path) = input.extract::<PathBuf>(py) {
PyReadable::from_file(&path, py).map_err(map_io_err)? PyReadable::from_file(&path, py).map_err(map_io_err)?
} else { } else {
PyReadable::from_data(input, py) PyReadable::from_data(input, py)
}; };
if let Some(graph_format) = GraphFormat::from_media_type(mime_type) { let mut parser = RdfParser::from_format(format);
let mut parser = GraphParser::from_format(graph_format);
if let Some(base_iri) = base_iri { if let Some(base_iri) = base_iri {
parser = parser parser = parser
.with_base_iri(base_iri) .with_base_iri(base_iri)
.map_err(|e| PyValueError::new_err(e.to_string()))?; .map_err(|e| PyValueError::new_err(e.to_string()))?;
} }
Ok(PyTripleReader { if without_named_graphs {
inner: parser.read_triples(input), parser = parser.without_named_graphs();
} }
.into_py(py)) if rename_blank_nodes {
} else if let Some(dataset_format) = DatasetFormat::from_media_type(mime_type) { parser = parser.rename_blank_nodes();
let mut parser = DatasetParser::from_format(dataset_format);
if let Some(base_iri) = base_iri {
parser = parser
.with_base_iri(base_iri)
.map_err(|e| PyValueError::new_err(e.to_string()))?;
} }
Ok(PyQuadReader { Ok(PyQuadReader {
inner: parser.read_quads(input), inner: parser.parse_read(input),
} }
.into_py(py)) .into_py(py))
} else {
Err(PyValueError::new_err(format!(
"Not supported MIME type: {mime_type}"
)))
}
} }
/// Serializes an RDF graph or dataset. /// Serializes an RDF graph or dataset.
@ -151,30 +148,9 @@ pub fn serialize(input: &PyAny, output: PyObject, mime_type: &str, py: Python<'_
writer.finish().map_err(map_io_err) writer.finish().map_err(map_io_err)
} }
#[pyclass(name = "TripleReader", module = "pyoxigraph")]
pub struct PyTripleReader {
inner: TripleReader<PyReadable>,
}
#[pymethods]
impl PyTripleReader {
fn __iter__(slf: PyRef<'_, Self>) -> PyRef<Self> {
slf
}
fn __next__(&mut self, py: Python<'_>) -> PyResult<Option<PyTriple>> {
py.allow_threads(|| {
self.inner
.next()
.map(|q| Ok(q.map_err(map_parse_error)?.into()))
.transpose()
})
}
}
#[pyclass(name = "QuadReader", module = "pyoxigraph")] #[pyclass(name = "QuadReader", module = "pyoxigraph")]
pub struct PyQuadReader { pub struct PyQuadReader {
inner: QuadReader<PyReadable>, inner: FromReadQuadReader<PyReadable>,
} }
#[pymethods] #[pymethods]

@ -3,7 +3,7 @@
use crate::io::{allow_threads_unsafe, map_io_err, map_parse_error, PyReadable, PyWritable}; use crate::io::{allow_threads_unsafe, map_io_err, map_parse_error, PyReadable, PyWritable};
use crate::model::*; use crate::model::*;
use crate::sparql::*; use crate::sparql::*;
use oxigraph::io::{DatasetFormat, GraphFormat, RdfFormat}; use oxigraph::io::RdfFormat;
use oxigraph::model::{GraphName, GraphNameRef}; use oxigraph::model::{GraphName, GraphNameRef};
use oxigraph::sparql::Update; use oxigraph::sparql::Update;
use oxigraph::store::{self, LoaderError, SerializerError, StorageError, Store}; use oxigraph::store::{self, LoaderError, SerializerError, StorageError, Store};
@ -366,7 +366,7 @@ impl PyStore {
/// :param to_graph: if it is a file composed of triples, the graph in which the triples should be stored. By default, the default graph is used. /// :param to_graph: if it is a file composed of triples, the graph in which the triples should be stored. By default, the default graph is used.
/// :type to_graph: NamedNode or BlankNode or DefaultGraph or None, optional /// :type to_graph: NamedNode or BlankNode or DefaultGraph or None, optional
/// :rtype: None /// :rtype: None
/// :raises ValueError: if the MIME type is not supported or the `to_graph` parameter is given with a quad file. /// :raises ValueError: if the MIME type is not supported.
/// :raises SyntaxError: if the provided data is invalid. /// :raises SyntaxError: if the provided data is invalid.
/// :raises IOError: if an I/O error happens during a quad insertion. /// :raises IOError: if an I/O error happens during a quad insertion.
/// ///
@ -383,6 +383,11 @@ impl PyStore {
to_graph: Option<&PyAny>, to_graph: Option<&PyAny>,
py: Python<'_>, py: Python<'_>,
) -> PyResult<()> { ) -> PyResult<()> {
let Some(format) = RdfFormat::from_media_type(mime_type) else {
return Err(PyValueError::new_err(format!(
"Not supported MIME type: {mime_type}"
)));
};
let to_graph_name = if let Some(graph_name) = to_graph { let to_graph_name = if let Some(graph_name) = to_graph {
Some(GraphName::from(&PyGraphNameRef::try_from(graph_name)?)) Some(GraphName::from(&PyGraphNameRef::try_from(graph_name)?))
} else { } else {
@ -394,29 +399,13 @@ impl PyStore {
PyReadable::from_data(input, py) PyReadable::from_data(input, py)
}; };
py.allow_threads(|| { py.allow_threads(|| {
if let Some(graph_format) = GraphFormat::from_media_type(mime_type) { if let Some(to_graph_name) = to_graph_name {
self.inner
.load_graph(
input,
graph_format,
to_graph_name.as_ref().unwrap_or(&GraphName::DefaultGraph),
base_iri,
)
.map_err(map_loader_error)
} else if let Some(dataset_format) = DatasetFormat::from_media_type(mime_type) {
if to_graph_name.is_some() {
return Err(PyValueError::new_err(
"The target graph name parameter is not available for dataset formats",
));
}
self.inner self.inner
.load_dataset(input, dataset_format, base_iri) .load_graph(input, format, to_graph_name, base_iri)
.map_err(map_loader_error)
} else { } else {
Err(PyValueError::new_err(format!( self.inner.load_dataset(input, format, base_iri)
"Not supported MIME type: {mime_type}"
)))
} }
.map_err(map_loader_error)
}) })
} }
@ -448,7 +437,7 @@ impl PyStore {
/// :param to_graph: if it is a file composed of triples, the graph in which the triples should be stored. By default, the default graph is used. /// :param to_graph: if it is a file composed of triples, the graph in which the triples should be stored. By default, the default graph is used.
/// :type to_graph: NamedNode or BlankNode or DefaultGraph or None, optional /// :type to_graph: NamedNode or BlankNode or DefaultGraph or None, optional
/// :rtype: None /// :rtype: None
/// :raises ValueError: if the MIME type is not supported or the `to_graph` parameter is given with a quad file. /// :raises ValueError: if the MIME type is not supported.
/// :raises SyntaxError: if the provided data is invalid. /// :raises SyntaxError: if the provided data is invalid.
/// :raises IOError: if an I/O error happens during a quad insertion. /// :raises IOError: if an I/O error happens during a quad insertion.
/// ///
@ -465,6 +454,11 @@ impl PyStore {
to_graph: Option<&PyAny>, to_graph: Option<&PyAny>,
py: Python<'_>, py: Python<'_>,
) -> PyResult<()> { ) -> PyResult<()> {
let Some(format) = RdfFormat::from_media_type(mime_type) else {
return Err(PyValueError::new_err(format!(
"Not supported MIME type: {mime_type}"
)));
};
let to_graph_name = if let Some(graph_name) = to_graph { let to_graph_name = if let Some(graph_name) = to_graph {
Some(GraphName::from(&PyGraphNameRef::try_from(graph_name)?)) Some(GraphName::from(&PyGraphNameRef::try_from(graph_name)?))
} else { } else {
@ -476,31 +470,16 @@ impl PyStore {
PyReadable::from_data(input, py) PyReadable::from_data(input, py)
}; };
py.allow_threads(|| { py.allow_threads(|| {
if let Some(graph_format) = GraphFormat::from_media_type(mime_type) { if let Some(to_graph_name) = to_graph_name {
self.inner self.inner
.bulk_loader() .bulk_loader()
.load_graph( .load_graph(input, format, to_graph_name, base_iri)
input, } else {
graph_format,
&to_graph_name.unwrap_or(GraphName::DefaultGraph),
base_iri,
)
.map_err(map_loader_error)
} else if let Some(dataset_format) = DatasetFormat::from_media_type(mime_type) {
if to_graph_name.is_some() {
return Err(PyValueError::new_err(
"The target graph name parameter is not available for dataset formats",
));
}
self.inner self.inner
.bulk_loader() .bulk_loader()
.load_dataset(input, dataset_format, base_iri) .load_dataset(input, format, base_iri)
.map_err(map_loader_error)
} else {
Err(PyValueError::new_err(format!(
"Not supported MIME type: {mime_type}"
)))
} }
.map_err(map_loader_error)
}) })
} }
@ -542,11 +521,6 @@ impl PyStore {
from_graph: Option<&PyAny>, from_graph: Option<&PyAny>,
py: Python<'_>, py: Python<'_>,
) -> PyResult<()> { ) -> PyResult<()> {
let output = if let Ok(path) = output.extract::<PathBuf>(py) {
PyWritable::from_file(&path, py).map_err(map_io_err)?
} else {
PyWritable::from_data(output)
};
let Some(format) = RdfFormat::from_media_type(mime_type) else { let Some(format) = RdfFormat::from_media_type(mime_type) else {
return Err(PyValueError::new_err(format!( return Err(PyValueError::new_err(format!(
"Not supported MIME type: {mime_type}" "Not supported MIME type: {mime_type}"
@ -557,6 +531,11 @@ impl PyStore {
} else { } else {
None None
}; };
let output = if let Ok(path) = output.extract::<PathBuf>(py) {
PyWritable::from_file(&path, py).map_err(map_io_err)?
} else {
PyWritable::from_data(output)
};
py.allow_threads(|| { py.allow_threads(|| {
if let Some(from_graph_name) = &from_graph_name { if let Some(from_graph_name) = &from_graph_name {
self.inner.dump_graph(output, format, from_graph_name) self.inner.dump_graph(output, format, from_graph_name)
@ -860,6 +839,7 @@ pub fn map_loader_error(error: LoaderError) -> PyErr {
match error { match error {
LoaderError::Storage(error) => map_storage_error(error), LoaderError::Storage(error) => map_storage_error(error),
LoaderError::Parsing(error) => map_parse_error(error), LoaderError::Parsing(error) => map_parse_error(error),
LoaderError::InvalidBaseIri { .. } => PyValueError::new_err(error.to_string()),
} }
} }

@ -2,9 +2,9 @@ import unittest
from io import BytesIO, StringIO, UnsupportedOperation from io import BytesIO, StringIO, UnsupportedOperation
from tempfile import NamedTemporaryFile, TemporaryFile from tempfile import NamedTemporaryFile, TemporaryFile
from pyoxigraph import Literal, NamedNode, Quad, Triple, parse, serialize from pyoxigraph import Literal, NamedNode, Quad, parse, serialize
EXAMPLE_TRIPLE = Triple( EXAMPLE_TRIPLE = Quad(
NamedNode("http://example.com/foo"), NamedNode("http://example.com/foo"),
NamedNode("http://example.com/p"), NamedNode("http://example.com/p"),
Literal("éù"), Literal("éù"),
@ -83,11 +83,40 @@ class TestParse(unittest.TestCase):
[EXAMPLE_QUAD], [EXAMPLE_QUAD],
) )
def test_parse_without_named_graphs(self) -> None:
with self.assertRaises(SyntaxError) as _:
list(
parse(
StringIO('<g> { <foo> <p> "1" }'),
"application/trig",
base_iri="http://example.com/",
without_named_graphs=True,
)
)
def test_parse_rename_blank_nodes(self) -> None:
self.assertNotEqual(
list(
parse(
StringIO('_:s <http://example.com/p> "o" .'),
"application/n-triples",
rename_blank_nodes=True,
)
),
list(
parse(
StringIO('_:s <http://example.com/p> "o" .'),
"application/n-triples",
rename_blank_nodes=True,
)
),
)
class TestSerialize(unittest.TestCase): class TestSerialize(unittest.TestCase):
def test_serialize_to_bytes_io(self) -> None: def test_serialize_to_bytes_io(self) -> None:
output = BytesIO() output = BytesIO()
serialize([EXAMPLE_TRIPLE], output, "text/turtle") serialize([EXAMPLE_TRIPLE.triple], output, "text/turtle")
self.assertEqual( self.assertEqual(
output.getvalue().decode(), output.getvalue().decode(),
'<http://example.com/foo> <http://example.com/p> "éù" .\n', '<http://example.com/foo> <http://example.com/p> "éù" .\n',

@ -1,10 +1,10 @@
#![allow(clippy::print_stderr, clippy::cast_precision_loss, clippy::use_debug)] #![allow(clippy::print_stderr, clippy::cast_precision_loss, clippy::use_debug)]
use anyhow::{anyhow, bail, ensure, Context, Error}; use anyhow::{anyhow, bail, ensure, Context};
use clap::{Parser, Subcommand}; use clap::{Parser, Subcommand};
use flate2::read::MultiGzDecoder; use flate2::read::MultiGzDecoder;
use oxhttp::model::{Body, HeaderName, HeaderValue, Method, Request, Response, Status}; use oxhttp::model::{Body, HeaderName, HeaderValue, Method, Request, Response, Status};
use oxhttp::Server; use oxhttp::Server;
use oxigraph::io::{DatasetFormat, GraphFormat, RdfFormat, RdfSerializer}; use oxigraph::io::{RdfFormat, RdfSerializer};
use oxigraph::model::{ use oxigraph::model::{
GraphName, GraphNameRef, IriParseError, NamedNode, NamedNodeRef, NamedOrBlankNode, GraphName, GraphNameRef, IriParseError, NamedNode, NamedNodeRef, NamedOrBlankNode,
}; };
@ -295,13 +295,13 @@ pub fn main() -> anyhow::Result<()> {
Store::new() Store::new()
}?; }?;
let format = if let Some(format) = format { let format = if let Some(format) = format {
Some(GraphOrDatasetFormat::from_str(&format)?) Some(rdf_format_from_name(&format)?)
} else { } else {
None None
}; };
let graph = if let Some(iri) = &graph { let graph = if let Some(iri) = &graph {
Some( Some(
NamedNodeRef::new(iri) NamedNode::new(iri)
.with_context(|| format!("The target graph name {iri} is invalid"))?, .with_context(|| format!("The target graph name {iri} is invalid"))?,
) )
} else { } else {
@ -342,6 +342,7 @@ pub fn main() -> anyhow::Result<()> {
.scope(|s| { .scope(|s| {
for file in file { for file in file {
let store = store.clone(); let store = store.clone();
let graph = graph.clone();
s.spawn(move |_| { s.spawn(move |_| {
let f = file.clone(); let f = file.clone();
let start = Instant::now(); let start = Instant::now();
@ -379,9 +380,7 @@ pub fn main() -> anyhow::Result<()> {
&loader, &loader,
MultiGzDecoder::new(fp), MultiGzDecoder::new(fp),
format.unwrap_or_else(|| { format.unwrap_or_else(|| {
GraphOrDatasetFormat::from_path( rdf_format_from_path(&file.with_extension(""))
&file.with_extension(""),
)
.unwrap() .unwrap()
}), }),
None, None,
@ -392,7 +391,7 @@ pub fn main() -> anyhow::Result<()> {
&loader, &loader,
fp, fp,
format.unwrap_or_else(|| { format.unwrap_or_else(|| {
GraphOrDatasetFormat::from_path(&file).unwrap() rdf_format_from_path(&file).unwrap()
}), }),
None, None,
graph, graph,
@ -432,7 +431,7 @@ pub fn main() -> anyhow::Result<()> {
}; };
let graph = if let Some(graph) = &graph { let graph = if let Some(graph) = &graph {
Some(if graph.eq_ignore_ascii_case("default") { Some(if graph.eq_ignore_ascii_case("default") {
GraphName::DefaultGraph GraphNameRef::DefaultGraph
} else { } else {
NamedNodeRef::new(graph) NamedNodeRef::new(graph)
.with_context(|| format!("The target graph name {graph} is invalid"))? .with_context(|| format!("The target graph name {graph} is invalid"))?
@ -637,24 +636,15 @@ pub fn main() -> anyhow::Result<()> {
fn bulk_load( fn bulk_load(
loader: &BulkLoader, loader: &BulkLoader,
reader: impl Read, reader: impl Read,
format: GraphOrDatasetFormat, format: RdfFormat,
base_iri: Option<&str>, base_iri: Option<&str>,
to_graph_name: Option<NamedNodeRef<'_>>, to_graph_name: Option<NamedNode>,
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
match format { if let Some(to_graph_name) = to_graph_name {
GraphOrDatasetFormat::Graph(format) => loader.load_graph( loader.load_graph(reader, format, to_graph_name, base_iri)
reader, } else {
format, loader.load_dataset(reader, format, base_iri)
to_graph_name.map_or(GraphNameRef::DefaultGraph, GraphNameRef::from), }?;
base_iri,
)?,
GraphOrDatasetFormat::Dataset(format) => {
if to_graph_name.is_some() {
bail!("The --graph option is not allowed when loading a dataset format like NQuads or TriG");
}
loader.load_dataset(reader, format, base_iri)?
}
}
Ok(()) Ok(())
} }
@ -662,57 +652,17 @@ fn dump(
store: &Store, store: &Store,
writer: impl Write, writer: impl Write,
format: RdfFormat, format: RdfFormat,
to_graph_name: Option<GraphName>, from_graph_name: Option<GraphNameRef<'_>>,
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
ensure!(format.supports_datasets() || to_graph_name.is_some(), "The --graph option is required when writing a format not supporting datasets like NTriples, Turtle or RDF/XML"); ensure!(format.supports_datasets() || from_graph_name.is_some(), "The --graph option is required when writing a format not supporting datasets like NTriples, Turtle or RDF/XML");
if let Some(to_graph_name) = &to_graph_name { if let Some(from_graph_name) = from_graph_name {
store.dump_graph(writer, format, to_graph_name) store.dump_graph(writer, format, from_graph_name)
} else { } else {
store.dump_dataset(writer, format) store.dump_dataset(writer, format)
}?; }?;
Ok(()) Ok(())
} }
#[derive(Copy, Clone)]
enum GraphOrDatasetFormat {
Graph(GraphFormat),
Dataset(DatasetFormat),
}
impl GraphOrDatasetFormat {
fn from_path(path: &Path) -> anyhow::Result<Self> {
format_from_path(path, Self::from_extension)
}
fn from_extension(name: &str) -> anyhow::Result<Self> {
Ok(match (GraphFormat::from_extension(name), DatasetFormat::from_extension(name)) {
(Some(g), Some(d)) => bail!("The file extension '{name}' can be resolved to both '{}' and '{}', not sure what to pick", g.file_extension(), d.file_extension()),
(Some(g), None) => Self::Graph(g),
(None, Some(d)) => Self::Dataset(d),
(None, None) =>
bail!("The file extension '{name}' is unknown")
})
}
fn from_media_type(name: &str) -> anyhow::Result<Self> {
Ok(
match (
GraphFormat::from_media_type(name),
DatasetFormat::from_media_type(name),
) {
(Some(g), Some(d)) => bail!(
"The media type '{name}' can be resolved to both '{}' and '{}', not sure what to pick",
g.file_extension(),
d.file_extension()
),
(Some(g), None) => Self::Graph(g),
(None, Some(d)) => Self::Dataset(d),
(None, None) => bail!("The media type '{name}' is unknown"),
},
)
}
}
fn format_from_path<T>( fn format_from_path<T>(
path: &Path, path: &Path,
from_extension: impl FnOnce(&str) -> anyhow::Result<T>, from_extension: impl FnOnce(&str) -> anyhow::Result<T>,
@ -731,20 +681,6 @@ fn format_from_path<T>(
} }
} }
impl FromStr for GraphOrDatasetFormat {
type Err = Error;
fn from_str(name: &str) -> anyhow::Result<Self> {
if let Ok(t) = Self::from_extension(name) {
return Ok(t);
}
if let Ok(t) = Self::from_media_type(name) {
return Ok(t);
}
bail!("The file format '{name}' is unknown")
}
}
fn rdf_format_from_path(path: &Path) -> anyhow::Result<RdfFormat> { fn rdf_format_from_path(path: &Path) -> anyhow::Result<RdfFormat> {
format_from_path(path, |ext| { format_from_path(path, |ext| {
RdfFormat::from_extension(ext) RdfFormat::from_extension(ext)
@ -978,7 +914,7 @@ fn handle_request(
let content_type = let content_type =
content_type(request).ok_or_else(|| bad_request("No Content-Type given"))?; content_type(request).ok_or_else(|| bad_request("No Content-Type given"))?;
if let Some(target) = store_target(request)? { if let Some(target) = store_target(request)? {
let format = GraphFormat::from_media_type(&content_type) let format = RdfFormat::from_media_type(&content_type)
.ok_or_else(|| unsupported_media_type(&content_type))?; .ok_or_else(|| unsupported_media_type(&content_type))?;
let new = !match &target { let new = !match &target {
NamedGraphName::NamedNode(target) => { NamedGraphName::NamedNode(target) => {
@ -1002,7 +938,7 @@ fn handle_request(
true true
} }
}; };
web_load_graph(&store, request, format, GraphName::from(target).as_ref())?; web_load_graph(&store, request, format, &GraphName::from(target))?;
Ok(Response::builder(if new { Ok(Response::builder(if new {
Status::CREATED Status::CREATED
} else { } else {
@ -1010,7 +946,7 @@ fn handle_request(
}) })
.build()) .build())
} else { } else {
let format = DatasetFormat::from_media_type(&content_type) let format = RdfFormat::from_media_type(&content_type)
.ok_or_else(|| unsupported_media_type(&content_type))?; .ok_or_else(|| unsupported_media_type(&content_type))?;
store.clear().map_err(internal_server_error)?; store.clear().map_err(internal_server_error)?;
web_load_dataset(&store, request, format)?; web_load_dataset(&store, request, format)?;
@ -1054,10 +990,10 @@ fn handle_request(
let content_type = let content_type =
content_type(request).ok_or_else(|| bad_request("No Content-Type given"))?; content_type(request).ok_or_else(|| bad_request("No Content-Type given"))?;
if let Some(target) = store_target(request)? { if let Some(target) = store_target(request)? {
let format = GraphFormat::from_media_type(&content_type) let format = RdfFormat::from_media_type(&content_type)
.ok_or_else(|| unsupported_media_type(&content_type))?; .ok_or_else(|| unsupported_media_type(&content_type))?;
let new = assert_that_graph_exists(&store, &target).is_ok(); let new = assert_that_graph_exists(&store, &target).is_ok();
web_load_graph(&store, request, format, GraphName::from(target).as_ref())?; web_load_graph(&store, request, format, &GraphName::from(target))?;
Ok(Response::builder(if new { Ok(Response::builder(if new {
Status::CREATED Status::CREATED
} else { } else {
@ -1065,23 +1001,20 @@ fn handle_request(
}) })
.build()) .build())
} else { } else {
match GraphOrDatasetFormat::from_media_type(&content_type) let format = RdfFormat::from_media_type(&content_type)
.map_err(|_| unsupported_media_type(&content_type))? .ok_or_else(|| unsupported_media_type(&content_type))?;
{ if format.supports_datasets() {
GraphOrDatasetFormat::Graph(format) => { web_load_dataset(&store, request, format)?;
Ok(Response::builder(Status::NO_CONTENT).build())
} else {
let graph = let graph =
resolve_with_base(request, &format!("/store/{:x}", random::<u128>()))?; resolve_with_base(request, &format!("/store/{:x}", random::<u128>()))?;
web_load_graph(&store, request, format, graph.as_ref().into())?; web_load_graph(&store, request, format, &graph.clone().into())?;
Ok(Response::builder(Status::CREATED) Ok(Response::builder(Status::CREATED)
.with_header(HeaderName::LOCATION, graph.into_string()) .with_header(HeaderName::LOCATION, graph.as_str())
.unwrap() .unwrap()
.build()) .build())
} }
GraphOrDatasetFormat::Dataset(format) => {
web_load_dataset(&store, request, format)?;
Ok(Response::builder(Status::NO_CONTENT).build())
}
}
} }
} }
(path, "HEAD") if path.starts_with("/store") => { (path, "HEAD") if path.starts_with("/store") => {
@ -1531,10 +1464,10 @@ fn content_type(request: &Request) -> Option<String> {
fn web_load_graph( fn web_load_graph(
store: &Store, store: &Store,
request: &mut Request, request: &mut Request,
format: GraphFormat, format: RdfFormat,
to_graph_name: GraphNameRef<'_>, to_graph_name: &GraphName,
) -> Result<(), HttpError> { ) -> Result<(), HttpError> {
let base_iri = if let GraphNameRef::NamedNode(graph_name) = to_graph_name { let base_iri = if let GraphName::NamedNode(graph_name) = to_graph_name {
Some(graph_name.as_str()) Some(graph_name.as_str())
} else { } else {
None None
@ -1543,11 +1476,11 @@ fn web_load_graph(
web_bulk_loader(store, request).load_graph( web_bulk_loader(store, request).load_graph(
request.body_mut(), request.body_mut(),
format, format,
to_graph_name, to_graph_name.clone(),
base_iri, base_iri,
) )
} else { } else {
store.load_graph(request.body_mut(), format, to_graph_name, base_iri) store.load_graph(request.body_mut(), format, to_graph_name.clone(), base_iri)
} }
.map_err(loader_to_http_error) .map_err(loader_to_http_error)
} }
@ -1555,7 +1488,7 @@ fn web_load_graph(
fn web_load_dataset( fn web_load_dataset(
store: &Store, store: &Store,
request: &mut Request, request: &mut Request,
format: DatasetFormat, format: RdfFormat,
) -> Result<(), HttpError> { ) -> Result<(), HttpError> {
if url_query_parameter(request, "no_transaction").is_some() { if url_query_parameter(request, "no_transaction").is_some() {
web_bulk_loader(store, request).load_dataset(request.body_mut(), format, None) web_bulk_loader(store, request).load_dataset(request.body_mut(), format, None)
@ -1616,6 +1549,7 @@ fn loader_to_http_error(e: LoaderError) -> HttpError {
match e { match e {
LoaderError::Parsing(e) => bad_request(e), LoaderError::Parsing(e) => bad_request(e),
LoaderError::Storage(e) => internal_server_error(e), LoaderError::Storage(e) => internal_server_error(e),
LoaderError::InvalidBaseIri { .. } => bad_request(e),
} }
} }

@ -1,5 +1,5 @@
use anyhow::{anyhow, bail, Context, Result}; use anyhow::{anyhow, bail, Context, Result};
use oxigraph::io::{DatasetFormat, DatasetParser, GraphFormat, GraphParser}; use oxigraph::io::{RdfFormat, RdfParser};
use oxigraph::model::{Dataset, Graph}; use oxigraph::model::{Dataset, Graph};
use oxttl::n3::N3Quad; use oxttl::n3::N3Quad;
use oxttl::N3Parser; use oxttl::N3Parser;
@ -33,14 +33,14 @@ pub fn read_file_to_string(url: &str) -> Result<String> {
pub fn load_to_graph( pub fn load_to_graph(
url: &str, url: &str,
graph: &mut Graph, graph: &mut Graph,
format: GraphFormat, format: RdfFormat,
ignore_errors: bool, ignore_errors: bool,
) -> Result<()> { ) -> Result<()> {
let parser = GraphParser::from_format(format).with_base_iri(url)?; let parser = RdfParser::from_format(format).with_base_iri(url)?;
for t in parser.read_triples(read_file(url)?) { for t in parser.parse_read(read_file(url)?) {
match t { match t {
Ok(t) => { Ok(t) => {
graph.insert(&t); graph.insert(&t.into());
} }
Err(e) => { Err(e) => {
if !ignore_errors { if !ignore_errors {
@ -52,26 +52,20 @@ pub fn load_to_graph(
Ok(()) Ok(())
} }
pub fn load_graph(url: &str, format: GraphFormat, ignore_errors: bool) -> Result<Graph> { pub fn load_graph(url: &str, format: RdfFormat, ignore_errors: bool) -> Result<Graph> {
let mut graph = Graph::new(); let mut graph = Graph::new();
load_to_graph(url, &mut graph, format, ignore_errors)?; load_to_graph(url, &mut graph, format, ignore_errors)?;
Ok(graph) Ok(graph)
} }
pub fn guess_graph_format(url: &str) -> Result<GraphFormat> {
url.rsplit_once('.')
.and_then(|(_, extension)| GraphFormat::from_extension(extension))
.ok_or_else(|| anyhow!("Serialization type not found for {url}"))
}
pub fn load_to_dataset( pub fn load_to_dataset(
url: &str, url: &str,
dataset: &mut Dataset, dataset: &mut Dataset,
format: DatasetFormat, format: RdfFormat,
ignore_errors: bool, ignore_errors: bool,
) -> Result<()> { ) -> Result<()> {
let parser = DatasetParser::from_format(format).with_base_iri(url)?; let parser = RdfParser::from_format(format).with_base_iri(url)?;
for q in parser.read_quads(read_file(url)?) { for q in parser.parse_read(read_file(url)?) {
match q { match q {
Ok(q) => { Ok(q) => {
dataset.insert(&q); dataset.insert(&q);
@ -86,15 +80,15 @@ pub fn load_to_dataset(
Ok(()) Ok(())
} }
pub fn load_dataset(url: &str, format: DatasetFormat, ignore_errors: bool) -> Result<Dataset> { pub fn load_dataset(url: &str, format: RdfFormat, ignore_errors: bool) -> Result<Dataset> {
let mut dataset = Dataset::new(); let mut dataset = Dataset::new();
load_to_dataset(url, &mut dataset, format, ignore_errors)?; load_to_dataset(url, &mut dataset, format, ignore_errors)?;
Ok(dataset) Ok(dataset)
} }
pub fn guess_dataset_format(url: &str) -> Result<DatasetFormat> { pub fn guess_rdf_format(url: &str) -> Result<RdfFormat> {
url.rsplit_once('.') url.rsplit_once('.')
.and_then(|(_, extension)| DatasetFormat::from_extension(extension)) .and_then(|(_, extension)| RdfFormat::from_extension(extension))
.ok_or_else(|| anyhow!("Serialization type not found for {url}")) .ok_or_else(|| anyhow!("Serialization type not found for {url}"))
} }

@ -1,4 +1,4 @@
use crate::files::{guess_graph_format, load_to_graph}; use crate::files::{guess_rdf_format, load_to_graph};
use crate::vocab::*; use crate::vocab::*;
use anyhow::{bail, Result}; use anyhow::{bail, Result};
use oxigraph::model::vocab::*; use oxigraph::model::vocab::*;
@ -275,7 +275,7 @@ impl TestManifest {
return Ok(None); return Ok(None);
}; };
self.graph.clear(); self.graph.clear();
load_to_graph(&url, &mut self.graph, guess_graph_format(&url)?, false)?; load_to_graph(&url, &mut self.graph, guess_rdf_format(&url)?, false)?;
let manifests = self let manifests = self
.graph .graph

@ -1,27 +1,27 @@
use crate::evaluator::TestEvaluator; use crate::evaluator::TestEvaluator;
use crate::files::{guess_dataset_format, guess_graph_format, load_dataset, load_graph, load_n3}; use crate::files::{guess_rdf_format, load_dataset, load_n3};
use crate::manifest::Test; use crate::manifest::Test;
use crate::report::{dataset_diff, graph_diff}; use crate::report::dataset_diff;
use anyhow::{anyhow, bail, Result}; use anyhow::{anyhow, bail, Result};
use oxigraph::io::{DatasetFormat, GraphFormat}; use oxigraph::io::RdfFormat;
use oxigraph::model::{BlankNode, Dataset, Quad}; use oxigraph::model::{BlankNode, Dataset, Quad};
use oxttl::n3::{N3Quad, N3Term}; use oxttl::n3::{N3Quad, N3Term};
pub fn register_parser_tests(evaluator: &mut TestEvaluator) { pub fn register_parser_tests(evaluator: &mut TestEvaluator) {
evaluator.register( evaluator.register(
"http://www.w3.org/ns/rdftest#TestNTriplesPositiveSyntax", "http://www.w3.org/ns/rdftest#TestNTriplesPositiveSyntax",
|t| evaluate_positive_graph_syntax_test(t, GraphFormat::NTriples), |t| evaluate_positive_syntax_test(t, RdfFormat::NTriples),
); );
evaluator.register( evaluator.register(
"http://www.w3.org/ns/rdftest#TestNQuadsPositiveSyntax", "http://www.w3.org/ns/rdftest#TestNQuadsPositiveSyntax",
|t| evaluate_positive_dataset_syntax_test(t, DatasetFormat::NQuads), |t| evaluate_positive_syntax_test(t, RdfFormat::NQuads),
); );
evaluator.register( evaluator.register(
"http://www.w3.org/ns/rdftest#TestTurtlePositiveSyntax", "http://www.w3.org/ns/rdftest#TestTurtlePositiveSyntax",
|t| evaluate_positive_graph_syntax_test(t, GraphFormat::Turtle), |t| evaluate_positive_syntax_test(t, RdfFormat::Turtle),
); );
evaluator.register("http://www.w3.org/ns/rdftest#TestTrigPositiveSyntax", |t| { evaluator.register("http://www.w3.org/ns/rdftest#TestTrigPositiveSyntax", |t| {
evaluate_positive_dataset_syntax_test(t, DatasetFormat::TriG) evaluate_positive_syntax_test(t, RdfFormat::TriG)
}); });
evaluator.register( evaluator.register(
"https://w3c.github.io/N3/tests/test.n3#TestN3PositiveSyntax", "https://w3c.github.io/N3/tests/test.n3#TestN3PositiveSyntax",
@ -29,47 +29,47 @@ pub fn register_parser_tests(evaluator: &mut TestEvaluator) {
); );
evaluator.register( evaluator.register(
"http://www.w3.org/ns/rdftest#TestNTriplesNegativeSyntax", "http://www.w3.org/ns/rdftest#TestNTriplesNegativeSyntax",
|t| evaluate_negative_graph_syntax_test(t, GraphFormat::NTriples), |t| evaluate_negative_syntax_test(t, RdfFormat::NTriples),
); );
evaluator.register( evaluator.register(
"http://www.w3.org/ns/rdftest#TestNQuadsNegativeSyntax", "http://www.w3.org/ns/rdftest#TestNQuadsNegativeSyntax",
|t| evaluate_negative_dataset_syntax_test(t, DatasetFormat::NQuads), |t| evaluate_negative_syntax_test(t, RdfFormat::NQuads),
); );
evaluator.register( evaluator.register(
"http://www.w3.org/ns/rdftest#TestTurtleNegativeSyntax", "http://www.w3.org/ns/rdftest#TestTurtleNegativeSyntax",
|t| evaluate_negative_graph_syntax_test(t, GraphFormat::Turtle), |t| evaluate_negative_syntax_test(t, RdfFormat::Turtle),
); );
evaluator.register("http://www.w3.org/ns/rdftest#TestTrigNegativeSyntax", |t| { evaluator.register("http://www.w3.org/ns/rdftest#TestTrigNegativeSyntax", |t| {
evaluate_negative_dataset_syntax_test(t, DatasetFormat::TriG) evaluate_negative_syntax_test(t, RdfFormat::TriG)
}); });
evaluator.register("http://www.w3.org/ns/rdftest#TestXMLNegativeSyntax", |t| { evaluator.register("http://www.w3.org/ns/rdftest#TestXMLNegativeSyntax", |t| {
evaluate_negative_graph_syntax_test(t, GraphFormat::RdfXml) evaluate_negative_syntax_test(t, RdfFormat::RdfXml)
}); });
evaluator.register( evaluator.register(
"https://w3c.github.io/N3/tests/test.n3#TestN3NegativeSyntax", "https://w3c.github.io/N3/tests/test.n3#TestN3NegativeSyntax",
evaluate_negative_n3_syntax_test, evaluate_negative_n3_syntax_test,
); );
evaluator.register("http://www.w3.org/ns/rdftest#TestTurtleEval", |t| { evaluator.register("http://www.w3.org/ns/rdftest#TestTurtleEval", |t| {
evaluate_graph_eval_test(t, GraphFormat::Turtle, false) evaluate_eval_test(t, RdfFormat::Turtle, false)
}); });
evaluator.register("http://www.w3.org/ns/rdftest#TestTrigEval", |t| { evaluator.register("http://www.w3.org/ns/rdftest#TestTrigEval", |t| {
evaluate_dataset_eval_test(t, DatasetFormat::TriG, false) evaluate_eval_test(t, RdfFormat::TriG, false)
}); });
evaluator.register("http://www.w3.org/ns/rdftest#TestXMLEval", |t| { evaluator.register("http://www.w3.org/ns/rdftest#TestXMLEval", |t| {
evaluate_graph_eval_test(t, GraphFormat::RdfXml, false) evaluate_eval_test(t, RdfFormat::RdfXml, false)
}); });
evaluator.register("https://w3c.github.io/N3/tests/test.n3#TestN3Eval", |t| { evaluator.register("https://w3c.github.io/N3/tests/test.n3#TestN3Eval", |t| {
evaluate_n3_eval_test(t, false) evaluate_n3_eval_test(t, false)
}); });
evaluator.register("http://www.w3.org/ns/rdftest#TestTurtleNegativeEval", |t| { evaluator.register("http://www.w3.org/ns/rdftest#TestTurtleNegativeEval", |t| {
evaluate_negative_graph_syntax_test(t, GraphFormat::Turtle) evaluate_negative_syntax_test(t, RdfFormat::Turtle)
}); });
evaluator.register("http://www.w3.org/ns/rdftest#TestTrigNegativeEval", |t| { evaluator.register("http://www.w3.org/ns/rdftest#TestTrigNegativeEval", |t| {
evaluate_negative_dataset_syntax_test(t, DatasetFormat::TriG) evaluate_negative_syntax_test(t, RdfFormat::TriG)
}); });
evaluator.register( evaluator.register(
"https://w3c.github.io/rdf-canon/tests/vocab#RDFC10EvalTest", "https://w3c.github.io/rdf-canon/tests/vocab#RDFC10EvalTest",
|t| evaluate_positive_dataset_syntax_test(t, DatasetFormat::NQuads), //TODO: not a proper implementation! |t| evaluate_positive_syntax_test(t, RdfFormat::NQuads), //TODO: not a proper implementation!
); );
evaluator.register( evaluator.register(
"https://w3c.github.io/rdf-canon/tests/vocab#RDFC10NegativeEvalTest", "https://w3c.github.io/rdf-canon/tests/vocab#RDFC10NegativeEvalTest",
@ -81,11 +81,11 @@ pub fn register_parser_tests(evaluator: &mut TestEvaluator) {
); );
evaluator.register( evaluator.register(
"https://github.com/oxigraph/oxigraph/tests#TestNTripleRecovery", "https://github.com/oxigraph/oxigraph/tests#TestNTripleRecovery",
|t| evaluate_graph_eval_test(t, GraphFormat::NTriples, true), |t| evaluate_eval_test(t, RdfFormat::NTriples, true),
); );
evaluator.register( evaluator.register(
"https://github.com/oxigraph/oxigraph/tests#TestTurtleRecovery", "https://github.com/oxigraph/oxigraph/tests#TestTurtleRecovery",
|t| evaluate_graph_eval_test(t, GraphFormat::Turtle, true), |t| evaluate_eval_test(t, RdfFormat::Turtle, true),
); );
evaluator.register( evaluator.register(
"https://github.com/oxigraph/oxigraph/tests#TestN3Recovery", "https://github.com/oxigraph/oxigraph/tests#TestN3Recovery",
@ -93,16 +93,7 @@ pub fn register_parser_tests(evaluator: &mut TestEvaluator) {
); );
} }
fn evaluate_positive_graph_syntax_test(test: &Test, format: GraphFormat) -> Result<()> { fn evaluate_positive_syntax_test(test: &Test, format: RdfFormat) -> Result<()> {
let action = test
.action
.as_deref()
.ok_or_else(|| anyhow!("No action found for test {test}"))?;
load_graph(action, format, false).map_err(|e| anyhow!("Parse error: {e}"))?;
Ok(())
}
fn evaluate_positive_dataset_syntax_test(test: &Test, format: DatasetFormat) -> Result<()> {
let action = test let action = test
.action .action
.as_deref() .as_deref()
@ -120,18 +111,7 @@ fn evaluate_positive_n3_syntax_test(test: &Test) -> Result<()> {
Ok(()) Ok(())
} }
fn evaluate_negative_graph_syntax_test(test: &Test, format: GraphFormat) -> Result<()> { fn evaluate_negative_syntax_test(test: &Test, format: RdfFormat) -> Result<()> {
let action = test
.action
.as_deref()
.ok_or_else(|| anyhow!("No action found for test {test}"))?;
match load_graph(action, format, false) {
Ok(_) => bail!("File parsed without errors even if it should not"),
Err(_) => Ok(()),
}
}
fn evaluate_negative_dataset_syntax_test(test: &Test, format: DatasetFormat) -> Result<()> {
let action = test let action = test
.action .action
.as_deref() .as_deref()
@ -153,36 +133,7 @@ fn evaluate_negative_n3_syntax_test(test: &Test) -> Result<()> {
} }
} }
fn evaluate_graph_eval_test(test: &Test, format: GraphFormat, ignore_errors: bool) -> Result<()> { fn evaluate_eval_test(test: &Test, format: RdfFormat, ignore_errors: bool) -> Result<()> {
let action = test
.action
.as_deref()
.ok_or_else(|| anyhow!("No action found for test {test}"))?;
let mut actual_graph = load_graph(action, format, ignore_errors)
.map_err(|e| anyhow!("Parse error on file {action}: {e}"))?;
actual_graph.canonicalize();
let results = test
.result
.as_ref()
.ok_or_else(|| anyhow!("No tests result found"))?;
let mut expected_graph = load_graph(results, guess_graph_format(results)?, false)
.map_err(|e| anyhow!("Parse error on file {results}: {e}"))?;
expected_graph.canonicalize();
if expected_graph == actual_graph {
Ok(())
} else {
bail!(
"The two files are not isomorphic. Diff:\n{}",
graph_diff(&expected_graph, &actual_graph)
)
}
}
fn evaluate_dataset_eval_test(
test: &Test,
format: DatasetFormat,
ignore_errors: bool,
) -> Result<()> {
let action = test let action = test
.action .action
.as_deref() .as_deref()
@ -194,7 +145,7 @@ fn evaluate_dataset_eval_test(
.result .result
.as_ref() .as_ref()
.ok_or_else(|| anyhow!("No tests result found"))?; .ok_or_else(|| anyhow!("No tests result found"))?;
let mut expected_dataset = load_dataset(results, guess_dataset_format(results)?, false) let mut expected_dataset = load_dataset(results, guess_rdf_format(results)?, false)
.map_err(|e| anyhow!("Parse error on file {results}: {e}"))?; .map_err(|e| anyhow!("Parse error on file {results}: {e}"))?;
expected_dataset.canonicalize(); expected_dataset.canonicalize();
if expected_dataset == actual_dataset { if expected_dataset == actual_dataset {

@ -1,5 +1,5 @@
use anyhow::Result; use anyhow::Result;
use oxigraph::model::{Dataset, Graph, NamedNode}; use oxigraph::model::{Dataset, NamedNode};
use std::fmt::Write; use std::fmt::Write;
use text_diff::{diff, Difference}; use text_diff::{diff, Difference};
use time::format_description::well_known::Rfc3339; use time::format_description::well_known::Rfc3339;
@ -26,20 +26,6 @@ fn normalize_dataset_text(store: &Dataset) -> String {
quads.join("\n") quads.join("\n")
} }
pub(super) fn graph_diff(expected: &Graph, actual: &Graph) -> String {
format_diff(
&normalize_graph_text(expected),
&normalize_graph_text(actual),
"triples",
)
}
fn normalize_graph_text(store: &Graph) -> String {
let mut triples: Vec<_> = store.iter().map(|q| q.to_string()).collect();
triples.sort();
triples.join("\n")
}
pub(super) fn format_diff(expected: &str, actual: &str, kind: &str) -> String { pub(super) fn format_diff(expected: &str, actual: &str, kind: &str) -> String {
let (_, changeset) = diff(expected, actual, "\n"); let (_, changeset) = diff(expected, actual, "\n");
let mut ret = String::new(); let mut ret = String::new();

@ -158,7 +158,7 @@ fn evaluate_evaluation_test(test: &Test) -> Result<()> {
load_dataset_to_store(data, &store)?; load_dataset_to_store(data, &store)?;
} }
for (name, value) in &test.graph_data { for (name, value) in &test.graph_data {
load_graph_to_store(value, &store, name)?; load_graph_to_store(value, &store, name.clone())?;
} }
let query_file = test let query_file = test
.query .query
@ -251,7 +251,7 @@ fn evaluate_update_evaluation_test(test: &Test) -> Result<()> {
load_dataset_to_store(data, &store)?; load_dataset_to_store(data, &store)?;
} }
for (name, value) in &test.graph_data { for (name, value) in &test.graph_data {
load_graph_to_store(value, &store, name)?; load_graph_to_store(value, &store, name.clone())?;
} }
let result_store = Store::new()?; let result_store = Store::new()?;
@ -259,7 +259,7 @@ fn evaluate_update_evaluation_test(test: &Test) -> Result<()> {
load_dataset_to_store(data, &result_store)?; load_dataset_to_store(data, &result_store)?;
} }
for (name, value) in &test.result_graph_data { for (name, value) in &test.result_graph_data {
load_graph_to_store(value, &result_store, name)?; load_graph_to_store(value, &result_store, name.clone())?;
} }
let update_file = test let update_file = test
@ -301,7 +301,7 @@ fn load_sparql_query_result(url: &str) -> Result<StaticQueryResults> {
false, false,
) )
} else { } else {
StaticQueryResults::from_graph(&load_graph(url, guess_graph_format(url)?, false)?) StaticQueryResults::from_graph(&load_graph(url, guess_rdf_format(url)?, false)?)
} }
} }
@ -698,14 +698,14 @@ fn solutions_to_string(solutions: Vec<Vec<(Variable, Term)>>, ordered: bool) ->
lines.join("\n") lines.join("\n")
} }
fn load_graph_to_store<'a>( fn load_graph_to_store(
url: &str, url: &str,
store: &Store, store: &Store,
to_graph_name: impl Into<GraphNameRef<'a>>, to_graph_name: impl Into<GraphName>,
) -> Result<()> { ) -> Result<()> {
store.load_graph( store.load_graph(
read_file(url)?, read_file(url)?,
guess_graph_format(url)?, guess_rdf_format(url)?,
to_graph_name, to_graph_name,
Some(url), Some(url),
)?; )?;
@ -713,16 +713,7 @@ fn load_graph_to_store<'a>(
} }
fn load_dataset_to_store(url: &str, store: &Store) -> Result<()> { fn load_dataset_to_store(url: &str, store: &Store) -> Result<()> {
if let Ok(format) = guess_dataset_format(url) { store.load_dataset(read_file(url)?, guess_rdf_format(url)?, Some(url))?;
store.load_dataset(read_file(url)?, format, Some(url))
} else {
store.load_graph(
read_file(url)?,
guess_graph_format(url)?,
GraphNameRef::DefaultGraph,
Some(url),
)
}?;
Ok(()) Ok(())
} }

Loading…
Cancel
Save