From f262df9f5365e245a522ef08ff95f9c110acfa3f Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 28 Nov 2021 16:15:29 +0100 Subject: [PATCH] Improve Oxigraph server CLI API --- bench/bsbm_oxigraph.sh | 3 +- lib/src/io/format.rs | 90 ++++++++++++++++++++++++++++++++--------- lib/src/sparql/model.rs | 40 ++++++++++++------ server/README.md | 7 +++- server/src/main.rs | 86 +++++++++++++++++++++++++++++---------- 5 files changed, 169 insertions(+), 57 deletions(-) diff --git a/bench/bsbm_oxigraph.sh b/bench/bsbm_oxigraph.sh index 30e719c6..87ad1ce2 100755 --- a/bench/bsbm_oxigraph.sh +++ b/bench/bsbm_oxigraph.sh @@ -5,7 +5,8 @@ PARALLELISM=5 cd bsbm-tools ./generate -fc -pc ${DATASET_SIZE} -s nt -fn "explore-${DATASET_SIZE}" -ud -ufn "explore-update-${DATASET_SIZE}" cargo build --release --manifest-path="../../server/Cargo.toml" -./../../target/release/oxigraph_server --file oxigraph_data --bind 127.0.0.1:7878 & +./../../target/release/oxigraph_server --location oxigraph_data load --file "explore-${DATASET_SIZE}.nt" +./../../target/release/oxigraph_server --location oxigraph_data serve --bind 127.0.0.1:7878 & sleep 5 curl -f -X POST -H 'Content-Type:application/n-triples' --data-binary "@explore-${DATASET_SIZE}.nt" http://127.0.0.1:7878/store?default ./testdriver -mt ${PARALLELISM} -ucf usecases/explore/sparql.txt -o "../bsbm.explore.oxigraph.${DATASET_SIZE}.${PARALLELISM}.main.xml" http://127.0.0.1:7878/query diff --git a/lib/src/io/format.rs b/lib/src/io/format.rs index 08ca8cf8..3bd5afd1 100644 --- a/lib/src/io/format.rs +++ b/lib/src/io/format.rs @@ -72,15 +72,30 @@ impl GraphFormat { /// assert_eq!(GraphFormat::from_media_type("text/turtle; charset=utf-8"), Some(GraphFormat::Turtle)) /// ``` pub fn from_media_type(media_type: &str) -> Option { - if let Some(base_type) = media_type.split(';').next() { - match base_type.trim() { - "application/n-triples" | "text/plain" => Some(Self::NTriples), - "text/turtle" | "application/turtle" | "application/x-turtle" => Some(Self::Turtle), - "application/rdf+xml" | "application/xml" | "text/xml" => Some(Self::RdfXml), - _ => None, - } - } else { - None + match media_type.split(';').next()?.trim() { + "application/n-triples" | "text/plain" => Some(Self::NTriples), + "text/turtle" | "application/turtle" | "application/x-turtle" => Some(Self::Turtle), + "application/rdf+xml" | "application/xml" | "text/xml" => Some(Self::RdfXml), + _ => None, + } + } + + /// Looks for a known format from an extension. + /// + /// It supports some aliases. + /// + /// Example: + /// ``` + /// use oxigraph::io::GraphFormat; + /// + /// assert_eq!(GraphFormat::from_extension("nt"), Some(GraphFormat::NTriples)) + /// ``` + pub fn from_extension(extension: &str) -> Option { + match extension { + "nt" | "txt" => Some(Self::NTriples), + "ttl" => Some(Self::Turtle), + "rdf" | "xml" => Some(Self::RdfXml), + _ => None, } } } @@ -153,16 +168,53 @@ impl DatasetFormat { /// assert_eq!(DatasetFormat::from_media_type("application/n-quads; charset=utf-8"), Some(DatasetFormat::NQuads)) /// ``` pub fn from_media_type(media_type: &str) -> Option { - if let Some(base_type) = media_type.split(';').next() { - match base_type.trim() { - "application/n-quads" | "text/x-nquads" | "text/nquads" => { - Some(Self::NQuads) - } - "application/trig" | "application/x-trig" => Some(Self::TriG), - _ => None, - } - } else { - None + match media_type.split(';').next()?.trim() { + "application/n-quads" | "text/x-nquads" | "text/nquads" => Some(Self::NQuads), + "application/trig" | "application/x-trig" => Some(Self::TriG), + _ => None, + } + } + + /// Looks for a known format from an extension. + /// + /// It supports some aliases. + /// + /// Example: + /// ``` + /// use oxigraph::io::DatasetFormat; + /// + /// assert_eq!(DatasetFormat::from_extension("nq"), Some(DatasetFormat::NQuads)) + /// ``` + pub fn from_extension(extension: &str) -> Option { + match extension { + "nq" | "txt" => Some(Self::NQuads), + "trig" => Some(Self::TriG), + _ => None, + } + } +} + +impl TryFrom for GraphFormat { + type Error = (); + + /// Attempts to find a graph format that is a subset of this [`DatasetFormat`]. + fn try_from(value: DatasetFormat) -> Result { + match value { + DatasetFormat::NQuads => Ok(Self::NTriples), + DatasetFormat::TriG => Ok(Self::Turtle), + } + } +} + +impl TryFrom for DatasetFormat { + type Error = (); + + /// Attempts to find a dataset format that is a superset of this [`GraphFormat`]. + fn try_from(value: GraphFormat) -> Result { + match value { + GraphFormat::NTriples => Ok(Self::NQuads), + GraphFormat::Turtle => Ok(Self::TriG), + GraphFormat::RdfXml => Err(()), } } } diff --git a/lib/src/sparql/model.rs b/lib/src/sparql/model.rs index b84d9440..835053af 100644 --- a/lib/src/sparql/model.rs +++ b/lib/src/sparql/model.rs @@ -190,20 +190,34 @@ impl QueryResultsFormat { /// assert_eq!(QueryResultsFormat::from_media_type("application/sparql-results+json; charset=utf-8"), Some(QueryResultsFormat::Json)) /// ``` pub fn from_media_type(media_type: &str) -> Option { - if let Some(base_type) = media_type.split(';').next() { - match base_type { - "application/sparql-results+xml" | "application/xml" | "text/xml" => { - Some(Self::Xml) - } - "application/sparql-results+json" | "application/json" | "text/json" => { - Some(Self::Json) - } - "text/csv" => Some(Self::Csv), - "text/tab-separated-values" | "text/tsv" => Some(Self::Tsv), - _ => None, + match media_type.split(';').next()?.trim() { + "application/sparql-results+xml" | "application/xml" | "text/xml" => Some(Self::Xml), + "application/sparql-results+json" | "application/json" | "text/json" => { + Some(Self::Json) } - } else { - None + "text/csv" => Some(Self::Csv), + "text/tab-separated-values" | "text/tsv" => Some(Self::Tsv), + _ => None, + } + } + + /// Looks for a known format from an extension. + /// + /// It supports some aliases. + /// + /// Example: + /// ``` + /// use oxigraph::sparql::QueryResultsFormat; + /// + /// assert_eq!(QueryResultsFormat::from_extension("json"), Some(QueryResultsFormat::Json)) + /// ``` + pub fn from_extension(extension: &str) -> Option { + match extension { + "srx" | "xml" => Some(Self::Xml), + "srj" | "json" => Some(Self::Json), + "csv" | "txt" => Some(Self::Csv), + "tsv" => Some(Self::Tsv), + _ => None, } } } diff --git a/server/README.md b/server/README.md index 0786fffc..cba46769 100644 --- a/server/README.md +++ b/server/README.md @@ -44,7 +44,7 @@ It will create a fat binary in `target/release/oxigraph_server`. ## Usage -Run `oxigraph_server -f my_data_storage_directory` to start the server where `my_data_storage_directory` is the directory where you want Oxigraph data to be stored in. It listens by default on `localhost:7878`. +Run `oxigraph_server serve --location my_data_storage_directory` to start the server where `my_data_storage_directory` is the directory where you want Oxigraph data to be stored in. It listens by default on `localhost:7878`. The server provides an HTML UI with a form to execute SPARQL requests. @@ -62,6 +62,9 @@ It provides the following REST actions: Use `oxigraph_server --help` to see the possible options when starting the server. +It is also possible to load RDF data offline using bulk loading: +`oxigraph_server load --location my_data_storage_directory --file my_file.nq` + ## Using a Docker image ### Display the help menu @@ -72,7 +75,7 @@ docker run --rm oxigraph/oxigraph --help ### Run the Web server Expose the server on port `7878` of the host machine, and save data on the local `./data` folder ```sh -docker run --init --rm -v $PWD/data:/data -p 7878:7878 oxigraph/oxigraph -b 0.0.0.0:7878 -f /data +docker run --init --rm -v $PWD/data:/data -p 7878:7878 oxigraph/oxigraph serve --bind 0.0.0.0:7878 --location /data ``` You can then access it from your machine on port `7878`: diff --git a/server/src/main.rs b/server/src/main.rs index a9370d99..9af1faf7 100644 --- a/server/src/main.rs +++ b/server/src/main.rs @@ -9,7 +9,7 @@ unused_qualifications )] -use clap::{App, Arg}; +use clap::{App, AppSettings, Arg, SubCommand}; use oxhttp::model::{Body, HeaderName, HeaderValue, Request, Response, Status}; use oxhttp::Server; use oxigraph::io::{DatasetFormat, DatasetSerializer, GraphFormat, GraphSerializer}; @@ -20,6 +20,7 @@ use oxiri::Iri; use rand::random; use std::cell::RefCell; use std::cmp::min; +use std::fs::File; use std::io::{BufReader, Error, ErrorKind, Read, Write}; use std::rc::Rc; use std::str::FromStr; @@ -34,36 +35,77 @@ const LOGO: &str = include_str!("../logo.svg"); pub fn main() -> std::io::Result<()> { let matches = App::new("Oxigraph SPARQL server") .arg( - Arg::with_name("bind") - .short("b") - .long("bind") - .help("Sets a custom config file") - .takes_value(true), - ) - .arg( - Arg::with_name("file") - .short("f") - .long("file") + Arg::with_name("location") + .short("l") + .long("location") .help("directory in which persist the data") .takes_value(true), ) + .setting(AppSettings::SubcommandRequiredElseHelp) + .subcommand( + SubCommand::with_name("serve") + .about("Start Oxigraph HTTP server") + .arg( + Arg::with_name("bind") + .short("b") + .long("bind") + .help("Sets a custom config file") + .takes_value(true), + ), + ) + .subcommand( + SubCommand::with_name("load") + .about("Bulk loads a file into the store") + .arg( + Arg::with_name("file") + .short("f") + .long("file") + .help("The file to load") + .takes_value(true) + .required(true), + ), + ) .get_matches(); - let bind = matches.value_of("bind").unwrap_or("localhost:7878"); - let file = matches.value_of("file"); - let store = if let Some(file) = file { - Store::open(file) + let mut store = if let Some(path) = matches.value_of_os("location") { + Store::open(path) } else { Store::new() }?; - let mut server = Server::new(move |request| handle_request(request, store.clone())); - server.set_global_timeout(HTTP_TIMEOUT); - server - .set_server_name(concat!("Oxigraph/", env!("CARGO_PKG_VERSION"))) - .unwrap(); - println!("Listening for requests at http://{}", &bind); - server.listen(bind) + match matches.subcommand() { + ("load", Some(submatches)) => { + let file = submatches.value_of("file").unwrap(); + let format = file + .rsplit_once(".") + .and_then(|(_, extension)| { + DatasetFormat::from_extension(extension) + .or_else(|| GraphFormat::from_extension(extension)?.try_into().ok()) + }) + .ok_or_else(|| { + Error::new( + ErrorKind::InvalidInput, + "The server is not able to guess the file format of {} from its extension", + ) + })?; + store.bulk_load_dataset(BufReader::new(File::open(file)?), format, None)?; + store.optimize() + } + ("serve", Some(submatches)) => { + let bind = submatches.value_of("bind").unwrap_or("localhost:7878"); + let mut server = Server::new(move |request| handle_request(request, store.clone())); + server.set_global_timeout(HTTP_TIMEOUT); + server + .set_server_name(concat!("Oxigraph/", env!("CARGO_PKG_VERSION"))) + .unwrap(); + println!("Listening for requests at http://{}", &bind); + server.listen(bind) + } + (s, _) => { + eprintln!("Not supported subcommand: '{}'", s); + Ok(()) + } + } } fn handle_request(request: &mut Request, store: Store) -> Response {