Improve Oxigraph server CLI API

bench/bsbm_oxigraph.sh

@@ -5,7 +5,8 @@ PARALLELISM=5
 cd bsbm-tools
 ./generate -fc -pc ${DATASET_SIZE} -s nt -fn "explore-${DATASET_SIZE}" -ud -ufn "explore-update-${DATASET_SIZE}"
 cargo build --release --manifest-path="../../server/Cargo.toml"
-./../../target/release/oxigraph_server --file oxigraph_data --bind 127.0.0.1:7878 &
+./../../target/release/oxigraph_server --location oxigraph_data load --file "explore-${DATASET_SIZE}.nt"
+./../../target/release/oxigraph_server --location oxigraph_data serve --bind 127.0.0.1:7878 &
 sleep 5
 curl -f -X POST -H 'Content-Type:application/n-triples' --data-binary "@explore-${DATASET_SIZE}.nt" http://127.0.0.1:7878/store?default
 ./testdriver -mt ${PARALLELISM} -ucf usecases/explore/sparql.txt -o "../bsbm.explore.oxigraph.${DATASET_SIZE}.${PARALLELISM}.main.xml" http://127.0.0.1:7878/query

lib/src/io/format.rs

@@ -72,15 +72,30 @@ impl GraphFormat {
     /// assert_eq!(GraphFormat::from_media_type("text/turtle; charset=utf-8"), Some(GraphFormat::Turtle))
     /// ```
     pub fn from_media_type(media_type: &str) -> Option<Self> {
-        if let Some(base_type) = media_type.split(';').next() {
+        match media_type.split(';').next()?.trim() {
-            match base_type.trim() {
             "application/n-triples" | "text/plain" => Some(Self::NTriples),
             "text/turtle" | "application/turtle" | "application/x-turtle" => Some(Self::Turtle),
             "application/rdf+xml" | "application/xml" | "text/xml" => Some(Self::RdfXml),
             _ => None,
         }
-        } else {
+    }
-            None
+
+    /// Looks for a known format from an extension.
+    ///
+    /// It supports some aliases.
+    ///
+    /// Example:
+    /// ```
+    /// use oxigraph::io::GraphFormat;
+    ///
+    /// assert_eq!(GraphFormat::from_extension("nt"), Some(GraphFormat::NTriples))
+    /// ```
+    pub fn from_extension(extension: &str) -> Option<Self> {
+        match extension {
+            "nt" | "txt" => Some(Self::NTriples),
+            "ttl" => Some(Self::Turtle),
+            "rdf" | "xml" => Some(Self::RdfXml),
+            _ => None,
         }
     }
 }
@@ -153,16 +168,53 @@ impl DatasetFormat {
     /// assert_eq!(DatasetFormat::from_media_type("application/n-quads; charset=utf-8"), Some(DatasetFormat::NQuads))
     /// ```
     pub fn from_media_type(media_type: &str) -> Option<Self> {
-        if let Some(base_type) = media_type.split(';').next() {
+        match media_type.split(';').next()?.trim() {
-            match base_type.trim() {
+            "application/n-quads" | "text/x-nquads" | "text/nquads" => Some(Self::NQuads),
-                "application/n-quads" | "text/x-nquads" | "text/nquads" => {
-                    Some(Self::NQuads)
-                }
             "application/trig" | "application/x-trig" => Some(Self::TriG),
             _ => None,
         }
-        } else {
+    }
-            None
+
+    /// Looks for a known format from an extension.
+    ///
+    /// It supports some aliases.
+    ///
+    /// Example:
+    /// ```
+    /// use oxigraph::io::DatasetFormat;
+    ///
+    /// assert_eq!(DatasetFormat::from_extension("nq"), Some(DatasetFormat::NQuads))
+    /// ```
+    pub fn from_extension(extension: &str) -> Option<Self> {
+        match extension {
+            "nq" | "txt" => Some(Self::NQuads),
+            "trig" => Some(Self::TriG),
+            _ => None,
+        }
+    }
+}
+
+impl TryFrom<DatasetFormat> for GraphFormat {
+    type Error = ();
+
+    /// Attempts to find a graph format that is a subset of this [`DatasetFormat`].
+    fn try_from(value: DatasetFormat) -> Result<Self, ()> {
+        match value {
+            DatasetFormat::NQuads => Ok(Self::NTriples),
+            DatasetFormat::TriG => Ok(Self::Turtle),
+        }
+    }
+}
+
+impl TryFrom<GraphFormat> for DatasetFormat {
+    type Error = ();
+
+    /// Attempts to find a dataset format that is a superset of this [`GraphFormat`].
+    fn try_from(value: GraphFormat) -> Result<Self, ()> {
+        match value {
+            GraphFormat::NTriples => Ok(Self::NQuads),
+            GraphFormat::Turtle => Ok(Self::TriG),
+            GraphFormat::RdfXml => Err(()),
         }
     }
 }

lib/src/sparql/model.rs

@@ -190,11 +190,8 @@ impl QueryResultsFormat {
     /// assert_eq!(QueryResultsFormat::from_media_type("application/sparql-results+json; charset=utf-8"), Some(QueryResultsFormat::Json))
     /// ```
     pub fn from_media_type(media_type: &str) -> Option<Self> {
-        if let Some(base_type) = media_type.split(';').next() {
+        match media_type.split(';').next()?.trim() {
-            match base_type {
+            "application/sparql-results+xml" | "application/xml" | "text/xml" => Some(Self::Xml),
-                "application/sparql-results+xml" | "application/xml" | "text/xml" => {
-                    Some(Self::Xml)
-                }
             "application/sparql-results+json" | "application/json" | "text/json" => {
                 Some(Self::Json)
             }
@@ -202,8 +199,25 @@ impl QueryResultsFormat {
             "text/tab-separated-values" | "text/tsv" => Some(Self::Tsv),
             _ => None,
         }
-        } else {
+    }
-            None
+
+    /// Looks for a known format from an extension.
+    ///
+    /// It supports some aliases.
+    ///
+    /// Example:
+    /// ```
+    /// use oxigraph::sparql::QueryResultsFormat;
+    ///
+    /// assert_eq!(QueryResultsFormat::from_extension("json"), Some(QueryResultsFormat::Json))
+    /// ```
+    pub fn from_extension(extension: &str) -> Option<Self> {
+        match extension {
+            "srx" | "xml" => Some(Self::Xml),
+            "srj" | "json" => Some(Self::Json),
+            "csv" | "txt" => Some(Self::Csv),
+            "tsv" => Some(Self::Tsv),
+            _ => None,
         }
     }
 }

server/README.md

@@ -44,7 +44,7 @@ It will create a fat binary in `target/release/oxigraph_server`.
 
 ## Usage
 
-Run `oxigraph_server -f my_data_storage_directory` to start the server where `my_data_storage_directory` is the directory where you want Oxigraph data to be stored in. It listens by default on `localhost:7878`.
+Run `oxigraph_server serve --location my_data_storage_directory` to start the server where `my_data_storage_directory` is the directory where you want Oxigraph data to be stored in. It listens by default on `localhost:7878`.
 
 The server provides an HTML UI with a form to execute SPARQL requests.
 
@@ -62,6 +62,9 @@ It provides the following REST actions:
 
 Use `oxigraph_server --help` to see the possible options when starting the server.
 
+It is also possible to load RDF data offline using bulk loading:
+`oxigraph_server load --location my_data_storage_directory --file my_file.nq`
+
 ## Using a Docker image
 
 ### Display the help menu
@@ -72,7 +75,7 @@ docker run --rm oxigraph/oxigraph --help
 ### Run the Web server
 Expose the server on port `7878` of the host machine, and save data on the local `./data` folder
 ```sh
-docker run --init --rm -v $PWD/data:/data -p 7878:7878 oxigraph/oxigraph -b 0.0.0.0:7878 -f /data
+docker run --init --rm -v $PWD/data:/data -p 7878:7878 oxigraph/oxigraph serve --bind 0.0.0.0:7878 --location /data
 ```
 
 You can then access it from your machine on port `7878`:

server/src/main.rs

@@ -9,7 +9,7 @@
     unused_qualifications
 )]
 
-use clap::{App, Arg};
+use clap::{App, AppSettings, Arg, SubCommand};
 use oxhttp::model::{Body, HeaderName, HeaderValue, Request, Response, Status};
 use oxhttp::Server;
 use oxigraph::io::{DatasetFormat, DatasetSerializer, GraphFormat, GraphSerializer};
@@ -20,6 +20,7 @@ use oxiri::Iri;
 use rand::random;
 use std::cell::RefCell;
 use std::cmp::min;
+use std::fs::File;
 use std::io::{BufReader, Error, ErrorKind, Read, Write};
 use std::rc::Rc;
 use std::str::FromStr;
@@ -33,30 +34,65 @@ const LOGO: &str = include_str!("../logo.svg");
 
 pub fn main() -> std::io::Result<()> {
     let matches = App::new("Oxigraph SPARQL server")
+        .arg(
+            Arg::with_name("location")
+                .short("l")
+                .long("location")
+                .help("directory in which persist the data")
+                .takes_value(true),
+        )
+        .setting(AppSettings::SubcommandRequiredElseHelp)
+        .subcommand(
+            SubCommand::with_name("serve")
+                .about("Start Oxigraph HTTP server")
                 .arg(
                     Arg::with_name("bind")
                         .short("b")
                         .long("bind")
                         .help("Sets a custom config file")
                         .takes_value(true),
+                ),
         )
+        .subcommand(
+            SubCommand::with_name("load")
+                .about("Bulk loads a file into the store")
                 .arg(
                     Arg::with_name("file")
                         .short("f")
                         .long("file")
-                .help("directory in which persist the data")
+                        .help("The file to load")
-                .takes_value(true),
+                        .takes_value(true)
+                        .required(true),
+                ),
         )
         .get_matches();
-    let bind = matches.value_of("bind").unwrap_or("localhost:7878");
-    let file = matches.value_of("file");
 
-    let store = if let Some(file) = file {
+    let mut store = if let Some(path) = matches.value_of_os("location") {
-        Store::open(file)
+        Store::open(path)
     } else {
         Store::new()
     }?;
 
+    match matches.subcommand() {
+        ("load", Some(submatches)) => {
+            let file = submatches.value_of("file").unwrap();
+            let format = file
+                .rsplit_once(".")
+                .and_then(|(_, extension)| {
+                    DatasetFormat::from_extension(extension)
+                        .or_else(|| GraphFormat::from_extension(extension)?.try_into().ok())
+                })
+                .ok_or_else(|| {
+                    Error::new(
+                        ErrorKind::InvalidInput,
+                        "The server is not able to guess the file format of {} from its extension",
+                    )
+                })?;
+            store.bulk_load_dataset(BufReader::new(File::open(file)?), format, None)?;
+            store.optimize()
+        }
+        ("serve", Some(submatches)) => {
+            let bind = submatches.value_of("bind").unwrap_or("localhost:7878");
             let mut server = Server::new(move |request| handle_request(request, store.clone()));
             server.set_global_timeout(HTTP_TIMEOUT);
             server
@@ -64,6 +100,12 @@ pub fn main() -> std::io::Result<()> {
                 .unwrap();
             println!("Listening for requests at http://{}", &bind);
             server.listen(bind)
+        }
+        (s, _) => {
+            eprintln!("Not supported subcommand: '{}'", s);
+            Ok(())
+        }
+    }
 }
 
 fn handle_request(request: &mut Request, store: Store) -> Response {