Server: Allows loading from stdin

server/src/main.rs

@@ -1,4 +1,4 @@
-use anyhow::{bail, Context, Error};
+use anyhow::{anyhow, bail, Context, Error};
 use clap::{Parser, Subcommand};
 use flate2::read::MultiGzDecoder;
 use oxhttp::model::{Body, HeaderName, HeaderValue, Request, Response, Status};
@@ -21,7 +21,7 @@ use std::cmp::{max, min};
 use std::ffi::OsStr;
 use std::fmt;
 use std::fs::File;
-use std::io::{self, BufReader, Read, Write};
+use std::io::{self, stdin, BufRead, BufReader, Read, Write};
 use std::path::{Path, PathBuf};
 use std::rc::Rc;
 use std::str;
@@ -132,6 +132,33 @@ pub fn main() -> anyhow::Result<()> {
             } else {
                 None
             };
+            if file.is_empty() {
+                // We read from stdin
+                let start = Instant::now();
+                let mut loader = store.bulk_loader().on_progress(move |size| {
+                    let elapsed = start.elapsed();
+                    eprintln!(
+                        "{size} triples loaded in {}s ({} t/s)",
+                        elapsed.as_secs(),
+                        ((size as f64) / elapsed.as_secs_f64()).round()
+                    )
+                });
+                if lenient {
+                    loader = loader.on_parse_error(move |e| {
+                        eprintln!("Parsing error: {e}");
+                        Ok(())
+                    })
+                }
+                bulk_load(
+                    loader,
+                    stdin().lock(),
+                    format.ok_or_else(|| {
+                        anyhow!("The --format option must be set when loading from stdin")
+                    })?,
+                    None,
+                    graph,
+                )
+            } else {
                 ThreadPoolBuilder::new()
                     .num_threads(max(1, available_parallelism()?.get() / 2))
                     .thread_name(|i| format!("Oxigraph bulk loader thread {i}"))
@@ -174,7 +201,7 @@ pub fn main() -> anyhow::Result<()> {
                                     if file.extension().map_or(false, |e| e == OsStr::new("gz")) {
                                         bulk_load(
                                             loader,
-                                        MultiGzDecoder::new(fp),
+                                            BufReader::new(MultiGzDecoder::new(fp)),
                                             format.unwrap_or_else(|| {
                                                 GraphOrDatasetFormat::from_path(
                                                     &file.with_extension(""),
@@ -187,16 +214,20 @@ pub fn main() -> anyhow::Result<()> {
                                     } else {
                                         bulk_load(
                                             loader,
-                                        fp,
+                                            BufReader::new(fp),
                                             format.unwrap_or_else(|| {
                                                 GraphOrDatasetFormat::from_path(&file).unwrap()
                                             }),
                                             None,
-                                        None,
+                                            graph,
                                         )
                                     }
                                 } {
-                                eprintln!("Error while loading file {}: {}", file.display(), error)
+                                    eprintln!(
+                                        "Error while loading file {}: {}",
+                                        file.display(),
+                                        error
+                                    )
                                     //TODO: hard fail
                                 }
                             })
@@ -204,6 +235,7 @@ pub fn main() -> anyhow::Result<()> {
                     });
                 Ok(())
             }
+        }
         Command::Serve { bind } => {
             let mut server = Server::new(move |request| {
                 handle_request(request, store.clone())
@@ -220,12 +252,11 @@ pub fn main() -> anyhow::Result<()> {
 
 fn bulk_load(
     loader: BulkLoader,
-    reader: impl Read,
+    reader: impl BufRead,
     format: GraphOrDatasetFormat,
     base_iri: Option<&str>,
     to_graph_name: Option<NamedNodeRef<'_>>,
 ) -> anyhow::Result<()> {
-    let reader = BufReader::new(reader);
     match format {
         GraphOrDatasetFormat::Graph(format) => loader.load_graph(
             reader,
@@ -1259,6 +1290,20 @@ mod tests {
         Ok(())
     }
 
+    #[test]
+    fn cli_load_from_stdin() -> Result<()> {
+        cli_command()?
+            .arg("load")
+            .arg("--format")
+            .arg("nq")
+            .write_stdin("<http://example.com/s> <http://example.com/p> <http://example.com/o> .")
+            .assert()
+            .success()
+            .stdout("")
+            .stderr(predicate::str::starts_with("1 triples loaded"));
+        Ok(())
+    }
+
     #[test]
     fn get_ui() -> Result<()> {
         ServerTest::new()?.test_status(