parent
77edc05ced
commit
fdcaf65a8d
@ -1,45 +0,0 @@ |
||||
[package] |
||||
name = "oxigraph-cli" |
||||
version.workspace = true |
||||
authors.workspace = true |
||||
license.workspace = true |
||||
readme = "README.md" |
||||
keywords = ["RDF", "SPARQL", "graph-database", "database"] |
||||
categories = ["command-line-utilities", "database"] |
||||
repository = "https://github.com/oxigraph/oxigraph/tree/main/cli" |
||||
homepage = "https://oxigraph.org/cli/" |
||||
description = """ |
||||
Oxigraph CLI tool and SPARQL HTTP server |
||||
""" |
||||
edition.workspace = true |
||||
rust-version.workspace = true |
||||
|
||||
[[bin]] |
||||
name = "oxigraph" |
||||
path = "src/main.rs" |
||||
doc = false |
||||
|
||||
[features] |
||||
default = [] |
||||
native-tls = ["oxigraph/http-client-native-tls"] |
||||
rustls-native = ["oxigraph/http-client-rustls-native"] |
||||
rustls-webpki = ["oxigraph/http-client-rustls-webpki"] |
||||
|
||||
[dependencies] |
||||
anyhow.workspace = true |
||||
clap = { workspace = true, features = ["derive"] } |
||||
flate2.workspace = true |
||||
oxhttp = { workspace = true, features = ["flate2"] } |
||||
oxigraph.workspace = true |
||||
oxiri.workspace = true |
||||
rand.workspace = true |
||||
rayon-core.workspace = true |
||||
url.workspace = true |
||||
|
||||
[dev-dependencies] |
||||
assert_cmd.workspace = true |
||||
assert_fs.workspace = true |
||||
predicates.workspace = true |
||||
|
||||
[lints] |
||||
workspace = true |
||||
@ -1,25 +0,0 @@ |
||||
FROM --platform=$BUILDPLATFORM rust:1-bookworm as builder |
||||
ARG BUILDARCH TARGETARCH |
||||
RUN apt-get update && \ |
||||
apt-get install -y libclang-dev clang && \ |
||||
if [ "$BUILDARCH" != "$TARGETARCH" ] && [ "$TARGETARCH" = "arm64" ] ; \ |
||||
then \ |
||||
apt-get install -y g++-aarch64-linux-gnu && \ |
||||
rustup target add aarch64-unknown-linux-gnu ; \ |
||||
fi |
||||
COPY . /oxigraph |
||||
WORKDIR /oxigraph/cli |
||||
RUN if [ "$BUILDARCH" != "$TARGETARCH" ] && [ "$TARGETARCH" = "arm64" ] ; \ |
||||
then \ |
||||
export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc && \ |
||||
export BINDGEN_EXTRA_CLANG_ARGS="--sysroot /usr/aarch64-linux-gnu" && \ |
||||
cargo build --release --target aarch64-unknown-linux-gnu --no-default-features --features rustls-webpki && \ |
||||
mv /oxigraph/target/aarch64-unknown-linux-gnu/release/oxigraph /oxigraph/target/release/oxigraph ; \ |
||||
else \ |
||||
cargo build --release --no-default-features --features rustls-webpki ; \ |
||||
fi |
||||
|
||||
FROM --platform=$TARGETPLATFORM gcr.io/distroless/cc-debian12 |
||||
COPY --from=builder /oxigraph/target/release/oxigraph /usr/local/bin/oxigraph |
||||
ENTRYPOINT [ "/usr/local/bin/oxigraph" ] |
||||
CMD [ "serve", "--location", "/data", "--bind", "0.0.0.0:7878" ] |
||||
@ -1,284 +0,0 @@ |
||||
Oxigraph CLI |
||||
============ |
||||
|
||||
[](https://crates.io/crates/oxigraph-cli) |
||||
[](https://crates.io/crates/oxigraph-cli) |
||||
[](https://anaconda.org/conda-forge/oxigraph-server) |
||||
[](https://github.com/oxigraph/oxigraph/actions) |
||||
[](https://gitter.im/oxigraph/community) |
||||
|
||||
Oxigraph CLI is a graph database implementing the [SPARQL](https://www.w3.org/TR/sparql11-overview/) standard. |
||||
It is packaged as a command-line tool allowing to manipulate RDF files, query them using SPARQL... |
||||
It also allows to spawn a HTTP server on top of the database. |
||||
|
||||
Oxigraph is in heavy development and SPARQL query evaluation has not been optimized yet. |
||||
|
||||
Oxigraph provides different installation methods for Oxigraph CLI: |
||||
* [`cargo install`](#installation) (multiplatform) |
||||
* [A Docker image](#using-a-docker-image) |
||||
* [A Homebrew formula](#homebrew) |
||||
* [A conda-forge package](https://anaconda.org/conda-forge/oxigraph-server) |
||||
* [Pre-built binaries](https://github.com/oxigraph/oxigraph/releases/latest) |
||||
|
||||
It is also usable as [a Rust library](https://crates.io/crates/oxigraph) and as [a Python library](https://pyoxigraph.readthedocs.io/). |
||||
|
||||
Oxigraph implements the following specifications: |
||||
* [SPARQL 1.1 Query](https://www.w3.org/TR/sparql11-query/), [SPARQL 1.1 Update](https://www.w3.org/TR/sparql11-update/), and [SPARQL 1.1 Federated Query](https://www.w3.org/TR/sparql11-federated-query/). |
||||
* [Turtle](https://www.w3.org/TR/turtle/), [TriG](https://www.w3.org/TR/trig/), [N-Triples](https://www.w3.org/TR/n-triples/), [N-Quads](https://www.w3.org/TR/n-quads/), and [RDF/XML](https://www.w3.org/TR/rdf-syntax-grammar/) RDF serialization formats for both data ingestion and retrieval. |
||||
* [SPARQL Query Results XML Format](https://www.w3.org/TR/rdf-sparql-XMLres/), [SPARQL 1.1 Query Results JSON Format](https://www.w3.org/TR/sparql11-results-json/) and [SPARQL 1.1 Query Results CSV and TSV Formats](https://www.w3.org/TR/sparql11-results-csv-tsv/). |
||||
* [SPARQL 1.1 Protocol](https://www.w3.org/TR/sparql11-protocol/#query-operation) and [SPARQL 1.1 Graph Store HTTP Protocol](https://www.w3.org/TR/sparql11-http-rdf-update/). |
||||
|
||||
A preliminary benchmark [is provided](../bench/README.md). |
||||
|
||||
Note that Oxigraph CLI was previously named Oxigraph Server before version 0.4. Older versions are available under [this name](https://crates.io/crates/oxigraph_server). |
||||
|
||||
[](https://repology.org/project/oxigraph/versions) |
||||
|
||||
## Installation |
||||
|
||||
You need to have [a recent stable version of Rust and Cargo installed](https://www.rust-lang.org/tools/install). |
||||
|
||||
To download, build and install the latest released version run `cargo install oxigraph-cli`. |
||||
There is no need to clone the git repository. |
||||
|
||||
To compile the command-line tool from source, clone this git repository including its submodules (`git clone --recursive https://github.com/oxigraph/oxigraph.git`), and execute `cargo build --release` in the `cli` directory to compile the full binary after having downloaded its dependencies. |
||||
It will create a fat binary in `target/release/oxigraph`. |
||||
|
||||
Some build options (cargo features) are available: |
||||
- `rocksdb-pkg-config`: links against an already compiled rocksdb shared library found using [pkg-config](https://crates.io/crates/pkg-config). |
||||
- `native-tls`: Enables Oxigraph HTTP client for query federation using the host OS TLS stack (enabled by default). |
||||
- `rustls-native` Enables Oxigraph HTTP client for query federation using [Rustls](https://crates.io/crates/rustls) and the native certificates. |
||||
- `rustls-webpki` Enables Oxigraph HTTP client for query federation using [Rustls](https://crates.io/crates/rustls) and the [Common CA Database](https://www.ccadb.org/) certificates. |
||||
|
||||
|
||||
## Usage |
||||
|
||||
Run `oxigraph serve --location my_data_storage_directory` to start the server where `my_data_storage_directory` is the directory where you want Oxigraph data to be stored. It listens by default on `localhost:7878`. |
||||
|
||||
The server provides an HTML UI, based on [YASGUI](https://yasgui.triply.cc), with a form to execute SPARQL requests. |
||||
|
||||
It provides the following REST actions: |
||||
* `/query` allows evaluating SPARQL queries against the server repository following the [SPARQL 1.1 Protocol](https://www.w3.org/TR/sparql11-protocol/#query-operation). |
||||
For example: |
||||
```bash |
||||
curl -X POST -H 'Content-Type:application/sparql-query' \ |
||||
--data 'SELECT * WHERE { ?s ?p ?o } LIMIT 10' http://localhost:7878/query |
||||
``` |
||||
This action supports content negotiation and could return [Turtle](https://www.w3.org/TR/turtle/), [N-Triples](https://www.w3.org/TR/n-triples/), [RDF/XML](https://www.w3.org/TR/rdf-syntax-grammar/), [SPARQL Query Results XML Format](https://www.w3.org/TR/rdf-sparql-XMLres/) and [SPARQL Query Results JSON Format](https://www.w3.org/TR/sparql11-results-json/). |
||||
* `/update` allows to execute SPARQL updates against the server repository following the [SPARQL 1.1 Protocol](https://www.w3.org/TR/sparql11-protocol/#update-operation). |
||||
For example: |
||||
```sh |
||||
curl -X POST -H 'Content-Type: application/sparql-update' \ |
||||
--data 'DELETE WHERE { <http://example.com/s> ?p ?o }' http://localhost:7878/update |
||||
``` |
||||
* `/store` allows to retrieve and change the server content using the [SPARQL 1.1 Graph Store HTTP Protocol](https://www.w3.org/TR/sparql11-http-rdf-update/). |
||||
For example: |
||||
```sh |
||||
curl -f -X POST -H 'Content-Type:application/n-triples' \ |
||||
-T MY_FILE.nt "http://localhost:7878/store?graph=http://example.com/g" |
||||
``` |
||||
will add the N-Triples file `MY_FILE.nt` to the server dataset inside of the `http://example.com/g` named graph. |
||||
[Turtle](https://www.w3.org/TR/turtle/), [N-Triples](https://www.w3.org/TR/n-triples/) and [RDF/XML](https://www.w3.org/TR/rdf-syntax-grammar/) are supported. |
||||
It is also possible to `POST`, `PUT` and `GET` the complete RDF dataset on the server using RDF dataset formats ([TriG](https://www.w3.org/TR/trig/) and [N-Quads](https://www.w3.org/TR/n-quads/)) against the `/store` endpoint. |
||||
For example: |
||||
```sh |
||||
curl -f -X POST -H 'Content-Type:application/n-quads' \ |
||||
-T MY_FILE.nq http://localhost:7878/store |
||||
``` |
||||
will add the N-Quads file `MY_FILE.nq` to the server dataset. |
||||
|
||||
Use `oxigraph --help` to see the possible options when starting the server. |
||||
|
||||
It is also possible to load RDF data offline using bulk loading: |
||||
`oxigraph load --location my_data_storage_directory --file my_file.nq` |
||||
|
||||
## Using a Docker image |
||||
|
||||
### Display the help menu |
||||
```sh |
||||
docker run --rm ghcr.io/oxigraph/oxigraph --help |
||||
``` |
||||
|
||||
### Run the Webserver |
||||
Expose the server on port `7878` of the host machine, and save data on the local `./data` folder |
||||
```sh |
||||
docker run --rm -v $PWD/data:/data -p 7878:7878 ghcr.io/oxigraph/oxigraph serve --location /data --bind 0.0.0.0:7878 |
||||
``` |
||||
|
||||
You can then access it from your machine on port `7878`: |
||||
|
||||
```sh |
||||
# Open the GUI in a browser |
||||
firefox http://localhost:7878 |
||||
|
||||
# Post some data |
||||
curl http://localhost:7878/store?default -H 'Content-Type: text/turtle' -T ./data.ttl |
||||
|
||||
# Make a query |
||||
curl -X POST -H 'Accept: application/sparql-results+json' -H 'Content-Type: application/sparql-query' --data 'SELECT * WHERE { ?s ?p ?o } LIMIT 10' http://localhost:7878/query |
||||
|
||||
# Make an UPDATE |
||||
curl -X POST -H 'Content-Type: application/sparql-update' --data 'DELETE WHERE { <http://example.com/s> ?p ?o }' http://localhost:7878/update |
||||
``` |
||||
|
||||
### Run the Web server with basic authentication |
||||
|
||||
It can be useful to make Oxigraph SPARQL endpoint available publicly, with a layer of authentication on `/update` to be able to add data. |
||||
|
||||
You can do so by using a nginx basic authentication in an additional docker container with `docker-compose`. First create a `nginx.conf` file: |
||||
|
||||
```nginx |
||||
daemon off; |
||||
events { |
||||
worker_connections 1024; |
||||
} |
||||
http { |
||||
server { |
||||
server_name localhost; |
||||
listen 7878; |
||||
rewrite ^/(.*) /$1 break; |
||||
proxy_ignore_client_abort on; |
||||
proxy_set_header X-Real-IP $remote_addr; |
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; |
||||
proxy_set_header Host $http_host; |
||||
proxy_set_header Access-Control-Allow-Origin "*"; |
||||
location ~ ^(/|/query)$ { |
||||
proxy_pass http://oxigraph:7878; |
||||
proxy_pass_request_headers on; |
||||
} |
||||
location ~ ^(/update|/store)$ { |
||||
auth_basic "Oxigraph Administrator's Area"; |
||||
auth_basic_user_file /etc/nginx/.htpasswd; |
||||
proxy_pass http://oxigraph:7878; |
||||
proxy_pass_request_headers on; |
||||
} |
||||
} |
||||
} |
||||
``` |
||||
|
||||
Then a `docker-compose.yml` in the same folder, you can change the default user and password in the `environment` section: |
||||
|
||||
```yaml |
||||
version: "3" |
||||
services: |
||||
oxigraph: |
||||
image: ghcr.io/oxigraph/oxigraph:latest |
||||
## To build from local source code: |
||||
# build: |
||||
# context: . |
||||
# dockerfile: server/Dockerfile |
||||
volumes: |
||||
- ./data:/data |
||||
|
||||
nginx-auth: |
||||
image: nginx:1.21.4 |
||||
environment: |
||||
- OXIGRAPH_USER=oxigraph |
||||
- OXIGRAPH_PASSWORD=oxigraphy |
||||
volumes: |
||||
- ./nginx.conf:/etc/nginx/nginx.conf |
||||
## For multiple users: uncomment this line to mount a pre-generated .htpasswd |
||||
# - ./.htpasswd:/etc/nginx/.htpasswd |
||||
ports: |
||||
- "7878:7878" |
||||
entrypoint: "bash -c 'echo -n $OXIGRAPH_USER: >> /etc/nginx/.htpasswd && echo $OXIGRAPH_PASSWORD | openssl passwd -stdin -apr1 >> /etc/nginx/.htpasswd && /docker-entrypoint.sh nginx'" |
||||
``` |
||||
|
||||
Once the `docker-compose.yaml` and `nginx.conf` are ready, start the Oxigraph server and nginx proxy for authentication on http://localhost:7878: |
||||
|
||||
```sh |
||||
docker-compose up |
||||
``` |
||||
|
||||
Then it is possible to update the graph using basic authentication mechanisms. For example with `curl`: change `$OXIGRAPH_USER` and `$OXIGRAPH_PASSWORD`, or set them as environment variables, then run this command to insert a simple triple: |
||||
|
||||
```sh |
||||
curl -X POST -u $OXIGRAPH_USER:$OXIGRAPH_PASSWORD -H 'Content-Type: application/sparql-update' --data 'INSERT DATA { <http://example.com/s> <http://example.com/p> <http://example.com/o> }' http://localhost:7878/update |
||||
``` |
||||
|
||||
In case you want to have multiple users, you can comment the `entrypoint:` line in the `docker-compose.yml` file, uncomment the `.htpasswd` volume, then generate each user in the `.htpasswd` file with this command: |
||||
|
||||
```sh |
||||
htpasswd -Bbn $OXIGRAPH_USER $OXIGRAPH_PASSWORD >> .htpasswd |
||||
``` |
||||
|
||||
### Build the image |
||||
|
||||
You could easily build your own Docker image by cloning this repository with its submodules, and going to the root folder: |
||||
|
||||
```sh |
||||
git clone --recursive https://github.com/oxigraph/oxigraph.git |
||||
cd oxigraph |
||||
``` |
||||
|
||||
Then run this command to build the image locally: |
||||
|
||||
````sh |
||||
docker build -t ghcr.io/oxigraph/oxigraph -f server/Dockerfile . |
||||
```` |
||||
|
||||
## Homebrew |
||||
|
||||
Oxigraph maintains a [Homebrew](https://brew.sh) formula in [a custom tap](https://github.com/oxigraph/homebrew-oxigraph). |
||||
|
||||
To install Oxigraph server using Homebrew do: |
||||
```sh |
||||
brew tap oxigraph/oxigraph |
||||
brew install oxigraph |
||||
``` |
||||
It installs the `oxigraph` binary. [See the usage documentation to know how to use it](#usage). |
||||
|
||||
|
||||
## Systemd |
||||
|
||||
It is possible to run Oxigraph in the background using systemd. |
||||
|
||||
For that, you can use the following `oxigraph.service` file (it might be inserted into `/etc/systemd/system/` or `$HOME/.config/systemd/user`): |
||||
```ini |
||||
[Unit] |
||||
Description=Oxigraph database server |
||||
After=network-online.target |
||||
Wants=network-online.target |
||||
|
||||
[Service] |
||||
Type=notify |
||||
ExecStart=/PATH/TO/oxigraph serve --location /PATH/TO/OXIGRAPH/DATA |
||||
|
||||
[Install] |
||||
WantedBy=multi-user.target |
||||
``` |
||||
|
||||
## Migration guide |
||||
|
||||
### From 0.2 to 0.3 |
||||
* The cli API has been completely rewritten. To start the server run `oxigraph serve --location MY_STORAGE` instead of `oxigraph --file MY_STORAGE`. |
||||
* Fast data bulk loading is not supported using `oxigraph load --location MY_STORAGE --file MY_FILE`. The file format is guessed from the extension (`.nt`, `.ttl`, `.nq`...). |
||||
* [RDF-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html) is now implemented. |
||||
* All operations are now transactional using the "repeatable read" isolation level: |
||||
the store only exposes changes that have been "committed" (i.e. no partial writes) |
||||
and the exposed state does not change for the complete duration of a read operation (e.g. a SPARQL query) or a read/write operation (e.g. a SPARQL update). |
||||
|
||||
|
||||
## Help |
||||
|
||||
Feel free to use [GitHub discussions](https://github.com/oxigraph/oxigraph/discussions) or [the Gitter chat](https://gitter.im/oxigraph/community) to ask questions or talk about Oxigraph. |
||||
[Bug reports](https://github.com/oxigraph/oxigraph/issues) are also very welcome. |
||||
|
||||
If you need advanced support or are willing to pay to get some extra features, feel free to reach out to [Tpt](https://github.com/Tpt). |
||||
|
||||
|
||||
## License |
||||
|
||||
This project is licensed under either of |
||||
|
||||
* Apache License, Version 2.0, ([LICENSE-APACHE](../LICENSE-APACHE) or |
||||
http://www.apache.org/licenses/LICENSE-2.0) |
||||
* MIT license ([LICENSE-MIT](../LICENSE-MIT) or |
||||
http://opensource.org/licenses/MIT) |
||||
|
||||
at your option. |
||||
|
||||
|
||||
### Contribution |
||||
|
||||
Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in Oxigraph by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. |
||||
|
Before Width: | Height: | Size: 4.5 KiB |
File diff suppressed because it is too large
Load Diff
@ -1,27 +0,0 @@ |
||||
<!DOCTYPE html> |
||||
<html lang="en"> |
||||
<head> |
||||
<meta charset="utf-8"> |
||||
<title>Oxigraph server</title> |
||||
<link href="https://unpkg.com/@triply/yasgui@4/build/yasgui.min.css" rel="stylesheet" type="text/css" /> |
||||
<script src="https://unpkg.com/@triply/yasgui@4/build/yasgui.min.js"></script> |
||||
<link rel="icon" type="image/svg+xml" href="/logo.svg"> |
||||
</head> |
||||
<body> |
||||
<div id="yasgui"></div> |
||||
<script> |
||||
const url = window.location.href.endsWith('/') ? window.location.href.slice(0, -1) : window.location.href; |
||||
new Yasgui(document.getElementById("yasgui"), { |
||||
requestConfig: { endpoint: url + "/query" }, |
||||
endpointCatalogueOptions: { |
||||
getData: function () { |
||||
return [ |
||||
{ endpoint: url + "/query" }, |
||||
{ endpoint: url + "/update" }, |
||||
]; |
||||
}, |
||||
keys: [], |
||||
} |
||||
}); |
||||
</script> |
||||
</body> |
||||
@ -1,249 +0,0 @@ |
||||
#![allow(clippy::panic)] |
||||
|
||||
use codspeed_criterion_compat::{criterion_group, criterion_main, Criterion, Throughput}; |
||||
use oxhttp::model::{Method, Request, Status}; |
||||
use oxigraph::io::{RdfFormat, RdfParser}; |
||||
use oxigraph::sparql::{Query, QueryResults, Update}; |
||||
use oxigraph::store::Store; |
||||
use rand::random; |
||||
use std::env::temp_dir; |
||||
use std::fs::{remove_dir_all, File}; |
||||
use std::io::Read; |
||||
use std::path::{Path, PathBuf}; |
||||
use std::str; |
||||
|
||||
fn parse_nt(c: &mut Criterion) { |
||||
let data = read_data("explore-1000.nt.zst"); |
||||
let mut group = c.benchmark_group("parse"); |
||||
group.throughput(Throughput::Bytes(data.len() as u64)); |
||||
group.sample_size(50); |
||||
group.bench_function("parse BSBM explore 1000", |b| { |
||||
b.iter(|| { |
||||
for r in RdfParser::from_format(RdfFormat::NTriples).parse_read(data.as_slice()) { |
||||
r.unwrap(); |
||||
} |
||||
}) |
||||
}); |
||||
group.bench_function("parse BSBM explore 1000 unchecked", |b| { |
||||
b.iter(|| { |
||||
for r in RdfParser::from_format(RdfFormat::NTriples) |
||||
.unchecked() |
||||
.parse_read(data.as_slice()) |
||||
{ |
||||
r.unwrap(); |
||||
} |
||||
}) |
||||
}); |
||||
} |
||||
|
||||
fn store_load(c: &mut Criterion) { |
||||
let data = read_data("explore-1000.nt.zst"); |
||||
let mut group = c.benchmark_group("store load"); |
||||
group.throughput(Throughput::Bytes(data.len() as u64)); |
||||
group.sample_size(10); |
||||
group.bench_function("load BSBM explore 1000 in memory", |b| { |
||||
b.iter(|| { |
||||
let store = Store::new().unwrap(); |
||||
do_load(&store, &data); |
||||
}) |
||||
}); |
||||
group.bench_function("load BSBM explore 1000 in on disk", |b| { |
||||
b.iter(|| { |
||||
let path = TempDir::default(); |
||||
let store = Store::open(&path).unwrap(); |
||||
do_load(&store, &data); |
||||
}) |
||||
}); |
||||
group.bench_function("load BSBM explore 1000 in on disk with bulk load", |b| { |
||||
b.iter(|| { |
||||
let path = TempDir::default(); |
||||
let store = Store::open(&path).unwrap(); |
||||
do_bulk_load(&store, &data); |
||||
}) |
||||
}); |
||||
} |
||||
|
||||
fn do_load(store: &Store, data: &[u8]) { |
||||
store.load_from_read(RdfFormat::NTriples, data).unwrap(); |
||||
store.optimize().unwrap(); |
||||
} |
||||
|
||||
fn do_bulk_load(store: &Store, data: &[u8]) { |
||||
store |
||||
.bulk_loader() |
||||
.load_from_read(RdfFormat::NTriples, data) |
||||
.unwrap(); |
||||
store.optimize().unwrap(); |
||||
} |
||||
|
||||
fn store_query_and_update(c: &mut Criterion) { |
||||
let data = read_data("explore-1000.nt.zst"); |
||||
let operations = bsbm_sparql_operation() |
||||
.into_iter() |
||||
.map(|op| match op { |
||||
RawOperation::Query(q) => Operation::Query(Query::parse(&q, None).unwrap()), |
||||
RawOperation::Update(q) => Operation::Update(Update::parse(&q, None).unwrap()), |
||||
}) |
||||
.collect::<Vec<_>>(); |
||||
let query_operations = operations |
||||
.iter() |
||||
.filter(|o| matches!(o, Operation::Query(_))) |
||||
.cloned() |
||||
.collect::<Vec<_>>(); |
||||
|
||||
let mut group = c.benchmark_group("store operations"); |
||||
group.throughput(Throughput::Elements(operations.len() as u64)); |
||||
group.sample_size(10); |
||||
|
||||
{ |
||||
let memory_store = Store::new().unwrap(); |
||||
do_bulk_load(&memory_store, &data); |
||||
group.bench_function("BSBM explore 1000 query in memory", |b| { |
||||
b.iter(|| run_operation(&memory_store, &query_operations)) |
||||
}); |
||||
group.bench_function("BSBM explore 1000 queryAndUpdate in memory", |b| { |
||||
b.iter(|| run_operation(&memory_store, &operations)) |
||||
}); |
||||
} |
||||
|
||||
{ |
||||
let path = TempDir::default(); |
||||
let disk_store = Store::open(&path).unwrap(); |
||||
do_bulk_load(&disk_store, &data); |
||||
group.bench_function("BSBM explore 1000 query on disk", |b| { |
||||
b.iter(|| run_operation(&disk_store, &query_operations)) |
||||
}); |
||||
group.bench_function("BSBM explore 1000 queryAndUpdate on disk", |b| { |
||||
b.iter(|| run_operation(&disk_store, &operations)) |
||||
}); |
||||
} |
||||
} |
||||
|
||||
fn run_operation(store: &Store, operations: &[Operation]) { |
||||
for operation in operations { |
||||
match operation { |
||||
Operation::Query(q) => match store.query(q.clone()).unwrap() { |
||||
QueryResults::Boolean(_) => (), |
||||
QueryResults::Solutions(s) => { |
||||
for s in s { |
||||
s.unwrap(); |
||||
} |
||||
} |
||||
QueryResults::Graph(g) => { |
||||
for t in g { |
||||
t.unwrap(); |
||||
} |
||||
} |
||||
}, |
||||
Operation::Update(u) => store.update(u.clone()).unwrap(), |
||||
} |
||||
} |
||||
} |
||||
|
||||
fn sparql_parsing(c: &mut Criterion) { |
||||
let operations = bsbm_sparql_operation(); |
||||
let mut group = c.benchmark_group("sparql parsing"); |
||||
group.sample_size(10); |
||||
group.throughput(Throughput::Bytes( |
||||
operations |
||||
.iter() |
||||
.map(|o| match o { |
||||
RawOperation::Query(q) => q.len(), |
||||
RawOperation::Update(u) => u.len(), |
||||
}) |
||||
.sum::<usize>() as u64, |
||||
)); |
||||
group.bench_function("BSBM query and update set", |b| { |
||||
b.iter(|| { |
||||
for operation in &operations { |
||||
match operation { |
||||
RawOperation::Query(q) => { |
||||
Query::parse(q, None).unwrap(); |
||||
} |
||||
RawOperation::Update(u) => { |
||||
Update::parse(u, None).unwrap(); |
||||
} |
||||
} |
||||
} |
||||
}) |
||||
}); |
||||
} |
||||
|
||||
criterion_group!(parse, parse_nt); |
||||
criterion_group!(store, sparql_parsing, store_query_and_update, store_load); |
||||
|
||||
criterion_main!(parse, store); |
||||
|
||||
fn read_data(file: &str) -> Vec<u8> { |
||||
if !Path::new(file).exists() { |
||||
let client = oxhttp::Client::new().with_redirection_limit(5); |
||||
let url = format!("https://github.com/Tpt/bsbm-tools/releases/download/v0.2/{file}"); |
||||
let request = Request::builder(Method::GET, url.parse().unwrap()).build(); |
||||
let response = client.request(request).unwrap(); |
||||
assert_eq!( |
||||
response.status(), |
||||
Status::OK, |
||||
"{}", |
||||
response.into_body().to_string().unwrap() |
||||
); |
||||
std::io::copy(&mut response.into_body(), &mut File::create(file).unwrap()).unwrap(); |
||||
} |
||||
let mut buf = Vec::new(); |
||||
zstd::Decoder::new(File::open(file).unwrap()) |
||||
.unwrap() |
||||
.read_to_end(&mut buf) |
||||
.unwrap(); |
||||
buf |
||||
} |
||||
|
||||
fn bsbm_sparql_operation() -> Vec<RawOperation> { |
||||
String::from_utf8(read_data("mix-exploreAndUpdate-1000.tsv.zst")) |
||||
.unwrap() |
||||
.lines() |
||||
.rev() |
||||
.take(300) // We take only 10 groups
|
||||
.map(|l| { |
||||
let mut parts = l.trim().split('\t'); |
||||
let kind = parts.next().unwrap(); |
||||
let operation = parts.next().unwrap(); |
||||
match kind { |
||||
"query" => RawOperation::Query(operation.into()), |
||||
"update" => RawOperation::Update(operation.into()), |
||||
_ => panic!("Unexpected operation kind {kind}"), |
||||
} |
||||
}) |
||||
.collect() |
||||
} |
||||
|
||||
#[derive(Clone)] |
||||
enum RawOperation { |
||||
Query(String), |
||||
Update(String), |
||||
} |
||||
|
||||
#[allow(clippy::large_enum_variant)] |
||||
#[derive(Clone)] |
||||
enum Operation { |
||||
Query(Query), |
||||
Update(Update), |
||||
} |
||||
|
||||
struct TempDir(PathBuf); |
||||
|
||||
impl Default for TempDir { |
||||
fn default() -> Self { |
||||
Self(temp_dir().join(format!("oxigraph-bench-{}", random::<u128>()))) |
||||
} |
||||
} |
||||
|
||||
impl AsRef<Path> for TempDir { |
||||
fn as_ref(&self) -> &Path { |
||||
&self.0 |
||||
} |
||||
} |
||||
|
||||
impl Drop for TempDir { |
||||
fn drop(&mut self) { |
||||
remove_dir_all(&self.0).unwrap() |
||||
} |
||||
} |
||||
Loading…
Reference in new issue