parent
77edc05ced
commit
fdcaf65a8d
@ -1,45 +0,0 @@ |
|||||||
[package] |
|
||||||
name = "oxigraph-cli" |
|
||||||
version.workspace = true |
|
||||||
authors.workspace = true |
|
||||||
license.workspace = true |
|
||||||
readme = "README.md" |
|
||||||
keywords = ["RDF", "SPARQL", "graph-database", "database"] |
|
||||||
categories = ["command-line-utilities", "database"] |
|
||||||
repository = "https://github.com/oxigraph/oxigraph/tree/main/cli" |
|
||||||
homepage = "https://oxigraph.org/cli/" |
|
||||||
description = """ |
|
||||||
Oxigraph CLI tool and SPARQL HTTP server |
|
||||||
""" |
|
||||||
edition.workspace = true |
|
||||||
rust-version.workspace = true |
|
||||||
|
|
||||||
[[bin]] |
|
||||||
name = "oxigraph" |
|
||||||
path = "src/main.rs" |
|
||||||
doc = false |
|
||||||
|
|
||||||
[features] |
|
||||||
default = [] |
|
||||||
native-tls = ["oxigraph/http-client-native-tls"] |
|
||||||
rustls-native = ["oxigraph/http-client-rustls-native"] |
|
||||||
rustls-webpki = ["oxigraph/http-client-rustls-webpki"] |
|
||||||
|
|
||||||
[dependencies] |
|
||||||
anyhow.workspace = true |
|
||||||
clap = { workspace = true, features = ["derive"] } |
|
||||||
flate2.workspace = true |
|
||||||
oxhttp = { workspace = true, features = ["flate2"] } |
|
||||||
oxigraph.workspace = true |
|
||||||
oxiri.workspace = true |
|
||||||
rand.workspace = true |
|
||||||
rayon-core.workspace = true |
|
||||||
url.workspace = true |
|
||||||
|
|
||||||
[dev-dependencies] |
|
||||||
assert_cmd.workspace = true |
|
||||||
assert_fs.workspace = true |
|
||||||
predicates.workspace = true |
|
||||||
|
|
||||||
[lints] |
|
||||||
workspace = true |
|
||||||
@ -1,25 +0,0 @@ |
|||||||
FROM --platform=$BUILDPLATFORM rust:1-bookworm as builder |
|
||||||
ARG BUILDARCH TARGETARCH |
|
||||||
RUN apt-get update && \ |
|
||||||
apt-get install -y libclang-dev clang && \ |
|
||||||
if [ "$BUILDARCH" != "$TARGETARCH" ] && [ "$TARGETARCH" = "arm64" ] ; \ |
|
||||||
then \ |
|
||||||
apt-get install -y g++-aarch64-linux-gnu && \ |
|
||||||
rustup target add aarch64-unknown-linux-gnu ; \ |
|
||||||
fi |
|
||||||
COPY . /oxigraph |
|
||||||
WORKDIR /oxigraph/cli |
|
||||||
RUN if [ "$BUILDARCH" != "$TARGETARCH" ] && [ "$TARGETARCH" = "arm64" ] ; \ |
|
||||||
then \ |
|
||||||
export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc && \ |
|
||||||
export BINDGEN_EXTRA_CLANG_ARGS="--sysroot /usr/aarch64-linux-gnu" && \ |
|
||||||
cargo build --release --target aarch64-unknown-linux-gnu --no-default-features --features rustls-webpki && \ |
|
||||||
mv /oxigraph/target/aarch64-unknown-linux-gnu/release/oxigraph /oxigraph/target/release/oxigraph ; \ |
|
||||||
else \ |
|
||||||
cargo build --release --no-default-features --features rustls-webpki ; \ |
|
||||||
fi |
|
||||||
|
|
||||||
FROM --platform=$TARGETPLATFORM gcr.io/distroless/cc-debian12 |
|
||||||
COPY --from=builder /oxigraph/target/release/oxigraph /usr/local/bin/oxigraph |
|
||||||
ENTRYPOINT [ "/usr/local/bin/oxigraph" ] |
|
||||||
CMD [ "serve", "--location", "/data", "--bind", "0.0.0.0:7878" ] |
|
||||||
@ -1,284 +0,0 @@ |
|||||||
Oxigraph CLI |
|
||||||
============ |
|
||||||
|
|
||||||
[](https://crates.io/crates/oxigraph-cli) |
|
||||||
[](https://crates.io/crates/oxigraph-cli) |
|
||||||
[](https://anaconda.org/conda-forge/oxigraph-server) |
|
||||||
[](https://github.com/oxigraph/oxigraph/actions) |
|
||||||
[](https://gitter.im/oxigraph/community) |
|
||||||
|
|
||||||
Oxigraph CLI is a graph database implementing the [SPARQL](https://www.w3.org/TR/sparql11-overview/) standard. |
|
||||||
It is packaged as a command-line tool allowing to manipulate RDF files, query them using SPARQL... |
|
||||||
It also allows to spawn a HTTP server on top of the database. |
|
||||||
|
|
||||||
Oxigraph is in heavy development and SPARQL query evaluation has not been optimized yet. |
|
||||||
|
|
||||||
Oxigraph provides different installation methods for Oxigraph CLI: |
|
||||||
* [`cargo install`](#installation) (multiplatform) |
|
||||||
* [A Docker image](#using-a-docker-image) |
|
||||||
* [A Homebrew formula](#homebrew) |
|
||||||
* [A conda-forge package](https://anaconda.org/conda-forge/oxigraph-server) |
|
||||||
* [Pre-built binaries](https://github.com/oxigraph/oxigraph/releases/latest) |
|
||||||
|
|
||||||
It is also usable as [a Rust library](https://crates.io/crates/oxigraph) and as [a Python library](https://pyoxigraph.readthedocs.io/). |
|
||||||
|
|
||||||
Oxigraph implements the following specifications: |
|
||||||
* [SPARQL 1.1 Query](https://www.w3.org/TR/sparql11-query/), [SPARQL 1.1 Update](https://www.w3.org/TR/sparql11-update/), and [SPARQL 1.1 Federated Query](https://www.w3.org/TR/sparql11-federated-query/). |
|
||||||
* [Turtle](https://www.w3.org/TR/turtle/), [TriG](https://www.w3.org/TR/trig/), [N-Triples](https://www.w3.org/TR/n-triples/), [N-Quads](https://www.w3.org/TR/n-quads/), and [RDF/XML](https://www.w3.org/TR/rdf-syntax-grammar/) RDF serialization formats for both data ingestion and retrieval. |
|
||||||
* [SPARQL Query Results XML Format](https://www.w3.org/TR/rdf-sparql-XMLres/), [SPARQL 1.1 Query Results JSON Format](https://www.w3.org/TR/sparql11-results-json/) and [SPARQL 1.1 Query Results CSV and TSV Formats](https://www.w3.org/TR/sparql11-results-csv-tsv/). |
|
||||||
* [SPARQL 1.1 Protocol](https://www.w3.org/TR/sparql11-protocol/#query-operation) and [SPARQL 1.1 Graph Store HTTP Protocol](https://www.w3.org/TR/sparql11-http-rdf-update/). |
|
||||||
|
|
||||||
A preliminary benchmark [is provided](../bench/README.md). |
|
||||||
|
|
||||||
Note that Oxigraph CLI was previously named Oxigraph Server before version 0.4. Older versions are available under [this name](https://crates.io/crates/oxigraph_server). |
|
||||||
|
|
||||||
[](https://repology.org/project/oxigraph/versions) |
|
||||||
|
|
||||||
## Installation |
|
||||||
|
|
||||||
You need to have [a recent stable version of Rust and Cargo installed](https://www.rust-lang.org/tools/install). |
|
||||||
|
|
||||||
To download, build and install the latest released version run `cargo install oxigraph-cli`. |
|
||||||
There is no need to clone the git repository. |
|
||||||
|
|
||||||
To compile the command-line tool from source, clone this git repository including its submodules (`git clone --recursive https://github.com/oxigraph/oxigraph.git`), and execute `cargo build --release` in the `cli` directory to compile the full binary after having downloaded its dependencies. |
|
||||||
It will create a fat binary in `target/release/oxigraph`. |
|
||||||
|
|
||||||
Some build options (cargo features) are available: |
|
||||||
- `rocksdb-pkg-config`: links against an already compiled rocksdb shared library found using [pkg-config](https://crates.io/crates/pkg-config). |
|
||||||
- `native-tls`: Enables Oxigraph HTTP client for query federation using the host OS TLS stack (enabled by default). |
|
||||||
- `rustls-native` Enables Oxigraph HTTP client for query federation using [Rustls](https://crates.io/crates/rustls) and the native certificates. |
|
||||||
- `rustls-webpki` Enables Oxigraph HTTP client for query federation using [Rustls](https://crates.io/crates/rustls) and the [Common CA Database](https://www.ccadb.org/) certificates. |
|
||||||
|
|
||||||
|
|
||||||
## Usage |
|
||||||
|
|
||||||
Run `oxigraph serve --location my_data_storage_directory` to start the server where `my_data_storage_directory` is the directory where you want Oxigraph data to be stored. It listens by default on `localhost:7878`. |
|
||||||
|
|
||||||
The server provides an HTML UI, based on [YASGUI](https://yasgui.triply.cc), with a form to execute SPARQL requests. |
|
||||||
|
|
||||||
It provides the following REST actions: |
|
||||||
* `/query` allows evaluating SPARQL queries against the server repository following the [SPARQL 1.1 Protocol](https://www.w3.org/TR/sparql11-protocol/#query-operation). |
|
||||||
For example: |
|
||||||
```bash |
|
||||||
curl -X POST -H 'Content-Type:application/sparql-query' \ |
|
||||||
--data 'SELECT * WHERE { ?s ?p ?o } LIMIT 10' http://localhost:7878/query |
|
||||||
``` |
|
||||||
This action supports content negotiation and could return [Turtle](https://www.w3.org/TR/turtle/), [N-Triples](https://www.w3.org/TR/n-triples/), [RDF/XML](https://www.w3.org/TR/rdf-syntax-grammar/), [SPARQL Query Results XML Format](https://www.w3.org/TR/rdf-sparql-XMLres/) and [SPARQL Query Results JSON Format](https://www.w3.org/TR/sparql11-results-json/). |
|
||||||
* `/update` allows to execute SPARQL updates against the server repository following the [SPARQL 1.1 Protocol](https://www.w3.org/TR/sparql11-protocol/#update-operation). |
|
||||||
For example: |
|
||||||
```sh |
|
||||||
curl -X POST -H 'Content-Type: application/sparql-update' \ |
|
||||||
--data 'DELETE WHERE { <http://example.com/s> ?p ?o }' http://localhost:7878/update |
|
||||||
``` |
|
||||||
* `/store` allows to retrieve and change the server content using the [SPARQL 1.1 Graph Store HTTP Protocol](https://www.w3.org/TR/sparql11-http-rdf-update/). |
|
||||||
For example: |
|
||||||
```sh |
|
||||||
curl -f -X POST -H 'Content-Type:application/n-triples' \ |
|
||||||
-T MY_FILE.nt "http://localhost:7878/store?graph=http://example.com/g" |
|
||||||
``` |
|
||||||
will add the N-Triples file `MY_FILE.nt` to the server dataset inside of the `http://example.com/g` named graph. |
|
||||||
[Turtle](https://www.w3.org/TR/turtle/), [N-Triples](https://www.w3.org/TR/n-triples/) and [RDF/XML](https://www.w3.org/TR/rdf-syntax-grammar/) are supported. |
|
||||||
It is also possible to `POST`, `PUT` and `GET` the complete RDF dataset on the server using RDF dataset formats ([TriG](https://www.w3.org/TR/trig/) and [N-Quads](https://www.w3.org/TR/n-quads/)) against the `/store` endpoint. |
|
||||||
For example: |
|
||||||
```sh |
|
||||||
curl -f -X POST -H 'Content-Type:application/n-quads' \ |
|
||||||
-T MY_FILE.nq http://localhost:7878/store |
|
||||||
``` |
|
||||||
will add the N-Quads file `MY_FILE.nq` to the server dataset. |
|
||||||
|
|
||||||
Use `oxigraph --help` to see the possible options when starting the server. |
|
||||||
|
|
||||||
It is also possible to load RDF data offline using bulk loading: |
|
||||||
`oxigraph load --location my_data_storage_directory --file my_file.nq` |
|
||||||
|
|
||||||
## Using a Docker image |
|
||||||
|
|
||||||
### Display the help menu |
|
||||||
```sh |
|
||||||
docker run --rm ghcr.io/oxigraph/oxigraph --help |
|
||||||
``` |
|
||||||
|
|
||||||
### Run the Webserver |
|
||||||
Expose the server on port `7878` of the host machine, and save data on the local `./data` folder |
|
||||||
```sh |
|
||||||
docker run --rm -v $PWD/data:/data -p 7878:7878 ghcr.io/oxigraph/oxigraph serve --location /data --bind 0.0.0.0:7878 |
|
||||||
``` |
|
||||||
|
|
||||||
You can then access it from your machine on port `7878`: |
|
||||||
|
|
||||||
```sh |
|
||||||
# Open the GUI in a browser |
|
||||||
firefox http://localhost:7878 |
|
||||||
|
|
||||||
# Post some data |
|
||||||
curl http://localhost:7878/store?default -H 'Content-Type: text/turtle' -T ./data.ttl |
|
||||||
|
|
||||||
# Make a query |
|
||||||
curl -X POST -H 'Accept: application/sparql-results+json' -H 'Content-Type: application/sparql-query' --data 'SELECT * WHERE { ?s ?p ?o } LIMIT 10' http://localhost:7878/query |
|
||||||
|
|
||||||
# Make an UPDATE |
|
||||||
curl -X POST -H 'Content-Type: application/sparql-update' --data 'DELETE WHERE { <http://example.com/s> ?p ?o }' http://localhost:7878/update |
|
||||||
``` |
|
||||||
|
|
||||||
### Run the Web server with basic authentication |
|
||||||
|
|
||||||
It can be useful to make Oxigraph SPARQL endpoint available publicly, with a layer of authentication on `/update` to be able to add data. |
|
||||||
|
|
||||||
You can do so by using a nginx basic authentication in an additional docker container with `docker-compose`. First create a `nginx.conf` file: |
|
||||||
|
|
||||||
```nginx |
|
||||||
daemon off; |
|
||||||
events { |
|
||||||
worker_connections 1024; |
|
||||||
} |
|
||||||
http { |
|
||||||
server { |
|
||||||
server_name localhost; |
|
||||||
listen 7878; |
|
||||||
rewrite ^/(.*) /$1 break; |
|
||||||
proxy_ignore_client_abort on; |
|
||||||
proxy_set_header X-Real-IP $remote_addr; |
|
||||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; |
|
||||||
proxy_set_header Host $http_host; |
|
||||||
proxy_set_header Access-Control-Allow-Origin "*"; |
|
||||||
location ~ ^(/|/query)$ { |
|
||||||
proxy_pass http://oxigraph:7878; |
|
||||||
proxy_pass_request_headers on; |
|
||||||
} |
|
||||||
location ~ ^(/update|/store)$ { |
|
||||||
auth_basic "Oxigraph Administrator's Area"; |
|
||||||
auth_basic_user_file /etc/nginx/.htpasswd; |
|
||||||
proxy_pass http://oxigraph:7878; |
|
||||||
proxy_pass_request_headers on; |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
``` |
|
||||||
|
|
||||||
Then a `docker-compose.yml` in the same folder, you can change the default user and password in the `environment` section: |
|
||||||
|
|
||||||
```yaml |
|
||||||
version: "3" |
|
||||||
services: |
|
||||||
oxigraph: |
|
||||||
image: ghcr.io/oxigraph/oxigraph:latest |
|
||||||
## To build from local source code: |
|
||||||
# build: |
|
||||||
# context: . |
|
||||||
# dockerfile: server/Dockerfile |
|
||||||
volumes: |
|
||||||
- ./data:/data |
|
||||||
|
|
||||||
nginx-auth: |
|
||||||
image: nginx:1.21.4 |
|
||||||
environment: |
|
||||||
- OXIGRAPH_USER=oxigraph |
|
||||||
- OXIGRAPH_PASSWORD=oxigraphy |
|
||||||
volumes: |
|
||||||
- ./nginx.conf:/etc/nginx/nginx.conf |
|
||||||
## For multiple users: uncomment this line to mount a pre-generated .htpasswd |
|
||||||
# - ./.htpasswd:/etc/nginx/.htpasswd |
|
||||||
ports: |
|
||||||
- "7878:7878" |
|
||||||
entrypoint: "bash -c 'echo -n $OXIGRAPH_USER: >> /etc/nginx/.htpasswd && echo $OXIGRAPH_PASSWORD | openssl passwd -stdin -apr1 >> /etc/nginx/.htpasswd && /docker-entrypoint.sh nginx'" |
|
||||||
``` |
|
||||||
|
|
||||||
Once the `docker-compose.yaml` and `nginx.conf` are ready, start the Oxigraph server and nginx proxy for authentication on http://localhost:7878: |
|
||||||
|
|
||||||
```sh |
|
||||||
docker-compose up |
|
||||||
``` |
|
||||||
|
|
||||||
Then it is possible to update the graph using basic authentication mechanisms. For example with `curl`: change `$OXIGRAPH_USER` and `$OXIGRAPH_PASSWORD`, or set them as environment variables, then run this command to insert a simple triple: |
|
||||||
|
|
||||||
```sh |
|
||||||
curl -X POST -u $OXIGRAPH_USER:$OXIGRAPH_PASSWORD -H 'Content-Type: application/sparql-update' --data 'INSERT DATA { <http://example.com/s> <http://example.com/p> <http://example.com/o> }' http://localhost:7878/update |
|
||||||
``` |
|
||||||
|
|
||||||
In case you want to have multiple users, you can comment the `entrypoint:` line in the `docker-compose.yml` file, uncomment the `.htpasswd` volume, then generate each user in the `.htpasswd` file with this command: |
|
||||||
|
|
||||||
```sh |
|
||||||
htpasswd -Bbn $OXIGRAPH_USER $OXIGRAPH_PASSWORD >> .htpasswd |
|
||||||
``` |
|
||||||
|
|
||||||
### Build the image |
|
||||||
|
|
||||||
You could easily build your own Docker image by cloning this repository with its submodules, and going to the root folder: |
|
||||||
|
|
||||||
```sh |
|
||||||
git clone --recursive https://github.com/oxigraph/oxigraph.git |
|
||||||
cd oxigraph |
|
||||||
``` |
|
||||||
|
|
||||||
Then run this command to build the image locally: |
|
||||||
|
|
||||||
````sh |
|
||||||
docker build -t ghcr.io/oxigraph/oxigraph -f server/Dockerfile . |
|
||||||
```` |
|
||||||
|
|
||||||
## Homebrew |
|
||||||
|
|
||||||
Oxigraph maintains a [Homebrew](https://brew.sh) formula in [a custom tap](https://github.com/oxigraph/homebrew-oxigraph). |
|
||||||
|
|
||||||
To install Oxigraph server using Homebrew do: |
|
||||||
```sh |
|
||||||
brew tap oxigraph/oxigraph |
|
||||||
brew install oxigraph |
|
||||||
``` |
|
||||||
It installs the `oxigraph` binary. [See the usage documentation to know how to use it](#usage). |
|
||||||
|
|
||||||
|
|
||||||
## Systemd |
|
||||||
|
|
||||||
It is possible to run Oxigraph in the background using systemd. |
|
||||||
|
|
||||||
For that, you can use the following `oxigraph.service` file (it might be inserted into `/etc/systemd/system/` or `$HOME/.config/systemd/user`): |
|
||||||
```ini |
|
||||||
[Unit] |
|
||||||
Description=Oxigraph database server |
|
||||||
After=network-online.target |
|
||||||
Wants=network-online.target |
|
||||||
|
|
||||||
[Service] |
|
||||||
Type=notify |
|
||||||
ExecStart=/PATH/TO/oxigraph serve --location /PATH/TO/OXIGRAPH/DATA |
|
||||||
|
|
||||||
[Install] |
|
||||||
WantedBy=multi-user.target |
|
||||||
``` |
|
||||||
|
|
||||||
## Migration guide |
|
||||||
|
|
||||||
### From 0.2 to 0.3 |
|
||||||
* The cli API has been completely rewritten. To start the server run `oxigraph serve --location MY_STORAGE` instead of `oxigraph --file MY_STORAGE`. |
|
||||||
* Fast data bulk loading is not supported using `oxigraph load --location MY_STORAGE --file MY_FILE`. The file format is guessed from the extension (`.nt`, `.ttl`, `.nq`...). |
|
||||||
* [RDF-star](https://w3c.github.io/rdf-star/cg-spec/2021-12-17.html) is now implemented. |
|
||||||
* All operations are now transactional using the "repeatable read" isolation level: |
|
||||||
the store only exposes changes that have been "committed" (i.e. no partial writes) |
|
||||||
and the exposed state does not change for the complete duration of a read operation (e.g. a SPARQL query) or a read/write operation (e.g. a SPARQL update). |
|
||||||
|
|
||||||
|
|
||||||
## Help |
|
||||||
|
|
||||||
Feel free to use [GitHub discussions](https://github.com/oxigraph/oxigraph/discussions) or [the Gitter chat](https://gitter.im/oxigraph/community) to ask questions or talk about Oxigraph. |
|
||||||
[Bug reports](https://github.com/oxigraph/oxigraph/issues) are also very welcome. |
|
||||||
|
|
||||||
If you need advanced support or are willing to pay to get some extra features, feel free to reach out to [Tpt](https://github.com/Tpt). |
|
||||||
|
|
||||||
|
|
||||||
## License |
|
||||||
|
|
||||||
This project is licensed under either of |
|
||||||
|
|
||||||
* Apache License, Version 2.0, ([LICENSE-APACHE](../LICENSE-APACHE) or |
|
||||||
http://www.apache.org/licenses/LICENSE-2.0) |
|
||||||
* MIT license ([LICENSE-MIT](../LICENSE-MIT) or |
|
||||||
http://opensource.org/licenses/MIT) |
|
||||||
|
|
||||||
at your option. |
|
||||||
|
|
||||||
|
|
||||||
### Contribution |
|
||||||
|
|
||||||
Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in Oxigraph by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. |
|
||||||
|
Before Width: | Height: | Size: 4.5 KiB |
File diff suppressed because it is too large
Load Diff
@ -1,27 +0,0 @@ |
|||||||
<!DOCTYPE html> |
|
||||||
<html lang="en"> |
|
||||||
<head> |
|
||||||
<meta charset="utf-8"> |
|
||||||
<title>Oxigraph server</title> |
|
||||||
<link href="https://unpkg.com/@triply/yasgui@4/build/yasgui.min.css" rel="stylesheet" type="text/css" /> |
|
||||||
<script src="https://unpkg.com/@triply/yasgui@4/build/yasgui.min.js"></script> |
|
||||||
<link rel="icon" type="image/svg+xml" href="/logo.svg"> |
|
||||||
</head> |
|
||||||
<body> |
|
||||||
<div id="yasgui"></div> |
|
||||||
<script> |
|
||||||
const url = window.location.href.endsWith('/') ? window.location.href.slice(0, -1) : window.location.href; |
|
||||||
new Yasgui(document.getElementById("yasgui"), { |
|
||||||
requestConfig: { endpoint: url + "/query" }, |
|
||||||
endpointCatalogueOptions: { |
|
||||||
getData: function () { |
|
||||||
return [ |
|
||||||
{ endpoint: url + "/query" }, |
|
||||||
{ endpoint: url + "/update" }, |
|
||||||
]; |
|
||||||
}, |
|
||||||
keys: [], |
|
||||||
} |
|
||||||
}); |
|
||||||
</script> |
|
||||||
</body> |
|
||||||
@ -1,249 +0,0 @@ |
|||||||
#![allow(clippy::panic)] |
|
||||||
|
|
||||||
use codspeed_criterion_compat::{criterion_group, criterion_main, Criterion, Throughput}; |
|
||||||
use oxhttp::model::{Method, Request, Status}; |
|
||||||
use oxigraph::io::{RdfFormat, RdfParser}; |
|
||||||
use oxigraph::sparql::{Query, QueryResults, Update}; |
|
||||||
use oxigraph::store::Store; |
|
||||||
use rand::random; |
|
||||||
use std::env::temp_dir; |
|
||||||
use std::fs::{remove_dir_all, File}; |
|
||||||
use std::io::Read; |
|
||||||
use std::path::{Path, PathBuf}; |
|
||||||
use std::str; |
|
||||||
|
|
||||||
fn parse_nt(c: &mut Criterion) { |
|
||||||
let data = read_data("explore-1000.nt.zst"); |
|
||||||
let mut group = c.benchmark_group("parse"); |
|
||||||
group.throughput(Throughput::Bytes(data.len() as u64)); |
|
||||||
group.sample_size(50); |
|
||||||
group.bench_function("parse BSBM explore 1000", |b| { |
|
||||||
b.iter(|| { |
|
||||||
for r in RdfParser::from_format(RdfFormat::NTriples).parse_read(data.as_slice()) { |
|
||||||
r.unwrap(); |
|
||||||
} |
|
||||||
}) |
|
||||||
}); |
|
||||||
group.bench_function("parse BSBM explore 1000 unchecked", |b| { |
|
||||||
b.iter(|| { |
|
||||||
for r in RdfParser::from_format(RdfFormat::NTriples) |
|
||||||
.unchecked() |
|
||||||
.parse_read(data.as_slice()) |
|
||||||
{ |
|
||||||
r.unwrap(); |
|
||||||
} |
|
||||||
}) |
|
||||||
}); |
|
||||||
} |
|
||||||
|
|
||||||
fn store_load(c: &mut Criterion) { |
|
||||||
let data = read_data("explore-1000.nt.zst"); |
|
||||||
let mut group = c.benchmark_group("store load"); |
|
||||||
group.throughput(Throughput::Bytes(data.len() as u64)); |
|
||||||
group.sample_size(10); |
|
||||||
group.bench_function("load BSBM explore 1000 in memory", |b| { |
|
||||||
b.iter(|| { |
|
||||||
let store = Store::new().unwrap(); |
|
||||||
do_load(&store, &data); |
|
||||||
}) |
|
||||||
}); |
|
||||||
group.bench_function("load BSBM explore 1000 in on disk", |b| { |
|
||||||
b.iter(|| { |
|
||||||
let path = TempDir::default(); |
|
||||||
let store = Store::open(&path).unwrap(); |
|
||||||
do_load(&store, &data); |
|
||||||
}) |
|
||||||
}); |
|
||||||
group.bench_function("load BSBM explore 1000 in on disk with bulk load", |b| { |
|
||||||
b.iter(|| { |
|
||||||
let path = TempDir::default(); |
|
||||||
let store = Store::open(&path).unwrap(); |
|
||||||
do_bulk_load(&store, &data); |
|
||||||
}) |
|
||||||
}); |
|
||||||
} |
|
||||||
|
|
||||||
fn do_load(store: &Store, data: &[u8]) { |
|
||||||
store.load_from_read(RdfFormat::NTriples, data).unwrap(); |
|
||||||
store.optimize().unwrap(); |
|
||||||
} |
|
||||||
|
|
||||||
fn do_bulk_load(store: &Store, data: &[u8]) { |
|
||||||
store |
|
||||||
.bulk_loader() |
|
||||||
.load_from_read(RdfFormat::NTriples, data) |
|
||||||
.unwrap(); |
|
||||||
store.optimize().unwrap(); |
|
||||||
} |
|
||||||
|
|
||||||
fn store_query_and_update(c: &mut Criterion) { |
|
||||||
let data = read_data("explore-1000.nt.zst"); |
|
||||||
let operations = bsbm_sparql_operation() |
|
||||||
.into_iter() |
|
||||||
.map(|op| match op { |
|
||||||
RawOperation::Query(q) => Operation::Query(Query::parse(&q, None).unwrap()), |
|
||||||
RawOperation::Update(q) => Operation::Update(Update::parse(&q, None).unwrap()), |
|
||||||
}) |
|
||||||
.collect::<Vec<_>>(); |
|
||||||
let query_operations = operations |
|
||||||
.iter() |
|
||||||
.filter(|o| matches!(o, Operation::Query(_))) |
|
||||||
.cloned() |
|
||||||
.collect::<Vec<_>>(); |
|
||||||
|
|
||||||
let mut group = c.benchmark_group("store operations"); |
|
||||||
group.throughput(Throughput::Elements(operations.len() as u64)); |
|
||||||
group.sample_size(10); |
|
||||||
|
|
||||||
{ |
|
||||||
let memory_store = Store::new().unwrap(); |
|
||||||
do_bulk_load(&memory_store, &data); |
|
||||||
group.bench_function("BSBM explore 1000 query in memory", |b| { |
|
||||||
b.iter(|| run_operation(&memory_store, &query_operations)) |
|
||||||
}); |
|
||||||
group.bench_function("BSBM explore 1000 queryAndUpdate in memory", |b| { |
|
||||||
b.iter(|| run_operation(&memory_store, &operations)) |
|
||||||
}); |
|
||||||
} |
|
||||||
|
|
||||||
{ |
|
||||||
let path = TempDir::default(); |
|
||||||
let disk_store = Store::open(&path).unwrap(); |
|
||||||
do_bulk_load(&disk_store, &data); |
|
||||||
group.bench_function("BSBM explore 1000 query on disk", |b| { |
|
||||||
b.iter(|| run_operation(&disk_store, &query_operations)) |
|
||||||
}); |
|
||||||
group.bench_function("BSBM explore 1000 queryAndUpdate on disk", |b| { |
|
||||||
b.iter(|| run_operation(&disk_store, &operations)) |
|
||||||
}); |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
fn run_operation(store: &Store, operations: &[Operation]) { |
|
||||||
for operation in operations { |
|
||||||
match operation { |
|
||||||
Operation::Query(q) => match store.query(q.clone()).unwrap() { |
|
||||||
QueryResults::Boolean(_) => (), |
|
||||||
QueryResults::Solutions(s) => { |
|
||||||
for s in s { |
|
||||||
s.unwrap(); |
|
||||||
} |
|
||||||
} |
|
||||||
QueryResults::Graph(g) => { |
|
||||||
for t in g { |
|
||||||
t.unwrap(); |
|
||||||
} |
|
||||||
} |
|
||||||
}, |
|
||||||
Operation::Update(u) => store.update(u.clone()).unwrap(), |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
fn sparql_parsing(c: &mut Criterion) { |
|
||||||
let operations = bsbm_sparql_operation(); |
|
||||||
let mut group = c.benchmark_group("sparql parsing"); |
|
||||||
group.sample_size(10); |
|
||||||
group.throughput(Throughput::Bytes( |
|
||||||
operations |
|
||||||
.iter() |
|
||||||
.map(|o| match o { |
|
||||||
RawOperation::Query(q) => q.len(), |
|
||||||
RawOperation::Update(u) => u.len(), |
|
||||||
}) |
|
||||||
.sum::<usize>() as u64, |
|
||||||
)); |
|
||||||
group.bench_function("BSBM query and update set", |b| { |
|
||||||
b.iter(|| { |
|
||||||
for operation in &operations { |
|
||||||
match operation { |
|
||||||
RawOperation::Query(q) => { |
|
||||||
Query::parse(q, None).unwrap(); |
|
||||||
} |
|
||||||
RawOperation::Update(u) => { |
|
||||||
Update::parse(u, None).unwrap(); |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
}) |
|
||||||
}); |
|
||||||
} |
|
||||||
|
|
||||||
criterion_group!(parse, parse_nt); |
|
||||||
criterion_group!(store, sparql_parsing, store_query_and_update, store_load); |
|
||||||
|
|
||||||
criterion_main!(parse, store); |
|
||||||
|
|
||||||
fn read_data(file: &str) -> Vec<u8> { |
|
||||||
if !Path::new(file).exists() { |
|
||||||
let client = oxhttp::Client::new().with_redirection_limit(5); |
|
||||||
let url = format!("https://github.com/Tpt/bsbm-tools/releases/download/v0.2/{file}"); |
|
||||||
let request = Request::builder(Method::GET, url.parse().unwrap()).build(); |
|
||||||
let response = client.request(request).unwrap(); |
|
||||||
assert_eq!( |
|
||||||
response.status(), |
|
||||||
Status::OK, |
|
||||||
"{}", |
|
||||||
response.into_body().to_string().unwrap() |
|
||||||
); |
|
||||||
std::io::copy(&mut response.into_body(), &mut File::create(file).unwrap()).unwrap(); |
|
||||||
} |
|
||||||
let mut buf = Vec::new(); |
|
||||||
zstd::Decoder::new(File::open(file).unwrap()) |
|
||||||
.unwrap() |
|
||||||
.read_to_end(&mut buf) |
|
||||||
.unwrap(); |
|
||||||
buf |
|
||||||
} |
|
||||||
|
|
||||||
fn bsbm_sparql_operation() -> Vec<RawOperation> { |
|
||||||
String::from_utf8(read_data("mix-exploreAndUpdate-1000.tsv.zst")) |
|
||||||
.unwrap() |
|
||||||
.lines() |
|
||||||
.rev() |
|
||||||
.take(300) // We take only 10 groups
|
|
||||||
.map(|l| { |
|
||||||
let mut parts = l.trim().split('\t'); |
|
||||||
let kind = parts.next().unwrap(); |
|
||||||
let operation = parts.next().unwrap(); |
|
||||||
match kind { |
|
||||||
"query" => RawOperation::Query(operation.into()), |
|
||||||
"update" => RawOperation::Update(operation.into()), |
|
||||||
_ => panic!("Unexpected operation kind {kind}"), |
|
||||||
} |
|
||||||
}) |
|
||||||
.collect() |
|
||||||
} |
|
||||||
|
|
||||||
#[derive(Clone)] |
|
||||||
enum RawOperation { |
|
||||||
Query(String), |
|
||||||
Update(String), |
|
||||||
} |
|
||||||
|
|
||||||
#[allow(clippy::large_enum_variant)] |
|
||||||
#[derive(Clone)] |
|
||||||
enum Operation { |
|
||||||
Query(Query), |
|
||||||
Update(Update), |
|
||||||
} |
|
||||||
|
|
||||||
struct TempDir(PathBuf); |
|
||||||
|
|
||||||
impl Default for TempDir { |
|
||||||
fn default() -> Self { |
|
||||||
Self(temp_dir().join(format!("oxigraph-bench-{}", random::<u128>()))) |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
impl AsRef<Path> for TempDir { |
|
||||||
fn as_ref(&self) -> &Path { |
|
||||||
&self.0 |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
impl Drop for TempDir { |
|
||||||
fn drop(&mut self) { |
|
||||||
remove_dir_all(&self.0).unwrap() |
|
||||||
} |
|
||||||
} |
|
||||||
Loading…
Reference in new issue