From 4c2eafbfe0a95d86d19580e1ed7f8c625f384ae3 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 9 Aug 2020 17:13:12 +0200 Subject: [PATCH] Updates BSBM benchmark --- bench/README.md | 6 +- bench/bsbm.explore.svg | 804 +++++++++++++++++++++------------------ bench/bsbm_blazegraph.sh | 4 +- bench/bsbm_graphdb.sh | 10 +- bench/bsbm_oxigraph.sh | 13 +- bench/bsbm_virtuoso.sh | 2 +- 6 files changed, 458 insertions(+), 381 deletions(-) diff --git a/bench/README.md b/bench/README.md index 7fa9f334..dbe7c546 100644 --- a/bench/README.md +++ b/bench/README.md @@ -10,7 +10,7 @@ It provides a dataset generator and multiple set of queries grouped by "use case We compare here Oxigraph with some existing SPARQL implementations (Blazegraph, Virtuoso and GraphDB). The dataset used in the following charts is generated with 10k "products" (see [its spec](http://wifo5-03.informatik.uni-mannheim.de/bizer/berlinsparqlbenchmark/spec/Dataset/index.html)). It leads to the creation of 3.5M triples. -It has been executed on a Dell Precision 5520 with 16GB of RAM. For Oxigraph, available memory has been limited to 1GB. +It has been executed on a PrevailPro P3000 with 32GB of RAM. ### Explore The [explore use case](http://wifo5-03.informatik.uni-mannheim.de/bizer/berlinsparqlbenchmark/spec/ExploreUseCase/index.html) is composed of 11 queries that do simple data retrieval. @@ -19,6 +19,7 @@ Query 6 existed in previous versions of the benchmark as is now removed. ![explore use case results](bsbm.explore.svg) + ## How to reproduce the benchmark The code of the benchmark is in the `bsbm-tools` submodule. You should pull it with a `git submodule update` before running the benchmark. To run the benchmark for Oxigraph run `bash bsbm_oxigraph.sh`. It will compile the current Oxigraph code and run the benchmark against it. -You could tweak the number of products in the dataset and the available memory using the environment variables at the beginning of `bsbm_oxigraph.sh`. +You could tweak the number of products in the dataset using the environment variables at the beginning of `bsbm_oxigraph.sh`. To generate the plots run `python3 bsbsm-plot.py`. diff --git a/bench/bsbm.explore.svg b/bench/bsbm.explore.svg index 4bef93e5..f230b089 100644 --- a/bench/bsbm.explore.svg +++ b/bench/bsbm.explore.svg @@ -4,9 +4,7 @@ - + @@ -27,215 +25,215 @@ z " style="fill:#ffffff;"/> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -384,16 +382,17 @@ z +" id="ma87655f08b" style="stroke:#000000;stroke-width:0.8;"/> - + - - + + - - + @@ -426,13 +424,14 @@ z - + - - + + - - + @@ -458,13 +456,14 @@ z - + - - + + - - + @@ -503,13 +501,14 @@ z - + - - + + - - + @@ -557,13 +555,14 @@ z - + - - + + - - - + @@ -607,7 +605,7 @@ z - + @@ -620,8 +618,9 @@ z - - + + - - - - - - + - - - + @@ -780,23 +778,23 @@ z +" id="md44c5b90c8" style="stroke:#000000;stroke-width:0.8;"/> - + - - + + - - + @@ -807,71 +805,164 @@ z - + - - + + - + - + - - + + - + - + - - + + + + + - + - + - - + + + + + + + + + + + + + + + + + + + + + - + - + + + + + + + + + + + + + + + + + - - + + - - - - - - - - - - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + @@ -1097,30 +1187,31 @@ L 414.72 41.472 - - - + - - + + - - - - - - - - - + @@ -1324,50 +1390,18 @@ z - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + - - + + - - - + @@ -1474,25 +1554,25 @@ z - - - - - - - - - - - - + + + + + + + + + + + + - + diff --git a/bench/bsbm_blazegraph.sh b/bench/bsbm_blazegraph.sh index 6f4d6fd1..7663c333 100755 --- a/bench/bsbm_blazegraph.sh +++ b/bench/bsbm_blazegraph.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash -DATASET_SIZE=100000 +DATASET_SIZE=10000 cd bsbm-tools ./generate -fc -pc ${DATASET_SIZE} -s nt -fn "explore-${DATASET_SIZE}" wget https://github.com/blazegraph/database/releases/download/BLAZEGRAPH_RELEASE_2_1_5/blazegraph.jar -/usr/lib/jvm/java-8-openjdk-amd64/bin/java -server -jar blazegraph.jar & +/usr/lib/jvm/java-8-openjdk/bin/java -server -jar blazegraph.jar & sleep 10 curl -f -X POST -H 'Content-Type:text/plain' --data-binary "@explore-${DATASET_SIZE}.nt" http://localhost:9999/blazegraph/sparql ./testdriver -ucf usecases/explore/sparql.txt -o "../bsbm.explore.blazegraph.${DATASET_SIZE}.2.1.5.xml" http://localhost:9999/blazegraph/sparql diff --git a/bench/bsbm_graphdb.sh b/bench/bsbm_graphdb.sh index 51aefd55..b185f525 100755 --- a/bench/bsbm_graphdb.sh +++ b/bench/bsbm_graphdb.sh @@ -1,19 +1,19 @@ #!/usr/bin/env bash -DATASET_SIZE=100000 +DATASET_SIZE=10000 cd bsbm-tools ./generate -fc -pc ${DATASET_SIZE} -s nt -fn "explore-${DATASET_SIZE}" export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 -../graphdb-free-8.11.0/bin/graphdb -s -Dgraphdb.logger.root.level=WARN & +../graphdb-free-9.3.3/bin/graphdb -s -Dgraphdb.logger.root.level=WARN & sleep 10 curl -f -X POST http://localhost:7200/rest/repositories -H 'Content-Type:application/json' -d ' {"id":"test","params":{"ruleset":{"label":"Ruleset","name":"ruleset","value":"empty"},"title":{"label":"Repository title","name":"title","value":"GraphDB Free repository"},"checkForInconsistencies":{"label":"Check for inconsistencies","name":"checkForInconsistencies","value":"false"},"disableSameAs":{"label":"Disable owl:sameAs","name":"disableSameAs","value":"true"},"baseURL":{"label":"Base URL","name":"baseURL","value":"http://example.org/owlim#"},"repositoryType":{"label":"Repository type","name":"repositoryType","value":"file-repository"},"id":{"label":"Repository ID","name":"id","value":"repo-test"},"storageFolder":{"label":"Storage folder","name":"storageFolder","value":"storage"}},"title":"Test","type":"free"} ' curl -f -X PUT -H 'Content-Type:application/n-triples' --data-binary "@explore-${DATASET_SIZE}.nt" http://localhost:7200/repositories/test/statements -./testdriver -ucf usecases/explore/sparql.txt -o "../bsbm.explore.graphdb.${DATASET_SIZE}.8.11.0.xml" http://localhost:7200/repositories/test -./testdriver -ucf usecases/businessIntelligence/sparql.txt -o "../bsbm.businessIntelligence.graphdb.${DATASET_SIZE}.8.11.0.xml" http://localhost:7200/repositories/test +./testdriver -ucf usecases/explore/sparql.txt -o "../bsbm.explore.graphdb.${DATASET_SIZE}.9.3.3.xml" http://localhost:7200/repositories/test +./testdriver -ucf usecases/businessIntelligence/sparql.txt -o "../bsbm.businessIntelligence.graphdb.${DATASET_SIZE}.9.3.3.xml" http://localhost:7200/repositories/test kill $! sleep 5 -rm -r ../graphdb-free-8.11.0/data +rm -r ../graphdb-free-9.3.3/data rm "explore-${DATASET_SIZE}.nt" rm -r td_data diff --git a/bench/bsbm_oxigraph.sh b/bench/bsbm_oxigraph.sh index a2bec3f9..69e0833b 100755 --- a/bench/bsbm_oxigraph.sh +++ b/bench/bsbm_oxigraph.sh @@ -1,19 +1,14 @@ #!/usr/bin/env bash -DATASET_SIZE=100000 # number of products in the dataset. There is around 350 triples generated by product. -MEMORY_SIZE=1000000 # available memory for Oxigraph in KB. Useful to simulate low RAM machines. - +DATASET_SIZE=10000 # number of products in the dataset. There is around 350 triples generated by product. cd bsbm-tools ./generate -fc -pc ${DATASET_SIZE} -s nt -fn "explore-${DATASET_SIZE}" cargo build --release --manifest-path="../../server/Cargo.toml" -( - ulimit -d ${MEMORY_SIZE} - ./../../target/release/oxigraph_server --file oxigraph_data --bind 127.0.0.1:7878 -) & +./../../target/release/oxigraph_server --file oxigraph_data --bind 127.0.0.1:7878 & sleep 5 curl -f -X POST -H 'Content-Type:application/n-triples' --data-binary "@explore-${DATASET_SIZE}.nt" http://127.0.0.1:7878/ -./testdriver -ucf usecases/explore/sparql.txt -o "../bsbm.explore.oxigraph.${DATASET_SIZE}.${MEMORY_SIZE}.$(date +'%Y-%m-%d').xml" http://127.0.0.1:7878/query -./testdriver -ucf usecases/businessIntelligence/sparql.txt -o "../bsbm.businessIntelligence.oxigraph.${DATASET_SIZE}.${MEMORY_SIZE}.$(date +'%Y-%m-%d').xml" "http://127.0.0.1:7878/query" +./testdriver -ucf usecases/explore/sparql.txt -o "../bsbm.explore.oxigraph.${DATASET_SIZE}.0.1.0-rocksdb.xml" http://127.0.0.1:7878/query +./testdriver -ucf usecases/businessIntelligence/sparql.txt -o "../bsbm.businessIntelligence.${DATASET_SIZE}.0.1.0-rocksdb.xml" "http://127.0.0.1:7878/query" kill $! rm -r oxigraph_data rm "explore-${DATASET_SIZE}.nt" diff --git a/bench/bsbm_virtuoso.sh b/bench/bsbm_virtuoso.sh index 6dade7eb..cfe55d11 100755 --- a/bench/bsbm_virtuoso.sh +++ b/bench/bsbm_virtuoso.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -DATASET_SIZE=100000 +DATASET_SIZE=10000 cd bsbm-tools ./generate -fc -pc ${DATASET_SIZE} -s nt -fn "explore-${DATASET_SIZE}" cp ../virtuoso-opensource/database/virtuoso.ini.sample virtuoso.ini