From 5b54f45194473ec147a0fb05685150a495ac7151 Mon Sep 17 00:00:00 2001
From: Tpt <thomaspt@hotmail.fr>
Date: Mon, 13 Sep 2021 21:07:33 +0200
Subject: [PATCH] Improves BSBM benchmark reports

---
 bench/README.md          | 8 ++++++++
 bench/bsbm_blazegraph.sh | 4 ++--
 bench/bsbm_graphdb.sh    | 2 +-
 bench/bsbm_jena.sh       | 2 +-
 bench/bsbm_oxigraph.sh   | 4 ++--
 bench/bsbm_virtuoso.sh   | 5 +++--
 6 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/bench/README.md b/bench/README.md
index 7175d71d..76bc5c15 100644
--- a/bench/README.md
+++ b/bench/README.md
@@ -11,6 +11,7 @@ We compare here Oxigraph with some existing SPARQL implementations (Blazegraph,
 
 The dataset used in the following charts is generated with 10k "products" (see [its spec](http://wifo5-03.informatik.uni-mannheim.de/bizer/berlinsparqlbenchmark/spec/Dataset/index.html)). It leads to the creation of 3.5M triples.
 It has been executed on a PrevailPro P3000 with 32GB of RAM.
+The tests have been executed with a concurrency factor of 5 (i.e. at most 5 queries are send at the same time to the server).
 
 ### Explore
 The [explore use case](http://wifo5-03.informatik.uni-mannheim.de/bizer/berlinsparqlbenchmark/spec/ExploreUseCase/index.html) is composed of 11 queries that do simple data retrieval.
@@ -19,6 +20,13 @@ Query 6 existed in previous versions of the benchmark but is now removed.
 
 ![explore use case results](bsbm.explore.svg)
 
+### Explore and update
+The [explore and update use case](http://wifo5-03.informatik.uni-mannheim.de/bizer/berlinsparqlbenchmark/spec/index.html#usecase_explore_and_update) is composed of the 2 operations of the [update use case](http://wifo5-03.informatik.uni-mannheim.de/bizer/berlinsparqlbenchmark/spec/UpdateUseCase/index.html) (`INSERT DATA` and `DELETE WHERE`) and the 11 queries of the [explore use case](http://wifo5-03.informatik.uni-mannheim.de/bizer/berlinsparqlbenchmark/spec/ExploreUseCase/index.html) and the .
+
+The first two char elements (1 and 2) are the 2 updates and the other (3 to 14) are the 11 queries.
+
+![explore use case results](bsbm.exploreAndUpdate.svg)
+
 <!--
 ### Business Intelligence
 The [business intelligence use case](http://wifo5-03.informatik.uni-mannheim.de/bizer/berlinsparqlbenchmark/spec/BusinessIntelligenceUseCase/index.html) is composed of 8 complex analytics queries.
diff --git a/bench/bsbm_blazegraph.sh b/bench/bsbm_blazegraph.sh
index fff54285..00acbb94 100755
--- a/bench/bsbm_blazegraph.sh
+++ b/bench/bsbm_blazegraph.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 DATASET_SIZE=10000
-PARALLELISM=1
+PARALLELISM=5
 cd bsbm-tools
 ./generate -fc -pc ${DATASET_SIZE} -s nt -fn "explore-${DATASET_SIZE}" -ud -ufn "explore-update-${DATASET_SIZE}"
 wget https://github.com/blazegraph/database/releases/download/BLAZEGRAPH_RELEASE_2_1_5/blazegraph.jar
@@ -10,7 +10,7 @@ sleep 10
 curl -f -X POST -H 'Content-Type:text/plain' --data-binary "@explore-${DATASET_SIZE}.nt" http://localhost:9999/blazegraph/sparql
 ./testdriver -mt ${PARALLELISM} -ucf usecases/explore/sparql.txt -o "../bsbm.explore.blazegraph.${DATASET_SIZE}.${PARALLELISM}.2.1.5.xml" http://localhost:9999/blazegraph/sparql
 ./testdriver -mt ${PARALLELISM} -ucf usecases/exploreAndUpdate/sparql.txt -o "../bsbm.exploreAndUpdate.blazegraph.${DATASET_SIZE}.${PARALLELISM}.2.1.5.xml" http://localhost:9999/blazegraph/sparql -u http://localhost:9999/blazegraph/sparql -udataset "explore-update-${DATASET_SIZE}.nt"
-./testdriver -mt ${PARALLELISM} -ucf usecases/businessIntelligence/sparql.txt -o "../bsbm.businessIntelligence.blazegraph.${DATASET_SIZE}.${PARALLELISM}.2.1.5.xml" http://localhost:9999/blazegraph/sparql
+#./testdriver -mt ${PARALLELISM} -ucf usecases/businessIntelligence/sparql.txt -o "../bsbm.businessIntelligence.blazegraph.${DATASET_SIZE}.${PARALLELISM}.2.1.5.xml" http://localhost:9999/blazegraph/sparql
 kill $!
 rm blazegraph.jar
 rm blazegraph.jnl
diff --git a/bench/bsbm_graphdb.sh b/bench/bsbm_graphdb.sh
index 78872d60..86c4ba47 100755
--- a/bench/bsbm_graphdb.sh
+++ b/bench/bsbm_graphdb.sh
@@ -13,7 +13,7 @@ curl -f -X POST http://localhost:7200/rest/repositories -H 'Content-Type:applica
 curl -f -X PUT -H 'Content-Type:application/n-triples' --data-binary "@explore-${DATASET_SIZE}.nt" http://localhost:7200/repositories/test/statements
 ./testdriver -mt ${PARALLELISM} -ucf usecases/explore/sparql.txt -o "../bsbm.explore.graphdb.${DATASET_SIZE}.${PARALLELISM}.9.3.3.xml" http://localhost:7200/repositories/test
 ./testdriver -mt ${PARALLELISM} -ucf usecases/explore/sparql.txt -o "../bsbm.exploreAndUpdate.graphdb.${DATASET_SIZE}.${PARALLELISM}.9.3.3.xml" http://localhost:7200/repositories/test -u http://localhost:7200/repositories/test -udataset "explore-update-${DATASET_SIZE}.nt"
-./testdriver -mt ${PARALLELISM} -ucf usecases/businessIntelligence/sparql.txt -o "../bsbm.businessIntelligence.graphdb.${DATASET_SIZE}.${PARALLELISM}.9.3.3.xml" http://localhost:7200/repositories/test
+#./testdriver -mt ${PARALLELISM} -ucf usecases/businessIntelligence/sparql.txt -o "../bsbm.businessIntelligence.graphdb.${DATASET_SIZE}.${PARALLELISM}.9.3.3.xml" http://localhost:7200/repositories/test
 kill $!
 sleep 5
 rm -r ../graphdb-free-9.3.3/data
diff --git a/bench/bsbm_jena.sh b/bench/bsbm_jena.sh
index b1f218e0..d1d76def 100755
--- a/bench/bsbm_jena.sh
+++ b/bench/bsbm_jena.sh
@@ -14,7 +14,7 @@ curl -f -X POST -H 'Content-Type:text/plain' --data-binary "@explore-${DATASET_S
 sleep 60
 ./testdriver -mt ${PARALLELISM} -ucf usecases/explore/sparql.txt -o "../bsbm.explore.jena.${DATASET_SIZE}.${PARALLELISM}.4.1.0.xml" http://localhost:3030/bsbm/query
 ./testdriver -mt ${PARALLELISM} -ucf usecases/exploreAndUpdate/sparql.txt -o "../bsbm.exploreAndUpdate.jena.${DATASET_SIZE}.${PARALLELISM}.4.1.0.xml" http://localhost:3030/bsbm/query -u http://localhost:3030/bsbm/update -udataset "explore-update-${DATASET_SIZE}.nt"
-./testdriver -mt ${PARALLELISM} -ucf usecases/businessIntelligence/sparql.txt -o "../bsbm.businessIntelligence.jena.${DATASET_SIZE}.${PARALLELISM}.4.1.0.xml" http://localhost:3030/bsbm/query
+#./testdriver -mt ${PARALLELISM} -ucf usecases/businessIntelligence/sparql.txt -o "../bsbm.businessIntelligence.jena.${DATASET_SIZE}.${PARALLELISM}.4.1.0.xml" http://localhost:3030/bsbm/query
 kill $!
 rm "explore-${DATASET_SIZE}.nt"
 rm -r td_data
diff --git a/bench/bsbm_oxigraph.sh b/bench/bsbm_oxigraph.sh
index c7d9174f..2e190439 100755
--- a/bench/bsbm_oxigraph.sh
+++ b/bench/bsbm_oxigraph.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 DATASET_SIZE=10000 # number of products in the dataset. There is around 350 triples generated by product.
-PARALLELISM=1
+PARALLELISM=5
 cd bsbm-tools
 ./generate -fc -pc ${DATASET_SIZE} -s nt -fn "explore-${DATASET_SIZE}" -ud -ufn "explore-update-${DATASET_SIZE}"
 cargo build --release --manifest-path="../../server/Cargo.toml"
@@ -10,7 +10,7 @@ sleep 5
 curl -f -X POST -H 'Content-Type:application/n-triples' --data-binary "@explore-${DATASET_SIZE}.nt" http://127.0.0.1:7878/store?default
 ./testdriver -mt ${PARALLELISM} -ucf usecases/explore/sparql.txt -o "../bsbm.explore.oxigraph.${DATASET_SIZE}.${PARALLELISM}.main.xml" http://127.0.0.1:7878/query
 ./testdriver -mt ${PARALLELISM} -ucf usecases/exploreAndUpdate/sparql.txt -o "../bsbm.exploreAndUpdate.oxigraph.${DATASET_SIZE}.${PARALLELISM}.main.xml" http://127.0.0.1:7878/query -u http://127.0.0.1:7878/update -udataset "explore-update-${DATASET_SIZE}.nt"
-./testdriver -mt ${PARALLELISM} -ucf usecases/businessIntelligence/sparql.txt -o "../bsbm.businessIntelligence.${DATASET_SIZE}.${PARALLELISM}.main.xml" "http://127.0.0.1:7878/query"
+#./testdriver -mt ${PARALLELISM} -ucf usecases/businessIntelligence/sparql.txt -o "../bsbm.businessIntelligence.${DATASET_SIZE}.${PARALLELISM}.main.xml" "http://127.0.0.1:7878/query"
 kill $!
 rm -r oxigraph_data
 rm "explore-${DATASET_SIZE}.nt"
diff --git a/bench/bsbm_virtuoso.sh b/bench/bsbm_virtuoso.sh
index cfe55d11..ddbf7d2f 100755
--- a/bench/bsbm_virtuoso.sh
+++ b/bench/bsbm_virtuoso.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 DATASET_SIZE=10000
+PARALLELISM=5
 cd bsbm-tools
 ./generate -fc -pc ${DATASET_SIZE} -s nt -fn "explore-${DATASET_SIZE}"
 cp ../virtuoso-opensource/database/virtuoso.ini.sample virtuoso.ini
@@ -12,8 +13,8 @@ SPARQL CREATE GRAPH <urn:graph:test>;
 ld_dir('$(realpath .)', 'explore-${DATASET_SIZE}.nt', 'urn:graph:test');
 rdf_loader_run();
 EOF
-./testdriver -ucf usecases/explore/sparql.txt -o "../bsbm.explore.virtuoso.${DATASET_SIZE}.7.2.5.xml" 'http://localhost:8890/sparql?graph-uri=urn:graph:test'
-./testdriver -ucf usecases/businessIntelligence/sparql.txt -o "../bsbm.businessIntelligence.virtuoso.${DATASET_SIZE}.7.2.5.xml" 'http://localhost:8890/sparql?graph-uri=urn:graph:test'
+./testdriver -mt ${PARALLELISM} -ucf usecases/explore/sparql.txt -o "../bsbm.explore.virtuoso.${DATASET_SIZE}.7.2.5.xml" 'http://localhost:8890/sparql?graph-uri=urn:graph:test'
+#./testdriver -mt ${PARALLELISM} -ucf usecases/businessIntelligence/sparql.txt -o "../bsbm.businessIntelligence.virtuoso.${DATASET_SIZE}.7.2.5.xml" 'http://localhost:8890/sparql?graph-uri=urn:graph:test'
 kill $!
 rm -r ../database
 rm "explore-${DATASET_SIZE}.nt"