From b69f84709759b5f3933ce2e9fb158b10450b9965 Mon Sep 17 00:00:00 2001 From: Tpt Date: Sun, 21 Mar 2021 21:11:09 +0100 Subject: [PATCH] Adds BSBM update benchmark --- bench/bsbm-plot.py | 34 +++++++++++++++------------------- bench/bsbm_blazegraph.sh | 8 +++++--- bench/bsbm_graphdb.sh | 10 ++++++---- bench/bsbm_oxigraph.sh | 8 +++++--- 4 files changed, 31 insertions(+), 29 deletions(-) diff --git a/bench/bsbm-plot.py b/bench/bsbm-plot.py index 2362c67a..470b03cb 100644 --- a/bench/bsbm-plot.py +++ b/bench/bsbm-plot.py @@ -4,6 +4,7 @@ from collections import defaultdict from glob import glob from numpy import array + def plot_y_per_x_per_plot(data, xlabel, ylabel, file, log=False): plt.figure(file) @@ -19,24 +20,19 @@ def plot_y_per_x_per_plot(data, xlabel, ylabel, file, log=False): plt.savefig(file) -# BSBM explore -aqet = defaultdict(dict) -for file in glob('bsbm.explore.*.xml'): - run = file.replace('bsbm.explore.', '').replace('.xml', '') - for query in ET.parse(file).getroot().find('queries').findall('query'): - val = float(query.find('aqet').text) - if val > 0: - aqet[run][int(query.attrib['nr'])] = val -plot_y_per_x_per_plot(aqet, 'query id', 'execution time (s)', 'bsbm.explore.svg') - -# BSBM business intelligence -aqet = defaultdict(dict) -for file in glob('bsbm.businessIntelligence.*.xml'): - run = file.replace('bsbm.businessIntelligence.', '').replace('.xml', '') - for query in ET.parse(file).getroot().find('queries').findall('query'): - val = float(query.find('aqet').text) - if val > 0: - aqet[run][int(query.attrib['nr'])] = val -plot_y_per_x_per_plot(aqet, 'query id', 'execution time (s) - log scale', 'bsbm.businessIntelligence.svg', log=True) +def plot_usecase(name: str): + aqet = defaultdict(dict) + for file in glob('bsbm.{}.*.xml'.format(name)): + run = file.replace('bsbm.{}.'.format(name), '').replace('.xml', '') + for query in ET.parse(file).getroot().find('queries').findall('query'): + val = float(query.find('aqet').text) + if val > 0: + aqet[run][int(query.attrib['nr'])] = val + plot_y_per_x_per_plot(aqet, 'query id', 'execution time (s)', 'bsbm.{}.svg'.format(name)) + + +plot_usecase('explore') +plot_usecase('exploreAndUpdate') +plot_usecase('businessIntelligence') plt.show() diff --git a/bench/bsbm_blazegraph.sh b/bench/bsbm_blazegraph.sh index 7663c333..fff54285 100755 --- a/bench/bsbm_blazegraph.sh +++ b/bench/bsbm_blazegraph.sh @@ -1,14 +1,16 @@ #!/usr/bin/env bash DATASET_SIZE=10000 +PARALLELISM=1 cd bsbm-tools -./generate -fc -pc ${DATASET_SIZE} -s nt -fn "explore-${DATASET_SIZE}" +./generate -fc -pc ${DATASET_SIZE} -s nt -fn "explore-${DATASET_SIZE}" -ud -ufn "explore-update-${DATASET_SIZE}" wget https://github.com/blazegraph/database/releases/download/BLAZEGRAPH_RELEASE_2_1_5/blazegraph.jar /usr/lib/jvm/java-8-openjdk/bin/java -server -jar blazegraph.jar & sleep 10 curl -f -X POST -H 'Content-Type:text/plain' --data-binary "@explore-${DATASET_SIZE}.nt" http://localhost:9999/blazegraph/sparql -./testdriver -ucf usecases/explore/sparql.txt -o "../bsbm.explore.blazegraph.${DATASET_SIZE}.2.1.5.xml" http://localhost:9999/blazegraph/sparql -./testdriver -ucf usecases/businessIntelligence/sparql.txt -o "../bsbm.businessIntelligence.blazegraph.${DATASET_SIZE}.2.1.5.xml" http://localhost:9999/blazegraph/sparql +./testdriver -mt ${PARALLELISM} -ucf usecases/explore/sparql.txt -o "../bsbm.explore.blazegraph.${DATASET_SIZE}.${PARALLELISM}.2.1.5.xml" http://localhost:9999/blazegraph/sparql +./testdriver -mt ${PARALLELISM} -ucf usecases/exploreAndUpdate/sparql.txt -o "../bsbm.exploreAndUpdate.blazegraph.${DATASET_SIZE}.${PARALLELISM}.2.1.5.xml" http://localhost:9999/blazegraph/sparql -u http://localhost:9999/blazegraph/sparql -udataset "explore-update-${DATASET_SIZE}.nt" +./testdriver -mt ${PARALLELISM} -ucf usecases/businessIntelligence/sparql.txt -o "../bsbm.businessIntelligence.blazegraph.${DATASET_SIZE}.${PARALLELISM}.2.1.5.xml" http://localhost:9999/blazegraph/sparql kill $! rm blazegraph.jar rm blazegraph.jnl diff --git a/bench/bsbm_graphdb.sh b/bench/bsbm_graphdb.sh index b185f525..78872d60 100755 --- a/bench/bsbm_graphdb.sh +++ b/bench/bsbm_graphdb.sh @@ -1,17 +1,19 @@ #!/usr/bin/env bash -DATASET_SIZE=10000 +DATASET_SIZE=10000 # number of products in the dataset. There is around 350 triples generated by product. +PARALLELISM=5 cd bsbm-tools ./generate -fc -pc ${DATASET_SIZE} -s nt -fn "explore-${DATASET_SIZE}" -export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 +export JAVA_HOME=/usr/lib/jvm/java-11-openjdk ../graphdb-free-9.3.3/bin/graphdb -s -Dgraphdb.logger.root.level=WARN & sleep 10 curl -f -X POST http://localhost:7200/rest/repositories -H 'Content-Type:application/json' -d ' {"id":"test","params":{"ruleset":{"label":"Ruleset","name":"ruleset","value":"empty"},"title":{"label":"Repository title","name":"title","value":"GraphDB Free repository"},"checkForInconsistencies":{"label":"Check for inconsistencies","name":"checkForInconsistencies","value":"false"},"disableSameAs":{"label":"Disable owl:sameAs","name":"disableSameAs","value":"true"},"baseURL":{"label":"Base URL","name":"baseURL","value":"http://example.org/owlim#"},"repositoryType":{"label":"Repository type","name":"repositoryType","value":"file-repository"},"id":{"label":"Repository ID","name":"id","value":"repo-test"},"storageFolder":{"label":"Storage folder","name":"storageFolder","value":"storage"}},"title":"Test","type":"free"} ' curl -f -X PUT -H 'Content-Type:application/n-triples' --data-binary "@explore-${DATASET_SIZE}.nt" http://localhost:7200/repositories/test/statements -./testdriver -ucf usecases/explore/sparql.txt -o "../bsbm.explore.graphdb.${DATASET_SIZE}.9.3.3.xml" http://localhost:7200/repositories/test -./testdriver -ucf usecases/businessIntelligence/sparql.txt -o "../bsbm.businessIntelligence.graphdb.${DATASET_SIZE}.9.3.3.xml" http://localhost:7200/repositories/test +./testdriver -mt ${PARALLELISM} -ucf usecases/explore/sparql.txt -o "../bsbm.explore.graphdb.${DATASET_SIZE}.${PARALLELISM}.9.3.3.xml" http://localhost:7200/repositories/test +./testdriver -mt ${PARALLELISM} -ucf usecases/explore/sparql.txt -o "../bsbm.exploreAndUpdate.graphdb.${DATASET_SIZE}.${PARALLELISM}.9.3.3.xml" http://localhost:7200/repositories/test -u http://localhost:7200/repositories/test -udataset "explore-update-${DATASET_SIZE}.nt" +./testdriver -mt ${PARALLELISM} -ucf usecases/businessIntelligence/sparql.txt -o "../bsbm.businessIntelligence.graphdb.${DATASET_SIZE}.${PARALLELISM}.9.3.3.xml" http://localhost:7200/repositories/test kill $! sleep 5 rm -r ../graphdb-free-9.3.3/data diff --git a/bench/bsbm_oxigraph.sh b/bench/bsbm_oxigraph.sh index d279da84..b8e26348 100755 --- a/bench/bsbm_oxigraph.sh +++ b/bench/bsbm_oxigraph.sh @@ -1,14 +1,16 @@ #!/usr/bin/env bash DATASET_SIZE=10000 # number of products in the dataset. There is around 350 triples generated by product. +PARALLELISM=1 cd bsbm-tools -./generate -fc -pc ${DATASET_SIZE} -s nt -fn "explore-${DATASET_SIZE}" +./generate -fc -pc ${DATASET_SIZE} -s nt -fn "explore-${DATASET_SIZE}" -ud -ufn "explore-update-${DATASET_SIZE}" cargo build --release --manifest-path="../../server/Cargo.toml" ./../../target/release/oxigraph_server --file oxigraph_data --bind 127.0.0.1:7878 & sleep 5 curl -f -X POST -H 'Content-Type:application/n-triples' --data-binary "@explore-${DATASET_SIZE}.nt" http://127.0.0.1:7878/store?default -./testdriver -ucf usecases/explore/sparql.txt -o "../bsbm.explore.oxigraph.${DATASET_SIZE}.main-rocksdb.xml" http://127.0.0.1:7878/query -./testdriver -ucf usecases/businessIntelligence/sparql.txt -o "../bsbm.businessIntelligence.${DATASET_SIZE}.0.1.0-rocksdb.xml" "http://127.0.0.1:7878/query" +./testdriver -mt ${PARALLELISM} -ucf usecases/explore/sparql.txt -o "../bsbm.explore.oxigraph.${DATASET_SIZE}.${PARALLELISM}.main-rocksdb.xml" http://127.0.0.1:7878/query +./testdriver -mt ${PARALLELISM} -ucf usecases/exploreAndUpdate/sparql.txt -o "../bsbm.exploreAndUpdate.oxigraph.${DATASET_SIZE}.${PARALLELISM}.main-rocksdb.xml" http://127.0.0.1:7878/query -u http://127.0.0.1:7878/update -udataset "explore-update-${DATASET_SIZE}.nt" +./testdriver -mt ${PARALLELISM} -ucf usecases/businessIntelligence/sparql.txt -o "../bsbm.businessIntelligence.${DATASET_SIZE}.${PARALLELISM}.main-rocksdb.xml" "http://127.0.0.1:7878/query" kill $! rm -r oxigraph_data rm "explore-${DATASET_SIZE}.nt"