diff --git a/.gitignore b/.gitignore index 702da3b..a27ca10 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,6 @@ hs_err_pid* **/*.iml **/.idea venv/ + +# Project specific files +runlogs diff --git a/scripts/STwavequide.json b/conf/STwavequide.json similarity index 97% rename from scripts/STwavequide.json rename to conf/STwavequide.json index c7da4ee..36cd767 100644 --- a/scripts/STwavequide.json +++ b/conf/STwavequide.json @@ -2,7 +2,9 @@ "name" : "stwaveguide", "dataset" : "/hdd1/apacaci/datasets/yago2s/tsv/yago2s_full_shuffle_virtuoso_tem53.hash", "input-type" : "hash", - "report-folder" : "/home/apacaci/sgraffito/rpq/results-stwaveguide", + "report-folder" : "/home/apacaci/sgraffito/rpq/results-stwaveguide-50M10M", + "window-size" : 50000000, + "slide-size" : 10000000, "buffer-size" : 50000000, "heap-size" : 128, "timeout" : 600, diff --git a/conf/StackOverflow-simple.json b/conf/StackOverflow-simple.json new file mode 100644 index 0000000..3f1f4ad --- /dev/null +++ b/conf/StackOverflow-simple.json @@ -0,0 +1,156 @@ +{ + "name" : "stackoverflow", + "dataset" : "/hdd1/apacaci/datasets/stackoverflow/labeled-sorted.txt", + "input-type" : "snap-sx", + "report-folder" : "/home/apacaci/sgraffito/results/stackoverflow", + "buffer-size" : 50000000, + "heap-size" : 192, + "timeout" : 150, + "executable" : "/home/apacaci/sgraffito/rpq/target/streamingrpq-1.0-SNAPSHOT-jar-with-dependencies.jar", + "runs" : [ + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "simple", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "simple", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "simple", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q", + "a2q" + ] + }, + { + "query-name" : "robotic4", + "index" : 1, + "semantics" : "simple", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "simple", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "simple", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "simple", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic8", + "index" : 1, + "semantics" : "simple", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic9", + "index" : 1, + "semantics" : "simple", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic10", + "index" : 1, + "semantics" : "simple", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "simple", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + } + ] +} diff --git a/conf/StackOverflow.json b/conf/StackOverflow.json new file mode 100644 index 0000000..aafe1c7 --- /dev/null +++ b/conf/StackOverflow.json @@ -0,0 +1,1164 @@ +{ + "name" : "stackoverflow", + "dataset" : "/hdd1/apacaci/datasets/stackoverflow/labeled-sorted.txt", + "input-type" : "snap-sx", + "report-folder" : "/home/apacaci/sgraffito/results/stackoverflow", + "buffer-size" : 50000000, + "heap-size" : 192, + "timeout" : 300, + "executable" : "/home/apacaci/sgraffito/rpq/target/streamingrpq-1.0-SNAPSHOT-jar-with-dependencies.jar", + "runs" : [ + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q", + "a2q" + ] + }, + { + "query-name" : "robotic4", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic8", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic9", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic10", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 864000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 864000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 864000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q", + "a2q" + ] + }, + { + "query-name" : "robotic4", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 864000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 864000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 864000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 864000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic8", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 864000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic9", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 864000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic10", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 864000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 864000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 1728000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 1728000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 1728000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q", + "a2q" + ] + }, + { + "query-name" : "robotic4", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 1728000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 1728000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 1728000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 1728000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic8", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 1728000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic9", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 1728000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic10", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 1728000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 1728000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 3456000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 3456000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 3456000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q", + "a2q" + ] + }, + { + "query-name" : "robotic4", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 3456000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 3456000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 3456000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 3456000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic8", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 3456000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic9", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 3456000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic10", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 3456000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 3456000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 4320000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 4320000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 4320000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q", + "a2q" + ] + }, + { + "query-name" : "robotic4", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 4320000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 4320000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 4320000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 4320000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic8", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 4320000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic9", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 4320000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic10", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 4320000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 4320000, + "slide-size" : 86400, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 43200, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 43200, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 43200, + "labels" : [ + "a2q", + "c2a", + "c2q", + "a2q" + ] + }, + { + "query-name" : "robotic4", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 43200, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 43200, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 43200, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 43200, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic8", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 43200, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic9", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 43200, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic10", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 43200, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 43200, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 129600, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 129600, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 129600, + "labels" : [ + "a2q", + "c2a", + "c2q", + "a2q" + ] + }, + { + "query-name" : "robotic4", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 129600, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 129600, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 129600, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 129600, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic8", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 129600, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic9", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 129600, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic10", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 129600, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 129600, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 172800, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 172800, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 172800, + "labels" : [ + "a2q", + "c2a", + "c2q", + "a2q" + ] + }, + { + "query-name" : "robotic4", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 172800, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 172800, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 172800, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 172800, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic8", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 172800, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic9", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 172800, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic10", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 172800, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "arbitrary", + "thread-count" : 32, + "window-size" : 2629743, + "slide-size" : 172800, + "labels" : [ + "a2q", + "c2a", + "c2q" + ] + } + ] +} diff --git a/conf/ldbc-sf1-arbitrary.json b/conf/ldbc-sf1-arbitrary.json new file mode 100644 index 0000000..9087ca1 --- /dev/null +++ b/conf/ldbc-sf1-arbitrary.json @@ -0,0 +1,101 @@ +{ + "name" : "ldbc-sf1", + "dataset" : "/hdd1/apacaci/datasets/ldbc/sf1/update-sorted-unixts.csv", + "input-type" : "ldbc", + "report-folder" : "/home/apacaci/sgraffito/results/ldbc-sf1/", + "buffer-size" : 50000000, + "heap-size" : 192, + "timeout" : 300, + "executable" : "/home/apacaci/sgraffito/rpq/target/streamingrpq-1.0-SNAPSHOT-jar-with-dependencies.jar", + "runs" : [ + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "replyOf" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf" + ] + }, + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf", + "hasCreator", + "knows" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf", + "hasCreator" + ] + }, + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "replyOf", + "hasCreator", + "knows" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "knows", + "likes", + "replyOf" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "creatorOf", + "replyOf", + "hasCreator" + ] + } + ] +} diff --git a/conf/ldbc-sf10-arbitrary.json b/conf/ldbc-sf10-arbitrary.json new file mode 100644 index 0000000..fc27169 --- /dev/null +++ b/conf/ldbc-sf10-arbitrary.json @@ -0,0 +1,813 @@ +{ + "name" : "ldbc-sf10", + "dataset" : "/hdd1/apacaci/datasets/ldbc/sf10/update-sorted-unixts.csv", + "input-type" : "ldbc", + "report-folder" : "/home/apacaci/sgraffito/results/ldbc-sf10/", + "buffer-size" : 50000000, + "heap-size" : 192, + "timeout" : 300, + "executable" : "/home/apacaci/sgraffito/rpq/target/streamingrpq-1.0-SNAPSHOT-jar-with-dependencies.jar", + "runs" : [ + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "replyOf" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf" + ] + }, + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf", + "hasCreator", + "knows" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf", + "hasCreator" + ] + }, + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "replyOf", + "hasCreator", + "knows" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "knows", + "likes", + "replyOf" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "creatorOf", + "replyOf", + "hasCreator" + ] + }, + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 1728000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "replyOf" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 1728000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf" + ] + }, + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 1728000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf", + "hasCreator", + "knows" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 1728000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf", + "hasCreator" + ] + }, + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 1728000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "replyOf", + "hasCreator", + "knows" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 1728000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "knows", + "likes", + "replyOf" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 1728000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "creatorOf", + "replyOf", + "hasCreator" + ] + }, + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 259200, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "replyOf" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 259200, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf" + ] + }, + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 259200, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf", + "hasCreator", + "knows" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 259200, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf", + "hasCreator" + ] + }, + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 259200, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "replyOf", + "hasCreator", + "knows" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 259200, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "knows", + "likes", + "replyOf" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 259200, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "creatorOf", + "replyOf", + "hasCreator" + ] + }, + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 3456000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "replyOf" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 3456000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf" + ] + }, + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 3456000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf", + "hasCreator", + "knows" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 3456000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf", + "hasCreator" + ] + }, + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 3456000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "replyOf", + "hasCreator", + "knows" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 3456000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "knows", + "likes", + "replyOf" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 3456000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "creatorOf", + "replyOf", + "hasCreator" + ] + }, + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 4320000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "replyOf" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 4320000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf" + ] + }, + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 4320000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf", + "hasCreator", + "knows" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 4320000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf", + "hasCreator" + ] + }, + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 4320000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "replyOf", + "hasCreator", + "knows" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 4320000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "knows", + "likes", + "replyOf" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 4320000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "creatorOf", + "replyOf", + "hasCreator" + ] + }, + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 43200, + "thread-count" : 32, + "labels" : [ + "replyOf" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 43200, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf" + ] + }, + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 43200, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf", + "hasCreator", + "knows" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 43200, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf", + "hasCreator" + ] + }, + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 43200, + "thread-count" : 32, + "labels" : [ + "replyOf", + "hasCreator", + "knows" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 43200, + "thread-count" : 32, + "labels" : [ + "knows", + "likes", + "replyOf" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 43200, + "thread-count" : 32, + "labels" : [ + "creatorOf", + "replyOf", + "hasCreator" + ] + }, + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 129600, + "thread-count" : 32, + "labels" : [ + "replyOf" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 129600, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf" + ] + }, + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 129600, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf", + "hasCreator", + "knows" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 129600, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf", + "hasCreator" + ] + }, + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 129600, + "thread-count" : 32, + "labels" : [ + "replyOf", + "hasCreator", + "knows" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 129600, + "thread-count" : 32, + "labels" : [ + "knows", + "likes", + "replyOf" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 129600, + "thread-count" : 32, + "labels" : [ + "creatorOf", + "replyOf", + "hasCreator" + ] + }, + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 172800, + "thread-count" : 32, + "labels" : [ + "replyOf" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 172800, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf" + ] + }, + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 172800, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf", + "hasCreator", + "knows" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 172800, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf", + "hasCreator" + ] + }, + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 172800, + "thread-count" : 32, + "labels" : [ + "replyOf", + "hasCreator", + "knows" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 172800, + "thread-count" : 32, + "labels" : [ + "knows", + "likes", + "replyOf" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 172800, + "thread-count" : 32, + "labels" : [ + "creatorOf", + "replyOf", + "hasCreator" + ] + }, + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "simple", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "replyOf" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "simple", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf" + ] + }, + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "simple", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf", + "hasCreator", + "knows" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "simple", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf", + "hasCreator" + ] + }, + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "simple", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "replyOf", + "hasCreator", + "knows" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "simple", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "knows", + "likes", + "replyOf" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "simple", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "creatorOf", + "replyOf", + "hasCreator" + ] + } + ] +} diff --git a/conf/ldbc-sf10-delete.json b/conf/ldbc-sf10-delete.json new file mode 100644 index 0000000..360e2f6 --- /dev/null +++ b/conf/ldbc-sf10-delete.json @@ -0,0 +1,259 @@ +{ + "name" : "ldbc-sf10", + "dataset" : "/hdd1/apacaci/datasets/ldbc/sf10/update-sorted-unixts.csv", + "input-type" : "ldbc", + "report-folder" : "/home/apacaci/sgraffito/results/ldbc-sf10/", + "buffer-size" : 50000000, + "heap-size" : 192, + "timeout" : 300, + "executable" : "/home/apacaci/sgraffito/rpq/target/streamingrpq-1.0-SNAPSHOT-jar-with-dependencies.jar", + "runs" : [ + + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "delete-ratio" : 2, + "labels" : [ + "likes", + "replyOf", + "hasCreator", + "knows" + ] + }, + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "delete-ratio" : 2, + "labels" : [ + "replyOf", + "hasCreator", + "knows" + ] + }, + + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "delete-ratio" : 1, + "labels" : [ + "likes", + "replyOf", + "hasCreator", + "knows" + ] + }, + + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "delete-ratio" : 1, + "labels" : [ + "replyOf", + "hasCreator", + "knows" + ] + }, + + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "delete-ratio" : 3, + "labels" : [ + "likes", + "replyOf", + "hasCreator", + "knows" + ] + }, + + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "delete-ratio" : 3, + "labels" : [ + "replyOf", + "hasCreator", + "knows" + ] + }, + + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "delete-ratio" : 4, + "labels" : [ + "likes", + "replyOf", + "hasCreator", + "knows" + ] + }, + + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "delete-ratio" : 4, + "labels" : [ + "replyOf", + "hasCreator", + "knows" + ] + }, + + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "delete-ratio" : 5, + "labels" : [ + "likes", + "replyOf", + "hasCreator", + "knows" + ] + }, + + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "delete-ratio" : 5, + "labels" : [ + "replyOf", + "hasCreator", + "knows" + ] + }, + + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "delete-ratio" : 6, + "labels" : [ + "likes", + "replyOf", + "hasCreator", + "knows" + ] + }, + + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "delete-ratio" : 6, + "labels" : [ + "replyOf", + "hasCreator", + "knows" + ] + }, + + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "delete-ratio" : 8, + "labels" : [ + "likes", + "replyOf", + "hasCreator", + "knows" + ] + }, + + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "delete-ratio" : 8, + "labels" : [ + "replyOf", + "hasCreator", + "knows" + ] + }, + + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "delete-ratio" : 10, + "labels" : [ + "likes", + "replyOf", + "hasCreator", + "knows" + ] + }, + + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "delete-ratio" : 10, + "labels" : [ + "replyOf", + "hasCreator", + "knows" + ] + } + ] +} diff --git a/conf/ldbc-sf3-arbitrary.json b/conf/ldbc-sf3-arbitrary.json new file mode 100644 index 0000000..1a15b61 --- /dev/null +++ b/conf/ldbc-sf3-arbitrary.json @@ -0,0 +1,101 @@ +{ + "name" : "ldbc-sf1", + "dataset" : "/hdd1/apacaci/datasets/ldbc/sf3/update-sorted-unixts.csv", + "input-type" : "ldbc", + "report-folder" : "/home/apacaci/sgraffito/results/ldbc-sf3/", + "buffer-size" : 50000000, + "heap-size" : 192, + "timeout" : 300, + "executable" : "/home/apacaci/sgraffito/rpq/target/streamingrpq-1.0-SNAPSHOT-jar-with-dependencies.jar", + "runs" : [ + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "replyOf" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf" + ] + }, + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf", + "hasCreator", + "knows" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf", + "hasCreator" + ] + }, + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "replyOf", + "hasCreator", + "knows" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "knows", + "likes", + "replyOf" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "creatorOf", + "replyOf", + "hasCreator" + ] + } + ] +} diff --git a/conf/ldbc-sf30-arbitrary.json b/conf/ldbc-sf30-arbitrary.json new file mode 100644 index 0000000..939c88b --- /dev/null +++ b/conf/ldbc-sf30-arbitrary.json @@ -0,0 +1,101 @@ +{ + "name" : "ldbc-sf30", + "dataset" : "/hdd1/apacaci/datasets/ldbc/sf30/update-sorted-unixts.csv", + "input-type" : "ldbc", + "report-folder" : "/home/apacaci/sgraffito/results/ldbc-sf30/", + "buffer-size" : 50000000, + "heap-size" : 192, + "timeout" : 300, + "executable" : "/home/apacaci/sgraffito/rpq/target/streamingrpq-1.0-SNAPSHOT-jar-with-dependencies.jar", + "runs" : [ + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "replyOf" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf" + ] + }, + { + "query-name" : "robotic3", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf", + "hasCreator", + "knows" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "likes", + "replyOf", + "hasCreator" + ] + }, + { + "query-name" : "robotic6", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "replyOf", + "hasCreator", + "knows" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "knows", + "likes", + "replyOf" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 864000, + "slide-size" : 86400, + "thread-count" : 32, + "labels" : [ + "creatorOf", + "replyOf", + "hasCreator" + ] + } + ] +} diff --git a/conf/watdiv-sf100.json b/conf/watdiv-sf100.json new file mode 100644 index 0000000..4e7c38d --- /dev/null +++ b/conf/watdiv-sf100.json @@ -0,0 +1,79 @@ +{ + "name" : "watdiv-sf100", + "dataset" : "/hdd1/apacaci/datasets/stream-watdiv/stream-sf100", + "input-type" : "ldbc", + "report-folder" : "/home/apacaci/sgraffito/results/watdiv-sf100/", + "buffer-size" : 50000000, + "heap-size" : 192, + "timeout" : 600, + "executable" : "/home/apacaci/sgraffito/rpq/target/streamingrpq-1.0-SNAPSHOT-jar-with-dependencies.jar", + "runs" : [ + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 36000000, + "slide-size" : 3600000, + "thread-count" : 32, + "delete-ratio" : 0, + "labels" : [ + "http://db.uwaterloo.ca/~galuc/wsdbm/follows" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 36000000, + "slide-size" : 3600000, + "thread-count" : 32, + "delete-ratio" : 0, + "labels" : [ + "http://purl.org/stuff/rev#reviewer", + "http://db.uwaterloo.ca/~galuc/wsdbm/follows" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 36000000, + "slide-size" : 3600000, + "thread-count" : 32, + "delete-ratio" : 0, + "labels" : [ + "http://purl.org/stuff/rev#reviewer", + "http://db.uwaterloo.ca/~galuc/wsdbm/follows", + "http://db.uwaterloo.ca/~galuc/wsdbm/likes" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 36000000, + "slide-size" : 3600000, + "thread-count" : 32, + "delete-ratio" : 0, + "labels" : [ + "http://purl.org/stuff/rev#hasReview", + "http://purl.org/stuff/rev#reviewer", + "http://db.uwaterloo.ca/~galuc/wsdbm/follows" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 36000000, + "slide-size" : 3600000, + "thread-count" : 32, + "delete-ratio" : 0, + "labels" : [ + "http://db.uwaterloo.ca/~galuc/wsdbm/likes", + "http://purl.org/stuff/rev#hasReview", + "http://purl.org/stuff/rev#reviewer" + ] + } + ] +} diff --git a/conf/watdiv-sf150.json b/conf/watdiv-sf150.json new file mode 100644 index 0000000..b39399b --- /dev/null +++ b/conf/watdiv-sf150.json @@ -0,0 +1,79 @@ +{ + "name" : "watdiv-sf150", + "dataset" : "/hdd1/apacaci/datasets/stream-watdiv/stream-sf150", + "input-type" : "ldbc", + "report-folder" : "/home/apacaci/sgraffito/results/watdiv-sf150/", + "buffer-size" : 50000000, + "heap-size" : 192, + "timeout" : 600, + "executable" : "/home/apacaci/sgraffito/rpq/target/streamingrpq-1.0-SNAPSHOT-jar-with-dependencies.jar", + "runs" : [ + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 36000000, + "slide-size" : 3600000, + "thread-count" : 32, + "delete-ratio" : 0, + "labels" : [ + "http://db.uwaterloo.ca/~galuc/wsdbm/follows" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 36000000, + "slide-size" : 3600000, + "thread-count" : 32, + "delete-ratio" : 0, + "labels" : [ + "http://purl.org/stuff/rev#reviewer", + "http://db.uwaterloo.ca/~galuc/wsdbm/follows" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 36000000, + "slide-size" : 3600000, + "thread-count" : 32, + "delete-ratio" : 0, + "labels" : [ + "http://purl.org/stuff/rev#reviewer", + "http://db.uwaterloo.ca/~galuc/wsdbm/follows", + "http://db.uwaterloo.ca/~galuc/wsdbm/likes" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 36000000, + "slide-size" : 3600000, + "thread-count" : 32, + "delete-ratio" : 0, + "labels" : [ + "http://purl.org/stuff/rev#hasReview", + "http://purl.org/stuff/rev#reviewer", + "http://db.uwaterloo.ca/~galuc/wsdbm/follows" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 36000000, + "slide-size" : 3600000, + "thread-count" : 32, + "delete-ratio" : 0, + "labels" : [ + "http://db.uwaterloo.ca/~galuc/wsdbm/likes", + "http://purl.org/stuff/rev#hasReview", + "http://purl.org/stuff/rev#reviewer" + ] + } + ] +} diff --git a/conf/watdiv-sf200.json b/conf/watdiv-sf200.json new file mode 100644 index 0000000..c6de805 --- /dev/null +++ b/conf/watdiv-sf200.json @@ -0,0 +1,79 @@ +{ + "name" : "watdiv-sf200", + "dataset" : "/hdd1/apacaci/datasets/stream-watdiv/stream-sf200", + "input-type" : "ldbc", + "report-folder" : "/home/apacaci/sgraffito/results/watdiv-sf200/", + "buffer-size" : 50000000, + "heap-size" : 192, + "timeout" : 600, + "executable" : "/home/apacaci/sgraffito/rpq/target/streamingrpq-1.0-SNAPSHOT-jar-with-dependencies.jar", + "runs" : [ + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 36000000, + "slide-size" : 3600000, + "thread-count" : 32, + "delete-ratio" : 0, + "labels" : [ + "http://db.uwaterloo.ca/~galuc/wsdbm/follows" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 36000000, + "slide-size" : 3600000, + "thread-count" : 32, + "delete-ratio" : 0, + "labels" : [ + "http://purl.org/stuff/rev#reviewer", + "http://db.uwaterloo.ca/~galuc/wsdbm/follows" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 36000000, + "slide-size" : 3600000, + "thread-count" : 32, + "delete-ratio" : 0, + "labels" : [ + "http://purl.org/stuff/rev#reviewer", + "http://db.uwaterloo.ca/~galuc/wsdbm/follows", + "http://db.uwaterloo.ca/~galuc/wsdbm/likes" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 36000000, + "slide-size" : 3600000, + "thread-count" : 32, + "delete-ratio" : 0, + "labels" : [ + "http://purl.org/stuff/rev#hasReview", + "http://purl.org/stuff/rev#reviewer", + "http://db.uwaterloo.ca/~galuc/wsdbm/follows" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 36000000, + "slide-size" : 3600000, + "thread-count" : 32, + "delete-ratio" : 0, + "labels" : [ + "http://db.uwaterloo.ca/~galuc/wsdbm/likes", + "http://purl.org/stuff/rev#hasReview", + "http://purl.org/stuff/rev#reviewer" + ] + } + ] +} diff --git a/conf/watdiv-sf250.json b/conf/watdiv-sf250.json new file mode 100644 index 0000000..b9fb969 --- /dev/null +++ b/conf/watdiv-sf250.json @@ -0,0 +1,79 @@ +{ + "name" : "watdiv-sf250", + "dataset" : "/hdd1/apacaci/datasets/stream-watdiv/stream-sf250", + "input-type" : "ldbc", + "report-folder" : "/home/apacaci/sgraffito/results/watdiv-sf250/", + "buffer-size" : 50000000, + "heap-size" : 192, + "timeout" : 600, + "executable" : "/home/apacaci/sgraffito/rpq/target/streamingrpq-1.0-SNAPSHOT-jar-with-dependencies.jar", + "runs" : [ + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 36000000, + "slide-size" : 3600000, + "thread-count" : 32, + "delete-ratio" : 0, + "labels" : [ + "http://db.uwaterloo.ca/~galuc/wsdbm/follows" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 36000000, + "slide-size" : 3600000, + "thread-count" : 32, + "delete-ratio" : 0, + "labels" : [ + "http://purl.org/stuff/rev#reviewer", + "http://db.uwaterloo.ca/~galuc/wsdbm/follows" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 36000000, + "slide-size" : 3600000, + "thread-count" : 32, + "delete-ratio" : 0, + "labels" : [ + "http://purl.org/stuff/rev#reviewer", + "http://db.uwaterloo.ca/~galuc/wsdbm/follows", + "http://db.uwaterloo.ca/~galuc/wsdbm/likes" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 36000000, + "slide-size" : 3600000, + "thread-count" : 32, + "delete-ratio" : 0, + "labels" : [ + "http://purl.org/stuff/rev#hasReview", + "http://purl.org/stuff/rev#reviewer", + "http://db.uwaterloo.ca/~galuc/wsdbm/follows" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 36000000, + "slide-size" : 3600000, + "thread-count" : 32, + "delete-ratio" : 0, + "labels" : [ + "http://db.uwaterloo.ca/~galuc/wsdbm/likes", + "http://purl.org/stuff/rev#hasReview", + "http://purl.org/stuff/rev#reviewer" + ] + } + ] +} diff --git a/conf/watdiv-sf50.json b/conf/watdiv-sf50.json new file mode 100644 index 0000000..d1c4bee --- /dev/null +++ b/conf/watdiv-sf50.json @@ -0,0 +1,79 @@ +{ + "name" : "watdiv-sf50", + "dataset" : "/hdd1/apacaci/datasets/stream-watdiv/stream-sf50", + "input-type" : "ldbc", + "report-folder" : "/home/apacaci/sgraffito/results/watdiv-sf50/", + "buffer-size" : 50000000, + "heap-size" : 192, + "timeout" : 600, + "executable" : "/home/apacaci/sgraffito/rpq/target/streamingrpq-1.0-SNAPSHOT-jar-with-dependencies.jar", + "runs" : [ + { + "query-name" : "robotic1", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 36000000, + "slide-size" : 3600000, + "thread-count" : 32, + "delete-ratio" : 0, + "labels" : [ + "http://db.uwaterloo.ca/~galuc/wsdbm/follows" + ] + }, + { + "query-name" : "robotic2", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 36000000, + "slide-size" : 3600000, + "thread-count" : 32, + "delete-ratio" : 0, + "labels" : [ + "http://purl.org/stuff/rev#reviewer", + "http://db.uwaterloo.ca/~galuc/wsdbm/follows" + ] + }, + { + "query-name" : "robotic5", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 36000000, + "slide-size" : 3600000, + "thread-count" : 32, + "delete-ratio" : 0, + "labels" : [ + "http://purl.org/stuff/rev#reviewer", + "http://db.uwaterloo.ca/~galuc/wsdbm/follows", + "http://db.uwaterloo.ca/~galuc/wsdbm/likes" + ] + }, + { + "query-name" : "robotic7", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 36000000, + "slide-size" : 3600000, + "thread-count" : 32, + "delete-ratio" : 0, + "labels" : [ + "http://purl.org/stuff/rev#hasReview", + "http://purl.org/stuff/rev#reviewer", + "http://db.uwaterloo.ca/~galuc/wsdbm/follows" + ] + }, + { + "query-name" : "robotic11", + "index" : 1, + "semantics" : "arbitrary", + "window-size" : 36000000, + "slide-size" : 3600000, + "thread-count" : 32, + "delete-ratio" : 0, + "labels" : [ + "http://db.uwaterloo.ca/~galuc/wsdbm/likes", + "http://purl.org/stuff/rev#hasReview", + "http://purl.org/stuff/rev#reviewer" + ] + } + ] +} diff --git a/scripts/waveguideq5q6.json b/conf/waveguideq5q6.json similarity index 100% rename from scripts/waveguideq5q6.json rename to conf/waveguideq5q6.json diff --git a/conf/yago2s-arbitrary.json b/conf/yago2s-arbitrary.json new file mode 100644 index 0000000..69d0187 --- /dev/null +++ b/conf/yago2s-arbitrary.json @@ -0,0 +1,732 @@ +{ + "name" : "yago2s", + "dataset" : "/hdd1/apacaci/datasets/yago2s/tsv/yago2s_full_shuffle_virtuoso_tem53.tsv", + "input-type" : "tsv", + "report-folder" : "/home/apacaci/sgraffito/results/yago2s/", + "buffer-size" : 50000000, + "heap-size" : 192, + "timeout" : 600, + "executable" : "/home/apacaci/sgraffito/rpq/target/streamingrpq-1.0-SNAPSHOT-jar-with-dependencies.jar", + "runs" : [ + { + "query-name" : "robotic1", + "index" : "B3-8", + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/owns", + "http://yago-knowledge.org/resource/isConnectedTo", + "http://yago-knowledge.org/resource/isLocatedIn" + ] + }, + { + "query-name" : "robotic2", + "index" : "B3-8", + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/owns", + "http://yago-knowledge.org/resource/isConnectedTo", + "http://yago-knowledge.org/resource/isLocatedIn" + ] + }, + { + "query-name" : "robotic3", + "index" : "B3-8", + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/owns", + "http://yago-knowledge.org/resource/isConnectedTo", + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/wasBornIn" + ] + }, + { + "query-name" : "robotic4", + "index" : "B3-8", + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/owns", + "http://yago-knowledge.org/resource/isConnectedTo", + "http://yago-knowledge.org/resource/isLocatedIn" + ] + }, + { + "query-name" : "robotic5", + "index" : "B3-8", + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/owns", + "http://yago-knowledge.org/resource/isConnectedTo", + "http://yago-knowledge.org/resource/isLocatedIn" + ] + }, + { + "query-name" : "robotic6", + "index" : "B3-8", + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/owns", + "http://yago-knowledge.org/resource/isConnectedTo", + "http://yago-knowledge.org/resource/isLocatedIn" + ] + }, + { + "query-name" : "robotic7", + "index" : "B3-8", + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/owns", + "http://yago-knowledge.org/resource/isConnectedTo", + "http://yago-knowledge.org/resource/isLocatedIn" + ] + }, + { + "query-name" : "robotic8", + "index" : "B3-8", + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/owns", + "http://yago-knowledge.org/resource/isConnectedTo", + "http://yago-knowledge.org/resource/isLocatedIn" + ] + }, + { + "query-name" : "robotic9", + "index" : "B3-8", + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/owns", + "http://yago-knowledge.org/resource/isConnectedTo", + "http://yago-knowledge.org/resource/isLocatedIn" + ] + }, + { + "query-name" : "robotic10", + "index" : "B3-8", + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/owns", + "http://yago-knowledge.org/resource/isConnectedTo", + "http://yago-knowledge.org/resource/isLocatedIn" + ] + }, + { + "query-name" : "robotic11", + "index" : "B3-8", + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/owns", + "http://yago-knowledge.org/resource/isConnectedTo", + "http://yago-knowledge.org/resource/isLocatedIn" + ] + }, + { + "query-name" : "robotic1", + "index" : 5, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic2", + "index" : 5, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic3", + "index" : 5, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn", + "http://yago-knowledge.org/resource/happenedIn" + ] + }, + { + "query-name" : "robotic4", + "index" : 5, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic5", + "index" : 5, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic6", + "index" : 5, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic7", + "index" : 5, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic8", + "index" : 5, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic9", + "index" : 5, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic10", + "index" : 5, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic11", + "index" : 5, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic1", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic2", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic3", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/wasBornIn" + ] + }, + { + "query-name" : "robotic4", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic5", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic6", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic7", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic8", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic9", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic10", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic11", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic1", + "index" : 40, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/hasChild", + "http://yago-knowledge.org/resource/influences", + "http://yago-knowledge.org/resource/marriedTo" + ] + }, + { + "query-name" : "robotic2", + "index" : 40, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/hasChild", + "http://yago-knowledge.org/resource/influences", + "http://yago-knowledge.org/resource/marriedTo" + ] + }, + { + "query-name" : "robotic3", + "index" : 40, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/hasChild", + "http://yago-knowledge.org/resource/influences", + "http://yago-knowledge.org/resource/marriedTo", + "http://yago-knowledge.org/resource/wasBornIn" + ] + }, + { + "query-name" : "robotic4", + "index" : 40, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/hasChild", + "http://yago-knowledge.org/resource/influences", + "http://yago-knowledge.org/resource/marriedTo" + ] + }, + { + "query-name" : "robotic5", + "index" : 40, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/hasChild", + "http://yago-knowledge.org/resource/influences", + "http://yago-knowledge.org/resource/marriedTo" + ] + }, + { + "query-name" : "robotic6", + "index" : 40, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/hasChild", + "http://yago-knowledge.org/resource/influences", + "http://yago-knowledge.org/resource/marriedTo" + ] + }, + { + "query-name" : "robotic7", + "index" : 40, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/hasChild", + "http://yago-knowledge.org/resource/influences", + "http://yago-knowledge.org/resource/marriedTo" + ] + }, + { + "query-name" : "robotic8", + "index" : 40, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/hasChild", + "http://yago-knowledge.org/resource/influences", + "http://yago-knowledge.org/resource/marriedTo" + ] + }, + { + "query-name" : "robotic9", + "index" : 40, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/hasChild", + "http://yago-knowledge.org/resource/influences", + "http://yago-knowledge.org/resource/marriedTo" + ] + }, + { + "query-name" : "robotic10", + "index" : 40, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/hasChild", + "http://yago-knowledge.org/resource/influences", + "http://yago-knowledge.org/resource/marriedTo" + ] + }, + { + "query-name" : "robotic11", + "index" : 40, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/hasChild", + "http://yago-knowledge.org/resource/influences", + "http://yago-knowledge.org/resource/marriedTo" + ] + }, + { + "query-name" : "robotic1", + "index" : 51, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isKnownFor", + "http://yago-knowledge.org/resource/influences", + "http://yago-knowledge.org/resource/hasChild" + ] + }, + { + "query-name" : "robotic2", + "index" : 51, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isKnownFor", + "http://yago-knowledge.org/resource/influences", + "http://yago-knowledge.org/resource/hasChild" + ] + }, + { + "query-name" : "robotic3", + "index" : 51, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isKnownFor", + "http://yago-knowledge.org/resource/influences", + "http://yago-knowledge.org/resource/hasChild", + "http://yago-knowledge.org/resource/wasBornIn" + ] + }, + { + "query-name" : "robotic4", + "index" : 51, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isKnownFor", + "http://yago-knowledge.org/resource/influences", + "http://yago-knowledge.org/resource/hasChild" + ] + }, + { + "query-name" : "robotic5", + "index" : 51, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isKnownFor", + "http://yago-knowledge.org/resource/influences", + "http://yago-knowledge.org/resource/hasChild" + ] + }, + { + "query-name" : "robotic6", + "index" : 51, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isKnownFor", + "http://yago-knowledge.org/resource/influences", + "http://yago-knowledge.org/resource/hasChild" + ] + }, + { + "query-name" : "robotic7", + "index" : 51, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isKnownFor", + "http://yago-knowledge.org/resource/influences", + "http://yago-knowledge.org/resource/hasChild" + ] + }, + { + "query-name" : "robotic8", + "index" : 51, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isKnownFor", + "http://yago-knowledge.org/resource/influences", + "http://yago-knowledge.org/resource/hasChild" + ] + }, + { + "query-name" : "robotic9", + "index" : 51, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isKnownFor", + "http://yago-knowledge.org/resource/influences", + "http://yago-knowledge.org/resource/hasChild" + ] + }, + { + "query-name" : "robotic10", + "index" : 51, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isKnownFor", + "http://yago-knowledge.org/resource/influences", + "http://yago-knowledge.org/resource/hasChild" + ] + }, + { + "query-name" : "robotic11", + "index" : 51, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isKnownFor", + "http://yago-knowledge.org/resource/influences", + "http://yago-knowledge.org/resource/hasChild" + ] + } + ] +} diff --git a/conf/yago2s-delete.json b/conf/yago2s-delete.json new file mode 100644 index 0000000..08b48fd --- /dev/null +++ b/conf/yago2s-delete.json @@ -0,0 +1,477 @@ +{ + "name" : "yago2s", + "dataset" : "/hdd1/apacaci/datasets/yago2s/tsv/yago2s_full_shuffle_virtuoso_tem53.tsv", + "input-type" : "tsv", + "report-folder" : "/home/apacaci/sgraffito/results/yago2s/", + "buffer-size" : 50000000, + "heap-size" : 192, + "timeout" : 300, + "executable" : "/home/apacaci/sgraffito/rpq/target/streamingrpq-1.0-SNAPSHOT-jar-with-dependencies.jar", + "runs" : [ + { + "query-name" : "robotic1", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 1, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic2", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 1, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic3", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 1, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn", + "http://yago-knowledge.org/resource/happenedIn" + ] + }, + { + "query-name" : "robotic4", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 1, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic5", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 1, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic6", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 1, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic7", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 1, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic8", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 1, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic9", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 1, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic10", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 1, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic11", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 1, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic1", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 3, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic2", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 3, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic3", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 3, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn", + "http://yago-knowledge.org/resource/happenedIn" + ] + }, + { + "query-name" : "robotic4", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 3, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic5", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 3, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic6", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 3, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic7", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 3, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic8", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 3, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic9", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 3, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic10", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 3, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic11", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 3, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic1", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 5, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic2", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 5, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic3", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 5, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn", + "http://yago-knowledge.org/resource/happenedIn" + ] + }, + { + "query-name" : "robotic4", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 5, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic5", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 5, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic6", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 5, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic7", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 5, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic8", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 5, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic9", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 5, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic10", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 5, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic11", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "delete-ratio" : 5, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + } + ] +} diff --git a/conf/yago2s-scalability-slide.json b/conf/yago2s-scalability-slide.json new file mode 100644 index 0000000..079eab7 --- /dev/null +++ b/conf/yago2s-scalability-slide.json @@ -0,0 +1,588 @@ +{ + "name" : "yago2s", + "dataset" : "/hdd1/apacaci/datasets/yago2s/tsv/yago2s_full_shuffle_virtuoso_tem53.tsv", + "input-type" : "tsv", + "report-folder" : "/home/apacaci/sgraffito/results/yago2s/", + "buffer-size" : 50000000, + "heap-size" : 192, + "timeout" : 300, + "executable" : "/home/apacaci/sgraffito/rpq/target/streamingrpq-1.0-SNAPSHOT-jar-with-dependencies.jar", + "runs" : [ + { + "query-name" : "robotic1", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 500000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic2", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 500000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic3", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 500000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/wasBornIn" + ] + }, + { + "query-name" : "robotic4", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 500000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic5", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 500000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic6", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 500000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic7", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 500000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic8", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 500000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic9", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 500000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic10", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 500000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic11", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 500000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic1", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic2", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic3", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/wasBornIn" + ] + }, + { + "query-name" : "robotic4", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic5", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic6", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic7", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic8", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic9", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic10", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic11", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic1", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1500000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic2", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1500000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic3", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1500000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/wasBornIn" + ] + }, + { + "query-name" : "robotic4", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1500000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic5", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1500000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic6", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1500000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic7", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1500000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic8", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1500000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic9", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1500000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic10", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1500000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic11", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 1500000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic1", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 2000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic2", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 2000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic3", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 2000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/wasBornIn" + ] + }, + { + "query-name" : "robotic4", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 2000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic5", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 2000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic6", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 2000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic7", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 2000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic8", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 2000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic9", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 2000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic10", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 2000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic11", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 10000000, + "slide-size" : 2000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + } + ] +} diff --git a/conf/yago2s-scalability-window.json b/conf/yago2s-scalability-window.json new file mode 100644 index 0000000..fbee465 --- /dev/null +++ b/conf/yago2s-scalability-window.json @@ -0,0 +1,444 @@ +{ + "name" : "yago2s", + "dataset" : "/hdd1/apacaci/datasets/yago2s/tsv/yago2s_full_shuffle_virtuoso_tem53.tsv", + "input-type" : "tsv", + "report-folder" : "/home/apacaci/sgraffito/results/yago2s/", + "buffer-size" : 50000000, + "heap-size" : 192, + "timeout" : 300, + "executable" : "/home/apacaci/sgraffito/rpq/target/streamingrpq-1.0-SNAPSHOT-jar-with-dependencies.jar", + "runs" : [ + { + "query-name" : "robotic1", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 5000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic2", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 5000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic3", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 5000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/wasBornIn" + ] + }, + { + "query-name" : "robotic4", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 5000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic5", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 5000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic6", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 5000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic7", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 5000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic8", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 5000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic9", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 5000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic10", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 5000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic11", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 5000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic1", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 15000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic2", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 15000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic3", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 15000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/wasBornIn" + ] + }, + { + "query-name" : "robotic4", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 15000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic5", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 15000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic6", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 15000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic7", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 15000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic8", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 15000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic9", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 15000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic10", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 15000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic11", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 15000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic1", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 20000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic2", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 20000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic3", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 20000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/wasBornIn" + ] + }, + { + "query-name" : "robotic4", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 20000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic5", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 20000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic6", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 20000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic7", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 20000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic8", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 20000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic9", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 20000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic10", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 20000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + }, + { + "query-name" : "robotic11", + "index" : 21, + "semantics" : "arbitrary", + "window-size" : 20000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/isLocatedIn", + "http://yago-knowledge.org/resource/dealsWith", + "http://yago-knowledge.org/resource/hasCapital" + ] + } + ] +} diff --git a/conf/yago2s-simple.json b/conf/yago2s-simple.json new file mode 100644 index 0000000..1211d28 --- /dev/null +++ b/conf/yago2s-simple.json @@ -0,0 +1,156 @@ +{ + "name" : "yago2s", + "dataset" : "/hdd1/apacaci/datasets/yago2s/tsv/yago2s_full_shuffle_virtuoso_tem53.tsv", + "input-type" : "tsv", + "report-folder" : "/home/apacaci/sgraffito/results/yago2s/", + "buffer-size" : 50000000, + "heap-size" : 192, + "timeout" : 300, + "executable" : "/home/apacaci/sgraffito/rpq/target/streamingrpq-1.0-SNAPSHOT-jar-with-dependencies.jar", + "runs" : [ + { + "query-name" : "robotic1", + "index" : 21, + "semantics" : "simple", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic2", + "index" : 21, + "semantics" : "simple", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic3", + "index" : 21, + "semantics" : "simple", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn", + "http://yago-knowledge.org/resource/happenedIn" + ] + }, + { + "query-name" : "robotic4", + "index" : 21, + "semantics" : "simple", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic5", + "index" : 21, + "semantics" : "simple", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic6", + "index" : 21, + "semantics" : "simple", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic7", + "index" : 21, + "semantics" : "simple", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic8", + "index" : 21, + "semantics" : "simple", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic9", + "index" : 21, + "semantics" : "simple", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic10", + "index" : 21, + "semantics" : "simple", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + }, + { + "query-name" : "robotic11", + "index" : 21, + "semantics" : "simple", + "window-size" : 10000000, + "slide-size" : 1000000, + "thread-count" : 32, + "labels" : [ + "http://yago-knowledge.org/resource/happenedIn", + "http://yago-knowledge.org/resource/hasCapital", + "http://yago-knowledge.org/resource/participatedIn" + ] + } + ] +} diff --git a/pom.xml b/pom.xml index 9146b84..ad5b7d8 100644 --- a/pom.xml +++ b/pom.xml @@ -56,9 +56,9 @@ maven-compiler-plugin + 3.8.0 - 1.8 - 1.8 + 13 diff --git a/scripts/LDBCConverter.py b/scripts/LDBCConverter.py new file mode 100755 index 0000000..5dd42b2 --- /dev/null +++ b/scripts/LDBCConverter.py @@ -0,0 +1,60 @@ +#!/usr/bin/python + +import csv +import sys +import os + +# update filename +fileNameTemplate = "updateStream_0_X_forum.csv"; + +# read the command line arguments +if len(sys.argv) != 3: + print "Provide input folder and output files" + sys.exit() + +# second argument is the input folder file +input_folder = sys.argv[1] +output_folder = sys.argv[2] + +for i in range(8): + # replace x with the actual range + inputFileName = os.path.join(input_folder, fileNameTemplate.replace("X", str(i))) + outputFileName = os.path.join(output_folder, fileNameTemplate.replace("X", str(i))) + + with open(outputFileName, 'w') as outputcsvfile: + csvwriter = csv.writer(outputcsvfile, delimiter='\t') + # open the input csv file + with open(inputFileName, 'rb') as inputcsvfile: + csvreader = csv.reader(inputcsvfile, delimiter='|') + for row in csvreader: + if(row[2] == "2"): + csvwriter.writerow([row[3], "likes", row[4],row[5]]) + csvwriter.writerow([row[4], "isLikedBy", row[3],row[5]]) + elif(row[2] == "3"): + csvwriter.writerow([row[3], "likes", row[4], row[5]]) + csvwriter.writerow([row[4], "isLikedBy", row[3],row[5]]) + elif(row[2] == "4"): + csvwriter.writerow([row[3], "hasModerator", row[6], row[5]]) + csvwriter.writerow([row[6], "moderatorOf", row[3], row[5]]) + elif(row[2] == "5"): + csvwriter.writerow([row[3], "hasMember", row[4], row[5]]) + csvwriter.writerow([row[4], "memberOf", row[3], row[5]]) + elif(row[2] == "6"): + csvwriter.writerow([row[3], "hasCreator", row[11], row[5]]) + csvwriter.writerow([row[11], "creatorOf", row[3], row[5]]) + csvwriter.writerow([row[3], "hasContainer", row[12], row[5]]) + csvwriter.writerow([row[12], "containerOf", row[3], row[5]]) + elif(row[2] == "7"): + csvwriter.writerow([row[3], "hasCreator", row[9], row[4]]) + csvwriter.writerow([row[9], "creatorOf", row[3], row[4]]) + if(row[11] == "-1"): + csvwriter.writerow([row[3], "replyOf", row[12], row[4]]) + csvwriter.writerow([row[12], "hasReply", row[3], row[4]]) + else: + csvwriter.writerow([row[3], "replyOf", row[11], row[4]]) + csvwriter.writerow([row[11], "hasReply", row[3], row[4]]) + elif(row[2] == "8"): + csvwriter.writerow([row[3], "knows", row[4], row[5]]) + csvwriter.writerow([row[4], "knows", row[3], row[5]]) + + diff --git a/scripts/cycle.json b/scripts/cycle.json deleted file mode 100644 index 9fddc8e..0000000 --- a/scripts/cycle.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "name" : "yago2s", - "dataset" : "/hdd1/apacaci/datasets/yago2s/tsv/yago2s_full_shuffle_virtuoso_tem53.tsv", - "heap-size" : 160, - "timeout" : 750, - "executable" : "/home/apacaci/sgraffito/rpq/target/streamingrpq-1.0-SNAPSHOT-jar-with-dependencies.jar", - "runs" : [ - { - "index" : 1, - "labels" : [ - "http://yago-knowledge.org/resource/byTransport", - "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", - "http://www.w3.org/2002/07/owl#disjointWith" - ] - }, - { - "index" : 2, - "labels" : [ - "http://yago-knowledge.org/resource/happenedIn", - "http://yago-knowledge.org/resource/hasCapital", - "http://yago-knowledge.org/resource/created" - ] - }, - { - "index" : 21, - "labels" : [ - "http://yago-knowledge.org/resource/isLocatedIn", - "http://yago-knowledge.org/resource/dealsWith", - "http://yago-knowledge.org/resource/hasCapital" - ] - }, - { - "index" : 47, - "labels" : [ - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - },{ - "index" : 51, - "labels" : [ - "http://yago-knowledge.org/resource/isKnownFor", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - } - ] -} diff --git a/scripts/cycle2.json b/scripts/cycle2.json deleted file mode 100644 index 1eacdb2..0000000 --- a/scripts/cycle2.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "name" : "yago2s", - "dataset" : "/home/apacaci/sgraffito/rpq/src/main/resources/diamondgraph.txt", - "heap-size" : 160, - "timeout" : 750, - "executable" : "/home/apacaci/sgraffito/rpq/target/streamingrpq-1.0-SNAPSHOT-jar-with-dependencies.jar", - "runs" : [ - { - "index" : 1, - "labels" : [ - "a", - "b" - ] - } - ] -} diff --git a/scripts/restricted.json b/scripts/restricted.json deleted file mode 100644 index 64c5951..0000000 --- a/scripts/restricted.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "name" : "yago2s", - "dataset" : "/hdd1/apacaci/datasets/yago2s/tsv/yago2s_full_shuffle_virtuoso_tem53.hash", - "input-type" : "hash", - "report-folder" : "/home/apacaci/sgraffito/rpq/results", - "buffer-size" : 50000000, - "heap-size" : 160, - "timeout" : 750, - "executable" : "/home/apacaci/sgraffito/rpq/target/streamingrpq-1.0-SNAPSHOT-jar-with-dependencies.jar", - "runs" : [ - { - "query-name" : "restricted", - "index" : 1, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/byTransport", - "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", - "http://www.w3.org/2002/07/owl#disjointWith" - ] - }, - { - "query-name" : "restricted", - "index" : 1, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/byTransport", - "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", - "http://www.w3.org/2002/07/owl#disjointWith" - ] - }, - { - "query-name" : "restricted", - "index" : 2, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/happenedIn", - "http://yago-knowledge.org/resource/hasCapital", - "http://yago-knowledge.org/resource/created" - ] - }, - { - "query-name" : "restricted", - "index" : 2, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/happenedIn", - "http://yago-knowledge.org/resource/hasCapital", - "http://yago-knowledge.org/resource/created" - ] - }, - { - "query-name" : "restricted", - "index" : 21, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/isLocatedIn", - "http://yago-knowledge.org/resource/dealsWith", - "http://yago-knowledge.org/resource/hasCapital" - ] - }, - { - "query-name" : "restricted", - "index" : 21, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/isLocatedIn", - "http://yago-knowledge.org/resource/dealsWith", - "http://yago-knowledge.org/resource/hasCapital" - ] - }, - { - "query-name" : "restricted", - "index" : 47, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name" : "restricted", - "index" : 47, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - },{ - "query-name" : "restricted", - "index" : 51, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/isKnownFor", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name" : "restricted", - "index" : 51, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/isKnownFor", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - } - ] -} diff --git a/scripts/result-aggregator.py b/scripts/result-aggregator.py index 660db41..ac7bbbd 100644 --- a/scripts/result-aggregator.py +++ b/scripts/result-aggregator.py @@ -1,6 +1,7 @@ import csv import sys import os +import re def get_immediate_subdirectories(a_dir): return [name for name in os.listdir(a_dir) @@ -18,55 +19,110 @@ def get_immediate_subdirectories(a_dir): log_folders = get_immediate_subdirectories(results_folder) with open(aggregated_results_file, 'w') as csv_file: - fieldnames = ['query', 'semantics', 'binding', 'full-edges', 'edges', 'delta', 'mean', 'median', 'p99'] + fieldnames = ['query', 'semantics', 'binding', 'window-size', 'slide-size', 'containing-tree-mean', 'results', + 'tree-count', 'tree-size-mean', 'tree-size-max', 'window-mean', 'window-p99', 'slide-count-mean', + 'processed-edge-count', 'delete-ratio' , 'delete-mean', 'delete-p99', 'processed-mean', 'processed-min', 'processed-p50', 'processed-p75', 'processed-p95', + 'processed-p98', 'processed-p99', 'processed-p999', 'time'] writer = csv.DictWriter(csv_file, fieldnames=fieldnames) writer.writeheader() for log_folder in log_folders: - edges = "" - dfst = "" - full_edges = "" - mean = "" - median = "" - p99 = "" - - full_histogram = os.path.join(results_folder, log_folder, "full-histogram.csv") - dfst_counter = os.path.join(results_folder, log_folder, "dfst-counter.csv") + containing_tree_counter = os.path.join(results_folder, log_folder, "containing-tree-counter.csv") + edgecount_histogram = os.path.join(results_folder, log_folder, "edgecount-histogram.csv") processed_histogram = os.path.join(results_folder, log_folder, "processed-histogram.csv") - edge_counter = os.path.join(results_folder, log_folder, "edge-counter.csv") + result_counter = os.path.join(results_folder, log_folder, "result-counter.csv") + tree_counter = os.path.join(results_folder, log_folder, "tree-counter.csv") + treesize_histogram = os.path.join(results_folder, log_folder, "tree-size-histogram.csv") + window_histogram = os.path.join(results_folder, log_folder, "window-histogram.csv") + deletion_histogram = os.path.join(results_folder, log_folder, "explicit-deletion-histogram.csv") - with open(full_histogram, 'r') as f: + print "Opening {}".format(log_folder) + + with open(containing_tree_counter, 'r') as f: row = reversed(list(csv.reader(f))).next() - full_edges = row[1] + containing_tree_mean = row[3] - with open(dfst_counter, 'r') as f: + with open(result_counter, 'r') as f: row = reversed(list(csv.reader(f))).next() - dfst = row[1] + results = row[1] - with open(edge_counter, 'r') as f: + with open(tree_counter, 'r') as f: row = reversed(list(csv.reader(f))).next() - edges = row[1] + tree_count = row[1] - with open(processed_histogram, 'r') as f: + with open(treesize_histogram, 'r') as f: + row = reversed(list(csv.reader(f))).next() + tree_size_mean = row[3] + tree_size_max = row[2] + + with open(window_histogram, 'r') as f: + row = reversed(list(csv.reader(f))).next() + window_mean = row[3] + window_p99 = row[10] + + with open(edgecount_histogram, 'r') as f: row = reversed(list(csv.reader(f))).next() - mean = row[3] - median = row[6] - p99 = row[10] + slide_count_mean = row[3] + + with open(processed_histogram, 'r') as f: + row_list = list(csv.reader(f)) + row = reversed(row_list).next() + processed_edge_count = row[1] + processed_mean = row[3] + processed_min = row[4] + processed_p50 = row[6] + processed_p75 = row[7] + processed_p95 = row[8] + processed_p98 = row[9] + processed_p99 = row[10] + processed_p999 = row[11] + time = len(row_list) query = log_folder.split("-")[0] predicates = log_folder.split("-")[1] semantics = log_folder.split("-")[2] + window_size = re.search('ws:(.*)-ss', log_folder).group(1) + slide_size = re.search('ss:(.*)-tc', log_folder).group(1) + + delete_ratio = 0 + delete_mean = 0 + delete_p99 = 0 + + if "-dr" in log_folder: + delete_ratio = re.search('-dr:(.*)', log_folder).group(1) + with open(deletion_histogram, 'r') as f: + row = reversed(list(csv.reader(f))).next() + delete_mean = row[3] + delete_p99 = row[10] + print delete_ratio writer.writerow({ 'query' : query, 'semantics' : semantics, 'binding' : predicates, - 'full-edges' : full_edges, - 'edges' : edges, - 'delta' : dfst, - 'mean' : mean, - 'median' : median, - 'p99' : p99 + 'window-size' : window_size, + 'slide-size' : slide_size, + 'containing-tree-mean' : containing_tree_mean, + 'results' : results, + 'tree-count' : tree_count, + 'tree-size-mean' : tree_size_mean, + 'tree-size-max' : tree_size_max, + 'window-mean' : window_mean, + 'window-p99' : window_p99, + 'slide-count-mean' : slide_count_mean, + 'delete-ratio' : delete_ratio, + 'delete-mean' : delete_mean, + 'delete-p99' : delete_p99, + 'processed-edge-count' : processed_edge_count, + 'processed-mean' : processed_mean, + 'processed-min' : processed_min, + 'processed-p50' : processed_p50, + 'processed-p75' : processed_p75, + 'processed-p95' : processed_p95, + 'processed-p98' : processed_p98, + 'processed-p99' : processed_p99, + 'processed-p999' : processed_p999, + 'time' : time }) print "Result aggregated for {}".format(log_folder) diff --git a/scripts/streaming-rpq.py b/scripts/streaming-rpq.py index 4ae6de1..07832d6 100755 --- a/scripts/streaming-rpq.py +++ b/scripts/streaming-rpq.py @@ -12,23 +12,33 @@ class RPQRun: input_type = "" report = "" buffer_size = 100000000 + window_size = 50000000 + slide_size = 10000000 semantics = "" labels = [] - def __init__(self, name, input, input_type, report, buffer_size, semantics, labels): + def __init__(self, name, input, input_type, report, buffer_size, window_size, slide_size, thread_count, delete_ratio, semantics, labels): self.name = name self.input = input self.input_type = input_type self.report_file = report self.buffer_size = buffer_size + self.window_size = window_size + self.slide_size = slide_size self.semantics = semantics self.labels = labels + self.thread_count = thread_count + self.delete_ratio = delete_ratio def produceCommandString(self): - command = "-f {} -t {} -s {} -n {} -ps {} -r {} -l {} ".format( + command = "-f {} -t {} -tc {} -dr {} -s {} -ws {} -ss {} -n {} -ps {} -r {} -l {}".format( self.input, self.input_type, + str(self.thread_count), + str(self.delete_ratio), str(self.buffer_size), + str(self.window_size), + str(self.slide_size), self.name, self.semantics, self.report_file, @@ -71,20 +81,24 @@ def produceCommandString(self): index = run_config["index"] semantics = run_config["semantics"] labels = run_config["labels"] - + window_size = run_config["window-size"] + slide_size = run_config["slide-size"] + thread_count = run_config["thread-count"] + delete_ratio = run_config["delete-ratio"] # reporting folder - report_csv_path = os.path.join(report_folder, query_name + "-" + str(index) + "-" + semantics) + report_csv_path = os.path.join(report_folder, query_name + "-" + str(index) + "-" + semantics + "-ws:" + str(window_size) + "-ss:" + str(slide_size) + "-tc:" + str(thread_count) + "-dr:" + str(delete_ratio)) # create the run object - run_list.append(RPQRun(query_name, dataset_location, input_type, report_csv_path, buffer_size, semantics, labels)) + run_list.append(RPQRun(query_name, dataset_location, input_type, report_csv_path, buffer_size, window_size, slide_size, thread_count, delete_ratio, semantics, labels)) # iterate over runs and run the experiments for run in run_list: commandString = run.produceCommandString() - javaCommand = "java -XX:+UnlockDiagnosticVMOptions -XX:ParGCCardsPerStrideChunk=32768 -Xms{}g -Xmx{}g -jar {} {}".format(heap_size, heap_size, executable, commandString) + javaCommand = "java -XX:+UnlockExperimentalVMOptions -XX:+UseEpsilonGC -Xms{}g -Xmx{}g -jar {} {}".format(heap_size, heap_size, executable, commandString) print "Executing command {} ".format(javaCommand) + sys.stdout.flush() elapsedTime = 0 interval = 5 @@ -98,7 +112,10 @@ def produceCommandString(self): # kill after timeout if process is still alive if elapsedTime > timeout and proc.poll() is None: print "Killing pid {} after timeout {}".format(str(proc.pid), str(timeout)) + sys.stdout.flush() proc.kill() + # sleep before starting new job for java to release the memory + time.sleep(interval) break if proc.poll() is not None: diff --git a/scripts/waveguide.json b/scripts/waveguide.json deleted file mode 100644 index e2c2e28..0000000 --- a/scripts/waveguide.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name" : "yago2s", - "dataset" : "/Users/apacaci/Projects/sgraffito/streamingrpq/dataset/yago2s/xab", - "input-type": "tsv", - "report-folder" : "/Users/apacaci/Projects/sgraffito/streamingrpq/results", - "buffer-size" : 100000000, - "heap-size" : 16, - "timeout" : 20, - "executable" : "/Users/apacaci/Projects/sgraffito/streamingrpq/codebase/target/streamingrpq-1.0-SNAPSHOT-jar-with-dependencies.jar", - "runs" : [ - { - "query-name" : "q5", - "index" : 1, - "semantics" : "simple", - "labels" : [ - "a", "b", "c" - ] - }, - { - "query-name" : "q6", - "index" : 2, - "semantics" : "simple", - "labels" : [ - "x", "y", "z" - ] - } - ] -} \ No newline at end of file diff --git a/scripts/wikidata2.json b/scripts/wikidata2.json deleted file mode 100644 index da09c78..0000000 --- a/scripts/wikidata2.json +++ /dev/null @@ -1,408 +0,0 @@ -{ - "name" : "yago2s", - "dataset" : "/hdd1/apacaci/datasets/yago2s/tsv/yago2s_full_shuffle_virtuoso_tem53.hash", - "input-type" : "hash", - "report-folder" : "/home/apacaci/sgraffito/rpq/results", - "buffer-size" : 50000000, - "heap-size" : 160, - "timeout" : 750, - "executable" : "/home/apacaci/sgraffito/rpq/target/streamingrpq-1.0-SNAPSHOT-jar-with-dependencies.jar", - "runs" : [ - { - "query-name" : "pvldbq4", - "index" : 1, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/byTransport", - "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", - "http://www.w3.org/2002/07/owl#disjointWith" - ] - }, - { - "query-name" : "pvldbq4", - "index" : 1, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/byTransport", - "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", - "http://www.w3.org/2002/07/owl#disjointWith" - ] - }, - { - "query-name" : "pvldbq4", - "index" : 2, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/happenedIn", - "http://yago-knowledge.org/resource/hasCapital", - "http://yago-knowledge.org/resource/created" - ] - }, - { - "query-name" : "pvldbq4", - "index" : 2, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/happenedIn", - "http://yago-knowledge.org/resource/hasCapital", - "http://yago-knowledge.org/resource/created" - ] - }, - { - "query-name" : "pvldbq4", - "index" : 21, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/isLocatedIn", - "http://yago-knowledge.org/resource/dealsWith", - "http://yago-knowledge.org/resource/hasCapital" - ] - }, - { - "query-name" : "pvldbq4", - "index" : 21, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/isLocatedIn", - "http://yago-knowledge.org/resource/dealsWith", - "http://yago-knowledge.org/resource/hasCapital" - ] - }, - { - "query-name" : "pvldbq4", - "index" : 47, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name" : "pvldbq4", - "index" : 47, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - },{ - "query-name" : "pvldbq4", - "index" : 51, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/isKnownFor", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name" : "pvldbq4", - "index" : 51, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/isKnownFor", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name" : "pvldbq5", - "index" : 1, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/byTransport", - "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", - "http://www.w3.org/2002/07/owl#disjointWith" - ] - }, - { - "query-name" : "pvldbq5", - "index" : 1, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/byTransport", - "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", - "http://www.w3.org/2002/07/owl#disjointWith" - ] - }, - { - "query-name" : "pvldbq5", - "index" : 2, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/happenedIn", - "http://yago-knowledge.org/resource/hasCapital", - "http://yago-knowledge.org/resource/created" - ] - }, - { - "query-name" : "pvldbq5", - "index" : 2, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/happenedIn", - "http://yago-knowledge.org/resource/hasCapital", - "http://yago-knowledge.org/resource/created" - ] - }, - { - "query-name" : "pvldbq5", - "index" : 21, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/isLocatedIn", - "http://yago-knowledge.org/resource/dealsWith", - "http://yago-knowledge.org/resource/hasCapital" - ] - }, - { - "query-name" : "pvldbq5", - "index" : 21, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/isLocatedIn", - "http://yago-knowledge.org/resource/dealsWith", - "http://yago-knowledge.org/resource/hasCapital" - ] - }, - { - "query-name" : "pvldbq5", - "index" : 47, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name" : "pvldbq5", - "index" : 47, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - },{ - "query-name" : "pvldbq5", - "index" : 51, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/isKnownFor", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name" : "pvldbq5", - "index" : 51, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/isKnownFor", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name" : "pvldbq21", - "index" : 1, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/byTransport", - "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", - "http://www.w3.org/2002/07/owl#disjointWith" - ] - }, - { - "query-name" : "pvldbq21", - "index" : 1, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/byTransport", - "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", - "http://www.w3.org/2002/07/owl#disjointWith" - ] - }, - { - "query-name" : "pvldbq21", - "index" : 2, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/happenedIn", - "http://yago-knowledge.org/resource/hasCapital", - "http://yago-knowledge.org/resource/created" - ] - }, - { - "query-name" : "pvldbq21", - "index" : 2, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/happenedIn", - "http://yago-knowledge.org/resource/hasCapital", - "http://yago-knowledge.org/resource/created" - ] - }, - { - "query-name" : "pvldbq21", - "index" : 21, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/isLocatedIn", - "http://yago-knowledge.org/resource/dealsWith", - "http://yago-knowledge.org/resource/hasCapital" - ] - }, - { - "query-name" : "pvldbq21", - "index" : 21, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/isLocatedIn", - "http://yago-knowledge.org/resource/dealsWith", - "http://yago-knowledge.org/resource/hasCapital" - ] - }, - { - "query-name" : "pvldbq21", - "index" : 47, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name" : "pvldbq21", - "index" : 47, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - },{ - "query-name" : "pvldbq21", - "index" : 51, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/isKnownFor", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name" : "pvldbq21", - "index" : 51, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/isKnownFor", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name" : "wwwq3", - "index" : 1, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/byTransport", - "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", - "http://www.w3.org/2002/07/owl#disjointWith" - ] - }, - { - "query-name" : "wwwq3", - "index" : 1, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/byTransport", - "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", - "http://www.w3.org/2002/07/owl#disjointWith" - ] - }, - { - "query-name" : "wwwq3", - "index" : 2, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/happenedIn", - "http://yago-knowledge.org/resource/hasCapital", - "http://yago-knowledge.org/resource/created" - ] - }, - { - "query-name" : "wwwq3", - "index" : 2, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/happenedIn", - "http://yago-knowledge.org/resource/hasCapital", - "http://yago-knowledge.org/resource/created" - ] - }, - { - "query-name" : "wwwq3", - "index" : 21, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/isLocatedIn", - "http://yago-knowledge.org/resource/dealsWith", - "http://yago-knowledge.org/resource/hasCapital" - ] - }, - { - "query-name" : "wwwq3", - "index" : 21, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/isLocatedIn", - "http://yago-knowledge.org/resource/dealsWith", - "http://yago-knowledge.org/resource/hasCapital" - ] - }, - { - "query-name" : "wwwq3", - "index" : 47, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name" : "wwwq3", - "index" : 47, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - },{ - "query-name" : "wwwq3", - "index" : 51, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/isKnownFor", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name" : "wwwq3", - "index" : 51, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/isKnownFor", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - } - ] -} diff --git a/scripts/wikidata2_rerun.json b/scripts/wikidata2_rerun.json deleted file mode 100644 index d753873..0000000 --- a/scripts/wikidata2_rerun.json +++ /dev/null @@ -1,598 +0,0 @@ -{ - "name": "yago2s", - "dataset": "/hdd1/apacaci/datasets/yago2s/tsv/yago2s_full_shuffle_virtuoso_tem53.hash", - "input-type": "hash", - "report-folder": "/home/apacaci/sgraffito/rpq/results_wikidata_rerun", - "buffer-size": 50000000, - "heap-size": 160, - "timeout": 750, - "executable": "/home/apacaci/sgraffito/rpq/target/streamingrpq-1.0-SNAPSHOT-jar-with-dependencies.jar", - "runs": [ - { - "query-name": "pvldbq1", - "index": 1, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/byTransport", - "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", - "http://www.w3.org/2002/07/owl#disjointWith" - ] - }, - { - "query-name": "pvldbq1", - "index": 2, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/happenedIn", - "http://yago-knowledge.org/resource/hasCapital", - "http://yago-knowledge.org/resource/created" - ] - }, - { - "query-name": "pvldbq1", - "index": 21, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/isLocatedIn", - "http://yago-knowledge.org/resource/dealsWith", - "http://yago-knowledge.org/resource/hasCapital" - ] - }, - { - "query-name": "pvldbq1", - "index": 47, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name": "pvldbq1", - "index": 51, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/isKnownFor", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name": "pvldbq2", - "index": 1, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/byTransport", - "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", - "http://www.w3.org/2002/07/owl#disjointWith" - ] - }, - { - "query-name": "pvldbq2", - "index": 2, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/happenedIn", - "http://yago-knowledge.org/resource/hasCapital", - "http://yago-knowledge.org/resource/created" - ] - }, - { - "query-name": "pvldbq2", - "index": 21, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/isLocatedIn", - "http://yago-knowledge.org/resource/dealsWith", - "http://yago-knowledge.org/resource/hasCapital" - ] - }, - { - "query-name": "pvldbq2", - "index": 47, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name": "pvldbq2", - "index": 51, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/isKnownFor", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name": "pvldbq3", - "index": 1, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/byTransport", - "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", - "http://www.w3.org/2002/07/owl#disjointWith" - ] - }, - { - "query-name": "pvldbq3", - "index": 2, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/happenedIn", - "http://yago-knowledge.org/resource/hasCapital", - "http://yago-knowledge.org/resource/created" - ] - }, - { - "query-name": "pvldbq3", - "index": 21, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/isLocatedIn", - "http://yago-knowledge.org/resource/dealsWith", - "http://yago-knowledge.org/resource/hasCapital" - ] - }, - { - "query-name": "pvldbq3", - "index": 47, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name": "pvldbq3", - "index": 51, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/isKnownFor", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name": "wwwq2", - "index": 1, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/byTransport", - "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", - "http://www.w3.org/2002/07/owl#disjointWith" - ] - }, - { - "query-name": "wwwq2", - "index": 2, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/happenedIn", - "http://yago-knowledge.org/resource/hasCapital", - "http://yago-knowledge.org/resource/created" - ] - }, - { - "query-name": "wwwq2", - "index": 21, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/isLocatedIn", - "http://yago-knowledge.org/resource/dealsWith", - "http://yago-knowledge.org/resource/hasCapital" - ] - }, - { - "query-name": "wwwq2", - "index": 47, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name": "wwwq2", - "index": 51, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/isKnownFor", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name": "pvldbq4", - "index": 1, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/byTransport", - "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", - "http://www.w3.org/2002/07/owl#disjointWith" - ] - }, - { - "query-name": "pvldbq4", - "index": 2, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/happenedIn", - "http://yago-knowledge.org/resource/hasCapital", - "http://yago-knowledge.org/resource/created" - ] - }, - { - "query-name": "pvldbq4", - "index": 21, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/isLocatedIn", - "http://yago-knowledge.org/resource/dealsWith", - "http://yago-knowledge.org/resource/hasCapital" - ] - }, - { - "query-name": "pvldbq4", - "index": 47, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name": "pvldbq4", - "index": 51, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/isKnownFor", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name": "pvldbq5", - "index": 1, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/byTransport", - "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", - "http://www.w3.org/2002/07/owl#disjointWith" - ] - }, - { - "query-name": "pvldbq5", - "index": 2, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/happenedIn", - "http://yago-knowledge.org/resource/hasCapital", - "http://yago-knowledge.org/resource/created" - ] - }, - { - "query-name": "pvldbq5", - "index": 21, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/isLocatedIn", - "http://yago-knowledge.org/resource/dealsWith", - "http://yago-knowledge.org/resource/hasCapital" - ] - }, - { - "query-name": "pvldbq5", - "index": 47, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name": "pvldbq5", - "index": 51, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/isKnownFor", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name": "pvldbq21", - "index": 1, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/byTransport", - "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", - "http://www.w3.org/2002/07/owl#disjointWith" - ] - }, - { - "query-name": "pvldbq21", - "index": 2, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/happenedIn", - "http://yago-knowledge.org/resource/hasCapital", - "http://yago-knowledge.org/resource/created" - ] - }, - { - "query-name": "pvldbq21", - "index": 21, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/isLocatedIn", - "http://yago-knowledge.org/resource/dealsWith", - "http://yago-knowledge.org/resource/hasCapital" - ] - }, - { - "query-name": "pvldbq21", - "index": 47, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name": "pvldbq21", - "index": 51, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/isKnownFor", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name": "wwwq3", - "index": 1, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/byTransport", - "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", - "http://www.w3.org/2002/07/owl#disjointWith" - ] - }, - { - "query-name": "wwwq3", - "index": 2, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/happenedIn", - "http://yago-knowledge.org/resource/hasCapital", - "http://yago-knowledge.org/resource/created" - ] - }, - { - "query-name": "wwwq3", - "index": 21, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/isLocatedIn", - "http://yago-knowledge.org/resource/dealsWith", - "http://yago-knowledge.org/resource/hasCapital" - ] - }, - { - "query-name": "wwwq3", - "index": 47, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name": "wwwq3", - "index": 51, - "semantics": "arbitrary", - "labels": [ - "http://yago-knowledge.org/resource/isKnownFor", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - },{ - "query-name" : "waveguide5", - "index" : 1, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/byTransport", - "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", - "http://www.w3.org/2002/07/owl#disjointWith" - ] - }, - { - "query-name" : "waveguide5", - "index" : 2, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/happenedIn", - "http://yago-knowledge.org/resource/hasCapital", - "http://yago-knowledge.org/resource/created" - ] - },{ - "query-name" : "waveguide5", - "index" : 21, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/isLocatedIn", - "http://yago-knowledge.org/resource/dealsWith", - "http://yago-knowledge.org/resource/hasCapital" - ] - },{ - "query-name" : "waveguide5", - "index" : 47, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - },{ - "query-name" : "waveguide5", - "index" : 51, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/isKnownFor", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name" : "waveguide5", - "index" : 1, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/byTransport", - "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", - "http://www.w3.org/2002/07/owl#disjointWith" - ] - }, - { - "query-name" : "waveguide5", - "index" : 2, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/happenedIn", - "http://yago-knowledge.org/resource/hasCapital", - "http://yago-knowledge.org/resource/created" - ] - },{ - "query-name" : "waveguide5", - "index" : 21, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/isLocatedIn", - "http://yago-knowledge.org/resource/dealsWith", - "http://yago-knowledge.org/resource/hasCapital" - ] - },{ - "query-name" : "waveguide5", - "index" : 47, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - },{ - "query-name" : "waveguide5", - "index" : 51, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/isKnownFor", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - },{ - "query-name" : "waveguide6", - "index" : 1, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/byTransport", - "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", - "http://www.w3.org/2002/07/owl#disjointWith" - ] - }, - { - "query-name" : "waveguide6", - "index" : 2, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/happenedIn", - "http://yago-knowledge.org/resource/hasCapital", - "http://yago-knowledge.org/resource/created" - ] - },{ - "query-name" : "waveguide6", - "index" : 21, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/isLocatedIn", - "http://yago-knowledge.org/resource/dealsWith", - "http://yago-knowledge.org/resource/hasCapital" - ] - },{ - "query-name" : "waveguide6", - "index" : 47, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - },{ - "query-name" : "waveguide6", - "index" : 51, - "semantics" : "arbitrary", - "labels" : [ - "http://yago-knowledge.org/resource/isKnownFor", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - }, - { - "query-name" : "waveguide6", - "index" : 1, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/byTransport", - "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", - "http://www.w3.org/2002/07/owl#disjointWith" - ] - }, - { - "query-name" : "waveguide6", - "index" : 2, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/happenedIn", - "http://yago-knowledge.org/resource/hasCapital", - "http://yago-knowledge.org/resource/created" - ] - },{ - "query-name" : "waveguide6", - "index" : 21, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/isLocatedIn", - "http://yago-knowledge.org/resource/dealsWith", - "http://yago-knowledge.org/resource/hasCapital" - ] - },{ - "query-name" : "waveguide6", - "index" : 47, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - },{ - "query-name" : "waveguide6", - "index" : 51, - "semantics" : "simple", - "labels" : [ - "http://yago-knowledge.org/resource/isKnownFor", - "http://yago-knowledge.org/resource/influences", - "http://yago-knowledge.org/resource/hasChild" - ] - } - ] -} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/input/InputTuple.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/input/InputTuple.java index 1fa5aa5..59e74a6 100644 --- a/src/main/java/ca/uwaterloo/cs/streamingrpq/input/InputTuple.java +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/input/InputTuple.java @@ -8,6 +8,7 @@ public class InputTuple { private S source; private T target; private L label; + private long timestamp; private TupleType type; @@ -18,10 +19,15 @@ public class InputTuple { * @param label */ public InputTuple(S source, T target, L label) { + this(source, target, label, 0); + } + + public InputTuple(S source, T target, L label, long timestamp) { this.source = source; this.target = target; this.label = label; this.type = TupleType.INSERT; + this.timestamp = timestamp; } /** @@ -31,10 +37,11 @@ public InputTuple(S source, T target, L label) { * @param label * @param type @{@link TupleType}, INSERT by default */ - public InputTuple(S source, T target, L label, TupleType type) { + public InputTuple(S source, T target, L label, long timestamp, TupleType type) { this.source = source; this.target = target; this.label = label; + this.timestamp = timestamp; this.type = type; } @@ -50,6 +57,26 @@ public L getLabel() { return label; } + public void setSource(S source) { + this.source = source; + } + + public void setTarget(T target) { + this.target = target; + } + + public void setLabel(L label) { + this.label = label; + } + + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + + public void setType(TupleType type) { + this.type = type; + } + public boolean isDeletion() { return this.type == TupleType.DELETE; } @@ -59,6 +86,10 @@ public String toString() { return new StringBuilder("<").append(this.source).append(",").append(this.target).append(",").append(this.label).append(",").append(this.type).append(">").toString(); } + public long getTimestamp() { + return timestamp; + } + /** * Created by anilpacaci on 2019-02-22. */ diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/input/LDBCStream.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/input/LDBCStream.java new file mode 100644 index 0000000..c9ea4b5 --- /dev/null +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/input/LDBCStream.java @@ -0,0 +1,169 @@ +package ca.uwaterloo.cs.streamingrpq.input; + +import com.google.common.base.Splitter; +import com.google.common.collect.Iterables; + +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Iterator; +import java.util.Queue; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; + +public class LDBCStream implements TextStream{ + + + FileReader fileStream; + BufferedReader bufferedReader; + + String filename; + + ScheduledExecutorService executor; + + Integer localCounter = 0; + Integer globalCounter = 0; + + Integer deleteCounter = 0; + + Long startTimestamp = -1L; + + long lastTimetamp = Long.MIN_VALUE; + + InputTuple tuple = null; + + private String splitResults[]; + + Queue deletionBuffer = new ArrayDeque<>(); + int deletionPercentage = 0; + + + public boolean isOpen() { + return false; + } + + public void open(String filename, int maxSize) { + this.startTimestamp = 0L; + open(filename); + } + + public void open(String filename, int maxSize, long startTimestamp, int deletionPercentage) { + this.startTimestamp = startTimestamp; + this.deletionPercentage = deletionPercentage; + open(filename); + } + + public void open(String filename) { + this.filename = filename; + try { + fileStream = new FileReader(filename); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + bufferedReader = new BufferedReader(fileStream, 20*1024*1024); + + Runnable counterRunnable = new Runnable() { + private int seconds = 0; + + @Override + public void run() { + System.out.println("Second " + ++seconds + " : " + localCounter + " / " + globalCounter + " -- deletes: " + deleteCounter); + localCounter = 0; + deleteCounter = 0; + } + }; + + executor = Executors.newScheduledThreadPool(1); + executor.scheduleAtFixedRate(counterRunnable, 1, 1, TimeUnit.SECONDS); + + splitResults = new String[4]; + + tuple = new InputTuple(null, null, null, 0); + } + + public void close() { + try { + bufferedReader.close(); + fileStream.close(); + } catch (IOException e) { + e.printStackTrace(); + } + + executor.shutdown(); + } + + public InputTuple next() { + String line = null; + int i = 0; + + //generate negative tuple if necessary + if(!deletionBuffer.isEmpty() && ThreadLocalRandom.current().nextInt(100) < deletionPercentage) { + line = deletionBuffer.poll(); + Iterator iterator = Splitter.on('\t').trimResults().split(line).iterator(); + for (i = 0; iterator.hasNext() && i < 4; i++) { + splitResults[i] = iterator.next(); + } + // only if we fully + if (i == 4) { +// tuple = new InputTuple(1,2,3); + tuple.setSource(splitResults[0].hashCode()); + tuple.setLabel(splitResults[1]); + tuple.setTarget(splitResults[2].hashCode()); + tuple.setType(InputTuple.TupleType.DELETE); + tuple.setTimestamp(lastTimetamp); + deleteCounter++; + return tuple; + } + } + + try { + while((line = bufferedReader.readLine()) != null) { + Iterator iterator = Splitter.on('\t').trimResults().split(line).iterator(); + for(i = 0; iterator.hasNext() && i < 4; i++) { + splitResults[i] = iterator.next(); + } + // only if we fully + if(i == 4) { +// tuple = new InputTuple(1,2,3); + lastTimetamp = Long.parseLong(splitResults[3]) - startTimestamp; + + tuple.setSource(splitResults[0].hashCode()); + tuple.setLabel(splitResults[1]); + tuple.setTarget(splitResults[2].hashCode()); + tuple.setType(InputTuple.TupleType.INSERT); + tuple.setTimestamp(lastTimetamp); + localCounter++; + globalCounter++; +// tuple = new InputTuple(Integer.parseInt(splitResults[0]), Integer.parseInt(splitResults[2]), splitResults[1]); + + // store this tuple for later deletion + if(ThreadLocalRandom.current().nextInt(100) < 2 * deletionPercentage) { + deletionBuffer.offer(line); + } + + break; + } + } + } catch (IOException e) { + return null; + } + if (line == null) { + return null; + } + + return tuple; + } + + public void reset() { + close(); + + open(this.filename); + + localCounter = 0; + globalCounter = 0; + } +} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/input/SimpleTextStream.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/input/SimpleTextStream.java index e3c7d01..918ee3b 100644 --- a/src/main/java/ca/uwaterloo/cs/streamingrpq/input/SimpleTextStream.java +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/input/SimpleTextStream.java @@ -7,6 +7,12 @@ import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Iterator; +import java.util.Queue; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; public class SimpleTextStream implements TextStream{ @@ -16,9 +22,17 @@ public class SimpleTextStream implements TextStream{ String filename; + ScheduledExecutorService executor; + Integer localCounter = 0; Integer globalCounter = 0; + private String splitResults[]; + + + Queue deletionBuffer = new ArrayDeque<>(); + int deletionPercentage = 0; + public boolean isOpen() { return false; @@ -28,6 +42,11 @@ public void open(String filename, int maxSize) { open(filename); } + @Override + public void open(String filename, int size, long startTimestamp, int deletionPercentage) { + open(filename, size); + } + public void open(String filename) { this.filename = filename; try { @@ -35,7 +54,22 @@ public void open(String filename) { } catch (FileNotFoundException e) { e.printStackTrace(); } - bufferedReader = new BufferedReader(fileStream, 1024*1024); + bufferedReader = new BufferedReader(fileStream, 20*1024*1024); + + Runnable counterRunnable = new Runnable() { + private int seconds = 0; + + @Override + public void run() { + System.out.println("Second " + ++seconds + " : " + localCounter + " / " + globalCounter); + localCounter = 0; + } + }; + + executor = Executors.newScheduledThreadPool(1); + executor.scheduleAtFixedRate(counterRunnable, 1, 1, TimeUnit.SECONDS); + + splitResults = new String[4]; } @@ -43,6 +77,7 @@ public void close() { try { bufferedReader.close(); fileStream.close(); + this.executor.shutdownNow(); } catch (IOException e) { e.printStackTrace(); } @@ -54,10 +89,18 @@ public InputTuple next() { InputTuple tuple = null; try { while((line = bufferedReader.readLine()) != null) { - String[] splitResults = Iterables.toArray(Splitter.on('\t').split(line), String.class); - if(splitResults.length == 3) { + Iterator iterator = Splitter.on('\t').trimResults().split(line).iterator(); + int i = 0; + for(i = 0; iterator.hasNext() && i < 4; i++) { + splitResults[i] = iterator.next(); + } + if(i == 4) { // tuple = new InputTuple(1,2,3); - tuple = new InputTuple(Integer.parseInt(splitResults[0]), Integer.parseInt(splitResults[2]), splitResults[1]); + if(splitResults[3].equals("+")) { + tuple = new InputTuple(Integer.parseInt(splitResults[0]), Integer.parseInt(splitResults[2]), splitResults[1], globalCounter); + } else { + tuple = new InputTuple(Integer.parseInt(splitResults[0]), Integer.parseInt(splitResults[2]), splitResults[1], globalCounter, InputTuple.TupleType.DELETE); + } break; } } diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/input/StackOverflowStream.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/input/StackOverflowStream.java new file mode 100644 index 0000000..a555ad5 --- /dev/null +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/input/StackOverflowStream.java @@ -0,0 +1,118 @@ +package ca.uwaterloo.cs.streamingrpq.input; + +import com.google.common.base.Splitter; +import com.google.common.collect.Iterables; + +import java.io.BufferedReader; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Queue; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +public class StackOverflowStream implements TextStream{ + + + FileReader fileStream; + BufferedReader bufferedReader; + + String filename; + + ScheduledExecutorService executor; + + Integer localCounter = 0; + Integer globalCounter = 0; + + Long startTimestamp = -1L; + + + Queue deletionBuffer = new ArrayDeque<>(); + int deletionPercentage = 0; + + + + public boolean isOpen() { + return false; + } + + public void open(String filename, int maxSize) { + this.startTimestamp = 0L; + open(filename); + } + + public void open(String filename, int maxSize, long startTimestamp, int deletionPercentage) { + this.startTimestamp = startTimestamp; + open(filename); + } + + public void open(String filename) { + this.filename = filename; + try { + fileStream = new FileReader(filename); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + bufferedReader = new BufferedReader(fileStream, 20*1024*1024); + + Runnable counterRunnable = new Runnable() { + private int seconds = 0; + + @Override + public void run() { + System.out.println("Second " + ++seconds + " : " + localCounter + " / " + globalCounter); + localCounter = 0; + } + }; + + executor = Executors.newScheduledThreadPool(1); + executor.scheduleAtFixedRate(counterRunnable, 1, 1, TimeUnit.SECONDS); + } + + public void close() { + try { + bufferedReader.close(); + fileStream.close(); + } catch (IOException e) { + e.printStackTrace(); + } + + executor.shutdown(); + } + + public InputTuple next() { + String line = null; + InputTuple tuple = null; + try { + while((line = bufferedReader.readLine()) != null) { + String[] splitResults = Iterables.toArray(Splitter.on('\t').trimResults().split(line), String.class); + if(splitResults.length == 4) { +// tuple = new InputTuple(1,2,3); + tuple = new InputTuple(Integer.parseInt(splitResults[0]), Integer.parseInt(splitResults[2]), splitResults[1], Long.parseLong(splitResults[3]) - startTimestamp); + localCounter++; + globalCounter++; +// tuple = new InputTuple(Integer.parseInt(splitResults[0]), Integer.parseInt(splitResults[2]), splitResults[1]); + break; + } + } + } catch (IOException e) { + return null; + } + if (line == null) { + return null; + } + + return tuple; + } + + public void reset() { + close(); + + open(this.filename); + + localCounter = 0; + globalCounter = 0; + } +} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/input/TextStream.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/input/TextStream.java index ed94b46..98f10f2 100644 --- a/src/main/java/ca/uwaterloo/cs/streamingrpq/input/TextStream.java +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/input/TextStream.java @@ -1,17 +1,21 @@ package ca.uwaterloo.cs.streamingrpq.input; +import java.util.ArrayDeque; +import java.util.Queue; + /** * Created by anilpacaci on 2019-01-31. */ public interface TextStream { - public boolean isOpen(); public void open(String filename); public void open(String filename, int size); + public void open(String filename, int size, long startTimestamp, int deletionPercentage); + public InputTuple next(); public void close(); diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/input/Yago2sHashStream.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/input/Yago2sHashStream.java index 2b76e7c..fbb08b6 100644 --- a/src/main/java/ca/uwaterloo/cs/streamingrpq/input/Yago2sHashStream.java +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/input/Yago2sHashStream.java @@ -7,6 +7,8 @@ import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Queue; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -25,6 +27,11 @@ public class Yago2sHashStream implements TextStream{ Integer globalCounter = 0; + Queue deletionBuffer = new ArrayDeque<>(); + int deletionPercentage = 0; + + + public boolean isOpen() { return false; } @@ -33,6 +40,11 @@ public void open(String filename, int maxSize) { open(filename); } + @Override + public void open(String filename, int size, long startTimestamp, int deletionPercentage) { + open(filename, size); + } + public void open(String filename) { this.filename = filename; try { @@ -76,7 +88,9 @@ public InputTuple next() { String[] splitResults = Iterables.toArray(Splitter.on(' ').split(line), String.class); if(splitResults.length == 3) { // tuple = new InputTuple(1,2,3); - tuple = new InputTuple(Integer.parseInt(splitResults[0]), Integer.parseInt(splitResults[2]), Integer.parseInt(splitResults[1])); + tuple = new InputTuple(Integer.parseInt(splitResults[0]), Integer.parseInt(splitResults[2]), Integer.parseInt(splitResults[1]), globalCounter); + localCounter++; + globalCounter++; // tuple = new InputTuple(Integer.parseInt(splitResults[0]), Integer.parseInt(splitResults[2]), splitResults[1]); break; } @@ -87,8 +101,7 @@ public InputTuple next() { if (line == null) { return null; } - localCounter++; - globalCounter++; + return tuple; } diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/input/Yago2sInMemoryTSVStream.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/input/Yago2sInMemoryTSVStream.java index 69e39f1..a808937 100644 --- a/src/main/java/ca/uwaterloo/cs/streamingrpq/input/Yago2sInMemoryTSVStream.java +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/input/Yago2sInMemoryTSVStream.java @@ -7,6 +7,8 @@ import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Queue; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -31,6 +33,9 @@ public class Yago2sInMemoryTSVStream implements TextStream{ Integer globalCounter = 0; + Queue deletionBuffer = new ArrayDeque<>(); + int deletionPercentage = 0; + public boolean isOpen() { return false; @@ -40,6 +45,11 @@ public void open(String filename) { open(filename, MAX_STREAM_SIZE); } + @Override + public void open(String filename, int size, long startTimestamp, int deletionPercentage) { + open(filename, size); + } + public void open(String filename, int maxSize) { try { fileStream = new FileReader(filename); diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/input/Yago2sTSVStream.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/input/Yago2sTSVStream.java index 8df6dd4..2cbfc6e 100644 --- a/src/main/java/ca/uwaterloo/cs/streamingrpq/input/Yago2sTSVStream.java +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/input/Yago2sTSVStream.java @@ -7,8 +7,12 @@ import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Iterator; +import java.util.Queue; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; public class Yago2sTSVStream implements TextStream{ @@ -23,6 +27,16 @@ public class Yago2sTSVStream implements TextStream{ Integer localCounter = 0; Integer globalCounter = 0; + Integer deleteCounter = 0; + + private String splitResults[]; + + InputTuple tuple = null; + + + Queue deletionBuffer = new ArrayDeque<>(); + int deletionPercentage = 0; + long lastTimetamp = Long.MIN_VALUE; public boolean isOpen() { @@ -33,6 +47,12 @@ public void open(String filename, int maxSize) { open(filename); } + @Override + public void open(String filename, int size, long startTimestamp, int deletionPercentage) { + this.deletionPercentage = deletionPercentage; + open(filename, size); + } + public void open(String filename) { this.filename = filename; try { @@ -47,14 +67,18 @@ public void open(String filename) { @Override public void run() { - System.out.println("Second " + ++seconds + " : " + localCounter + " / " + globalCounter); + System.out.println("Second " + ++seconds + " : " + localCounter + " / " + globalCounter + " -- deletes: " + deleteCounter); localCounter = 0; + deleteCounter++; } }; executor = Executors.newScheduledThreadPool(1); executor.scheduleAtFixedRate(counterRunnable, 1, 1, TimeUnit.SECONDS); + splitResults = new String[4]; + + tuple = new InputTuple(null, null, null, 0); } public void close() { @@ -68,16 +92,55 @@ public void close() { executor.shutdown(); } - public InputTuple next() { + public InputTuple next() { String line = null; - InputTuple tuple = null; + + //generate negative tuple if necessary + if(!deletionBuffer.isEmpty() && ThreadLocalRandom.current().nextInt(100) < deletionPercentage) { + line = deletionBuffer.poll(); + Iterator iterator = Splitter.on('\t').trimResults().split(line).iterator(); + int i = 0; + for(i = 0; iterator.hasNext() && i < 3; i++) { + splitResults[i] = iterator.next(); + } + // only if we fully + if(i == 3) { + tuple.setSource(splitResults[0].hashCode()); + tuple.setTarget(splitResults[2].hashCode()); + tuple.setLabel(splitResults[1]); + tuple.setTimestamp(globalCounter); + tuple.setType(InputTuple.TupleType.DELETE); + deleteCounter++; + return tuple; + } + } + try { while((line = bufferedReader.readLine()) != null) { - String[] splitResults = Iterables.toArray(Splitter.on('\t').split(line), String.class); - if(splitResults.length == 3) { + Iterator iterator = Splitter.on('\t').trimResults().split(line).iterator(); + int i = 0; + for(i = 0; iterator.hasNext() && i < 3; i++) { + splitResults[i] = iterator.next(); + } + // only if we fully + if(i == 3) { // tuple = new InputTuple(1,2,3); - tuple = new InputTuple(splitResults[0].hashCode(), splitResults[2].hashCode(), splitResults[1].hashCode()); + lastTimetamp = globalCounter; + + tuple.setSource(splitResults[0].hashCode()); + tuple.setTarget(splitResults[2].hashCode()); + tuple.setLabel(splitResults[1]); + tuple.setTimestamp(globalCounter); + tuple.setType(InputTuple.TupleType.INSERT); // tuple = new InputTuple(Integer.parseInt(splitResults[0]), Integer.parseInt(splitResults[2]), splitResults[1]); + localCounter++; + globalCounter++; + + // store this tuple for later deletion + if(ThreadLocalRandom.current().nextInt(100) < 2 * deletionPercentage) { + deletionBuffer.offer(line); + } + break; } } @@ -87,8 +150,7 @@ public InputTuple next() { if (line == null) { return null; } - localCounter++; - globalCounter++; + return tuple; } diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/runtime/MazeQueries.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/runtime/MazeQueries.java new file mode 100644 index 0000000..6f846e8 --- /dev/null +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/runtime/MazeQueries.java @@ -0,0 +1,146 @@ +package ca.uwaterloo.cs.streamingrpq.runtime; + +import ca.uwaterloo.cs.streamingrpq.stree.data.QueryAutomata; + +/** + * Created by anilpacaci on 2019-10-13. + */ +public class MazeQueries { + + public static QueryAutomata getMazeQuery(String queryName, L... predicateString) { + QueryAutomata query; + + switch (queryName) { + case "robotic1" : + checkArguments(queryName, 1, predicateString); + query = new QueryAutomata<>(2); + query.addTransition(0, predicateString[0], 1); + query.addTransition(1, predicateString[0], 1); + query.addFinalState(0); + query.addFinalState(1); + break; + case "robotic2": + checkArguments(queryName, 2, predicateString); + query = new QueryAutomata<>(2); + query.addTransition(0, predicateString[0], 1); + query.addTransition(1, predicateString[1], 1); + query.addFinalState(1); + break; + case "robotic3": + checkArguments(queryName, 4, predicateString); + query = new QueryAutomata<>(4); + query.addTransition(0, predicateString[0], 1); + query.addTransition(1, predicateString[1], 1); + query.addTransition(1, predicateString[2], 2); + query.addTransition(2, predicateString[3], 3); + query.addTransition(3, predicateString[3], 3); + query.addFinalState(1); + query.addFinalState(2); + query.addFinalState(3); + break; + case "robotic4": + checkArguments(queryName, 3, predicateString); + query = new QueryAutomata<>(2); + query.addTransition(0, predicateString[0], 1); + query.addTransition(0, predicateString[1], 1); + query.addTransition(0, predicateString[2], 1); + query.addTransition(1, predicateString[0], 1); + query.addTransition(1, predicateString[1], 1); + query.addTransition(1, predicateString[2], 1); + query.addFinalState(0); + query.addFinalState(1); + break; + case "robotic5": + checkArguments(queryName, 3, predicateString); + query = new QueryAutomata<>(3); + query.addTransition(0, predicateString[0], 1); + query.addTransition(1, predicateString[1], 1); + query.addTransition(1, predicateString[2], 2); + query.addFinalState(2); + break; + case "robotic6": + checkArguments(queryName, 3, predicateString); + query = new QueryAutomata<>(4); + query.addTransition(0, predicateString[0], 1); + query.addTransition(1, predicateString[0], 1); + query.addTransition(1, predicateString[1], 2); + query.addTransition(2, predicateString[2], 3); + query.addTransition(3, predicateString[2], 3); + query.addFinalState(1); + query.addFinalState(3); + break; + case "robotic7": + checkArguments(queryName, 3, predicateString); + query = new QueryAutomata<>(3); + query.addTransition(0, predicateString[0], 1); + query.addTransition(1, predicateString[1], 2); + query.addTransition(2, predicateString[2], 2); + query.addFinalState(2); + break; + case "robotic8": + checkArguments(queryName, 2, predicateString); + query = new QueryAutomata<>(2); + query.addTransition(0, predicateString[0], 1); + query.addTransition(0, predicateString[1], 1); + query.addTransition(1, predicateString[1], 1); + query.addFinalState(1); + break; + case "robotic9": + checkArguments(queryName, 3, predicateString); + query = new QueryAutomata<>(2); + query.addTransition(0, predicateString[0], 1); + query.addTransition(0, predicateString[1], 1); + query.addTransition(0, predicateString[2], 1); + query.addTransition(1, predicateString[0], 1); + query.addTransition(1, predicateString[1], 1); + query.addTransition(1, predicateString[2], 1); + query.addFinalState(1); + break; + case "robotic10": + checkArguments(queryName, 3, predicateString); + query = new QueryAutomata<>(2); + query.addTransition(0, predicateString[0], 1); + query.addTransition(0, predicateString[1], 1); + query.addTransition(1, predicateString[2], 1); + query.addFinalState(1); + break; + case "robotic11": + checkArguments(queryName, 3, predicateString); + query = new QueryAutomata<>(4); + query.addTransition(0, predicateString[0], 1); + query.addTransition(1, predicateString[1], 2); + query.addTransition(2, predicateString[2], 3); + query.addFinalState(3); + break; + case "robotic12": + checkArguments(queryName, 3, predicateString); + query = new QueryAutomata<>(2); + query.addTransition(0, predicateString[0], 1); + query.addTransition(0, predicateString[1], 1); + query.addTransition(0, predicateString[2], 1); + query.addFinalState(1); + break; + case "robotic13": + checkArguments(queryName, 3, predicateString); + query = new QueryAutomata<>(4); + query.addTransition(0, predicateString[0], 1); + query.addTransition(1, predicateString[1], 2); + query.addTransition(1, predicateString[2], 3); + query.addTransition(2, predicateString[2], 3); + query.addFinalState(1); + query.addFinalState(2); + query.addFinalState(3); + break; + default: + throw new IllegalArgumentException(queryName + " is not a valid query name"); + } + + return query; + } + + private static void checkArguments(String queryName, int requiredArguments, Object... arguments) throws IllegalArgumentException{ + if(arguments.length < requiredArguments) { + throw new IllegalArgumentException("Query " + queryName + " requires " + requiredArguments + " predicates"); + } + } +} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/runtime/STQueryRunner.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/runtime/STQueryRunner.java index 8f839c0..8162546 100644 --- a/src/main/java/ca/uwaterloo/cs/streamingrpq/runtime/STQueryRunner.java +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/runtime/STQueryRunner.java @@ -1,10 +1,11 @@ package ca.uwaterloo.cs.streamingrpq.runtime; -import ca.uwaterloo.cs.streamingrpq.input.TextStream; -import ca.uwaterloo.cs.streamingrpq.input.Yago2sHashStream; -import ca.uwaterloo.cs.streamingrpq.input.Yago2sTSVStream; +import ca.uwaterloo.cs.streamingrpq.input.*; import ca.uwaterloo.cs.streamingrpq.stree.data.QueryAutomata; -import ca.uwaterloo.cs.streamingrpq.stree.engine.IncrementalRAPQ; +import ca.uwaterloo.cs.streamingrpq.stree.engine.RPQEngine; +import ca.uwaterloo.cs.streamingrpq.stree.engine.WindowedRAPQ; +import ca.uwaterloo.cs.streamingrpq.stree.engine.WindowedRSPQ; +import ca.uwaterloo.cs.streamingrpq.stree.util.Semantics; import ca.uwaterloo.cs.streamingrpq.transitiontable.util.PathSemantics; import ca.uwaterloo.cs.streamingrpq.transitiontable.waveguide.SingleThreadedRun; import com.codahale.metrics.CsvReporter; @@ -39,9 +40,14 @@ public static void main(String[] args) { Integer inputSize = Integer.parseInt(line.getOptionValue("s")); Integer maxSize = Integer.parseInt(line.getOptionValue("s")); String queryName = line.getOptionValue("n"); + Long windowSize = Long.parseLong(line.getOptionValue("ws")); + Long slideSize = Long.parseLong(line.getOptionValue("ss")); + Long startTimestamp = Long.parseLong(line.getOptionValue("st", "0")); + Integer threadCount = Integer.parseInt(line.getOptionValue("tc", "1")); + Integer deletionPercentage = Integer.parseInt(line.getOptionValue("dr", "0")); String semantics = line.getOptionValue("ps"); - PathSemantics pathSemantics = PathSemantics.fromValue(semantics); + Semantics pathSemantics = Semantics.fromValue(semantics); String recordCSVFilePath = line.getOptionValue("r"); @@ -57,48 +63,48 @@ public static void main(String[] args) { case "hash": stream = new Yago2sHashStream(); break; + case "text": + stream = new SimpleTextStream(); + break; + case "snap-sx": + stream = new StackOverflowStream(); + break; + case "ldbc": + stream = new LDBCStream(); + break; default: stream = new Yago2sTSVStream(); } - stream.open(filename, inputSize); - - - QueryAutomata query; + RPQEngine rpq; + QueryAutomata query; + SingleThreadedRun task; + try { + query = MazeQueries.getMazeQuery(queryName, predicateString); + } catch (IllegalArgumentException e) { + logger.error("Error creating the query", e); + return; + } - if(queryName.equals("waveguide6")) { - query = new QueryAutomata<>(4); - query.addTransition(0, predicates[0], 1); - query.addTransition(1, predicates[1], 2); - query.addTransition(2, predicates[2], 3); - query.addTransition(3, predicates[0], 1); - query.addFinalState(3); - } else if(queryName.equals("waveguide5")) { - query = new QueryAutomata<>(4); - query.addTransition(0, predicates[0], 1); - query.addTransition(1, predicates[1], 2); - query.addTransition(2, predicates[1], 2); - query.addTransition(2, predicates[2], 3); - query.addTransition(3, predicates[2], 3); - query.addFinalState(3); + if(pathSemantics.equals(Semantics.ARBITRARY)) { + rpq = new WindowedRAPQ(query, maxSize, windowSize, slideSize, threadCount); } else { - logger.error("Not a valid queryname: " + queryName); - return; + rpq = new WindowedRSPQ(query, maxSize, windowSize, slideSize, threadCount); } - IncrementalRAPQ rapq = new IncrementalRAPQ(query, maxSize); + stream.open(filename, inputSize, startTimestamp, deletionPercentage); + task = new SingleThreadedRun(queryName, stream, rpq); MetricRegistry metricRegistry = new MetricRegistry(); - rapq.addMetricRegistry(metricRegistry); + rpq.addMetricRegistry(metricRegistry); // create the metrics directory File resultDirectory = new File(recordCSVFilePath); resultDirectory.mkdirs(); CsvReporter reporter = CsvReporter.forRegistry(metricRegistry).convertRatesTo(TimeUnit.SECONDS).convertDurationsTo(TimeUnit.MICROSECONDS).build(resultDirectory); - reporter.start(10, TimeUnit.SECONDS); + reporter.start(1, TimeUnit.SECONDS); - SingleThreadedRun task = new SingleThreadedRun(queryName, stream, rapq); try { task.call(); } catch (Exception e) { @@ -109,6 +115,9 @@ public static void main(String[] args) { reporter.stop(); reporter.close(); + //shutdown the engine + rpq.shutDown(); + //shut down the executor //reset the stream so we can reuse it for the next query stream.reset(); @@ -126,6 +135,11 @@ private static Options getCLIOptions() { options.addRequiredOption("n", "name", true, "name of the query to be run"); options.addRequiredOption("ps", "semantics", true, "path semantics"); options.addRequiredOption("r", "report-path", true, "CSV file to record execution metrics"); + options.addRequiredOption("ws", "window-size", true, "Size of the window in milliseconds"); + options.addRequiredOption("ss", "slide-size", true, "Slide of the window in milliseconds"); + options.addOption("st", "start-timestamp", true, "Starting timestamp, 0 by default"); + options.addOption("tc", "threadCount", true, "# of Threads for inter-query parallelism"); + options.addOption("dr", "deleteRatio", true, "percentage of deletions in the stream"); Option labelOption = new Option("l", "labels", true, "list of labels in order"); labelOption.setArgs(Option.UNLIMITED_VALUES); diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/Delta.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/Delta.java deleted file mode 100644 index acf22a4..0000000 --- a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/Delta.java +++ /dev/null @@ -1,50 +0,0 @@ -package ca.uwaterloo.cs.streamingrpq.stree.data; - -import ca.uwaterloo.cs.streamingrpq.stree.util.Constants; -import ca.uwaterloo.cs.streamingrpq.stree.util.Hasher; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Multimap; - -import java.util.Collection; -import java.util.HashMap; - -public class Delta { - - private HashMap treeIndex; - private Multimap treeNodeIndex; - - - public Delta(int capacity) { - treeIndex = new HashMap<>(capacity); - treeNodeIndex = HashMultimap.create(capacity, Constants.EXPECTED_TREES); - } - - public SpanningTree getTree(V vertex) { - SpanningTree tree = treeIndex.get(vertex); - return tree; - } - - public Collection getTrees() { - return treeIndex.values(); - } - - public Collection getTrees(V vertex, int state) { - return treeNodeIndex.get(Hasher.TreeNodeHasher(vertex.hashCode(), state)); - } - - public boolean exists(V vertex) { - return treeIndex.containsKey(vertex); - } - - public void addTree(V vertex) { - if(exists(vertex)) { - // TODO one spanner per root vertex - } - SpanningTree tree = new SpanningTree<>(this, vertex); - treeIndex.put(vertex, tree); - } - - protected void updateTreeNodeIndex(SpanningTree tree, TreeNode treeNode) { - treeNodeIndex.put(treeNode.hashCode(), tree); - } -} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/Graph.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/Graph.java deleted file mode 100644 index 79933a4..0000000 --- a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/Graph.java +++ /dev/null @@ -1,52 +0,0 @@ -package ca.uwaterloo.cs.streamingrpq.stree.data; - -import ca.uwaterloo.cs.streamingrpq.stree.util.Constants; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Multimap; - -import java.util.HashMap; - -public class Graph { - private HashMap> forwardAdjacency; - private HashMap> backwardAdjacency; - - - public Graph(int capacity) { - forwardAdjacency = new HashMap<>(capacity); - backwardAdjacency = new HashMap<>(capacity); - } - - public void addEdge(V source, V target, L label) { - if(!forwardAdjacency.containsKey(source)) { - forwardAdjacency.put(source, HashMultimap.create(Constants.EXPECTED_LABELS, Constants.EXPECTED_NEIGHBOURS)); - } - - Multimap forwardLabels = forwardAdjacency.get(source); - forwardLabels.put(label, target); - - if(!backwardAdjacency.containsKey(target)) { - backwardAdjacency.put(target, HashMultimap.create(Constants.EXPECTED_LABELS, Constants.EXPECTED_NEIGHBOURS)); - } - - Multimap backwardLabels = backwardAdjacency.get(target); - backwardLabels.put(label, source); - } - - public Multimap getForwardEdges(V source) { - if(forwardAdjacency.containsKey(source)) { - return forwardAdjacency.get(source); - } - - // TODO edge does not exist - return null; - } - - public Multimap getBackwardEdges(V source) { - if(backwardAdjacency.containsKey(source)) { - return backwardAdjacency.get(source); - } - - // TODO edge does not exist - return null; - } -} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/GraphEdge.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/GraphEdge.java new file mode 100644 index 0000000..ebc7e8c --- /dev/null +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/GraphEdge.java @@ -0,0 +1,52 @@ +package ca.uwaterloo.cs.streamingrpq.stree.data; + +/** + * Created by anilpacaci on 2019-10-02. + */ +public class GraphEdge { + + private int h = 0; + + private V source; + private V target; + private long timestamp; + + public GraphEdge(V source, V target, long timestamp) { + this.source = source; + this.target = target; + this.timestamp = timestamp; + } + + public V getSource() { + return source; + } + + public V getTarget() { + return target; + } + + public long getTimestamp() { + return timestamp; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + GraphEdge graphEdge = (GraphEdge) o; + + if (!source.equals(graphEdge.source)) return false; + return (target.equals(graphEdge.target)); + } + + @Override + public int hashCode() { + if(h == 0) { + int result = source.hashCode(); + result = 31 * result + target.hashCode(); + h = result; + } + return h; + } +} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/ProductGraph.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/ProductGraph.java new file mode 100644 index 0000000..c26959c --- /dev/null +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/ProductGraph.java @@ -0,0 +1,125 @@ +package ca.uwaterloo.cs.streamingrpq.stree.data; + +import ca.uwaterloo.cs.streamingrpq.stree.util.Hasher; +import com.codahale.metrics.MetricRegistry; +import com.google.common.collect.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.Map; + +public class ProductGraph { + + private QueryAutomata automata; + + private Map, ProductGraphNode> nodeIndex; + + private int edgeCount; + + private LinkedList>> timeOrderedEdges; + + private final Logger LOG = LoggerFactory.getLogger(ProductGraph.class); + + public ProductGraph(int capacity, QueryAutomata automata) { + timeOrderedEdges = new LinkedList>>(); + nodeIndex = Maps.newHashMapWithExpectedSize(capacity); + this.automata = automata; + this.edgeCount = 0; + } + + public void addEdge(V source, V target, L label, long timestamp) { + Map transitions = automata.getTransition(label); + for(Map.Entry transition : transitions.entrySet()) { + int sourceState = transition.getKey(); + int targetState = transition.getValue(); + ProductGraphNode sourceNode = this.getNode(source, sourceState); + ProductGraphNode targetNode = this.getNode(target, targetState); + GraphEdge> forwardEdge = new GraphEdge<>(sourceNode, targetNode, timestamp); + sourceNode.addForwardEdge(forwardEdge); + targetNode.addBackwardEdge(forwardEdge); + timeOrderedEdges.add(forwardEdge); + edgeCount++; + } + } + + public void removeEdge(V source, V target, L label, long timestamp) { + Map transitions = automata.getTransition(label); + for(Map.Entry transition : transitions.entrySet()) { + int sourceState = transition.getKey(); + int targetState = transition.getValue(); + ProductGraphNode sourceNode = this.getNode(source, sourceState); + ProductGraphNode targetNode = this.getNode(target, targetState); + GraphEdge> forwardEdge = new GraphEdge<>(sourceNode, targetNode, timestamp); + sourceNode.removeForwardEdge(forwardEdge); + targetNode.removeBackwardEdge(forwardEdge); + //timeOrderedEdges.add(forwardEdge); + edgeCount--; + } + } + + private ProductGraphNode getNode(V vertex, int state) { + ProductGraphNode node = this.nodeIndex.get(Hasher.getThreadLocalTreeNodePairKey(vertex, state)); + if(node == null) { + node = new ProductGraphNode<>(vertex, state); + this.nodeIndex.put(Hasher.createTreeNodePairKey(vertex, state), node); + } + return node; + } + + private void removeEdgeFromHashIndexes(GraphEdge> edge) { + edge.getSource().removeForwardEdge(edge); + edge.getTarget().removeBackwardEdge(edge); + + } + + public Collection> getVertices() { + return nodeIndex.values(); + } + + public Collection>> getForwardEdges(V source, int state) { + return getForwardEdges(this.getNode(source, state)); + } + + public Collection>> getForwardEdges(ProductGraphNode node) { + return node.getForwardEdges(); + } + + public Collection>> getBackwardEdges(V source, int state) { + return getBackwardEdges(this.getNode(source, state)); + } + + public Collection>> getBackwardEdges(ProductGraphNode node) { + return node.getBackwardEdges(); + } + + /** + * removes old edges from the productGraph, used during window management + * @param minTimestamp lower bound of the window interval. Any edge whose timestamp is smaller will be removed + */ + public void removeOldEdges(long minTimestamp) { + LOG.info("Graph expiry at {}", minTimestamp); + // it suffices to linearly scan from the oldest edge as we assume ordered arrival + Iterator>> edgeIterator = timeOrderedEdges.iterator(); + while(edgeIterator.hasNext()) { + GraphEdge> oldestEdge = edgeIterator.next(); + if(oldestEdge.getTimestamp() <= minTimestamp) { + edgeIterator.remove(); + removeEdgeFromHashIndexes(oldestEdge); + edgeCount--; + } else { + // as we assume ordered arrival, we can stop the search + break; + } + } + } + + public int getEdgeCount() { + return edgeCount; + } + + public void addMetricRegistry(MetricRegistry metricRegistry) { + } +} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/ProductGraphNode.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/ProductGraphNode.java new file mode 100644 index 0000000..5f48704 --- /dev/null +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/ProductGraphNode.java @@ -0,0 +1,89 @@ +package ca.uwaterloo.cs.streamingrpq.stree.data; + +import ca.uwaterloo.cs.streamingrpq.stree.util.Constants; + +import java.util.*; + +/** + * Created by anilpacaci on 2019-10-08. + */ +public class ProductGraphNode { + + private V vertex; + private int state; + + private Queue>> forwardEdges; + private Queue>> backwardEdges; + + private int hash = 0; + + public ProductGraphNode(V vertex, int state) { + this.vertex = vertex; + this.state = state; + + this.forwardEdges = new ArrayDeque<>(Constants.EXPECTED_NEIGHBOURS); + this.backwardEdges = new ArrayDeque<>(Constants.EXPECTED_NEIGHBOURS); + } + + public V getVertex() { + return vertex; + } + + public int getState() { + return state; + } + + protected void addForwardEdge(GraphEdge> forwardEdge) { + this.forwardEdges.add(forwardEdge); + } + + protected void addBackwardEdge(GraphEdge> backwardEdge) { + this.backwardEdges.add(backwardEdge); + } + + protected void removeForwardEdge(GraphEdge> forwardEdge) { + this.forwardEdges.remove(forwardEdge); + } + + protected void removeBackwardEdge(GraphEdge> backwardEdge) { + this.backwardEdges.remove(backwardEdge); + } + + public Collection>> getForwardEdges() { + return forwardEdges; + } + + public Collection>> getBackwardEdges() { + return backwardEdges; + } + + @Override + + public boolean equals(Object o) { + if (o == this) return true; + if (! (o instanceof ProductGraphNode)) return false; + + ProductGraphNode pair = (ProductGraphNode) o; + + return pair.vertex .equals(this.vertex) && pair.state == this.state; + } + + // implementation from effective Java : Item 9 + @Override + public int hashCode() { + int h = hash; + if (h == 0) { + h = 17; + h = 31 * h + state; + h = 31 * h + vertex.hashCode(); + hash = h; + } + return h; + } + + @Override + public String toString() { + return new StringBuilder("<").append(this.vertex) + .append(",").append(this.state).append(">").toString(); + } +} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/QueryAutomata.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/QueryAutomata.java index e9c9219..e0a7a49 100644 --- a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/QueryAutomata.java +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/QueryAutomata.java @@ -1,22 +1,34 @@ package ca.uwaterloo.cs.streamingrpq.stree.data; -import com.google.common.collect.HashBasedTable; -import com.google.common.collect.Table; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; +import java.util.*; public class QueryAutomata { public int numOfStates; public Set finalStates; - Table transitions; + HashMap> transitions; + + HashMap> reverseTransitions; + + HashMap> labelTransitions; + + private boolean containmentMark[][]; public QueryAutomata(int numOfStates) { + this.containmentMark = new boolean[numOfStates][numOfStates]; finalStates = new HashSet<>(); - transitions = HashBasedTable.create(); + transitions = new HashMap<>(); + reverseTransitions = new HashMap<>(); + labelTransitions = new HashMap<>(); this.numOfStates = numOfStates; + // initialize transition maps for all + for(int i = 0; i < numOfStates; i++) { + transitions.put(i, new HashMap()); + reverseTransitions.put(i, HashMultimap.create()); + } } public void addFinalState(int state) { @@ -31,19 +43,170 @@ public boolean isFinalState(int state) { } public void addTransition(int source, L label, int target) { - transitions.put(source, label, target); + HashMap forwardMap = transitions.get(source); + forwardMap.put(label, target); + Multimap backwardMap = reverseTransitions.get(target); + backwardMap.put(label, source); + if(!labelTransitions.containsKey(label)) { + labelTransitions.put(label, new HashMap<>()); + } + HashMap labelMap = labelTransitions.get(label); + labelMap.put(source, target); } public Integer getTransition(int source, L label) { - return transitions.get(source, label); + HashMap forwardMap = transitions.get(source); + return forwardMap.get(label); } - public Map getTransitions(int source) { - return transitions.row(source); + /** + * + * @param target + * @param label + * @return an empty collection, not null in case target, label entry does not exists + */ + public Collection getReverseTransitions(int target, L label) { + Multimap backwardMap = reverseTransitions.get(target); + + return backwardMap.get(label); } public Map getTransition(L label) { - return transitions.column(label); + return labelTransitions.getOrDefault(label, new HashMap<>()); + } + + public boolean hasContainment(Integer stateQ, Integer stateT) { + if(stateQ == null) { + return true; + } + return !this.containmentMark[stateQ][stateT]; + } + + /** + * Optimization procedure for the autamaton, including minimization, containment relationship + * MUST be called after automaton is constructed + */ + public void computeContainmentRelationship() { + int alphabetSize = labelTransitions.keySet().size(); + + // once we construct the minimized DFA, we can easily compute the sufflix language containment relationship + // Algorithm S of Wood'95 + ArrayList[][] statePairMatrix = new ArrayList[this.numOfStates][this.numOfStates]; + for(int s = 0; s < this.numOfStates; s++) { + for (int t = 0; t < this.numOfStates; t++) { + statePairMatrix[s][t] = new ArrayList<>(); + } + } + // first create a transition matrix for the DFA + int[][] transitionMatrix = new int[this.numOfStates][alphabetSize]; + for(int i = 0; i < this.numOfStates; i++) { + for(int j = 0; j < alphabetSize; j++) { + transitionMatrix[i][j] = -1; + } + } + + Iterator edgeIterator = labelTransitions.keySet().iterator();; + for(int j = 0 ; j < alphabetSize; j++) { + HashMap edges = labelTransitions.get(edgeIterator.next()); + for (Map.Entry edge : edges.entrySet()) { + transitionMatrix[edge.getKey()][j] = edge.getValue(); + } + } + + // initialize: line 1 of Algorithm S + for(int s = 0; s < this.numOfStates; s++) { + for (int t = 0; t < this.numOfStates; t++) { + // for s \in S-F and t \in F + if(!finalStates.contains(s) && finalStates.contains(t)) { + containmentMark[s][t] = true; + } + } + } + + // line 2-7 of Algorithm S0 + for(int s = 0; s < this.numOfStates; s++) { + for (int t = 0; t < this.numOfStates; t++) { + // for s,t \in ((SxS) - ((S-F)xF)) + if(finalStates.contains(s) || !finalStates.contains(t)) { + // implement line 3, + boolean isMarked = false; + Queue markQueue = new ArrayDeque<>(); + for(int j = 0; j < alphabetSize; j++) { + if(transitionMatrix[s][j] == transitionMatrix[t][j] && transitionMatrix[s][j] != -1) { + isMarked = true; + markQueue.add(StatePair.createInstance(s,t)); + } + } + + // recursively mark all the pairs on the list of pairs that are marked in this step + // line 5 of the Algorithm S + while(!markQueue.isEmpty()) { + StatePair pair = markQueue.poll(); + List pairList = statePairMatrix[pair.stateS][pair.stateT]; + for(StatePair candidate : pairList) { + if(!containmentMark[candidate.stateS][candidate.stateT]) { + markQueue.add(candidate); + containmentMark[candidate.stateS][candidate.stateT] = true; + } + } + } + + // if there is no marked, then populate the lists + // line 6 of Algorithm S + if(!isMarked) { + for(int j = 0; j < alphabetSize; j++) { + int sEndpoint = transitionMatrix[s][j]; + int tEndpoint = transitionMatrix[t][j]; + if(sEndpoint != -1 && tEndpoint != -1 && sEndpoint != tEndpoint) { + // Line 7 of Algorithm S + statePairMatrix[sEndpoint][tEndpoint].add(StatePair.createInstance(s,t)); + } + } + } + + } + } + } + + } + + private static class StatePair { + public final int stateS, stateT; + + private StatePair(int stateS, int stateT) { + this.stateS = stateS; + this.stateT = stateT; + } + + public static StatePair createInstance(int stateS, int stateT) { + return new StatePair(stateS,stateT); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if (!(o instanceof StatePair)) { + return false; + } + + StatePair sp = (StatePair) o; + return this.stateS == sp.stateS && this.stateT == sp.stateT; + } + + @Override + public int hashCode() { + int result = 17; + result = 31 * result + this.stateS; + result = 31 * result + this.stateT; + return result; + } + + @Override + public String toString() { + return new StringBuilder(). + append("<").append(stateS).append(",").append(stateT) + .append(">").toString(); + } } } diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/ResultPair.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/ResultPair.java new file mode 100644 index 0000000..708ba13 --- /dev/null +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/ResultPair.java @@ -0,0 +1,33 @@ +package ca.uwaterloo.cs.streamingrpq.stree.data; + +public class ResultPair { + + private final V source; + private final V target; + + private boolean isDeletion; + + public ResultPair(final V source, final V target) { + this.source = source; + this.target = target; + this.isDeletion = false; + } + + public ResultPair(final V source, final V target, boolean isDeletion) { + this.source = source; + this.target = target; + this.isDeletion = isDeletion; + } + + public V getSource() { + return source; + } + + public V getTarget() { + return target; + } + + public boolean isDeletion() { + return isDeletion; + } +} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/SpanningTree.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/SpanningTree.java deleted file mode 100644 index ea37812..0000000 --- a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/SpanningTree.java +++ /dev/null @@ -1,50 +0,0 @@ -package ca.uwaterloo.cs.streamingrpq.stree.data; - -import com.google.common.collect.HashBasedTable; -import com.google.common.collect.Table; - -public class SpanningTree { - - private TreeNode rootNode; - private Delta delta; - - Table nodeIndex; - - protected SpanningTree(Delta delta, V rootVertex) { - this.rootNode = new TreeNode(rootVertex, 0, null, this); - this.delta = delta; - this.nodeIndex = HashBasedTable.create(); - nodeIndex.put(rootVertex, 0, rootNode); - this.delta.updateTreeNodeIndex(this, rootNode); - } - - - public void addNode(TreeNode parentNode, V childVertex, int childState, long timestamp) { - if(parentNode == null) { - // TODO no object found - } - if(parentNode.getTree().equals(this)) { - // TODO wrong tree - } - - TreeNode child = new TreeNode<>(childVertex, childState, parentNode, this); - parentNode.addChildren(child); - nodeIndex.put(childVertex, childState, child); - - // a new node is added to the spanning tree. update delta index - this.delta.updateTreeNodeIndex(this, child); - } - - public boolean exists(V vertex, int state) { - return nodeIndex.contains(vertex, state); - } - - public TreeNode getNode(V vertex, int state) { - TreeNode node = nodeIndex.get(vertex, state); - return node; - } - - public V getRootVertex() { - return this.rootNode.getVertex(); - } -} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/TreeNode.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/TreeNode.java deleted file mode 100644 index e164c83..0000000 --- a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/TreeNode.java +++ /dev/null @@ -1,89 +0,0 @@ -package ca.uwaterloo.cs.streamingrpq.stree.data; - -import ca.uwaterloo.cs.streamingrpq.stree.util.Hasher; - -import java.util.Collection; -import java.util.HashSet; - -public class TreeNode { - - private int hash = 0; - - SpanningTree tree; - - private V vertex; - private int state; - private long timestamp; - - private TreeNode parent; - - private Collection children; - - public TreeNode(V vertex, int state, TreeNode parent, SpanningTree t) { - this.vertex = vertex; - this.state = state; - this.parent = parent; - this.children = new HashSet<>(); - this.tree = t; - } - - public SpanningTree getTree() { - return tree; - } - - public V getVertex() { - return vertex; - } - - - public int getState() { - return state; - } - - public long getTimestamp() { - return timestamp; - } - - public void setTimestamp(long timestamp) { - this.timestamp = timestamp; - } - - public TreeNode getParent() { - return parent; - } - - public void setParent(TreeNode parent) { - this.parent = parent; - } - - public Collection getChildren() { - return children; - } - - protected void addChildren(TreeNode child) { - this.children.add(child); - child.setParent(this); - } - - @Override - public boolean equals(Object o) { - if (o == this) return true; - if (!(o instanceof TreeNode)) { - return false; - } - - TreeNode tuple = (TreeNode) o; - - return tuple.vertex == vertex && tuple.state == state; - } - - @Override - public int hashCode() { - int h = hash; - if(h == 0) { - h = Hasher.TreeNodeHasher(vertex.hashCode(), state); - hash = h; - } - return h; - } -} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/arbitrary/DeltaRAPQ.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/arbitrary/DeltaRAPQ.java new file mode 100644 index 0000000..06cd25d --- /dev/null +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/arbitrary/DeltaRAPQ.java @@ -0,0 +1,199 @@ +package ca.uwaterloo.cs.streamingrpq.stree.data.arbitrary; + +import ca.uwaterloo.cs.streamingrpq.stree.data.*; +import ca.uwaterloo.cs.streamingrpq.stree.util.Constants; +import ca.uwaterloo.cs.streamingrpq.stree.util.Hasher; +import com.codahale.metrics.Counter; +import com.codahale.metrics.Histogram; +import com.codahale.metrics.MetricRegistry; +import com.google.common.collect.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.concurrent.*; + +public class DeltaRAPQ{ + + private Map treeIndex; + private Map, Set> nodeToTreeIndex; + + protected Counter treeCounter; + protected Histogram treeSizeHistogram; + + private final Logger LOG = LoggerFactory.getLogger(DeltaRAPQ.class); + + public DeltaRAPQ(int capacity) { + treeIndex = new ConcurrentHashMap<>(Constants.EXPECTED_TREES); + nodeToTreeIndex = new ConcurrentHashMap<>(Constants.EXPECTED_TREES); + } + + public Collection getTrees(V vertex, int state) { + Set containingTrees = nodeToTreeIndex.computeIfAbsent(Hasher.createTreeNodePairKey(vertex, state), key -> Collections.newSetFromMap(new ConcurrentHashMap()) ); + return containingTrees; + } + + public boolean exists(V vertex) { + return treeIndex.containsKey(vertex); + } + + + + public SpanningTreeRAPQ addTree(V vertex, long timestamp) { + if(exists(vertex)) { + // TODO one spanner per root vertex + } + SpanningTreeRAPQ tree = new SpanningTreeRAPQ<>(this, vertex, timestamp); + treeIndex.put(vertex, tree); + addToTreeNodeIndex(tree, tree.getRootNode()); + + treeCounter.inc(); + return tree; + } + + public void removeTree(SpanningTreeRAPQ tree) { + TreeNode rootNode = tree.getRootNode(); + this.treeIndex.remove(rootNode.getVertex()); + + Collection containingTrees = getTrees(rootNode.getVertex(), rootNode.getState()); + containingTrees.remove(tree); + if(containingTrees.isEmpty()) { + nodeToTreeIndex.remove(Hasher.getThreadLocalTreeNodePairKey(rootNode.getVertex(), rootNode.getState())); + } + + treeCounter.dec(); + } + + protected void addToTreeNodeIndex(SpanningTreeRAPQ tree, TreeNode treeNode) { + Collection containingTrees = getTrees(treeNode.getVertex(), treeNode.getState()); + containingTrees.add(tree); + } + + protected void removeFromTreeIndex(TreeNode removedNode, SpanningTreeRAPQ tree) { + Collection containingTrees = this.getTrees(removedNode.getVertex(), removedNode.getState()); + containingTrees.remove(tree); + if(containingTrees.isEmpty()) { + this.nodeToTreeIndex.remove(Hasher.getThreadLocalTreeNodePairKey(removedNode.getVertex(), removedNode.getState())); + } + } + + /** + * Simply performs a full bFS traversal and re-create all edges + * @param minTimestamp + * @param productGraph + * @param + */ + public void batchExpiry(Long minTimestamp, ProductGraph productGraph, CompletionService executorService) { + // clear both indexes + nodeToTreeIndex.clear(); + treeIndex.clear(); + LOG.info("Batch expiry at {}, indices are cleard", minTimestamp); + + // retrieve all the nodes with start state 0 + Collection> productGraphNodes = productGraph.getVertices(); + + for(ProductGraphNode node : productGraphNodes) { + if(node.getState() != 0) { + // create tree only for the nodes with start state 0 + continue; + } + + Collection>> edges = productGraph.getForwardEdges(node); + Optional maxTimestamp = edges.stream().map(e -> e.getTimestamp()).max(Long::compare); + if(!maxTimestamp.isPresent() || maxTimestamp.get() <= minTimestamp) { + // has no valid edge, skip this node + continue; + } + + // create a spanning tree + SpanningTreeRAPQ tree = this.addTree(node.getVertex(), maxTimestamp.get()); + // traverse this tree + Queue> queue = new LinkedList<>(); + queue.offer(node); + + while(!queue.isEmpty()) { + ProductGraphNode currentNode = queue.remove(); + TreeNode currentTreeNode = tree.getNode(currentNode.getVertex(), currentNode.getState()); + + Collection>> forwardEdges = productGraph.getForwardEdges(currentNode); + for(GraphEdge> forwardEdge : forwardEdges) { + if(forwardEdge.getTimestamp() <= minTimestamp) { + // this edge is not valid, skip it + continue; + } + + ProductGraphNode targetNode = forwardEdge.getTarget(); + if(!tree.exists(targetNode.getVertex(), targetNode.getState())) { + tree.addNode(currentTreeNode, targetNode.getVertex(), targetNode.getState(), Long.min(currentTreeNode.getTimestamp(), forwardEdge.getTimestamp())); + queue.offer(targetNode); + } + } + + } + //entire tree is traversed, now add it to the tree index + + } + + } + + /** + * Updates Perform window expiry on each spanning tree + * @param minTimestamp lower bound on the window interval + * @param productGraph snapshotGraph + * @param executorService + * @param + */ + public void expiry(Long minTimestamp, ProductGraph productGraph, ExecutorService executorService) { + Collection trees = treeIndex.values(); + List> futures = new ArrayList<>(trees.size()); + CompletionService completionService = new ExecutorCompletionService<>(executorService); + + LOG.info("{} of trees in DeltaRAPQ", trees.size()); + for(SpanningTreeRAPQ tree : trees) { + treeSizeHistogram.update(tree.getSize()); + if (tree.getMinTimestamp() > minTimestamp) { + // this tree does not have any node to be deleted, so just skip it + continue; + } + + RAPQSpanningTreeExpiryJob RAPQSpanningTreeExpiryJob = new RAPQSpanningTreeExpiryJob(minTimestamp, productGraph, tree); + futures.add(completionService.submit(RAPQSpanningTreeExpiryJob)); + } + + for(int i = 0; i < futures.size(); i++) { + try { + completionService.take().get(); + } catch (InterruptedException | ExecutionException e) { + LOG.error("SpanningTreeExpiry interrupted during execution", e); + } + } + + LOG.info("Expiry at {}: # of trees {}, # of edges in the productGraph {}", minTimestamp, treeIndex.size(), productGraph.getEdgeCount()); + } + + public void addMetricRegistry(MetricRegistry metricRegistry) { + this.treeCounter = metricRegistry.counter("tree-counter"); + this.treeSizeHistogram = metricRegistry.histogram("tree-size-histogram"); + } + + private static class RAPQSpanningTreeExpiryJob implements Callable { + + private Long minTimestamp; + private ProductGraph productGraph; + + private Table> nodeToTreeIndex; + private SpanningTreeRAPQ tree; + + public RAPQSpanningTreeExpiryJob(Long minTimestamp, ProductGraph productGraph, SpanningTreeRAPQ tree) { + this.minTimestamp = minTimestamp; + this.productGraph = productGraph; + this.tree = tree; + } + + @Override + public Void call() throws Exception { + tree.removeOldEdges(minTimestamp, productGraph); + return null; + } + } +} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/arbitrary/SpanningTreeRAPQ.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/arbitrary/SpanningTreeRAPQ.java new file mode 100644 index 0000000..97e15df --- /dev/null +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/arbitrary/SpanningTreeRAPQ.java @@ -0,0 +1,250 @@ +package ca.uwaterloo.cs.streamingrpq.stree.data.arbitrary; + +import ca.uwaterloo.cs.streamingrpq.stree.data.GraphEdge; +import ca.uwaterloo.cs.streamingrpq.stree.data.ProductGraph; +import ca.uwaterloo.cs.streamingrpq.stree.data.ProductGraphNode; +import ca.uwaterloo.cs.streamingrpq.stree.util.Constants; +import ca.uwaterloo.cs.streamingrpq.stree.util.Hasher; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +public class SpanningTreeRAPQ { + + private TreeNode rootNode; + private DeltaRAPQ deltaRAPQ; + + private Map, TreeNode> nodeIndex; + + private final Logger LOG = LoggerFactory.getLogger(SpanningTreeRAPQ.class); + + private long minTimestamp; + + //expiry related data structures + private HashSet> candidates; + private HashSet> candidateRemoval; + private HashSet> visited; + + protected SpanningTreeRAPQ() { + // empty constructor + } + + protected SpanningTreeRAPQ(DeltaRAPQ deltaRAPQ, V rootVertex, long timestamp) { + this.rootNode = new TreeNode(rootVertex, 0, null, this, timestamp); + this.deltaRAPQ = deltaRAPQ; + this.nodeIndex = new HashMap<>(Constants.EXPECTED_TREE_SIZE); + nodeIndex.put(Hasher.createTreeNodePairKey(rootVertex, 0), rootNode); + this.minTimestamp = timestamp; + + candidates = new HashSet<>(Constants.EXPECTED_TREE_SIZE); + candidateRemoval = new HashSet<>(Constants.EXPECTED_TREE_SIZE); + visited = new HashSet<>(Constants.EXPECTED_TREE_SIZE); + } + + protected int getSize() { + return nodeIndex.size(); + } + + public TreeNode addNode(TreeNode parentNode, V childVertex, int childState, long timestamp) { + if(parentNode == null) { + // TODO no object found + } + if(parentNode.getTree().equals(this)) { + // TODO wrong tree + } + + TreeNode child = new TreeNode<>(childVertex, childState, parentNode, this, timestamp); + nodeIndex.put(Hasher.createTreeNodePairKey(childVertex, childState), child); + + // a new node is added to the spanning tree. update delta index + this.deltaRAPQ.addToTreeNodeIndex(this, child); + + this.updateTimestamp(timestamp); + + return child; + } + + public boolean exists(V vertex, int state) { + return nodeIndex.containsKey(Hasher.getThreadLocalTreeNodePairKey(vertex, state)); + } + + public TreeNode getNode(V vertex, int state) { + TreeNode node = nodeIndex.get(Hasher.getThreadLocalTreeNodePairKey(vertex, state )); + return node; + } + + public V getRootVertex() { + return this.rootNode.getVertex(); + } + + public TreeNode getRootNode() { + return this.rootNode; + } + + protected void updateTimestamp(long timestamp) { + if(timestamp < minTimestamp) { + this.minTimestamp = timestamp; + } + } + + public long getMinTimestamp() { + return minTimestamp; + } + + /** + * removes old edges from the productGraph, used during window management. + * This function assumes that expired edges are removed from the productGraph, so traversal assumes that it is guarenteed to + * traverse valid edges + * @param minTimestamp lower bound of the window interval. Any edge whose timestamp is smaller will be removed + * @return The set of nodes that have expired from the window as there is no other path + */ + public Collection> removeOldEdges(long minTimestamp, ProductGraph productGraph) { + // if root is expired (root node timestamp is its youngest edge), then the entire tree needs to be removed +// if(this.rootNode.getTimestamp() <= minTimestamp) { +// return this.nodeIndex.values(); +// } + + // potentially expired nodes + candidates.clear(); + candidateRemoval.clear(); + + // perform a bfs traversal on tree, no need for visited as it is a three + LinkedList queue = new LinkedList<>(); + // minTimestamp of the tree should be updated, find the lowest timestamp in the tree higher than the minTimestmap + // because after this maintenance, there is not going to be a node in the tree lower than the minTimestamp + long minimumValidTimetamp = Long.MAX_VALUE; + queue.addAll(rootNode.getChildren()); + while(!queue.isEmpty()) { + // populate the queue with children + TreeNode currentVertex = queue.remove(); + queue.addAll(currentVertex.getChildren()); + + // check time timestamp to decide whether it is expired + if(currentVertex.getTimestamp() <= minTimestamp) { + candidates.add(currentVertex); + } + // find minValidTimestamp for filtering for the next maintenance window + if(currentVertex.getTimestamp() > minTimestamp && currentVertex.getTimestamp() < minimumValidTimetamp) { + minimumValidTimetamp = currentVertex.getTimestamp(); + } + } + //update the lowest minimum timestamp for this tree + this.minTimestamp = minimumValidTimetamp; + + Iterator> candidateIterator = candidates.iterator(); + this.visited.clear(); + + LOG.debug("Expiry for spanning tree {}, # of candidates {} out of {} nodes", toString(), candidates.size(), nodeIndex.size()); + + //scan over potential nodes once. + // For each potential, check they have a valid non-tree edge in the original productGraph + // If there is traverse down from here (in the productGraph) and remove all children from potentials + while(candidateIterator.hasNext()) { + TreeNode candidate = candidateIterator.next(); + // check if a previous traversal already found a path for the candidate + if(candidate.getTimestamp() > minTimestamp) { + continue; + } + //check if there exists any incoming edge from a valid state + Collection>> backwardEdges = productGraph.getBackwardEdges(candidate.getVertex(), candidate.getState()); + TreeNode newParent = null; + GraphEdge> newParentEdge = null; + for(GraphEdge> backwardEdge : backwardEdges) { + newParent = this.getNode(backwardEdge.getSource().getVertex(), backwardEdge.getSource().getState()); + if (newParent != null && (!candidates.contains(newParent) || candidateRemoval.contains(newParent))) { + // there is an incoming edge with valid source + // source is valid (in the tree) and not in candidate + newParentEdge = backwardEdge; + break; + } + } + + //if this node becomes valid, just traverse everything down in the productGraph. If somehow I traverse an edge who would + // be an incoming edge of some candidate, then it is removed from candidate so I never check it there. + // If that edge is checked during incoming edge search, than it might be only examined again with a traversal which makes sure + // that edge cannot be visited again. Therefore it is O(m) + if(newParentEdge != null) { + // means that there was a tree node that is not in the candidates but in the tree as a valid node + candidate.setParent(newParent); + candidate.setTimestamp(Long.min(newParent.getTimestamp(), newParentEdge.getTimestamp())); + // current vertex has a valid incoming edge, so it needs to be removed from candidates + candidateRemoval.add(candidate); + + //now traverse the productGraph down from this node, and remove any visited node from candidates + LinkedList traversalQueue = new LinkedList(); + + // initial node is the current candidate, because now it is reachable + traversalQueue.add(candidate); + while(!traversalQueue.isEmpty()){ + TreeNode currentVertex = traversalQueue.remove(); + visited.add(currentVertex); + + Collection>> forwardEdges = productGraph.getForwardEdges(currentVertex.getVertex(), currentVertex.getState()); + // for each potential child + for(GraphEdge> forwardEdge : forwardEdges) { + // I can simply retrieve from the tree index because any node that is reachable are in tree index + TreeNode outgoingTreeNode = this.getNode(forwardEdge.getTarget().getVertex(), forwardEdge.getTarget().getState()); + // there exists such node in the tree & the edge we are traversing is valid & this node has not been visited before + if (outgoingTreeNode != null && forwardEdge.getTimestamp() > minTimestamp && !visited.contains(outgoingTreeNode)) { + if (candidates.contains(outgoingTreeNode)) { + // remove this node from potentials as now there is a younger path + candidateRemoval.add(outgoingTreeNode); + } + if (outgoingTreeNode.getTimestamp() < Long.min(currentVertex.getTimestamp(), forwardEdge.getTimestamp())) { + // note anything in the candidates has a lower timestamp then + // min(currentVertex, forwardEdge) as currentVertex and forward edge are guarenteed to be larger than minTimestamp + outgoingTreeNode.setParent(currentVertex); + outgoingTreeNode.setTimestamp(Long.min(currentVertex.getTimestamp(), forwardEdge.getTimestamp())); + traversalQueue.add(outgoingTreeNode); + } + } + // nodes with forward edge smaller than minTimestamp with already younger paths no need to be visited + // so no need to add them into the queue + } + } + + } + + } + + // update candidates + candidates.removeAll(candidateRemoval); + + // now if there is any potential remanining, it is guarenteed that they are not reachable + // so simply clean the indexes and generate negative result if necessary + for(TreeNode currentVertex : candidates) { + // remove this node from the node index + nodeIndex.remove(Hasher.getThreadLocalTreeNodePairKey(currentVertex.getVertex(), currentVertex.getState())); + //remove this node from parent's chilren list + currentVertex.setParent(null); + + //remove this entry from nodeToTreeIndex + this.deltaRAPQ.removeFromTreeIndex(currentVertex, this); + } + + if(this.isExpired(minTimestamp)) { + TreeNode removedTuple = this.getRootNode(); + deltaRAPQ.removeTree(this); + } + + LOG.debug("Spanning tree rooted at {}, remove {} nodes at timestamp {} ", getRootVertex(), candidates.size(), minTimestamp); + + return candidates; + } + + /** + * Checks whether the entire three has expired, i.e. there is no active edge from the root node + * @return true if there is no active edge from the root node + */ + public boolean isExpired(long minTimestamp) { + boolean expired = rootNode.getChildren().stream().allMatch(c -> c.getTimestamp() <= minTimestamp); + return expired; + } + + @Override + public String toString() { + return new StringBuilder().append("Root:").append(rootNode.getVertex()).append("-TS:").append(rootNode.getTimestamp()).toString(); + } + +} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/arbitrary/TreeNode.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/arbitrary/TreeNode.java new file mode 100644 index 0000000..c9128aa --- /dev/null +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/arbitrary/TreeNode.java @@ -0,0 +1,130 @@ +package ca.uwaterloo.cs.streamingrpq.stree.data.arbitrary; + +import ca.uwaterloo.cs.streamingrpq.stree.data.simple.SpanningTreeRSPQ; +import ca.uwaterloo.cs.streamingrpq.stree.util.Hasher; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.concurrent.ConcurrentHashMap; + +public class TreeNode { + + private int hash = 0; + + private SpanningTreeRAPQ tree; + + protected V vertex; + protected int state; + protected long timestamp; + + private TreeNode parent; + + private Collection children; + + protected TreeNode() { + + } + + protected TreeNode(V vertex, int state, TreeNode parent, SpanningTreeRAPQ t, long timestamp) { + this.vertex = vertex; + this.state = state; + this.parent = parent; + this.children = Collections.newSetFromMap(new ConcurrentHashMap()); + this.tree = t; + this.timestamp = timestamp; + // set this as a child of the parent if it is not null + if(parent != null) { + this.parent.addChildren(this); + } + } + + public SpanningTreeRAPQ getTree() { + return tree; + } + + public V getVertex() { + return vertex; + } + + + public int getState() { + return state; + } + + public long getTimestamp() { + return timestamp; + } + + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + this.tree.updateTimestamp(timestamp); + } + + /** + * Sets the timestamp of this node to Long.MIN for expiry ro remove after an explicit deletion + * it does not set the timestamp for the tree + */ + public void setDeleted() { + this.timestamp = Long.MIN_VALUE; + } + + public TreeNode getParent() { + return parent; + } + + /** + * Changes the parent of the current node. Removes this node from previous parent's children nodes, and + * adds it into new parent's children nodes. + * @param parent new parent. null only if this node is deleted + */ + public void setParent(TreeNode parent) { + // remove this node from previous parent + if(this.parent != null) { + this.parent.children.remove(this); + } + // set a new parent + this.parent = parent; + // if it is set null, then it is time to remove this node + if(this.parent != null) { + // add this as a child to new parent + if (parent != null) ; + this.parent.addChildren(this); + } + } + + public Collection getChildren() { + return children; + } + + private void addChildren(TreeNode child) { + this.children.add(child); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if (!(o instanceof TreeNode)) { + return false; + } + + TreeNode tuple = (TreeNode) o; + + return tuple.vertex.equals(vertex) && tuple.state == state; + } + + @Override + public int hashCode() { + int h = hash; + if(h == 0) { + h = Hasher.TreeNodeHasher(vertex.hashCode(), state); + hash = h; + } + return h; + } + + @Override + public String toString() { + return new StringBuilder().append("Node:").append(getVertex()).append(",").append(getState()).append("-TS:").append(getTimestamp()).toString(); + } +} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/simple/DeltaRSPQ.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/simple/DeltaRSPQ.java new file mode 100644 index 0000000..8dc23c8 --- /dev/null +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/simple/DeltaRSPQ.java @@ -0,0 +1,141 @@ +package ca.uwaterloo.cs.streamingrpq.stree.data.simple; + +import ca.uwaterloo.cs.streamingrpq.stree.data.*; +import ca.uwaterloo.cs.streamingrpq.stree.data.arbitrary.SpanningTreeRAPQ; +import ca.uwaterloo.cs.streamingrpq.stree.util.Constants; +import ca.uwaterloo.cs.streamingrpq.stree.util.Hasher; +import com.codahale.metrics.Counter; +import com.codahale.metrics.Histogram; +import com.codahale.metrics.MetricRegistry; +import com.google.common.collect.Table; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.concurrent.*; + +public class DeltaRSPQ { + + private Map treeIndex; + private Map, Set> nodeToTreeIndex; + + protected Counter treeCounter; + protected Histogram treeSizeHistogram; + + private final Logger LOG = LoggerFactory.getLogger(DeltaRSPQ.class); + + public DeltaRSPQ(int capacity) { + treeIndex = new ConcurrentHashMap<>(Constants.EXPECTED_TREES); + nodeToTreeIndex = new ConcurrentHashMap<>(Constants.EXPECTED_TREES); + } + + public Collection getTrees(V vertex, int state) { + Set containingTrees = nodeToTreeIndex.computeIfAbsent(Hasher.createTreeNodePairKey(vertex, state), key -> + Collections.newSetFromMap(new ConcurrentHashMap()) ); + return containingTrees; + } + + public boolean exists(V vertex) { + return treeIndex.containsKey(vertex); + } + + public SpanningTreeRSPQ addTree(V vertex, long timestamp) { + if(exists(vertex)) { + // TODO one spanner per root vertex + } + SpanningTreeRSPQ tree = new SpanningTreeRSPQ<>(this, vertex, timestamp); + treeIndex.put(vertex, tree); + addToTreeNodeIndex(tree, tree.getRootNode()); + + treeCounter.inc(); + return tree; + } + + public void removeTree(SpanningTreeRSPQ tree) { + TreeNodeRSPQ rootNode = tree.getRootNode(); + this.treeIndex.remove(rootNode.getVertex()); + + Collection containingTrees = getTrees(rootNode.getVertex(), rootNode.getState()); + containingTrees.remove(tree); + if(containingTrees.isEmpty()) { + nodeToTreeIndex.remove(Hasher.getThreadLocalTreeNodePairKey(rootNode.getVertex(), rootNode.getState())); + } + + treeCounter.dec(); + } + + public void addToTreeNodeIndex(SpanningTreeRSPQ tree, TreeNodeRSPQ treeNode) { + Collection containingTrees = getTrees(treeNode.getVertex(), treeNode.getState()); + containingTrees.add(tree); + } + + public void removeFromTreeIndex(TreeNodeRSPQ removedNode, SpanningTreeRSPQ tree) { + Collection containingTrees = this.getTrees(removedNode.getVertex(), removedNode.getState()); + containingTrees.remove(tree); + if(containingTrees.isEmpty()) { + this.nodeToTreeIndex.remove(Hasher.getThreadLocalTreeNodePairKey(removedNode.getVertex(), removedNode.getState())); + } + } + + + /** + * Updates Perform window expiry on each spanning tree + * @param minTimestamp lower bound on the window interval + * @param productGraph snapshotGraph + * @param executorService + * @param + */ + public void expiry(Long minTimestamp, ProductGraph productGraph, ExecutorService executorService) { + Collection trees = treeIndex.values(); + List> futures = new ArrayList<>(trees.size()); + CompletionService completionService = new ExecutorCompletionService<>(executorService); + + LOG.info("{} of trees in DeltaRAPQ", trees.size()); + for(SpanningTreeRSPQ tree : trees) { + treeSizeHistogram.update(tree.getSize()); + if (tree.getMinTimestamp() > minTimestamp) { + // this tree does not have any node to be deleted, so just skip it + continue; + } + + RSPQSpanningTreeExpiryJob RSPQSpanningTreeExpiryJob = new RSPQSpanningTreeExpiryJob(minTimestamp, productGraph, tree); + futures.add(completionService.submit(RSPQSpanningTreeExpiryJob)); + } + + for(int i = 0; i < futures.size(); i++) { + try { + completionService.take().get(); + } catch (InterruptedException | ExecutionException e) { + LOG.error("SpanningTreeExpiry interrupted during execution", e); + } + } + + LOG.info("Expiry at {}: # of trees {}, # of edges in the productGraph {}", minTimestamp, treeIndex.size(), productGraph.getEdgeCount()); + } + + public void addMetricRegistry(MetricRegistry metricRegistry) { + this.treeCounter = metricRegistry.counter("tree-counter"); + this.treeSizeHistogram = metricRegistry.histogram("tree-size-histogram"); + } + + private static class RSPQSpanningTreeExpiryJob implements Callable { + + private Long minTimestamp; + private ProductGraph productGraph; + + private Table> nodeToTreeIndex; + private SpanningTreeRSPQ tree; + + public RSPQSpanningTreeExpiryJob(Long minTimestamp, ProductGraph productGraph, SpanningTreeRSPQ tree) { + this.minTimestamp = minTimestamp; + this.productGraph = productGraph; + this.tree = tree; + } + + @Override + public Void call() throws Exception { + tree.removeOldEdges(minTimestamp, productGraph); + return null; + } + } +} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/simple/SpanningTreeRSPQ.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/simple/SpanningTreeRSPQ.java new file mode 100644 index 0000000..75c2e84 --- /dev/null +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/simple/SpanningTreeRSPQ.java @@ -0,0 +1,331 @@ +package ca.uwaterloo.cs.streamingrpq.stree.data.simple; + +import ca.uwaterloo.cs.streamingrpq.stree.data.*; +import ca.uwaterloo.cs.streamingrpq.stree.util.Constants; +import ca.uwaterloo.cs.streamingrpq.stree.util.Hasher; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; +import com.google.common.collect.Sets; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +public class SpanningTreeRSPQ { + + private TreeNodeRSPQ rootNode; + private DeltaRSPQ delta; + + private Multimap, TreeNodeRSPQ> nodeIndex; + + private HashSet> markings; + + private final Logger LOG = LoggerFactory.getLogger(SpanningTreeRSPQ.class); + + private long minTimestamp; + + //expiry related data structures + private HashSet> candidates; + private HashSet> candidateRemoval; + private HashSet> visited; + + protected SpanningTreeRSPQ(DeltaRSPQ delta, V rootVertex, long timestamp) { + this.rootNode = new TreeNodeRSPQ(rootVertex, 0, null, this, timestamp); + this.delta = delta; + this.nodeIndex = HashMultimap.create(Constants.EXPECTED_TREE_SIZE, Constants.EXPECTED_LABELS); + nodeIndex.put(Hasher.createTreeNodePairKey(rootVertex, 0), rootNode); + this.markings = Sets.newHashSet(); + this.minTimestamp = timestamp; + + candidates = new HashSet<>(Constants.EXPECTED_TREE_SIZE); + candidateRemoval = new HashSet<>(Constants.EXPECTED_TREE_SIZE); + visited = new HashSet<>(Constants.EXPECTED_TREE_SIZE); + } + + protected int getSize() { + return nodeIndex.size(); + } + + public TreeNodeRSPQ addNode(TreeNodeRSPQ parentNode, V childVertex, int childState, long timestamp) { + if(parentNode == null) { + // TODO no object found + } + if(parentNode.getTree().equals(this)) { + // TODO wrong tree + } + + TreeNodeRSPQ child = new TreeNodeRSPQ<>(childVertex, childState, parentNode, this, timestamp); + nodeIndex.put(Hasher.createTreeNodePairKey(childVertex, childState), child); + + // a new node is added to the spanning tree. update delta index + this.delta.addToTreeNodeIndex(this, child); + + this.updateTimestamp(timestamp); + + return child; + } + + public boolean exists(V vertex, int state) { + return nodeIndex.containsKey(Hasher.getThreadLocalTreeNodePairKey(vertex, state)); + } + + public Collection> getNodes(V vertex, int state) { + Collection> node = nodeIndex.get(Hasher.getThreadLocalTreeNodePairKey(vertex, state )); + return node; + } + + public V getRootVertex() { + return this.rootNode.getVertex(); + } + + public TreeNodeRSPQ getRootNode() { + return this.rootNode; + } + + protected void updateTimestamp(long timestamp) { + if(timestamp < minTimestamp) { + this.minTimestamp = timestamp; + } + } + + /** + * removes the node from current nodeIndex. + * If there is no such node remaining, then remove it from delta tree index + * @param node + */ + private void removeNode(TreeNodeRSPQ node) { + Hasher.MapKey nodeKey = Hasher.getThreadLocalTreeNodePairKey(node.getVertex(), node.getState()); + this.nodeIndex.remove(nodeKey, node); + //remove this node from parent's chilren list + node.setParent(null); + if(this.nodeIndex.get(nodeKey).isEmpty()) { + this.delta.removeFromTreeIndex(node, this); + } + } + + public long getMinTimestamp() { + return minTimestamp; + } + + public void addMarking(V vertex, int state) { + markings.add(Hasher.createTreeNodePairKey(vertex, state)); + } + + public boolean isMarked(V vertex, int state) { + return markings.contains(Hasher.getThreadLocalTreeNodePairKey(vertex, state)); + } + + public void removeMarking(V vertex, int state) { + markings.remove(Hasher.getThreadLocalTreeNodePairKey(vertex, state)); + } + + /** + * Implements unmarking procedure for nodes with conflicts + * @param productGraph + * @param parentNode + */ + protected void unmark(ProductGraph productGraph, TreeNodeRSPQ parentNode) { + + Stack> unmarkedNodes = new Stack<>(); + + TreeNodeRSPQ currentNode = parentNode; + + // first unmark all marked nodes upto parent + while (currentNode != null && this.isMarked(currentNode.getVertex(), currentNode.getState())) { + this.removeMarking(currentNode.getVertex(), currentNode.getState()); + unmarkedNodes.push(currentNode); + currentNode = currentNode.getParent(); + } + + // now simply traverse back edges of the candidates and invoke processTransition + while (!unmarkedNodes.isEmpty()) { + currentNode = unmarkedNodes.pop(); + + // get backward edges of the unmarked node + Collection>> backwardEdges = productGraph.getBackwardEdges(currentNode.getVertex(), currentNode.getState()); + for (GraphEdge> backwardEdge : backwardEdges) { + V sourceVertex = backwardEdge.getSource().getVertex(); + int sourceState = backwardEdge.getSource().getState(); + // find all the nodes that are pruned due to previously marking + Collection> parentNodes = this.getNodes(sourceVertex, sourceState); + for (TreeNodeRSPQ parent : parentNodes) { + // try to extend if it is not a cycle in the product graph + if (!parent.containsCM(currentNode.getVertex(), currentNode.getState())) { + extendPrefixPath(productGraph, parent, currentNode.getVertex(), currentNode.getState(), backwardEdge.getTimestamp()); + } + } + } + } + } + + protected void extendPrefixPath(ProductGraph productGraph, TreeNodeRSPQ parentNode, V targetVertex, int targetState, long edgeTimestamp) { + + } + + /** + * removes old edges from the productGraph, used during window management. + * This function assumes that expired edges are removed from the productGraph, so traversal assumes that it is guarenteed to + * traverse valid edges + * @param minTimestamp lower bound of the window interval. Any edge whose timestamp is smaller will be removed + * @return The set of nodes that have expired from the window as there is no other path + */ + protected void removeOldEdges(long minTimestamp, ProductGraph productGraph) { + // if root is expired (root node timestamp is its youngest edge), then the entire tree needs to be removed +// if(this.rootNode.getTimestamp() <= minTimestamp) { +// return this.nodeIndex.values(); +// } + + // potentially expired nodes + candidates.clear(); + candidateRemoval.clear(); + + // perform a bfs traversal on tree, no need for visited as it is a three + LinkedList> queue = new LinkedList<>(); + // minTimestamp of the tree should be updated, find the lowest timestamp in the tree higher than the minTimestmap + // because after this maintenance, there is not going to be a node in the tree lower than the minTimestamp + long minimumValidTimetamp = Long.MAX_VALUE; + queue.addAll(rootNode.getChildren()); + while(!queue.isEmpty()) { + // populate the queue with children + TreeNodeRSPQ currentVertex = queue.remove(); + queue.addAll(currentVertex.getChildren()); + + // check time timestamp to decide whether it is expired + if(currentVertex.getTimestamp() <= minTimestamp) { + // if node is unmarked simply remove it here + if(this.isMarked(currentVertex.getVertex(), currentVertex.getState())) { + // unmarked nodes are guarenteed to have no cross-edges + this.removeNode(currentVertex); + } else { + // marked nodes are just candidate for removal + candidates.add(currentVertex); + } + + } + // find minValidTimestamp for filtering for the next maintenance window + if(currentVertex.getTimestamp() > minTimestamp && currentVertex.getTimestamp() < minimumValidTimetamp) { + minimumValidTimetamp = currentVertex.getTimestamp(); + } + } + //update the lowest minimum timestamp for this tree + this.minTimestamp = minimumValidTimetamp; + + Iterator> candidateIterator = candidates.iterator(); + visited.clear(); + + LOG.debug("Expiry for spanning tree {}, # of candidates {} out of {} nodes", toString(), candidates.size(), nodeIndex.size()); + + //scan over potential nodes once. + // For each potential, check they have a valid non-tree edge in the original productGraph + // If there is traverse down from here (in the productGraph) and remove all children from potentials + while(candidateIterator.hasNext()) { + TreeNodeRSPQ candidate = candidateIterator.next(); + // check if a previous traversal already found a path for the candidate + if(candidate.getTimestamp() > minTimestamp) { + continue; + } + //check if there exists any incoming edge from a valid state + Collection>> backwardEdges = productGraph.getBackwardEdges(candidate.getVertex(), candidate.getState()); + TreeNodeRSPQ newParent = null; + GraphEdge> newParentEdge = null; + for(GraphEdge> backwardEdge : backwardEdges) { + Collection> newParents = this.getNodes(backwardEdge.getSource().getVertex(), backwardEdge.getSource().getState()); + // candidate is a marked node, therefore these edges cannot form a cycle or register conflict + for(TreeNodeRSPQ newParentCandidate : newParents) { + if (!candidates.contains(newParentCandidate) || candidateRemoval.contains(newParentCandidate)) { + // there is an incoming edge with valid source + // source is valid (in the tree) and not in candidate + newParent = newParentCandidate; + newParentEdge = backwardEdge; + break; + } + } + if(newParentEdge != null) { + // a valid backward edge is found + break; + } + } + + //if this node becomes valid, just traverse everything down in the productGraph. If somehow I traverse an edge who would + // be an incoming edge of some candidate, then it is removed from candidate so I never check it there. + // If that edge is checked during incoming edge search, than it might be only examined again with a traversal which makes sure + // that edge cannot be visited again. Therefore it is O(m) + if(newParentEdge != null) { + // means that there was a tree node that is not in the candidates but in the tree as a valid node + candidate.setParent(newParent); + candidate.setTimestamp(Long.min(newParent.getTimestamp(), newParentEdge.getTimestamp())); + // current vertex has a valid incoming edge, so it needs to be removed from candidates + candidateRemoval.add(candidate); + + //now traverse the productGraph down from this node, and remove any visited node from candidates + LinkedList> traversalQueue = new LinkedList<>(); + + // initial node is the current candidate, because now it is reachable + traversalQueue.add(candidate); + while(!traversalQueue.isEmpty()){ + TreeNodeRSPQ currentVertex = traversalQueue.remove(); + visited.add(currentVertex); + + Collection>> forwardEdges = productGraph.getForwardEdges(currentVertex.getVertex(), currentVertex.getState()); + // for each potential child + for(GraphEdge> forwardEdge : forwardEdges) { + // I can simply retrieve from the tree index because any node that is reachable are in tree index + Collection> outgoingTreeNodes = this.getNodes(forwardEdge.getTarget().getVertex(), forwardEdge.getTarget().getState()); + for (TreeNodeRSPQ outgoingTreeNode : outgoingTreeNodes) { + // there exists such node in the tree & the edge we are traversing is valid & this node has not been visited before + if (forwardEdge.getTimestamp() > minTimestamp && !visited.contains(outgoingTreeNode)) { + if (candidates.contains(outgoingTreeNode)) { + // remove this node from potentials as now there is a younger path + candidateRemoval.add(outgoingTreeNode); + } + if (outgoingTreeNode.getTimestamp() < Long.min(currentVertex.getTimestamp(), forwardEdge.getTimestamp())) { + // note anything in the candidates has a lower timestamp then + // min(currentVertex, forwardEdge) as currentVertex and forward edge are guarenteed to be larger than minTimestamp + outgoingTreeNode.setParent(currentVertex); + outgoingTreeNode.setTimestamp(Long.min(currentVertex.getTimestamp(), forwardEdge.getTimestamp())); + traversalQueue.add(outgoingTreeNode); + } + } + // nodes with forward edge smaller than minTimestamp with already younger paths no need to be visited + // so no need to add them into the queue + } + } + } + + } + + } + + // update candidates + candidates.removeAll(candidateRemoval); + + // now if there is any potential remanining, it is guarenteed that they are not reachable + // so simply clean the indexes and generate negative result if necessary + for(TreeNodeRSPQ currentVertex : candidates) { + // remove this node from the node index + this.removeNode(currentVertex); + } + + if(this.isExpired(minTimestamp)) { + TreeNodeRSPQ removedTuple = this.getRootNode(); + delta.removeTree(this); + } + + LOG.debug("Spanning tree rooted at {}, remove {} nodes at timestamp {} ", getRootVertex(), candidates.size(), minTimestamp); + } + + /** + * Checks whether the entire three has expired, i.e. there is no active edge from the root node + * @return true if there is no active edge from the root node + */ + public boolean isExpired(long minTimestamp) { + boolean expired = rootNode.getChildren().stream().allMatch(c -> c.getTimestamp() <= minTimestamp); + return expired; + } + + @Override + public String toString() { + return new StringBuilder().append("Root:").append(rootNode.getVertex()).append("-TS:").append(rootNode.getTimestamp()).toString(); + } + +} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/simple/TreeNodeRSPQ.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/simple/TreeNodeRSPQ.java new file mode 100644 index 0000000..d71b32c --- /dev/null +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/data/simple/TreeNodeRSPQ.java @@ -0,0 +1,158 @@ +package ca.uwaterloo.cs.streamingrpq.stree.data.simple; + +import ca.uwaterloo.cs.streamingrpq.stree.data.arbitrary.TreeNode; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Maps; +import com.google.common.collect.SetMultimap; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class TreeNodeRSPQ { + + private int hash = 0; + + protected V vertex; + protected int state; + protected long timestamp; + + private SpanningTreeRSPQ tree; + + private TreeNodeRSPQ parent; + + private Collection> children; + + private Map firstMarkings; + private SetMultimap currentMarkings; + + protected TreeNodeRSPQ(V vertex, int state, TreeNodeRSPQ parent, SpanningTreeRSPQ t, long timestamp) { + this.vertex = vertex; + this.state = state; + this.parent = parent; + this.children = Collections.newSetFromMap(new ConcurrentHashMap, Boolean>());; + this.tree = t; + this.timestamp = timestamp; + // set this as a child of the parent if it is not null + if(parent != null) { + this.firstMarkings = Maps.newHashMap(parent.firstMarkings); + this.currentMarkings = HashMultimap.create(parent.currentMarkings); + } else { + this.firstMarkings = Maps.newHashMap(); + this.currentMarkings = HashMultimap.create(); + } + + // populate with root node + this.firstMarkings.putIfAbsent(vertex, state); + this.currentMarkings.put(vertex, state); + + if(parent != null) { + this.parent.addChildren(this); + } + } + + public SpanningTreeRSPQ getTree() { + return tree; + } + + public V getVertex() { + return vertex; + } + + + public int getState() { + return state; + } + + public long getTimestamp() { + return timestamp; + } + + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + this.tree.updateTimestamp(timestamp); + } + + public TreeNodeRSPQ getParent() { + return parent; + } + + /** + * Changes the parent of the current node. Removes this node from previous parent's children nodes, and + * adds it into new parent's children nodes. + * @param parent new parent. null only if this node is deleted + */ + public void setParent(TreeNodeRSPQ parent) { + // remove this node from previous parent + if(this.parent != null) { + this.parent.children.remove(this); + } + // set a new parent + this.parent = parent; + // if it is set null, then it is time to remove this node + if(this.parent != null) { + // add this as a child to new parent + if (parent != null) ; + this.parent.addChildren(this); + } + } + + public Collection> getChildren() { + return children; + } + + private void addChildren(TreeNodeRSPQ child) { + this.children.add(child); + } + + /** + * Check whether current markings contain this pair of vertex-state + * @param vertex + * @param state + * @return true if this vertex has already been visited on this state on the path to this node from the root + */ + public boolean containsCM(V vertex, int state) { + return currentMarkings.containsEntry(vertex, state); + } + + /** + * The first state this vertex is visited on the path from root to this node + * @param vertex + * @return null if no such vertex exists + */ + public Integer getFirstCM(V vertex) { + return firstMarkings.get(vertex); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if (!(o instanceof TreeNodeRSPQ)) { + return false; + } + + TreeNodeRSPQ tuple = (TreeNodeRSPQ) o; + + return tuple.vertex.equals(vertex) && + tuple.state == state && + tuple.firstMarkings.equals(firstMarkings) && + tuple.currentMarkings.equals(currentMarkings); + } + + @Override + public int hashCode() { + int h = hash; + if(h == 0) { + h = 17; + h = 31 * h + vertex.hashCode(); + h = 31 * h + state; + h = 31 * h + firstMarkings.hashCode(); + h = 31 * h + currentMarkings.hashCode(); + hash = h; + } + return h; + } + +} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/engine/AbstractTreeExpansionJob.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/engine/AbstractTreeExpansionJob.java new file mode 100644 index 0000000..d6a8f2b --- /dev/null +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/engine/AbstractTreeExpansionJob.java @@ -0,0 +1,24 @@ +package ca.uwaterloo.cs.streamingrpq.stree.engine; + +import ca.uwaterloo.cs.streamingrpq.stree.data.arbitrary.SpanningTreeRAPQ; +import ca.uwaterloo.cs.streamingrpq.stree.data.arbitrary.TreeNode; + +import java.util.concurrent.Callable; + +public abstract class AbstractTreeExpansionJob implements Callable { + + /** + * Populates the job array + * @param spanningTree + * @param parentNode + * @param targetVertex + * @param targetState + * @param edgeTimestamp + * @return false whenever job array is full and cannot be further populated + */ + abstract boolean addJob(T spanningTree, N parentNode, int targetVertex, int targetState, long edgeTimestamp) throws IllegalStateException; + + abstract boolean isFull(); + + abstract boolean isEmpty(); +} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/engine/IncrementalRAPQ.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/engine/IncrementalRAPQ.java deleted file mode 100644 index def0ab0..0000000 --- a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/engine/IncrementalRAPQ.java +++ /dev/null @@ -1,126 +0,0 @@ -package ca.uwaterloo.cs.streamingrpq.stree.engine; - -import ca.uwaterloo.cs.streamingrpq.input.InputTuple; -import ca.uwaterloo.cs.streamingrpq.stree.data.*; -import com.codahale.metrics.*; -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Multimap; - -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -public class IncrementalRAPQ { - - // metric tools - private MetricRegistry metricRegistry; - private Counter expansionCounter; - private Histogram fullHistogram; - private Histogram processedHistogram; - private Timer fullTimer; - private Meter queueMeter; - - private Delta delta; - private Graph graph; - private QueryAutomata automata; - - private Multimap results; - - public IncrementalRAPQ(QueryAutomata query, int capacity) { - delta = new Delta<>(capacity); - automata = query; - results = HashMultimap.create(); - graph = new Graph<>(capacity); - } - - public void processTransition(SpanningTree tree, int parentVertex, int parentState, int childVertex, int childState) { - TreeNode parentNode = tree.getNode(parentVertex, parentState); - // extend the spanning tree with incoming noe - tree.addNode(parentNode, childVertex, childState, parentNode.getTimestamp()); - - // add this pair to results if it is a final state - if(automata.isFinalState(childState)) { - results.put(tree.getRootVertex(), childVertex); - } - - // get all the forward edges of the new extended node - Multimap forwardEdges = graph.getForwardEdges(childVertex); - - if(forwardEdges == null) { - // TODO better nul handling - // end recursion if node has no forward edges - return; - } - - for(Map.Entry forwardEdge : forwardEdges.entries()) { - Integer targetState = automata.getTransition(childState, forwardEdge.getKey()); - if(targetState != null && !tree.exists(forwardEdge.getValue(), targetState)) { - // recursive call as the target of the forwardEdge has not been visited in state targetState before - processTransition(tree, childVertex, childState, forwardEdge.getValue(), targetState); - } - } - } - - public void processEdge(InputTuple inputTuple) { - Long startTime = System.nanoTime(); - Timer.Context timer = fullTimer.time(); - - // retrieve all transition that can be performed with this label - Map transitions = automata.getTransition(inputTuple.getLabel()); - - if(transitions.isEmpty()) { - // there is no transition with given label, simply return - return; - } else { - // add edge to the snapshot graph - graph.addEdge(inputTuple.getSource(), inputTuple.getTarget(), inputTuple.getLabel()); - } - - //create a spanning tree for the source node in case it does not exists - if(transitions.keySet().contains(0) && !delta.exists(inputTuple.getSource())) { - // if there exists a start transition with given label, there should be a spanning tree rooted at source vertex - delta.addTree(inputTuple.getSource()); - } - - List> transitionList = transitions.entrySet().stream().collect(Collectors.toList()); - - // for each transition that given label satisy - for(Map.Entry transition : transitionList) { - int sourceState = transition.getKey(); - int targetState = transition.getValue(); - - // iterate over spanning trees that include the source node - for(SpanningTree spanningTree : delta.getTrees(inputTuple.getSource(), sourceState)) { - // if the source already exists, but not the target - if (spanningTree.exists(inputTuple.getSource(), sourceState) && !spanningTree.exists(inputTuple.getTarget(), targetState)) { - processTransition(spanningTree, inputTuple.getSource(), sourceState, inputTuple.getTarget(), targetState); - } - } - } - - - // metric recording - Long elapsedTime = System.nanoTime() - startTime; - //populate histograms - fullHistogram.update(elapsedTime); - timer.stop(); - if(!transitions.isEmpty()) { - // it implies that edge is processed - processedHistogram.update(elapsedTime); - } - } - - public Multimap getResults() { - return results; - } - - public void addMetricRegistry(MetricRegistry metricRegistry) { - this.metricRegistry = metricRegistry; - // register all the matrics - this.expansionCounter = metricRegistry.counter("expansion-counter"); - this.fullHistogram = metricRegistry.histogram("full-histogram"); - this.processedHistogram = metricRegistry.histogram("processed-histogram"); - this.fullTimer = metricRegistry.timer("full-timer"); - this.queueMeter = metricRegistry.meter("queue-meter"); - } -} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/engine/RPQEngine.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/engine/RPQEngine.java new file mode 100644 index 0000000..0f97e18 --- /dev/null +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/engine/RPQEngine.java @@ -0,0 +1,76 @@ +package ca.uwaterloo.cs.streamingrpq.stree.engine; + +import ca.uwaterloo.cs.streamingrpq.input.InputTuple; +import ca.uwaterloo.cs.streamingrpq.stree.data.*; +import ca.uwaterloo.cs.streamingrpq.stree.util.Constants; +import com.codahale.metrics.*; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; +import com.google.common.collect.Queues; + +import java.sql.Time; +import java.util.Queue; +import java.util.concurrent.TimeUnit; + +/** + * Created by anilpacaci on 2019-10-02. + */ +public abstract class RPQEngine { + + protected MetricRegistry metricRegistry; + protected Counter resultCounter; + protected Histogram containingTreeHistogram; + protected Histogram fullHistogram; + protected Histogram processedHistogram; + protected Histogram explicitDeletionHistogram; + protected Histogram fullProcessedHistogram; + protected Histogram windowManagementHistogram; + protected Histogram edgeCountHistogram; + protected Timer fullTimer; + + protected ProductGraph productGraph; + protected QueryAutomata automata; + + protected Queue> results; + + protected int edgeCount = 0; + + + protected RPQEngine(QueryAutomata query, int capacity) { + automata = query; + results = Queues.newConcurrentLinkedQueue(); + productGraph = new ProductGraph<>(capacity, query); + } + + public Queue> getResults() { + return results; + } + + public long getResultCount() { + return resultCounter.getCount(); + } + + public void addMetricRegistry(MetricRegistry metricRegistry) { + this.metricRegistry = metricRegistry; + // register all the matrics + this.resultCounter = metricRegistry.counter("result-counter"); + this.fullHistogram = metricRegistry.histogram("full-histogram"); + this.processedHistogram = new Histogram(new SlidingTimeWindowArrayReservoir(10, TimeUnit.MINUTES)); + metricRegistry.register("processed-histogram", this.processedHistogram); + this.explicitDeletionHistogram = new Histogram(new SlidingTimeWindowArrayReservoir(10, TimeUnit.MINUTES)); + metricRegistry.register("explicit-deletion-histogram", this.explicitDeletionHistogram); + this.fullProcessedHistogram = new Histogram(new SlidingTimeWindowArrayReservoir(10, TimeUnit.MINUTES)); + metricRegistry.register("full-processed-histogram", this.fullProcessedHistogram); + this.containingTreeHistogram = metricRegistry.histogram("containing-tree-counter"); + this.fullTimer = metricRegistry.timer("full-timer"); + this.windowManagementHistogram = new Histogram(new SlidingTimeWindowArrayReservoir(10, TimeUnit.MINUTES)); + metricRegistry.register("window-histogram", windowManagementHistogram); + edgeCountHistogram = metricRegistry.histogram("edgecount-histogram"); + + this.productGraph.addMetricRegistry(metricRegistry); + } + + public abstract void processEdge(InputTuple inputTuple); + + public abstract void shutDown(); +} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/engine/TreeNodeRAPQDeletionJob.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/engine/TreeNodeRAPQDeletionJob.java new file mode 100644 index 0000000..9201af7 --- /dev/null +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/engine/TreeNodeRAPQDeletionJob.java @@ -0,0 +1,112 @@ +package ca.uwaterloo.cs.streamingrpq.stree.engine; + +import ca.uwaterloo.cs.streamingrpq.stree.data.*; +import ca.uwaterloo.cs.streamingrpq.stree.data.arbitrary.SpanningTreeRAPQ; +import ca.uwaterloo.cs.streamingrpq.stree.data.arbitrary.TreeNode; +import ca.uwaterloo.cs.streamingrpq.stree.util.Constants; + +import java.util.ArrayDeque; +import java.util.Collection; +import java.util.Queue; + +public class TreeNodeRAPQDeletionJob extends AbstractTreeExpansionJob, TreeNode>{ + + private ProductGraph productGraph; + private QueryAutomata automata; + private SpanningTreeRAPQ spanningTree[]; + private TreeNode parentNode[]; + private int targetVertex[]; + private int targetState[]; + private long edgeTimestamp[]; + + private int currentSize; + + private int resultCount; + + private Queue> results; + + public TreeNodeRAPQDeletionJob(ProductGraph productGraph, QueryAutomata automata, Queue> results) { + this.productGraph = productGraph; + this.automata = automata; + this.spanningTree = new SpanningTreeRAPQ[Constants.EXPECTED_BATCH_SIZE]; + this.parentNode = new TreeNode[Constants.EXPECTED_BATCH_SIZE]; + this.targetVertex = new int[Constants.EXPECTED_BATCH_SIZE]; + this.targetState = new int[Constants.EXPECTED_BATCH_SIZE]; + this.edgeTimestamp = new long[Constants.EXPECTED_BATCH_SIZE]; + this.results = results; + this.currentSize = 0; + this.resultCount = 0; + } + + /** + * Populates the job array + * @param spanningTree + * @param parentNode + * @param targetVertex + * @param targetState + * @param edgeTimestamp + * @return false whenever job array is full and cannot be further populated + */ + public boolean addJob(SpanningTreeRAPQ spanningTree, TreeNode parentNode, int targetVertex, int targetState, long edgeTimestamp) throws IllegalStateException{ + if(this.currentSize >= Constants.EXPECTED_BATCH_SIZE) { + throw new IllegalStateException("Job capacity exceed limit " + currentSize); + } + + this.spanningTree[currentSize] = spanningTree; + this.parentNode[currentSize] = parentNode; + this.targetVertex[currentSize] = targetVertex; + this.targetState[currentSize] = targetState; + this.edgeTimestamp[currentSize] = edgeTimestamp; + this.currentSize++; + + if(currentSize == Constants.EXPECTED_BATCH_SIZE - 1) { + return false; + } + + return true; + } + + public boolean isFull() { + return currentSize == Constants.EXPECTED_BATCH_SIZE - 1; + } + + public boolean isEmpty() { + return currentSize == 0; + } + + @Override + public Integer call() throws Exception { + + // call each job in teh buffer + for(int i = 0; i < currentSize; i++) { + markExpired(spanningTree[i], parentNode[i], targetVertex[i], targetState[i], edgeTimestamp[i]); + } + + return this.resultCount; + } + + public void markExpired(SpanningTreeRAPQ tree, TreeNode parentNode, int childVertex, int childState, long edgeTimestamp) { + // update the timestamp of the entire subtree of such node exists + if(tree.exists(childVertex, childState)) { + // if the child node already exists, we might need to update timestamp + TreeNode childNode = tree.getNode(childVertex, childState); + + Queue> queue = new ArrayDeque<>(); + queue.offer(childNode); + while(!queue.isEmpty()) { + TreeNode currentNode = queue.poll(); + currentNode.setTimestamp(Long.MIN_VALUE); + queue.addAll(currentNode.getChildren()); + } + + // allnodes are marked, + // simply call expiry on the spanning tree + + tree.removeOldEdges(edgeTimestamp, productGraph); + + + } else { + // there is no such edge so no need for deletion + } + } +} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/engine/TreeNodeRAPQTreeExpansionJob.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/engine/TreeNodeRAPQTreeExpansionJob.java new file mode 100644 index 0000000..c73c2f8 --- /dev/null +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/engine/TreeNodeRAPQTreeExpansionJob.java @@ -0,0 +1,187 @@ +package ca.uwaterloo.cs.streamingrpq.stree.engine; + +import ca.uwaterloo.cs.streamingrpq.input.InputTuple; +import ca.uwaterloo.cs.streamingrpq.stree.data.*; +import ca.uwaterloo.cs.streamingrpq.stree.data.arbitrary.SpanningTreeRAPQ; +import ca.uwaterloo.cs.streamingrpq.stree.data.arbitrary.TreeNode; +import ca.uwaterloo.cs.streamingrpq.stree.util.Constants; + +import java.util.ArrayDeque; +import java.util.Collection; +import java.util.Queue; + +public class TreeNodeRAPQTreeExpansionJob extends AbstractTreeExpansionJob, TreeNode>{ + + private ProductGraph productGraph; + private QueryAutomata automata; + private SpanningTreeRAPQ spanningTree[]; + private TreeNode parentNode[]; + private int targetVertex[]; + private int targetState[]; + private long edgeTimestamp[]; + private boolean isDeletion; + + private int currentSize; + + private int resultCount; + + private Queue> results; + + public TreeNodeRAPQTreeExpansionJob(ProductGraph productGraph, QueryAutomata automata, Queue> results, boolean isDeletion) { + this.productGraph = productGraph; + this.automata = automata; + this.spanningTree = new SpanningTreeRAPQ[Constants.EXPECTED_BATCH_SIZE]; + this.parentNode = new TreeNode[Constants.EXPECTED_BATCH_SIZE]; + this.targetVertex = new int[Constants.EXPECTED_BATCH_SIZE]; + this.targetState = new int[Constants.EXPECTED_BATCH_SIZE]; + this.edgeTimestamp = new long[Constants.EXPECTED_BATCH_SIZE]; + this.isDeletion = isDeletion; + this.results = results; + this.currentSize = 0; + this.resultCount = 0; + } + + /** + * Populates the job array + * @param spanningTree + * @param parentNode + * @param targetVertex + * @param targetState + * @param edgeTimestamp + * @return false whenever job array is full and cannot be further populated + */ + public boolean addJob(SpanningTreeRAPQ spanningTree, TreeNode parentNode, int targetVertex, int targetState, long edgeTimestamp) throws IllegalStateException{ + if(this.currentSize >= Constants.EXPECTED_BATCH_SIZE) { + throw new IllegalStateException("Job capacity exceed limit " + currentSize); + } + + this.spanningTree[currentSize] = spanningTree; + this.parentNode[currentSize] = parentNode; + this.targetVertex[currentSize] = targetVertex; + this.targetState[currentSize] = targetState; + this.edgeTimestamp[currentSize] = edgeTimestamp; + this.currentSize++; + + if(currentSize == Constants.EXPECTED_BATCH_SIZE - 1) { + return false; + } + + return true; + } + + public boolean isFull() { + return currentSize == Constants.EXPECTED_BATCH_SIZE - 1; + } + + public boolean isEmpty() { + return currentSize == 0; + } + + @Override + public Integer call() throws Exception { + + if(isDeletion) { + // call each job in teh buffer + for (int i = 0; i < currentSize; i++) { + markExpired(spanningTree[i], parentNode[i], targetVertex[i], targetState[i], edgeTimestamp[i]); + } + } else { + // call each job in teh buffer + for (int i = 0; i < currentSize; i++) { + processTransition(spanningTree[i], parentNode[i], targetVertex[i], targetState[i], edgeTimestamp[i]); + } + } + return this.resultCount; + } + + public void processTransition(SpanningTreeRAPQ tree, TreeNode parentNode, int childVertex, int childState, long edgeTimestamp) { + // either update timestamp, or create the node + if(tree.exists(childVertex, childState)) { + // if the child node already exists, we might need to update timestamp + TreeNode childNode = tree.getNode(childVertex, childState); + + // root's children have timestamp equal to the edge timestamp + // root timestmap always higher than any node in the tree + if(parentNode.equals(tree.getRootNode())) { + childNode.setTimestamp(edgeTimestamp); + parentNode.setTimestamp( edgeTimestamp); + // properly update the parent pointer + childNode.setParent(parentNode); + } + // child node cannot be the root because parent has to be at least + else if(childNode.getTimestamp() < Long.min(parentNode.getTimestamp(), edgeTimestamp)) { + // only update its timestamp if there is a younger path, back edge is guarenteed to be at smaller or equal + childNode.setTimestamp(Long.min(parentNode.getTimestamp(), edgeTimestamp)); + // properly update the parent pointer + childNode.setParent(parentNode); + } + + } else { + // extend the spanning tree with incoming node + + // root's children have timestamp equal to the edge timestamp + // root timestmap always higher than any node in the tree + TreeNode childNode; + if(parentNode.equals(tree.getRootNode())) { + childNode = tree.addNode(parentNode, childVertex, childState, edgeTimestamp); + parentNode.setTimestamp(edgeTimestamp); + } + else { + childNode = tree.addNode(parentNode, childVertex, childState, Long.min(parentNode.getTimestamp(), edgeTimestamp)); + } + // add this pair to results if it is a final state + if (automata.isFinalState(childState)) { + results.offer(new ResultPair<>(tree.getRootVertex(), childVertex)); + resultCount++; + } + + // get all the forward edges of the new extended node + Collection>> forwardEdges = productGraph.getForwardEdges(childVertex, childState); + + if (forwardEdges == null) { + // TODO better nul handling + // end recursion if node has no forward edges + return; + } else { + // there are forward edges, iterate over them + for (GraphEdge> forwardEdge : forwardEdges) { + // recursive call as the target of the forwardEdge has not been visited in state targetState before + //processTransition(tree, childNode, forwardEdge.getTarget(), targetState, forwardEdge.getTimestamp()); + processTransition(tree, childNode, forwardEdge.getTarget().getVertex(), forwardEdge.getTarget().getState(), forwardEdge.getTimestamp()); + } + } + } + } + + public void markExpired(SpanningTreeRAPQ tree, TreeNode parentNode, int childVertex, int childState, long edgeTimestamp) { + // update the timestamp of the entire subtree of such node exists + if(tree.exists(childVertex, childState)) { + // if the child node already exists, we might need to update timestamp + TreeNode childNode = tree.getNode(childVertex, childState); + + Queue> queue = new ArrayDeque<>(); + queue.offer(childNode); + while(!queue.isEmpty()) { + TreeNode currentNode = queue.poll(); + currentNode.setDeleted(); + queue.addAll(currentNode.getChildren()); + } + + // allnodes are marked, + // simply call expiry on the spanning tree + + // it is OK to remove the deletion with timestamp 0 because we only want to delete nodes that are set to Long.MIN + Collection> removedNodes = tree.removeOldEdges(0, productGraph); + + for(TreeNode removedNode : removedNodes) { + if(automata.isFinalState(removedNode.getState())) { + results.offer(new ResultPair(tree.getRootVertex(), removedNode.getVertex(), true)); + resultCount--; + } + } + + } else { + // there is no such edge so no need for deletion + } + } +} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/engine/TreeNodeRSPQTreeExpansionJob.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/engine/TreeNodeRSPQTreeExpansionJob.java new file mode 100644 index 0000000..95f186c --- /dev/null +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/engine/TreeNodeRSPQTreeExpansionJob.java @@ -0,0 +1,173 @@ +package ca.uwaterloo.cs.streamingrpq.stree.engine; + +import ca.uwaterloo.cs.streamingrpq.stree.data.*; +import ca.uwaterloo.cs.streamingrpq.stree.data.simple.SpanningTreeRSPQ; +import ca.uwaterloo.cs.streamingrpq.stree.data.simple.TreeNodeRSPQ; +import ca.uwaterloo.cs.streamingrpq.stree.util.Constants; + +import java.util.Collection; +import java.util.Queue; +import java.util.Stack; + +public class TreeNodeRSPQTreeExpansionJob extends AbstractTreeExpansionJob, TreeNodeRSPQ>{ + + private ProductGraph productGraph; + private QueryAutomata automata; + private SpanningTreeRSPQ spanningTree[]; + private TreeNodeRSPQ parentNode[]; + private int targetVertex[]; + private int targetState[]; + private long edgeTimestamp[]; + + private int currentSize; + + private int resultCount; + + private Queue> results; + + public TreeNodeRSPQTreeExpansionJob(ProductGraph productGraph, QueryAutomata automata, Queue> results) { + this.productGraph = productGraph; + this.automata = automata; + this.spanningTree = new SpanningTreeRSPQ[Constants.EXPECTED_BATCH_SIZE]; + this.parentNode = new TreeNodeRSPQ[Constants.EXPECTED_BATCH_SIZE]; + this.targetVertex = new int[Constants.EXPECTED_BATCH_SIZE]; + this.targetState = new int[Constants.EXPECTED_BATCH_SIZE]; + this.edgeTimestamp = new long[Constants.EXPECTED_BATCH_SIZE]; + this.results = results; + this.currentSize = 0; + this.resultCount = 0; + } + + /** + * Populates the job array + * @param spanningTree + * @param parentNode + * @param targetVertex + * @param targetState + * @param edgeTimestamp + * @return false whenever job array is full and cannot be further populated + */ + public boolean addJob(SpanningTreeRSPQ spanningTree, TreeNodeRSPQ parentNode, int targetVertex, int targetState, long edgeTimestamp) throws IllegalStateException{ + if(this.currentSize >= Constants.EXPECTED_BATCH_SIZE) { + throw new IllegalStateException("Job capacity exceed limit " + currentSize); + } + + this.spanningTree[currentSize] = spanningTree; + this.parentNode[currentSize] = parentNode; + this.targetVertex[currentSize] = targetVertex; + this.targetState[currentSize] = targetState; + this.edgeTimestamp[currentSize] = edgeTimestamp; + this.currentSize++; + + if(currentSize == Constants.EXPECTED_BATCH_SIZE - 1) { + return false; + } + + return true; + } + + public boolean isFull() { + return currentSize == Constants.EXPECTED_BATCH_SIZE - 1; + } + + public boolean isEmpty() { + return currentSize == 0; + } + + @Override + public Integer call() throws Exception { + + // call each job in teh buffer + for(int i = 0; i < currentSize; i++) { + if(!parentNode[i].containsCM(targetVertex[i], targetState[i]) && !spanningTree[i].isMarked(targetVertex[i], targetState[i])) { + processTransition(spanningTree[i], parentNode[i], targetVertex[i], targetState[i], edgeTimestamp[i]); + } + } + + return this.resultCount; + } + + public void processTransition(SpanningTreeRSPQ tree, TreeNodeRSPQ parentNode, int childVertex, int childState, long edgeTimestamp) { + if(!automata.hasContainment(parentNode.getFirstCM(childVertex), childState)) { + // detected conflict parent node needs to be unmarked + unmark(tree, parentNode); + } else { + // if this is the first target product node is visited, simply add it to the markings + if(!tree.exists(childVertex, childState)) { + tree.addMarking(childVertex, childState); + } + + //add new node to the tree as a new child + TreeNodeRSPQ childNode; + if(parentNode.equals(tree.getRootNode())) { + childNode = tree.addNode(parentNode, childVertex, childState, edgeTimestamp); + parentNode.setTimestamp(edgeTimestamp); + } else { + childNode = tree.addNode(parentNode, childVertex, childState, Long.min(parentNode.getTimestamp(), edgeTimestamp)); + } + + if(automata.isFinalState(childState)) { + results.offer(new ResultPair<>(tree.getRootVertex(), childVertex)); + resultCount++; + } + + // get all the forward edges of the new extended node + Collection>> forwardEdges = productGraph.getForwardEdges(childVertex, childState); + + if(forwardEdges == null) { + return; + } else { + // there are forward edges, iterate over them + for (GraphEdge> forwardEdge : forwardEdges) { + int targetVertex = forwardEdge.getTarget().getVertex(); + int targetState = forwardEdge.getTarget().getState(); + if(!childNode.containsCM(targetVertex, targetState) && !tree.isMarked(targetVertex, targetState)) { + // visit a node only if that same node is not visited at the same state before + // simply prevent cycles in product graph + processTransition(tree, childNode, targetVertex, targetState, forwardEdge.getTimestamp()); + } + } + } + } + } + + /** + * Implements unmarking procedure for nodes with conflicts + * @param tree + * @param parentNode + */ + public void unmark(SpanningTreeRSPQ tree, TreeNodeRSPQ parentNode) { + + Stack> unmarkedNodes = new Stack<>(); + + TreeNodeRSPQ currentNode = parentNode; + + // first unmark all marked nodes upto parent + while(currentNode != null && tree.isMarked(currentNode.getVertex(), currentNode.getState())) { + tree.removeMarking(currentNode.getVertex(), currentNode.getState()); + unmarkedNodes.push(currentNode); + currentNode = currentNode.getParent(); + } + + // now simply traverse back edges of the candidates and invoke processTransition + while(!unmarkedNodes.isEmpty()) { + currentNode = unmarkedNodes.pop(); + + // get backward edges of the unmarked node + Collection>> backwardEdges = productGraph.getBackwardEdges(currentNode.getVertex(), currentNode.getState()); + for(GraphEdge> backwardEdge : backwardEdges) { + int sourceVertex = backwardEdge.getSource().getVertex(); + int sourceState = backwardEdge.getSource().getState(); + // find all the nodes that are pruned due to previously marking + Collection> parentNodes = tree.getNodes(sourceVertex, sourceState); + for(TreeNodeRSPQ parent : parentNodes) { + // try to extend if it is not a cycle in the product graph + if(!parent.containsCM(currentNode.getVertex(), currentNode.getState())) { + processTransition(tree, parent, currentNode.getVertex(), currentNode.getState(), backwardEdge.getTimestamp()); + } + } + } + } + + } +} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/engine/WindowedRAPQ.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/engine/WindowedRAPQ.java new file mode 100644 index 0000000..d8fdf56 --- /dev/null +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/engine/WindowedRAPQ.java @@ -0,0 +1,210 @@ +package ca.uwaterloo.cs.streamingrpq.stree.engine; + +import ca.uwaterloo.cs.streamingrpq.input.InputTuple; +import ca.uwaterloo.cs.streamingrpq.stree.data.*; +import ca.uwaterloo.cs.streamingrpq.stree.data.arbitrary.DeltaRAPQ; +import ca.uwaterloo.cs.streamingrpq.stree.data.arbitrary.SpanningTreeRAPQ; +import ca.uwaterloo.cs.streamingrpq.stree.data.arbitrary.TreeNode; +import ca.uwaterloo.cs.streamingrpq.stree.util.Constants; +import com.codahale.metrics.Histogram; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Timer; +import com.google.common.collect.Lists; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.concurrent.*; +import java.util.stream.Collectors; + +/** + * Created by anilpacaci on 2019-10-02. + */ +public class WindowedRAPQ extends RPQEngine { + + private long windowSize; + private long slideSize; + private long lastExpiry = 0; + + protected DeltaRAPQ deltaRAPQ; + + + private ExecutorService executorService; + + private int numOfThreads; + + + private final Logger LOG = LoggerFactory.getLogger(WindowedRAPQ.class); + + public WindowedRAPQ(QueryAutomata query, int capacity, long windowSize, long slideSize) { + this(query, capacity, windowSize, slideSize, 1); + } + + public WindowedRAPQ(QueryAutomata query, int capacity, long windowSize, long slideSize, int numOfThreads) { + super(query, capacity); + this.deltaRAPQ = new DeltaRAPQ<>(capacity); + this.windowSize = windowSize; + this.slideSize = slideSize; + this.executorService = Executors.newFixedThreadPool(numOfThreads); + this.numOfThreads = numOfThreads; + } + + @Override + public void addMetricRegistry(MetricRegistry metricRegistry) { + this.deltaRAPQ.addMetricRegistry(metricRegistry); + // call super function to include all other histograms + super.addMetricRegistry(metricRegistry); + } + + @Override + public void processEdge(InputTuple inputTuple) { + // total number of trees expanded due this edge insertion + int treeCount = 0; + //for now window processing is done inside edge processing + long currentTimestamp = inputTuple.getTimestamp(); + if(currentTimestamp - slideSize >= lastExpiry && currentTimestamp >= windowSize ) { + // its slide time, maintain the window + Long windowStartTime = System.nanoTime(); + expiry(currentTimestamp - windowSize); + lastExpiry = currentTimestamp; + Long windowElapsedTime = System.nanoTime() - windowStartTime; + windowManagementHistogram.update(windowElapsedTime); + + //reset the edge counter + edgeCountHistogram.update(edgeCount); + edgeCount = 0; + } + + + // restart time for edge processing + Long edgeStartTime = System.nanoTime(); + Timer.Context timer = fullTimer.time(); + // retrieve all transition that can be performed with this label + Map transitions = automata.getTransition(inputTuple.getLabel()); + + if(transitions.isEmpty()) { + // there is no transition with given label, simply return + return; + } else { + // add edge to the snapshot productGraph + if(inputTuple.isDeletion()) { + productGraph.removeEdge(inputTuple.getSource(), inputTuple.getTarget(), inputTuple.getLabel(), inputTuple.getTimestamp()); + edgeCount--; + } else { + productGraph.addEdge(inputTuple.getSource(), inputTuple.getTarget(), inputTuple.getLabel(), inputTuple.getTimestamp()); + edgeCount++; + } + } + + // edge is an insertion + //create a spanning tree for the source node in case it does not exists + if (transitions.keySet().contains(0) && !deltaRAPQ.exists(inputTuple.getSource())) { + // if there exists a start transition with given label, there should be a spanning tree rooted at source vertex + deltaRAPQ.addTree(inputTuple.getSource(), inputTuple.getTimestamp()); + } + + List> futureList = Lists.newArrayList(); + CompletionService completionService = new ExecutorCompletionService<>(this.executorService); + + List> transitionList = transitions.entrySet().stream().collect(Collectors.toList()); + TreeNodeRAPQTreeExpansionJob treeExpansionJob = new TreeNodeRAPQTreeExpansionJob<>(productGraph, automata, results, inputTuple.isDeletion()); + + // for each transition that given label satisy + for (Map.Entry transition : transitionList) { + int sourceState = transition.getKey(); + int targetState = transition.getValue(); + + Collection containingTrees = deltaRAPQ.getTrees(inputTuple.getSource(), sourceState); + treeCount += containingTrees.size(); + + boolean runParallel = containingTrees.size() > Constants.EXPECTED_BATCH_SIZE * this.numOfThreads; + // iterate over spanning trees that include the source node + for (SpanningTreeRAPQ spanningTree : containingTrees) { + // source is guarenteed to exists due to above loop, + // we do not check target here as even if it exist, we might update its timetsap + TreeNode parentNode = spanningTree.getNode(inputTuple.getSource(), sourceState); + //processTransition(spanningTree, parentNode, inputTuple.getTarget(), targetState, inputTuple.getTimestamp()); + treeExpansionJob.addJob(spanningTree, parentNode, inputTuple.getTarget(), targetState, inputTuple.getTimestamp()); + // check whether the job is full and ready to submit + if (treeExpansionJob.isFull()) { + if (runParallel) { + futureList.add(completionService.submit(treeExpansionJob)); + treeExpansionJob = new TreeNodeRAPQTreeExpansionJob<>(productGraph, automata, results, inputTuple.isDeletion()); + } else { + try { + Integer partialResultCount = treeExpansionJob.call(); + treeExpansionJob = new TreeNodeRAPQTreeExpansionJob<>(productGraph, automata, results, inputTuple.isDeletion()); + resultCounter.inc(partialResultCount); + } catch (Exception e) { + LOG.error("SpanningTreeExpansion exception on main thread", e); + } + } + } + } + + // wait for results of al jobs before moving to next transition to ensure that there is a single thread working on a tree + for (int i = 0; i < futureList.size(); i++) { + try { + Integer partialResultCount = completionService.take().get(); + resultCounter.inc(partialResultCount); + } catch (InterruptedException | ExecutionException e) { + LOG.error("SpanningTreeExpansion interrupted during execution", e); + } + } + + // if there is any remaining job in the buffer, run them in main thread + if (!treeExpansionJob.isEmpty()) { + try { + Integer partialResultCount = treeExpansionJob.call(); + resultCounter.inc(partialResultCount); + } catch (Exception e) { + LOG.error("SpanningTreeExpansion exception on main thread", e); + } + } + + } + + + // metric recording + Long edgeElapsedTime = System.nanoTime() - edgeStartTime; + //populate histograms + fullHistogram.update(edgeElapsedTime); + timer.stop(); + // if the incoming edge is not discarded + if(!transitions.isEmpty()) { + // it implies that edge is processed + processedHistogram.update(edgeElapsedTime); + containingTreeHistogram.update(treeCount); + } + + if(inputTuple.isDeletion()) { + // log explicit deletion time separately + explicitDeletionHistogram.update(edgeElapsedTime); + } + + } + + private void processEdgeRAPQ() { + + } + + @Override + public void shutDown() { + // shutdown executors + this.executorService.shutdown(); + } + + /** + * updates DeltaRAPQ and Spanning Trees and removes any node that is lower than the window endpoint + * might need to traverse the entire spanning tree to make sure that there does not exists an alternative path + */ + private void expiry(long minTimestamp) { + LOG.info("Expiry procedure at timestamp: {}", minTimestamp); + // first remove the expired edges from the productGraph + productGraph.removeOldEdges(minTimestamp); + // then maintain the spanning trees, not that spanning trees are maintained without knowing which edge is deleted + deltaRAPQ.expiry(minTimestamp, productGraph, this.executorService); + //delta.batchExpiry(minTimestamp, productGraph, this.executorService); + } + +} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/engine/WindowedRSPQ.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/engine/WindowedRSPQ.java new file mode 100644 index 0000000..1348d06 --- /dev/null +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/engine/WindowedRSPQ.java @@ -0,0 +1,208 @@ +package ca.uwaterloo.cs.streamingrpq.stree.engine; + +import ca.uwaterloo.cs.streamingrpq.input.InputTuple; +import ca.uwaterloo.cs.streamingrpq.stree.data.QueryAutomata; +import ca.uwaterloo.cs.streamingrpq.stree.data.arbitrary.DeltaRAPQ; +import ca.uwaterloo.cs.streamingrpq.stree.data.arbitrary.SpanningTreeRAPQ; +import ca.uwaterloo.cs.streamingrpq.stree.data.arbitrary.TreeNode; +import ca.uwaterloo.cs.streamingrpq.stree.data.simple.DeltaRSPQ; +import ca.uwaterloo.cs.streamingrpq.stree.data.simple.SpanningTreeRSPQ; +import ca.uwaterloo.cs.streamingrpq.stree.data.simple.TreeNodeRSPQ; +import ca.uwaterloo.cs.streamingrpq.stree.util.Constants; +import com.codahale.metrics.Histogram; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Timer; +import com.google.common.collect.Lists; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.*; +import java.util.stream.Collectors; + +/** + * Created by anilpacaci on 2019-10-02. + */ +public class WindowedRSPQ extends RPQEngine { + + private long windowSize; + private long slideSize; + private long lastExpiry = 0; + + protected DeltaRSPQ delta; + + private ExecutorService executorService; + + private int numOfThreads; + + + private final Logger LOG = LoggerFactory.getLogger(WindowedRSPQ.class); + + public WindowedRSPQ(QueryAutomata query, int capacity, long windowSize, long slideSize) { + this(query, capacity, windowSize, slideSize, 1); + } + + public WindowedRSPQ(QueryAutomata query, int capacity, long windowSize, long slideSize, int numOfThreads) { + super(query, capacity); + this.delta = new DeltaRSPQ<>(capacity); + this.windowSize = windowSize; + this.slideSize = slideSize; + this.executorService = Executors.newFixedThreadPool(numOfThreads); + this.numOfThreads = numOfThreads; + // make sure to call query optimizer to compute containment relationship + query.computeContainmentRelationship(); + } + + @Override + public void addMetricRegistry(MetricRegistry metricRegistry) { + + // call metric registry on delta + this.delta.addMetricRegistry(metricRegistry); + // call super function to include all other histograms + super.addMetricRegistry(metricRegistry); + } + + @Override + public void processEdge(InputTuple inputTuple) { + // total number of trees expanded due this edge insertion + int treeCount = 0; + //for now window processing is done inside edge processing + long currentTimestamp = inputTuple.getTimestamp(); + if(currentTimestamp - slideSize >= lastExpiry && currentTimestamp >= windowSize ) { + // its slide time, maintain the window + Long windowStartTime = System.nanoTime(); + expiry(currentTimestamp - windowSize); + lastExpiry = currentTimestamp; + Long windowElapsedTime = System.nanoTime() - windowStartTime; + windowManagementHistogram.update(windowElapsedTime); + + //reset the edge counter + edgeCountHistogram.update(edgeCount); + edgeCount = 0; + } + + + // restart time for edge processing + Long edgeStartTime = System.nanoTime(); + Timer.Context timer = fullTimer.time(); + // retrieve all transition that can be performed with this label + Map transitions = automata.getTransition(inputTuple.getLabel()); + + if(transitions.isEmpty()) { + // there is no transition with given label, simply return + return; + } else { + // add edge to the snapshot productGraph + productGraph.addEdge(inputTuple.getSource(), inputTuple.getTarget(), inputTuple.getLabel(), inputTuple.getTimestamp()); + edgeCount++; + } + + //create a spanning tree for the source node in case it does not exists + if(transitions.keySet().contains(0) && !delta.exists(inputTuple.getSource())) { + // if there exists a start transition with given label, there should be a spanning tree rooted at source vertex + delta.addTree(inputTuple.getSource(), inputTuple.getTimestamp()); + } + + List> futureList = Lists.newArrayList(); + CompletionService completionService = new ExecutorCompletionService<>(this.executorService); + + List> transitionList = transitions.entrySet().stream().collect(Collectors.toList()); + TreeNodeRSPQTreeExpansionJob treeExpansionJob = new TreeNodeRSPQTreeExpansionJob<>(productGraph, automata, results); + + // for each transition that given label satisy + for (Map.Entry transition : transitionList) { + int sourceState = transition.getKey(); + int targetState = transition.getValue(); + + Collection containingTrees = delta.getTrees(inputTuple.getSource(), sourceState); + treeCount += containingTrees.size(); + + boolean runParallel = containingTrees.size() > Constants.EXPECTED_BATCH_SIZE * this.numOfThreads; + // iterate over spanning trees that include the source node + for (SpanningTreeRSPQ spanningTree : containingTrees) { + // source is guarenteed to exists due to above loop, + // we do not check target here as even if it exist, we might update its timetsap + Collection> parentNodes = spanningTree.getNodes(inputTuple.getSource(), sourceState); + for(TreeNodeRSPQ parentNode : parentNodes) { + //processTransition(spanningTree, parentNode, inputTuple.getTarget(), targetState, inputTuple.getTimestamp()); + treeExpansionJob.addJob(spanningTree, parentNode, inputTuple.getTarget(), targetState, inputTuple.getTimestamp()); + // check whether the job is full and ready to submit + if (treeExpansionJob.isFull()) { + if (runParallel) { + futureList.add(completionService.submit(treeExpansionJob)); + treeExpansionJob = new TreeNodeRSPQTreeExpansionJob<>(productGraph, automata, results); + } else { + try { + Integer partialResultCount = treeExpansionJob.call(); + treeExpansionJob = new TreeNodeRSPQTreeExpansionJob<>(productGraph, automata, results); + resultCounter.inc(partialResultCount); + } catch (Exception e) { + LOG.error("SpanningTreeExpansion exception on main thread", e); + } + } + } + } + } + } + + + for (int i = 0; i < futureList.size(); i++) { + try { + Integer partialResultCount = completionService.take().get(); + resultCounter.inc(partialResultCount); + } catch (InterruptedException | ExecutionException e) { + LOG.error("SpanningTreeExpansion interrupted during execution", e); + } + } + + + // if there is any remaining job in the buffer, run them in main thread + if (!treeExpansionJob.isEmpty()) { + try { + Integer partialResultCount = treeExpansionJob.call(); + resultCounter.inc(partialResultCount); + } catch (Exception e) { + LOG.error("SpanningTreeExpansion exception on main thread", e); + } + } + + // metric recording + Long edgeElapsedTime = System.nanoTime() - edgeStartTime; + //populate histograms + fullHistogram.update(edgeElapsedTime); + timer.stop(); + // if the incoming edge is not discarded + if(!transitions.isEmpty()) { + // it implies that edge is processed + processedHistogram.update(edgeElapsedTime); + containingTreeHistogram.update(treeCount); + } + + } + + private void processEdgeRAPQ() { + + } + + @Override + public void shutDown() { + // shutdown executors + this.executorService.shutdown(); + } + + /** + * updates DeltaRAPQ and Spanning Trees and removes any node that is lower than the window endpoint + * might need to traverse the entire spanning tree to make sure that there does not exists an alternative path + */ + private void expiry(long minTimestamp) { + LOG.info("Expiry procedure at timestamp: {}", minTimestamp); + // first remove the expired edges from the productGraph + productGraph.removeOldEdges(minTimestamp); + // then maintain the spanning trees, not that spanning trees are maintained without knowing which edge is deleted + delta.expiry(minTimestamp, productGraph, this.executorService); + //delta.batchExpiry(minTimestamp, productGraph, this.executorService); + } + +} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/util/Constants.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/util/Constants.java index 58b909e..d9eb7c9 100644 --- a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/util/Constants.java +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/util/Constants.java @@ -4,12 +4,21 @@ public class Constants { // average number of neighbours per node public static final int EXPECTED_NEIGHBOURS = 12; - // expected number of nodes in the graph + // expected number of nodes in the productGraph public static final int EXPECTED_NODES = 50000000; // expected number of tree a single edge is in - public static final int EXPECTED_TREES = 1000; + public static final int EXPECTED_TREES = 65536; - // expected number of labels in a graph + // expected number of labels in a productGraph public static final int EXPECTED_LABELS = 8; + + /** + * Total number of SpanningTreeExpansion jobs to be assigned to a thread + */ + public static final int EXPECTED_BATCH_SIZE = 64; + + public static final int EXPECTED_TREE_SIZE = 8192; + + public static final int HISTOGRAM_BUCKET_SIZE = 526336; } diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/util/Hasher.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/util/Hasher.java index e2d148b..4a3f9ca 100644 --- a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/util/Hasher.java +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/util/Hasher.java @@ -1,11 +1,68 @@ package ca.uwaterloo.cs.streamingrpq.stree.util; +import java.util.Map; + public class Hasher { + private static ThreadLocal threadLocalKey = new ThreadLocal<>(); + public static int TreeNodeHasher(int vertex, int state) { int h = 17; h = 31 * h + vertex; h = 31 * h + state; return h; } + + public static int TreeNodeHasher(V vertex, int state) { + int h = 17; + h = 31 * h + vertex.hashCode(); + h = 31 * h + state; + return h; + } + + public static MapKey createTreeNodePairKey(V vertex, int state) { + MapKey mapKey = new MapKey(vertex, state); + return mapKey; + } + + public static MapKey getThreadLocalTreeNodePairKey(V vertex, int state) { + + MapKey mapKey = threadLocalKey.get(); + if(mapKey == null) { + mapKey = new MapKey(vertex, state); + threadLocalKey.set(mapKey); + } else { + mapKey.X = vertex; + mapKey.Y = state; + } + return mapKey; + } + + public static class MapKey { + + public V X; + public int Y; + + public MapKey(V X, int Y) { + this.X = X; + this.Y = Y; + } + + @Override + public boolean equals (final Object O) { + if (!(O instanceof MapKey)) return false; + if (!((MapKey) O).X.equals(X)) return false; + if (((MapKey) O).Y != Y) return false; + return true; + } + + @Override + public int hashCode() { + int h = 17; + h = 31 * h + X.hashCode(); + h = 31 * h + Y; + return h; + } + + } } diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/util/Semantics.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/util/Semantics.java new file mode 100644 index 0000000..8177f45 --- /dev/null +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/stree/util/Semantics.java @@ -0,0 +1,36 @@ +package ca.uwaterloo.cs.streamingrpq.stree.util; + +import java.util.HashMap; +import java.util.Map; + +/** + * Created by anilpacaci on 2019-10-11. + */ +public enum Semantics { + + SIMPLE ("simple"), + ARBITRARY ("arbitrary"); + + private final String semantics; + + private static final Map BY_LABEL = new HashMap<>(); + + static { + for(Semantics p : values()) { + BY_LABEL.put(p.semantics, p); + } + } + + Semantics(String semantics) { + this.semantics = semantics; + } + + @Override + public String toString() { + return semantics; + } + + public static Semantics fromValue(String semantics) { + return BY_LABEL.get(semantics.toLowerCase()); + } +} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/transitiontable/dfa/DFA.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/transitiontable/dfa/DFA.java index a0d72e8..aa45e8b 100644 --- a/src/main/java/ca/uwaterloo/cs/streamingrpq/transitiontable/dfa/DFA.java +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/transitiontable/dfa/DFA.java @@ -47,7 +47,7 @@ public DFA(int capacity, PathSemantics semantics) { PATH_SEMANTICS = semantics; - // data structures for the automaton and the graph + // data structures for the automaton and the productGraph dfaTransitions = HashMultimap.create(); dfaNodes = new HashMap<>(); results = HashMultimap.create(); diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/transitiontable/util/cycle/Cycle.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/transitiontable/util/cycle/Cycle.java deleted file mode 100644 index ad1b016..0000000 --- a/src/main/java/ca/uwaterloo/cs/streamingrpq/transitiontable/util/cycle/Cycle.java +++ /dev/null @@ -1,4 +0,0 @@ -package ca.uwaterloo.cs.streamingrpq.transitiontable.util.cycle; - -public class Cycle { -} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/transitiontable/util/cycle/Edge.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/transitiontable/util/cycle/Edge.java deleted file mode 100644 index 6ed54a6..0000000 --- a/src/main/java/ca/uwaterloo/cs/streamingrpq/transitiontable/util/cycle/Edge.java +++ /dev/null @@ -1,37 +0,0 @@ -package ca.uwaterloo.cs.streamingrpq.transitiontable.util.cycle; - -public class Edge { - - N source, target; - L label; - - public Edge(N source, N target, L label) { - this.source = source; - this.target = target; - this.label = label; - } - - public N getSource() { - return source; - } - - public void setSource(N source) { - this.source = source; - } - - public N getTarget() { - return target; - } - - public void setTarget(N target) { - this.target = target; - } - - public L getLabel() { - return label; - } - - public void setLabel(L label) { - this.label = label; - } -} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/transitiontable/util/cycle/FilteredSimpleTextStream.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/transitiontable/util/cycle/FilteredSimpleTextStream.java deleted file mode 100644 index c85394a..0000000 --- a/src/main/java/ca/uwaterloo/cs/streamingrpq/transitiontable/util/cycle/FilteredSimpleTextStream.java +++ /dev/null @@ -1,98 +0,0 @@ -package ca.uwaterloo.cs.streamingrpq.transitiontable.util.cycle; - -import ca.uwaterloo.cs.streamingrpq.input.InputTuple; -import ca.uwaterloo.cs.streamingrpq.input.TextStream; -import com.google.common.base.Splitter; -import com.google.common.collect.Iterables; - -import java.io.BufferedReader; -import java.io.FileNotFoundException; -import java.io.FileReader; -import java.io.IOException; -import java.util.HashSet; - -public class FilteredSimpleTextStream implements TextStream{ - - - FileReader fileStream; - BufferedReader bufferedReader; - - String filename; - - Integer localCounter = 0; - Integer globalCounter = 0; - - HashSet predicates; - - - public FilteredSimpleTextStream(String[] predicates) { - this.predicates = new HashSet<>(); - - for(String p : predicates) { - this.predicates.add(p); - } - } - - public boolean isOpen() { - return false; - } - - public void open(String filename, int maxSize) { - open(filename); - } - - public void open(String filename) { - this.filename = filename; - try { - fileStream = new FileReader(filename); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } - bufferedReader = new BufferedReader(fileStream, 1024*1024); - - } - - public void close() { - try { - bufferedReader.close(); - fileStream.close(); - } catch (IOException e) { - e.printStackTrace(); - } - - } - - public InputTuple next() { - String line = null; - InputTuple tuple = null; - try { - while((line = bufferedReader.readLine()) != null) { - String[] splitResults = Iterables.toArray(Splitter.on('\t').split(line), String.class); - if(splitResults.length == 3) { - if (this.predicates.contains(splitResults[1])) { -// tuple = new InputTuple(1,2,3); - tuple = new InputTuple(splitResults[0].hashCode(), splitResults[2].hashCode(), splitResults[1]); - break; - } - } - } - } catch (IOException e) { - return null; - } - if (line == null) { - return null; - } - localCounter++; - globalCounter++; - return tuple; - } - - public void reset() { - close(); - - open(this.filename); - - localCounter = 0; - globalCounter = 0; - } -} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/transitiontable/util/cycle/Graph.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/transitiontable/util/cycle/Graph.java deleted file mode 100644 index 9376128..0000000 --- a/src/main/java/ca/uwaterloo/cs/streamingrpq/transitiontable/util/cycle/Graph.java +++ /dev/null @@ -1,37 +0,0 @@ -package ca.uwaterloo.cs.streamingrpq.transitiontable.util.cycle; - -import com.google.common.collect.HashMultimap; - -import java.util.Collection; -import java.util.Set; - -public class Graph { - - private HashMultimap> edges; - - public Graph() { - edges = HashMultimap.create(); - } - - public void addEdge(N source, N target, L label) { - Edge edge = new Edge(source, target, label); - edges.put(source, edge); - } - - public Collection getVertices() { - return edges.keySet(); - } - - public Collection> getChildren(N vertex) { - return edges.get(vertex); - } - - public L getLabel(N source, N target) { - Set> neighbours = edges.get(source); - return neighbours.stream().filter(e -> e.getTarget().equals(target)).findFirst().get().getLabel(); - } - - public int getVertexCount() { - return edges.size(); - } -} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/transitiontable/util/cycle/SimpleCycleDFS.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/transitiontable/util/cycle/SimpleCycleDFS.java deleted file mode 100644 index f51a039..0000000 --- a/src/main/java/ca/uwaterloo/cs/streamingrpq/transitiontable/util/cycle/SimpleCycleDFS.java +++ /dev/null @@ -1,113 +0,0 @@ -package ca.uwaterloo.cs.streamingrpq.transitiontable.util.cycle; - -import ca.uwaterloo.cs.streamingrpq.input.InputTuple; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.*; - -public class SimpleCycleDFS { - - private static Logger logger = LoggerFactory.getLogger(SimpleCycleDFS.class); - - private Graph graph; - private List> cycles; - - - public SimpleCycleDFS(Graph graph) { - this.graph = graph; - this.cycles = new ArrayList<>(); - } - - - public void findCycles() { - HashSet visited = new HashSet<>(); - List path = new ArrayList<>(); - - Collection vertices = graph.getVertices(); - - for(N startVertex : vertices) { - visit(startVertex, visited, path); - } - } - - private void visit(N vertex, HashSet visited, List path) { - if(path.contains(vertex)) { - // cycle is detected, report it - addCycle(vertex, path); - } - - if(visited.contains(vertex)) { - // vertex has been visited before, so simply continue with the next element - return; - } - - visited.add(vertex); - path.add(vertex); - - Collection> children = graph.getChildren(vertex); - for(Edge child : children) { - visit(child.getTarget(), visited, path); - } - - path.remove(vertex); - - } - - private void addCycle(N vertex, List path) { - int firstIndex = path.indexOf(vertex); - List cycleLabels = new ArrayList<>(); - if(firstIndex < 0) { - logger.error("Element {} is not in the path, no cycle formed", vertex); - return; - } - - ListIterator iterator = path.listIterator(firstIndex); - N source = iterator.next(); - N firstVertex = source; - while(iterator.hasNext()) { - N target = iterator.next(); - L label = graph.getLabel(source, target); - cycleLabels.add(label); - source = target; - } - - L lastLabel = graph.getLabel(source, firstVertex); - cycleLabels.add(lastLabel); - - cycles.add(cycleLabels); - } - - public List> getCycles() { - return cycles; - } - - public static void main(String[] argv) { - String inputFilePath = argv[0]; - String index = argv[1]; - FilteredSimpleTextStream stream = new FilteredSimpleTextStream(Arrays.copyOfRange(argv, 2, argv.length)); - stream.open(inputFilePath); - - Graph graph = new Graph<>(); - SimpleCycleDFS cycleDFS = new SimpleCycleDFS<>(graph); - - InputTuple input = stream.next(); - - - while(input != null) { - graph.addEdge(input.getSource(), input.getTarget(), input.getLabel()); - input = stream.next(); - } - logger.info("Dataset loaded for predicate set {}", index); - logger.info("Number of vertices is {}", graph.getVertexCount()); - cycleDFS.findCycles(); - - logger.info("Cycles for index {}:", index); - List> result = cycleDFS.getCycles(); - - for(List cycle : result) { - logger.info(cycle.toString()); - } - logger.info("All cycles for {} completed !!!", index); - } -} diff --git a/src/main/java/ca/uwaterloo/cs/streamingrpq/transitiontable/waveguide/SingleThreadedRun.java b/src/main/java/ca/uwaterloo/cs/streamingrpq/transitiontable/waveguide/SingleThreadedRun.java index 6a2c759..bf88815 100644 --- a/src/main/java/ca/uwaterloo/cs/streamingrpq/transitiontable/waveguide/SingleThreadedRun.java +++ b/src/main/java/ca/uwaterloo/cs/streamingrpq/transitiontable/waveguide/SingleThreadedRun.java @@ -2,7 +2,7 @@ import ca.uwaterloo.cs.streamingrpq.input.InputTuple; import ca.uwaterloo.cs.streamingrpq.input.TextStream; -import ca.uwaterloo.cs.streamingrpq.stree.engine.IncrementalRAPQ; +import ca.uwaterloo.cs.streamingrpq.stree.engine.RPQEngine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -11,16 +11,16 @@ /** * Created by anilpacaci on 2019-06-07. */ -public class SingleThreadedRun implements Callable { +public class SingleThreadedRun implements Callable { private static Logger logger = LoggerFactory.getLogger(SingleThreadedRun.class); private String queryName; private TextStream stream; - private IncrementalRAPQ query; + private RPQEngine query; - public SingleThreadedRun(String queryName, TextStream stream, IncrementalRAPQ query) { + public SingleThreadedRun(String queryName, TextStream stream, RPQEngine query) { this.queryName = queryName; this.stream = stream; this.query = query; @@ -29,7 +29,7 @@ public SingleThreadedRun(String queryName, TextStream stream, IncrementalRAPQ input = stream.next(); + InputTuple input = stream.next(); logger.info("Query " + queryName + " is starting!"); while (input != null) { @@ -44,7 +44,7 @@ public Object call() throws Exception { } } - logger.info("total number of results for query " + queryName + " : " + query.getResults().size()); + logger.info("total number of results for query " + queryName + " : " + query.getResultCount()); return null; } diff --git a/src/main/resources/diamondgraph.txt b/src/main/resources/diamondgraph.txt index 36b82b7..2b64023 100644 --- a/src/main/resources/diamondgraph.txt +++ b/src/main/resources/diamondgraph.txt @@ -1,11 +1,13 @@ -4 b 1 -3 a 4 -2 b 3 -1 a 2 -1 a 5 -5 b 3 -5 b 6 -6 a 7 -7 b 3 -5 a 8 -8 b 6 \ No newline at end of file +1 a 2 + +4 b 1 + +3 a 4 + +1 a 5 + +1 a 7 + +5 b 6 + +2 b 3 + +6 a 7 + +5 a 8 + +5 b 3 + +5 a 8 - +8 b 6 + +7 b 3 + \ No newline at end of file diff --git a/src/main/resources/examplegraph.txt b/src/main/resources/examplegraph.txt new file mode 100644 index 0000000..2f2df2a --- /dev/null +++ b/src/main/resources/examplegraph.txt @@ -0,0 +1,5 @@ +3 m 4 +4 m 1 +2 m 4 +5 m 2 +1 m 2 \ No newline at end of file diff --git a/src/misc/virtuoso-queries.txt b/src/misc/virtuoso-queries.txt index f3b75b6..5a2c50d 100644 --- a/src/misc/virtuoso-queries.txt +++ b/src/misc/virtuoso-queries.txt @@ -18,7 +18,7 @@ rdf_loader_run (); & sparql select count(*) from where { ?s ?p ?o }; -select * {graph ?g {}} +select * {productGraph ?g {}} sparql SELECT DISTINCT ?g WHERE { GRAPH ?g {?s ?p ?o} } ORDER BY ?g diff --git a/src/test/java/ca/uwaterloo/cs/streamingrpq/SpanningTreeTest.java b/src/test/java/ca/uwaterloo/cs/streamingrpq/WindowedSpanningTreeTest.java similarity index 72% rename from src/test/java/ca/uwaterloo/cs/streamingrpq/SpanningTreeTest.java rename to src/test/java/ca/uwaterloo/cs/streamingrpq/WindowedSpanningTreeTest.java index 5f6a9e2..1000717 100644 --- a/src/test/java/ca/uwaterloo/cs/streamingrpq/SpanningTreeTest.java +++ b/src/test/java/ca/uwaterloo/cs/streamingrpq/WindowedSpanningTreeTest.java @@ -3,15 +3,16 @@ import ca.uwaterloo.cs.streamingrpq.input.InputTuple; import ca.uwaterloo.cs.streamingrpq.input.SimpleTextStream; import ca.uwaterloo.cs.streamingrpq.input.TextStream; -import ca.uwaterloo.cs.streamingrpq.input.Yago2sTSVStream; import ca.uwaterloo.cs.streamingrpq.stree.data.QueryAutomata; -import ca.uwaterloo.cs.streamingrpq.stree.engine.IncrementalRAPQ; +import ca.uwaterloo.cs.streamingrpq.stree.engine.RPQEngine; +import ca.uwaterloo.cs.streamingrpq.stree.engine.WindowedRAPQ; +import ca.uwaterloo.cs.streamingrpq.stree.engine.WindowedRSPQ; import com.codahale.metrics.ConsoleReporter; import com.codahale.metrics.MetricRegistry; import java.util.concurrent.TimeUnit; -public class SpanningTreeTest { +public class WindowedSpanningTreeTest { static String filename = "src/main/resources/diamondgraph.txt"; @@ -22,7 +23,7 @@ public static void main(String[] args) { query.addTransition(1, "b", 2); query.addTransition(2, "a", 1); - IncrementalRAPQ rapqEngine = new IncrementalRAPQ(query, 100); + RPQEngine rapqEngine = new WindowedRAPQ<>(query, 100, 5, 1, 10); MetricRegistry metricRegistry = new MetricRegistry(); rapqEngine.addMetricRegistry(metricRegistry); @@ -42,7 +43,13 @@ public static void main(String[] args) { input = stream.next(); } - rapqEngine.getResults().entries().iterator().forEachRemaining(t-> {System.out.println(t.getKey() + " --> " + t.getValue());}); + rapqEngine.getResults().iterator().forEachRemaining(t-> {System.out.println(t.getSource() + " --> " + t.getTarget() + " " + !t.isDeletion());}); + + rapqEngine.shutDown(); + + stream.close(); + + reporter.stop(); } } diff --git a/src/test/java/ca/uwaterloo/cs/streamingrpq/WindowedSpanningTreeTestRSPQ.java b/src/test/java/ca/uwaterloo/cs/streamingrpq/WindowedSpanningTreeTestRSPQ.java new file mode 100644 index 0000000..6a4e65e --- /dev/null +++ b/src/test/java/ca/uwaterloo/cs/streamingrpq/WindowedSpanningTreeTestRSPQ.java @@ -0,0 +1,58 @@ +package ca.uwaterloo.cs.streamingrpq; + +import ca.uwaterloo.cs.streamingrpq.input.InputTuple; +import ca.uwaterloo.cs.streamingrpq.input.SimpleTextStream; +import ca.uwaterloo.cs.streamingrpq.input.TextStream; +import ca.uwaterloo.cs.streamingrpq.stree.data.QueryAutomata; +import ca.uwaterloo.cs.streamingrpq.stree.engine.RPQEngine; +import ca.uwaterloo.cs.streamingrpq.stree.engine.WindowedRSPQ; +import com.codahale.metrics.ConsoleReporter; +import com.codahale.metrics.MetricRegistry; + +import java.util.concurrent.TimeUnit; + +public class WindowedSpanningTreeTestRSPQ { + + static String filename = "src/main/resources/examplegraph.txt"; + + public static void main(String[] args) { + QueryAutomata query = new QueryAutomata(3); + query.addFinalState(0); + query.addFinalState(1); + query.addFinalState(2); + query.addTransition(0, "m", 0); + query.addTransition(0, "f", 1); + query.addTransition(1, "f", 1); + query.addTransition(1, "m", 2); + query.addTransition(2, "m", 2); + + RPQEngine rapqEngine = new WindowedRSPQ<>(query, 100, 10, 1, 10); + MetricRegistry metricRegistry = new MetricRegistry(); + rapqEngine.addMetricRegistry(metricRegistry); + + final ConsoleReporter reporter = ConsoleReporter.forRegistry(metricRegistry) + .convertRatesTo(TimeUnit.SECONDS) + .convertDurationsTo(TimeUnit.MILLISECONDS) + .build(); + reporter.start(1, TimeUnit.MINUTES); + + + TextStream stream = new SimpleTextStream(); + stream.open(filename); + InputTuple input = stream.next(); + + while (input != null) { + rapqEngine.processEdge(input); + input = stream.next(); + } + + rapqEngine.getResults().iterator().forEachRemaining(t-> {System.out.println(t.getSource() + " --> " + t.getTarget());}); + + rapqEngine.shutDown(); + + stream.close(); + + reporter.stop(); + + } +} diff --git a/watdiv-run.sh b/watdiv-run.sh new file mode 100755 index 0000000..465883b --- /dev/null +++ b/watdiv-run.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +scripts/streaming-rpq.py conf/watdiv-sf250.json |& tee watdiv-250.run ; +scripts/streaming-rpq.py conf/watdiv-sf200.json |& tee watdiv-200.run ; +scripts/streaming-rpq.py conf/watdiv-sf150.json |& tee watdiv-150.run ; +scripts/streaming-rpq.py conf/watdiv-sf100.json |& tee watdiv-100.run ; +scripts/streaming-rpq.py conf/watdiv-sf50.json |& tee watdiv-50.run ; +