add notebook that demonstrates learning of cost functions

sekruse · Sep 7, 2016 · b9511a0 · b9511a0
1 parent 22e7f51
commit b9511a0
Show file tree

Hide file tree

Showing 10 changed files with 1,619 additions and 73 deletions.
diff --git a/notebooks/cost-functions/cardinalities.json b/notebooks/cost-functions/cardinalities.json
diff --git a/notebooks/cost-functions/cost-functions.properties b/notebooks/cost-functions/cost-functions.properties
@@ -0,0 +1,55 @@
+# Here go the learned cost functions...
+rheem.java.reduceby.load = {\
+ "in":1,\
+ "out":1,\
+ "cpu":"${0.9691075617570686*in0 + 179.58079684227764*out0 + 23.652186277726912}",\
+ "ram":"0",\
+ "disk":"0",\
+ "net":"0",\
+ "p":0.9\
+}
+rheem.java.localcallbacksink.load = {\
+ "in":1,\
+ "out":0,\
+ "cpu":"${10.662676041238914*in0 + 698273.3774416123}",\
+ "ram":"0",\
+ "disk":"0",\
+ "net":"0",\
+ "p":0.9\
+}
+rheem.java.textfilesource.load = {\
+ "in":0,\
+ "out":1,\
+ "cpu":"${6.706230156985611*out0 + 116946.86108530796}",\
+ "ram":"0",\
+ "disk":"0",\
+ "net":"0",\
+ "p":0.9\
+}
+rheem.java.filter.load = {\
+ "in":1,\
+ "out":1,\
+ "cpu":"${1.8101285013854862*in0 + 0.10522799511388539*out0 + 174508.14960646234}",\
+ "ram":"0",\
+ "disk":"0",\
+ "net":"0",\
+ "p":0.9\
+}
+rheem.java.map.load = {\
+ "in":1,\
+ "out":1,\
+ "cpu":"${3.786263829086864*in0 + 712651.7791312812}",\
+ "ram":"0",\
+ "disk":"0",\
+ "net":"0",\
+ "p":0.9\
+}
+rheem.java.flatmap.load = {\
+ "in":1,\
+ "out":1,\
+ "cpu":"${7.6590772571940064*in0 + 3.1164436836959934*out0 + 464218.1441863766}",\
+ "ram":"0",\
+ "disk":"0",\
+ "net":"0",\
+ "p":0.9\
+}
diff --git a/notebooks/cost-functions/executions.json b/notebooks/cost-functions/executions.json
diff --git a/notebooks/cost-functions/profiledb.json b/notebooks/cost-functions/profiledb.json
diff --git a/notebooks/cost-functions/rheem.properties b/notebooks/cost-functions/rheem.properties
@@ -0,0 +1,67 @@
+spark.master = local
+rheem.spark.init.ms = 10000
+rheem.core.optimizer.instrumentation = org.qcri.rheem.core.profiling.FullInstrumentationStrategy
+rheem.core.log.executions = /Users/basti/Work/Notebooks/boss-2016/cost-functions/executions.json
+rheem.core.log.cardinalities = /Users/basti/Work/Notebooks/boss-2016/cost-functions/cardinalities.json
+
+rheem.profiler.ga.population.size = 20
+rheem.profiler.ga.population.elite = 1
+rheem.profiler.ga.selection.ratio = 1
+rheem.profiler.ga.mutation.ratio = 1
+rheem.profiler.ga.mutation.alteration = 0.1
+rheem.profiler.ga.mutation.reset = 0.05
+
+rheem.java.localcallbacksink.load = {\
+ "in":1,\
+ "out":0,\
+ "cpu":"${10000*in0 + 20}",\
+ "ram":"0",\
+ "disk":"0",\
+ "net":"0",\
+ "p":0.9\
+}
+rheem.java.filter.load = {\
+ "in":1,\
+ "out":1,\
+ "cpu":"${10000*in0 + 10*out0 + 0}",\
+ "ram":"0",\
+ "disk":"0",\
+ "net":"0",\
+ "p":0.9\
+}
+rheem.java.map.load = {\
+ "in":1,\
+ "out":1,\
+ "cpu":"${10000*in0 + 20}",\
+ "ram":"0",\
+ "disk":"0",\
+ "net":"0",\
+ "p":0.9\
+}
+rheem.java.reduceby.load = {\
+ "in":1,\
+ "out":1,\
+ "cpu":"${10000*in0 + 2*out0 + 20}",\
+ "ram":"0",\
+ "disk":"0",\
+ "net":"0",\
+ "p":0.9\
+}
+rheem.java.textfilesource.load = {\
+ "in":0,\
+ "out":1,\
+ "cpu":"${70000*out0 + 10}",\
+ "ram":"0",\
+ "disk":"0",\
+ "net":"0",\
+ "p":0.9\
+}
+rheem.java.flatmap.load = {\
+ "in":1,\
+ "out":1,\
+ "cpu":"${10000*in0 + 5*out0 + 0}",\
+ "ram":"0",\
+ "disk":"0",\
+ "net":"0",\
+ "p":0.9\
+}