Jiff

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Boston University - Software & Application Innovation Lab, Nikolaj Volgushev, Malte Schwarzkopf
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.rst

@@ -11,7 +11,7 @@ This framework allows users to define data analysis workflows in familiar fronte
 Dependencies
 ------------
 
-Conclave requires a Python 3.x environment. On Ubuntu (14.04+), installing the `python3`, `python3-pystache`, and `python3-nose` should get everything that's needed.
+Conclave requires a Python 3.x environment. On Ubuntu (14.04+), installing the `python3`, `python3-pystache` should get everything that's needed.
 
 Testing
 -------

conclave/codegen/jiff.py

@@ -46,6 +46,13 @@ def generate_party_code(self):
             "SERVER_IP_PORT": "{0}:{1}".format(self.jiff_config.server_ip, self.jiff_config.server_port)
         }
 
+        nodes = self.dag.top_sort()
+        for node in nodes:
+            if isinstance(node, Open):
+                if self.pid in node.out_rel.stored_with:
+                    data['OUTPUT_FILE'] = node.out_rel.name
+                    break
+
         self.party_code += pystache.render(template, data)
 
         return self
@@ -185,10 +192,12 @@ def _generate_open(self, open_op: Open):
         template = open(
             "{0}/open.tmpl".format(self.template_directory), 'r').read()
 
-        data = {}
+        data = {
+            "INREL": open_op.get_in_rel().name,
+            "OUTREL": open_op.out_rel.name
+        }
 
-        # return pystache.render(template, data)
-        return ''
+        return pystache.render(template, data)
 
     def _generate_project(self, project_op: Project):
 
@@ -275,10 +284,15 @@ def _generate_sort_by(self, sort_op: SortBy):
 
         return pystache.render(template, data)
 
-    def _generate_bash(self):
-        # TODO: need clear way to run jiff server/party/mpc files outside of jiff repository
+    def _write_bash(self):
+        template = open("{}/bash.tmpl"
+                        .format(self.template_directory), 'r').read()
+
+        data = {
+            "JIFF_PATH": self.jiff_config.jiff_path
+        }
 
-        bash_code = ''
+        bash_code = pystache.render(template, data)
 
         if self.pid == self.jiff_config.server_pid:
             bash_code += 'node server.js\n'
@@ -302,7 +316,7 @@ def _write_code(self, code, job_name):
         protocol_file = open("{}/{}/mpc.js".format(self.config.code_path, job_name), 'w')
         protocol_file.write(code)
 
-        bash_code = self._generate_bash()
+        bash_code = self._write_bash()
         bash_file = open("{}/{}/run.sh".format(self.config.code_path, job_name), 'w')
         bash_file.write(bash_code)
 

conclave/codegen/templates/jiff/bash.tmpl

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-{{{OP_CODE}}}
+export NODE_PATH="{{{JIFF_PATH}}}/node_modules"

conclave/codegen/templates/jiff/create.tmpl

@@ -1,2 +1,2 @@
 
-    var {{{OUTREL}}} = datasets[{{{ID}}}];
+    var {{{OUTREL}}} = shares[{{{ID}}}];

conclave/codegen/templates/jiff/mpc_top_level.tmpl

@@ -14,24 +14,6 @@
     return saved_instance;
   };
 
-  const splitDatasets = function(shares, jiff_instance)
-  {
-    var datasets = [];
-
-    for (var k = 1; k <= jiff_instance.party_count; k++)
-    {
-      datasets.push([]);
-    }
-
-    for (var i = 0; i < shares.length; i++) {
-      for (var j = 0; j < jiff_instance.party_count; j++) {
-        datasets[j].push(shares[i][j+1]);
-      }
-    }
-
-    return datasets;
-
-  };
 
   const project = function(inputRel, projCols)
   {
@@ -50,6 +32,8 @@
         result[j].push(inputRel[j][projCols[k]]);
       }
     }
+    return result;
+
   };
 
   const concatenate = function(inRels)
@@ -120,6 +104,196 @@
 
   }
 
+
+  const sortBy = function(inRel)
+  {
+    var numRows = inRel.length;
+    var numCols = inRel[0].length;
+
+  var sortArray = [];
+  for (var i = 0; i < numRows; i++)
+  {
+    sortArray.push([]);
+    for (var k = 0; j < numCols; j++)
+    {
+      sortArray[i].push(inRel[k]);
+    }
+  }
+	oddEvenSort(array, keyCol, numCols, 0, numRows, true);
+  }
+  const oddEvenSort = function(array, keyCol, numCols, lo, n, ascending)
+  {
+    if (n > 1)
+    {
+      var m = n / 2;
+      oddEvenSort(array, keyCol, numCols, lo, m, ascending);
+      oddEvenSort(array, keyCol, numCols, lo + m, m, ascending);
+      oddEvenMerge(array, keyCol, numCols, lo, n, 1, ascending);
+    }
+  }
+
+  //TODO
+  const oddEvenMerge = function(array, keyCol, numCols, lo, n, ascending)
+  {
+    var m = r * 2;
+    if (m < n)
+    {
+      oddEvenMerge(array, keyCol, numCols, lo, n, m, ascending);
+      oddEvenMerge(array, keyCol, numCols, lo + r, n, m, ascending);
+
+      for (var i = lo + r; (i + r) < (lo + n); i += m)
+      {
+        compareExchange(array, keyCol, numCols, i, i + r, ascending);
+      }
+    } else {
+      compareExchange(array, keyCol, numCols, lo, lo + r, ascending);
+    }
+  }
+
+  //TODO
+  const compareExchange = function(array, keyCol, numCols, i, j, ascending)
+  {
+    var x = array[i][keyCol];
+    var y = array[j][keyCol];
+
+    var c = 1;
+    var d = 1;
+    if (ascending)
+    {
+      c = (x.slt(y)).mult(c);
+      d = (y.slt(x)).mult(d);
+    }
+    else
+    {
+      d = (x.slt(y)).mult(d);
+      c = (y.slt(x)).mult(c);
+    }
+
+    var temp1 = [];
+    var temp2 = [];
+
+    for (var k = 0; k < numCols; k++)
+    {
+      var a = array[i][k];
+      var b = array[j][k];
+
+      temp1.push((c * a) + (d * b));
+      temp2.push((d * a) + (c * b));
+    }
+    array[i] = temp1;
+    array[j] = temp2;
+  }
+
+  //TODO: make this work similar to OblivC
+  const agg = function(inRel)
+  {
+    var numCols = 2;
+    var numRows = inRel.length;
+    var array = [];
+
+    for (var i = 0; i < numRows; i++)
+    {
+      array.push([]);
+      array[i].push(inRel[i][0]);
+      array[i].push(inRel[i][1]);
+    }
+
+    if (!(numRows && (numRows & (numRows - 1)) === 0))
+    {
+      var paddedVal = nextPowerOf2(numRows);
+      var paddedArray = padInput(array, numCols, numRows, paddedVal);
+      return agg_helper(inRel, numCols, paddedVal);
+    } else
+    {
+      return agg_helper(inRel, numCols, numRows);
+    }
+  }
+
+  const oddEvenAgg = function(array, lo, n)
+  {
+    if (n > 1)
+    {
+      var m = n / 2;
+      oddEvenAgg(array, lo, m);
+      oddEvenAgg(array, lo + m, m);
+      for (var i = 0; i < m; i++)
+      {
+        aggIfEq(array, lo + i);
+      }
+    }
+    else
+    {
+      aggIfEq(array, 0);
+    }
+  }
+
+  const aggIfEq = function(array, index)
+  {
+    var c = array[index][0].eq(array[index+1][0]);
+    array[index][0] = array[index][1].add(c.mult(array[index+1][1]));
+  }
+
+  const agg_helper = function(inRel, numCols, numRows)
+  {
+
+		oddEvenSort(array, 0, numCols, 0, numRows, true);
+		oddEvenAgg(array, 0, numRows);
+
+		oddEvenSort(array, 1, numCols, 0, numRows, false);
+
+		var outRows = 0;
+
+		for (var i = 0; i < numRows; i++)
+		{
+      var add = 1;
+      add = array[i][1].neq(0).mult(add);
+      outRows += add;
+		}
+
+    var returnArray = []
+
+		for (var i = 0; i < numOutRows; i++)
+		{
+			returnArray.push([])
+			array[i].push(array[i][0]);
+			array[i].push(array[i][1]);
+		}
+
+    return returnArray;
+  }
+
+  const padInput = function(array, numCols, numRows, padVal)
+  {
+    var paddedArray = [];
+    for (var i = 0; i < numRows; i++) {
+      paddedArray.push([]);
+      for (var j = 0; j < numCols; j++)
+      {
+        paddedArray[i].push(array[i][j]);
+      }
+    }
+
+    for (var i = numRows; i < pad; i++)
+    {
+      paddedArray.push([]);
+      for (var j = 0; j < numCols; j++)
+      {
+        paddedArray[i].push(0);
+      }
+    }
+    return paddedArray;
+  }
+
+  const nextPowerOf2 = function(n)
+  {
+    var p = 1;
+    if (n && !(n & (n - 1)))
+      return n;
+    while (p < n)
+      p <<= 1;
+    return p;
+  }
+
   const divide = function(inRel, newCol, targetCol, operands, scalar)
   {
     var result = [];
@@ -146,24 +320,10 @@
     return result;
   }
 
-  const open = function(inRel, jiff_instance)
+  const open = function(inRel)
   {
-    var result = [];
-
-    for (var k = 0; k < inRel.length; k++)
-    {
-      result.push([]);
-    }
-
-    for (var i = 0; i < inRel.length; i++)
-    {
-      for (var j = 0; j < inRel[i].length; j++)
-      {
-        result[i].push(jiff_instance.open(inRel[i][j]));
-      }
-    }
-
-    return result
+    var results = saved_instance.open_2D_array(inRel);
+    return results;
   };
 
   /**
@@ -177,7 +337,7 @@
     // NOTE: assuming 1 file per party here
 
     var inputData = [];
-    var unparsedData = fs.readFileSync(input, 'UTF-8');
+    var unparsedData = (fs.readFileSync(input, 'UTF-8')).trim();
     var rows = unparsedData.split('\n');
 
     // start at one, skip header row
@@ -187,21 +347,10 @@
       inputData.push(arr);
     }
 
-    var allShares = [];
-
-    for (let j = 0; j < inputData.length; j++)
-    {
-      allShares.push(jiff_instance.share_array(inputData[j]));
-    }
-
-    Promise.all(allShares).then(function(shares) {
-
-      var datasets = splitDatasets(shares, jiff_instance);
-
-{{{OP_CODE}}}
+    var promise = jiff_instance.share_2D_array(inputData);
 
+    return promise.then(function(shares) {
+        {{{OP_CODE}}}
     });
-
-    return Promise.all(allShares);
   };
 }((typeof exports == 'undefined' ? this.mpc = {} : exports), typeof exports != 'undefined'));

conclave/codegen/templates/jiff/open.tmpl

@@ -0,0 +1,3 @@
+
+    var {{{OUTREL}}} = open({{{INREL}}});
+    return {{{OUTREL}}};