From 6e604ca1aba8fe45a517ebaf7b7e1df59add9bca Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 9 May 2018 11:29:21 -0400 Subject: [PATCH 01/16] rm cfg file --- examples/oblivc/local-cfg-two.yml | 54 ------------------------------- 1 file changed, 54 deletions(-) delete mode 100644 examples/oblivc/local-cfg-two.yml diff --git a/examples/oblivc/local-cfg-two.yml b/examples/oblivc/local-cfg-two.yml deleted file mode 100644 index e5ad4dba..00000000 --- a/examples/oblivc/local-cfg-two.yml +++ /dev/null @@ -1,54 +0,0 @@ -# GENERAL -pid: 2 -all_pids: [1,2] -workflow_name: oc_test -use_swift: False - -# TODO: these could be hardcoded to /tmp/_in/, -# /tmp/_in/, and /tmp/_code -# for use on OS containers. -input_path: /Users/ben/Desktop/ -output_path: /Users/ben/Desktop/ -code_path: /Users/ben/Desktop/protocol2/ - -# SPARK -spark: - master_url: local - -# OBLIV-C -oblivc: - oc_path: /Users/ben/Desktop/dev/obliv-c/bin/oblivcc - ip_port: localhost:9000 - -# NET -net: - parties: - - host: localhost - port: 9001 - - host: localhost - port: 9002 - -# SWIFT -swift: - source: - AUTHORIZATION: - osAuthUrl: - username: - password: - PROJECT: - osProjectDomain: - osProjectName: - DATA: - container_name: - files: - dest: - AUTHORIZATION: - osAuthUrl: - username: - password: - PROJECT: - osProjectDomain: - osProjectName: - DATA: - container_name: - files: \ No newline at end of file From 7f1cb8bd9de493addb4c03e2e112dd5666ab0ac9 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 9 May 2018 11:30:16 -0400 Subject: [PATCH 02/16] update --- .gitignore | 2 ++ conclave/dispatch/oblivc.py | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index adecf125..34751dfd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ # Added manually. .idea/ .DS_Store +examples/oblivc/local-cfg-one.yml +examples/oblivc/local-cfg-two.yml # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/conclave/dispatch/oblivc.py b/conclave/dispatch/oblivc.py index 3d14db53..604e9670 100644 --- a/conclave/dispatch/oblivc.py +++ b/conclave/dispatch/oblivc.py @@ -27,15 +27,12 @@ def _dispatch(self, job): except Exception as e: print(e) - @asyncio.coroutine def party_one_dispatch(self, job): - # TODO: parties hardcoded as 1 & 2 right now - might be different in future - yield from self.loop.run_in_executor(None, self._dispatch(job)) - - # notify other party that we're done self.peer.send_done_msg(2, job.name + ".party_one") + self._dispatch(job) + def party_two_dispatch(self, job): for input_party in job.input_parties: @@ -45,6 +42,9 @@ def party_two_dispatch(self, job): futures = self.to_wait_on.values() self.loop.run_until_complete(asyncio.gather(*futures)) + # hack + time.sleep(15) + self._dispatch(job) def dispatch(self, job): From 0324e2f9e0214a69616ba6bd2c5c2a2dad75c5fa Mon Sep 17 00:00:00 2001 From: nicolas3355 Date: Wed, 6 Jun 2018 10:54:33 -0400 Subject: [PATCH 03/16] moved nosetest to the requirements to better support virtualenv. --- README.rst | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index e09a4aa1..3c44ba37 100644 --- a/README.rst +++ b/README.rst @@ -11,7 +11,7 @@ This framework allows users to define data analysis workflows in familiar fronte Dependencies ------------ -Conclave requires a Python 3.x environment. On Ubuntu (14.04+), installing the `python3`, `python3-pystache`, and `python3-nose` should get everything that's needed. +Conclave requires a Python 3.x environment. On Ubuntu (14.04+), installing the `python3`, `python3-pystache` should get everything that's needed. Testing ------- diff --git a/requirements.txt b/requirements.txt index 1e276c6f..ab473a55 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,4 @@ pystache>=0.5.4 PyYAML>=3.12 python-keystoneclient python-swiftclient - +nose From b9878faca556546a32676915e4a06585e7aaa078 Mon Sep 17 00:00:00 2001 From: bengetch Date: Mon, 11 Jun 2018 11:35:52 -0400 Subject: [PATCH 04/16] add license --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..6a8f1bb5 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Ben Getchell + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From 2266f7d0518d39a902aebcd38ea176fb16fd83f7 Mon Sep 17 00:00:00 2001 From: bengetch Date: Mon, 11 Jun 2018 11:59:55 -0400 Subject: [PATCH 05/16] Update LICENSE --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 6a8f1bb5..fe174c97 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2018 Ben Getchell +Copyright (c) 2018 Boston University - Software & Application Innovation Lab, Nikolaj Volgushev, Malte Schwarzkopf Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal From a87902d528815ac1e5a2cfb43868852f0c8066f2 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 5 Jul 2018 10:46:52 -0400 Subject: [PATCH 06/16] rework moc ex --- examples/apps/moc/conf-ca.yml | 22 ---------------------- examples/apps/moc/conf-cb.yml | 22 ---------------------- examples/apps/moc/conf-cc.yml | 22 ---------------------- examples/apps/moc/moc_example.py | 22 ++++++++-------------- 4 files changed, 8 insertions(+), 80 deletions(-) delete mode 100644 examples/apps/moc/conf-ca.yml delete mode 100644 examples/apps/moc/conf-cb.yml delete mode 100644 examples/apps/moc/conf-cc.yml diff --git a/examples/apps/moc/conf-ca.yml b/examples/apps/moc/conf-ca.yml deleted file mode 100644 index 4d5a0e13..00000000 --- a/examples/apps/moc/conf-ca.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -# Party ID -pid: &pid 1 -workflow_name: &workflow_name !join [job-, *pid] -name: taxi -delimiter: ',' -code_path: /mnt/shared/ -spark: - cluster_prefix: &prefix ca - hdfs: - node_name: !join [*prefix, '-spark-node-0:8020'] - root: /home/ubuntu - master_url: !join ['spark://', *prefix, '-spark-node-0:7077'] - -sharemind: - parties: - - host: ca-spark-node-0 - port: 9001 - - host: cb-spark-node-0 - port: 9002 - - host: cc-spark-node-0 - port: 9003 \ No newline at end of file diff --git a/examples/apps/moc/conf-cb.yml b/examples/apps/moc/conf-cb.yml deleted file mode 100644 index d3f5b903..00000000 --- a/examples/apps/moc/conf-cb.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -# Party ID -pid: &pid 2 -workflow_name: &workflow_name !join [job-, *pid] -name: taxi -delimiter: ',' -code_path: /mnt/shared/ -spark: - cluster_prefix: &prefix cb - hdfs: - node_name: !join [*prefix, '-spark-node-0:8020'] - root: /home/ubuntu - master_url: !join ['spark://', *prefix, '-spark-node-0:7077'] - -sharemind: - parties: - - host: ca-spark-node-0 - port: 9001 - - host: cb-spark-node-0 - port: 9002 - - host: cc-spark-node-0 - port: 9003 diff --git a/examples/apps/moc/conf-cc.yml b/examples/apps/moc/conf-cc.yml deleted file mode 100644 index 53a87607..00000000 --- a/examples/apps/moc/conf-cc.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -# Party ID -pid: &pid 3 -workflow_name: &workflow_name !join [job-, *pid] -name: taxi -delimiter: ',' -code_path: /mnt/shared/ -spark: - cluster_prefix: &prefix cc - hdfs: - node_name: !join [*prefix, '-spark-node-0:8020'] - root: /home/ubuntu - master_url: !join ['spark://', *prefix, '-spark-node-0:7077'] - -sharemind: - parties: - - host: ca-spark-node-0 - port: 9001 - - host: cb-spark-node-0 - port: 9002 - - host: cc-spark-node-0 - port: 9003 \ No newline at end of file diff --git a/examples/apps/moc/moc_example.py b/examples/apps/moc/moc_example.py index 8707fd1b..fa14eb3a 100644 --- a/examples/apps/moc/moc_example.py +++ b/examples/apps/moc/moc_example.py @@ -13,34 +13,28 @@ def protocol(): # define input columns cols_in_a = [ - defCol('a', 'INTEGER', [2]), - defCol('b', 'INTEGER', [2]), + defCol('a', 'INTEGER', [1]), + defCol('b', 'INTEGER', [1]), ] cols_in_b = [ - defCol('a', 'INTEGER', [2]), - defCol('c', 'INTEGER', [2]), - ] - cols_in_c = [ - defCol('a', 'INTEGER', [2]), - defCol('d', 'INTEGER', [2]) + defCol('a', 'INTEGER', [1]), + defCol('c', 'INTEGER', [1]), ] # instantiate input columns # NOTE: input file names will correspond to the 0th arg of each create call ("in1", "in2", etc.) - in1 = sal.create("in1", cols_in_a, {2}) - in2 = sal.create("in2", cols_in_b, {2}) - in3 = sal.create("in3", cols_in_c, {2}) + in1 = sal.create("in1", cols_in_a, {1}) + in2 = sal.create("in2", cols_in_b, {1}) # operate on columns # join in1 & in2 over the column 'a', name output relation 'join1' join1 = sal.join(in1, in2, 'join1', ['a'], ['a']) - join2 = sal.join(join1, in3, 'join2', ['a'], ['a']) # collect leaf node - out = sal.collect(join2, 2) + out = sal.collect(join1, 2) # return root nodes - return {in1, in2, in3} + return {in1, in2} if __name__ == "__main__": From eaa29b7f5a210aa256a1353219313562d5658745 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 25 Jul 2018 14:25:42 -0400 Subject: [PATCH 07/16] workflow for openshift --- examples/apps/local/conf-one.yaml | 64 ++++++++++++++++++++++++++ examples/apps/local/conf-two.yaml | 64 ++++++++++++++++++++++++++ examples/apps/local/in1.csv | 3 ++ examples/apps/local/in2.csv | 3 ++ examples/apps/local/local.py | 43 +++++++++++++++++ examples/apps/local/timing_results.csv | 2 + examples/cfg-ex.yml | 8 ++++ 7 files changed, 187 insertions(+) create mode 100644 examples/apps/local/conf-one.yaml create mode 100644 examples/apps/local/conf-two.yaml create mode 100644 examples/apps/local/in1.csv create mode 100644 examples/apps/local/in2.csv create mode 100644 examples/apps/local/local.py create mode 100644 examples/apps/local/timing_results.csv diff --git a/examples/apps/local/conf-one.yaml b/examples/apps/local/conf-one.yaml new file mode 100644 index 00000000..f95f1ede --- /dev/null +++ b/examples/apps/local/conf-one.yaml @@ -0,0 +1,64 @@ +# GENERAL +all_pids: [1,2] +pid: 1 +workflow_name: test-workflow +name: test +delimiter: ',' + +# SWIFT +use_swift: False + +# SPARK +spark: + master_url: local + +# OBLIV-C +oblivc: + oc_path: // + ip_port: localhost:9000 + +# JIFF +jiff: + jiff_path: /Users/ben/Desktop/dev/jiff/ + party_count: 2 + server_pid: 1 + server_ip: localhost + server_port: 9002 + +# NET +net: + parties: + - host: 127.0.0.1 + port: 9000 + - host: 127.0.0.1 + port: 9001 + +# INPUT & OUTPUT +input_path: /Users/ben/Desktop/dev/cici-conclave/conclave/examples/apps/local/ +output_path: /tmp/ +code_path: /tmp/ + +# SWIFT +swift: + source: + AUTHORIZATION: + osAuthUrl: + username: + password: + PROJECT: + osProjectDomain: + osProjectName: + DATA: + container_name: + files: + dest: + AUTHORIZATION: + osAuthUrl: + username: + password: + PROJECT: + osProjectDomain: + osProjectName: + DATA: + container_name: + files: \ No newline at end of file diff --git a/examples/apps/local/conf-two.yaml b/examples/apps/local/conf-two.yaml new file mode 100644 index 00000000..e90af43c --- /dev/null +++ b/examples/apps/local/conf-two.yaml @@ -0,0 +1,64 @@ +# GENERAL +all_pids: [1,2] +pid: 2 +workflow_name: test-workflow +name: test +delimiter: ',' + +# SWIFT +use_swift: False + +# SPARK +spark: + master_url: local + +# OBLIV-C +oblivc: + oc_path: // + ip_port: localhost:9000 + +# JIFF +jiff: + jiff_path: /Users/ben/Desktop/dev/jiff/ + party_count: 2 + server_pid: 1 + server_ip: localhost + server_port: 9002 + +# NET +net: + parties: + - host: 127.0.0.1 + port: 9000 + - host: 127.0.0.1 + port: 9001 + +# INPUT & OUTPUT +input_path: /Users/ben/Desktop/dev/cici-conclave/conclave/examples/apps/local/ +output_path: /tmp/ +code_path: /tmp/ + +# SWIFT +swift: + source: + AUTHORIZATION: + osAuthUrl: + username: + password: + PROJECT: + osProjectDomain: + osProjectName: + DATA: + container_name: + files: + dest: + AUTHORIZATION: + osAuthUrl: + username: + password: + PROJECT: + osProjectDomain: + osProjectName: + DATA: + container_name: + files: \ No newline at end of file diff --git a/examples/apps/local/in1.csv b/examples/apps/local/in1.csv new file mode 100644 index 00000000..38139df2 --- /dev/null +++ b/examples/apps/local/in1.csv @@ -0,0 +1,3 @@ +a,b +1,1 +2,1 \ No newline at end of file diff --git a/examples/apps/local/in2.csv b/examples/apps/local/in2.csv new file mode 100644 index 00000000..d63e6a3e --- /dev/null +++ b/examples/apps/local/in2.csv @@ -0,0 +1,3 @@ +a,c +1,1 +2,1 \ No newline at end of file diff --git a/examples/apps/local/local.py b/examples/apps/local/local.py new file mode 100644 index 00000000..fa14eb3a --- /dev/null +++ b/examples/apps/local/local.py @@ -0,0 +1,43 @@ +""" +Simple example workflow for MOC deployment of Conclave +""" +import conclave.lang as sal +from conclave.utils import * +from conclave import workflow + + +def protocol(): + """ + Define inputs and operations to be performed between them. + """ + + # define input columns + cols_in_a = [ + defCol('a', 'INTEGER', [1]), + defCol('b', 'INTEGER', [1]), + ] + cols_in_b = [ + defCol('a', 'INTEGER', [1]), + defCol('c', 'INTEGER', [1]), + ] + + # instantiate input columns + # NOTE: input file names will correspond to the 0th arg of each create call ("in1", "in2", etc.) + in1 = sal.create("in1", cols_in_a, {1}) + in2 = sal.create("in2", cols_in_b, {1}) + + # operate on columns + # join in1 & in2 over the column 'a', name output relation 'join1' + join1 = sal.join(in1, in2, 'join1', ['a'], ['a']) + + # collect leaf node + out = sal.collect(join1, 2) + + # return root nodes + return {in1, in2} + + +if __name__ == "__main__": + + workflow.run(protocol) + diff --git a/examples/apps/local/timing_results.csv b/examples/apps/local/timing_results.csv new file mode 100644 index 00000000..c91c0710 --- /dev/null +++ b/examples/apps/local/timing_results.csv @@ -0,0 +1,2 @@ +test-workflow,0.0,0:00:00.000073 +test-workflow,0.045,0:00:00.044935 diff --git a/examples/cfg-ex.yml b/examples/cfg-ex.yml index ec40d467..c89640ac 100644 --- a/examples/cfg-ex.yml +++ b/examples/cfg-ex.yml @@ -20,6 +20,14 @@ oblivc: oc_path: ip_port: : +# JIFF +jiff: + jiff_path: + party_count: + server_pid: + server_ip: + server_port: + # NET net: parties: From 22281377430c0356bcab08ca1efdeac2e4e014f2 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 25 Jul 2018 14:25:53 -0400 Subject: [PATCH 08/16] jiff config --- conclave/workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conclave/workflow.py b/conclave/workflow.py index 01447910..d9410940 100644 --- a/conclave/workflow.py +++ b/conclave/workflow.py @@ -34,7 +34,7 @@ def setup(conf: Dict): oc_config = OblivcConfig(oc_path, ip_port) # JIFF - jiff_path = conf["jiff"]["jiff_path"] + jiff_path = conf["jiff"]["jiff_path"] party_count = conf["jiff"]["party_count"] server_pid = conf["jiff"]["server_pid"] server_ip = conf["jiff"]["server_ip"] From 78bc0ad2a01f0f7a14032961cb2c4d9af1fa7a97 Mon Sep 17 00:00:00 2001 From: nicolas3355 Date: Wed, 25 Jul 2018 16:58:12 -0400 Subject: [PATCH 09/16] bug fixes in the generated jiff code --- conclave/codegen/jiff.py | 13 +++++++++---- conclave/codegen/templates/jiff/bash.tmpl | 2 +- conclave/codegen/templates/jiff/server.tmpl | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/conclave/codegen/jiff.py b/conclave/codegen/jiff.py index 14b1d2d5..a29a5e8e 100644 --- a/conclave/codegen/jiff.py +++ b/conclave/codegen/jiff.py @@ -275,10 +275,15 @@ def _generate_sort_by(self, sort_op: SortBy): return pystache.render(template, data) - def _generate_bash(self): - # TODO: need clear way to run jiff server/party/mpc files outside of jiff repository + def _write_bash(self): + template = open("{}/bash.tmpl" + .format(self.template_directory), 'r').read() - bash_code = '' + data = { + "JIFF_PATH": self.jiff_config.jiff_path + } + + bash_code = pystache.render(template, data) if self.pid == self.jiff_config.server_pid: bash_code += 'node server.js\n' @@ -302,7 +307,7 @@ def _write_code(self, code, job_name): protocol_file = open("{}/{}/mpc.js".format(self.config.code_path, job_name), 'w') protocol_file.write(code) - bash_code = self._generate_bash() + bash_code = self._write_bash() bash_file = open("{}/{}/run.sh".format(self.config.code_path, job_name), 'w') bash_file.write(bash_code) diff --git a/conclave/codegen/templates/jiff/bash.tmpl b/conclave/codegen/templates/jiff/bash.tmpl index c38184eb..c8b6dbed 100644 --- a/conclave/codegen/templates/jiff/bash.tmpl +++ b/conclave/codegen/templates/jiff/bash.tmpl @@ -1,3 +1,3 @@ #!/bin/bash -{{{OP_CODE}}} \ No newline at end of file +export NODE_PATH="{{{JIFF_PATH}}}/node_modules" diff --git a/conclave/codegen/templates/jiff/server.tmpl b/conclave/codegen/templates/jiff/server.tmpl index 879b7bb8..deb9160f 100644 --- a/conclave/codegen/templates/jiff/server.tmpl +++ b/conclave/codegen/templates/jiff/server.tmpl @@ -10,4 +10,4 @@ app.use("{{{JIFF_PATH}}}/lib/ext", express.static("lib/ext")); http.listen({{{PORT}}}, function() { console.log('listening on *:{{{PORT}}}'); -} +}); From 17c28a09a581a7b7745e7580b1b580a77087d7a0 Mon Sep 17 00:00:00 2001 From: Ben Date: Fri, 27 Jul 2018 14:46:42 -0400 Subject: [PATCH 10/16] python3 --> python3.5 in python dispatch --- conclave/dispatch/python.py | 2 +- conclave/net/__init__.py | 2 +- examples/apps/local/conf-one.yaml | 64 ------------------------------- examples/apps/local/conf-two.yaml | 64 ------------------------------- 4 files changed, 2 insertions(+), 130 deletions(-) delete mode 100644 examples/apps/local/conf-one.yaml delete mode 100644 examples/apps/local/conf-two.yaml diff --git a/conclave/dispatch/python.py b/conclave/dispatch/python.py index 21188ff0..e7c1b7cc 100644 --- a/conclave/dispatch/python.py +++ b/conclave/dispatch/python.py @@ -12,6 +12,6 @@ def dispatch(self, job): .format(job.name, job.code_dir)) try: - call(["python3", cmd]) + call(["python3.5", cmd]) except Exception as e: print(e) diff --git a/conclave/net/__init__.py b/conclave/net/__init__.py index 6fe60c16..9640bb6a 100644 --- a/conclave/net/__init__.py +++ b/conclave/net/__init__.py @@ -1,7 +1,7 @@ import asyncio import functools import pickle -import sys + class IAMMsg: """ Message identifying peer. """ diff --git a/examples/apps/local/conf-one.yaml b/examples/apps/local/conf-one.yaml deleted file mode 100644 index f95f1ede..00000000 --- a/examples/apps/local/conf-one.yaml +++ /dev/null @@ -1,64 +0,0 @@ -# GENERAL -all_pids: [1,2] -pid: 1 -workflow_name: test-workflow -name: test -delimiter: ',' - -# SWIFT -use_swift: False - -# SPARK -spark: - master_url: local - -# OBLIV-C -oblivc: - oc_path: // - ip_port: localhost:9000 - -# JIFF -jiff: - jiff_path: /Users/ben/Desktop/dev/jiff/ - party_count: 2 - server_pid: 1 - server_ip: localhost - server_port: 9002 - -# NET -net: - parties: - - host: 127.0.0.1 - port: 9000 - - host: 127.0.0.1 - port: 9001 - -# INPUT & OUTPUT -input_path: /Users/ben/Desktop/dev/cici-conclave/conclave/examples/apps/local/ -output_path: /tmp/ -code_path: /tmp/ - -# SWIFT -swift: - source: - AUTHORIZATION: - osAuthUrl: - username: - password: - PROJECT: - osProjectDomain: - osProjectName: - DATA: - container_name: - files: - dest: - AUTHORIZATION: - osAuthUrl: - username: - password: - PROJECT: - osProjectDomain: - osProjectName: - DATA: - container_name: - files: \ No newline at end of file diff --git a/examples/apps/local/conf-two.yaml b/examples/apps/local/conf-two.yaml deleted file mode 100644 index e90af43c..00000000 --- a/examples/apps/local/conf-two.yaml +++ /dev/null @@ -1,64 +0,0 @@ -# GENERAL -all_pids: [1,2] -pid: 2 -workflow_name: test-workflow -name: test -delimiter: ',' - -# SWIFT -use_swift: False - -# SPARK -spark: - master_url: local - -# OBLIV-C -oblivc: - oc_path: // - ip_port: localhost:9000 - -# JIFF -jiff: - jiff_path: /Users/ben/Desktop/dev/jiff/ - party_count: 2 - server_pid: 1 - server_ip: localhost - server_port: 9002 - -# NET -net: - parties: - - host: 127.0.0.1 - port: 9000 - - host: 127.0.0.1 - port: 9001 - -# INPUT & OUTPUT -input_path: /Users/ben/Desktop/dev/cici-conclave/conclave/examples/apps/local/ -output_path: /tmp/ -code_path: /tmp/ - -# SWIFT -swift: - source: - AUTHORIZATION: - osAuthUrl: - username: - password: - PROJECT: - osProjectDomain: - osProjectName: - DATA: - container_name: - files: - dest: - AUTHORIZATION: - osAuthUrl: - username: - password: - PROJECT: - osProjectDomain: - osProjectName: - DATA: - container_name: - files: \ No newline at end of file From 151dd11d337752ca7276fb0411d4fb0d390a18f9 Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 28 Jul 2018 16:28:19 -0400 Subject: [PATCH 11/16] compute party change --- examples/apps/local/local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/apps/local/local.py b/examples/apps/local/local.py index fa14eb3a..1511f91b 100644 --- a/examples/apps/local/local.py +++ b/examples/apps/local/local.py @@ -31,7 +31,7 @@ def protocol(): join1 = sal.join(in1, in2, 'join1', ['a'], ['a']) # collect leaf node - out = sal.collect(join1, 2) + out = sal.collect(join1, 1) # return root nodes return {in1, in2} From da8b908bb5813919ad429206cf06bcc3a8bc9117 Mon Sep 17 00:00:00 2001 From: nicolas3355 Date: Tue, 31 Jul 2018 12:57:05 -0400 Subject: [PATCH 12/16] fixed typo that was causing party not connecting to server bug --- conclave/codegen/templates/jiff/party.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conclave/codegen/templates/jiff/party.tmpl b/conclave/codegen/templates/jiff/party.tmpl index cc1feb7d..cb099218 100644 --- a/conclave/codegen/templates/jiff/party.tmpl +++ b/conclave/codegen/templates/jiff/party.tmpl @@ -22,4 +22,4 @@ options.onConnect = function(jiff_instance) { }; // Connect -mpc.connect({{{SERVER_IP_PORT}}}, computation_id, options); \ No newline at end of file +mpc.connect("http://{{{SERVER_IP_PORT}}}", computation_id, options); \ No newline at end of file From b0801d468a89c21573a605b0d2616ed3ed6268e2 Mon Sep 17 00:00:00 2001 From: nicolas3355 Date: Thu, 2 Aug 2018 12:15:11 -0400 Subject: [PATCH 13/16] implemented open shares, using the new share and open --- conclave/codegen/jiff.py | 8 +-- conclave/codegen/templates/jiff/create.tmpl | 2 +- .../codegen/templates/jiff/mpc_top_level.tmpl | 53 +++---------------- conclave/codegen/templates/jiff/open.tmpl | 2 + conclave/codegen/templates/jiff/party.tmpl | 4 +- 5 files changed, 18 insertions(+), 51 deletions(-) diff --git a/conclave/codegen/jiff.py b/conclave/codegen/jiff.py index a29a5e8e..a96725d3 100644 --- a/conclave/codegen/jiff.py +++ b/conclave/codegen/jiff.py @@ -185,10 +185,12 @@ def _generate_open(self, open_op: Open): template = open( "{0}/open.tmpl".format(self.template_directory), 'r').read() - data = {} + data = { + "INREL": open_op.get_in_rel().name, + "OUTREL": open_op.out_rel.name + } - # return pystache.render(template, data) - return '' + return pystache.render(template, data) def _generate_project(self, project_op: Project): diff --git a/conclave/codegen/templates/jiff/create.tmpl b/conclave/codegen/templates/jiff/create.tmpl index 7af61daa..a1b60509 100644 --- a/conclave/codegen/templates/jiff/create.tmpl +++ b/conclave/codegen/templates/jiff/create.tmpl @@ -1,2 +1,2 @@ - var {{{OUTREL}}} = datasets[{{{ID}}}]; \ No newline at end of file + var {{{OUTREL}}} = shares[{{{ID}}}]; \ No newline at end of file diff --git a/conclave/codegen/templates/jiff/mpc_top_level.tmpl b/conclave/codegen/templates/jiff/mpc_top_level.tmpl index 6538f375..57b4b102 100644 --- a/conclave/codegen/templates/jiff/mpc_top_level.tmpl +++ b/conclave/codegen/templates/jiff/mpc_top_level.tmpl @@ -14,24 +14,6 @@ return saved_instance; }; - const splitDatasets = function(shares, jiff_instance) - { - var datasets = []; - - for (var k = 1; k <= jiff_instance.party_count; k++) - { - datasets.push([]); - } - - for (var i = 0; i < shares.length; i++) { - for (var j = 0; j < jiff_instance.party_count; j++) { - datasets[j].push(shares[i][j+1]); - } - } - - return datasets; - - }; const project = function(inputRel, projCols) { @@ -50,6 +32,8 @@ result[j].push(inputRel[j][projCols[k]]); } } + return result; + }; const concatenate = function(inRels) @@ -148,20 +132,8 @@ const open = function(inRel, jiff_instance) { - var result = []; + var results = jiff_instance.open_2D_array(inRel); - for (var k = 0; k < inRel.length; k++) - { - result.push([]); - } - - for (var i = 0; i < inRel.length; i++) - { - for (var j = 0; j < inRel[i].length; j++) - { - result[i].push(jiff_instance.open(inRel[i][j])); - } - } return result }; @@ -177,7 +149,7 @@ // NOTE: assuming 1 file per party here var inputData = []; - var unparsedData = fs.readFileSync(input, 'UTF-8'); + var unparsedData = (fs.readFileSync(input, 'UTF-8')).trim(); var rows = unparsedData.split('\n'); // start at one, skip header row @@ -187,21 +159,10 @@ inputData.push(arr); } - var allShares = []; - - for (let j = 0; j < inputData.length; j++) - { - allShares.push(jiff_instance.share_array(inputData[j])); - } - - Promise.all(allShares).then(function(shares) { - - var datasets = splitDatasets(shares, jiff_instance); - -{{{OP_CODE}}} + var promise = jiff_instance.share_2D_array(inputData); + return promise.then(function(shares) { + {{{OP_CODE}}} }); - - return Promise.all(allShares); }; }((typeof exports == 'undefined' ? this.mpc = {} : exports), typeof exports != 'undefined')); diff --git a/conclave/codegen/templates/jiff/open.tmpl b/conclave/codegen/templates/jiff/open.tmpl index e69de29b..bd2bd449 100644 --- a/conclave/codegen/templates/jiff/open.tmpl +++ b/conclave/codegen/templates/jiff/open.tmpl @@ -0,0 +1,2 @@ + + var {{{OUTREL}}} = open({{{INREL}}}); diff --git a/conclave/codegen/templates/jiff/party.tmpl b/conclave/codegen/templates/jiff/party.tmpl index cb099218..253048e1 100644 --- a/conclave/codegen/templates/jiff/party.tmpl +++ b/conclave/codegen/templates/jiff/party.tmpl @@ -18,7 +18,9 @@ if(party_id != null) party_id = parseInt(party_id, 10); // JIFF options var options = {party_count: party_count, party_id: party_id}; options.onConnect = function(jiff_instance) { - mpc.compute(input); + mpc.compute(input).then(function(p){ + p.then(console.log); + }); }; // Connect From ffdb84ea8ddbfa3609714605ae9748c087f4fa6f Mon Sep 17 00:00:00 2001 From: nicolas3355 Date: Sat, 4 Aug 2018 11:55:54 -0400 Subject: [PATCH 14/16] bug fixes + hack to make open work --- conclave/codegen/templates/jiff/mpc_top_level.tmpl | 8 +++----- conclave/codegen/templates/jiff/open.tmpl | 1 + examples/jiff_codegen/jiff_ex.py | 5 +---- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/conclave/codegen/templates/jiff/mpc_top_level.tmpl b/conclave/codegen/templates/jiff/mpc_top_level.tmpl index 57b4b102..9d6ea733 100644 --- a/conclave/codegen/templates/jiff/mpc_top_level.tmpl +++ b/conclave/codegen/templates/jiff/mpc_top_level.tmpl @@ -130,12 +130,10 @@ return result; } - const open = function(inRel, jiff_instance) + const open = function(inRel) { - var results = jiff_instance.open_2D_array(inRel); - - - return result + var results = saved_instance.open_2D_array(inRel); + return results; }; /** diff --git a/conclave/codegen/templates/jiff/open.tmpl b/conclave/codegen/templates/jiff/open.tmpl index bd2bd449..1523d478 100644 --- a/conclave/codegen/templates/jiff/open.tmpl +++ b/conclave/codegen/templates/jiff/open.tmpl @@ -1,2 +1,3 @@ var {{{OUTREL}}} = open({{{INREL}}}); + return {{{OUTREL}}}; \ No newline at end of file diff --git a/examples/jiff_codegen/jiff_ex.py b/examples/jiff_codegen/jiff_ex.py index 81f9e5ca..eb904ce3 100644 --- a/examples/jiff_codegen/jiff_ex.py +++ b/examples/jiff_codegen/jiff_ex.py @@ -92,10 +92,7 @@ def project(): in1 = in_rels[0] in2 = in_rels[1] - cl1 = sal._close(in1, "cl1", set([1, 2])) - cl2 = sal._close(in2, "cl2", set([1, 2])) - - rel = sal.concat([cl1, cl2], "rel") + rel = sal.concat([in1, in2], "rel") proj = sal.project(rel, 'proj1', ['b', 'a', 'c']) From b5b6a7703272978f2b8beecaa454147d1ea55b3a Mon Sep 17 00:00:00 2001 From: nicolas3355 Date: Tue, 7 Aug 2018 11:15:32 -0400 Subject: [PATCH 15/16] jiff now prints the output of open to a file --- conclave/codegen/jiff.py | 7 +++++ conclave/codegen/templates/jiff/party.tmpl | 31 +++++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/conclave/codegen/jiff.py b/conclave/codegen/jiff.py index a96725d3..caffa602 100644 --- a/conclave/codegen/jiff.py +++ b/conclave/codegen/jiff.py @@ -46,6 +46,13 @@ def generate_party_code(self): "SERVER_IP_PORT": "{0}:{1}".format(self.jiff_config.server_ip, self.jiff_config.server_port) } + nodes = self.dag.top_sort() + for node in nodes: + if isinstance(node, Open): + if self.pid in node.out_rel.stored_with: + data['OUTPUT_FILE'] = node.out_rel.name + break + self.party_code += pystache.render(template, data) return self diff --git a/conclave/codegen/templates/jiff/party.tmpl b/conclave/codegen/templates/jiff/party.tmpl index 253048e1..6314e518 100644 --- a/conclave/codegen/templates/jiff/party.tmpl +++ b/conclave/codegen/templates/jiff/party.tmpl @@ -15,11 +15,40 @@ if(computation_id == null) computation_id = 'test'; var party_id = process.argv[5]; if(party_id != null) party_id = parseInt(party_id, 10); + +var format_2d = function (array_2d){ + var output = ""; + + for (var i=0; i < array_2d.length; i++){ + for (var j=0; j < array_2d[i].length; j++){ + output += array_2d[i][j] + ","; + } + //remove the last comma and replace it with a new line + output = output.slice(0, -1) + '\n'; + } + //remove trailing new line + return output.slice(0, -1); +} + // JIFF options var options = {party_count: party_count, party_id: party_id}; options.onConnect = function(jiff_instance) { mpc.compute(input).then(function(p){ - p.then(console.log); + p.then(function(v){ + + var output = "{{{OUTPUT_FILE}}}"; + if(output.length > 0){ + output += ".csv"; + var fs = require('fs'); + fs.writeFile(output, format_2d(v) , function(err) { + if(err) { + return console.log(err); + } + }); + } + + }); + }); }; From a36e4ffca0b188838c12b309a3d6b15b8edb1fef Mon Sep 17 00:00:00 2001 From: p-flock Date: Thu, 9 Aug 2018 16:05:26 -0400 Subject: [PATCH 16/16] add aggregate and sortby function (and helpers) into jiff templates for code generation --- .../codegen/templates/jiff/mpc_top_level.tmpl | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) diff --git a/conclave/codegen/templates/jiff/mpc_top_level.tmpl b/conclave/codegen/templates/jiff/mpc_top_level.tmpl index 9d6ea733..cc856412 100644 --- a/conclave/codegen/templates/jiff/mpc_top_level.tmpl +++ b/conclave/codegen/templates/jiff/mpc_top_level.tmpl @@ -104,6 +104,196 @@ } + + const sortBy = function(inRel) + { + var numRows = inRel.length; + var numCols = inRel[0].length; + + var sortArray = []; + for (var i = 0; i < numRows; i++) + { + sortArray.push([]); + for (var k = 0; j < numCols; j++) + { + sortArray[i].push(inRel[k]); + } + } + oddEvenSort(array, keyCol, numCols, 0, numRows, true); + } + const oddEvenSort = function(array, keyCol, numCols, lo, n, ascending) + { + if (n > 1) + { + var m = n / 2; + oddEvenSort(array, keyCol, numCols, lo, m, ascending); + oddEvenSort(array, keyCol, numCols, lo + m, m, ascending); + oddEvenMerge(array, keyCol, numCols, lo, n, 1, ascending); + } + } + + //TODO + const oddEvenMerge = function(array, keyCol, numCols, lo, n, ascending) + { + var m = r * 2; + if (m < n) + { + oddEvenMerge(array, keyCol, numCols, lo, n, m, ascending); + oddEvenMerge(array, keyCol, numCols, lo + r, n, m, ascending); + + for (var i = lo + r; (i + r) < (lo + n); i += m) + { + compareExchange(array, keyCol, numCols, i, i + r, ascending); + } + } else { + compareExchange(array, keyCol, numCols, lo, lo + r, ascending); + } + } + + //TODO + const compareExchange = function(array, keyCol, numCols, i, j, ascending) + { + var x = array[i][keyCol]; + var y = array[j][keyCol]; + + var c = 1; + var d = 1; + if (ascending) + { + c = (x.slt(y)).mult(c); + d = (y.slt(x)).mult(d); + } + else + { + d = (x.slt(y)).mult(d); + c = (y.slt(x)).mult(c); + } + + var temp1 = []; + var temp2 = []; + + for (var k = 0; k < numCols; k++) + { + var a = array[i][k]; + var b = array[j][k]; + + temp1.push((c * a) + (d * b)); + temp2.push((d * a) + (c * b)); + } + array[i] = temp1; + array[j] = temp2; + } + + //TODO: make this work similar to OblivC + const agg = function(inRel) + { + var numCols = 2; + var numRows = inRel.length; + var array = []; + + for (var i = 0; i < numRows; i++) + { + array.push([]); + array[i].push(inRel[i][0]); + array[i].push(inRel[i][1]); + } + + if (!(numRows && (numRows & (numRows - 1)) === 0)) + { + var paddedVal = nextPowerOf2(numRows); + var paddedArray = padInput(array, numCols, numRows, paddedVal); + return agg_helper(inRel, numCols, paddedVal); + } else + { + return agg_helper(inRel, numCols, numRows); + } + } + + const oddEvenAgg = function(array, lo, n) + { + if (n > 1) + { + var m = n / 2; + oddEvenAgg(array, lo, m); + oddEvenAgg(array, lo + m, m); + for (var i = 0; i < m; i++) + { + aggIfEq(array, lo + i); + } + } + else + { + aggIfEq(array, 0); + } + } + + const aggIfEq = function(array, index) + { + var c = array[index][0].eq(array[index+1][0]); + array[index][0] = array[index][1].add(c.mult(array[index+1][1])); + } + + const agg_helper = function(inRel, numCols, numRows) + { + + oddEvenSort(array, 0, numCols, 0, numRows, true); + oddEvenAgg(array, 0, numRows); + + oddEvenSort(array, 1, numCols, 0, numRows, false); + + var outRows = 0; + + for (var i = 0; i < numRows; i++) + { + var add = 1; + add = array[i][1].neq(0).mult(add); + outRows += add; + } + + var returnArray = [] + + for (var i = 0; i < numOutRows; i++) + { + returnArray.push([]) + array[i].push(array[i][0]); + array[i].push(array[i][1]); + } + + return returnArray; + } + + const padInput = function(array, numCols, numRows, padVal) + { + var paddedArray = []; + for (var i = 0; i < numRows; i++) { + paddedArray.push([]); + for (var j = 0; j < numCols; j++) + { + paddedArray[i].push(array[i][j]); + } + } + + for (var i = numRows; i < pad; i++) + { + paddedArray.push([]); + for (var j = 0; j < numCols; j++) + { + paddedArray[i].push(0); + } + } + return paddedArray; + } + + const nextPowerOf2 = function(n) + { + var p = 1; + if (n && !(n & (n - 1))) + return n; + while (p < n) + p <<= 1; + return p; + } + const divide = function(inRel, newCol, targetCol, operands, scalar) { var result = [];