Merge pull request #62 from common-workflow-language/attachment-subdirs

Support subdirs in attachments.

Author: tetron

setup.py

@@ -11,7 +11,7 @@
     long_description = readmeFile.read()
 
 setup(name='wes-service',
-      version='2.8',
+      version='2.9',
       description='GA4GH Workflow Execution Service reference implementation',
       long_description=long_description,
       author='GA4GH Containers and Workflows task team',

wes_client/util.py

@@ -129,10 +129,12 @@ def build_wes_request(workflow_file, json_path, attachments=None):
             if attachment.startswith("file://"):
                 attachment = attachment[7:]
                 attach_f = open(attachment, "rb")
+                relpath = os.path.relpath(attachment, os.getcwd())
             elif attachment.startswith("http"):
                 attach_f = urlopen(attachment)
+                relpath = os.path.basename(attach_f)
 
-            parts.append(("workflow_attachment", (os.path.basename(attachment), attach_f)))
+            parts.append(("workflow_attachment", (relpath, attach_f)))
 
     return parts
 

wes_client/wes_client_main.py

@@ -66,7 +66,7 @@ def main(argv=sys.argv[1:]):
 
     if args.log:
         response = client.get_run_log(run_id=args.log)
-        sys.stdout.write(response["workflow_log"]["stderr"])
+        sys.stdout.write(requests.get(response["run_log"]["stderr"], headers=auth).text)
         return 0
 
     if args.get:

wes_service/arvados_wes.py

@@ -165,8 +165,8 @@ def invoke_cwl_runner(self, cr_uuid, workflow_url, workflow_params,
 
                 self.log_for_run(cr_uuid, stderrdata, env['ARVADOS_API_TOKEN'])
 
-                if tempdir:
-                    shutil.rmtree(tempdir)
+            if tempdir:
+                shutil.rmtree(tempdir)
 
         except subprocess.CalledProcessError as e:
             api.container_requests().update(uuid=cr_uuid, body={"priority": 0,
@@ -272,11 +272,10 @@ def log_object(cr):
                 "exit_code": containerlog["exit_code"] or 0
             }
             if containerlog["log"]:
-                r["stdout"] = "%sc=%s/_/%s" % (api._resourceDesc["keepWebServiceUrl"], containerlog["log"], "stdout.txt")  # NOQA
-                r["stderr"] = "%sc=%s/_/%s" % (api._resourceDesc["keepWebServiceUrl"], containerlog["log"], "stderr.txt")  # NOQA
-            else:
-                r["stdout"] = "%s/x-dynamic-logs/stdout" % (connexion.request.url)
-                r["stderr"] = "%s/x-dynamic-logs/stderr" % (connexion.request.url)
+                r["stdout_keep"] = "%sc=%s/_/%s" % (api._resourceDesc["keepWebServiceUrl"], containerlog["log"], "stdout.txt")  # NOQA
+                r["stderr_keep"] = "%sc=%s/_/%s" % (api._resourceDesc["keepWebServiceUrl"], containerlog["log"], "stderr.txt")  # NOQA
+            r["stdout"] = "%s/x-dynamic-logs/stdout" % (connexion.request.url)
+            r["stderr"] = "%s/x-dynamic-logs/stderr" % (connexion.request.url)
 
             return r
 

wes_service/cwl_runner.py

@@ -59,9 +59,9 @@ def run(self, request, tempdir, opts):
 
         # link the cwl and json into the tempdir/cwd
         if workflow_url.startswith('file://'):
-            os.link(workflow_url[7:], os.path.join(tempdir, "wes_workflow.cwl"))
+            os.symlink(workflow_url[7:], os.path.join(tempdir, "wes_workflow.cwl"))
             workflow_url = os.path.join(tempdir, "wes_workflow.cwl")
-        os.link(inputtemp.name, os.path.join(tempdir, "cwl.input.json"))
+        os.symlink(inputtemp.name, os.path.join(tempdir, "cwl.input.json"))
         jsonpath = os.path.join(tempdir, "cwl.input.json")
 
         # build args and run

wes_service/util.py

@@ -52,8 +52,14 @@ def collect_attachments(self, run_id=None):
         for k, ls in iterlists(connexion.request.files):
             for v in ls:
                 if k == "workflow_attachment":
-                    filename = secure_filename(v.filename)
-                    dest = os.path.join(tempdir, filename)
+                    sp = v.filename.split("/")
+                    fn = []
+                    for p in sp:
+                        if p not in ("", ".", ".."):
+                            fn.append(secure_filename(p))
+                    dest = os.path.join(tempdir, *fn)
+                    if not os.path.isdir(os.path.dirname(dest)):
+                        os.makedirs(os.path.dirname(dest))
                     self.log_for_run(run_id, "Staging attachment '%s' to '%s'" % (v.filename, dest))
                     v.save(dest)
                     body[k] = "file://%s" % tempdir  # Reference to temp working dir.