Merge branch 'master' into badge-for-jenkins

Author: tom-tan

MANIFEST.in

@@ -9,6 +9,7 @@ include tests/wf/*
 include tests/override/*
 include tests/checker_wf/*
 include tests/subgraph/*
+include tests/trs/*
 include cwltool/schemas/v1.0/*.yml
 include cwltool/schemas/v1.0/*.yml
 include cwltool/schemas/v1.0/*.md

README.rst

@@ -498,7 +498,7 @@ Running tests locally
 
 -  Running basic tests ``(/tests)``:
 
-To run the basis tests after installing `cwltool` execute the following:
+To run the basic tests after installing `cwltool` execute the following:
 
 .. code:: bash
 

cwltool/docker.py

@@ -259,7 +259,7 @@ def add_writable_directory_volume(self,
                 self.append_volume(runtime, new_dir, volume.target,
                                    writable=True)
             elif not os.path.exists(host_outdir_tgt):
-                os.makedirs(host_outdir_tgt, 0o0755)
+                os.makedirs(host_outdir_tgt)
         else:
             if self.inplace_update:
                 self.append_volume(runtime, volume.resolved, volume.target,

cwltool/job.py

@@ -533,7 +533,7 @@ def create_file_and_add_volume(self,
             contents = volume.resolved
         dirname = os.path.dirname(host_outdir_tgt or new_file)
         if not os.path.exists(dirname):
-            os.makedirs(dirname, 0o0755)
+            os.makedirs(dirname)
         with open(host_outdir_tgt or new_file, "wb") as file_literal:
             file_literal.write(contents.encode("utf-8"))
         if not host_outdir_tgt:

cwltool/process.py

@@ -225,7 +225,7 @@ def stage_files(pathmapper,             # type: PathMapper
         if not entry.staged:
             continue
         if not os.path.exists(os.path.dirname(entry.target)):
-            os.makedirs(os.path.dirname(entry.target), 0o0755)
+            os.makedirs(os.path.dirname(entry.target))
         if entry.type in ("File", "Directory") and os.path.exists(entry.resolved):
             if symlink:  # Use symlink func if allowed
                 if onWindows():
@@ -242,13 +242,13 @@ def stage_files(pathmapper,             # type: PathMapper
                 stage_func(entry.resolved, entry.target)
         elif entry.type == "Directory" and not os.path.exists(entry.target) \
                 and entry.resolved.startswith("_:"):
-            os.makedirs(entry.target, 0o0755)
+            os.makedirs(entry.target)
         elif entry.type == "WritableFile" and not ignore_writable:
             shutil.copy(entry.resolved, entry.target)
             ensure_writable(entry.target)
         elif entry.type == "WritableDirectory" and not ignore_writable:
             if entry.resolved.startswith("_:"):
-                os.makedirs(entry.target, 0o0755)
+                os.makedirs(entry.target)
             else:
                 shutil.copytree(entry.resolved, entry.target)
                 ensure_writable(entry.target)

cwltool/resolver.py

@@ -13,10 +13,17 @@
 else:
     from pathlib import Path
 
+if not getattr(__builtins__, "WindowsError", None):
+    class WindowsError(OSError): pass
 
 def resolve_local(document_loader, uri):
     pathpart, frag = urllib.parse.urldefrag(uri)
-    pathobj = Path(pathpart).resolve()
+
+    try:
+        pathobj = Path(pathpart).resolve()
+    except (WindowsError, OSError):
+        _logger.debug("local resolver could not resolve %s", uri)
+        return None
 
     if pathobj.is_file():
         if frag:
@@ -48,19 +55,35 @@ def tool_resolver(document_loader, uri):
 
 
 ga4gh_tool_registries = ["https://dockstore.org/api"]
-GA4GH_TRS = "{0}/api/ga4gh/v2/tools/{1}/versions/{2}/plain-CWL/descriptor"
+# in the TRS registry, a primary descriptor can be reached at {0}/api/ga4gh/v2/tools/{1}/versions/{2}/plain-CWL/descriptor
+# The primary descriptor is a CommandLineTool in the case that the files endpoint only describes one file
+# When the primary descriptor is a Workflow, files need to be imported without stripping off "descriptor", looking at the files endpoint is a workaround
+# tested with TRS version 2.0.0-beta.2 
+# TODO not stripping off "descriptor" when looking for local imports would also work https://github.com/ga4gh/tool-registry-service-schemas/blob/2.0.0-beta.2/src/main/resources/swagger/ga4gh-tool-discovery.yaml#L273
+GA4GH_TRS_FILES = "{0}/api/ga4gh/v2/tools/{1}/versions/{2}/CWL/files"
+GA4GH_TRS_PRIMARY_DESCRIPTOR = "{0}/api/ga4gh/v2/tools/{1}/versions/{2}/plain-CWL/descriptor/{3}"
+
 
 def resolve_ga4gh_tool(document_loader, uri):
     path, version = uri.partition(":")[::2]
     if not version:
         version = "latest"
     for reg in ga4gh_tool_registries:
-        ds = GA4GH_TRS.format(reg, urllib.parse.quote(path, ""),
-                              urllib.parse.quote(version, ""))
+        ds = GA4GH_TRS_FILES.format(reg, urllib.parse.quote(path, ""), urllib.parse.quote(version, ""))
         try:
+            _logger.debug("Head path is %s", ds)
             resp = document_loader.session.head(ds)
             resp.raise_for_status()
-            return ds
+
+            _logger.debug("Passed head path of %s", ds)
+
+            resp = document_loader.session.get(ds)
+            for file_listing in resp.json():
+                if file_listing.get('file_type') == 'PRIMARY_DESCRIPTOR':
+                    primary_path = file_listing.get('path')
+                    ds2 = GA4GH_TRS_PRIMARY_DESCRIPTOR.format(reg, urllib.parse.quote(path, ""), urllib.parse.quote(version, ""), urllib.parse.quote(primary_path, ""))
+                    _logger.debug("Resolved %s", ds2)
+                    return ds2
         except Exception:
             pass
     return None

cwltool/singularity.py

@@ -226,7 +226,7 @@ def add_writable_directory_volume(self,
                 new_dir = os.path.join(
                     tempfile.mkdtemp(prefix=tmp_prefix, dir=tmp_dir),
                     os.path.basename(volume.resolved))
-            os.makedirs(new_dir, 0o0755)
+            os.makedirs(new_dir)
         else:
             if host_outdir_tgt is not None:
                 # workaround for lack of overlapping mounts in Singularity

tests/test_examples.py

@@ -1,6 +1,7 @@
 import json
 import logging
 import os
+import stat
 import sys
 from io import BytesIO, StringIO
 import pytest
@@ -792,10 +793,15 @@ class TestSecondaryFiles():
     def test_secondary_files(self):
         test_file = "secondary-files.cwl"
         test_job_file = "secondary-files-job.yml"
-        error_code, _, stderr = get_main_output(
-            ["--enable-dev",
-             get_data(os.path.join("tests", test_file)),
-             get_data(os.path.join("tests", test_job_file))])
+        try:
+            old_umask = os.umask(stat.S_IWOTH)  # test run with umask 002
+            error_code, _, stderr = get_main_output(
+                ["--enable-dev",
+                get_data(os.path.join("tests", test_file)),
+                get_data(os.path.join("tests", test_job_file))])
+        finally:
+            assert stat.S_IMODE(os.stat('lsout').st_mode) == 436  # 664 in octal, '-rw-rw-r--'
+            os.umask(old_umask)  # revert back to original umask
         assert "completed success" in stderr
         assert error_code == 0
 

tests/test_trs.py

@@ -0,0 +1,76 @@
+from __future__ import absolute_import
+
+import mock
+
+from cwltool.main import main
+from .util import get_data
+
+def mocked_requests_head(*args):
+
+    class MockResponse:
+        def __init__(self, json_data, status_code, raise_for_status=None):
+            self.json_data = json_data
+            self.status_code = status_code
+            self.raise_for_status = mock.Mock()
+            self.raise_for_status.side_effect = raise_for_status
+
+        def json(self):
+            return self.json_data
+
+    return MockResponse(None, 200)
+
+
+def mocked_requests_get(*args):
+
+    class MockResponse:
+        def __init__(self, json_data, status_code, raise_for_status=None):
+            self.json_data = json_data
+            self.text = json_data
+            self.status_code = status_code
+            self.raise_for_status = mock.Mock()
+            self.raise_for_status.side_effect = raise_for_status
+
+        def json(self):
+            return self.json_data
+
+    if args[0] == 'https://dockstore.org/api/api/ga4gh/v2/tools/quay.io%2Fbriandoconnor%2Fdockstore-tool-md5sum/versions/1.0.4/CWL/files':
+        return MockResponse(
+            [{"file_type": "CONTAINERFILE", "path": "Dockerfile"}, {"file_type": "PRIMARY_DESCRIPTOR", "path": "Dockstore.cwl"},
+             {"file_type": "TEST_FILE", "path": "test.json"}], 200)
+    elif args[0] == 'https://dockstore.org/api/api/ga4gh/v2/tools/quay.io%2Fbriandoconnor%2Fdockstore-tool-md5sum/versions/1.0.4/plain-CWL/descriptor/Dockstore.cwl':
+        string = open(get_data("tests/trs/Dockstore.cwl"), "r").read()
+        return MockResponse(string, 200)
+    elif args[0] == 'https://dockstore.org/api/api/ga4gh/v2/tools/%23workflow%2Fgithub.com%2Fdockstore-testing%2Fmd5sum-checker/versions/develop/plain-CWL/descriptor/md5sum-tool.cwl':
+        string = open(get_data("tests/trs/md5sum-tool.cwl"), "r").read()
+        return MockResponse(string, 200)
+    elif args[0] == 'https://dockstore.org/api/api/ga4gh/v2/tools/%23workflow%2Fgithub.com%2Fdockstore-testing%2Fmd5sum-checker/versions/develop/plain-CWL/descriptor/md5sum-workflow.cwl':
+        string = open(get_data("tests/trs/md5sum-workflow.cwl"), "r").read()
+        return MockResponse(string, 200)
+    elif args[
+        0] == 'https://dockstore.org/api/api/ga4gh/v2/tools/%23workflow%2Fgithub.com%2Fdockstore-testing%2Fmd5sum-checker/versions/develop/CWL/files':
+        return MockResponse(
+            [{"file_type": "TEST_FILE", "path": "md5sum-input-cwl.json"}, {"file_type": "SECONDARY_DESCRIPTOR", "path": "md5sum-tool.cwl"},
+             {"file_type": "PRIMARY_DESCRIPTOR", "path": "md5sum-workflow.cwl"}], 200)
+
+    print ("A mocked call to TRS missed, target was %s", args[0])
+    return MockResponse(None, 404)
+
+
+@mock.patch('requests.Session.head', side_effect=mocked_requests_head)
+@mock.patch('requests.Session.get', side_effect=mocked_requests_get)
+def test_tool_trs_template(mock_head, mock_get):
+    params = ["--make-template", r"quay.io/briandoconnor/dockstore-tool-md5sum:1.0.4"]
+    return_value = main(params)
+    mock_head.assert_called()
+    mock_get.assert_called()
+    assert return_value == 0
+
+
+@mock.patch('requests.Session.head', side_effect=mocked_requests_head)
+@mock.patch('requests.Session.get', side_effect=mocked_requests_get)
+def test_workflow_trs_template(mock_head, mock_get):
+    params = ["--make-template", r"#workflow/github.com/dockstore-testing/md5sum-checker:develop"]
+    return_value = main(params)
+    mock_head.assert_called()
+    mock_get.assert_called()
+    assert return_value == 0

tests/trs/Dockstore.cwl

@@ -0,0 +1,50 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+id: Md5sum
+label: Simple md5sum tool
+cwlVersion: v1.0
+
+$namespaces:
+  dct: http://purl.org/dc/terms/
+  foaf: http://xmlns.com/foaf/0.1/
+
+doc: |
+  [![Docker Repository on Quay.io](https://quay.io/repository/briandoconnor/dockstore-tool-md5sum/status "Docker Repository on Quay.io")](https://quay.io/repository/briandoconnor/dockstore-tool-md5sum)
+  [![Build Status](https://travis-ci.org/briandoconnor/dockstore-tool-md5sum.svg)](https://travis-ci.org/briandoconnor/dockstore-tool-md5sum)
+  A very, very simple Docker container for the md5sum command. See the [README](https://github.com/briandoconnor/dockstore-tool-md5sum/blob/master/README.md) for more information.
+
+
+#dct:creator:
+#  '@id': http://orcid.org/0000-0002-7681-6415
+#  foaf:name: Brian O'Connor
+#  foaf:mbox: briandoconnor@gmail.com
+
+requirements:
+- class: DockerRequirement
+  dockerPull: quay.io/briandoconnor/dockstore-tool-md5sum:1.0.4
+- class: InlineJavascriptRequirement
+
+hints:
+- class: ResourceRequirement
+  # The command really requires very little resources.
+  coresMin: 1
+  ramMin: 1024
+  outdirMin: 1024
+
+inputs:
+  input_file:
+    type: File
+    inputBinding:
+      position: 1
+    doc: The file that will have its md5sum calculated.
+
+outputs:
+  output_file:
+    type: File
+    format: http://edamontology.org/data_3671
+    outputBinding:
+      glob: md5sum.txt
+    doc: A text file that contains a single line that is the md5sum of the input file.
+
+baseCommand: [/bin/my_md5sum]