test_integration also tests Arvados backend

Author: Peter Amstutz

test/test_integration.py

@@ -62,7 +62,7 @@ def test_dockstore_md5sum(self):
                                           workflow_attachment=self.cwl_attachments)
         state = self.wait_for_finish(run_id)
         self.check_complete(run_id)
-        self.assertTrue(check_for_file(outfile_path), 'Output file was not found: ' + str(outfile_path))
+        self.assertTrue(self.check_for_file(outfile_path), 'Output file was not found: ' + str(outfile_path))
 
     def test_local_md5sum(self):
         """LOCAL md5sum cwl to the wes-service server, and check for the correct output."""
@@ -71,7 +71,7 @@ def test_local_md5sum(self):
                                                workflow_attachment=self.cwl_attachments)
         state = self.wait_for_finish(run_id)
         self.check_complete(run_id)
-        self.assertTrue(check_for_file(outfile_path), 'Output file was not found: ' + str(outfile_path))
+        self.assertTrue(self.check_for_file(outfile_path), 'Output file was not found: ' + str(outfile_path))
 
     def test_run_attachments(self):
         """LOCAL md5sum cwl to the wes-service server, check for attachments."""
@@ -81,9 +81,9 @@ def test_run_attachments(self):
         get_response = self.client.get_run_log(run_id)["request"]
         state = self.wait_for_finish(run_id)
         self.check_complete(run_id)
-        self.assertTrue(check_for_file(outfile_path), 'Output file was not found: ' + get_response["workflow_attachment"])
+        self.assertTrue(self.check_for_file(outfile_path), 'Output file was not found: ' + get_response["workflow_attachment"])
         attachment_tool_path = get_response["workflow_attachment"][7:] + "/dockstore-tool-md5sum.cwl"
-        self.assertTrue(check_for_file(attachment_tool_path), 'Attachment file was not found: ' + get_response["workflow_attachment"])
+        self.assertTrue(self.check_for_file(attachment_tool_path), 'Attachment file was not found: ' + get_response["workflow_attachment"])
 
     def test_get_service_info(self):
         """
@@ -143,10 +143,21 @@ def check_complete(self, run_id):
         if s["state"] != "COMPLETE":
             logging.info(str(s["run_log"]["stderr"]))
             if str(s["run_log"]["stderr"]).startswith("http"):
-                logs = requests.get(s["run_log"]["stderr"], headers=auth).text
+                logs = requests.get(s["run_log"]["stderr"], headers=self.client.auth).text
                 logging.info("Run log:\n" + logs)
         assert s["state"] == "COMPLETE"
 
+    def check_for_file(self, filepath, seconds=120):
+        """Return True if a file exists within a certain amount of time."""
+        wait_counter = 0
+        while not os.path.exists(filepath):
+            time.sleep(1)
+            wait_counter += 1
+            if wait_counter > seconds:
+                return False
+        return True
+
+
 def get_server_pids():
     try:
         pids = subprocess.check_output(['pgrep', '-f', 'wes_service_main.py']).strip().split()
@@ -155,16 +166,6 @@ def get_server_pids():
     return pids
 
 
-def check_for_file(filepath, seconds=120):
-    """Return True if a file exists within a certain amount of time."""
-    wait_counter = 0
-    while not os.path.exists(filepath):
-        time.sleep(1)
-        wait_counter += 1
-        if wait_counter > seconds:
-            return False
-    return True
-
 
 class CwltoolTest(IntegrationTest):
     """Test using cwltool."""
@@ -205,8 +206,31 @@ def test_local_wdl(self):
             outfile_path, run_id = self.run_md5sum(wf_input=self.wdl_local_path,
                                                    json_input=self.wdl_json_input,
                                                    workflow_attachment=self.wdl_attachments)
-            self.assertTrue(check_for_file(outfile_path), 'Output file was not found: ' + str(outfile_path))
+            self.assertTrue(self.check_for_file(outfile_path), 'Output file was not found: ' + str(outfile_path))
+
+
+class ArvadosTest(IntegrationTest):
+    """Test using arvados-cwl-runner."""
+
+    def setUp(self):
+        """
+        Start a (local) wes-service server to make requests against.
+        Use arvados-cwl-runner as the wes-service server 'backend'.
+        Requires ARVADOS_API_HOST and ARVADOS_API_TOKEN to be set in the environment.
+        """
+        if os.path.exists('workflows'):
+            shutil.rmtree('workflows')
+        self.wes_server_process = subprocess.Popen(
+            ['python', os.path.abspath('wes_service/wes_service_main.py'),
+             '--backend=wes_service.arvados_wes',
+             '--port=8080',
+             '--debug'])
+        self.client.auth = {"Authorization": "Bearer " + os.environ["ARVADOS_API_TOKEN"]}
+        time.sleep(5)
 
+    def check_for_file(self, filepath, seconds=120):
+        # Doesn't make sense for arvados
+        return True
 
 # Prevent pytest/unittest's discovery from attempting to discover the base test class.
 del IntegrationTest

wes_service/arvados_wes.py

@@ -145,7 +145,7 @@ def invoke_cwl_runner(self, cr_uuid, workflow_url, workflow_params,
                 # a-c-r submission processes from seeing each other.
 
                 cmd = ["arvados-cwl-runner", "--submit-request-uuid="+cr_uuid,
-                       "--submit", "--no-wait", "--api=containers"]
+                       "--submit", "--no-wait", "--api=containers", "--debug"]
 
                 if project_uuid:
                     cmd.append("--project-uuid="+project_uuid)