From 346b9ca3de119bcaec8c4efb942ce2492dcd9d7c Mon Sep 17 00:00:00 2001
From: mtaghiza <mtaghiza@jhu.edu>
Date: Tue, 24 Mar 2020 11:48:18 -0400
Subject: [PATCH 01/10] allow writing into Data Volume in job submission

---
 py2/SciServer/Jobs.py | 36 ++++++++++++++++++++++++++++++------
 py3/SciServer/Jobs.py | 34 +++++++++++++++++++++++++++++-----
 2 files changed, 59 insertions(+), 11 deletions(-)

diff --git a/py2/SciServer/Jobs.py b/py2/SciServer/Jobs.py
index 62b7364..07862b9 100644
--- a/py2/SciServer/Jobs.py
+++ b/py2/SciServer/Jobs.py
@@ -410,7 +410,10 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
         datVols = [];
         if dataVolumes is None:
             for vol in dockerComputeDomain.get('volumes'):
-                datVols.append({'name': vol.get('name')});
+                if 'write' in vol.get('allowedActions'):
+                    datVols.append({'name': vol.get('name'), 'needsWriteAccess': True});
+                else:
+                    datVols.append({'name': vol.get('name'), 'needsWriteAccess': False});
 
         else:
             for dVol in dataVolumes:
@@ -418,7 +421,16 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
                 for vol in dockerComputeDomain.get('volumes'):
                     if vol.get('name') == dVol.get('name'):
                         found = True;
-                        datVols.append({'name': vol.get('name')});
+                        if (dVol.get('needsWriteAccess')):
+                            if dVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'):
+                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True});
+                            else:
+                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False});
+                        else:
+                            if 'write' in vol.get('allowedActions'):
+                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True});
+                            else:
+                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False});
 
                 if not found:
                     raise Exception("Data volume '" + dVol.get('name') + "' not found within Compute domain")
@@ -507,9 +519,9 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
             for uVol in userVolumes:
                 found = False;
                 for vol in dockerComputeDomain.get('userVolumes'):
-                    if vol.get('name') == uVol.get('name'):
+                    if vol.get('name') == uVol.get('name') and vol.get('rootVolumeName') == uVol.get('rootVolumeName') and vol.get('owner') == uVol.get('owner'):
                         found = True;
-                        if (uVol.has_key('needsWriteAccess')):
+                        if (uVol.get('needsWriteAccess')):
                             if uVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'):
                                 uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True});
                             else:
@@ -526,7 +538,10 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
         datVols = [];
         if dataVolumes is None:
             for vol in dockerComputeDomain.get('volumes'):
-                datVols.append({'name': vol.get('name')});
+                if 'write' in vol.get('allowedActions'):
+                    datVols.append({'name': vol.get('name'), 'needsWriteAccess': True});
+                else:
+                    datVols.append({'name': vol.get('name'), 'needsWriteAccess': False});
 
         else:
             for dVol in dataVolumes:
@@ -534,7 +549,16 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
                 for vol in dockerComputeDomain.get('volumes'):
                     if vol.get('name') == dVol.get('name'):
                         found = True;
-                        datVols.append({'name': vol.get('name')});
+                        if (dVol.get('needsWriteAccess')):
+                            if dVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'):
+                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True});
+                            else:
+                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False});
+                        else:
+                            if 'write' in vol.get('allowedActions'):
+                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True});
+                            else:
+                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False});
 
                 if not found:
                     raise Exception("Data volume '" + dVol.get('name') + "' not found within Compute domain")
diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py
index 7a88684..b8c344a 100644
--- a/py3/SciServer/Jobs.py
+++ b/py3/SciServer/Jobs.py
@@ -410,7 +410,10 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
         datVols = [];
         if dataVolumes is None:
             for vol in dockerComputeDomain.get('volumes'):
-                datVols.append({'name': vol.get('name')});
+                if 'write' in vol.get('allowedActions'):
+                    datVols.append({'name': vol.get('name'), 'needsWriteAccess': True});
+                else:
+                    datVols.append({'name': vol.get('name'), 'needsWriteAccess': False});
 
         else:
             for dVol in dataVolumes:
@@ -418,7 +421,16 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
                 for vol in dockerComputeDomain.get('volumes'):
                     if vol.get('name') == dVol.get('name'):
                         found = True;
-                        datVols.append({'name': vol.get('name')});
+                        if (dVol.get('needsWriteAccess')):
+                            if dVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'):
+                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True});
+                            else:
+                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False});
+                        else:
+                            if 'write' in vol.get('allowedActions'):
+                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True});
+                            else:
+                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False});
 
                 if not found:
                     raise Exception("Data volume '" + dVol.get('name') + "' not found within Compute domain")
@@ -507,7 +519,7 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
             for uVol in userVolumes:
                 found = False;
                 for vol in dockerComputeDomain.get('userVolumes'):
-                    if vol.get('name') == uVol.get('name'):
+                    if vol.get('name') == uVol.get('name') and vol.get('rootVolumeName') == uVol.get('rootVolumeName') and vol.get('owner') == uVol.get('owner'):
                         found = True;
                         if (uVol.get('needsWriteAccess')):
                             if uVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'):
@@ -526,7 +538,10 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
         datVols = [];
         if dataVolumes is None:
             for vol in dockerComputeDomain.get('volumes'):
-                datVols.append({'name': vol.get('name')});
+                if 'write' in vol.get('allowedActions'):
+                    datVols.append({'name': vol.get('name'), 'needsWriteAccess': True});
+                else:
+                    datVols.append({'name': vol.get('name'), 'needsWriteAccess': False});
 
         else:
             for dVol in dataVolumes:
@@ -534,7 +549,16 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
                 for vol in dockerComputeDomain.get('volumes'):
                     if vol.get('name') == dVol.get('name'):
                         found = True;
-                        datVols.append({'name': vol.get('name')});
+                        if (dVol.get('needsWriteAccess')):
+                            if dVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'):
+                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True});
+                            else:
+                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False});
+                        else:
+                            if 'write' in vol.get('allowedActions'):
+                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True});
+                            else:
+                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False});
 
                 if not found:
                     raise Exception("Data volume '" + dVol.get('name') + "' not found within Compute domain")

From 59348ba1b8261cb43e3dc5a4e0508c5ee7e2b06b Mon Sep 17 00:00:00 2001
From: mtaghiza <mtaghiza@jhu.edu>
Date: Fri, 3 Apr 2020 08:39:08 -0400
Subject: [PATCH 02/10] add name to data volume definition in job submitting
 methods

---
 py2/SciServer/Jobs.py | 24 ++++++++++++------------
 py3/SciServer/Jobs.py | 24 ++++++++++++------------
 2 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/py2/SciServer/Jobs.py b/py2/SciServer/Jobs.py
index 07862b9..67b5b6d 100644
--- a/py2/SciServer/Jobs.py
+++ b/py2/SciServer/Jobs.py
@@ -411,9 +411,9 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
         if dataVolumes is None:
             for vol in dockerComputeDomain.get('volumes'):
                 if 'write' in vol.get('allowedActions'):
-                    datVols.append({'name': vol.get('name'), 'needsWriteAccess': True});
+                    datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                 else:
-                    datVols.append({'name': vol.get('name'), 'needsWriteAccess': False});
+                    datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
 
         else:
             for dVol in dataVolumes:
@@ -423,14 +423,14 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
                         found = True;
                         if (dVol.get('needsWriteAccess')):
                             if dVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'):
-                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True});
+                                datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                             else:
-                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False});
+                                datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
                         else:
                             if 'write' in vol.get('allowedActions'):
-                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True});
+                                datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                             else:
-                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False});
+                                datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
 
                 if not found:
                     raise Exception("Data volume '" + dVol.get('name') + "' not found within Compute domain")
@@ -539,9 +539,9 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
         if dataVolumes is None:
             for vol in dockerComputeDomain.get('volumes'):
                 if 'write' in vol.get('allowedActions'):
-                    datVols.append({'name': vol.get('name'), 'needsWriteAccess': True});
+                    datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                 else:
-                    datVols.append({'name': vol.get('name'), 'needsWriteAccess': False});
+                    datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
 
         else:
             for dVol in dataVolumes:
@@ -551,14 +551,14 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
                         found = True;
                         if (dVol.get('needsWriteAccess')):
                             if dVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'):
-                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True});
+                                datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                             else:
-                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False});
+                                datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
                         else:
                             if 'write' in vol.get('allowedActions'):
-                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True});
+                                datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                             else:
-                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False});
+                                datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
 
                 if not found:
                     raise Exception("Data volume '" + dVol.get('name') + "' not found within Compute domain")
diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py
index b8c344a..0992e2c 100644
--- a/py3/SciServer/Jobs.py
+++ b/py3/SciServer/Jobs.py
@@ -411,9 +411,9 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
         if dataVolumes is None:
             for vol in dockerComputeDomain.get('volumes'):
                 if 'write' in vol.get('allowedActions'):
-                    datVols.append({'name': vol.get('name'), 'needsWriteAccess': True});
+                    datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                 else:
-                    datVols.append({'name': vol.get('name'), 'needsWriteAccess': False});
+                    datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
 
         else:
             for dVol in dataVolumes:
@@ -423,14 +423,14 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
                         found = True;
                         if (dVol.get('needsWriteAccess')):
                             if dVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'):
-                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True});
+                                datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                             else:
-                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False});
+                                datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
                         else:
                             if 'write' in vol.get('allowedActions'):
-                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True});
+                                datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                             else:
-                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False});
+                                datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
 
                 if not found:
                     raise Exception("Data volume '" + dVol.get('name') + "' not found within Compute domain")
@@ -539,9 +539,9 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
         if dataVolumes is None:
             for vol in dockerComputeDomain.get('volumes'):
                 if 'write' in vol.get('allowedActions'):
-                    datVols.append({'name': vol.get('name'), 'needsWriteAccess': True});
+                    datVols.append({'id': vol.get('id'),'name': vol.get('name'), 'writable': True});
                 else:
-                    datVols.append({'name': vol.get('name'), 'needsWriteAccess': False});
+                    datVols.append({'id': vol.get('id'),'name': vol.get('name'), 'writable': False});
 
         else:
             for dVol in dataVolumes:
@@ -551,14 +551,14 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
                         found = True;
                         if (dVol.get('needsWriteAccess')):
                             if dVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'):
-                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True});
+                                datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                             else:
-                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False});
+                                datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
                         else:
                             if 'write' in vol.get('allowedActions'):
-                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True});
+                                datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                             else:
-                                datVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': False});
+                                datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
 
                 if not found:
                     raise Exception("Data volume '" + dVol.get('name') + "' not found within Compute domain")

From 804b3cecc1b69be4a53fb957b09b4c229de7a774 Mon Sep 17 00:00:00 2001
From: mtaghiza <mtaghiza@jhu.edu>
Date: Fri, 3 Apr 2020 09:33:32 -0400
Subject: [PATCH 03/10] fixed typo

---
 py2/SciServer/Jobs.py |  8 ++++----
 py3/SciServer/Jobs.py | 12 ++++++------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/py2/SciServer/Jobs.py b/py2/SciServer/Jobs.py
index 67b5b6d..7c6770c 100644
--- a/py2/SciServer/Jobs.py
+++ b/py2/SciServer/Jobs.py
@@ -421,8 +421,8 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
                 for vol in dockerComputeDomain.get('volumes'):
                     if vol.get('name') == dVol.get('name'):
                         found = True;
-                        if (dVol.get('needsWriteAccess')):
-                            if dVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'):
+                        if dVol.get('needsWriteAccess'):
+                            if dVol.get('needsWriteAccess') is True and 'write' in vol.get('allowedActions'):
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                             else:
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
@@ -549,8 +549,8 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
                 for vol in dockerComputeDomain.get('volumes'):
                     if vol.get('name') == dVol.get('name'):
                         found = True;
-                        if (dVol.get('needsWriteAccess')):
-                            if dVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'):
+                        if dVol.get('needsWriteAccess'):
+                            if dVol.get('needsWriteAccess') is True and 'write' in vol.get('allowedActions'):
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                             else:
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py
index 0992e2c..7edea70 100644
--- a/py3/SciServer/Jobs.py
+++ b/py3/SciServer/Jobs.py
@@ -421,8 +421,8 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
                 for vol in dockerComputeDomain.get('volumes'):
                     if vol.get('name') == dVol.get('name'):
                         found = True;
-                        if (dVol.get('needsWriteAccess')):
-                            if dVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'):
+                        if dVol.get('needsWriteAccess'):
+                            if dVol.get('needsWriteAccess') is True and 'write' in vol.get('allowedActions'):
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                             else:
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
@@ -539,9 +539,9 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
         if dataVolumes is None:
             for vol in dockerComputeDomain.get('volumes'):
                 if 'write' in vol.get('allowedActions'):
-                    datVols.append({'id': vol.get('id'),'name': vol.get('name'), 'writable': True});
+                    datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                 else:
-                    datVols.append({'id': vol.get('id'),'name': vol.get('name'), 'writable': False});
+                    datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
 
         else:
             for dVol in dataVolumes:
@@ -549,8 +549,8 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
                 for vol in dockerComputeDomain.get('volumes'):
                     if vol.get('name') == dVol.get('name'):
                         found = True;
-                        if (dVol.get('needsWriteAccess')):
-                            if dVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'):
+                        if dVol.get('needsWriteAccess'):
+                            if dVol.get('needsWriteAccess') is True and 'write' in vol.get('allowedActions'):
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                             else:
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});

From b86912008988245a74ea2511e106699e4f9d7279 Mon Sep 17 00:00:00 2001
From: mtaghiza <mtaghiza@jhu.edu>
Date: Tue, 21 Apr 2020 17:36:45 -0400
Subject: [PATCH 04/10] fixed documentation, open jobs, and allowed actions
 checks

---
 py2/SciServer/Jobs.py | 20 +++++++++-----------
 py3/SciServer/Jobs.py | 22 +++++++++++-----------
 2 files changed, 20 insertions(+), 22 deletions(-)

diff --git a/py2/SciServer/Jobs.py b/py2/SciServer/Jobs.py
index 7c6770c..14ff4dc 100644
--- a/py2/SciServer/Jobs.py
+++ b/py2/SciServer/Jobs.py
@@ -210,7 +210,7 @@ def getJobsList(top=10, open=None, start=None, end=None, type='all'):
         if(type=='docker'):
             url = Config.RacmApiURL + "/jobm/rest/dockerjobs?"
 
-        url = url + topString + startString + endString + "TaskName=" + taskName;
+        url = url + topString + startString + endString + openString  + "TaskName=" + taskName;
 
         headers = {'X-Auth-Token': token, "Content-Type": "application/json"}
         res = requests.get(url, headers=headers, stream=True)
@@ -345,7 +345,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
     :param dockerComputeDomain: object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains().
     :param dockerImageName: name (string) of the Docker image for executing the notebook. E.g.,  dockerImageName="Python (astro)". An array of available Docker images is defined as the 'images' property in the dockerComputeDomain object.
     :param userVolumes: a list with the names of user volumes (with optional write permissions) that will be mounted to the docker Image. E.g.: userVolumes = [{'name':'persistent', 'needsWriteAccess':False},{'name':'scratch', , 'needsWriteAccess':True}] . A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes=None, then all available user volumes are mounted, with 'needsWriteAccess' = True if the user has Write permissions on the volume.
-    :param dataVolumes: a list with the names of data volumes that will be mounted to the docker Image. E.g.: dataVolumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}]. A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted.
+    :param dataVolumes: a list with the names of data volumes (with optional write permissions) that will be mounted to the docker Image. E.g.: dataVolumes=[{"name":"SDSS_DAS", 'needsWriteAccess':False}, {"name":"Recount"}]. A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted.
     :param resultsFolderPath: full path to results folder (string) where the original notebook is copied to and executed. E.g.: /home/idies/workspace/rootVolume/username/userVolume/jobsFolder. If not set, then a default folder will be set automatically.
     :param parameters: string containing parameters that the notebook might need during its execution. This string is written in the 'parameters.txt' file in the same directory level where the notebook is being executed.
     :param jobAlias: alias (string) of job, defined by the user.
@@ -410,7 +410,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
         datVols = [];
         if dataVolumes is None:
             for vol in dockerComputeDomain.get('volumes'):
-                if 'write' in vol.get('allowedActions'):
+                if vol.get('writable') is True:
                     datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                 else:
                     datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
@@ -422,12 +422,12 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
                     if vol.get('name') == dVol.get('name'):
                         found = True;
                         if dVol.get('needsWriteAccess'):
-                            if dVol.get('needsWriteAccess') is True and 'write' in vol.get('allowedActions'):
+                            if dVol.get('needsWriteAccess') is True:
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                             else:
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
                         else:
-                            if 'write' in vol.get('allowedActions'):
+                            if vol.get('writable') is True:
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                             else:
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
@@ -471,9 +471,7 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
     :param userVolumes: a list with the names of user volumes (with optional write permissions) that will be mounted to the docker Image.
            E.g., userVolumes = [{'name':'persistent', 'needsWriteAccess':False},{'name':'scratch', , 'needsWriteAccess':True}]
            A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes=None, then all available user volumes are mounted, with 'needsWriteAccess' = True if the user has Write permissions on the volume.
-    :param dataVolumes: a list with the names of data volumes that will be mounted to the docker Image.
-           E.g., dataVolumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}].
-           A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted.
+    :param dataVolumes: a list with the names of data volumes (with optional write permissions) that will be mounted to the docker Image. E.g.: dataVolumes=[{"name":"SDSS_DAS", 'needsWriteAccess':False}, {"name":"Recount"}]. A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted.
     :param resultsFolderPath: full path to results folder (string) where the shell command is executed. E.g.: /home/idies/workspace/rootVolume/username/userVolume/jobsFolder. If not set, then a default folder will be set automatically.
     :param jobAlias: alias (string) of job, defined by the user.
     :return: the job ID (int)
@@ -538,7 +536,7 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
         datVols = [];
         if dataVolumes is None:
             for vol in dockerComputeDomain.get('volumes'):
-                if 'write' in vol.get('allowedActions'):
+                if vol.get('writable') is True:
                     datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                 else:
                     datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
@@ -550,12 +548,12 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
                     if vol.get('name') == dVol.get('name'):
                         found = True;
                         if dVol.get('needsWriteAccess'):
-                            if dVol.get('needsWriteAccess') is True and 'write' in vol.get('allowedActions'):
+                            if dVol.get('needsWriteAccess') is True:
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                             else:
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
                         else:
-                            if 'write' in vol.get('allowedActions'):
+                            if vol.get('writable') is True:
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                             else:
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py
index 7edea70..e46d610 100644
--- a/py3/SciServer/Jobs.py
+++ b/py3/SciServer/Jobs.py
@@ -210,7 +210,7 @@ def getJobsList(top=10, open=None, start=None, end=None, type='all'):
         if(type=='docker'):
             url = Config.RacmApiURL + "/jobm/rest/dockerjobs?"
 
-        url = url + topString + startString + endString + "TaskName=" + taskName;
+        url = url + topString + startString + endString + openString + "TaskName=" + taskName;
 
         headers = {'X-Auth-Token': token, "Content-Type": "application/json"}
         res = requests.get(url, headers=headers, stream=True)
@@ -345,7 +345,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
     :param dockerComputeDomain: object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains().
     :param dockerImageName: name (string) of the Docker image for executing the notebook. E.g.,  dockerImageName="Python (astro)". An array of available Docker images is defined as the 'images' property in the dockerComputeDomain object.
     :param userVolumes: a list with the names of user volumes (with optional write permissions) that will be mounted to the docker Image. E.g.: userVolumes = [{'name':'persistent', 'needsWriteAccess':False},{'name':'scratch', , 'needsWriteAccess':True}] . A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes=None, then all available user volumes are mounted, with 'needsWriteAccess' = True if the user has Write permissions on the volume.
-    :param dataVolumes: a list with the names of data volumes that will be mounted to the docker Image. E.g.: dataVolumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}]. A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted.
+    :param dataVolumes: a list with the names of data volumes (with optional write permissions) that will be mounted to the docker Image. E.g.: dataVolumes=[{"name":"SDSS_DAS", 'needsWriteAccess':False}, {"name":"Recount"}]. A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted.
     :param resultsFolderPath: full path to results folder (string) where the original notebook is copied to and executed. E.g.: /home/idies/workspace/rootVolume/username/userVolume/jobsFolder. If not set, then a default folder will be set automatically.
     :param parameters: string containing parameters that the notebook might need during its execution. This string is written in the 'parameters.txt' file in the same directory level where the notebook is being executed.
     :param jobAlias: alias (string) of job, defined by the user.
@@ -410,7 +410,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
         datVols = [];
         if dataVolumes is None:
             for vol in dockerComputeDomain.get('volumes'):
-                if 'write' in vol.get('allowedActions'):
+                if vol.get('writable') is True:
                     datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                 else:
                     datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
@@ -422,12 +422,12 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
                     if vol.get('name') == dVol.get('name'):
                         found = True;
                         if dVol.get('needsWriteAccess'):
-                            if dVol.get('needsWriteAccess') is True and 'write' in vol.get('allowedActions'):
+                            if dVol.get('needsWriteAccess') is True:
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                             else:
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
                         else:
-                            if 'write' in vol.get('allowedActions'):
+                            if vol.get('writable') is True:
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                             else:
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
@@ -469,10 +469,10 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
     :param dockerComputeDomain: object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains().
     :param dockerImageName: name (string) of the Docker image for executing the notebook. E.g.,  dockerImageName="Python (astro)". An array of available Docker images is defined as the 'images' property in the dockerComputeDomain object.
     :param userVolumes: a list with the names of user volumes (with optional write permissions) that will be mounted to the docker Image.
-           E.g., userVolumes = [{'name':'persistent', 'needsWriteAccess':False},{'name':'scratch', , 'needsWriteAccess':True}]
+           E.g., userVolumes = [{'name':'persistent', 'needsWriteAccess':False},{'name':'scratch', 'needsWriteAccess':True}]
            A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes=None, then all available user volumes are mounted, with 'needsWriteAccess' = True if the user has Write permissions on the volume.
-    :param dataVolumes: a list with the names of data volumes that will be mounted to the docker Image.
-           E.g., dataVolumes=[{"name":"SDSS_DAS"}, {"name":"Recount"}].
+    :param dataVolumes: a list with the names of data volumes (with optional write permissions) that will be mounted to the docker Image.
+           E.g., dataVolumes=[{"name":"SDSS_DAS", 'needsWriteAccess':False}, {"name":"Recount"}].
            A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted.
     :param resultsFolderPath: full path to results folder (string) where the shell command is executed. E.g.: /home/idies/workspace/rootVolume/username/userVolume/jobsFolder. If not set, then a default folder will be set automatically.
     :param jobAlias: alias (string) of job, defined by the user.
@@ -538,7 +538,7 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
         datVols = [];
         if dataVolumes is None:
             for vol in dockerComputeDomain.get('volumes'):
-                if 'write' in vol.get('allowedActions'):
+                if vol.get('writable') is True:
                     datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                 else:
                     datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
@@ -550,12 +550,12 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
                     if vol.get('name') == dVol.get('name'):
                         found = True;
                         if dVol.get('needsWriteAccess'):
-                            if dVol.get('needsWriteAccess') is True and 'write' in vol.get('allowedActions'):
+                            if dVol.get('needsWriteAccess') is True:
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                             else:
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
                         else:
-                            if 'write' in vol.get('allowedActions'):
+                            if vol.get('writable') is True:
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                             else:
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});

From 9d0f19a86e8a49b7b87ab2cc49c974e87333eae4 Mon Sep 17 00:00:00 2001
From: mtaghiza <mtaghiza@jhu.edu>
Date: Tue, 12 May 2020 11:45:59 -0400
Subject: [PATCH 05/10] added getJobQueues function, and improved way of
 setting/defining result outputs types for rdb domain queries.

---
 py2/SciServer/Jobs.py | 109 ++++++++++++++++++++++++++++++++++--------
 py3/SciServer/Jobs.py | 101 +++++++++++++++++++++++++++++++-------
 2 files changed, 173 insertions(+), 37 deletions(-)

diff --git a/py2/SciServer/Jobs.py b/py2/SciServer/Jobs.py
index 14ff4dc..a2567f2 100644
--- a/py2/SciServer/Jobs.py
+++ b/py2/SciServer/Jobs.py
@@ -10,6 +10,29 @@
 import pandas as pd
 
 
+class QueryOutputType:
+    FILE_CSV = "FILE_CSV"
+    FILE_TSV = "FILE_TSV"
+    FILE_BSV = "FILE_BSV"
+    FILE_JSON = "FILE_JSON"
+    FILE_FITS = "FILE_FITS"
+    FILE_VOTABLE = "FILE_VOTABLE"
+    TABLE = "TABLE"
+
+class QueryOutput:
+
+    def __init__(self):
+        self.outputs = []
+
+    def add_output(self, output_name="query_result.json", output_type = QueryOutputType.FILE_JSON, result_number=1):
+        if output_type not in [type for type in dir(Jobs.QueryOutputType) if not type.startswith("__")]:
+            raise Exception("Unsupported output_type " + str(output_type))
+        self.outputs.append({'location':output_name, 'type':output_type , 'resultNumber':result_number})
+
+    def get_outputs(self):
+        return self.outputs
+
+
 
 def getDockerComputeDomains():
     """
@@ -210,7 +233,7 @@ def getJobsList(top=10, open=None, start=None, end=None, type='all'):
         if(type=='docker'):
             url = Config.RacmApiURL + "/jobm/rest/dockerjobs?"
 
-        url = url + topString + startString + endString + openString  + "TaskName=" + taskName;
+        url = url + topString + startString + endString + openString + "TaskName=" + taskName;
 
         headers = {'X-Auth-Token': token, "Content-Type": "application/json"}
         res = requests.get(url, headers=headers, stream=True)
@@ -273,6 +296,54 @@ def getDockerJobsListQuick(top=10, open=None, start=None, end=None, labelReg=Non
 
 
 
+def getJobQueues(returnType="pandas"):
+    """
+    Gets information about queues of jobs submitted to all compute domains, including the ranking jobs that users have
+    already submitted to the queue for execution, as well as the ranking that a new job would get if submitted to a queue.
+    :param returnType: if set To "pandas" (default setting), then it returns a pandas dataframe. Else, it will return a dictionary object.
+    :return: a pandas dataframe by default, with each row containing information about a submitted job. See the 'returnType' parameter for other return options.
+    The resultset columns are: the name of the compute domain, the type or class of compute domain, the queue Id
+    (that uniquely identifies the compute domain), the ranking of a job in the queue (smaller ranking has higher priority),
+    the name of the user owner of the job (names other than the requestor's are hashed as a number), the job status
+    (rows with status=0 are placeholders to see where a newly submitted job to the queue would be placed in the ranking),
+    job submission time, and job start time.
+    :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error.
+    :example: queues = Jobs.getJobQueues();
+    .. seealso:: Jobs.getJobsList, Jobs.submitNotebookJob, Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob,
+    """
+    token = Authentication.getToken()
+    if token is not None and token != "":
+
+        if Config.isSciServerComputeEnvironment():
+            taskName = "Compute.SciScript-Python.Jobs.getJobQueues"
+        else:
+            taskName = "SciScript-Python.Jobs.getJobQueues"
+
+
+        url = Config.RacmApiURL + "/jobm/rest/jobs/queues?TaskName=" + taskName
+
+        headers = {'X-Auth-Token': token, "Content-Type": "application/json"}
+        res = requests.get(url, headers=headers, stream=True)
+
+        if res.status_code != 200:
+            raise Exception("Error when getting job queues from JOBM API.\nHttp Response from JOBM API returned status code " +
+                str(res.status_code) + ":\n" + res.content.decode());
+        j = json.loads(res.content.decode())
+        # change dict 'j' so it can be used to create a pandas dataframe
+        if returnType=="pandas":
+            j['data']=j.pop('rows')
+            j.pop('hrNames')
+            df=pd.read_json(json.dumps(j), orient='split')
+            df.fillna('',inplace=True)
+            return df
+        else:
+            return j
+    else:
+        raise Exception("User token is not defined. First log into SciServer.")
+
+
+
+
 def getJobDescription(jobId):
     """
     Gets the definition of the job,
@@ -469,9 +540,11 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
     :param dockerComputeDomain: object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains().
     :param dockerImageName: name (string) of the Docker image for executing the notebook. E.g.,  dockerImageName="Python (astro)". An array of available Docker images is defined as the 'images' property in the dockerComputeDomain object.
     :param userVolumes: a list with the names of user volumes (with optional write permissions) that will be mounted to the docker Image.
-           E.g., userVolumes = [{'name':'persistent', 'needsWriteAccess':False},{'name':'scratch', , 'needsWriteAccess':True}]
+           E.g., userVolumes = [{'name':'persistent', 'needsWriteAccess':False},{'name':'scratch', 'needsWriteAccess':True}]
            A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes=None, then all available user volumes are mounted, with 'needsWriteAccess' = True if the user has Write permissions on the volume.
-    :param dataVolumes: a list with the names of data volumes (with optional write permissions) that will be mounted to the docker Image. E.g.: dataVolumes=[{"name":"SDSS_DAS", 'needsWriteAccess':False}, {"name":"Recount"}]. A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted.
+    :param dataVolumes: a list with the names of data volumes (with optional write permissions) that will be mounted to the docker Image.
+           E.g., dataVolumes=[{"name":"SDSS_DAS", 'needsWriteAccess':False}, {"name":"Recount"}].
+           A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted.
     :param resultsFolderPath: full path to results folder (string) where the shell command is executed. E.g.: /home/idies/workspace/rootVolume/username/userVolume/jobsFolder. If not set, then a default folder will be set automatically.
     :param jobAlias: alias (string) of job, defined by the user.
     :return: the job ID (int)
@@ -585,14 +658,14 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
     else:
         raise Exception("User token is not defined. First log into SciServer.")
 
-def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = None, resultsName='queryResults', resultsFolderPath="", jobAlias = ""):
+def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = None, query_output = None, resultsFolderPath="", jobAlias = ""):
     """
     Submits a sql query for execution (as an asynchronous job) inside a relational database (RDB) compute domain.
 
-    :param sqlQuery: sql query (string)
+    :param sqlQuery: sql query statement (string)
     :param rdbComputeDomain: object (dictionary) that defines a relational database (RDB) compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains().
     :param databaseContextName: database context name (string) on which the sql query is executed.
-    :param resultsName: name (string) of the table or file (without file type ending) that contains the query result. In case the sql query has multiple statements, should be set to a list of names (e.g., ['result1','result2']).
+    :param query_output: if a string, then it is the name of the json file that contains the query output result. Otherwise, it is a parameter of class QueryOutput, which contains the definitions of the output result types for each each subquery in the sql query statement.
     :param resultsFolderPath: full path to results folder (string) where query output tables are written into. E.g.: /home/idies/workspace/rootVOlume/username/userVolume/jobsFolder . If not set, then a default folder will be set automatically.
     :param jobAlias: alias (string) of job, defined by the user.
     :return: a dictionary containing the definition of the submitted job.
@@ -624,21 +697,17 @@ def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = Non
             else:
                 raise Exception("rbdComputeDomain has no database contexts available for the user.");
 
-        targets = [];
-        if type(resultsName) == str:
-            targets.append({'location': resultsName, 'type': 'FILE_CSV', 'resultNumber': 1});
-        elif type(resultsName) == list:
-            if len(set(resultsName)) != len(resultsName):
-                raise Exception("Elements of parameter 'resultsName' must be unique");
-
-            for i in range(len(resultsName)):
-                if type(resultsName[i]) == str:
-                    targets.append({'location': resultsName[i], 'type': 'FILE_CSV', 'resultNumber': i+1});
-                else:
-                    raise Exception("Elements of array 'resultsName' are not strings");
-
+        if type(query_output) == str or query_output is None:
+            targets = QueryOutput()
+            if type(query_output) == str:
+                name = query_output
+            else:
+                name = "queryResult.json"
+            targets.add_output(output_name=name, output_type=QueryOutputType.FILE_JSON, result_number=1)
+        elif type(query_output) == QueryOutput:
+            targets = query_output
         else:
-            raise Exception("Type of parameter 'resultsName' is not supported");
+            raise Exception("Unsupported query_output type " + str(type(query_output)))
 
 
         rdbDomainId = rdbComputeDomain.get('id');
diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py
index e46d610..9bea1f8 100644
--- a/py3/SciServer/Jobs.py
+++ b/py3/SciServer/Jobs.py
@@ -10,6 +10,29 @@
 import pandas as pd
 
 
+class QueryOutputType:
+    FILE_CSV = "FILE_CSV"
+    FILE_TSV = "FILE_TSV"
+    FILE_BSV = "FILE_BSV"
+    FILE_JSON = "FILE_JSON"
+    FILE_FITS = "FILE_FITS"
+    FILE_VOTABLE = "FILE_VOTABLE"
+    TABLE = "TABLE"
+
+class QueryOutput:
+
+    def __init__(self):
+        self.outputs = []
+
+    def add_output(self, output_name="query_result.json", output_type = QueryOutputType.FILE_JSON, result_number=1):
+        if output_type not in [type for type in dir(Jobs.QueryOutputType) if not type.startswith("__")]:
+            raise Exception("Unsupported output_type " + str(output_type))
+        self.outputs.append({'location':output_name, 'type':output_type , 'resultNumber':result_number})
+
+    def get_outputs(self):
+        return self.outputs
+
+
 
 def getDockerComputeDomains():
     """
@@ -273,6 +296,54 @@ def getDockerJobsListQuick(top=10, open=None, start=None, end=None, labelReg=Non
 
 
 
+def getJobQueues(returnType="pandas"):
+    """
+    Gets information about queues of jobs submitted to all compute domains, including the ranking jobs that users have
+    already submitted to the queue for execution, as well as the ranking that a new job would get if submitted to a queue.
+    :param returnType: if set To "pandas" (default setting), then it returns a pandas dataframe. Else, it will return a dictionary object.
+    :return: a pandas dataframe by default, with each row containing information about a submitted job. See the 'returnType' parameter for other return options.
+    The resultset columns are: the name of the compute domain, the type or class of compute domain, the queue Id
+    (that uniquely identifies the compute domain), the ranking of a job in the queue (smaller ranking has higher priority),
+    the name of the user owner of the job (names other than the requestor's are hashed as a number), the job status
+    (rows with status=0 are placeholders to see where a newly submitted job to the queue would be placed in the ranking),
+    job submission time, and job start time.
+    :raises: Throws an exception if the HTTP request to the Authentication URL returns an error, and if the HTTP request to the JOBM API returns an error.
+    :example: queues = Jobs.getJobQueues();
+    .. seealso:: Jobs.getJobsList, Jobs.submitNotebookJob, Jobs.submitShellCommandJob, Jobs.getJobStatus, Jobs.getDockerComputeDomains, Jobs.cancelJob,
+    """
+    token = Authentication.getToken()
+    if token is not None and token != "":
+
+        if Config.isSciServerComputeEnvironment():
+            taskName = "Compute.SciScript-Python.Jobs.getJobQueues"
+        else:
+            taskName = "SciScript-Python.Jobs.getJobQueues"
+
+
+        url = Config.RacmApiURL + "/jobm/rest/jobs/queues?TaskName=" + taskName
+
+        headers = {'X-Auth-Token': token, "Content-Type": "application/json"}
+        res = requests.get(url, headers=headers, stream=True)
+
+        if res.status_code != 200:
+            raise Exception("Error when getting job queues from JOBM API.\nHttp Response from JOBM API returned status code " +
+                str(res.status_code) + ":\n" + res.content.decode());
+        j = json.loads(res.content.decode())
+        # change dict 'j' so it can be used to create a pandas dataframe
+        if returnType=="pandas":
+            j['data']=j.pop('rows')
+            j.pop('hrNames')
+            df=pd.read_json(json.dumps(j), orient='split')
+            df.fillna('',inplace=True)
+            return df
+        else:
+            return j
+    else:
+        raise Exception("User token is not defined. First log into SciServer.")
+
+
+
+
 def getJobDescription(jobId):
     """
     Gets the definition of the job,
@@ -587,14 +658,14 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
     else:
         raise Exception("User token is not defined. First log into SciServer.")
 
-def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = None, resultsName='queryResults', resultsFolderPath="", jobAlias = ""):
+def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = None, query_output = None, resultsFolderPath="", jobAlias = ""):
     """
     Submits a sql query for execution (as an asynchronous job) inside a relational database (RDB) compute domain.
 
-    :param sqlQuery: sql query (string)
+    :param sqlQuery: sql query statement (string)
     :param rdbComputeDomain: object (dictionary) that defines a relational database (RDB) compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getRDBComputeDomains().
     :param databaseContextName: database context name (string) on which the sql query is executed.
-    :param resultsName: name (string) of the table or file (without file type ending) that contains the query result. In case the sql query has multiple statements, should be set to a list of names (e.g., ['result1','result2']).
+    :param query_output: if a string, then it is the name of the json file that contains the query output result. Otherwise, it is a parameter of class QueryOutput, which contains the definitions of the output result types for each each subquery in the sql query statement.
     :param resultsFolderPath: full path to results folder (string) where query output tables are written into. E.g.: /home/idies/workspace/rootVOlume/username/userVolume/jobsFolder . If not set, then a default folder will be set automatically.
     :param jobAlias: alias (string) of job, defined by the user.
     :return: a dictionary containing the definition of the submitted job.
@@ -626,21 +697,17 @@ def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = Non
             else:
                 raise Exception("rbdComputeDomain has no database contexts available for the user.");
 
-        targets = [];
-        if type(resultsName) == str:
-            targets.append({'location': resultsName, 'type': 'FILE_CSV', 'resultNumber': 1});
-        elif type(resultsName) == list:
-            if len(set(resultsName)) != len(resultsName):
-                raise Exception("Elements of parameter 'resultsName' must be unique");
-
-            for i in range(len(resultsName)):
-                if type(resultsName[i]) == str:
-                    targets.append({'location': resultsName[i], 'type': 'FILE_CSV', 'resultNumber': i+1});
-                else:
-                    raise Exception("Elements of array 'resultsName' are not strings");
-
+        if type(query_output) == str or query_output is None:
+            targets = QueryOutput()
+            if type(query_output) == str:
+                name = query_output
+            else:
+                name = "queryResult.json"
+            targets.add_output(output_name=name, output_type=QueryOutputType.FILE_JSON, result_number=1)
+        elif type(query_output) == QueryOutput:
+            targets = query_output
         else:
-            raise Exception("Type of parameter 'resultsName' is not supported");
+            raise Exception("Unsupported query_output type " + str(type(query_output)))
 
 
         rdbDomainId = rdbComputeDomain.get('id');

From 127fcac669a691f961ab7fa543b2d225f6bafb2b Mon Sep 17 00:00:00 2001
From: mtaghiza <mtaghiza@jhu.edu>
Date: Wed, 13 May 2020 16:21:18 -0400
Subject: [PATCH 06/10] small fix

---
 py2/SciServer/Jobs.py | 8 ++++----
 py3/SciServer/Jobs.py | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/py2/SciServer/Jobs.py b/py2/SciServer/Jobs.py
index a2567f2..d89367b 100644
--- a/py2/SciServer/Jobs.py
+++ b/py2/SciServer/Jobs.py
@@ -464,7 +464,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
                 for vol in dockerComputeDomain.get('userVolumes'):
                     if vol.get('name') == uVol.get('name') and vol.get('rootVolumeName') == uVol.get('rootVolumeName') and vol.get('owner') == uVol.get('owner'):
                         found = True;
-                        if (uVol.has_key('needsWriteAccess')):
+                        if uVol.get('needsWriteAccess') is not None:
                             if uVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'):
                                 uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True});
                             else:
@@ -492,7 +492,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
                 for vol in dockerComputeDomain.get('volumes'):
                     if vol.get('name') == dVol.get('name'):
                         found = True;
-                        if dVol.get('needsWriteAccess'):
+                        if dVol.get('needsWriteAccess') is not None:
                             if dVol.get('needsWriteAccess') is True:
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                             else:
@@ -592,7 +592,7 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
                 for vol in dockerComputeDomain.get('userVolumes'):
                     if vol.get('name') == uVol.get('name') and vol.get('rootVolumeName') == uVol.get('rootVolumeName') and vol.get('owner') == uVol.get('owner'):
                         found = True;
-                        if (uVol.get('needsWriteAccess')):
+                        if uVol.get('needsWriteAccess') is not None:
                             if uVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'):
                                 uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True});
                             else:
@@ -620,7 +620,7 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
                 for vol in dockerComputeDomain.get('volumes'):
                     if vol.get('name') == dVol.get('name'):
                         found = True;
-                        if dVol.get('needsWriteAccess'):
+                        if dVol.get('needsWriteAccess') is not None:
                             if dVol.get('needsWriteAccess') is True:
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                             else:
diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py
index 9bea1f8..806db94 100644
--- a/py3/SciServer/Jobs.py
+++ b/py3/SciServer/Jobs.py
@@ -464,7 +464,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
                 for vol in dockerComputeDomain.get('userVolumes'):
                     if vol.get('name') == uVol.get('name') and vol.get('rootVolumeName') == uVol.get('rootVolumeName') and vol.get('owner') == uVol.get('owner'):
                         found = True;
-                        if (uVol.get('needsWriteAccess')):
+                        if uVol.get('needsWriteAccess') is not None:
                             if uVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'):
                                 uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True});
                             else:
@@ -492,7 +492,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
                 for vol in dockerComputeDomain.get('volumes'):
                     if vol.get('name') == dVol.get('name'):
                         found = True;
-                        if dVol.get('needsWriteAccess'):
+                        if dVol.get('needsWriteAccess') is not None:
                             if dVol.get('needsWriteAccess') is True:
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                             else:
@@ -592,7 +592,7 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
                 for vol in dockerComputeDomain.get('userVolumes'):
                     if vol.get('name') == uVol.get('name') and vol.get('rootVolumeName') == uVol.get('rootVolumeName') and vol.get('owner') == uVol.get('owner'):
                         found = True;
-                        if (uVol.get('needsWriteAccess')):
+                        if uVol.get('needsWriteAccess') is not None:
                             if uVol.get('needsWriteAccess') == True and 'write' in vol.get('allowedActions'):
                                 uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True});
                             else:
@@ -620,7 +620,7 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
                 for vol in dockerComputeDomain.get('volumes'):
                     if vol.get('name') == dVol.get('name'):
                         found = True;
-                        if dVol.get('needsWriteAccess'):
+                        if dVol.get('needsWriteAccess') is not None:
                             if dVol.get('needsWriteAccess') is True:
                                 datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
                             else:

From 4781501b80b01a3ac989fddd520e371ecacd2973 Mon Sep 17 00:00:00 2001
From: mtaghiza <mtaghiza@jhu.edu>
Date: Tue, 9 Mar 2021 14:36:51 -0500
Subject: [PATCH 07/10] allow including all data volumes and user volumes by
 specifying 'all' parameter value (but give data volumes only read
 permissions)

---
 py2/SciServer/Jobs.py | 26 ++++++++++----------------
 py3/SciServer/Jobs.py | 28 +++++++++++-----------------
 2 files changed, 21 insertions(+), 33 deletions(-)

diff --git a/py2/SciServer/Jobs.py b/py2/SciServer/Jobs.py
index d89367b..fbf3963 100644
--- a/py2/SciServer/Jobs.py
+++ b/py2/SciServer/Jobs.py
@@ -415,8 +415,8 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
     :param notebookPath: path of the notebook within the filesystem mounted in SciServer-Compute (string). Example: notebookPath = '/home/idies/worskpace/persistent/JupyterNotebook.ipynb'
     :param dockerComputeDomain: object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains().
     :param dockerImageName: name (string) of the Docker image for executing the notebook. E.g.,  dockerImageName="Python (astro)". An array of available Docker images is defined as the 'images' property in the dockerComputeDomain object.
-    :param userVolumes: a list with the names of user volumes (with optional write permissions) that will be mounted to the docker Image. E.g.: userVolumes = [{'name':'persistent', 'needsWriteAccess':False},{'name':'scratch', , 'needsWriteAccess':True}] . A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes=None, then all available user volumes are mounted, with 'needsWriteAccess' = True if the user has Write permissions on the volume.
-    :param dataVolumes: a list with the names of data volumes (with optional write permissions) that will be mounted to the docker Image. E.g.: dataVolumes=[{"name":"SDSS_DAS", 'needsWriteAccess':False}, {"name":"Recount"}]. A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted.
+    :param userVolumes: a list with the names of user volumes (with optional write permissions) that will be mounted to the docker Image. E.g.: userVolumes = [{'name':'persistent', 'needsWriteAccess':False},{'name':'scratch', , 'needsWriteAccess':True}] . A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes='all', then all available user volumes are mounted, with 'needsWriteAccess' = True if the user has Write permissions on the volume.
+    :param dataVolumes: a list with the names of data volumes (with optional write permissions) that will be mounted to the docker Image. E.g.: dataVolumes=[{"name":"SDSS_DAS", 'needsWriteAccess':False}, {"name":"Recount"}]. A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes='all', then all available data volumes are mounted (without write permissions).
     :param resultsFolderPath: full path to results folder (string) where the original notebook is copied to and executed. E.g.: /home/idies/workspace/rootVolume/username/userVolume/jobsFolder. If not set, then a default folder will be set automatically.
     :param parameters: string containing parameters that the notebook might need during its execution. This string is written in the 'parameters.txt' file in the same directory level where the notebook is being executed.
     :param jobAlias: alias (string) of job, defined by the user.
@@ -450,7 +450,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
                 raise Exception("dockerComputeDomain has no docker images available for the user.");
 
         uVols = [];
-        if userVolumes is None:
+        if userVolumes == 'all':
             for vol in dockerComputeDomain.get('userVolumes'):
                 if 'write' in vol.get('allowedActions'):
                     uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True});
@@ -479,12 +479,9 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
                     raise Exception("User volume '" + uVol.get('name') + "' not found within Compute domain")
 
         datVols = [];
-        if dataVolumes is None:
+        if dataVolumes == 'all':
             for vol in dockerComputeDomain.get('volumes'):
-                if vol.get('writable') is True:
-                    datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
-                else:
-                    datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
+                datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
 
         else:
             for dVol in dataVolumes:
@@ -541,10 +538,10 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
     :param dockerImageName: name (string) of the Docker image for executing the notebook. E.g.,  dockerImageName="Python (astro)". An array of available Docker images is defined as the 'images' property in the dockerComputeDomain object.
     :param userVolumes: a list with the names of user volumes (with optional write permissions) that will be mounted to the docker Image.
            E.g., userVolumes = [{'name':'persistent', 'needsWriteAccess':False},{'name':'scratch', 'needsWriteAccess':True}]
-           A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes=None, then all available user volumes are mounted, with 'needsWriteAccess' = True if the user has Write permissions on the volume.
+           A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes='all', then all available user volumes are mounted, with 'needsWriteAccess' = True if the user has Write permissions on the volume.
     :param dataVolumes: a list with the names of data volumes (with optional write permissions) that will be mounted to the docker Image.
            E.g., dataVolumes=[{"name":"SDSS_DAS", 'needsWriteAccess':False}, {"name":"Recount"}].
-           A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted.
+           A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes='all', then all available data volumes are mounted (without write permissions).
     :param resultsFolderPath: full path to results folder (string) where the shell command is executed. E.g.: /home/idies/workspace/rootVolume/username/userVolume/jobsFolder. If not set, then a default folder will be set automatically.
     :param jobAlias: alias (string) of job, defined by the user.
     :return: the job ID (int)
@@ -578,7 +575,7 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
                 raise Exception("dockerComputeDomain has no docker images available for the user.");
 
         uVols = [];
-        if userVolumes is None:
+        if userVolumes == 'all':
             for vol in dockerComputeDomain.get('userVolumes'):
                 if 'write' in vol.get('allowedActions'):
                     uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True});
@@ -607,12 +604,9 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
                     raise Exception("User volume '" + uVol.get('name') + "' not found within Compute domain")
 
         datVols = [];
-        if dataVolumes is None:
+        if dataVolumes == 'all':
             for vol in dockerComputeDomain.get('volumes'):
-                if vol.get('writable') is True:
-                    datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
-                else:
-                    datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
+                datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
 
         else:
             for dVol in dataVolumes:
diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py
index 806db94..bfcd881 100644
--- a/py3/SciServer/Jobs.py
+++ b/py3/SciServer/Jobs.py
@@ -408,15 +408,15 @@ def getJobStatus(jobId):
         raise Exception("Invalid integer value given to job status.")
 
 
-def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=None, userVolumes=None,  dataVolumes=None, resultsFolderPath="", parameters="", jobAlias= ""):
+def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=None, userVolumes='all',  dataVolumes='all', resultsFolderPath="", parameters="", jobAlias= ""):
     """
     Submits a Jupyter Notebook for execution (as an asynchronous job) inside a Docker compute domain.
 
     :param notebookPath: path of the notebook within the filesystem mounted in SciServer-Compute (string). Example: notebookPath = '/home/idies/worskpace/persistent/JupyterNotebook.ipynb'
     :param dockerComputeDomain: object (dictionary) that defines a Docker compute domain. A list of these kind of objects available to the user is returned by the function Jobs.getDockerComputeDomains().
     :param dockerImageName: name (string) of the Docker image for executing the notebook. E.g.,  dockerImageName="Python (astro)". An array of available Docker images is defined as the 'images' property in the dockerComputeDomain object.
-    :param userVolumes: a list with the names of user volumes (with optional write permissions) that will be mounted to the docker Image. E.g.: userVolumes = [{'name':'persistent', 'needsWriteAccess':False},{'name':'scratch', , 'needsWriteAccess':True}] . A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes=None, then all available user volumes are mounted, with 'needsWriteAccess' = True if the user has Write permissions on the volume.
-    :param dataVolumes: a list with the names of data volumes (with optional write permissions) that will be mounted to the docker Image. E.g.: dataVolumes=[{"name":"SDSS_DAS", 'needsWriteAccess':False}, {"name":"Recount"}]. A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted.
+    :param userVolumes: a list with the names of user volumes (with optional write permissions) that will be mounted to the docker Image. E.g.: userVolumes = [{'name':'persistent', 'needsWriteAccess':False},{'name':'scratch', , 'needsWriteAccess':True}] . A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes='all', then all available user volumes are mounted, with 'needsWriteAccess' = True if the user has Write permissions on the volume.
+    :param dataVolumes: a list with the names of data volumes (with optional write permissions) that will be mounted to the docker Image. E.g.: dataVolumes=[{"name":"SDSS_DAS", 'needsWriteAccess':False}, {"name":"Recount"}]. A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes='all', then all available data volumes are mounted (without write permissions).
     :param resultsFolderPath: full path to results folder (string) where the original notebook is copied to and executed. E.g.: /home/idies/workspace/rootVolume/username/userVolume/jobsFolder. If not set, then a default folder will be set automatically.
     :param parameters: string containing parameters that the notebook might need during its execution. This string is written in the 'parameters.txt' file in the same directory level where the notebook is being executed.
     :param jobAlias: alias (string) of job, defined by the user.
@@ -450,7 +450,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
                 raise Exception("dockerComputeDomain has no docker images available for the user.");
 
         uVols = [];
-        if userVolumes is None:
+        if userVolumes == 'all':
             for vol in dockerComputeDomain.get('userVolumes'):
                 if 'write' in vol.get('allowedActions'):
                     uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True});
@@ -479,12 +479,9 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
                     raise Exception("User volume '" + uVol.get('name') + "' not found within Compute domain")
 
         datVols = [];
-        if dataVolumes is None:
+        if dataVolumes == "all":
             for vol in dockerComputeDomain.get('volumes'):
-                if vol.get('writable') is True:
-                    datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
-                else:
-                    datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
+                datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
 
         else:
             for dVol in dataVolumes:
@@ -541,10 +538,10 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
     :param dockerImageName: name (string) of the Docker image for executing the notebook. E.g.,  dockerImageName="Python (astro)". An array of available Docker images is defined as the 'images' property in the dockerComputeDomain object.
     :param userVolumes: a list with the names of user volumes (with optional write permissions) that will be mounted to the docker Image.
            E.g., userVolumes = [{'name':'persistent', 'needsWriteAccess':False},{'name':'scratch', 'needsWriteAccess':True}]
-           A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes=None, then all available user volumes are mounted, with 'needsWriteAccess' = True if the user has Write permissions on the volume.
+           A list of available user volumes can be found as the 'userVolumes' property in the dockerComputeDomain object. If userVolumes='all', then all available user volumes are mounted, with 'needsWriteAccess' = True if the user has Write permissions on the volume.
     :param dataVolumes: a list with the names of data volumes (with optional write permissions) that will be mounted to the docker Image.
            E.g., dataVolumes=[{"name":"SDSS_DAS", 'needsWriteAccess':False}, {"name":"Recount"}].
-           A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes=None, then all available data volumes are mounted.
+           A list of available data volumes can be found as the 'volumes' property in the dockerComputeDomain object. If dataVolumes='all', then all available data volumes are mounted (without write permissions).
     :param resultsFolderPath: full path to results folder (string) where the shell command is executed. E.g.: /home/idies/workspace/rootVolume/username/userVolume/jobsFolder. If not set, then a default folder will be set automatically.
     :param jobAlias: alias (string) of job, defined by the user.
     :return: the job ID (int)
@@ -578,7 +575,7 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
                 raise Exception("dockerComputeDomain has no docker images available for the user.");
 
         uVols = [];
-        if userVolumes is None:
+        if userVolumes == 'all':
             for vol in dockerComputeDomain.get('userVolumes'):
                 if 'write' in vol.get('allowedActions'):
                     uVols.append({'userVolumeId': vol.get('id'), 'needsWriteAccess': True});
@@ -607,12 +604,9 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
                     raise Exception("User volume '" + uVol.get('name') + "' not found within Compute domain")
 
         datVols = [];
-        if dataVolumes is None:
+        if dataVolumes == "all":
             for vol in dockerComputeDomain.get('volumes'):
-                if vol.get('writable') is True:
-                    datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': True});
-                else:
-                    datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
+                datVols.append({'id': vol.get('id'), 'name': vol.get('name'), 'writable': False});
 
         else:
             for dVol in dataVolumes:

From 6db23714d4e8c0dd398181cb27d7e073c531d735 Mon Sep 17 00:00:00 2001
From: mtaghiza <mtaghiza@jhu.edu>
Date: Tue, 9 Mar 2021 15:40:12 -0500
Subject: [PATCH 08/10] raise exception with None values for user/data volumes
 in docker job submission.

---
 py2/SciServer/Jobs.py | 6 ++++++
 py3/SciServer/Jobs.py | 6 ++++++
 2 files changed, 12 insertions(+)

diff --git a/py2/SciServer/Jobs.py b/py2/SciServer/Jobs.py
index fbf3963..61f93c5 100644
--- a/py2/SciServer/Jobs.py
+++ b/py2/SciServer/Jobs.py
@@ -554,6 +554,9 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
     token = Authentication.getToken()
     if token is not None and token != "":
 
+        if userVolumes is None or dataVolumes is None:
+            raise ValueError("None value not supported for userVolumes or dataVolumes parameters. Use 'all' instead.")
+
         if Config.isSciServerComputeEnvironment():
             taskName = "Compute.SciScript-Python.Jobs.submitShellCommandJob"
         else:
@@ -672,6 +675,9 @@ def submitRDBQueryJob(sqlQuery, rdbComputeDomain=None, databaseContextName = Non
     token = Authentication.getToken()
     if token is not None and token != "":
 
+        if userVolumes is None or dataVolumes is None:
+            raise ValueError("None value not supported for userVolumes or dataVolumes parameters. Use 'all' instead.")
+
         if Config.isSciServerComputeEnvironment():
             taskName = "Compute.SciScript-Python.Jobs.submitRDBQueryJob"
         else:
diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py
index bfcd881..5a8a645 100644
--- a/py3/SciServer/Jobs.py
+++ b/py3/SciServer/Jobs.py
@@ -430,6 +430,9 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
     token = Authentication.getToken()
     if token is not None and token != "":
 
+        if userVolumes is None or dataVolumes is None:
+            raise ValueError("None value not supported for userVolumes or dataVolumes parameters. Use 'all' instead.")
+
         if Config.isSciServerComputeEnvironment():
             taskName = "Compute.SciScript-Python.Jobs.submitNotebookJob"
         else:
@@ -554,6 +557,9 @@ def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageN
     token = Authentication.getToken()
     if token is not None and token != "":
 
+        if userVolumes is None or dataVolumes is None:
+            raise ValueError("None value not supported for userVolumes or dataVolumes parameters. Use 'all' instead.")
+
         if Config.isSciServerComputeEnvironment():
             taskName = "Compute.SciScript-Python.Jobs.submitShellCommandJob"
         else:

From b2ad78c3e1975ff6ea4c2d7236b6a4e5c2a5ac6d Mon Sep 17 00:00:00 2001
From: mtaghiza <mtaghiza@jhu.edu>
Date: Tue, 9 Mar 2021 15:54:00 -0500
Subject: [PATCH 09/10] include 'all' option in missing remaining methods

---
 py2/SciServer/Jobs.py | 4 ++--
 py3/SciServer/Jobs.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/py2/SciServer/Jobs.py b/py2/SciServer/Jobs.py
index 61f93c5..15e8fd0 100644
--- a/py2/SciServer/Jobs.py
+++ b/py2/SciServer/Jobs.py
@@ -408,7 +408,7 @@ def getJobStatus(jobId):
         raise Exception("Invalid integer value given to job status.")
 
 
-def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=None, userVolumes=None,  dataVolumes=None, resultsFolderPath="", parameters="", jobAlias= ""):
+def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName='all', userVolumes='all',  dataVolumes=None, resultsFolderPath="", parameters="", jobAlias= ""):
     """
     Submits a Jupyter Notebook for execution (as an asynchronous job) inside a Docker compute domain.
 
@@ -529,7 +529,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
         raise Exception("User token is not defined. First log into SciServer.")
 
 
-def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageName = None, userVolumes = None, dataVolumes = None, resultsFolderPath = "", jobAlias = ""):
+def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageName = None, userVolumes = 'all', dataVolumes = 'all', resultsFolderPath = "", jobAlias = ""):
     """
     Submits a shell command for execution (as an asynchronous job) inside a Docker compute domain.
 
diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py
index 5a8a645..b72fe5a 100644
--- a/py3/SciServer/Jobs.py
+++ b/py3/SciServer/Jobs.py
@@ -532,7 +532,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
         raise Exception("User token is not defined. First log into SciServer.")
 
 
-def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageName = None, userVolumes = None, dataVolumes = None, resultsFolderPath = "", jobAlias = ""):
+def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageName = 'all', userVolumes = 'all', dataVolumes = None, resultsFolderPath = "", jobAlias = ""):
     """
     Submits a shell command for execution (as an asynchronous job) inside a Docker compute domain.
 

From ace43f8245cbc73db30fd7fa2d3b1feefa4e1312 Mon Sep 17 00:00:00 2001
From: mtaghiza <mtaghiza@jhu.edu>
Date: Tue, 9 Mar 2021 15:57:21 -0500
Subject: [PATCH 10/10] fifed 'all' assignement

---
 py2/SciServer/Jobs.py | 2 +-
 py3/SciServer/Jobs.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/py2/SciServer/Jobs.py b/py2/SciServer/Jobs.py
index 15e8fd0..d1731e6 100644
--- a/py2/SciServer/Jobs.py
+++ b/py2/SciServer/Jobs.py
@@ -408,7 +408,7 @@ def getJobStatus(jobId):
         raise Exception("Invalid integer value given to job status.")
 
 
-def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName='all', userVolumes='all',  dataVolumes=None, resultsFolderPath="", parameters="", jobAlias= ""):
+def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=None, userVolumes='all',  dataVolumes='all', resultsFolderPath="", parameters="", jobAlias= ""):
     """
     Submits a Jupyter Notebook for execution (as an asynchronous job) inside a Docker compute domain.
 
diff --git a/py3/SciServer/Jobs.py b/py3/SciServer/Jobs.py
index b72fe5a..ca296f8 100644
--- a/py3/SciServer/Jobs.py
+++ b/py3/SciServer/Jobs.py
@@ -532,7 +532,7 @@ def submitNotebookJob(notebookPath, dockerComputeDomain=None, dockerImageName=No
         raise Exception("User token is not defined. First log into SciServer.")
 
 
-def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageName = 'all', userVolumes = 'all', dataVolumes = None, resultsFolderPath = "", jobAlias = ""):
+def submitShellCommandJob(shellCommand, dockerComputeDomain = None, dockerImageName = None, userVolumes = 'all', dataVolumes = 'all', resultsFolderPath = "", jobAlias = ""):
     """
     Submits a shell command for execution (as an asynchronous job) inside a Docker compute domain.