Merge pull request #87 from mgermain/job_logs

Job logs

Author: MarcCote

scripts/smart_dispatch.py

@@ -99,7 +99,7 @@ def main():
                       'mem_per_command': None  # args.memPerCommand
                       }
 
-    job_generator = job_generator_factory(queue, commands, command_params, CLUSTER_NAME)
+    job_generator = job_generator_factory(queue, commands, command_params, CLUSTER_NAME, path_job)
     pbs_filenames = job_generator.write_pbs_files(path_job_commands)
 
     # Launch the jobs
@@ -175,6 +175,8 @@ def get_job_folders(jobname):
         os.makedirs(path_job_logs)
     if not os.path.exists(os.path.join(path_job_logs, "worker")):
         os.makedirs(os.path.join(path_job_logs, "worker"))
+    if not os.path.exists(os.path.join(path_job_logs, "job")):
+        os.makedirs(os.path.join(path_job_logs, "job"))
 
     return path_job, path_job_logs, path_job_commands
 
@@ -189,6 +191,7 @@ def create_job_folders(jobname):
     if not os.path.exists(path_job_logs):
         os.makedirs(path_job_logs)
         os.makedirs(os.path.join(path_job_logs, "worker"))
+        os.makedirs(os.path.join(path_job_logs, "job"))
 
     return path_job, path_job_logs, path_job_commands
 

smartdispatch/job_generator.py

@@ -6,15 +6,15 @@
 from smartdispatch import utils
 
 
-def job_generator_factory(queue, commands, command_params={}, cluster_name=None):
+def job_generator_factory(queue, commands, command_params={}, cluster_name=None, base_path="./"):
     if cluster_name == "guillimin":
-        return GuilliminJobGenerator(queue, commands, command_params)
+        return GuilliminJobGenerator(queue, commands, command_params, base_path)
     elif cluster_name == "mammouth":
-        return MammouthJobGenerator(queue, commands, command_params)
+        return MammouthJobGenerator(queue, commands, command_params, base_path)
     elif cluster_name == "helios":
-        return HeliosJobGenerator(queue, commands, command_params)
+        return HeliosJobGenerator(queue, commands, command_params, base_path)
 
-    return JobGenerator(queue, commands, command_params)
+    return JobGenerator(queue, commands, command_params, base_path)
 
 
 class JobGenerator(object):
@@ -31,9 +31,10 @@ class JobGenerator(object):
         information about the commands
     """
 
-    def __init__(self, queue, commands, command_params={}):
+    def __init__(self, queue, commands, command_params={}, base_path="./"):
         self.commands = commands
         self.queue = queue
+        self.job_log_filename = '"{base_path}/logs/job/"$PBS_JOBID".{{ext}}"'.format(base_path=base_path)
 
         self.nb_cores_per_command = command_params.get('nb_cores_per_command', 1)
         self.nb_gpus_per_command = command_params.get('nb_gpus_per_command', 1)
@@ -51,6 +52,9 @@ def generate_pbs(self):
         for i, commands in enumerate(utils.chunks(self.commands, n=nb_commands_per_node)):
             pbs = PBS(self.queue.name, self.queue.walltime)
 
+            # TODO Move the add_options into the JobManager once created.
+            pbs.add_options(o=self.job_log_filename.format(ext='out'), e=self.job_log_filename.format(ext='err'))
+
             # Set resource: nodes
             resource = "1:ppn={ppn}".format(ppn=len(commands) * self.nb_cores_per_command)
             if self.queue.nb_gpus_per_node > 0:
@@ -126,6 +130,6 @@ def generate_pbs(self):
             # Nb of GPUs has to be a multiple of 2
             nb_gpus = int(re.findall("gpus=([0-9]+)", pbs.resources['nodes'])[0])
             if nb_gpus % 2 != 0:
-                pbs.resources['nodes'] = re.sub("gpus=[0-9]+", "gpus={0}".format(nb_gpus+1), pbs.resources['nodes'])
+                pbs.resources['nodes'] = re.sub("gpus=[0-9]+", "gpus={0}".format(nb_gpus + 1), pbs.resources['nodes'])
 
         return pbs_list