Added unit tests for --commandsFile option.

Author: MarcCote

scripts/smart_dispatch.py

@@ -150,7 +150,8 @@ def parse_arguments():
     parser.add_argument('-c', '--coresPerCommand', type=int, required=False, help='How many cores a command needs.', default=1)
     parser.add_argument('-g', '--gpusPerCommand', type=int, required=False, help='How many gpus a command needs.', default=1)
     # parser.add_argument('-m', '--memPerCommand', type=float, required=False, help='How much memory a command needs (in Gb).')
-    parser.add_argument('-f', '--commandsFile', type=str, required=False, help='File containing commands to launch. Each command must be on a seperate line. (Replaces commandAndOptions)')
+    parser.add_argument('-f', '--commandsFile', type=argparse.FileType('r'), required=False,
+                        help='File containing commands to launch. Each command must be on a seperate line. (Replaces commandAndOptions)')
 
     parser.add_argument('-l', '--modules', type=str, required=False, help='List of additional modules to load.', nargs='+')
     parser.add_argument('-x', '--doNotLaunch', action='store_true', help='Creates the QSUB files without launching them.')
@@ -174,9 +175,6 @@ def parse_arguments():
         if args.queueName not in AVAILABLE_QUEUES and ((args.coresPerNode is None and args.gpusPerNode is None) or args.walltime is None):
             parser.error("Unknown queue, --coresPerNode/--gpusPerNode and --walltime must be set.")
 
-        if args.commandsFile is not None and not os.path.isfile(args.commandsFile):
-            parser.error("Command file does not exist: '{0}'.".format(args.commandsFile))
-
     return args
 
 

tests/test_smart_dispatch.py

@@ -15,15 +15,19 @@ def setUp(self):
         self.testing_dir = tempfile.mkdtemp()
         self.logs_dir = os.path.join(self.testing_dir, 'SMART_DISPATCH_LOGS')
 
-        self.commands = 'echo "[1 2 3 4]" "[6 7 8]" "[9 0]"'
-        self.nb_commands = 4*3*2
+        self.folded_commands = 'echo "[1 2 3 4]" "[6 7 8]" "[9 0]"'
+        self.commands = ["echo 1 6 9", "echo 1 6 0", "echo 1 7 9", "echo 1 7 0", "echo 1 8 9", "echo 1 8 0",
+                         "echo 2 6 9", "echo 2 6 0", "echo 2 7 9", "echo 2 7 0", "echo 2 8 9", "echo 2 8 0",
+                         "echo 3 6 9", "echo 3 6 0", "echo 3 7 9", "echo 3 7 0", "echo 3 8 9", "echo 3 8 0",
+                         "echo 4 6 9", "echo 4 6 0", "echo 4 7 9", "echo 4 7 0", "echo 4 8 9", "echo 4 8 0"]
+        self.nb_commands = len(self.commands)
 
-        smart_dispatch_command = 'smart_dispatch.py -C 1 -q test -t 5:00 -x {0}'
-        self.launch_command = smart_dispatch_command.format('launch ' + self.commands)
-        self.resume_command = smart_dispatch_command.format('resume {0}')
+        self.smart_dispatch_command = 'smart_dispatch.py -C 1 -q test -t 5:00 -x'
+        self.launch_command = self.smart_dispatch_command + " launch " + self.folded_commands
+        self.resume_command = self.smart_dispatch_command + " resume {0}"
 
         smart_dispatch_command_with_pool = 'smart_dispatch.py --pool 10 -C 1 -q test -t 5:00 -x {0}'
-        self.launch_command_with_pool = smart_dispatch_command_with_pool.format('launch ' + self.commands)
+        self.launch_command_with_pool = smart_dispatch_command_with_pool.format('launch ' + self.folded_commands)
         self.nb_workers = 10
 
         self._cwd = os.getcwd()
@@ -46,6 +50,24 @@ def test_main_launch(self):
         path_job_commands = os.path.join(self.logs_dir, batch_uid, "commands")
         assert_equal(len(os.listdir(path_job_commands)), self.nb_commands + 1)
 
+    def test_launch_using_commands_file(self):
+        # Actual test
+        commands_filename = "commands_to_run.txt"
+        open(commands_filename, 'w').write("\n".join(self.commands))
+
+        launch_command = self.smart_dispatch_command + " -f {0} launch".format(commands_filename)
+        exit_status = call(launch_command, shell=True)
+
+        # Test validation
+        assert_equal(exit_status, 0)
+        assert_true(os.path.isdir(self.logs_dir))
+        assert_equal(len(os.listdir(self.logs_dir)), 1)
+
+        batch_uid = os.listdir(self.logs_dir)[0]
+        path_job_commands = os.path.join(self.logs_dir, batch_uid, "commands")
+        assert_equal(len(os.listdir(path_job_commands)), self.nb_commands + 1)
+        assert_equal(open(pjoin(path_job_commands, 'commands.txt')).read(), "\n".join(self.commands) + "\n")
+
     def test_main_launch_with_pool_of_workers(self):
         # Actual test
         exit_status = call(self.launch_command_with_pool, shell=True)