From 9f49f00b9e86078de69a504ae61ba47cb624052c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Alexandre=20C=C3=B4t=C3=A9?= Date: Fri, 9 Oct 2015 00:49:45 -0400 Subject: [PATCH 01/10] Changed print to be Python 3 compliant, convert Popen's output from bytes to str and removed function file in argparse --- scripts/smart-dispatch | 14 +++++++++----- smartdispatch/utils.py | 6 +++++- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/scripts/smart-dispatch b/scripts/smart-dispatch index 5e83f46..ba8dcb3 100755 --- a/scripts/smart-dispatch +++ b/scripts/smart-dispatch @@ -1,5 +1,6 @@ #!/usr/bin/env python2 # -*- coding: utf-8 -*- +from __future__ import print_function import os import sys @@ -119,8 +120,8 @@ def main(): pbs_filenames = job_generator.write_pbs_files(path_job_commands) # Launch the jobs - print "## {nb_commands} command(s) will be executed in {nb_jobs} job(s) ##".format(nb_commands=nb_commands, nb_jobs=len(pbs_filenames)) - print "Batch UID:\n{batch_uid}".format(batch_uid=jobname) + print("## {nb_commands} command(s) will be executed in {nb_jobs} job(s) ##".format(nb_commands=nb_commands, nb_jobs=len(pbs_filenames))) + print("Batch UID:\n{batch_uid}".format(batch_uid=jobname)) if not args.doNotLaunch: jobs_id = [] for pbs_filename in pbs_filenames: @@ -130,8 +131,8 @@ def main(): with open_with_lock(pjoin(path_job, "jobs_id.txt"), 'a') as jobs_id_file: jobs_id_file.writelines(t.strftime("## %Y-%m-%d %H:%M:%S ##\n")) jobs_id_file.writelines("\n".join(jobs_id) + "\n") - print "\nJobs id:\n{jobs_id}".format(jobs_id=" ".join(jobs_id)) - print "\nLogs, command, and jobs id related to this batch will be in:\n {smartdispatch_folder}".format(smartdispatch_folder=path_job) + print("\nJobs id:\n{jobs_id}".format(jobs_id=" ".join(jobs_id))) + print("\nLogs, command, and jobs id related to this batch will be in:\n {smartdispatch_folder}".format(smartdispatch_folder=path_job)) def parse_arguments(): @@ -146,7 +147,7 @@ def parse_arguments(): parser.add_argument('-c', '--coresPerCommand', type=int, required=False, help='How many cores a command needs.', default=1) parser.add_argument('-g', '--gpusPerCommand', type=int, required=False, help='How many gpus a command needs.', default=1) # parser.add_argument('-m', '--memPerCommand', type=float, required=False, help='How much memory a command needs (in Gb).') - parser.add_argument('-f', '--commandsFile', type=file, required=False, help='File containing commands to launch. Each command must be on a seperate line. (Replaces commandAndOptions)') + parser.add_argument('-f', '--commandsFile', type=str, required=False, help='File containing commands to launch. Each command must be on a seperate line. (Replaces commandAndOptions)') parser.add_argument('-l', '--modules', type=str, required=False, help='List of additional modules to load.', nargs='+') parser.add_argument('-x', '--doNotLaunch', action='store_true', help='Creates the QSUB files without launching them.') @@ -170,6 +171,9 @@ def parse_arguments(): if args.queueName not in AVAILABLE_QUEUES and ((args.coresPerNode is None and args.gpusPerNode is None) or args.walltime is None): parser.error("Unknown queue, --coresPerNode/--gpusPerNode and --walltime must be set.") + if args.commandsFile is not None and not os.path.isfile(args.commandsFile): + parser.error("Command file does not exist: '{0}'.".format(args.commandsFile)) + return args diff --git a/smartdispatch/utils.py b/smartdispatch/utils.py index 8178a15..3cec048 100644 --- a/smartdispatch/utils.py +++ b/smartdispatch/utils.py @@ -1,3 +1,5 @@ +from __future__ import print_function + import re import hashlib import unicodedata @@ -15,7 +17,7 @@ def print_boxed(string): out = u"\u250c" + box_line + u"\u2510\n" out += '\n'.join([u"\u2502 {} \u2502".format(line.ljust(max_len)) for line in splitted_string]) out += u"\n\u2514" + box_line + u"\u2518" - print out + print(out) def yes_no_prompt(query, default=None): @@ -92,6 +94,8 @@ def detect_cluster(): # Get server status try: output = Popen(["qstat", "-B"], stdout=PIPE).communicate()[0] + if isinstance(output, bytes): + output = output.decode("utf-8") except OSError: # If qstat is not available we assume that the cluster is unknown. return None From 4f112367e10774a8c212f1fde9f7ec0756df81bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Alexandre=20C=C3=B4t=C3=A9?= Date: Mon, 14 Dec 2015 18:23:34 -0500 Subject: [PATCH 02/10] Fixed UFT8 problems since Python3 handles strings differently --- scripts/smart-dispatch | 4 ++++ smartdispatch/utils.py | 17 ++++++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/scripts/smart-dispatch b/scripts/smart-dispatch index ba8dcb3..13ac6cd 100755 --- a/scripts/smart-dispatch +++ b/scripts/smart-dispatch @@ -126,6 +126,10 @@ def main(): jobs_id = [] for pbs_filename in pbs_filenames: qsub_output = check_output('{launcher} {pbs_filename}'.format(launcher=LAUNCHER if args.launcher is None else args.launcher, pbs_filename=pbs_filename), shell=True) + + if isinstance(qsub_output, bytes): + qsub_output = qsub_output.decode("utf-8") + jobs_id += [qsub_output.strip()] with open_with_lock(pjoin(path_job, "jobs_id.txt"), 'a') as jobs_id_file: diff --git a/smartdispatch/utils.py b/smartdispatch/utils.py index 3cec048..8bad52c 100644 --- a/smartdispatch/utils.py +++ b/smartdispatch/utils.py @@ -1,6 +1,7 @@ from __future__ import print_function import re +import binascii import hashlib import unicodedata import json @@ -37,13 +38,13 @@ def yes_no_prompt(query, default=None): def chunks(sequence, n): """ Yield successive n-sized chunks from sequence. """ - for i in xrange(0, len(sequence), n): + for i in range(0, len(sequence), n): yield sequence[i:i + n] def generate_uid_from_string(value): """ Create unique identifier from a string. """ - return hashlib.sha256(value).hexdigest() + return hashlib.sha256(value.encode()).hexdigest() def slugify(value): @@ -56,7 +57,12 @@ def slugify(value): --------- https://github.com/django/django/blob/1.7c3/django/utils/text.py#L436 """ - value = unicodedata.normalize('NFKD', unicode(value, "UTF-8")).encode('ascii', 'ignore').decode('ascii') + try: + value = unicode(value, "UTF-8") + except NameError: + pass # In Python 3 all strings are already unicode. + + value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii') value = re.sub('[^\w\s-]', '', value).strip().lower() return str(re.sub('[-\s]+', '_', value)) @@ -64,7 +70,8 @@ def slugify(value): def encode_escaped_characters(text, escaping_character="\\"): """ Escape the escaped character using its hex representation """ def hexify(match): - return "\\x{0}".format(match.group()[-1].encode("hex")) + # Reference: http://stackoverflow.com/questions/18298251/python-hex-values-to-convert-to-a-string-integer + return "\\x" + binascii.hexlify(match.group()[-1].encode()).decode() return re.sub(r"\\.", hexify, text) @@ -75,7 +82,7 @@ def decode_escaped_characters(text): return '' def unhexify(match): - return match.group()[2:].decode("hex") + return binascii.unhexlify(match.group()[2:]).decode() return re.sub(r"\\x..", unhexify, text) From c46df67d69195144583b676adbb3a19a241cb1b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Alexandre=20C=C3=B4t=C3=A9?= Date: Mon, 14 Dec 2015 22:58:06 -0500 Subject: [PATCH 03/10] Added more support for Python3 --- smartdispatch/argument_template.py | 2 +- smartdispatch/smartdispatch.py | 2 +- smartdispatch/tests/test_smartdispatch.py | 6 +++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/smartdispatch/argument_template.py b/smartdispatch/argument_template.py index 7ef67f4..12c833b 100644 --- a/smartdispatch/argument_template.py +++ b/smartdispatch/argument_template.py @@ -35,7 +35,7 @@ def unfold(self, match): start = int(groups[0]) end = int(groups[1]) step = 1 if groups[2] is None else int(groups[2]) - return map(str, range(start, end, step)) + return list(map(str, range(start, end, step))) argument_templates = build_argument_templates_dictionnary() diff --git a/smartdispatch/smartdispatch.py b/smartdispatch/smartdispatch.py index 40c499a..d703e8c 100644 --- a/smartdispatch/smartdispatch.py +++ b/smartdispatch/smartdispatch.py @@ -97,7 +97,7 @@ def unfold_command(command): pos = match.end() arguments.append([text[pos:]]) # Add remaining unfolded arguments - arguments = [map(utils.decode_escaped_characters, argvalues) for argvalues in arguments] + arguments = [list(map(utils.decode_escaped_characters, argvalues)) for argvalues in arguments] return ["".join(argvalues) for argvalues in itertools.product(*arguments)] diff --git a/smartdispatch/tests/test_smartdispatch.py b/smartdispatch/tests/test_smartdispatch.py index c066982..3294734 100644 --- a/smartdispatch/tests/test_smartdispatch.py +++ b/smartdispatch/tests/test_smartdispatch.py @@ -3,7 +3,11 @@ import shutil import time as t from os.path import join as pjoin -from StringIO import StringIO + +try: + from io import StringIO # Python 3 +except ImportError: + from StringIO import StringIO # Python 2 import tempfile from nose.tools import assert_true, assert_equal From c79d4fc30e97cb84d7ac490836fb645bed5e2cf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Alexandre=20C=C3=B4t=C3=A9?= Date: Mon, 8 Feb 2016 16:36:14 -0500 Subject: [PATCH 04/10] Made tests Python 3 compliant --- smartdispatch/tests/test_smartdispatch.py | 10 +++------- smartdispatch/workers/tests/test_base_worker.py | 8 ++++---- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/smartdispatch/tests/test_smartdispatch.py b/smartdispatch/tests/test_smartdispatch.py index 3294734..e224624 100644 --- a/smartdispatch/tests/test_smartdispatch.py +++ b/smartdispatch/tests/test_smartdispatch.py @@ -3,11 +3,7 @@ import shutil import time as t from os.path import join as pjoin - -try: - from io import StringIO # Python 3 -except ImportError: - from StringIO import StringIO # Python 2 +from io import StringIO, open import tempfile from nose.tools import assert_true, assert_equal @@ -47,11 +43,11 @@ def test_get_commands_from_file(): commands = ["command1 arg1 arg2", "command2", "command3 arg1 arg2 arg3 arg4"] - fileobj = StringIO("\n".join(commands)) + fileobj = StringIO(u"\n".join(commands)) assert_array_equal(smartdispatch.get_commands_from_file(fileobj), commands) # Test stripping last line if empty - fileobj = StringIO("\n".join(commands) + "\n") + fileobj = StringIO(u"\n".join(commands) + u"\n") assert_array_equal(smartdispatch.get_commands_from_file(fileobj), commands) diff --git a/smartdispatch/workers/tests/test_base_worker.py b/smartdispatch/workers/tests/test_base_worker.py index d3af9ae..b8ec5f5 100644 --- a/smartdispatch/workers/tests/test_base_worker.py +++ b/smartdispatch/workers/tests/test_base_worker.py @@ -25,7 +25,7 @@ def setUp(self): self.command_manager = CommandManager(os.path.join(self._commands_dir, "commands.txt")) self.command_manager.set_commands_to_run(self.commands) - self.commands_uid = map(utils.generate_uid_from_string, self.commands) + self.commands_uid = list(map(utils.generate_uid_from_string, self.commands)) def tearDown(self): shutil.rmtree(self._commands_dir) @@ -114,6 +114,6 @@ def test_lock(self): time.sleep(1) stdout, stderr = process.communicate() - assert_equal(stdout, "") - assert_true("write-lock" in stderr, msg="Forcing a race condition, try increasing sleeping time above.") - assert_true("Traceback" not in stderr) # Check that there are no errors. + assert_equal(stdout, b"") + assert_true("write-lock" in stderr.decode(), msg="Forcing a race condition, try increasing sleeping time above.") + assert_true("Traceback" not in stderr.decode()) # Check that there are no errors. From 9cfd937868cf07f560105b0efd2a601f7b938c72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Alexandre=20C=C3=B4t=C3=A9?= Date: Mon, 8 Feb 2016 17:46:50 -0500 Subject: [PATCH 05/10] Added Python 3.4 in Travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1df6490..e5923d2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: python python: - "2.7" - #- "3.4" + - "3.4" install: - pip install coveralls From 8beaaee5508f9429a6435a2c09ec06a6cd5e0936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Alexandre=20C=C3=B4t=C3=A9?= Date: Sat, 9 Apr 2016 16:12:17 -0400 Subject: [PATCH 06/10] Added Python 3.5 to Travis tests. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index e5923d2..04e1f67 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ language: python python: - "2.7" - "3.4" + - "3.5" install: - pip install coveralls From 70d679b4fd0526be20ef42eb73f9be7ffe0b8d0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Alexandre=20C=C3=B4t=C3=A9?= Date: Sat, 9 Apr 2016 16:13:30 -0400 Subject: [PATCH 07/10] Added unit tests for --commandsFile option. --- scripts/smart-dispatch | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/smart-dispatch b/scripts/smart-dispatch index 13ac6cd..06538d0 100755 --- a/scripts/smart-dispatch +++ b/scripts/smart-dispatch @@ -151,7 +151,8 @@ def parse_arguments(): parser.add_argument('-c', '--coresPerCommand', type=int, required=False, help='How many cores a command needs.', default=1) parser.add_argument('-g', '--gpusPerCommand', type=int, required=False, help='How many gpus a command needs.', default=1) # parser.add_argument('-m', '--memPerCommand', type=float, required=False, help='How much memory a command needs (in Gb).') - parser.add_argument('-f', '--commandsFile', type=str, required=False, help='File containing commands to launch. Each command must be on a seperate line. (Replaces commandAndOptions)') + parser.add_argument('-f', '--commandsFile', type=argparse.FileType('r'), required=False, + help='File containing commands to launch. Each command must be on a seperate line. (Replaces commandAndOptions)') parser.add_argument('-l', '--modules', type=str, required=False, help='List of additional modules to load.', nargs='+') parser.add_argument('-x', '--doNotLaunch', action='store_true', help='Creates the QSUB files without launching them.') @@ -175,9 +176,6 @@ def parse_arguments(): if args.queueName not in AVAILABLE_QUEUES and ((args.coresPerNode is None and args.gpusPerNode is None) or args.walltime is None): parser.error("Unknown queue, --coresPerNode/--gpusPerNode and --walltime must be set.") - if args.commandsFile is not None and not os.path.isfile(args.commandsFile): - parser.error("Command file does not exist: '{0}'.".format(args.commandsFile)) - return args From 6812ea1bfd34fad44b9f51789565271bcdafdbeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Alexandre=20C=C3=B4t=C3=A9?= Date: Sat, 9 Apr 2016 16:13:50 -0400 Subject: [PATCH 08/10] Addressed @ASalvail's comments --- smartdispatch/argument_template.py | 2 +- smartdispatch/smartdispatch.py | 2 +- smartdispatch/utils.py | 4 +++- smartdispatch/workers/tests/test_base_worker.py | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/smartdispatch/argument_template.py b/smartdispatch/argument_template.py index 12c833b..7486e82 100644 --- a/smartdispatch/argument_template.py +++ b/smartdispatch/argument_template.py @@ -35,7 +35,7 @@ def unfold(self, match): start = int(groups[0]) end = int(groups[1]) step = 1 if groups[2] is None else int(groups[2]) - return list(map(str, range(start, end, step))) + return [str(i) for i in range(start, end, step)] argument_templates = build_argument_templates_dictionnary() diff --git a/smartdispatch/smartdispatch.py b/smartdispatch/smartdispatch.py index d703e8c..1e4847b 100644 --- a/smartdispatch/smartdispatch.py +++ b/smartdispatch/smartdispatch.py @@ -97,7 +97,7 @@ def unfold_command(command): pos = match.end() arguments.append([text[pos:]]) # Add remaining unfolded arguments - arguments = [list(map(utils.decode_escaped_characters, argvalues)) for argvalues in arguments] + arguments = [[utils.decode_escaped_characters(v) for v in argvalues] for argvalues in arguments] return ["".join(argvalues) for argvalues in itertools.product(*arguments)] diff --git a/smartdispatch/utils.py b/smartdispatch/utils.py index 8bad52c..94ea3ae 100644 --- a/smartdispatch/utils.py +++ b/smartdispatch/utils.py @@ -57,11 +57,13 @@ def slugify(value): --------- https://github.com/django/django/blob/1.7c3/django/utils/text.py#L436 """ + # Convert `value` to Unicode so we can slugify it using the unicodedata module. try: value = unicode(value, "UTF-8") except NameError: - pass # In Python 3 all strings are already unicode. + pass # In Python 3, all strings are already stored as Unicode. + # Replace all compatibility characters with their equivalents. value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii') value = re.sub('[^\w\s-]', '', value).strip().lower() return str(re.sub('[-\s]+', '_', value)) diff --git a/smartdispatch/workers/tests/test_base_worker.py b/smartdispatch/workers/tests/test_base_worker.py index b8ec5f5..2d97027 100644 --- a/smartdispatch/workers/tests/test_base_worker.py +++ b/smartdispatch/workers/tests/test_base_worker.py @@ -25,7 +25,7 @@ def setUp(self): self.command_manager = CommandManager(os.path.join(self._commands_dir, "commands.txt")) self.command_manager.set_commands_to_run(self.commands) - self.commands_uid = list(map(utils.generate_uid_from_string, self.commands)) + self.commands_uid = [utils.generate_uid_from_string(c) for c in self.commands] def tearDown(self): shutil.rmtree(self._commands_dir) From 59cb2a620ddcbbec3046f7fbc2407c7b559aa7d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Alexandre=20C=C3=B4t=C3=A9?= Date: Sun, 10 Apr 2016 15:13:58 -0400 Subject: [PATCH 09/10] Rebase master --- smartdispatch/tests/test_filelock.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/smartdispatch/tests/test_filelock.py b/smartdispatch/tests/test_filelock.py index 691ac1f..6492990 100644 --- a/smartdispatch/tests/test_filelock.py +++ b/smartdispatch/tests/test_filelock.py @@ -38,9 +38,9 @@ def _test_open_with_lock(lock_func): time.sleep(1) stdout, stderr = process.communicate() - assert_equal(stdout, "") - assert_true("Traceback" not in stderr, msg="Unexpected error: '{}'".format(stderr)) - assert_true("write-lock" in stderr, msg="Forcing a race condition, try increasing sleeping time above.") + assert_equal(stdout, b"") + assert_true("Traceback" not in stderr.decode(), msg="Unexpected error: '{}'".format(stderr.decode())) + assert_true("write-lock" in stderr.decode(), msg="Forcing a race condition, try increasing sleeping time above.") shutil.rmtree(temp_dir) # Cleaning up. From 6eb7f8efa6fc9dc76651908ee4d52952f7337e31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc-Alexandre=20C=C3=B4t=C3=A9?= Date: Tue, 12 Apr 2016 18:15:08 -0400 Subject: [PATCH 10/10] Do not force the use of Python2 --- scripts/smart-dispatch | 4 ++-- smartdispatch/workers/base_worker.py | 2 +- smartdispatch/workers/tests/test_base_worker.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/smart-dispatch b/scripts/smart-dispatch index 06538d0..4643e1f 100755 --- a/scripts/smart-dispatch +++ b/scripts/smart-dispatch @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python # -*- coding: utf-8 -*- from __future__ import print_function @@ -102,7 +102,7 @@ def main(): # Generating all the worker commands worker_script = pjoin(os.path.dirname(smartdispatch.__file__), 'workers', 'base_worker.py') - COMMAND_STRING = 'cd "{cwd}"; python2 {worker_script} "{commands_file}" "{log_folder}" '\ + COMMAND_STRING = 'cd "{cwd}"; python {worker_script} "{commands_file}" "{log_folder}" '\ '1>> "{log_folder}/worker/$PBS_JOBID\"\"_worker_{{ID}}.o" '\ '2>> "{log_folder}/worker/$PBS_JOBID\"\"_worker_{{ID}}.e" ' COMMAND_STRING = COMMAND_STRING.format(cwd=os.getcwd(), worker_script=worker_script, commands_file=command_manager._commands_filename, log_folder=path_job_logs) diff --git a/smartdispatch/workers/base_worker.py b/smartdispatch/workers/base_worker.py index fce1610..393aeed 100755 --- a/smartdispatch/workers/base_worker.py +++ b/smartdispatch/workers/base_worker.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python # -*- coding: utf-8 -*- import os diff --git a/smartdispatch/workers/tests/test_base_worker.py b/smartdispatch/workers/tests/test_base_worker.py index 2d97027..59ba4af 100644 --- a/smartdispatch/workers/tests/test_base_worker.py +++ b/smartdispatch/workers/tests/test_base_worker.py @@ -32,7 +32,7 @@ def tearDown(self): shutil.rmtree(self.logs_dir) def test_main(self): - command = ['python2', self.base_worker_script, self.command_manager._commands_filename, self.logs_dir] + command = ['python', self.base_worker_script, self.command_manager._commands_filename, self.logs_dir] assert_equal(call(command), 0) # Simulate a resume, i.e. re-run the command, the output/error should be concatenated. self.command_manager.set_commands_to_run(self.commands) @@ -106,7 +106,7 @@ def test_main(self): assert_equal("", logfile.read()) def test_lock(self): - command = ['python2', self.base_worker_script, self.command_manager._commands_filename, self.logs_dir] + command = ['python', self.base_worker_script, self.command_manager._commands_filename, self.logs_dir] # Lock the commands file before running 'base_worker.py' with open_with_lock(self.command_manager._commands_filename, 'r+'):