diff --git a/.travis.yml b/.travis.yml index 1df6490..04e1f67 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,8 @@ language: python python: - "2.7" - #- "3.4" + - "3.4" + - "3.5" install: - pip install coveralls diff --git a/scripts/smart-dispatch b/scripts/smart-dispatch index 5e83f46..4643e1f 100755 --- a/scripts/smart-dispatch +++ b/scripts/smart-dispatch @@ -1,5 +1,6 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python # -*- coding: utf-8 -*- +from __future__ import print_function import os import sys @@ -101,7 +102,7 @@ def main(): # Generating all the worker commands worker_script = pjoin(os.path.dirname(smartdispatch.__file__), 'workers', 'base_worker.py') - COMMAND_STRING = 'cd "{cwd}"; python2 {worker_script} "{commands_file}" "{log_folder}" '\ + COMMAND_STRING = 'cd "{cwd}"; python {worker_script} "{commands_file}" "{log_folder}" '\ '1>> "{log_folder}/worker/$PBS_JOBID\"\"_worker_{{ID}}.o" '\ '2>> "{log_folder}/worker/$PBS_JOBID\"\"_worker_{{ID}}.e" ' COMMAND_STRING = COMMAND_STRING.format(cwd=os.getcwd(), worker_script=worker_script, commands_file=command_manager._commands_filename, log_folder=path_job_logs) @@ -119,19 +120,23 @@ def main(): pbs_filenames = job_generator.write_pbs_files(path_job_commands) # Launch the jobs - print "## {nb_commands} command(s) will be executed in {nb_jobs} job(s) ##".format(nb_commands=nb_commands, nb_jobs=len(pbs_filenames)) - print "Batch UID:\n{batch_uid}".format(batch_uid=jobname) + print("## {nb_commands} command(s) will be executed in {nb_jobs} job(s) ##".format(nb_commands=nb_commands, nb_jobs=len(pbs_filenames))) + print("Batch UID:\n{batch_uid}".format(batch_uid=jobname)) if not args.doNotLaunch: jobs_id = [] for pbs_filename in pbs_filenames: qsub_output = check_output('{launcher} {pbs_filename}'.format(launcher=LAUNCHER if args.launcher is None else args.launcher, pbs_filename=pbs_filename), shell=True) + + if isinstance(qsub_output, bytes): + qsub_output = qsub_output.decode("utf-8") + jobs_id += [qsub_output.strip()] with open_with_lock(pjoin(path_job, "jobs_id.txt"), 'a') as jobs_id_file: jobs_id_file.writelines(t.strftime("## %Y-%m-%d %H:%M:%S ##\n")) jobs_id_file.writelines("\n".join(jobs_id) + "\n") - print "\nJobs id:\n{jobs_id}".format(jobs_id=" ".join(jobs_id)) - print "\nLogs, command, and jobs id related to this batch will be in:\n {smartdispatch_folder}".format(smartdispatch_folder=path_job) + print("\nJobs id:\n{jobs_id}".format(jobs_id=" ".join(jobs_id))) + print("\nLogs, command, and jobs id related to this batch will be in:\n {smartdispatch_folder}".format(smartdispatch_folder=path_job)) def parse_arguments(): @@ -146,7 +151,8 @@ def parse_arguments(): parser.add_argument('-c', '--coresPerCommand', type=int, required=False, help='How many cores a command needs.', default=1) parser.add_argument('-g', '--gpusPerCommand', type=int, required=False, help='How many gpus a command needs.', default=1) # parser.add_argument('-m', '--memPerCommand', type=float, required=False, help='How much memory a command needs (in Gb).') - parser.add_argument('-f', '--commandsFile', type=file, required=False, help='File containing commands to launch. Each command must be on a seperate line. (Replaces commandAndOptions)') + parser.add_argument('-f', '--commandsFile', type=argparse.FileType('r'), required=False, + help='File containing commands to launch. Each command must be on a seperate line. (Replaces commandAndOptions)') parser.add_argument('-l', '--modules', type=str, required=False, help='List of additional modules to load.', nargs='+') parser.add_argument('-x', '--doNotLaunch', action='store_true', help='Creates the QSUB files without launching them.') diff --git a/smartdispatch/argument_template.py b/smartdispatch/argument_template.py index 7ef67f4..7486e82 100644 --- a/smartdispatch/argument_template.py +++ b/smartdispatch/argument_template.py @@ -35,7 +35,7 @@ def unfold(self, match): start = int(groups[0]) end = int(groups[1]) step = 1 if groups[2] is None else int(groups[2]) - return map(str, range(start, end, step)) + return [str(i) for i in range(start, end, step)] argument_templates = build_argument_templates_dictionnary() diff --git a/smartdispatch/smartdispatch.py b/smartdispatch/smartdispatch.py index 40c499a..1e4847b 100644 --- a/smartdispatch/smartdispatch.py +++ b/smartdispatch/smartdispatch.py @@ -97,7 +97,7 @@ def unfold_command(command): pos = match.end() arguments.append([text[pos:]]) # Add remaining unfolded arguments - arguments = [map(utils.decode_escaped_characters, argvalues) for argvalues in arguments] + arguments = [[utils.decode_escaped_characters(v) for v in argvalues] for argvalues in arguments] return ["".join(argvalues) for argvalues in itertools.product(*arguments)] diff --git a/smartdispatch/tests/test_filelock.py b/smartdispatch/tests/test_filelock.py index 691ac1f..6492990 100644 --- a/smartdispatch/tests/test_filelock.py +++ b/smartdispatch/tests/test_filelock.py @@ -38,9 +38,9 @@ def _test_open_with_lock(lock_func): time.sleep(1) stdout, stderr = process.communicate() - assert_equal(stdout, "") - assert_true("Traceback" not in stderr, msg="Unexpected error: '{}'".format(stderr)) - assert_true("write-lock" in stderr, msg="Forcing a race condition, try increasing sleeping time above.") + assert_equal(stdout, b"") + assert_true("Traceback" not in stderr.decode(), msg="Unexpected error: '{}'".format(stderr.decode())) + assert_true("write-lock" in stderr.decode(), msg="Forcing a race condition, try increasing sleeping time above.") shutil.rmtree(temp_dir) # Cleaning up. diff --git a/smartdispatch/tests/test_smartdispatch.py b/smartdispatch/tests/test_smartdispatch.py index c066982..e224624 100644 --- a/smartdispatch/tests/test_smartdispatch.py +++ b/smartdispatch/tests/test_smartdispatch.py @@ -3,7 +3,7 @@ import shutil import time as t from os.path import join as pjoin -from StringIO import StringIO +from io import StringIO, open import tempfile from nose.tools import assert_true, assert_equal @@ -43,11 +43,11 @@ def test_get_commands_from_file(): commands = ["command1 arg1 arg2", "command2", "command3 arg1 arg2 arg3 arg4"] - fileobj = StringIO("\n".join(commands)) + fileobj = StringIO(u"\n".join(commands)) assert_array_equal(smartdispatch.get_commands_from_file(fileobj), commands) # Test stripping last line if empty - fileobj = StringIO("\n".join(commands) + "\n") + fileobj = StringIO(u"\n".join(commands) + u"\n") assert_array_equal(smartdispatch.get_commands_from_file(fileobj), commands) diff --git a/smartdispatch/utils.py b/smartdispatch/utils.py index 8178a15..94ea3ae 100644 --- a/smartdispatch/utils.py +++ b/smartdispatch/utils.py @@ -1,4 +1,7 @@ +from __future__ import print_function + import re +import binascii import hashlib import unicodedata import json @@ -15,7 +18,7 @@ def print_boxed(string): out = u"\u250c" + box_line + u"\u2510\n" out += '\n'.join([u"\u2502 {} \u2502".format(line.ljust(max_len)) for line in splitted_string]) out += u"\n\u2514" + box_line + u"\u2518" - print out + print(out) def yes_no_prompt(query, default=None): @@ -35,13 +38,13 @@ def yes_no_prompt(query, default=None): def chunks(sequence, n): """ Yield successive n-sized chunks from sequence. """ - for i in xrange(0, len(sequence), n): + for i in range(0, len(sequence), n): yield sequence[i:i + n] def generate_uid_from_string(value): """ Create unique identifier from a string. """ - return hashlib.sha256(value).hexdigest() + return hashlib.sha256(value.encode()).hexdigest() def slugify(value): @@ -54,7 +57,14 @@ def slugify(value): --------- https://github.com/django/django/blob/1.7c3/django/utils/text.py#L436 """ - value = unicodedata.normalize('NFKD', unicode(value, "UTF-8")).encode('ascii', 'ignore').decode('ascii') + # Convert `value` to Unicode so we can slugify it using the unicodedata module. + try: + value = unicode(value, "UTF-8") + except NameError: + pass # In Python 3, all strings are already stored as Unicode. + + # Replace all compatibility characters with their equivalents. + value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii') value = re.sub('[^\w\s-]', '', value).strip().lower() return str(re.sub('[-\s]+', '_', value)) @@ -62,7 +72,8 @@ def slugify(value): def encode_escaped_characters(text, escaping_character="\\"): """ Escape the escaped character using its hex representation """ def hexify(match): - return "\\x{0}".format(match.group()[-1].encode("hex")) + # Reference: http://stackoverflow.com/questions/18298251/python-hex-values-to-convert-to-a-string-integer + return "\\x" + binascii.hexlify(match.group()[-1].encode()).decode() return re.sub(r"\\.", hexify, text) @@ -73,7 +84,7 @@ def decode_escaped_characters(text): return '' def unhexify(match): - return match.group()[2:].decode("hex") + return binascii.unhexlify(match.group()[2:]).decode() return re.sub(r"\\x..", unhexify, text) @@ -92,6 +103,8 @@ def detect_cluster(): # Get server status try: output = Popen(["qstat", "-B"], stdout=PIPE).communicate()[0] + if isinstance(output, bytes): + output = output.decode("utf-8") except OSError: # If qstat is not available we assume that the cluster is unknown. return None diff --git a/smartdispatch/workers/base_worker.py b/smartdispatch/workers/base_worker.py index fce1610..393aeed 100755 --- a/smartdispatch/workers/base_worker.py +++ b/smartdispatch/workers/base_worker.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python # -*- coding: utf-8 -*- import os diff --git a/smartdispatch/workers/tests/test_base_worker.py b/smartdispatch/workers/tests/test_base_worker.py index d3af9ae..59ba4af 100644 --- a/smartdispatch/workers/tests/test_base_worker.py +++ b/smartdispatch/workers/tests/test_base_worker.py @@ -25,14 +25,14 @@ def setUp(self): self.command_manager = CommandManager(os.path.join(self._commands_dir, "commands.txt")) self.command_manager.set_commands_to_run(self.commands) - self.commands_uid = map(utils.generate_uid_from_string, self.commands) + self.commands_uid = [utils.generate_uid_from_string(c) for c in self.commands] def tearDown(self): shutil.rmtree(self._commands_dir) shutil.rmtree(self.logs_dir) def test_main(self): - command = ['python2', self.base_worker_script, self.command_manager._commands_filename, self.logs_dir] + command = ['python', self.base_worker_script, self.command_manager._commands_filename, self.logs_dir] assert_equal(call(command), 0) # Simulate a resume, i.e. re-run the command, the output/error should be concatenated. self.command_manager.set_commands_to_run(self.commands) @@ -106,7 +106,7 @@ def test_main(self): assert_equal("", logfile.read()) def test_lock(self): - command = ['python2', self.base_worker_script, self.command_manager._commands_filename, self.logs_dir] + command = ['python', self.base_worker_script, self.command_manager._commands_filename, self.logs_dir] # Lock the commands file before running 'base_worker.py' with open_with_lock(self.command_manager._commands_filename, 'r+'): @@ -114,6 +114,6 @@ def test_lock(self): time.sleep(1) stdout, stderr = process.communicate() - assert_equal(stdout, "") - assert_true("write-lock" in stderr, msg="Forcing a race condition, try increasing sleeping time above.") - assert_true("Traceback" not in stderr) # Check that there are no errors. + assert_equal(stdout, b"") + assert_true("write-lock" in stderr.decode(), msg="Forcing a race condition, try increasing sleeping time above.") + assert_true("Traceback" not in stderr.decode()) # Check that there are no errors.