diff --git a/.travis.yml b/.travis.yml index 1df6490..a05ba99 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: python python: - "2.7" - #- "3.4" + - "3.5" install: - pip install coveralls @@ -12,4 +12,4 @@ script: - nosetests -v --with-coverage . smartdispatch/workers after_success: - - coveralls \ No newline at end of file + - coveralls diff --git a/scripts/smart-dispatch b/scripts/smart-dispatch index 86904fa..447699e 100755 --- a/scripts/smart-dispatch +++ b/scripts/smart-dispatch @@ -1,5 +1,6 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python # -*- coding: utf-8 -*- +from __future__ import print_function import os import sys @@ -178,11 +179,11 @@ def main(): pbs_filenames = job_generator.write_pbs_files(path_job_commands) # Launch the jobs - print "## {nb_commands} command(s) will be executed in {nb_jobs} job(s) ##".format(nb_commands=nb_commands, nb_jobs=len(pbs_filenames)) - print "Batch UID:\n{batch_uid}".format(batch_uid=jobname) + print("## {nb_commands} command(s) will be executed in {nb_jobs} job(s) ##".format(nb_commands=nb_commands, nb_jobs=len(pbs_filenames))) + print("Batch UID:\n{batch_uid}".format(batch_uid=jobname)) if not args.doNotLaunch: launch_jobs(LAUNCHER if args.launcher is None else args.launcher, pbs_filenames, CLUSTER_NAME, path_job) - print "\nLogs, command, and jobs id related to this batch will be in:\n {smartdispatch_folder}".format(smartdispatch_folder=path_job) + print("\nLogs, command, and jobs id related to this batch will be in:\n {smartdispatch_folder}".format(smartdispatch_folder=path_job)) def parse_arguments(): @@ -198,7 +199,7 @@ def parse_arguments(): parser.add_argument('-c', '--coresPerCommand', type=int, required=False, help='How many cores a command needs.', default=1) parser.add_argument('-g', '--gpusPerCommand', type=int, required=False, help='How many gpus a command needs.', default=1) # parser.add_argument('-m', '--memPerCommand', type=float, required=False, help='How much memory a command needs (in Gb).') - parser.add_argument('-f', '--commandsFile', type=file, required=False, help='File containing commands to launch. Each command must be on a seperate line. (Replaces commandAndOptions)') + parser.add_argument('-f', '--commandsFile', type=open, required=False, help='File containing commands to launch. Each command must be on a seperate line. (Replaces commandAndOptions)') parser.add_argument('-l', '--modules', type=str, required=False, help='List of additional modules to load.', nargs='+') parser.add_argument('-x', '--doNotLaunch', action='store_true', help='Generate all the files without launching the job.') diff --git a/setup.py b/setup.py index 1da3ffe..c59f37b 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,6 @@ license='LICENSE.txt', description='An easy to use job launcher for supercomputers with PBS compatible job manager.', long_description=open('README.md').read(), - install_requires=['psutil>=1'], + install_requires=['psutil>=1', 'future'], package_data={'smartdispatch': ['config/*.json']} ) diff --git a/smartdispatch/argument_template.py b/smartdispatch/argument_template.py index 7ef67f4..e37e469 100644 --- a/smartdispatch/argument_template.py +++ b/smartdispatch/argument_template.py @@ -1,3 +1,6 @@ +from builtins import map +from builtins import range +from builtins import object import re from collections import OrderedDict @@ -35,7 +38,7 @@ def unfold(self, match): start = int(groups[0]) end = int(groups[1]) step = 1 if groups[2] is None else int(groups[2]) - return map(str, range(start, end, step)) + return list(map(str, list(range(start, end, step)))) argument_templates = build_argument_templates_dictionnary() diff --git a/smartdispatch/command_manager.py b/smartdispatch/command_manager.py index cb13fdc..8e8b5f3 100644 --- a/smartdispatch/command_manager.py +++ b/smartdispatch/command_manager.py @@ -1,3 +1,4 @@ +from builtins import object import os from .filelock import open_with_lock diff --git a/smartdispatch/job_generator.py b/smartdispatch/job_generator.py index d2db23c..ab7d11c 100644 --- a/smartdispatch/job_generator.py +++ b/smartdispatch/job_generator.py @@ -1,5 +1,7 @@ from __future__ import absolute_import +from builtins import str +from builtins import object import os import re from smartdispatch.pbs import PBS diff --git a/smartdispatch/pbs.py b/smartdispatch/pbs.py index f8d7982..56b2eae 100644 --- a/smartdispatch/pbs.py +++ b/smartdispatch/pbs.py @@ -1,3 +1,5 @@ +from builtins import str +from builtins import object import re from collections import OrderedDict @@ -54,7 +56,7 @@ def add_options(self, **options): Declares a name for the job. It must consist of printable, non white space characters with the first character alphabetic. """ - for option_name, option_value in options.items(): + for option_name, option_value in list(options.items()): # If known option, validate it. if option_name.strip('-') == 'N': if len(option_name) > 64: @@ -81,7 +83,7 @@ def add_resources(self, **resources): *pmem*: pmem=[0-9]+(b|kb|mb|gb|tb) Specifies the maximum amount of physical memory used by any single process of the job. """ - for resource_name, resource_value in resources.items(): + for resource_name, resource_value in list(resources.items()): # If known ressource, validate it. if resource_name == 'nodes': if re.match(regex_resource_nodes, str(resource_value)) is None: @@ -150,13 +152,13 @@ def __str__(self): pbs = [] pbs += ["#!/bin/bash"] - for option_name, option_value in self.options.items(): + for option_name, option_value in list(self.options.items()): if option_value == "": pbs += ["#PBS {0}".format(option_name)] else: pbs += ["#PBS {0} {1}".format(option_name, option_value)] - for resource_name, resource_value in self.resources.items(): + for resource_name, resource_value in list(self.resources.items()): pbs += ["#PBS -l {0}={1}".format(resource_name, resource_value)] pbs += ["\n# Modules #"] diff --git a/smartdispatch/queue.py b/smartdispatch/queue.py index 091fa8d..bb0ff73 100644 --- a/smartdispatch/queue.py +++ b/smartdispatch/queue.py @@ -1,4 +1,6 @@ -from smartdispatch import get_available_queues +from __future__ import absolute_import +from builtins import object +from .smartdispatch import get_available_queues class Queue(object): diff --git a/smartdispatch/smartdispatch.py b/smartdispatch/smartdispatch.py index b38e8cf..2c584b5 100644 --- a/smartdispatch/smartdispatch.py +++ b/smartdispatch/smartdispatch.py @@ -1,5 +1,7 @@ -from __future__ import absolute_import +from __future__ import absolute_import, print_function +from builtins import next +from builtins import map import os import re import itertools @@ -89,7 +91,7 @@ def unfold_command(command): text = utils.encode_escaped_characters(command) # Build the master regex with all argument's regex - regex = "(" + "|".join(["(?P<{0}>{1})".format(name, arg.regex) for name, arg in argument_templates.items()]) + ")" + regex = "(" + "|".join(["(?P<{0}>{1})".format(name, arg.regex) for name, arg in list(argument_templates.items())]) + ")" pos = 0 arguments = [] @@ -98,12 +100,12 @@ def unfold_command(command): arguments.append([text[pos:match.start()]]) # Unfold argument - argument_template_name, matched_text = next((k, v) for k, v in match.groupdict().items() if v is not None) + argument_template_name, matched_text = next((k, v) for k, v in list(match.groupdict().items()) if v is not None) arguments.append(argument_templates[argument_template_name].unfold(matched_text)) pos = match.end() arguments.append([text[pos:]]) # Add remaining unfolded arguments - arguments = [map(utils.decode_escaped_characters, argvalues) for argvalues in arguments] + arguments = [list(map(utils.decode_escaped_characters, argvalues)) for argvalues in arguments] return ["".join(argvalues) for argvalues in itertools.product(*arguments)] @@ -185,11 +187,11 @@ def launch_jobs(launcher, pbs_filenames, cluster_name, path_job): # pragma: no for pbs_filename in pbs_filenames: launcher_output = check_output('PBS_FILENAME={pbs_filename} {launcher} {pbs_filename}'.format( launcher=launcher, pbs_filename=pbs_filename), shell=True) - jobs_id += [launcher_output.strip()] + jobs_id += [launcher_output.strip().decode()] # On some clusters, SRMJID and PBS_JOBID don't match if cluster_name in ['helios']: - launcher_output = check_output(['qstat', '-f']).split('Job Id: ') + launcher_output = check_output(['qstat', '-f']).decode().split('Job Id: ') for job in launcher_output: if re.search(r"SRMJID:{job_id}".format(job_id=jobs_id[-1]), job): pbs_job_id = re.match(r"[0-9a-zA-Z.-]*", job).group() @@ -198,4 +200,4 @@ def launch_jobs(launcher, pbs_filenames, cluster_name, path_job): # pragma: no with open_with_lock(pjoin(path_job, "jobs_id.txt"), 'a') as jobs_id_file: jobs_id_file.writelines(t.strftime("## %Y-%m-%d %H:%M:%S ##\n")) jobs_id_file.writelines("\n".join(jobs_id) + "\n") - print "\nJobs id:\n{jobs_id}".format(jobs_id=" ".join(jobs_id)) + print("\nJobs id:\n{jobs_id}".format(jobs_id=" ".join(jobs_id))) diff --git a/smartdispatch/tests/test_filelock.py b/smartdispatch/tests/test_filelock.py index 691ac1f..dcc92a7 100644 --- a/smartdispatch/tests/test_filelock.py +++ b/smartdispatch/tests/test_filelock.py @@ -38,7 +38,7 @@ def _test_open_with_lock(lock_func): time.sleep(1) stdout, stderr = process.communicate() - assert_equal(stdout, "") + assert_equal(stdout, b"") assert_true("Traceback" not in stderr, msg="Unexpected error: '{}'".format(stderr)) assert_true("write-lock" in stderr, msg="Forcing a race condition, try increasing sleeping time above.") diff --git a/smartdispatch/tests/test_job_generator.py b/smartdispatch/tests/test_job_generator.py index 7214d9d..b216683 100644 --- a/smartdispatch/tests/test_job_generator.py +++ b/smartdispatch/tests/test_job_generator.py @@ -1,3 +1,5 @@ +from builtins import str +from builtins import object from nose.tools import assert_true, assert_false, assert_equal, assert_raises import os diff --git a/smartdispatch/tests/test_pbs.py b/smartdispatch/tests/test_pbs.py index 6088052..b7a986e 100644 --- a/smartdispatch/tests/test_pbs.py +++ b/smartdispatch/tests/test_pbs.py @@ -1,3 +1,4 @@ +from builtins import str from nose.tools import assert_true, assert_equal, assert_raises from numpy.testing import assert_array_equal @@ -29,7 +30,7 @@ def test_constructor(self): def test_add_options(self): # Default options assert_equal(len(self.pbs.options), 2) - assert_true('-V' in self.pbs.options.keys()) + assert_true('-V' in list(self.pbs.options.keys())) assert_equal(self.pbs.options['-q'], self.queue_name) self.pbs.add_options(A="option1") diff --git a/smartdispatch/tests/test_queue.py b/smartdispatch/tests/test_queue.py index 4cc4bc6..5994f55 100644 --- a/smartdispatch/tests/test_queue.py +++ b/smartdispatch/tests/test_queue.py @@ -35,7 +35,7 @@ def test_constructor(self): # Test with missing information but referring to a known queue. for cluster_name in self.known_clusters: - for queue_name, queue_infos in get_available_queues(cluster_name).items(): + for queue_name, queue_infos in list(get_available_queues(cluster_name).items()): queue = Queue(queue_name, cluster_name) assert_equal(queue.name, queue_name) assert_equal(queue.cluster_name, cluster_name) diff --git a/smartdispatch/tests/test_smartdispatch.py b/smartdispatch/tests/test_smartdispatch.py index 8b5f6ec..d5b95c4 100644 --- a/smartdispatch/tests/test_smartdispatch.py +++ b/smartdispatch/tests/test_smartdispatch.py @@ -1,9 +1,11 @@ +from future import standard_library +standard_library.install_aliases() import os import re import shutil import time as t from os.path import join as pjoin -from StringIO import StringIO +from io import StringIO import tempfile from nose.tools import assert_true, assert_equal diff --git a/smartdispatch/tests/test_utils.py b/smartdispatch/tests/test_utils.py index 4eaef4e..0681705 100644 --- a/smartdispatch/tests/test_utils.py +++ b/smartdispatch/tests/test_utils.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +from builtins import zip +from builtins import range import unittest from smartdispatch import utils @@ -21,11 +23,11 @@ def test_print_boxed_empty(self): def test_chunks(): - sequence = range(10) + sequence = list(range(10)) for n in range(1, 11): expected = [] - for start, end in zip(range(0, len(sequence), n), range(n, len(sequence) + n, n)): + for start, end in zip(list(range(0, len(sequence), n)), list(range(n, len(sequence) + n, n))): expected.append(sequence[start:end]) assert_array_equal(list(utils.chunks(sequence, n)), expected, "n:{0}".format(n)) diff --git a/smartdispatch/utils.py b/smartdispatch/utils.py index 9135780..ce4f914 100644 --- a/smartdispatch/utils.py +++ b/smartdispatch/utils.py @@ -1,7 +1,21 @@ +from __future__ import print_function + +from builtins import str +from builtins import input +from builtins import map +from builtins import range + +try: + _unicode = unicode + _utf8 = lambda x: _unicode(x, 'UTF-8') +except NameError: + _utf8 = str + import re import hashlib import unicodedata import json +import codecs from distutils.util import strtobool from subprocess import Popen, PIPE @@ -17,7 +31,7 @@ def jobname_generator(jobname, job_id): Returns ------- str - The cropped version of the string. + The cropped version of the string. ''' # 64 - 1 since the total length including -1 should be less than 64 job_id = str(job_id) @@ -30,13 +44,13 @@ def jobname_generator(jobname, job_id): def print_boxed(string): splitted_string = string.split('\n') - max_len = max(map(len, splitted_string)) + max_len = max(list(map(len, splitted_string))) box_line = u"\u2500" * (max_len + 2) out = u"\u250c" + box_line + u"\u2510\n" out += '\n'.join([u"\u2502 {} \u2502".format(line.ljust(max_len)) for line in splitted_string]) out += u"\n\u2514" + box_line + u"\u2518" - print out + print(out) def yes_no_prompt(query, default=None): @@ -47,7 +61,7 @@ def yes_no_prompt(query, default=None): while True: try: - answer = raw_input("{0}{1}".format(query, available_prompts[default])) + answer = eval(input("{0}{1}".format(query, available_prompts[default]))) return strtobool(answer) except ValueError: if answer == '' and default is not None: @@ -56,13 +70,13 @@ def yes_no_prompt(query, default=None): def chunks(sequence, n): """ Yield successive n-sized chunks from sequence. """ - for i in xrange(0, len(sequence), n): + for i in range(0, len(sequence), n): yield sequence[i:i + n] def generate_uid_from_string(value): """ Create unique identifier from a string. """ - return hashlib.sha256(value).hexdigest() + return hashlib.sha256(value.encode('UTF-8')).hexdigest() def slugify(value): @@ -75,7 +89,7 @@ def slugify(value): --------- https://github.com/django/django/blob/1.7c3/django/utils/text.py#L436 """ - value = unicodedata.normalize('NFKD', unicode(value, "UTF-8")).encode('ascii', 'ignore').decode('ascii') + value = unicodedata.normalize('NFKD', _utf8(value)).encode('ascii', 'ignore').decode('ascii') value = re.sub('[^\w\s-]', '', value).strip().lower() return str(re.sub('[-\s]+', '_', value)) @@ -83,7 +97,7 @@ def slugify(value): def encode_escaped_characters(text, escaping_character="\\"): """ Escape the escaped character using its hex representation """ def hexify(match): - return "\\x{0}".format(match.group()[-1].encode("hex")) + return codecs.encode("\\x{0}".format(match.group()[-1], 'hex_codec')) return re.sub(r"\\.", hexify, text) @@ -117,7 +131,7 @@ def detect_cluster(): # If qstat is not available we assume that the cluster is unknown. return None # Get server name from status - server_name = output.split('\n')[2].split(' ')[0] + server_name = output.decode().split('\n')[2].split(' ')[0] # Cleanup the name and return it cluster_name = None if server_name.split('.')[-1] == 'm': diff --git a/smartdispatch/workers/tests/test_base_worker.py b/smartdispatch/workers/tests/test_base_worker.py index d3af9ae..5894f17 100644 --- a/smartdispatch/workers/tests/test_base_worker.py +++ b/smartdispatch/workers/tests/test_base_worker.py @@ -114,6 +114,6 @@ def test_lock(self): time.sleep(1) stdout, stderr = process.communicate() - assert_equal(stdout, "") + assert_equal(stdout, b"") assert_true("write-lock" in stderr, msg="Forcing a race condition, try increasing sleeping time above.") assert_true("Traceback" not in stderr) # Check that there are no errors.