Updates to make smart-dispatch Python 3 compatible

.travis.yml

@@ -1,7 +1,7 @@
 language: python
 python:
   - "2.7"
-  #- "3.4"
+  - "3.5"
 
 install:
   - pip install coveralls
@@ -12,4 +12,4 @@ script:
   - nosetests -v --with-coverage . smartdispatch/workers
 
 after_success:
-  - coveralls
+  - coveralls

scripts/smart-dispatch

@@ -1,5 +1,6 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
+from __future__ import print_function
 
 import os
 import sys
@@ -178,11 +179,11 @@ def main():
     pbs_filenames = job_generator.write_pbs_files(path_job_commands)
 
     # Launch the jobs
-    print "## {nb_commands} command(s) will be executed in {nb_jobs} job(s) ##".format(nb_commands=nb_commands, nb_jobs=len(pbs_filenames))
-    print "Batch UID:\n{batch_uid}".format(batch_uid=jobname)
+    print("## {nb_commands} command(s) will be executed in {nb_jobs} job(s) ##".format(nb_commands=nb_commands, nb_jobs=len(pbs_filenames)))
+    print("Batch UID:\n{batch_uid}".format(batch_uid=jobname))
     if not args.doNotLaunch:
         launch_jobs(LAUNCHER if args.launcher is None else args.launcher, pbs_filenames, CLUSTER_NAME, path_job)
-    print "\nLogs, command, and jobs id related to this batch will be in:\n {smartdispatch_folder}".format(smartdispatch_folder=path_job)
+    print("\nLogs, command, and jobs id related to this batch will be in:\n {smartdispatch_folder}".format(smartdispatch_folder=path_job))
 
 
 def parse_arguments():
@@ -198,7 +199,7 @@ def parse_arguments():
     parser.add_argument('-c', '--coresPerCommand', type=int, required=False, help='How many cores a command needs.', default=1)
     parser.add_argument('-g', '--gpusPerCommand', type=int, required=False, help='How many gpus a command needs.', default=1)
     # parser.add_argument('-m', '--memPerCommand', type=float, required=False, help='How much memory a command needs (in Gb).')
-    parser.add_argument('-f', '--commandsFile', type=file, required=False, help='File containing commands to launch. Each command must be on a seperate line. (Replaces commandAndOptions)')
+    parser.add_argument('-f', '--commandsFile', type=open, required=False, help='File containing commands to launch. Each command must be on a seperate line. (Replaces commandAndOptions)')
 
     parser.add_argument('-l', '--modules', type=str, required=False, help='List of additional modules to load.', nargs='+')
     parser.add_argument('-x', '--doNotLaunch', action='store_true', help='Generate all the files without launching the job.')

setup.py

@@ -14,6 +14,6 @@
     license='LICENSE.txt',
     description='An easy to use job launcher for supercomputers with PBS compatible job manager.',
     long_description=open('README.md').read(),
-    install_requires=['psutil>=1'],
+    install_requires=['psutil>=1', 'future'],
     package_data={'smartdispatch': ['config/*.json']}
 )

smartdispatch/argument_template.py

@@ -1,3 +1,6 @@
+from builtins import map
+from builtins import range
+from builtins import object
 import re
 from collections import OrderedDict
 
@@ -35,7 +38,7 @@ def unfold(self, match):
         start = int(groups[0])
         end = int(groups[1])
         step = 1 if groups[2] is None else int(groups[2])
-        return map(str, range(start, end, step))
+        return list(map(str, list(range(start, end, step))))
 
 
 argument_templates = build_argument_templates_dictionnary()

smartdispatch/command_manager.py

@@ -1,3 +1,4 @@
+from builtins import object
 import os
 from .filelock import open_with_lock
 

smartdispatch/job_generator.py

@@ -1,5 +1,7 @@
 from __future__ import absolute_import
 
+from builtins import str
+from builtins import object
 import os
 import re
 from smartdispatch.pbs import PBS

smartdispatch/pbs.py

@@ -1,3 +1,5 @@
+from builtins import str
+from builtins import object
 import re
 from collections import OrderedDict
 
@@ -54,7 +56,7 @@ def add_options(self, **options):
             Declares a name for the job. It must consist of printable,
             non white space characters with the first character alphabetic.
         """
-        for option_name, option_value in options.items():
+        for option_name, option_value in list(options.items()):
             # If known option, validate it.
             if option_name.strip('-') == 'N':
                 if len(option_name) > 64:
@@ -81,7 +83,7 @@ def add_resources(self, **resources):
         *pmem*: pmem=[0-9]+(b|kb|mb|gb|tb)
             Specifies the maximum amount of physical memory used by any single process of the job.
         """
-        for resource_name, resource_value in resources.items():
+        for resource_name, resource_value in list(resources.items()):
             # If known ressource, validate it.
             if resource_name == 'nodes':
                 if re.match(regex_resource_nodes, str(resource_value)) is None:
@@ -150,13 +152,13 @@ def __str__(self):
         pbs = []
         pbs += ["#!/bin/bash"]
 
-        for option_name, option_value in self.options.items():
+        for option_name, option_value in list(self.options.items()):
             if option_value == "":
                 pbs += ["#PBS {0}".format(option_name)]
             else:
                 pbs += ["#PBS {0} {1}".format(option_name, option_value)]
 
-        for resource_name, resource_value in self.resources.items():
+        for resource_name, resource_value in list(self.resources.items()):
             pbs += ["#PBS -l {0}={1}".format(resource_name, resource_value)]
 
         pbs += ["\n# Modules #"]

smartdispatch/queue.py

@@ -1,4 +1,6 @@
-from smartdispatch import get_available_queues
+from __future__ import absolute_import
+from builtins import object
+from .smartdispatch import get_available_queues
 
 
 class Queue(object):

smartdispatch/smartdispatch.py

@@ -1,5 +1,7 @@
-from __future__ import absolute_import
+from __future__ import absolute_import, print_function
 
+from builtins import next
+from builtins import map
 import os
 import re
 import itertools
@@ -89,7 +91,7 @@ def unfold_command(command):
     text = utils.encode_escaped_characters(command)
 
     # Build the master regex with all argument's regex
-    regex = "(" + "|".join(["(?P<{0}>{1})".format(name, arg.regex) for name, arg in argument_templates.items()]) + ")"
+    regex = "(" + "|".join(["(?P<{0}>{1})".format(name, arg.regex) for name, arg in list(argument_templates.items())]) + ")"
 
     pos = 0
     arguments = []
@@ -98,12 +100,12 @@ def unfold_command(command):
         arguments.append([text[pos:match.start()]])
 
         # Unfold argument
-        argument_template_name, matched_text = next((k, v) for k, v in match.groupdict().items() if v is not None)
+        argument_template_name, matched_text = next((k, v) for k, v in list(match.groupdict().items()) if v is not None)
         arguments.append(argument_templates[argument_template_name].unfold(matched_text))
         pos = match.end()
 
     arguments.append([text[pos:]])  # Add remaining unfolded arguments
-    arguments = [map(utils.decode_escaped_characters, argvalues) for argvalues in arguments]
+    arguments = [list(map(utils.decode_escaped_characters, argvalues)) for argvalues in arguments]
     return ["".join(argvalues) for argvalues in itertools.product(*arguments)]
 
 
@@ -185,11 +187,11 @@ def launch_jobs(launcher, pbs_filenames, cluster_name, path_job):  # pragma: no
     for pbs_filename in pbs_filenames:
         launcher_output = check_output('PBS_FILENAME={pbs_filename} {launcher} {pbs_filename}'.format(
             launcher=launcher, pbs_filename=pbs_filename), shell=True)
-        jobs_id += [launcher_output.strip()]
+        jobs_id += [launcher_output.strip().decode()]
 
         # On some clusters, SRMJID and PBS_JOBID don't match
         if cluster_name in ['helios']:
-            launcher_output = check_output(['qstat', '-f']).split('Job Id: ')
+            launcher_output = check_output(['qstat', '-f']).decode().split('Job Id: ')
             for job in launcher_output:
                 if re.search(r"SRMJID:{job_id}".format(job_id=jobs_id[-1]), job):
                     pbs_job_id = re.match(r"[0-9a-zA-Z.-]*", job).group()
@@ -198,4 +200,4 @@ def launch_jobs(launcher, pbs_filenames, cluster_name, path_job):  # pragma: no
     with open_with_lock(pjoin(path_job, "jobs_id.txt"), 'a') as jobs_id_file:
         jobs_id_file.writelines(t.strftime("## %Y-%m-%d %H:%M:%S ##\n"))
         jobs_id_file.writelines("\n".join(jobs_id) + "\n")
-    print "\nJobs id:\n{jobs_id}".format(jobs_id=" ".join(jobs_id))
+    print("\nJobs id:\n{jobs_id}".format(jobs_id=" ".join(jobs_id)))

smartdispatch/tests/test_filelock.py

@@ -38,7 +38,7 @@ def _test_open_with_lock(lock_func):
         time.sleep(1)
 
     stdout, stderr = process.communicate()
-    assert_equal(stdout, "")
+    assert_equal(stdout, b"")
     assert_true("Traceback" not in stderr, msg="Unexpected error: '{}'".format(stderr))
     assert_true("write-lock" in stderr, msg="Forcing a race condition, try increasing sleeping time above.")
 

smartdispatch/tests/test_job_generator.py

@@ -1,3 +1,5 @@
+from builtins import str
+from builtins import object
 from nose.tools import assert_true, assert_false, assert_equal, assert_raises
 
 import os

smartdispatch/tests/test_pbs.py

@@ -1,3 +1,4 @@
+from builtins import str
 from nose.tools import assert_true, assert_equal, assert_raises
 from numpy.testing import assert_array_equal
 
@@ -29,7 +30,7 @@ def test_constructor(self):
     def test_add_options(self):
         # Default options
         assert_equal(len(self.pbs.options), 2)
-        assert_true('-V' in self.pbs.options.keys())
+        assert_true('-V' in list(self.pbs.options.keys()))
         assert_equal(self.pbs.options['-q'], self.queue_name)
 
         self.pbs.add_options(A="option1")

smartdispatch/tests/test_queue.py

@@ -35,7 +35,7 @@ def test_constructor(self):
 
         # Test with missing information but referring to a known queue.
         for cluster_name in self.known_clusters:
-            for queue_name, queue_infos in get_available_queues(cluster_name).items():
+            for queue_name, queue_infos in list(get_available_queues(cluster_name).items()):
                 queue = Queue(queue_name, cluster_name)
                 assert_equal(queue.name, queue_name)
                 assert_equal(queue.cluster_name, cluster_name)

smartdispatch/tests/test_smartdispatch.py

@@ -1,9 +1,11 @@
+from future import standard_library
+standard_library.install_aliases()
 import os
 import re
 import shutil
 import time as t
 from os.path import join as pjoin
-from StringIO import StringIO
+from io import StringIO
 
 import tempfile
 from nose.tools import assert_true, assert_equal

smartdispatch/tests/test_utils.py

@@ -1,4 +1,6 @@
 # -*- coding: utf-8 -*-
+from builtins import zip
+from builtins import range
 import unittest
 
 from smartdispatch import utils
@@ -21,11 +23,11 @@ def test_print_boxed_empty(self):
 
 
 def test_chunks():
-    sequence = range(10)
+    sequence = list(range(10))
 
     for n in range(1, 11):
         expected = []
-        for start, end in zip(range(0, len(sequence), n), range(n, len(sequence) + n, n)):
+        for start, end in zip(list(range(0, len(sequence), n)), list(range(n, len(sequence) + n, n))):
             expected.append(sequence[start:end])
 
         assert_array_equal(list(utils.chunks(sequence, n)), expected, "n:{0}".format(n))

smartdispatch/utils.py

@@ -1,7 +1,21 @@
+from __future__ import print_function
+
+from builtins import str
+from builtins import input
+from builtins import map
+from builtins import range
+
+try:
+    _unicode = unicode
+    _utf8 = lambda x: _unicode(x, 'UTF-8')
+except NameError:
+    _utf8 = str
+
 import re
 import hashlib
 import unicodedata
 import json
+import codecs
 
 from distutils.util import strtobool
 from subprocess import Popen, PIPE
@@ -17,7 +31,7 @@ def jobname_generator(jobname, job_id):
     Returns
     -------
     str
-    The cropped version of the string.  
+    The cropped version of the string.
     '''
     # 64 - 1 since the total length including -1 should be less than 64
     job_id = str(job_id)
@@ -30,13 +44,13 @@ def jobname_generator(jobname, job_id):
 
 def print_boxed(string):
     splitted_string = string.split('\n')
-    max_len = max(map(len, splitted_string))
+    max_len = max(list(map(len, splitted_string)))
     box_line = u"\u2500" * (max_len + 2)
 
     out = u"\u250c" + box_line + u"\u2510\n"
     out += '\n'.join([u"\u2502 {} \u2502".format(line.ljust(max_len)) for line in splitted_string])
     out += u"\n\u2514" + box_line + u"\u2518"
-    print out
+    print(out)
 
 
 def yes_no_prompt(query, default=None):
@@ -47,7 +61,7 @@ def yes_no_prompt(query, default=None):
 
     while True:
         try:
-            answer = raw_input("{0}{1}".format(query, available_prompts[default]))
+            answer = eval(input("{0}{1}".format(query, available_prompts[default])))
             return strtobool(answer)
         except ValueError:
             if answer == '' and default is not None:
@@ -56,13 +70,13 @@ def yes_no_prompt(query, default=None):
 
 def chunks(sequence, n):
     """ Yield successive n-sized chunks from sequence. """
-    for i in xrange(0, len(sequence), n):
+    for i in range(0, len(sequence), n):
         yield sequence[i:i + n]
 
 
 def generate_uid_from_string(value):
     """ Create unique identifier from a string. """
-    return hashlib.sha256(value).hexdigest()
+    return hashlib.sha256(value.encode('UTF-8')).hexdigest()
 
 
 def slugify(value):
@@ -75,15 +89,15 @@ def slugify(value):
     ---------
     https://github.com/django/django/blob/1.7c3/django/utils/text.py#L436
     """
-    value = unicodedata.normalize('NFKD', unicode(value, "UTF-8")).encode('ascii', 'ignore').decode('ascii')
+    value = unicodedata.normalize('NFKD', _utf8(value)).encode('ascii', 'ignore').decode('ascii')
     value = re.sub('[^\w\s-]', '', value).strip().lower()
     return str(re.sub('[-\s]+', '_', value))
 
 
 def encode_escaped_characters(text, escaping_character="\\"):
     """ Escape the escaped character using its hex representation """
     def hexify(match):
-        return "\\x{0}".format(match.group()[-1].encode("hex"))
+        return codecs.encode("\\x{0}".format(match.group()[-1], 'hex_codec'))
 
     return re.sub(r"\\.", hexify, text)
 
@@ -117,7 +131,7 @@ def detect_cluster():
         # If qstat is not available we assume that the cluster is unknown.
         return None
     # Get server name from status
-    server_name = output.split('\n')[2].split(' ')[0]
+    server_name = output.decode().split('\n')[2].split(' ')[0]
     # Cleanup the name and return it
     cluster_name = None
     if server_name.split('.')[-1] == 'm':

smartdispatch/workers/tests/test_base_worker.py

@@ -114,6 +114,6 @@ def test_lock(self):
             time.sleep(1)
 
         stdout, stderr = process.communicate()
-        assert_equal(stdout, "")
+        assert_equal(stdout, b"")
         assert_true("write-lock" in stderr, msg="Forcing a race condition, try increasing sleeping time above.")
         assert_true("Traceback" not in stderr)  # Check that there are no errors.