diff --git a/.travis.yml b/.travis.yml index d88eee2..7d9be97 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,12 +5,13 @@ python: - "2.7" - "3.4" - "3.5" - - "3.5-dev" # 3.5 development branch - "3.6" - - "3.6-dev" # 3.6 development branch - - "3.7-dev" # 3.7 development branch + - "3.7" + - "3.8" + - "3.9-dev" # PyPy versions - - "pypy3.5" + - pypy + - pypy3 addons: apt: @@ -20,7 +21,8 @@ addons: - parallel install: - - pip install -r requirements.txt + - export PYTHONIOENCODING=UTF8 + - pip install -r requirements-testing.txt - pip install . - pip install pycodestyle diff --git a/qbatch/qbatch.py b/qbatch/qbatch.py index e90d692..df028ca 100755 --- a/qbatch/qbatch.py +++ b/qbatch/qbatch.py @@ -1,12 +1,8 @@ #!/usr/bin/env python -from __future__ import print_function -from __future__ import division -from __future__ import unicode_literals -from __future__ import absolute_import -from future import standard_library +from __future__ import (absolute_import, division, + print_function, unicode_literals) from builtins import * -from builtins import str -from builtins import range +from future import standard_library import argparse import math import os @@ -167,15 +163,13 @@ def run_command(command, logfile=None): if logfile: filehandle = open(logfile, 'w', encoding="utf-8") while True: - output = process.stdout.readline() - if output.decode('UTF-8') == '' and process.poll() is not None: + output = process.stdout.readline().decode('utf-8').strip() + if output == '' and process.poll() is not None: break if output and logfile: - print(output.decode('UTF-8').strip()) - filehandle.write(output.decode('UTF-8').strip()) + filehandle.write(output) filehandle.write("\n") - elif output: - print(output.decode('UTF-8').strip()) + print(output) rc = process.poll() if logfile: filehandle.close() @@ -196,6 +190,16 @@ def mkdirp(*p): return path +def unicode_str(string): + """Converts a bytestring to a unicode string""" + + try: + value = string.decode('utf-8') + except AttributeError: + value = string + return value + + def positive_int(string): """Checks agument is a positive integer""" msg = "Must be a positive integer" @@ -377,7 +381,9 @@ def qbatchDriver(**kwargs): else: sys.exit("qbatch: error: no command provided as last argument") elif command_file[0] == '-': - task_list = sys.stdin.readlines() + with open(getattr(sys.stdin, 'buffer', sys.stdin).fileno(), + encoding='utf8') as reader: + task_list = reader.readlines() job_name = job_name or 'STDIN' else: task_list = [] @@ -514,8 +520,8 @@ def qbatchDriver(**kwargs): ] scriptfile = os.path.join(SCRIPT_FOLDER, job_name + ".joblist") metafile = os.path.join(SCRIPT_FOLDER, job_name + ".meta") - script = open(scriptfile, 'w') - meta = open(metafile, 'w') + script = open(scriptfile, 'w', encoding="utf-8") + meta = open(metafile, 'w', encoding="utf-8") script.write('\n'.join(script_lines)) meta.write(" ".join(sys.argv[1:-1])) script.close() @@ -661,7 +667,7 @@ def qbatchParser(args=None): if -j is larger than --ppj (useful to make use of hyper-threading on some systems)""") parser.add_argument( - "-N", "--jobname", action="store", + "-N", "--jobname", action="store", type=unicode_str, help="""Set job name (defaults to name of command file, or STDIN)""") parser.add_argument( "--mem", default=MEM, diff --git a/requirements-testing.txt b/requirements-testing.txt new file mode 100644 index 0000000..76abfd6 --- /dev/null +++ b/requirements-testing.txt @@ -0,0 +1,3 @@ +nose>=1.0 +future +ushlex diff --git a/requirements.txt b/requirements.txt index 310bbdf..2c6edea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1 @@ -nose>=1.0 future -six diff --git a/setup.py b/setup.py index 7812de1..c2b0532 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ setup( name='qbatch', - version='2.2', + version='2.2.1', description='Execute shell command lines in parallel on Slurm, ' 'S(un|on of) Grid Engine (SGE) and PBS/Torque clusters', author="Jon Pipitone, Gabriel A. Devenyi", @@ -40,7 +40,6 @@ 'Topic :: Utilities', ], install_requires=[ - "six", "future", ], ) diff --git a/test/test_qbatch.py b/test/test_qbatch.py index b12a5ec..991f43a 100644 --- a/test/test_qbatch.py +++ b/test/test_qbatch.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -9,7 +10,11 @@ from builtins import range import os import shutil -import shlex +import sys +if sys.version_info < (3, 0): + import ushlex as shlex +else: + import shlex from subprocess import Popen, PIPE, STDOUT import tempfile import sys @@ -201,3 +206,18 @@ def test_run_qbatch_local_piped_commands(): "Return code = {0}".format(p.returncode) assert set(out.splitlines()) == set(expected.splitlines()), \ "Expected {0} but got {1}".format(expected, out) + +def test_run_qbatch_local_piped_commands_utf8(): + cmds = "\n".join(["echo hëllo"] * 24) + p = command_pipe('qbatch -N tëst_run_qbatch_local_piped_commands --env none -j2 -b local -') + out, _ = p.communicate(cmds.encode('utf-8')) + + expected, _ = command_pipe( + 'parallel --tag --line-buffer -j2').communicate(cmds.encode('utf-8')) + + print(out) + + assert p.returncode == 0, \ + "Return code = {0}".format(p.returncode) + assert set(out.splitlines()) == set(expected.splitlines()), \ + "Expected {0} but got {1}".format(expected, out)