Merge pull request #210 from gdevenyi/fix-unicode

.travis.yml

@@ -5,12 +5,13 @@ python:
   - "2.7"
   - "3.4"
   - "3.5"
-  - "3.5-dev"  # 3.5 development branch
   - "3.6"
-  - "3.6-dev"  # 3.6 development branch
-  - "3.7-dev"  # 3.7 development branch
+  - "3.7"
+  - "3.8"
+  - "3.9-dev"
   # PyPy versions
-  - "pypy3.5"
+  - pypy
+  - pypy3
 
 addons:
     apt:
@@ -20,7 +21,8 @@ addons:
         - parallel
 
 install:
-    - pip install -r requirements.txt
+    - export PYTHONIOENCODING=UTF8
+    - pip install -r requirements-testing.txt
     - pip install .
     - pip install pycodestyle
 

qbatch/qbatch.py

@@ -1,12 +1,8 @@
 #!/usr/bin/env python
-from __future__ import print_function
-from __future__ import division
-from __future__ import unicode_literals
-from __future__ import absolute_import
-from future import standard_library
+from __future__ import (absolute_import, division,
+                        print_function, unicode_literals)
 from builtins import *
-from builtins import str
-from builtins import range
+from future import standard_library
 import argparse
 import math
 import os
@@ -167,15 +163,13 @@ def run_command(command, logfile=None):
     if logfile:
         filehandle = open(logfile, 'w', encoding="utf-8")
     while True:
-        output = process.stdout.readline()
-        if output.decode('UTF-8') == '' and process.poll() is not None:
+        output = process.stdout.readline().decode('utf-8').strip()
+        if output == '' and process.poll() is not None:
             break
         if output and logfile:
-            print(output.decode('UTF-8').strip())
-            filehandle.write(output.decode('UTF-8').strip())
+            filehandle.write(output)
             filehandle.write("\n")
-        elif output:
-            print(output.decode('UTF-8').strip())
+        print(output)
     rc = process.poll()
     if logfile:
         filehandle.close()
@@ -196,6 +190,16 @@ def mkdirp(*p):
     return path
 
 
+def unicode_str(string):
+    """Converts a bytestring to a unicode string"""
+
+    try:
+        value = string.decode('utf-8')
+    except AttributeError:
+        value = string
+    return value
+
+
 def positive_int(string):
     """Checks agument is a positive integer"""
     msg = "Must be a positive integer"
@@ -377,7 +381,9 @@ def qbatchDriver(**kwargs):
             else:
                 sys.exit("qbatch: error: no command provided as last argument")
         elif command_file[0] == '-':
-            task_list = sys.stdin.readlines()
+            with open(getattr(sys.stdin, 'buffer', sys.stdin).fileno(),
+                      encoding='utf8') as reader:
+                task_list = reader.readlines()
             job_name = job_name or 'STDIN'
         else:
             task_list = []
@@ -514,8 +520,8 @@ def qbatchDriver(**kwargs):
         ]
         scriptfile = os.path.join(SCRIPT_FOLDER, job_name + ".joblist")
         metafile = os.path.join(SCRIPT_FOLDER, job_name + ".meta")
-        script = open(scriptfile, 'w')
-        meta = open(metafile, 'w')
+        script = open(scriptfile, 'w', encoding="utf-8")
+        meta = open(metafile, 'w', encoding="utf-8")
         script.write('\n'.join(script_lines))
         meta.write(" ".join(sys.argv[1:-1]))
         script.close()
@@ -661,7 +667,7 @@ def qbatchParser(args=None):
         if -j is larger than --ppj
         (useful to make use of hyper-threading on some systems)""")
     parser.add_argument(
-        "-N", "--jobname", action="store",
+        "-N", "--jobname", action="store", type=unicode_str,
         help="""Set job name (defaults to name of command file, or STDIN)""")
     parser.add_argument(
         "--mem", default=MEM,

requirements-testing.txt

@@ -0,0 +1,3 @@
+nose>=1.0
+future
+ushlex

requirements.txt

@@ -1,3 +1 @@
-nose>=1.0
 future
-six

setup.py

@@ -11,7 +11,7 @@
 
 setup(
     name='qbatch',
-    version='2.2',
+    version='2.2.1',
     description='Execute shell command lines in parallel on Slurm, '
     'S(un|on of) Grid Engine (SGE) and PBS/Torque clusters',
     author="Jon Pipitone, Gabriel A. Devenyi",
@@ -40,7 +40,6 @@
         'Topic :: Utilities',
     ],
     install_requires=[
-        "six",
         "future",
     ],
 )

test/test_qbatch.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
@@ -9,7 +10,11 @@
 from builtins import range
 import os
 import shutil
-import shlex
+import sys
+if sys.version_info < (3, 0):
+    import ushlex as shlex
+else:
+    import shlex
 from subprocess import Popen, PIPE, STDOUT
 import tempfile
 import sys
@@ -201,3 +206,18 @@ def test_run_qbatch_local_piped_commands():
         "Return code = {0}".format(p.returncode)
     assert set(out.splitlines()) == set(expected.splitlines()), \
         "Expected {0} but got {1}".format(expected, out)
+
+def test_run_qbatch_local_piped_commands_utf8():
+    cmds = "\n".join(["echo hëllo"] * 24)
+    p = command_pipe('qbatch -N tëst_run_qbatch_local_piped_commands --env none -j2 -b local -')
+    out, _ = p.communicate(cmds.encode('utf-8'))
+
+    expected, _ = command_pipe(
+        'parallel --tag --line-buffer -j2').communicate(cmds.encode('utf-8'))
+
+    print(out)
+
+    assert p.returncode == 0, \
+        "Return code = {0}".format(p.returncode)
+    assert set(out.splitlines()) == set(expected.splitlines()), \
+        "Expected {0} but got {1}".format(expected, out)