Fix python3 incompatibilities (#376)

.travis.yml

@@ -43,6 +43,7 @@ script: |
   time coverage run -a alibuild/aliBuild -c alibuild/tests/testdist build broken6 --no-system --disable GCC-Toolchain 2>&1 | grep scanning
   pushd alibuild
     time coverage run -a tests/test_parseRecipe.py
+    time coverage run -a tests/test_utilities.py
   popd
   time coverage run -a alibuild/aliBuild build zlib --no-system --disable GCC-Toolchain
   alibuild/alienv q

aliBuild

@@ -20,6 +20,7 @@ from alibuild_helpers.log import debug, error, warning, banner, info
 from alibuild_helpers.log import logger_handler, logger, LogFormatter, ProgressPrint, riemannStream
 from alibuild_helpers.utilities import format, getVersion, detectArch, dockerStatusOutput, parseDefaults, readDefaults
 from alibuild_helpers.utilities import parseRecipe, getPackageList, getRecipeReader
+from alibuild_helpers.utilities import Hasher
 from alibuild_helpers.analytics import decideAnalytics, askForAnalytics, report_screenview, report_exception, report_event
 from alibuild_helpers.analytics import enable_analytics, disable_analytics
 import traceback
@@ -47,19 +48,13 @@ def execute(command, printer=debug):
   popen = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
   lines_iterator = iter(popen.stdout.readline, "")
   for line in lines_iterator:
-    printer(line.strip("\n"))  # yield line
+    if not line: break
+    printer(line.decode('utf-8', 'ignore').strip("\n"))  # yield line
   output = popen.communicate()[0]
   printer(output)
   exitCode = popen.returncode
   return exitCode
 
-class Hasher:
-  def __init__(self):
-    self.h = hashlib.sha1()
-  def __call__(self, txt):
-    self.h.update(txt)
-  def hexdigest(self):
-    return self.h.hexdigest()
 
 def dieOnError(err, msg):
   if err:

aliDoctor

@@ -20,8 +20,9 @@ def execute(command):
   lines_iterator = iter(popen.stdout.readline, "")
   txt = ""
   for line in lines_iterator:
-    txt += line # yield line
-  txt += popen.communicate()[0]
+    if not line: break
+    txt += line.decode('utf-8', "ignore") # yield line
+  txt += popen.communicate()[0].decode('utf-8', 'ignore')
   return (popen.returncode, txt)
 
 def prunePaths(workDir):

alibuild_helpers/utilities.py

@@ -7,6 +7,7 @@
 from os.path import dirname, exists
 import platform
 import base64
+import hashlib
 from glob import glob
 from os.path import basename
 
@@ -278,3 +279,11 @@ def dockerStatusOutput(cmd, dockerImage=None, executor=getstatusoutput):
                  di=dockerImage,
                  c=base64.b64encode(cmd))
   return executor(cmd)
+
+class Hasher:
+  def __init__(self):
+    self.h = hashlib.sha1()
+  def __call__(self, txt):
+    self.h.update(txt.encode('utf-8', 'ignore'))
+  def hexdigest(self):
+    return self.h.hexdigest()

tests/test_utilities.py

@@ -1,6 +1,7 @@
 import unittest
 import platform
 from alibuild_helpers.utilities import doDetectArch
+from alibuild_helpers.utilities import Hasher
 
 UBUNTU_1510_OS_RELEASE = """
 NAME="Ubuntu"
@@ -101,11 +102,29 @@
   ['sabayon2_x86-64', True, SABAYON2_OS_RELEASE.split("\n"), ('gentoo', '2.2', ''), 'Linux', 'x86_64']
 ]
 
-class TestArchitectures(unittest.TestCase):
+class TestUtilities(unittest.TestCase):
   def test_osx(self):
     for payload in architecturePayloads:
       result, hasOsRelease, osReleaseLines, platformTuple, platformSystem, platformProcessor = payload
       self.assertEqual(result, doDetectArch(hasOsRelease, osReleaseLines, platformTuple, platformSystem, platformProcessor))
+  def test_Hasher(self):
+    h = Hasher()
+    h("foo")
+    self.assertEqual("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33", h.hexdigest())
+    h("")
+    self.assertEqual("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33", h.hexdigest())
+    self.assertRaises(AttributeError, h, 1)
+    h("bar")
+    self.assertEqual("8843d7f92416211de9ebb963ff4ce28125932878", h.hexdigest())
+
+  def test_UTF8_Hasher(self):
+    h1 = Hasher()
+    h2 = Hasher()
+    h1(u'\ua000')
+    h2(u'\ua001')
+    self.assertEqual(h1.hexdigest(), "2af8e41129115eb231a0af76ec5465d3a9184fc4")
+    self.assertEqual(h2.hexdigest(), "1619bcdbeff6828138ad9b6e43cc17e856457603")
+    self.assertNotEqual(h1.hexdigest(), h2.hexdigest())
 
 if __name__ == '__main__':
     unittest.main()