Make the build python scripts compatible with Python 3.xlinaro-master

Builds have been tested with both Python 3.5 and 2.7 - 3.x builds
still fail because of protobuf issues in external/nanopb-c, but the
build scripts themselves are working with both interpreters.

Change-Id: Iec45c120e069b9639661e99fd2500d8866418683
Signed-off-by: Bernhard Rosenkränzer <Bernhard.Rosenkranzer@linaro.org>

8 files changed, 83 insertions, 56 deletions

diff --git a/core/tasks/check_boot_jars/check_boot_jars.py b/core/tasks/check_boot_jars/check_boot_jars.py
index 5a0ec40163..1202bcf246 100755
--- a/core/tasks/check_boot_jars/check_boot_jars.py
+++ b/core/tasks/check_boot_jars/check_boot_jars.py
@@ -5,6 +5,7 @@ Check boot jars.
 
 Usage: check_boot_jars.py <package_whitelist_file> <jar1> <jar2> ...
 """
+from __future__ import print_function
 import logging
 import os.path
 import re
@@ -50,20 +51,22 @@ def CheckJar(jar):
     return False
   items = stdout.split()
   for f in items:
-    if f.endswith('.class'):
+    if f.endswith(b'.class'):
       package_name = os.path.dirname(f)
+      if sys.version_info[0] >= 3:
+        package_name = package_name.decode("utf-8")
       package_name = package_name.replace('/', '.')
       # Skip class without a package name
       if package_name and not whitelist_re.match(package_name):
-        print >> sys.stderr, ('Error: %s: unknown package name of class file %s'
-                              % (jar, f))
+        print(('Error: %s: unknown package name of class file %s'
+                              % (jar, f)), file=sys.stderr)
         return False
   return True
 
 
 def main(argv):
   if len(argv) < 2:
-    print __doc__
+    print(__doc__)
     return 1
 
   if not LoadWhitelist(argv[0]):
diff --git a/tools/check_radio_versions.py b/tools/check_radio_versions.py
index ebe621f466..1c236687c4 100755
--- a/tools/check_radio_versions.py
+++ b/tools/check_radio_versions.py
@@ -53,7 +53,7 @@ for item in sys.argv[2:]:
     f = open(fn + ".sha1")
   except IOError:
     if not bad: print
-    print "*** Error opening \"%s.sha1\"; can't verify %s" % (fn, key)
+    print("*** Error opening \"%s.sha1\"; can't verify %s" % (fn, key))
     bad = True
     continue
   for line in f:
@@ -64,16 +64,16 @@ for item in sys.argv[2:]:
 
   if digest not in versions:
     if not bad: print
-    print "*** SHA-1 hash of \"%s\" doesn't appear in \"%s.sha1\"" % (fn, fn)
+    print("*** SHA-1 hash of \"%s\" doesn't appear in \"%s.sha1\"" % (fn, fn))
     bad = True
     continue
 
   if versions[digest] not in values:
     if not bad: print
-    print "*** \"%s\" is version %s; not any %s allowed by \"%s\"." % (
-        fn, versions[digest], key, sys.argv[1])
+    print("*** \"%s\" is version %s; not any %s allowed by \"%s\"." % (
+        fn, versions[digest], key, sys.argv[1]))
     bad = True
 
 if bad:
-  print
+  print()
   sys.exit(1)
diff --git a/tools/diff_package_overlays.py b/tools/diff_package_overlays.py
index 0e2c773d1a..7d4358c8df 100755
--- a/tools/diff_package_overlays.py
+++ b/tools/diff_package_overlays.py
@@ -85,7 +85,7 @@ def main(argv):
 
   # Print out the package names that have overlay change.
   for r in result:
-    print r
+    print(r)
 
 if __name__ == "__main__":
   main(sys.argv)
diff --git a/tools/event_log_tags.py b/tools/event_log_tags.py
index 645839ec29..57aae38a1b 100644
--- a/tools/event_log_tags.py
+++ b/tools/event_log_tags.py
@@ -14,6 +14,7 @@
 
 """A module for reading and parsing event-log-tags files."""
 
+from __future__ import print_function
 import re
 import sys
 
@@ -55,7 +56,7 @@ class TagFile(object):
     if file_object is None:
       try:
         file_object = open(filename, "rb")
-      except (IOError, OSError), e:
+      except (IOError, OSError) as e:
         self.AddError(str(e))
         return
 
@@ -64,6 +65,8 @@ class TagFile(object):
         self.linenum += 1
 
         line = line.strip()
+        if sys.version_info[0] >= 3:
+          line = str(line, "utf-8")
         if not line or line[0] == '#': continue
         parts = re.split(r"\s+", line, 2)
 
@@ -100,7 +103,7 @@ class TagFile(object):
 
         self.tags.append(Tag(tag, tagname, description,
                              self.filename, self.linenum))
-    except (IOError, OSError), e:
+    except (IOError, OSError) as e:
       self.AddError(str(e))
 
 
@@ -120,7 +123,7 @@ def WriteOutput(output_file, data):
   """Write 'data' to the given output filename (which may be None to
   indicate stdout).  Emit an error message and die on any failure.
   'data' may be a string or a StringIO object."""
-  if not isinstance(data, str):
+  if not isinstance(data, str) and not isinstance(data, bytes):
     data = data.getvalue()
   try:
     if output_file is None:
@@ -128,8 +131,14 @@ def WriteOutput(output_file, data):
       output_file = "<stdout>"
     else:
       out = open(output_file, "wb")
-    out.write(data)
+    if isinstance(data, bytes):
+      out.write(data)
+    else:
+      if sys.version_info[0] < 3:
+        out.write(str(data))
+      else:
+        out.write(str(data, "utf-8"))
     out.close()
-  except (IOError, OSError), e:
-    print >> sys.stderr, "failed to write %s: %s" % (output_file, e)
+  except (IOError, OSError) as e:
+    print("failed to write %s: %s" % (output_file, e), file=sys.stderr)
     sys.exit(1)
diff --git a/tools/java-event-log-tags.py b/tools/java-event-log-tags.py
index f364751c4f..98f39c3d0f 100755
--- a/tools/java-event-log-tags.py
+++ b/tools/java-event-log-tags.py
@@ -23,7 +23,8 @@ tags in the given input file.
 -h to display this usage message and exit.
 """
 
-import cStringIO
+from __future__ import print_function
+from io import StringIO
 import getopt
 import os
 import os.path
@@ -34,26 +35,31 @@ import event_log_tags
 
 output_file = None
 
+if sys.version_info[0] >= 3:
+  # Py3 strings are already unicode, so...
+  def unicode(x):
+    return x
+
 try:
   opts, args = getopt.getopt(sys.argv[1:], "ho:")
-except getopt.GetoptError, err:
-  print str(err)
-  print __doc__
+except getopt.GetoptError as err:
+  print(str(err))
+  print(__doc__)
   sys.exit(2)
 
 for o, a in opts:
   if o == "-h":
-    print __doc__
+    print(__doc__)
     sys.exit(2)
   elif o == "-o":
     output_file = a
   else:
-    print >> sys.stderr, "unhandled option %s" % (o,)
+    print("unhandled option %s" % (o,), file=sys.stderr)
     sys.exit(1)
 
 if len(args) != 2:
-  print "need exactly two input files, not %d" % (len(args),)
-  print __doc__
+  print("need exactly two input files, not %d" % (len(args),))
+  print(__doc__)
   sys.exit(1)
 
 fn = args[0]
@@ -85,32 +91,32 @@ if "javadoc_hide" in tagfile.options:
 
 if tagfile.errors:
   for fn, ln, msg in tagfile.errors:
-    print >> sys.stderr, "%s:%d: error: %s" % (fn, ln, msg)
+    print("%s:%d: error: %s" % (fn, ln, msg), file=sys.stderr)
   sys.exit(1)
 
-buffer = cStringIO.StringIO()
+buffer = StringIO()
 buffer.write("/* This file is auto-generated.  DO NOT MODIFY.\n"
              " * Source file: %s\n"
-             " */\n\n" % (fn,))
+             " */\n\n" % (unicode(fn),))
 
-buffer.write("package %s;\n\n" % (tagfile.options["java_package"][0],))
+buffer.write("package %s;\n\n" % (unicode(tagfile.options["java_package"][0]),))
 
 basename, _ = os.path.splitext(os.path.basename(fn))
 
 if hide:
-  buffer.write("/**\n"
+  buffer.write(u"/**\n"
                " * @hide\n"
                " */\n")
-buffer.write("public class %s {\n" % (basename,))
-buffer.write("  private %s() { }  // don't instantiate\n" % (basename,))
+buffer.write(u"public class %s {\n" % (basename,))
+buffer.write(u"  private %s() { }  // don't instantiate\n" % (basename,))
 
 for t in tagfile.tags:
   if t.description:
-    buffer.write("\n  /** %d %s %s */\n" % (t.tagnum, t.tagname, t.description))
+    buffer.write(u"\n  /** %d %s %s */\n" % (t.tagnum, t.tagname, t.description))
   else:
-    buffer.write("\n  /** %d %s */\n" % (t.tagnum, t.tagname))
+    buffer.write(u"\n  /** %d %s */\n" % (t.tagnum, t.tagname))
 
-  buffer.write("  public static final int %s = %d;\n" %
+  buffer.write(u"  public static final int %s = %d;\n" %
                (t.tagname.upper(), t.tagnum))
 
 keywords = frozenset(["abstract", "continue", "for", "new", "switch", "assert",
@@ -138,15 +144,18 @@ for t in tagfile.tags:
     args = []
   argTypesNames = ", ".join([javaTypes[int(arg[1])] + " " + javaName(arg[0]) for arg in args])
   argNames = "".join([", " + javaName(arg[0]) for arg in args])
-  buffer.write("\n  public static void %s(%s) {" % (methodName, argTypesNames))
-  buffer.write("\n    android.util.EventLog.writeEvent(%s%s);" % (t.tagname.upper(), argNames))
-  buffer.write("\n  }\n")
+  buffer.write(u"\n  public static void %s(%s) {" % (methodName, argTypesNames))
+  buffer.write(u"\n    android.util.EventLog.writeEvent(%s%s);" % (t.tagname.upper(), argNames))
+  buffer.write(u"\n  }\n")
 
 
-buffer.write("}\n");
+buffer.write(u"}\n");
 
 output_dir = os.path.dirname(output_file)
 if not os.path.exists(output_dir):
   os.makedirs(output_dir)
 
+if sys.version_info[0] >= 3:
+  buffer=bytes(buffer.getvalue(), 'utf-8')
+
 event_log_tags.WriteOutput(output_file, buffer)
diff --git a/tools/merge-event-log-tags.py b/tools/merge-event-log-tags.py
index 64bad3f813..8e56843a15 100755
--- a/tools/merge-event-log-tags.py
+++ b/tools/merge-event-log-tags.py
@@ -24,7 +24,9 @@ and fails if they do.
 -h to display this usage message and exit.
 """
 
-import cStringIO
+from __future__ import print_function
+from six import iteritems
+from io import StringIO
 import getopt
 try:
   import hashlib
@@ -48,21 +50,21 @@ ASSIGN_LIMIT = 1000000
 
 try:
   opts, args = getopt.getopt(sys.argv[1:], "ho:m:")
-except getopt.GetoptError, err:
-  print str(err)
-  print __doc__
+except getopt.GetoptError as err:
+  print(str(err))
+  print(__doc__)
   sys.exit(2)
 
 for o, a in opts:
   if o == "-h":
-    print __doc__
+    print(__doc__)
     sys.exit(2)
   elif o == "-o":
     output_file = a
   elif o == "-m":
     pre_merged_file = a
   else:
-    print >> sys.stderr, "unhandled option %s" % (o,)
+    print("unhandled option %s" % (o,), file=sys.stderr)
     sys.exit(1)
 
 # Restrictions on tags:
@@ -133,12 +135,12 @@ for fn in args:
 
 if errors:
   for fn, ln, msg in errors:
-    print >> sys.stderr, "%s:%d: error: %s" % (fn, ln, msg)
+    print("%s:%d: error: %s" % (fn, ln, msg), file=sys.stderr)
   sys.exit(1)
 
 if warnings:
   for fn, ln, msg in warnings:
-    print >> sys.stderr, "%s:%d: warning: %s" % (fn, ln, msg)
+    print("%s:%d: warning: %s" % (fn, ln, msg), file=sys.stderr)
 
 # Python's hash function (a) isn't great and (b) varies between
 # versions of python.  Using md5 is overkill here but is the same from
@@ -154,14 +156,14 @@ def hashname(str):
 # If we were provided pre-merged tags (w/ the -m option), then don't
 # ever try to allocate one, just fail if we don't have a number
 
-for name, t in sorted(by_tagname.iteritems()):
+for name, t in sorted(iteritems(by_tagname)):
   if t.tagnum is None:
     if pre_merged_tags:
       try:
         t.tagnum = pre_merged_tags[t.tagname]
       except KeyError:
-        print >> sys.stderr, ("Error: Tag number not defined for tag `%s'."
-            +" Have you done a full build?") % t.tagname
+        print(("Error: Tag number not defined for tag `%s'."
+            +" Have you done a full build?") % t.tagname, file=sys.stderr)
         sys.exit(1)
     else:
       while True:
@@ -174,11 +176,14 @@ for name, t in sorted(by_tagname.iteritems()):
 
 # by_tagnum should be complete now; we've assigned numbers to all tags.
 
-buffer = cStringIO.StringIO()
-for n, t in sorted(by_tagnum.iteritems()):
+buffer = StringIO()
+for n, t in sorted(iteritems(by_tagnum)):
   if t.description:
-    buffer.write("%d %s %s\n" % (t.tagnum, t.tagname, t.description))
+    buffer.write("%d %s %s\n" % (t.tagnum, unicode(t.tagname), unicode(t.description)))
   else:
-    buffer.write("%d %s\n" % (t.tagnum, t.tagname))
+    buffer.write("%d %s\n" % (t.tagnum, unicode(t.tagname)))
+
+if sys.version_info[0] >= 3:
+  buffer=bytes(buffer.getvalue(), 'utf-8')
 
 event_log_tags.WriteOutput(output_file, buffer)
diff --git a/tools/normalize_path.py b/tools/normalize_path.py
index 6c4d5486dc..363df1f1fd 100755
--- a/tools/normalize_path.py
+++ b/tools/normalize_path.py
@@ -22,8 +22,8 @@ import sys
 
 if len(sys.argv) > 1:
   for p in sys.argv[1:]:
-    print os.path.normpath(p)
+    print(os.path.normpath(p))
   sys.exit(0)
 
 for line in sys.stdin:
-  print os.path.normpath(line.strip())
+  print(os.path.normpath(line.strip()))
diff --git a/tools/post_process_props.py b/tools/post_process_props.py
index 9dcaadfd0d..9c1b6d4da0 100755
--- a/tools/post_process_props.py
+++ b/tools/post_process_props.py
@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from six import iteritems
 import sys
 
 # Usage: post_process_props.py file.prop [blacklist_key, ...]
@@ -57,7 +58,7 @@ def validate(prop):
   """
   check_pass = True
   buildprops = prop.to_dict()
-  for key, value in buildprops.iteritems():
+  for key, value in iteritems(buildprops):
     # Check build properties' length.
     if len(key) > PROP_NAME_MAX:
       check_pass = False