Update text_encoder unicode utilities

PiperOrigin-RevId: 193703965

Author: Ryan Sepassi

tensor2tensor/data_generators/text_encoder.py

@@ -60,26 +60,37 @@
 _ESCAPE_CHARS = set(u"\\_u;0123456789")
 
 
-# Conversion between Unicode and UTF-8, if required (on Python2).
-if six.PY2:
-
-  def native_to_unicode(s):
-    return s if isinstance(s, unicode) else s.decode("utf-8")
+# Unicode utility functions that work with Python 2 and 3
+def native_to_unicode(s):
+  return s if is_unicode(s) else to_unicode(s)
 
-  def unicode_to_native(s):
-    return s.encode("utf-8") if isinstance(s, unicode) else s
-else:  # No conversion required on Python >= 3.
 
-  def native_to_unicode(s):
+def unicode_to_native(s):
+  if six.PY2:
+    return s.encode("utf-8") if is_unicode(s) else s
+  else:
     return s
 
-  def unicode_to_native(s):
+
+def is_unicode(s):
+  if six.PY2:
+    if isinstance(s, unicode):
+      return True
+  else:
+    if isinstance(s, str):
+      return True
+  return False
+
+
+def to_unicode(s, ignore_errors=False):
+  if is_unicode(s):
     return s
+  error_mode = "ignore" if ignore_errors else "strict"
+  return s.decode("utf-8", errors=error_mode)
 
 
 def to_unicode_ignore_errors(s):
-  return (unicode(s, "utf-8", errors="ignore")
-          if six.PY2 else s.decode("utf-8", "ignore"))
+  return to_unicode(s, ignore_errors=True)
 
 
 class TextEncoder(object):

tensor2tensor/utils/cloud_tpu.py

@@ -29,6 +29,7 @@
 import time
 
 from six.moves import input  # pylint: disable=redefined-builtin
+from tensor2tensor.data_generators import text_encoder
 import tensorflow as tf
 
 TPU_IP = "10.240.%d.2"
@@ -216,7 +217,7 @@ def shell_background(cmd_, **kwargs):
 
 
 def shell_output(cmd_, **kwargs):
-  return sp.check_output(format_cmd(cmd_, **kwargs))
+  return text_encoder.to_unicode(sp.check_output(format_cmd(cmd_, **kwargs)))
 
 
 def shell_run(cmd_, **kwargs):