Make Transformer decoding stable (don't fail on each Unicode error, work around missing hparams).

PiperOrigin-RevId: 219209875

Author: Lukasz Kaiser

tensor2tensor/data_generators/text_encoder.py

@@ -60,7 +60,14 @@
 
 # Unicode utility functions that work with Python 2 and 3
 def native_to_unicode(s):
-  return s if is_unicode(s) else to_unicode(s)
+  if is_unicode(s):
+    return s
+  try:
+    return to_unicode(s)
+  except UnicodeDecodeError:
+    res = to_unicode(s, ignore_errors=True)
+    tf.logging.info("Ignoring Unicode error, outputting: %s" % res)
+    return res
 
 
 def unicode_to_native(s):
@@ -1057,4 +1064,3 @@ def decode(self, ids, strip_extraneous=False):
     """
     del strip_extraneous
     return " ".join([str(i) for i in ids])
-

tensor2tensor/data_generators/translate.py

@@ -81,7 +81,8 @@ def compute_bleu_summaries(hook_args):
   decode_hparams = hook_args.decode_hparams
   estimator = hook_args.estimator
   current_step = estimator.get_variable_value(tf.GraphKeys.GLOBAL_STEP)
-  if current_step and decode_hparams.iterations_per_loop:
+  has_iters = hasattr(decode_hparams, "iterations_per_loop")
+  if current_step and has_iters and decode_hparams.iterations_per_loop:
     iterations_per_loop = decode_hparams.iterations_per_loop
     current_epoch = np.asscalar(current_step) // iterations_per_loop
   else: