t2t-decoder with --checkpoint_path (#524)

* no pip download progress bars in Travis log

see #523

* allow specifying --checkpoint_path with t2t-decoder

and allow keeping timestamp in that case.
This is needed for t2t-translate-all + t2t-bleu to work
as expected (I forgot to add this commit to #488).

* prevent tf.gfile.Glob crashes due to concurrent filesystem edits

tf.gfile.Glob may crash with
 tensorflow.python.framework.errors_impl.NotFoundError:
 xy/model.ckpt-1130761_temp_9cb4cb0b0f5f4382b5ea947aadfb7a40; No such file or directory
Let's use standard glob.glob instead, it seems to be more reliable.

* reintroducing FLAGS deleted by someone

this is needed for **locals() to work

* speedup BLEU tokenization

As I think about it, I would prefer my original implementation
https://github.com/tensorflow/tensor2tensor/blob/bb1173adce940e62c840970fa0f06f69fd9398db/tensor2tensor/utils/bleu_hook.py#L147-L157
But it seems there are some T2T/Google internal Python guidelines forbidding this,
so we have to live with the singleton.

* another solution of #523

* make save_checkpoints_secs work again

The functionality was broken during the adoption of TPU trainer_lib.py
instead of the original trainer_utils.py.
Currently, the default is to save checkpoints each 2000 steps,
while in previous T2T versions the default was each 10 minutes.

* adapt according to @rsepassi's review

* Update NotFoundError

Author: martinpopel

.travis.yml

@@ -6,8 +6,8 @@ before_install:
   - sudo apt-get update -qq
   - sudo apt-get install -qq libhdf5-dev
 install:
-  - pip install tensorflow
-  - pip install .[tests]
+  - pip install -q tensorflow
+  - pip install -q .[tests]
 env:
   global:
     - T2T_PROBLEM=algorithmic_reverse_binary40_test

tensor2tensor/bin/t2t_decoder.py

@@ -47,10 +47,14 @@
 FLAGS = flags.FLAGS
 
 # Additional flags in bin/t2t_trainer.py and utils/flags.py
+flags.DEFINE_string("checkpoint_path", None,
+                    "Path to the model checkpoint. Overrides output_dir.")
 flags.DEFINE_string("decode_from_file", None,
                     "Path to the source file for decoding")
 flags.DEFINE_string("decode_to_file", None,
                     "Path to the decoded (output) file")
+flags.DEFINE_bool("keep_timestamp", True,
+                  "Set the mtime of the decoded file to the checkpoint_path+'.index' mtime.")
 flags.DEFINE_bool("decode_interactive", False,
                   "Interactive local inference mode.")
 flags.DEFINE_integer("decode_shards", 1, "Number of decoding replicas.")
@@ -76,7 +80,11 @@ def decode(estimator, hparams, decode_hp):
     decoding.decode_interactively(estimator, hparams, decode_hp)
   elif FLAGS.decode_from_file:
     decoding.decode_from_file(estimator, FLAGS.decode_from_file, hparams,
-                              decode_hp, FLAGS.decode_to_file)
+                              decode_hp, FLAGS.decode_to_file,
+                              checkpoint_path=FLAGS.checkpoint_path)
+    if FLAGS.checkpoint_path and FLAGS.keep_timestamp:
+      ckpt_time = os.path.getmtime(FLAGS.checkpoint_path + '.index')
+      os.utime(FLAGS.decode_to_file, (ckpt_time, ckpt_time))
   else:
     decoding.decode_from_dataset(
         estimator,

tensor2tensor/bin/t2t_translate_all.py

@@ -81,10 +81,12 @@ def main(_):
   if not os.path.exists(flags_path):
     shutil.copy2(os.path.join(model_dir, "flags.txt"), flags_path)
 
+  locals_and_flags = {'FLAGS': FLAGS}
   for model in bleu_hook.stepfiles_iterator(model_dir, FLAGS.wait_minutes,
                                             FLAGS.min_steps):
     tf.logging.info("Translating " + model.filename)
     out_file = translated_base_file + "-" + str(model.steps)
+    locals_and_flags.update(locals())
     if os.path.exists(out_file):
       tf.logging.info(out_file + " already exists, so skipping it.")
     else:
@@ -96,7 +98,7 @@ def main(_):
           "--model={FLAGS.model} --hparams_set={FLAGS.hparams_set} "
           "--checkpoint_path={model.filename} --decode_from_file={source} "
           "--decode_to_file={out_file}"
-      ).format(**locals())
+      ).format(**locals_and_flags)
       command = FLAGS.decoder_command.format(**locals())
       tf.logging.info("Running:\n" + command)
       os.system(command)

tensor2tensor/utils/bleu_hook.py

@@ -24,6 +24,7 @@
 import re
 import sys
 import time
+import glob
 import unicodedata
 
 # Dependency imports
@@ -158,6 +159,7 @@ def property_chars(self, prefix):
     return "".join(six.unichr(x) for x in range(sys.maxunicode)
                    if unicodedata.category(six.unichr(x)).startswith(prefix))
 
+uregex = UnicodeRegex()
 
 def bleu_tokenize(string):
   r"""Tokenize a string following the official BLEU implementation.
@@ -183,7 +185,6 @@ def bleu_tokenize(string):
   Returns:
     a list of tokens
   """
-  uregex = UnicodeRegex()
   string = uregex.nondigit_punct_re.sub(r"\1 \2 ", string)
   string = uregex.punct_nondigit_re.sub(r" \1 \2", string)
   string = uregex.symbol_re.sub(r" \1 ", string)
@@ -205,11 +206,24 @@ def bleu_wrapper(ref_filename, hyp_filename, case_sensitive=False):
 
 StepFile = collections.namedtuple("StepFile", "filename mtime ctime steps")
 
+def _try_twice_tf_glob(pattern):
+  """tf.gfile.Glob may crash with
+  tensorflow.python.framework.errors_impl.NotFoundError:
+  xy/model.ckpt-1130761_temp_9cb4cb0b0f5f4382b5ea947aadfb7a40;
+  No such file or directory
+
+  Standard glob.glob does not have this bug, but does not hangle gs://...
+  So let's use tf.gfile.Glob twice to handle most concurrency problems.
+  """
+  try:
+    return tf.gfile.Glob(pattern)
+  except tf.errors.NotFoundError:
+    return tf.gfile.Glob(pattern)
 
 def _read_stepfiles_list(path_prefix, path_suffix=".index", min_steps=0):
   """Return list of StepFiles sorted by step from files at path_prefix."""
   stepfiles = []
-  for filename in tf.gfile.Glob(path_prefix + "*-[0-9]*" + path_suffix):
+  for filename in _try_twice_tf_glob(path_prefix + '*-[0-9]*' + path_suffix):
     basename = filename[:-len(path_suffix)] if len(path_suffix) else filename
     try:
       steps = int(basename.rsplit("-")[-1])

tensor2tensor/utils/decoding.py

@@ -219,7 +219,8 @@ def decode_from_file(estimator,
                      filename,
                      hparams,
                      decode_hp,
-                     decode_to_file=None):
+                     decode_to_file=None,
+                     checkpoint_path=None):
   """Compute predictions on entries in filename and write them out."""
   if not decode_hp.batch_size:
     decode_hp.batch_size = 32
@@ -248,7 +249,7 @@ def input_fn():
     return _decode_input_tensor_to_features_dict(example, hparams)
 
   decodes = []
-  result_iter = estimator.predict(input_fn)
+  result_iter = estimator.predict(input_fn, checkpoint_path=checkpoint_path)
   for result in result_iter:
     if decode_hp.return_beams:
       beam_decodes = []