fix performance issue in convert_ids_to_tokens (#37773)

Author: martin-harmonic

src/transformers/tokenization_utils_fast.py

@@ -410,9 +410,11 @@ def convert_ids_to_tokens(
         if isinstance(ids, int):
             return self._tokenizer.id_to_token(ids)
         tokens = []
+        # self.all_special_ids is an @property which may be slow, so only compute it once before the loop
+        ids_to_skip = set(self.all_special_ids) if skip_special_tokens else set()
         for index in ids:
             index = int(index)
-            if skip_special_tokens and index in self.all_special_ids:
+            if index in ids_to_skip:
                 continue
             tokens.append(self._tokenizer.id_to_token(index))
         return tokens