Replace try_import sentencepiece with direct import sentencepiece (#1961)

* direct import sentencepiece

* direct import sentencepiece

Author: yingyibiao

paddlenlp/transformers/albert/tokenizer.py

@@ -18,7 +18,8 @@
 import unicodedata
 from shutil import copyfile
 
-from paddle.utils import try_import
+import sentencepiece as spm
+
 from .. import PretrainedTokenizer, BertTokenizer, AddedToken
 
 __all__ = ['AlbertTokenizer']
@@ -581,8 +582,6 @@ def __init__(self,
         self.remove_space = remove_space
         self.keep_accents = keep_accents
         self.sentencepiece_model_file = sentencepiece_model_file
-
-        spm = try_import("sentencepiece")
         self.sp_model = spm.SentencePieceProcessor()
         self.sp_model.Load(sentencepiece_model_file)
 
@@ -597,7 +596,6 @@ def __getstate__(self):
 
     def __setstate__(self, d):
         self.__dict__ = d
-        spm = try_import("sentencepiece")
         self.sp_model = spm.SentencePieceProcessor()
         self.sp_model.Load(self.sentencepiece_model_file)
 

paddlenlp/transformers/bart/tokenizer.py

@@ -12,7 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import itertools
+
 from paddle.utils import try_import
 from .. import GPTTokenizer, AddedToken
 

paddlenlp/transformers/bigbird/tokenizer.py

@@ -18,8 +18,10 @@
 import six
 import re
 import numpy as np
-from paddle.utils import try_import
+
+import sentencepiece as spm
 from paddlenlp.data import Vocab
+
 from .. import PretrainedTokenizer, AddedToken
 
 __all__ = ['BigBirdTokenizer']
@@ -95,8 +97,7 @@ def __init__(self,
                 "`tokenizer = BigBirdTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"
                 .format(sentencepiece_model_file))
         self.encoding = encoding
-        mod = try_import('sentencepiece')
-        self.sp_model = mod.SentencePieceProcessor()
+        self.sp_model = spm.SentencePieceProcessor()
         if os.path.isfile(sentencepiece_model_file):
             self.sp_model.Load(sentencepiece_model_file)
         vocab_dict = {}

paddlenlp/transformers/ernie/tokenizer.py

@@ -18,7 +18,8 @@
 import six
 import shutil
 
-from paddle.utils import try_import
+import sentencepiece as spm
+
 from paddlenlp.utils.env import MODEL_HOME
 
 from .. import BasicTokenizer, PretrainedTokenizer, WordpieceTokenizer
@@ -425,8 +426,7 @@ def __init__(self,
                  cls_token="[CLS]",
                  mask_token="[MASK]",
                  **kwargs):
-        mod = try_import('sentencepiece')
-        self.sp_model = mod.SentencePieceProcessor()
+        self.sp_model = spm.SentencePieceProcessor()
         self.word_dict = word_dict
 
         self.do_lower_case = do_lower_case

paddlenlp/transformers/ernie_m/tokenizer.py

@@ -14,7 +14,7 @@
 
 import os
 
-from paddle.utils import try_import
+import sentencepiece as spm
 
 from .. import PretrainedTokenizer
 from ..tokenizer_utils import _is_control, _is_whitespace
@@ -91,8 +91,7 @@ def __init__(self,
                  pad_token="[PAD]",
                  cls_token="[CLS]",
                  mask_token="[MASK]"):
-        mod = try_import('sentencepiece')
-        self.sp_model = mod.SentencePieceProcessor()
+        self.sp_model = spm.SentencePieceProcessor()
 
         self.do_lower_case = do_lower_case
         self.encoding = encoding

paddlenlp/transformers/gpt/tokenizer.py

@@ -19,10 +19,9 @@
 import json
 import jieba
 import shutil
+import sentencepiece as spm
 from paddle.utils import try_import
 
-from paddlenlp.utils.log import logger
-
 from .. import PretrainedTokenizer, AddedToken
 
 __all__ = [
@@ -135,8 +134,8 @@ def __init__(
                 "`tokenizer = GPTTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"
                 .format(model_file))
         self.max_len = max_len if max_len is not None else int(1e12)
-        mod = try_import("sentencepiece")
-        self.sp = mod.SentencePieceProcessor(model_file=model_file)
+        self.sp = spm.SentencePieceProcessor()
+        self.sp.Load(model_file)
         self.translator = str.maketrans(" \n", "\u2582\u2583")
 
     '''

paddlenlp/transformers/layoutxlm/tokenizer.py

@@ -17,9 +17,10 @@
 import itertools
 from dataclasses import dataclass, field
 from collections import OrderedDict
-from paddle.utils import try_import
 from typing import List, Optional
 
+import sentencepiece as spm
+
 from .. import PretrainedTokenizer, AddedToken
 from ..tokenizer_utils import _is_punctuation, _is_control, _is_whitespace
 
@@ -90,7 +91,6 @@ def __init__(self,
         self._unk_token = unk_token
         self._pad_token = pad_token
         self._mask_token = mask_token
-        spm = try_import("sentencepiece")
         self.sp_model = spm.SentencePieceProcessor()
         self.sp_model.Load(vocab_file)
         self.vocab_file = vocab_file

paddlenlp/transformers/mbart/tokenizer.py

@@ -14,7 +14,8 @@
 
 import itertools
 from contextlib import contextmanager
-from paddle.utils import try_import
+import sentencepiece as spm
+
 from .. import PretrainedTokenizer, AddedToken
 
 __all__ = ['MBartTokenizer']
@@ -239,7 +240,6 @@ def __init__(self,
             mask_token, lstrip=True,
             rstrip=False) if isinstance(mask_token, str) else mask_token
         self._build_special_tokens_map_extended(mask_token=mask_token)
-        spm = try_import('sentencepiece')
         self.sp_model = spm.SentencePieceProcessor()
         self.sp_model.Load(str(vocab_file))
         self.fairseq_offset = 1
@@ -432,7 +432,6 @@ def __init__(self,
             mask_token, lstrip=True,
             rstrip=False) if isinstance(mask_token, str) else mask_token
         self._build_special_tokens_map_extended(mask_token=mask_token)
-        spm = try_import('sentencepiece')
         self.sp_model = spm.SentencePieceProcessor()
         self.sp_model.Load(str(vocab_file))
         self.fairseq_offset = 1

paddlenlp/transformers/reformer/tokenizer.py

@@ -12,7 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from paddle.utils import try_import
+import sentencepiece as spm
+
 from ..albert.tokenizer import AlbertEnglishTokenizer
 
 __all__ = ['ReformerTokenizer']
@@ -76,8 +77,6 @@ def __init__(self,
         self.remove_space = remove_space
         self.keep_accents = keep_accents
         self.sentencepiece_model_file = sentencepiece_model_file
-
-        spm = try_import("sentencepiece")
         self.sp_model = spm.SentencePieceProcessor()
         self.sp_model.Load(sentencepiece_model_file)
 

paddlenlp/transformers/rembert/tokenizer.py

@@ -16,10 +16,8 @@
 from shutil import copyfile
 from typing import List, Optional, Tuple
 
-try:
-    import sentencepiece as spm
-except:
-    pass
+import sentencepiece as spm
+
 from .. import PretrainedTokenizer
 
 __all__ = ['RemBertTokenizer']