-
- | 硬件 |
- 硬件对应的接口 |
- 可用的推理引擎 |
- 推理引擎对应的接口 |
- 是否支持 Paddle 新格式量化模型 |
- 是否支持 FP16 模式 |
-
-
- | CPU |
- use_cpu() |
- Paddle Inference |
- use_paddle_infer_backend() |
- ✅ |
- N/A |
-
-
- | ONNX Runtime |
- use_ort_backend() |
- ✅ |
- N/A |
-
-
- | OpenVINO |
- use_openvino_backend() |
- ❔ |
- N/A |
-
-
- | GPU |
- use_gpu() |
- Paddle Inference |
- use_paddle_infer_backend() |
- ✅ |
- N/A |
-
-
- | ONNX Runtime |
- use_ort_backend() |
- ✅ |
- ❔ |
-
-
- | Paddle TensorRT |
- use_paddle_infer_backend() + paddle_infer_option.enable_trt = True |
- ✅ |
- ✅ |
-
-
- | TensorRT |
- use_trt_backend() |
- ✅ |
- ✅ |
-
-
- | 昆仑芯 XPU |
- use_kunlunxin() |
- Paddle Lite |
- use_paddle_lite_backend() |
- N/A |
- ✅ |
-
-
- | 华为 昇腾 |
- use_ascend() |
- Paddle Lite |
- use_paddle_lite_backend() |
- ❔ |
- ✅ |
-
-
- | Graphcore IPU |
- use_ipu() |
- Paddle Inference |
- use_paddle_infer_backend() |
- ❔ |
- N/A |
-
-
diff --git a/slm/model_zoo/ernie-1.0/finetune/deploy/infer.py b/slm/model_zoo/ernie-1.0/finetune/deploy/infer.py
new file mode 100644
index 000000000000..fa83e0eb535c
--- /dev/null
+++ b/slm/model_zoo/ernie-1.0/finetune/deploy/infer.py
@@ -0,0 +1,112 @@
+# Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+
+import numpy as np
+from paddle import inference
+from scipy.special import softmax
+
+from paddlenlp.datasets import load_dataset
+from paddlenlp.transformers import AutoTokenizer
+from paddlenlp.utils.env import (
+ PADDLE_INFERENCE_MODEL_SUFFIX,
+ PADDLE_INFERENCE_WEIGHTS_SUFFIX,
+)
+
+
+def parse_arguments():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--model_dir", required=True, help="The directory of model.")
+ parser.add_argument("--vocab_path", type=str, default="", help="The path of tokenizer vocab.")
+ parser.add_argument("--model_prefix", type=str, default="model", help="The model and params file prefix.")
+ parser.add_argument("--device", type=str, default="cpu", choices=["gpu", "cpu"], help="Device type.")
+ parser.add_argument("--batch_size", type=int, default=1, help="The batch size of data.")
+ parser.add_argument("--max_length", type=int, default=128, help="The max length of sequence.")
+ parser.add_argument("--cpu_threads", type=int, default=1, help="Number of threads for CPU.")
+ parser.add_argument("--device_id", type=int, default=0, help="GPU device id.")
+ return parser.parse_args()
+
+
+def batchfy_text(texts, batch_size):
+ return [texts[i : i + batch_size] for i in range(0, len(texts), batch_size)]
+
+
+class Predictor(object):
+ def __init__(self, args):
+ if args.vocab_path and os.path.isdir(args.vocab_path):
+ self.tokenizer = AutoTokenizer.from_pretrained(args.vocab_path)
+ else:
+ self.tokenizer = AutoTokenizer.from_pretrained("ernie-1.0")
+
+ model_file = os.path.join(args.model_dir, args.model_prefix + f"{PADDLE_INFERENCE_MODEL_SUFFIX}")
+ params_file = os.path.join(args.model_dir, args.model_prefix + f"{PADDLE_INFERENCE_WEIGHTS_SUFFIX}")
+
+ config = inference.Config(model_file, params_file)
+ if args.device == "gpu":
+ config.enable_use_gpu(100, args.device_id)
+ else:
+ config.disable_gpu()
+ config.set_cpu_math_library_num_threads(args.cpu_threads)
+
+ config.switch_use_feed_fetch_ops(False)
+ self.predictor = inference.create_predictor(config)
+ self.input_handles = [self.predictor.get_input_handle(name) for name in self.predictor.get_input_names()]
+ self.output_handle = self.predictor.get_output_handle(self.predictor.get_output_names()[0])
+
+ self.batch_size = args.batch_size
+ self.max_length = args.max_length
+
+ def preprocess(self, texts):
+ encoded = self.tokenizer(
+ texts, max_length=self.max_length, padding=True, truncation=True, return_token_type_ids=True
+ )
+ input_ids = np.array(encoded["input_ids"], dtype="int64")
+ token_type_ids = np.array(encoded["token_type_ids"], dtype="int64")
+ return input_ids, token_type_ids
+
+ def infer(self, input_ids, token_type_ids):
+ self.input_handles[0].copy_from_cpu(input_ids)
+ self.input_handles[1].copy_from_cpu(token_type_ids)
+ self.predictor.run()
+ return self.output_handle.copy_to_cpu()
+
+ def postprocess(self, logits):
+ probs = softmax(logits, axis=1)
+ return {"label": probs.argmax(axis=1), "confidence": probs}
+
+ def predict(self, texts):
+ input_ids, token_type_ids = self.preprocess(texts)
+ logits = self.infer(input_ids, token_type_ids)
+ return self.postprocess(logits)
+
+
+if __name__ == "__main__":
+ args = parse_arguments()
+ predictor = Predictor(args)
+ test_ds = load_dataset("chnsenticorp", splits=["test"])
+ texts_ds = [d["text"] for d in test_ds]
+ label_map = {0: "negative", 1: "positive"}
+ batch_texts = batchfy_text(texts_ds, args.batch_size)
+
+ for bs, texts in enumerate(batch_texts):
+ outputs = predictor.predict(texts)
+ for i, sentence in enumerate(texts):
+ label = outputs["label"][i]
+ confidence = outputs["confidence"][i]
+ print(
+ f"Batch id: {bs}, example id: {i}, sentence: {sentence}, "
+ f"label: {label_map[label]}, negative prob: {confidence[0]:.4f}, positive prob: {confidence[1]:.4f}."
+ )
diff --git a/slm/model_zoo/ernie-1.0/finetune/deploy/seq_cls_infer.py b/slm/model_zoo/ernie-1.0/finetune/deploy/seq_cls_infer.py
deleted file mode 100644
index 2ba53e76b24c..000000000000
--- a/slm/model_zoo/ernie-1.0/finetune/deploy/seq_cls_infer.py
+++ /dev/null
@@ -1,149 +0,0 @@
-# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import distutils.util
-import os
-
-import fastdeploy as fd
-import numpy as np
-
-from paddlenlp.datasets import load_dataset
-from paddlenlp.transformers import AutoTokenizer
-
-
-def parse_arguments():
- import argparse
-
- parser = argparse.ArgumentParser()
- parser.add_argument("--model_dir", required=True, help="The directory of model.")
- parser.add_argument("--vocab_path", type=str, default="", help="The path of tokenizer vocab.")
- parser.add_argument("--model_prefix", type=str, default="model", help="The model and params file prefix.")
- parser.add_argument(
- "--device",
- type=str,
- default="cpu",
- choices=["gpu", "cpu"],
- help="Type of inference device, support 'cpu' or 'gpu'.",
- )
- parser.add_argument(
- "--backend",
- type=str,
- default="paddle",
- choices=["onnx_runtime", "paddle", "openvino", "tensorrt", "paddle_tensorrt"],
- help="The inference runtime backend.",
- )
- parser.add_argument("--batch_size", type=int, default=1, help="The batch size of data.")
- parser.add_argument("--max_length", type=int, default=128, help="The max length of sequence.")
- parser.add_argument("--log_interval", type=int, default=10, help="The interval of logging.")
- parser.add_argument("--use_fp16", type=distutils.util.strtobool, default=False, help="Wheter to use FP16 mode")
- parser.add_argument("--cpu_threads", type=int, default=1, help="Number of threads to predict when using cpu.")
- parser.add_argument("--device_id", type=int, default=0, help="Select which gpu device to train model.")
- return parser.parse_args()
-
-
-def batchfy_text(texts, batch_size):
- batch_texts = []
- batch_start = 0
- while batch_start < len(texts):
- batch_texts += [texts[batch_start : min(batch_start + batch_size, len(texts))]]
- batch_start += batch_size
- return batch_texts
-
-
-class Predictor(object):
- def __init__(self, args):
- self.tokenizer = AutoTokenizer.from_pretrained(args.model_dir)
- self.runtime = self.create_fd_runtime(args)
- self.batch_size = args.batch_size
- self.max_length = args.max_length
-
- def create_fd_runtime(self, args):
- option = fd.RuntimeOption()
- model_path = os.path.join(args.model_dir, args.model_prefix + ".pdmodel")
- params_path = os.path.join(args.model_dir, args.model_prefix + ".pdiparams")
- option.set_model_path(model_path, params_path)
- if args.device == "cpu":
- option.use_cpu()
- option.set_cpu_thread_num(args.cpu_threads)
- else:
- option.use_gpu(args.device_id)
- if args.backend == "paddle":
- option.use_paddle_infer_backend()
- elif args.backend == "onnx_runtime":
- option.use_ort_backend()
- elif args.backend == "openvino":
- option.use_openvino_backend()
- else:
- option.use_trt_backend()
- if args.backend == "paddle_tensorrt":
- option.use_paddle_infer_backend()
- option.paddle_infer_option.collect_trt_shape = True
- option.paddle_infer_option.enable_trt = True
- trt_file = os.path.join(args.model_dir, "model.trt")
- option.trt_option.set_shape(
- "input_ids", [1, 1], [args.batch_size, args.max_length], [args.batch_size, args.max_length]
- )
- option.trt_option.set_shape(
- "token_type_ids", [1, 1], [args.batch_size, args.max_length], [args.batch_size, args.max_length]
- )
- if args.use_fp16:
- option.trt_option.enable_fp16 = True
- trt_file = trt_file + ".fp16"
- option.trt_option.serialize_file = trt_file
- return fd.Runtime(option)
-
- def preprocess(self, text):
- data = self.tokenizer(text, max_length=self.max_length, padding=True, truncation=True)
- input_ids_name = self.runtime.get_input_info(0).name
- token_type_ids_name = self.runtime.get_input_info(1).name
- input_map = {
- input_ids_name: np.array(data["input_ids"], dtype="int64"),
- token_type_ids_name: np.array(data["token_type_ids"], dtype="int64"),
- }
- return input_map
-
- def infer(self, input_map):
- results = self.runtime.infer(input_map)
- return results
-
- def postprocess(self, infer_data):
- logits = np.array(infer_data[0])
- max_value = np.max(logits, axis=1, keepdims=True)
- exp_data = np.exp(logits - max_value)
- probs = exp_data / np.sum(exp_data, axis=1, keepdims=True)
- out_dict = {"label": probs.argmax(axis=-1), "confidence": probs}
- return out_dict
-
- def predict(self, texts):
- input_map = self.preprocess(texts)
- infer_result = self.infer(input_map)
- output = self.postprocess(infer_result)
- return output
-
-
-if __name__ == "__main__":
- args = parse_arguments()
- predictor = Predictor(args)
- test_ds = load_dataset("chnsenticorp", splits=["test"])
- texts_ds = [d["text"] for d in test_ds]
- label_map = {0: "negative", 1: "positive"}
- batch_texts = batchfy_text(texts_ds, args.batch_size)
- for bs, texts in enumerate(batch_texts):
- outputs = predictor.predict(texts)
- for i, sentence1 in enumerate(texts):
- print(
- f"Batch id: {bs}, example id: {i}, sentence: {sentence1}, "
- f"label: {label_map[outputs['label'][i]]}, negative prob: {outputs['confidence'][i][0]:.4f}, "
- f"positive prob: {outputs['confidence'][i][1]:.4f}."
- )