Support intermediate_api gpt-3 test (#9912)

* split from pr9882

* fix

Author: Function-Samuel

tests/test_tipc/static/auto_parallel/gpt3/N4C32/intermediate_api_gpt-3-13b_pretrain_dy2st_bs128_bf16_DP4_MP2_PP4.sh

@@ -0,0 +1,26 @@
+# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+param="model_item=intermediate_api_gpt-3-13b_pretrain_dy2st "
+param+="run_mode=DP4_MP2_PP4 "
+param+="device_num=N4C32 "
+param+="global_batch_size=128 "
+param+="nnodes=4 "
+param+="model_type=gpt3_13b "
+param+="intermediate_api=intermediate_api_ "
+
+cd ./tests
+bash ./test_tipc/static/auto_parallel/gpt3/benchmark_common/prepare.sh
+
+bash -c "${param} bash ./test_tipc/static/auto_parallel/gpt3/benchmark_common/run_benchmark.sh"

tests/test_tipc/static/auto_parallel/gpt3/benchmark_common/run_benchmark.sh

@@ -24,6 +24,9 @@ function _set_params(){
     fp_item="bf16"
     MODEL_TYPE=${model_type:-"gpt3_13b"}
 
+    # for intermediate api
+    intermediate_api=${intermediate_api:-""}
+
     ip_lists=($(echo $TRAINER_INSTANCES | tr ',' ' '))
     master_ip=${ip_lists[0]}
     nnodes=${nnodes:-1}
@@ -174,17 +177,17 @@ function _train(){
         train_cmd="python -u -m paddle.distributed.launch --gpus=0,1,2,3,4,5,6,7 \
             --nnodes 1 --nproc_per_node 8 \
             --log_dir mylog run_pretrain_auto.py \
-            ./pretrain_config_${MODEL_TYPE}/pretrain-${MODEL_TYPE}.json"
+            ./pretrain_config_${MODEL_TYPE}/${intermediate_api}pretrain-${MODEL_TYPE}.json"
         ;;
     N4C32) echo "Run with: device_num=${device_num} run_mode=${run_mode}"
         train_cmd="python -u -m paddle.distributed.launch --gpus=0,1,2,3,4,5,6,7 \
             --log_dir mylog run_pretrain_auto.py \
-            ./pretrain_config_${MODEL_TYPE}/pretrain-${MODEL_TYPE}.json"
+            ./pretrain_config_${MODEL_TYPE}/${intermediate_api}pretrain-${MODEL_TYPE}.json"
         ;;
     *) echo "Run with: device_num=${device_num}, run_mode=${run_mode}"
         train_cmd="python -u -m paddle.distributed.launch --gpus=0,1,2,3,4,5,6,7 \
             --log_dir mylog run_pretrain_auto.py \
-            ./pretrain_config_${MODEL_TYPE}/pretrain-${MODEL_TYPE}.json"
+            ./pretrain_config_${MODEL_TYPE}/${intermediate_api}pretrain-${MODEL_TYPE}.json"
         ;;
     esac
     cd ../llm/auto_parallel/gpt-3

tests/test_tipc/static/auto_parallel/gpt3/pretrain_config_gpt3_13b/intermediate_api_pretrain-gpt3_13b.json

@@ -0,0 +1,55 @@
+{
+    "model_name_or_path": "gpt3-13B-en",
+    "tokenizer_name_or_path": "gpt3-13B-en",
+    "to_static": true,
+    "enable_auto_parallel": 1,
+    "input_dir": "./data",
+    "output_dir": "./checkpoints/gpt_pretrain_ckpts",
+    "split": "949,50,1",
+    "max_seq_length": 4096,
+    "tensor_parallel_degree": 2,
+    "pipeline_parallel_degree": 4,
+    "per_device_train_batch_size": 1,
+    "per_device_eval_batch_size": 1,
+    "scale_loss": 1024,
+    "learning_rate": 0.00001,
+    "min_learning_rate": 0.000001,
+    "max_steps": 500,
+    "save_steps": 50000,
+    "weight_decay": 0.01,
+    "warmup_ratio": 0.01,
+    "logging_steps": 5,
+    "continue_training": 0,
+    "dataloader_num_workers": 1,
+    "eval_steps": 100000,
+    "report_to": "visualdl",
+    "disable_tqdm": true,
+    "do_train": true,
+    "do_eval": true,
+    "device": "gpu",
+    "model_type": "gpt_network",
+    "sequence_parallel": 1,
+    "use_flash_attention": 1,
+    "use_fast_layer_norm": 1,
+    "fused_linear": 1,
+    "fuse_attention_ffn": 1,
+    "fuse_attention_qkv": 1,
+    "fused_linear_param_grad_add": 1,
+    "use_fused_rope": true,
+    "use_fused_rms_norm": true,
+    "recompute": 0,
+    "recompute_use_reentrant": true,
+    "recompute_granularity": "full",
+    "pp_recompute_interval": 1,
+    "gradient_accumulation_steps": 32,
+    "max_grad_norm": 1.0,
+    "bf16": 1,
+    "fp16_opt_level": "O2",
+    "amp_master_grad": true,
+    "attention_probs_dropout_prob": 0.1,
+    "hidden_dropout_prob": 0.1,
+    "tensor_parallel_config": "enable_mp_async_allreduce replace_with_parallel_cross_entropy",
+    "data_parallel_config": "enable_allreduce_avg_in_gradinent_scale gradient_sync_after_accumulate",
+    "pipeline_parallel_config": "enable_send_recv_overlap enable_split_backward",
+    "use_intermediate_api": true
+}