Remove num_hidden_layers for baichuan (#10655)

* remove num_hidden_layers for baichuan

* add enable_linear_fused_grad_add

* remove fused_linear_param_grad_add

* add tensor_fusion

* add fused_linear_param_grad_add

Author: waliwali777

tests/test_tipc/static/auto_parallel/baichuan2/N4C32/baichuan-inc-baichuan-2-13b_pretrain_dynamic_auto_bs32_bf16_DP1_MP4_PP1_Sharding8_Stage1.sh

@@ -21,6 +21,8 @@ param+="model_type=baichuan2_13b "
 param+='dynamic_auto=_dynamic_auto '
 
 export FLAGS_fuse_reducescatter_in_opt=1
+export FLAGS_enable_sharding_overlap=1
+export FLAGS_enable_tensor_fusion=1
 
 cd ./tests
 bash ./test_tipc/static/auto_parallel/baichuan2/benchmark_common/prepare.sh

tests/test_tipc/static/auto_parallel/baichuan2/pretrain_config_baichuan2_13b/pretrain-baichuan2_13b_dynamic_auto.json

@@ -32,25 +32,25 @@
     "per_device_eval_batch_size": 1,
     "recompute": false,
     "recompute_use_reentrant": true,
-    "recompute_granularity": "full",
+    "recompute_granularity": "full_attn",
     "pp_recompute_interval": 0,
     "bf16": true,
     "fp16_opt_level": "O2",
     "amp_master_grad": true,
     "fuse_attention_ffn": true,
     "fuse_attention_qkv": true,
     "use_flash_attention": true,
-    "fused_linear": 1,
+    "fused_linear": true,
     "fused_linear_param_grad_add": 1,
+    "enable_linear_fused_grad_add": true,
     "use_fused_rope": true,
     "use_fused_rms_norm": true,
     "max_seq_length": 4096,
-    "sequence_parallel": 1,
+    "sequence_parallel": true,
     "sharding": "stage1",
     "sharding_parallel_degree": 8,
     "sharding_parallel_config": "enable_tensor_fusion enable_overlap",
     "tensor_parallel_config": "enable_mp_async_allreduce replace_with_parallel_cross_entropy",
     "data_parallel_config": "enable_allreduce_avg_in_gradinent_scale gradient_sync_after_accumulate",
-    "pipeline_parallel_config": "enable_send_recv_overlap enable_split_backward",
-    "num_hidden_layers": 20
+    "pipeline_parallel_config": "enable_send_recv_overlap enable_split_backward"
 }