[CI]skip unnecessary executions (#10509)

* skip unnecessary executions for unittest

* fix

* fix

* skip unnecessary executions for distribute

* fix unittest

* fix

Author: Liujie0926

scripts/distribute/run_ci.sh

@@ -63,6 +63,7 @@ install_paddle(){
 install_paddlenlp(){
     echo -e "\033[31m ---- Install paddlenlp by set PYTHONPATH  \033"
     export PYTHONPATH=${nlp_dir}:$PYTHONPATH
+    # python -m pip install -r ${nlp_dir}/requirements.txt
     sed -i -e "s/paddlenlp/#paddlenlp/g" model_zoo/gpt-3/requirements.txt
     # export http_proxy=${proxy} && export https_proxy=${proxy}
     # python -m pip uninstall paddlenlp -y
@@ -87,9 +88,9 @@ install_external_ops(){
 
 function is_a100() {
     if [ $(nvidia-smi|grep A100|wc -l)  -ne 0 ];then
-        echo 1
+        echo 1 # A100
     else
-        echo 0
+        echo 0 # not A100
     fi
 }
 
@@ -131,9 +132,6 @@ get_diff_TO_case(){
             fi
         done
     else
-        case_list[${#case_list[*]}]=gpt-3_auto
-        case_list[${#case_list[*]}]=llama_auto
-        case_list[${#case_list[*]}]=deepseek_auto
         for file_name in `git diff --numstat upstream/${AGILE_COMPILE_BRANCH} |awk '{print $NF}'`;do
             arr_file_name=(${file_name//// })
             dir1=${arr_file_name[0]}
@@ -147,6 +145,9 @@ get_diff_TO_case(){
             elif [[ ${file_name##*.} == "md" ]] || [[ ${file_name##*.} == "rst" ]] || [[ ${dir1} == "docs" ]];then
                 continue
             else
+                case_list[${#case_list[*]}]=gpt-3_auto
+                case_list[${#case_list[*]}]=llama_auto
+                case_list[${#case_list[*]}]=deepseek_auto
                 for ((i=0; i<${#target_lists_for_gpt[@]}; i++)); do
                     if [[ ! ${dir3} =~ "benchmarks" ]] && [[ ${file_item} == *${target_lists_for_gpt[i]}* ]];then
                         case_list[${#case_list[*]}]=gpt-3_dygraph

scripts/unit_test/ci_unit.sh

@@ -16,6 +16,7 @@
 
 set -e
 export paddle=$1
+export FLAGS_enable_CE=${2-false}
 export nlp_dir=/workspace/PaddleNLP
 export log_path=/workspace/PaddleNLP/unittest_logs
 cd $nlp_dir
@@ -49,10 +50,17 @@ set_env() {
     export FLAGS_cudnn_deterministic=1
     export HF_ENDPOINT=https://hf-mirror.com
     export FLAGS_use_cuda_managed_memory=true
+    export running_time=30m
+
     # for CE
-    # export CE_TEST_ENV=1
-    # export RUN_SLOW_TEST=1
-    # export PYTHONPATH=${nlp_dir}:${nlp_dir/llm/
+    if [[ ${FLAGS_enable_CE} == "true" ]];then
+        export CE_TEST_ENV=1
+        export RUN_SLOW_TEST=1
+        export PYTHONPATH=${nlp_dir}:${nlp_dir}/llm:${PYTHONPATH}
+        export running_time=5h
+    else
+        continue
+    fi
 }
 
 print_info() {
@@ -74,25 +82,49 @@ print_info() {
     fi
 }
 
-install_requirements
+get_diff_TO_case(){
+export FLAGS_enable_CI=false
+for file_name in `git diff --numstat ${AGILE_COMPILE_BRANCH} |awk '{print $NF}'`;do
+    arr_file_name=(${file_name//// })
+    dir1=${arr_file_name[0]}
+    dir2=${arr_file_name[1]}
+    dir3=${arr_file_name[2]}
+    dir4=${arr_file_name[3]}
+    file_item=$dir1/$dir2/$dir3/$dir4
+    echo "file_name:"${file_name}, "dir1:"${dir1}, "dir2:"${dir2},"dir3:"${dir3},".xx:" ${file_name##*.}
+    if [ ! -f ${file_name} ];then # 针对pr删掉文件
+        continue
+    elif [[ ${file_name##*.} == "md" ]] || [[ ${file_name##*.} == "rst" ]] || [[ ${dir1} == "docs" ]];then
+        continue
+    else
+        FLAGS_enable_CI=true
+    fi
+done
+}
+get_diff_TO_case
 set_env
-cd ${nlp_dir}
-echo ' Testing all unittest cases '
-export http_proxy=${proxy} && export https_proxy=${proxy}
-set +e
-timeout 30m python -m pytest -v -n 8 \
-  --dist loadgroup \
-  --retries 1 --retry-delay 1 \
-  --timeout 200 --durations 20 --alluredir=result \
-  --cov paddlenlp --cov-report xml:coverage.xml > ${log_path}/unittest.log 2>&1
-exit_code=$?
-print_info $exit_code unittest
-
-cd ${nlp_dir}
-echo -e "\033[35m ---- Genrate Allure Report  \033[0m"
-unset http_proxy && unset https_proxy
-cp scripts/regression/gen_allure_report.py ./
-python gen_allure_report.py > /dev/null
-echo -e "\033[35m ---- Report: https://xly.bce.baidu.com/ipipe/ipipe-report/report/${AGILE_JOB_BUILD_ID}/report/  \033[0m"
+if [[ ${FLAGS_enable_CI} == "true" ]] || [[ ${FLAGS_enable_CE} == "true" ]];then
+    install_requirements
+    cd ${nlp_dir}
+    echo ' Testing all unittest cases '
+    export http_proxy=${proxy} && export https_proxy=${proxy}
+    set +e
+    timeout ${running_time} python -m pytest -v -n 8 \
+    --dist loadgroup \
+    --retries 1 --retry-delay 1 \
+    --timeout 200 --durations 20 --alluredir=result \
+    --cov paddlenlp --cov-report xml:coverage.xml > ${log_path}/unittest.log 2>&1
+    exit_code=$?
+    print_info $exit_code unittest
 
+    cd ${nlp_dir}
+    echo -e "\033[35m ---- Genrate Allure Report  \033[0m"
+    unset http_proxy && unset https_proxy
+    cp scripts/regression/gen_allure_report.py ./
+    python gen_allure_report.py > /dev/null
+    echo -e "\033[35m ---- Report: https://xly.bce.baidu.com/ipipe/ipipe-report/report/${AGILE_JOB_BUILD_ID}/report/  \033[0m"
+else
+    echo -e "\033[32m Changed Not CI case, Skips \033[0m"
+    exit_code=0
+fi
 exit $exit_code