|
| 1 | +matchedNode = [] |
| 2 | +generateBaselineNode = [] |
| 3 | +for (label in pullRequest.labels) { |
| 4 | + listOfLabelNodeNames = jenkins.model.Jenkins.instance.nodes.collect { |
| 5 | + node -> node.getLabelString().contains(label) ? node.name : null |
| 6 | + |
| 7 | + if ((label.matches(node.getLabelString()+"-(.*)"))) { |
| 8 | + matchedNode += node.getLabelString() |
| 9 | + } |
| 10 | + |
| 11 | + if ((label.matches(node.getLabelString()+"(.*)-BL"))) { |
| 12 | + generateBaselineNode += node.getLabelString() |
| 13 | + } |
| 14 | + } |
| 15 | +} |
| 16 | + |
| 17 | +modifiedLabels = matchedNode.collect{"'" + it + "'"} |
| 18 | +baselineLabels = generateBaselineNode.collect{"'" + it + "'"} |
| 19 | +def generateStage(nodeLabel) { |
| 20 | + return { |
| 21 | + stage("Initialize on ${nodeLabel.replaceAll("'","")}") { |
| 22 | + node(nodeLabel) { |
| 23 | + script { |
| 24 | + currentBuild.displayName = "#${BUILD_NUMBER} ${nodeLabel.replaceAll("'","")} test=${params.WM_OPERATIONAL_TESTS}" |
| 25 | + } |
| 26 | + cleanWs() |
| 27 | + checkout scm |
| 28 | + script { |
| 29 | + def UFS_PLATFORM = nodeLabel.replaceAll("'","") |
| 30 | + echo "nodeLabel=${nodeLabel} NODE_NAME=${NODE_NAME} UFS_PLATFORM=${UFS_PLATFORM} UFS_COMPILER=${env.UFS_COMPILER}" |
| 31 | + sh 'bash --login "${WORKSPACE}/.cicd/scripts/wm_init.sh"' |
| 32 | + sh "STAGE_NAME='${env.STAGE_NAME}' " + 'bash --login "${WORKSPACE}/.cicd/scripts/disk_usage.sh"' |
| 33 | + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${UFS_PLATFORM}-*-time-wm_init.json", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] |
| 34 | + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${UFS_PLATFORM}-*-disk-usageInit*.csv", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] |
| 35 | + } |
| 36 | + } |
| 37 | + } |
| 38 | + stage("Build on ${nodeLabel.replaceAll("'","")}") { |
| 39 | + if (params.WM_BUILD == true ) { |
| 40 | + node(nodeLabel) { |
| 41 | + script { |
| 42 | + def UFS_PLATFORM = nodeLabel.replaceAll("'","") |
| 43 | + //currentBuild.displayName = "#${BUILD_NUMBER} ${nodeLabel.replaceAll("'","")} ${UFS_COMPILER}" |
| 44 | + currentBuild.description = "build ${UFS_PLATFORM}-${UFS_COMPILER}" |
| 45 | + |
| 46 | + echo "Building on ${nodeLabel}" |
| 47 | + sh 'bash --login "${WORKSPACE}/.cicd/scripts/wm_build.sh"' |
| 48 | + sh "STAGE_NAME='${env.STAGE_NAME}' " + 'bash --login "${WORKSPACE}/.cicd/scripts/disk_usage.sh"' |
| 49 | + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${UFS_PLATFORM}-*-time-wm_build.json", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] |
| 50 | + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${UFS_PLATFORM}-*-disk-usageBuild.csv", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] |
| 51 | + } |
| 52 | + } |
| 53 | + } else { |
| 54 | + echo "Building on ${nodeLabel} skipped" |
| 55 | + } |
| 56 | + } |
| 57 | + stage("Test on ${nodeLabel.replaceAll("'","")}") { |
| 58 | + if (params.WM_OPERATIONAL_TESTS != 'none' ) { |
| 59 | + node(nodeLabel) { |
| 60 | + script { |
| 61 | + def UFS_PLATFORM = nodeLabel.replaceAll("'","") |
| 62 | + try { |
| 63 | + echo "Running Tests on ${nodeLabel}" |
| 64 | + if (baselineLabels.contains(nodeLabel)) { |
| 65 | + sh "WM_CREATE_BASELINE=true " + 'bash --login "${WORKSPACE}/.cicd/scripts/wm_test.sh"' |
| 66 | + } |
| 67 | + else { |
| 68 | + sh "WM_CREATE_BASELINE=false " + 'bash --login "${WORKSPACE}/.cicd/scripts/wm_test.sh"' |
| 69 | + } |
| 70 | + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "wm_test_results-*-*.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] |
| 71 | + } |
| 72 | + catch(err) { |
| 73 | + sh ''' |
| 74 | + export machine=${NODE_NAME} |
| 75 | + export CHANGE_ID=${CHANGE_ID} |
| 76 | + GIT_OWNER=$(echo $GIT_URL | cut -d '/' -f4) |
| 77 | + GIT_REPO_NAME=$(echo $GIT_URL | cut -d '/' -f5 | cut -d '.' -f1) |
| 78 | + set +x |
| 79 | +
|
| 80 | + echo "Testing concluded...removing labels for $machine from $GIT_URL" |
| 81 | + echo "https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels /{$machine-RT,$machine-BL}" |
| 82 | + curl --silent -X DELETE -H "Accept: application/vnd.github.v3+json" -H "Authorization: Bearer ${GITHUB_TOKEN}" https://api.github.com/repos/${GIT_OWNER}/${GIT_REPO_NAME}/issues/${CHANGE_ID}/labels/{$machine-RT,$machine-BL} |
| 83 | + ''' |
| 84 | + currentBuild.result = 'FAILURE' |
| 85 | + } |
| 86 | + |
| 87 | + sh ''' |
| 88 | + export machine=${NODE_NAME} |
| 89 | + export CHANGE_ID=${CHANGE_ID} |
| 90 | + cd ${WORKSPACE}/tests |
| 91 | + export machine_name_logs=$(echo $machine | awk '{ print tolower($1) }') |
| 92 | + tar --create --gzip --verbose --dereference --file "${machine_name_logs}.tgz" ${WORKSPACE}/tests/logs/*.log |
| 93 | + ''' |
| 94 | + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: true, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "**/*tgz*", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] |
| 95 | + |
| 96 | + sh "STAGE_NAME='${env.STAGE_NAME}' " + 'bash --login "${WORKSPACE}/.cicd/scripts/disk_usage.sh"' |
| 97 | + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${UFS_PLATFORM}-*-time-wm_test.json", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] |
| 98 | + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${UFS_PLATFORM}-*-disk-usageTest.csv", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] |
| 99 | + |
| 100 | + sh "[[ -d tests/logs ]] && cd tests/logs && tar --create --gzip --verbose --dereference --file ../../wm_test_logs-${UFS_PLATFORM}-${env.UFS_COMPILER}.tgz log_${UFS_PLATFORM}/* RegressionTests_${UFS_PLATFORM}.log || cat /dev/null > ../../wm_test_logs-${UFS_PLATFORM}-${env.UFS_COMPILER}.tgz" |
| 101 | + s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "wm_test_logs-${UFS_PLATFORM}-${env.UFS_COMPILER}.tgz", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: [] |
| 102 | + } |
| 103 | + } |
| 104 | + } else { |
| 105 | + echo "Running Tests on ${nodeLabel} skipped" |
| 106 | + } |
| 107 | + } |
| 108 | + stage("Post from ${nodeLabel.replaceAll("'","")}") { |
| 109 | + if (params.WM_OPERATIONAL_TESTS != 'none' ) { |
| 110 | + node(nodeLabel) { |
| 111 | + script { |
| 112 | + def UFS_PLATFORM = nodeLabel.replaceAll("'","") |
| 113 | + try { |
| 114 | + echo "Post Results from ${nodeLabel}" |
| 115 | + if (baselineLabels.contains(nodeLabel)) { |
| 116 | + //sh 'bash --login "${WORKSPACE}/.cicd/scripts/post_test_results.sh ${NODE_NAME} BL"' |
| 117 | + sh "WM_TEST_LABEL=BL " + 'bash --login "${WORKSPACE}/.cicd/scripts/post_test_results.sh"' |
| 118 | + } |
| 119 | + else { |
| 120 | + sh "WM_TEST_LABEL=RT " + 'bash --login "${WORKSPACE}/.cicd/scripts/post_test_results.sh"' |
| 121 | + } |
| 122 | + } |
| 123 | + catch(err) { |
| 124 | + echo "Error: Post Results from ${nodeLabel}" |
| 125 | + } |
| 126 | + } |
| 127 | + } |
| 128 | + } |
| 129 | + } |
| 130 | + } |
| 131 | +} |
| 132 | + |
| 133 | +def parallelStagesMap = modifiedLabels.collectEntries { |
| 134 | + ["${it}" : generateStage(it)] |
| 135 | +} |
| 136 | + |
| 137 | +pipeline { |
| 138 | + agent none |
| 139 | + parameters { |
| 140 | + booleanParam name: 'WM_BUILD', defaultValue: false, description: 'Whether to attempt to compile the model code tests' |
| 141 | + // Regression Test Suite ? |
| 142 | + choice(name: 'WM_OPERATIONAL_TESTS', choices: ['default', 'control_p8', 'cpld_control_p8', 'comprehensive', 'rt.sh', 'none'], description: 'Specify the suite of tests to run') |
| 143 | + } |
| 144 | + environment { |
| 145 | + ACCNR = 'epic' |
| 146 | + AWS_PROD_ACCOUNT_ID = credentials('AWS_PROD_ACCOUNT_ID') |
| 147 | + AWS_PROD_SNS_TOPIC = credentials('AWS_PROD_SNS_TOPIC') |
| 148 | + GITHUB_TOKEN = credentials('GithubJenkinsNew') |
| 149 | + GIT_URL = 'https://github.com/ufs-community/ufs-weather-model.git' |
| 150 | + UFS_COMPILER = 'intel' |
| 151 | + } |
| 152 | + stages { |
| 153 | + stage('Launch SonarQube') { |
| 154 | + steps { |
| 155 | + script { |
| 156 | + echo "BRANCH_NAME=${env.CHANGE_BRANCH}" |
| 157 | + echo "FORK_NAME=${env.CHANGE_FORK}" |
| 158 | + echo "CHANGE_URL=${env.CHANGE_URL}" |
| 159 | + echo "CHANGE_ID=${env.CHANGE_ID}" |
| 160 | + build job: '/ufs-weather-model/ufs-wm-sonarqube', parameters: [ |
| 161 | + string(name: 'BRANCH_NAME', value: env.CHANGE_BRANCH ?: 'develop'), |
| 162 | + string(name: 'FORK_NAME', value: env.CHANGE_FORK ?: ''), |
| 163 | + string(name: 'CHANGE_URL', value: env.CHANGE_URL ?: ''), |
| 164 | + string(name: 'CHANGE_ID', value: env.CHANGE_ID ?: '') |
| 165 | + ], wait: false |
| 166 | + } |
| 167 | + } |
| 168 | + } |
| 169 | + stage('Run Regression Tests in Parallel') { |
| 170 | + steps { |
| 171 | + script { |
| 172 | + parallel parallelStagesMap |
| 173 | + } |
| 174 | + } |
| 175 | + } |
| 176 | + } |
| 177 | + post { |
| 178 | + success { |
| 179 | + node('built-in') { |
| 180 | + echo 'This will run only if successful.' |
| 181 | + sh ''' |
| 182 | + aws sns publish --topic-arn "arn:aws:sns:us-east-1:${AWS_PROD_ACCOUNT_ID}:${AWS_PROD_SNS_TOPIC}" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":sunny: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'* *succeeded*"}}' |
| 183 | + ''' |
| 184 | + } |
| 185 | + } |
| 186 | + failure { |
| 187 | + node('built-in') { |
| 188 | + echo 'This will run only if the run was marked as unstable.' |
| 189 | + sh ''' |
| 190 | + aws sns publish --topic-arn "arn:aws:sns:us-east-1:${AWS_PROD_ACCOUNT_ID}:${AWS_PROD_SNS_TOPIC}" --region us-east-1 --message '{"version":"1.0","source":"custom","content":{"description":":warning: Jenkins build *'"$JOB_NAME"' '"$BUILD_NUMBER"'* with *PR-'"$CHANGE_ID"'* *failed!*"}}' |
| 191 | + ''' |
| 192 | + } |
| 193 | + } |
| 194 | + always { |
| 195 | + script { |
| 196 | + // Trigger another job to collect all build statistics |
| 197 | + CI_JOB_NAME=env.JOB_NAME.replace("/${env.BRANCH_NAME}","") |
| 198 | + CI_BRANCH_NAME=env.BRANCH_NAME.replace("%2F","%252F") |
| 199 | + echo "post: Triggering ufs-weather-model/ufs-wm-metrics job for ${CI_JOB_NAME} on branch build ${CI_BRANCH_NAME}/${env.BUILD_NUMBER} ..." |
| 200 | + |
| 201 | + build job: '/ufs-weather-model/ufs-wm-metrics', parameters: [ |
| 202 | + string(name: 'CI_JOB_NAME', value: "${CI_JOB_NAME}"), |
| 203 | + string(name: 'CI_BUILD_NUMBER', value: "${CI_BRANCH_NAME}/${env.BUILD_NUMBER}") |
| 204 | + ], wait: false |
| 205 | + |
| 206 | + echo "#### post: ufs-weather-model/ufs-wm-metrics COMPLETE." |
| 207 | + } |
| 208 | + } |
| 209 | + } |
| 210 | +} |
0 commit comments