Skip to content

Commit 2e95cdd

Browse files
prudhvigodithiprudhvigodithi
authored
Creating Issue Reports for Flaky Test Failures in Gradle Check (#436)
Signed-off-by: Prudhvi Godithi <pgodithi@amazon.com>
1 parent cfba629 commit 2e95cdd

24 files changed

+1365
-126
lines changed

‎build.gradle

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ dependencies {
2727
testImplementation group: 'org.yaml', name: 'snakeyaml', version: '2.0'
2828
testImplementation group: 'org.assertj', name: 'assertj-core', version: '3.24.2'
2929
testImplementation group: 'com.lesfurets', name:'jenkins-pipeline-unit', version: '1.13'
30+
testImplementation group: 'org.mockito', name: 'mockito-core', version: '5.12.0'
3031
}
3132

3233
configurations.all {
@@ -44,13 +45,13 @@ configurations.all {
4445
sourceSets {
4546
main {
4647
groovy {
47-
srcDirs = ['src/jenkins']
48+
srcDirs = ['src/jenkins', 'src/gradlecheck']
4849
}
4950
}
5051

5152
test {
5253
groovy {
53-
srcDirs = ['tests/jenkins']
54+
srcDirs = ['tests/jenkins', 'tests/gradlecheck']
5455
}
5556
}
5657

@@ -84,8 +85,8 @@ sharedLibrary {
8485

8586
test {
8687
testLogging {
87-
events "failed"
88-
exceptionFormat "full"
88+
events "failed"
89+
exceptionFormat "full"
8990
}
9091
if (project.hasProperty('pipeline.stack.write')) {
9192
systemProperty 'pipeline.stack.write', project.getProperty('pipeline.stack.write')
@@ -109,14 +110,15 @@ jacocoTestReport {
109110

110111
afterEvaluate {
111112
classDirectories.from = fileTree(
112-
dir: "$buildDir/jacoco/classpathdumps",
113-
includes: [
114-
'**/*_Jenkinsfile.*',
115-
'**/jenkins/*'
116-
],
117-
excludes: [
118-
'**/*\$_get_closure*'
119-
]
113+
dir: "$buildDir/jacoco/classpathdumps",
114+
includes: [
115+
'**/*_Jenkinsfile.*',
116+
'**/jenkins/*',
117+
'**/gradlecheck/*'
118+
],
119+
excludes: [
120+
'**/*\$_get_closure*'
121+
]
120122
)
121123
}
122124

‎src/gradlecheck/CreateMarkDownTable.groovy

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*/
9+
10+
package gradlecheck
11+
12+
class CreateMarkDownTable {
13+
String failedTest
14+
ArrayList<String> tableData
15+
ArrayList<String> additionalPullRequests
16+
17+
CreateMarkDownTable(String failedTest, List<Map<String, Object>> tableData, List<String> additionalPullRequests) {
18+
this.failedTest = failedTest
19+
this.tableData = tableData
20+
this.additionalPullRequests = additionalPullRequests
21+
}
22+
23+
def createMarkdownTable() {
24+
25+
def tableHeader = """
26+
## Flaky Test Report for `${this.failedTest}`
27+
28+
Noticed the `${this.failedTest}` has some flaky, failing tests that failed during **post-merge actions**.
29+
30+
### Details
31+
32+
| Git Reference | Merged Pull Request | Build Details | Test Name |
33+
|---------------|----------------------|---------------|-----------|
34+
"""
35+
def tableRows = this.tableData.collect { row ->
36+
"| ${row.gitReference} | ${row.pullRequestLink} | ${row.buildDetailLink} | ${row.testNames.join('<br><br>')} |"
37+
}.join("\n")
38+
39+
def additionalPRSection = """
40+
\nThe other pull requests, besides those involved in post-merge actions, that contain failing tests with the `${this.failedTest}` class are:
41+
42+
${this.additionalPullRequests.collect { pr -> "- [${pr}](https://github.com/opensearch-project/OpenSearch/pull/${pr})" }.join('\n')}
43+
44+
For more details on the failed tests refer to [OpenSearch Gradle Check Metrics](https://metrics.opensearch.org/_dashboards/app/dashboards#/view/e5e64d40-ed31-11ee-be99-69d1dbc75083) dashboard.
45+
"""
46+
47+
return tableHeader + tableRows + additionalPRSection
48+
}
49+
50+
}

‎src/gradlecheck/FetchPostMergeFailedTestClass.groovy

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*/
9+
10+
package gradlecheck
11+
12+
import groovy.json.JsonOutput
13+
import gradlecheck.OpenSearchMetricsQuery
14+
15+
class FetchPostMergeFailedTestClass {
16+
String metricsUrl
17+
String awsAccessKey
18+
String awsSecretKey
19+
String awsSessionToken
20+
def script
21+
22+
FetchPostMergeFailedTestClass(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, def script) {
23+
this.metricsUrl = metricsUrl
24+
this.awsAccessKey = awsAccessKey
25+
this.awsSecretKey = awsSecretKey
26+
this.awsSessionToken = awsSessionToken
27+
this.script = script
28+
}
29+
30+
def getQuery() {
31+
def queryMap = [
32+
size: 200,
33+
query: [
34+
bool: [
35+
must: [
36+
[
37+
match: [
38+
"invoke_type.keyword": [
39+
query: "Post Merge Action",
40+
operator: "OR",
41+
prefix_length: 0,
42+
max_expansions: 50,
43+
fuzzy_transpositions: true,
44+
lenient: false,
45+
zero_terms_query: "NONE",
46+
auto_generate_synonyms_phrase_query: true,
47+
boost: 1
48+
]
49+
]
50+
],
51+
[
52+
match: [
53+
test_status: [
54+
query: "FAILED",
55+
operator: "OR"
56+
]
57+
]
58+
]
59+
]
60+
]
61+
],
62+
aggregations: [
63+
test_class_keyword_agg: [
64+
terms: [
65+
field: "test_class",
66+
size: 500
67+
]
68+
]
69+
]
70+
]
71+
72+
def query = JsonOutput.toJson(queryMap)
73+
return query.replace('"', '\\"')
74+
}
75+
76+
def getPostMergeFailedTestClass() {
77+
def jsonResponse = new OpenSearchMetricsQuery(metricsUrl,awsAccessKey, awsSecretKey, awsSessionToken, script).fetchMetrics(getQuery())
78+
def keys = jsonResponse.aggregations.test_class_keyword_agg.buckets.collect { it.key }
79+
return keys
80+
}
81+
}

‎src/gradlecheck/FetchPostMergeFailedTestName.groovy

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*/
9+
10+
package gradlecheck
11+
12+
import groovy.json.JsonOutput
13+
import gradlecheck.OpenSearchMetricsQuery
14+
15+
class FetchPostMergeFailedTestName {
16+
String metricsUrl
17+
String awsAccessKey
18+
String awsSecretKey
19+
String awsSessionToken
20+
def script
21+
22+
FetchPostMergeFailedTestName(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, def script) {
23+
this.metricsUrl = metricsUrl
24+
this.awsAccessKey = awsAccessKey
25+
this.awsSecretKey = awsSecretKey
26+
this.awsSessionToken = awsSessionToken
27+
this.script = script
28+
}
29+
30+
def getQuery(testName, gitReference) {
31+
def queryMap = [
32+
size: 200,
33+
query: [
34+
bool: [
35+
must: [
36+
[
37+
match: [
38+
"invoke_type.keyword": [
39+
query: "Post Merge Action",
40+
operator: "OR",
41+
prefix_length: 0,
42+
max_expansions: 50,
43+
fuzzy_transpositions: true,
44+
lenient: false,
45+
zero_terms_query: "NONE",
46+
auto_generate_synonyms_phrase_query: true,
47+
boost: 1
48+
]
49+
]
50+
],
51+
[
52+
match: [
53+
test_status: [
54+
query: "FAILED",
55+
operator: "OR",
56+
prefix_length: 0,
57+
max_expansions: 50,
58+
fuzzy_transpositions: true,
59+
lenient: false,
60+
zero_terms_query: "NONE",
61+
auto_generate_synonyms_phrase_query: true,
62+
boost: 1
63+
]
64+
]
65+
],
66+
[
67+
match: [
68+
test_class: [
69+
query: testName,
70+
operator: "OR",
71+
prefix_length: 0,
72+
max_expansions: 50,
73+
fuzzy_transpositions: true,
74+
lenient: false,
75+
zero_terms_query: "NONE",
76+
auto_generate_synonyms_phrase_query: true,
77+
boost: 1
78+
]
79+
]
80+
],
81+
[
82+
match: [
83+
"git_reference.keyword": [
84+
query: gitReference,
85+
operator: "OR",
86+
prefix_length: 0,
87+
max_expansions: 50,
88+
fuzzy_transpositions: true,
89+
lenient: false,
90+
zero_terms_query: "NONE",
91+
auto_generate_synonyms_phrase_query: true,
92+
boost: 1
93+
]
94+
]
95+
]
96+
],
97+
adjust_pure_negative: true,
98+
boost: 1
99+
]
100+
],
101+
aggregations: [
102+
test_name_keyword_agg: [
103+
terms: [
104+
field: "test_name",
105+
size: 500
106+
]
107+
],
108+
build_number_agg: [
109+
terms: [
110+
field: "build_number",
111+
size: 500
112+
]
113+
],
114+
pull_request_agg: [
115+
terms: [
116+
field: "pull_request",
117+
size: 500
118+
]
119+
]
120+
]
121+
]
122+
123+
def query = JsonOutput.toJson(queryMap)
124+
return query.replace('"', '\\"')
125+
126+
}
127+
def getPostMergeFailedTestName(testName, gitReference) {
128+
return new OpenSearchMetricsQuery(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, script).fetchMetrics(getQuery(testName, gitReference))
129+
}
130+
}

0 commit comments

Comments
 (0)