Skip to content

Use messages in rerun formatter #3028

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 3 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions cucumber-bom/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,12 @@
<properties>
<ci-environment.version>10.0.1</ci-environment.version>
<cucumber-expressions.version>18.0.1</cucumber-expressions.version>
<gherkin.version>33.0.0</gherkin.version>
<gherkin.version>33.0.1-SNAPSHOT</gherkin.version>
<html-formatter.version>21.13.0</html-formatter.version>
<junit-xml-formatter.version>0.8.0</junit-xml-formatter.version>
<messages.version>28.0.0</messages.version>
<messages.version>28.0.1-SNAPSHOT</messages.version>
<pretty-formatter.version>0.3.0</pretty-formatter.version>
<query.version>13.5.0</query.version>
<query.version>13.5.1-SNAPSHOT</query.version>
<tag-expressions.version>6.1.2</tag-expressions.version>
<testng-xml-formatter.version>0.4.0</testng-xml-formatter.version>
</properties>
Expand Down
173 changes: 134 additions & 39 deletions cucumber-core/src/main/java/io/cucumber/core/plugin/RerunFormatter.java
Original file line number Diff line number Diff line change
@@ -1,65 +1,59 @@
package io.cucumber.core.plugin;

import io.cucumber.core.feature.FeatureWithLines;
import io.cucumber.messages.types.Envelope;
import io.cucumber.messages.types.Pickle;
import io.cucumber.messages.types.TestCase;
import io.cucumber.messages.types.TestCaseFinished;
import io.cucumber.messages.types.TestCaseStarted;
import io.cucumber.messages.types.TestRunFinished;
import io.cucumber.messages.types.TestStepFinished;
import io.cucumber.messages.types.TestStepResultStatus;
import io.cucumber.plugin.ConcurrentEventListener;
import io.cucumber.plugin.event.EventPublisher;
import io.cucumber.plugin.event.TestCase;
import io.cucumber.plugin.event.TestCaseFinished;
import io.cucumber.plugin.event.TestRunFinished;

import java.io.File;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;

import static io.cucumber.core.feature.FeatureWithLines.create;
import static io.cucumber.messages.types.TestStepResultStatus.PASSED;
import static io.cucumber.messages.types.TestStepResultStatus.SKIPPED;
import static java.util.Collections.emptyList;
import static java.util.Comparator.comparing;
import static java.util.Objects.requireNonNull;

/**
* Formatter for reporting all failed test cases and print their locations
* Failed means: results that make the exit code non-zero.
* Formatter for reporting all failed test cases and print their locations.
*/
public final class RerunFormatter implements ConcurrentEventListener {

private final UTF8PrintWriter out;
private final Map<URI, Collection<Integer>> featureAndFailedLinesMapping = new LinkedHashMap<>();
private final Query query = new Query();
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Using query here seems over kill but as pickles don't have their location build in, this is the easiest way to get it. Consider putting the pickle location in the message.

private final Map<String, Set<Integer>> featureAndFailedLinesMapping = new HashMap<>();
private final PrintWriter writer;

public RerunFormatter(OutputStream out) {
this.out = new UTF8PrintWriter(out);
this.writer = createPrintWriter(out);
}

@Override
public void setEventPublisher(EventPublisher publisher) {
publisher.registerHandlerFor(TestCaseFinished.class, this::handleTestCaseFinished);
publisher.registerHandlerFor(TestRunFinished.class, event -> finishReport());
}

private void handleTestCaseFinished(TestCaseFinished event) {
if (!event.getResult().getStatus().isOk()) {
recordTestFailed(event.getTestCase());
}
}

private void finishReport() {
for (Map.Entry<URI, Collection<Integer>> entry : featureAndFailedLinesMapping.entrySet()) {
FeatureWithLines featureWithLines = create(relativize(entry.getKey()), entry.getValue());
out.println(featureWithLines.toString());
}

out.close();
}

private void recordTestFailed(TestCase testCase) {
URI uri = testCase.getUri();
Collection<Integer> failedTestCaseLines = getFailedTestCaseLines(uri);
failedTestCaseLines.add(testCase.getLocation().getLine());
}

private Collection<Integer> getFailedTestCaseLines(URI uri) {
return featureAndFailedLinesMapping.computeIfAbsent(uri, k -> new ArrayList<>());
private static PrintWriter createPrintWriter(OutputStream out) {
return new PrintWriter(
new OutputStreamWriter(
requireNonNull(out),
StandardCharsets.UTF_8));
}

static URI relativize(URI uri) {
Expand All @@ -79,4 +73,105 @@ static URI relativize(URI uri) {
throw new IllegalArgumentException(e.getMessage(), e);
}
}

@Override
public void setEventPublisher(EventPublisher publisher) {
publisher.registerHandlerFor(Envelope.class, event -> {
query.update(event);
event.getTestCaseFinished().ifPresent(this::handleTestCaseFinished);
event.getTestRunFinished().ifPresent(this::handleTestRunFinished);
});
}


private void handleTestCaseFinished(TestCaseFinished event) {
TestStepResultStatus status = query.findMostSevereTestStepResultBy(event)
// By definition
.orElse(PASSED);
if (status == PASSED || status == SKIPPED) {
return;
}
query.findPickleBy(event).ifPresent(pickle -> {
// Adds the entire feature for rerunning
Set<Integer> lines = featureAndFailedLinesMapping.computeIfAbsent(pickle.getUri(), s -> new HashSet<>());
pickle.getLocation().ifPresent(location -> {
// Adds the specific scenarios
// TODO: Messages are silly
lines.add((int) (long) location.getLine());
});
});
}

private void handleTestRunFinished(TestRunFinished testRunFinished) {
for (Map.Entry<String, Set<Integer>> entry : featureAndFailedLinesMapping.entrySet()) {
String key = entry.getKey();
// TODO: Should these be relative?
FeatureWithLines featureWithLines = create(relativize(URI.create(key)), entry.getValue());
writer.println(featureWithLines);
}

writer.close();
}

/**
* Miniaturized version of Cucumber Query.
* <p>
* The rerun plugin only needs a few things.
*/
private static class Query {

private final Map<String, TestCase> testCaseById = new HashMap<>();
private final Map<String, List<TestStepResultStatus>> testStepsResultStatusByTestCaseStartedId = new HashMap<>();
private final Map<String, TestCaseStarted> testCaseStartedById = new HashMap<>();
private final Map<String, Pickle> pickleById = new HashMap<>();

void update(Envelope envelope) {
envelope.getPickle().ifPresent(this::updatePickle);
envelope.getTestCase().ifPresent(this::updateTestCase);
envelope.getTestCaseStarted().ifPresent(this::updateTestCaseStarted);
envelope.getTestStepFinished().ifPresent(this::updateTestStepFinished);
}

private void updatePickle(Pickle event) {
pickleById.put(event.getId(), event);
}

private void updateTestCase(TestCase event) {
testCaseById.put(event.getId(), event);
}

private void updateTestCaseStarted(TestCaseStarted testCaseStarted) {
testCaseStartedById.put(testCaseStarted.getId(), testCaseStarted);
}

private void updateTestStepFinished(TestStepFinished event) {
String testCaseStartedId = event.getTestCaseStartedId();
testStepsResultStatusByTestCaseStartedId.computeIfAbsent(testCaseStartedId, s -> new ArrayList<>())
.add(event.getTestStepResult().getStatus());
}

public Optional<TestStepResultStatus> findMostSevereTestStepResultBy(TestCaseFinished testCaseFinished) {
List<TestStepResultStatus> statuses = testStepsResultStatusByTestCaseStartedId
.getOrDefault(testCaseFinished.getTestCaseStartedId(), emptyList());
if (statuses.isEmpty()) {
return Optional.empty();
}
return Optional.of(Collections.max(statuses, comparing(Enum::ordinal)));
}

public Optional<Pickle> findPickleBy(TestCaseFinished testCaseFinished) {
String testCaseStartedId = testCaseFinished.getTestCaseStartedId();
TestCaseStarted testCaseStarted = testCaseStartedById.get(testCaseStartedId);
if (testCaseStarted == null) {
return Optional.empty();
}
TestCase testCase = testCaseById.get(testCaseStarted.getTestCaseId());
if (testCase == null) {
return Optional.empty();
}
return Optional.ofNullable(pickleById.get(testCase.getPickleId()));
}

}

}
Original file line number Diff line number Diff line change
Expand Up @@ -379,7 +379,7 @@ private static String getHookName(Hook hook) {
}

private Optional<String> findSnippets(Pickle pickle) {
return query.findLocationOf(pickle)
return pickle.getLocation()
.map(location -> {
URI uri = URI.create(pickle.getUri());
List<Suggestion> suggestionForTestCase = suggestions.stream()
Expand Down
Loading