Failed Test Replay

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java

@@ -632,6 +632,7 @@ public void execute() {
       }
 
       maybeStartAppSec(scoClass, sco);
+      // start before debugger to enable Failed Test Replay correctly
       maybeStartCiVisibility(instrumentation, scoClass, sco);
       maybeStartLLMObs(instrumentation, scoClass, sco);
       // start debugger before remote config to subscribe to it before starting to poll
@@ -1310,10 +1311,6 @@ && isExplicitlyDisabled(TraceInstrumentationConfig.CODE_ORIGIN_FOR_SPANS_ENABLED
         && isExplicitlyDisabled(DebuggerConfig.DISTRIBUTED_DEBUGGER_ENABLED)) {
       return;
     }
-    if (!remoteConfigEnabled) {
-      log.warn("Cannot enable Dynamic Instrumentation because Remote Configuration is not enabled");
-      return;
-    }
     startDebuggerAgent(inst, scoClass, sco);
   }
 

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilitySystem.java

@@ -101,6 +101,10 @@ public static void start(Instrumentation inst, SharedCommunicationObjects sco) {
         inst.addTransformer(new CoverageClassTransformer(instrumentationFilter));
       }
 
+      if (executionSettings.isFailedTestReplayEnabled()) {
+        config.setCiVisibilityFailedTestReplayEnabled(true);
+      }
+
       CiVisibilityCoverageServices.Child coverageServices =
           new CiVisibilityCoverageServices.Child(services, repoServices, executionSettings);
       TestEventsHandlerFactory testEventsHandlerFactory =

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/CiVisibilitySettings.java

@@ -17,6 +17,7 @@ public class CiVisibilitySettings {
           false,
           false,
           false,
+          false,
           EarlyFlakeDetectionSettings.DEFAULT,
           TestManagementSettings.DEFAULT,
           null);
@@ -28,6 +29,7 @@ public class CiVisibilitySettings {
   private final boolean flakyTestRetriesEnabled;
   private final boolean impactedTestsDetectionEnabled;
   private final boolean knownTestsEnabled;
+  private final boolean failedTestReplayEnabled;
   private final EarlyFlakeDetectionSettings earlyFlakeDetectionSettings;
   private final TestManagementSettings testManagementSettings;
   @Nullable private final String defaultBranch;
@@ -40,6 +42,7 @@ public class CiVisibilitySettings {
       boolean flakyTestRetriesEnabled,
       boolean impactedTestsDetectionEnabled,
       boolean knownTestsEnabled,
+      boolean failedTestReplayEnabled,
       EarlyFlakeDetectionSettings earlyFlakeDetectionSettings,
       TestManagementSettings testManagementSettings,
       @Nullable String defaultBranch) {
@@ -50,6 +53,7 @@ public class CiVisibilitySettings {
     this.flakyTestRetriesEnabled = flakyTestRetriesEnabled;
     this.impactedTestsDetectionEnabled = impactedTestsDetectionEnabled;
     this.knownTestsEnabled = knownTestsEnabled;
+    this.failedTestReplayEnabled = failedTestReplayEnabled;
     this.earlyFlakeDetectionSettings = earlyFlakeDetectionSettings;
     this.testManagementSettings = testManagementSettings;
     this.defaultBranch = defaultBranch;
@@ -83,6 +87,10 @@ public boolean isKnownTestsEnabled() {
     return knownTestsEnabled;
   }
 
+  public boolean isFailedTestReplayEnabled() {
+    return failedTestReplayEnabled;
+  }
+
   public EarlyFlakeDetectionSettings getEarlyFlakeDetectionSettings() {
     return earlyFlakeDetectionSettings;
   }
@@ -112,6 +120,7 @@ public boolean equals(Object o) {
         && flakyTestRetriesEnabled == that.flakyTestRetriesEnabled
         && impactedTestsDetectionEnabled == that.impactedTestsDetectionEnabled
         && knownTestsEnabled == that.knownTestsEnabled
+        && failedTestReplayEnabled == that.failedTestReplayEnabled
         && Objects.equals(earlyFlakeDetectionSettings, that.earlyFlakeDetectionSettings)
         && Objects.equals(testManagementSettings, that.testManagementSettings)
         && Objects.equals(defaultBranch, that.defaultBranch);
@@ -127,6 +136,7 @@ public int hashCode() {
         flakyTestRetriesEnabled,
         impactedTestsDetectionEnabled,
         knownTestsEnabled,
+        failedTestReplayEnabled,
         earlyFlakeDetectionSettings,
         testManagementSettings,
         defaultBranch);
@@ -154,6 +164,7 @@ public CiVisibilitySettings fromJson(Map<String, Object> json) {
           getBoolean(json, "flaky_test_retries_enabled", false),
           getBoolean(json, "impacted_tests_enabled", false),
           getBoolean(json, "known_tests_enabled", false),
+          getBoolean(json, "di_enabled", false),
           EarlyFlakeDetectionSettings.JsonAdapter.INSTANCE.fromJson(
               (Map<String, Object>) json.get("early_flake_detection")),
           TestManagementSettings.JsonAdapter.INSTANCE.fromJson(

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApiImpl.java

@@ -17,6 +17,7 @@
 import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector;
 import datadog.trace.api.civisibility.telemetry.tag.CoverageEnabled;
 import datadog.trace.api.civisibility.telemetry.tag.EarlyFlakeDetectionEnabled;
+import datadog.trace.api.civisibility.telemetry.tag.FailedTestReplayEnabled;
 import datadog.trace.api.civisibility.telemetry.tag.FlakyTestRetriesEnabled;
 import datadog.trace.api.civisibility.telemetry.tag.ImpactedTestsDetectionEnabled;
 import datadog.trace.api.civisibility.telemetry.tag.ItrEnabled;
@@ -156,6 +157,7 @@ public CiVisibilitySettings getSettings(TracerEnvironment tracerEnvironment) thr
         settings.isKnownTestsEnabled() ? KnownTestsEnabled.TRUE : null,
         settings.isImpactedTestsDetectionEnabled() ? ImpactedTestsDetectionEnabled.TRUE : null,
         settings.getTestManagementSettings().isEnabled() ? TestManagementEnabled.TRUE : null,
+        settings.isFailedTestReplayEnabled() ? FailedTestReplayEnabled.SettingsMetric.TRUE : null,
         settings.isGitUploadRequired() ? RequireGit.TRUE : null);
 
     return settings;

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettings.java

@@ -26,6 +26,7 @@ public class ExecutionSettings {
           false,
           false,
           false,
+          false,
           EarlyFlakeDetectionSettings.DEFAULT,
           TestManagementSettings.DEFAULT,
           null,
@@ -43,6 +44,7 @@ public class ExecutionSettings {
   private final boolean testSkippingEnabled;
   private final boolean flakyTestRetriesEnabled;
   private final boolean impactedTestsDetectionEnabled;
+  private final boolean failedTestReplayEnabled;
   @Nonnull private final EarlyFlakeDetectionSettings earlyFlakeDetectionSettings;
   @Nonnull private final TestManagementSettings testManagementSettings;
   @Nullable private final String itrCorrelationId;
@@ -58,6 +60,7 @@ public ExecutionSettings(
       boolean testSkippingEnabled,
       boolean flakyTestRetriesEnabled,
       boolean impactedTestsDetectionEnabled,
+      boolean failedTestReplayEnabled,
       @Nonnull EarlyFlakeDetectionSettings earlyFlakeDetectionSettings,
       @Nonnull TestManagementSettings testManagementSettings,
       @Nullable String itrCorrelationId,
@@ -74,6 +77,7 @@ public ExecutionSettings(
     this.testSkippingEnabled = testSkippingEnabled;
     this.flakyTestRetriesEnabled = flakyTestRetriesEnabled;
     this.impactedTestsDetectionEnabled = impactedTestsDetectionEnabled;
+    this.failedTestReplayEnabled = failedTestReplayEnabled;
     this.earlyFlakeDetectionSettings = earlyFlakeDetectionSettings;
     this.testManagementSettings = testManagementSettings;
     this.itrCorrelationId = itrCorrelationId;
@@ -110,6 +114,7 @@ private ExecutionSettings(
       boolean testSkippingEnabled,
       boolean flakyTestRetriesEnabled,
       boolean impactedTestsDetectionEnabled,
+      boolean failedTestReplayEnabled,
       @Nonnull EarlyFlakeDetectionSettings earlyFlakeDetectionSettings,
       @Nonnull TestManagementSettings testManagementSettings,
       @Nullable String itrCorrelationId,
@@ -123,6 +128,7 @@ private ExecutionSettings(
     this.testSkippingEnabled = testSkippingEnabled;
     this.flakyTestRetriesEnabled = flakyTestRetriesEnabled;
     this.impactedTestsDetectionEnabled = impactedTestsDetectionEnabled;
+    this.failedTestReplayEnabled = failedTestReplayEnabled;
     this.earlyFlakeDetectionSettings = earlyFlakeDetectionSettings;
     this.testManagementSettings = testManagementSettings;
     this.itrCorrelationId = itrCorrelationId;
@@ -157,6 +163,10 @@ public boolean isImpactedTestsDetectionEnabled() {
     return impactedTestsDetectionEnabled;
   }
 
+  public boolean isFailedTestReplayEnabled() {
+    return failedTestReplayEnabled;
+  }
+
   @Nonnull
   public EarlyFlakeDetectionSettings getEarlyFlakeDetectionSettings() {
     return earlyFlakeDetectionSettings;
@@ -243,6 +253,7 @@ public boolean equals(Object o) {
         && testSkippingEnabled == that.testSkippingEnabled
         && flakyTestRetriesEnabled == that.flakyTestRetriesEnabled
         && impactedTestsDetectionEnabled == that.impactedTestsDetectionEnabled
+        && failedTestReplayEnabled == that.failedTestReplayEnabled
         && Objects.equals(earlyFlakeDetectionSettings, that.earlyFlakeDetectionSettings)
         && Objects.equals(testManagementSettings, that.testManagementSettings)
         && Objects.equals(itrCorrelationId, that.itrCorrelationId)
@@ -261,6 +272,7 @@ public int hashCode() {
         testSkippingEnabled,
         flakyTestRetriesEnabled,
         impactedTestsDetectionEnabled,
+        failedTestReplayEnabled,
         earlyFlakeDetectionSettings,
         testManagementSettings,
         itrCorrelationId,
@@ -278,6 +290,7 @@ public static class Serializer {
     private static final int TEST_SKIPPING_ENABLED_FLAG = 4;
     private static final int FLAKY_TEST_RETRIES_ENABLED_FLAG = 8;
     private static final int IMPACTED_TESTS_DETECTION_ENABLED_FLAG = 16;
+    private static final int FAILED_TEST_REPLAY_ENABLED_FLAG = 32;
 
     public static ByteBuffer serialize(ExecutionSettings settings) {
       datadog.trace.civisibility.ipc.serialization.Serializer s =
@@ -291,7 +304,8 @@ public static ByteBuffer serialize(ExecutionSettings settings) {
                   | (settings.flakyTestRetriesEnabled ? FLAKY_TEST_RETRIES_ENABLED_FLAG : 0)
                   | (settings.impactedTestsDetectionEnabled
                       ? IMPACTED_TESTS_DETECTION_ENABLED_FLAG
-                      : 0));
+                      : 0)
+                  | (settings.failedTestReplayEnabled ? FAILED_TEST_REPLAY_ENABLED_FLAG : 0));
       s.write(flags);
 
       EarlyFlakeDetectionSettings.Serializer.serialize(s, settings.earlyFlakeDetectionSettings);
@@ -330,6 +344,7 @@ public static ExecutionSettings deserialize(ByteBuffer buffer) {
       boolean testSkippingEnabled = (flags & TEST_SKIPPING_ENABLED_FLAG) != 0;
       boolean flakyTestRetriesEnabled = (flags & FLAKY_TEST_RETRIES_ENABLED_FLAG) != 0;
       boolean impactedTestsDetectionEnabled = (flags & IMPACTED_TESTS_DETECTION_ENABLED_FLAG) != 0;
+      boolean failedTestReplayEnabled = (flags & FAILED_TEST_REPLAY_ENABLED_FLAG) != 0;
 
       EarlyFlakeDetectionSettings earlyFlakeDetectionSettings =
           EarlyFlakeDetectionSettings.Serializer.deserialize(buffer);
@@ -372,6 +387,7 @@ public static ExecutionSettings deserialize(ByteBuffer buffer) {
           testSkippingEnabled,
           flakyTestRetriesEnabled,
           impactedTestsDetectionEnabled,
+          failedTestReplayEnabled,
           earlyFlakeDetectionSettings,
           testManagementSettings,
           itrCorrelationId,

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettingsFactoryImpl.java

@@ -177,6 +177,11 @@ private Map<String, ExecutionSettings> doCreate(
             settings,
             CiVisibilitySettings::isKnownTestsEnabled,
             Config::isCiVisibilityKnownTestsRequestEnabled);
+    boolean failedTestReplayEnabled =
+        isFeatureEnabled(
+            settings,
+            CiVisibilitySettings::isFailedTestReplayEnabled,
+            Config::isCiVisibilityFailedTestReplayEnabled);
 
     TestManagementSettings testManagementSettings = getTestManagementSettings(settings);
 
@@ -189,7 +194,8 @@ private Map<String, ExecutionSettings> doCreate(
             + "Impacted tests detection - {},\n"
             + "Known tests marking - {},\n"
             + "Auto test retries - {},\n"
-            + "Test Management - {}",
+            + "Test Management - {},\n"
+            + "Failed Test Replay - {}",
         repositoryRoot,
         tracerEnvironment.getConfigurations().getRuntimeName(),
         tracerEnvironment.getConfigurations().getRuntimeVersion(),
@@ -201,7 +207,8 @@ private Map<String, ExecutionSettings> doCreate(
         impactedTestsEnabled,
         knownTestsRequest,
         flakyTestRetriesEnabled,
-        testManagementSettings.isEnabled());
+        testManagementSettings.isEnabled(),
+        failedTestReplayEnabled);
 
     Future<SkippableTests> skippableTestsFuture =
         executor.submit(() -> getSkippableTests(tracerEnvironment, itrEnabled));
@@ -253,6 +260,7 @@ private Map<String, ExecutionSettings> doCreate(
               testSkippingEnabled,
               flakyTestRetriesEnabled,
               impactedTestsEnabled,
+              failedTestReplayEnabled,
               earlyFlakeDetectionEnabled
                   ? settings.getEarlyFlakeDetectionSettings()
                   : EarlyFlakeDetectionSettings.DEFAULT,

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/AbstractTestSession.java

@@ -130,7 +130,8 @@ public AbstractTestSession(
         config.isAgentlessLogSubmissionEnabled() ? AgentlessLogSubmissionEnabled.TRUE : null,
         CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(config.getCiVisibilityTestOrder())
             ? FailFastTestOrderEnabled.TRUE
-            : null);
+            : null,
+        null);
 
     if (instrumentationType == InstrumentationType.MANUAL_API) {
       metricCollector.add(CiVisibilityCountMetric.MANUAL_API_EVENTS, 1, EventType.SESSION);

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java

@@ -305,6 +305,7 @@ public void end(@Nullable Long endTime) {
         span.getTag(Tags.TEST_IS_RETRY) != null ? IsRetry.TRUE : null,
         span.getTag(Tags.TEST_HAS_FAILED_ALL_RETRIES) != null ? HasFailedAllRetries.TRUE : null,
         retryReason instanceof TagValue ? (TagValue) retryReason : null,
+        null,
         span.getTag(Tags.TEST_IS_RUM_ACTIVE) != null ? IsRum.TRUE : null,
         CIConstants.SELENIUM_BROWSER_DRIVER.equals(span.getTag(Tags.TEST_BROWSER_DRIVER))
             ? BrowserDriver.SELENIUM

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemModuleImpl.java

@@ -12,6 +12,7 @@
 import datadog.trace.api.civisibility.domain.JavaAgent;
 import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector;
 import datadog.trace.api.config.CiVisibilityConfig;
+import datadog.trace.api.config.DebuggerConfig;
 import datadog.trace.api.config.GeneralConfig;
 import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
 import datadog.trace.bootstrap.instrumentation.api.AgentSpanContext;
@@ -182,6 +183,17 @@ private Map<String, String> getPropertiesPropagatedToChildProcess(
         Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.TEST_MANAGEMENT_ENABLED),
         Boolean.toString(executionSettings.getTestManagementSettings().isEnabled()));
 
+    // enable exception replay if failed test replay is enabled
+    if (executionSettings.isFailedTestReplayEnabled()) {
+      propagatedSystemProperties.put(
+          Strings.propertyNameToSystemPropertyName(
+              CiVisibilityConfig.TEST_FAILED_TEST_REPLAY_ENABLED),
+          "true");
+      propagatedSystemProperties.put(
+          Strings.propertyNameToSystemPropertyName(DebuggerConfig.EXCEPTION_REPLAY_ENABLED),
+          "true");
+    }
+
     // explicitly disable build instrumentation in child processes,
     // because some projects run "embedded" Maven/Gradle builds as part of their integration tests,
     // and we don't want to show those as if they were regular build executions

dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ConfigurationApiImplTest.groovy

@@ -55,10 +55,10 @@ class ConfigurationApiImplTest extends Specification {
 
     where:
     agentless | compression | expectedSettings
-    false     | false       | new CiVisibilitySettings(false, false, false, false, false, false, false, EarlyFlakeDetectionSettings.DEFAULT, TestManagementSettings.DEFAULT, null)
-    false     | true        | new CiVisibilitySettings(true, true, true, true, true, true, true, EarlyFlakeDetectionSettings.DEFAULT, TestManagementSettings.DEFAULT, "main")
-    true      | false       | new CiVisibilitySettings(false, true, false, true, false, true, false, new EarlyFlakeDetectionSettings(true, [new ExecutionsByDuration(1000, 3)], 10), new TestManagementSettings(true, 10), "master")
-    true      | true        | new CiVisibilitySettings(false, false, true, true, false, false, true, new EarlyFlakeDetectionSettings(true, [new ExecutionsByDuration(5000, 3), new ExecutionsByDuration(120000, 2)], 10), new TestManagementSettings(true, 20), "prod")
+    false     | false       | new CiVisibilitySettings(false, false, false, false, false, false, false, false, EarlyFlakeDetectionSettings.DEFAULT, TestManagementSettings.DEFAULT, null)
+    false     | true        | new CiVisibilitySettings(true, true, true, true, true, true, true, true, EarlyFlakeDetectionSettings.DEFAULT, TestManagementSettings.DEFAULT, "main")
+    true      | false       | new CiVisibilitySettings(false, true, false, true, false, true, false, true, new EarlyFlakeDetectionSettings(true, [new ExecutionsByDuration(1000, 3)], 10), new TestManagementSettings(true, 10), "master")
+    true      | true        | new CiVisibilitySettings(false, false, true, true, false, false, true, false, new EarlyFlakeDetectionSettings(true, [new ExecutionsByDuration(5000, 3), new ExecutionsByDuration(120000, 2)], 10), new TestManagementSettings(true, 20), "prod")
   }
 
   def "test skippable tests request"() {

dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ExecutionSettingsTest.groovy

@@ -1,15 +1,13 @@
 package datadog.trace.civisibility.config
 
 import datadog.trace.api.civisibility.CIConstants
-import datadog.trace.api.civisibility.config.LibraryCapability
 import datadog.trace.api.civisibility.config.TestFQN
 import datadog.trace.api.civisibility.config.TestIdentifier
 import datadog.trace.api.civisibility.config.TestMetadata
 import datadog.trace.api.config.CiVisibilityConfig
 import datadog.trace.civisibility.diff.LineDiff
 import datadog.trace.test.util.DDSpecification
 
-import java.util.stream.Collectors
 
 import static datadog.trace.civisibility.TestUtils.lines
 
@@ -31,6 +29,7 @@ class ExecutionSettingsTest extends DDSpecification {
       false,
       false,
       false,
+      false,
       EarlyFlakeDetectionSettings.DEFAULT,
       TestManagementSettings.DEFAULT,
       null,
@@ -49,6 +48,7 @@ class ExecutionSettingsTest extends DDSpecification {
       false,
       true,
       true,
+      true,
       new EarlyFlakeDetectionSettings(true, [], 10),
       new TestManagementSettings(true, 20),
       "",
@@ -68,6 +68,7 @@ class ExecutionSettingsTest extends DDSpecification {
       true,
       false,
       true,
+      false,
       new EarlyFlakeDetectionSettings(true, [new ExecutionsByDuration(10, 20)], 10),
       new TestManagementSettings(true, 20),
       "itrCorrelationId",
@@ -91,6 +92,7 @@ class ExecutionSettingsTest extends DDSpecification {
       true,
       true,
       true,
+      true,
       new EarlyFlakeDetectionSettings(true, [new ExecutionsByDuration(10, 20), new ExecutionsByDuration(30, 40)], 10),
       new TestManagementSettings(true, 20),
       "itrCorrelationId",
@@ -127,6 +129,7 @@ class ExecutionSettingsTest extends DDSpecification {
     settingsEnabled,
     settingsEnabled,
     settingsEnabled,
+    settingsEnabled,
     earlyFlakeDetectionSettings,
     testManagementSettings,
     null,

dd-java-agent/agent-ci-visibility/src/test/resources/datadog/trace/civisibility/config/settings-response.ftl

@@ -10,6 +10,7 @@
       "flaky_test_retries_enabled": ${settings.flakyTestRetriesEnabled?c},
       "impacted_tests_enabled": ${settings.impactedTestsDetectionEnabled?c},
       "known_tests_enabled": ${settings.knownTestsEnabled?c},
+      "di_enabled": ${settings.failedTestReplayEnabled?c},
       <#if settings.defaultBranch??>
         "default_branch": "${settings.defaultBranch}",
       </#if>