Skip to content

Commit

Permalink
Implement flaky test retries for JUnit 4 (#6435)
Browse files Browse the repository at this point in the history
  • Loading branch information
nikita-tkachenko-datadog authored Jan 9, 2024
1 parent 9656cf7 commit 4d7d105
Show file tree
Hide file tree
Showing 80 changed files with 5,023 additions and 511 deletions.
Original file line number Diff line number Diff line change
@@ -1,20 +1,12 @@
package datadog.trace.instrumentation.junit4;

import datadog.trace.api.civisibility.coverage.CoverageBridge;
import datadog.trace.util.MethodHandles;
import datadog.trace.util.Strings;
import io.cucumber.core.gherkin.Feature;
import io.cucumber.core.gherkin.Pickle;
import io.cucumber.core.resource.ClassLoaders;
import java.io.InputStream;
import java.lang.invoke.MethodHandle;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.junit.Ignore;
import org.junit.runner.Description;
import org.junit.runner.notification.Failure;
Expand All @@ -28,50 +20,19 @@ public class CucumberTracingListener extends TracingListener {

private static final Logger LOGGER = LoggerFactory.getLogger(CucumberTracingListener.class);

private static final ClassLoader CUCUMBER_CLASS_LOADER = ClassLoaders.getDefaultClassLoader();
public static final String FRAMEWORK_NAME = "cucumber";
public static final String FRAMEWORK_VERSION = getVersion();

private static String getVersion() {
try (InputStream cucumberPropsStream =
CUCUMBER_CLASS_LOADER.getResourceAsStream(
"META-INF/maven/io.cucumber/cucumber-junit/pom.properties")) {
Properties cucumberProps = new Properties();
cucumberProps.load(cucumberPropsStream);
String version = cucumberProps.getProperty("version");
if (Strings.isNotBlank(version)) {
return version;
}
} catch (Exception e) {
// fallback below
}
return "unknown";
}

private static final MethodHandles REFLECTION = new MethodHandles(CUCUMBER_CLASS_LOADER);
private static final MethodHandle PICKLE_ID_CONSTRUCTOR =
REFLECTION.constructor("io.cucumber.junit.PickleRunners$PickleId", Pickle.class);
private static final MethodHandle PICKLE_ID_URI_GETTER =
REFLECTION.privateFieldGetter("io.cucumber.junit.PickleRunners$PickleId", "uri");
private static final MethodHandle FEATURE_GETTER =
REFLECTION.privateFieldGetter("io.cucumber.junit.FeatureRunner", "feature");
public static final String FRAMEWORK_VERSION = CucumberUtils.getVersion();

private final Map<Object, Pickle> pickleById = new HashMap<>();
private final Map<Object, Pickle> pickleById;

public CucumberTracingListener(List<ParentRunner<?>> featureRunners) {
for (ParentRunner<?> featureRunner : featureRunners) {
Feature feature = (Feature) REFLECTION.invoke(FEATURE_GETTER, featureRunner);
for (Pickle pickle : feature.getPickles()) {
Object pickleId = REFLECTION.invoke(PICKLE_ID_CONSTRUCTOR, pickle);
pickleById.put(pickleId, pickle);
}
}
pickleById = CucumberUtils.getPicklesById(featureRunners);
}

@Override
public void testSuiteStarted(final Description description) {
if (isFeature(description)) {
String testSuiteName = getTestSuiteNameForFeature(description);
String testSuiteName = CucumberUtils.getTestSuiteNameForFeature(description);
TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteStart(
testSuiteName, FRAMEWORK_NAME, FRAMEWORK_VERSION, null, Collections.emptyList(), false);
}
Expand All @@ -80,14 +41,14 @@ public void testSuiteStarted(final Description description) {
@Override
public void testSuiteFinished(final Description description) {
if (isFeature(description)) {
String testSuiteName = getTestSuiteNameForFeature(description);
String testSuiteName = CucumberUtils.getTestSuiteNameForFeature(description);
TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(testSuiteName, null);
}
}

@Override
public void testStarted(final Description description) {
String testSuiteName = getTestSuiteNameForScenario(description);
String testSuiteName = CucumberUtils.getTestSuiteNameForScenario(description);
String testName = description.getMethodName();
List<String> categories = getCategories(description);

Expand All @@ -106,31 +67,9 @@ public void testStarted(final Description description) {
recordFeatureFileCodeCoverage(description);
}

private static String getTestSuiteNameForFeature(Description featureDescription) {
Object uniqueId = JUnit4Utils.getUniqueId(featureDescription);
return (uniqueId != null ? uniqueId + ":" : "") + featureDescription.getClassName();
}

private static String getTestSuiteNameForScenario(Description scenarioDescription) {
URI featureUri = getFeatureUri(scenarioDescription);
return (featureUri != null ? featureUri + ":" : "") + scenarioDescription.getClassName();
}

private static URI getFeatureUri(Description scenarioDescription) {
try {
Object pickleId = JUnit4Utils.getUniqueId(scenarioDescription);
return REFLECTION.invoke(PICKLE_ID_URI_GETTER, pickleId);
} catch (Exception e) {
LOGGER.error(
"Could not retrieve unique ID from scenario description {}", scenarioDescription, e);
return null;
}
}

private static void recordFeatureFileCodeCoverage(Description scenarioDescription) {
try {
Object pickleId = JUnit4Utils.getUniqueId(scenarioDescription);
URI pickleUri = REFLECTION.invoke(PICKLE_ID_URI_GETTER, pickleId);
URI pickleUri = CucumberUtils.getPickleUri(scenarioDescription);
String featureRelativePath = pickleUri.getSchemeSpecificPart();
CoverageBridge.currentCoverageProbeStoreRecordNonCode(featureRelativePath);
} catch (Exception e) {
Expand All @@ -140,7 +79,7 @@ private static void recordFeatureFileCodeCoverage(Description scenarioDescriptio

@Override
public void testFinished(final Description description) {
String testSuiteName = getTestSuiteNameForScenario(description);
String testSuiteName = CucumberUtils.getTestSuiteNameForScenario(description);
String testName = description.getMethodName();
TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFinish(
testSuiteName, null, testName, null, null);
Expand All @@ -151,12 +90,12 @@ public void testFinished(final Description description) {
public void testFailure(final Failure failure) {
Description description = failure.getDescription();
if (isFeature(description)) {
String testSuiteName = getTestSuiteNameForFeature(description);
String testSuiteName = CucumberUtils.getTestSuiteNameForFeature(description);
Throwable throwable = failure.getException();
TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFailure(
testSuiteName, null, throwable);
} else {
String testSuiteName = getTestSuiteNameForScenario(description);
String testSuiteName = CucumberUtils.getTestSuiteNameForScenario(description);
String testName = description.getMethodName();
Throwable throwable = failure.getException();
TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFailure(
Expand All @@ -176,10 +115,10 @@ public void testAssumptionFailure(final Failure failure) {

Description description = failure.getDescription();
if (isFeature(description)) {
String testSuiteName = getTestSuiteNameForFeature(description);
String testSuiteName = CucumberUtils.getTestSuiteNameForFeature(description);
TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(testSuiteName, null, reason);
} else {
String testSuiteName = getTestSuiteNameForScenario(description);
String testSuiteName = CucumberUtils.getTestSuiteNameForScenario(description);
String testName = description.getMethodName();
TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSkip(
testSuiteName, null, testName, null, null, reason);
Expand All @@ -192,13 +131,13 @@ public void testIgnored(final Description description) {
String reason = ignore != null ? ignore.value() : null;

if (isFeature(description)) {
String testSuiteName = getTestSuiteNameForFeature(description);
String testSuiteName = CucumberUtils.getTestSuiteNameForFeature(description);
TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteStart(
testSuiteName, FRAMEWORK_NAME, FRAMEWORK_VERSION, null, Collections.emptyList(), false);
TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(testSuiteName, null, reason);
TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(testSuiteName, null);
} else {
String testSuiteName = getTestSuiteNameForScenario(description);
String testSuiteName = CucumberUtils.getTestSuiteNameForScenario(description);
String testName = description.getMethodName();
List<String> categories = getCategories(description);
TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestIgnore(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
package datadog.trace.instrumentation.junit4;

import datadog.trace.agent.tooling.muzzle.Reference;
import datadog.trace.api.civisibility.config.TestIdentifier;
import datadog.trace.util.MethodHandles;
import datadog.trace.util.Strings;
import io.cucumber.core.gherkin.Feature;
import io.cucumber.core.gherkin.Pickle;
import io.cucumber.core.resource.ClassLoaders;
import java.io.InputStream;
import java.lang.invoke.MethodHandle;
import java.net.URI;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.junit.runner.Description;
import org.junit.runners.ParentRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public abstract class CucumberUtils {

private static final Logger LOGGER = LoggerFactory.getLogger(CucumberUtils.class);

private static final ClassLoader CUCUMBER_CLASS_LOADER = ClassLoaders.getDefaultClassLoader();

public static String getVersion() {
try (InputStream cucumberPropsStream =
CUCUMBER_CLASS_LOADER.getResourceAsStream(
"META-INF/maven/io.cucumber/cucumber-junit/pom.properties")) {
Properties cucumberProps = new Properties();
cucumberProps.load(cucumberPropsStream);
String version = cucumberProps.getProperty("version");
if (Strings.isNotBlank(version)) {
return version;
}
} catch (Exception e) {
// fallback below
}
return "unknown";
}

private static final MethodHandles REFLECTION = new MethodHandles(CUCUMBER_CLASS_LOADER);
private static final MethodHandle FEATURE_GETTER =
REFLECTION.privateFieldGetter("io.cucumber.junit.FeatureRunner", "feature");
private static final MethodHandle PICKLE_ID_CONSTRUCTOR =
REFLECTION.constructor("io.cucumber.junit.PickleRunners$PickleId", Pickle.class);
private static final MethodHandle PICKLE_ID_URI_GETTER =
REFLECTION.privateFieldGetter("io.cucumber.junit.PickleRunners$PickleId", "uri");
private static final MethodHandle PICKLE_RUNNER_GET_DESCRIPTION =
REFLECTION.method("io.cucumber.junit.PickleRunners$PickleRunner", "getDescription");

private CucumberUtils() {}

public static Map<Object, Pickle> getPicklesById(List<ParentRunner<?>> featureRunners) {
Map<Object, Pickle> pickleById = new HashMap<>();
for (ParentRunner<?> featureRunner : featureRunners) {
Feature feature = (Feature) REFLECTION.invoke(FEATURE_GETTER, featureRunner);
for (Pickle pickle : feature.getPickles()) {
Object pickleId = REFLECTION.invoke(PICKLE_ID_CONSTRUCTOR, pickle);
pickleById.put(pickleId, pickle);
}
}
return pickleById;
}

public static URI getPickleUri(Description scenarioDescription) {
Object pickleId = JUnit4Utils.getUniqueId(scenarioDescription);
return REFLECTION.invoke(PICKLE_ID_URI_GETTER, pickleId);
}

public static String getTestSuiteNameForFeature(Description featureDescription) {
Object uniqueId = JUnit4Utils.getUniqueId(featureDescription);
return (uniqueId != null ? uniqueId + ":" : "") + featureDescription.getClassName();
}

public static String getTestSuiteNameForScenario(Description scenarioDescription) {
URI featureUri = getFeatureUri(scenarioDescription);
return (featureUri != null ? featureUri + ":" : "") + scenarioDescription.getClassName();
}

private static URI getFeatureUri(Description scenarioDescription) {
try {
Object pickleId = JUnit4Utils.getUniqueId(scenarioDescription);
return REFLECTION.invoke(PICKLE_ID_URI_GETTER, pickleId);
} catch (Exception e) {
LOGGER.error(
"Could not retrieve unique ID from scenario description {}", scenarioDescription, e);
return null;
}
}

public static Description getPickleRunnerDescription(
Object /* io.cucumber.junit.PickleRunners.PickleRunner */ runner) {
return REFLECTION.invoke(PICKLE_RUNNER_GET_DESCRIPTION, runner);
}

public static TestIdentifier toTestIdentifier(Description description) {
String suite = getTestSuiteNameForScenario(description);
String name = description.getMethodName();
return new TestIdentifier(suite, name, null, null);
}

public static final class MuzzleHelper {
public static Reference[] additionalMuzzleReferences() {
return new Reference[] {
new Reference.Builder("io.cucumber.junit.FeatureRunner")
.withField(new String[0], 0, "feature", "Lio/cucumber/core/gherkin/Feature;")
.build(),
new Reference.Builder("io.cucumber.junit.PickleRunners$PickleId")
.withField(new String[0], 0, "uri", "Ljava/net/URI;")
.build(),
new Reference.Builder("io.cucumber.junit.PickleRunners$PickleRunner")
.withMethod(new String[0], 0, "getDescription", "Lorg/junit/runner/Description;")
.build()
};
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import com.google.auto.service.AutoService;
import datadog.trace.agent.tooling.Instrumenter;
import datadog.trace.agent.tooling.muzzle.Reference;
import java.util.List;
import net.bytebuddy.asm.Advice;
import org.junit.runner.notification.RunListener;
Expand All @@ -27,6 +28,7 @@ public String instrumentedType() {
@Override
public String[] helperClassNames() {
return new String[] {
packageName + ".CucumberUtils",
packageName + ".TestEventsHandlerHolder",
packageName + ".SkippedByItr",
packageName + ".JUnit4Utils",
Expand All @@ -35,6 +37,11 @@ public String[] helperClassNames() {
};
}

@Override
public Reference[] additionalMuzzleReferences() {
return CucumberUtils.MuzzleHelper.additionalMuzzleReferences();
}

@Override
public void adviceTransformations(AdviceTransformation transformation) {
transformation.applyAdvice(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

import com.google.auto.service.AutoService;
import datadog.trace.agent.tooling.Instrumenter;
import datadog.trace.agent.tooling.muzzle.Reference;
import datadog.trace.api.Config;
import datadog.trace.api.civisibility.InstrumentationBridge;
import datadog.trace.api.civisibility.config.TestIdentifier;
Expand Down Expand Up @@ -47,6 +48,7 @@ public ElementMatcher<TypeDescription> hierarchyMatcher() {
@Override
public String[] helperClassNames() {
return new String[] {
packageName + ".CucumberUtils",
packageName + ".TestEventsHandlerHolder",
packageName + ".SkippedByItr",
packageName + ".JUnit4Utils",
Expand All @@ -55,6 +57,11 @@ public String[] helperClassNames() {
};
}

@Override
public Reference[] additionalMuzzleReferences() {
return CucumberUtils.MuzzleHelper.additionalMuzzleReferences();
}

@Override
public void adviceTransformations(AdviceTransformation transformation) {
transformation.applyAdvice(
Expand All @@ -77,7 +84,7 @@ public static Boolean run(
}
}

TestIdentifier test = JUnit4Utils.toTestIdentifier(description);
TestIdentifier test = CucumberUtils.toTestIdentifier(description);
if (TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skip(test)) {
notifier.fireTestAssumptionFailed(
new Failure(
Expand Down
Loading

0 comments on commit 4d7d105

Please sign in to comment.