diff --git a/dd-smoke-tests/gradle/src/test/java/datadog/smoketest/GradleDaemonSmokeTest.java b/dd-smoke-tests/gradle/src/test/java/datadog/smoketest/GradleDaemonSmokeTest.java index f86716bc8ec..eced6a275e0 100644 --- a/dd-smoke-tests/gradle/src/test/java/datadog/smoketest/GradleDaemonSmokeTest.java +++ b/dd-smoke-tests/gradle/src/test/java/datadog/smoketest/GradleDaemonSmokeTest.java @@ -13,13 +13,21 @@ import datadog.trace.api.config.TraceInstrumentationConfig; import datadog.trace.civisibility.CiVisibilityTableTestConverters; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; import org.gradle.testkit.runner.BuildResult; import org.gradle.testkit.runner.BuildTask; import org.gradle.testkit.runner.GradleRunner; @@ -42,6 +50,12 @@ class GradleDaemonSmokeTest extends AbstractGradleTest { private static final String TEST_SERVICE_NAME = "test-gradle-service"; + private static final Path GRADLE_DAEMON_DIAGNOSTICS_DIR = + Paths.get( + System.getProperty("datadog.smoketest.builddir", "build"), + "reports", + "gradle-daemon-diagnostics"); + private static final Pattern DAEMON_LOG_PATTERN = Pattern.compile("daemon-(.+)\\.out\\.log"); // Gradle's default timeout is 10s private static final int GRADLE_DISTRIBUTION_NETWORK_TIMEOUT = 30_000; @@ -346,6 +360,7 @@ private void givenGradleProjectProperties(Map additionalArgs) th effectiveAdditionalArgs.put(TraceInstrumentationConfig.TRACE_ENABLED, "false"); List arguments = buildJvmArguments(mockBackend.getIntakeUrl(), TEST_SERVICE_NAME, effectiveAdditionalArgs); + addCrashDiagnostics(arguments); String gradleProperties = "org.gradle.jvmargs=" + String.join(" ", arguments); // Write to projectFolder (per-test) instead of testKitFolder (shared), so each @@ -430,20 +445,168 @@ private BuildResult runGradle( try { return successExpected ? gradleRunner.build() : gradleRunner.buildAndFail(); } catch (Exception e) { - Path daemonLogDir = testKitFolder.resolve("test-kit-daemon/" + gradleVersion); - Path daemonLog = - Files.exists(daemonLogDir) - ? Files.list(daemonLogDir) - .filter(p -> p.toString().endsWith("log")) - .findAny() - .orElse(null) - : null; - if (daemonLog != null) { + try { + collectDaemonDiagnostics(gradleVersion, e); + } catch (Exception diagnosticsError) { + System.err.println("Failed to collect Gradle daemon diagnostics: " + diagnosticsError); + } + throw new RuntimeException(e); + } + } + + private static void addCrashDiagnostics(List arguments) { + try { + Files.createDirectories(GRADLE_DAEMON_DIAGNOSTICS_DIR); + if (JavaVirtualMachine.isJ9()) { + arguments.add("-Xdump:directory=" + GRADLE_DAEMON_DIAGNOSTICS_DIR.toAbsolutePath()); + } else { + arguments.add( + "-XX:ErrorFile=" + + GRADLE_DAEMON_DIAGNOSTICS_DIR.toAbsolutePath() + + "/hs_err_pid%p.log"); + } + } catch (IOException e) { + System.err.println("Failed to configure Gradle daemon crash diagnostics: " + e); + } + } + + private void collectDaemonDiagnostics(String gradleVersion, Exception failure) + throws IOException { + Path failureDir = + GRADLE_DAEMON_DIAGNOSTICS_DIR.resolve( + gradleVersion + "-" + projectFolder.getFileName().toString()); + Files.createDirectories(failureDir); + + List summary = new ArrayList<>(); + Throwable cause = failure; + int causeIndex = 0; + while (cause != null) { + String message = cause.getMessage(); + int lineBreak = message != null ? message.indexOf('\n') : -1; + if (lineBreak >= 0) { + message = message.substring(0, lineBreak); + } + summary.add("failure[" + causeIndex++ + "]=" + cause.getClass().getName() + ": " + message); + cause = cause.getCause(); + } + summary.add("gradleVersion=" + gradleVersion); + + Path cgroupMembership = Paths.get("/proc/self/cgroup"); + Path daemonLogDir = testKitFolder.resolve("test-kit-daemon/" + gradleVersion); + Path daemonLog = newestDaemonLog(daemonLogDir); + if (daemonLog != null) { + copyIfExists(daemonLog, failureDir.resolve(daemonLog.getFileName()), summary); + summary.add("daemonLog=" + daemonLog); + + Matcher matcher = DAEMON_LOG_PATTERN.matcher(daemonLog.getFileName().toString()); + if (matcher.matches()) { + String daemonIdentifier = matcher.group(1); + summary.add("daemonIdentifier=" + daemonIdentifier); + if (daemonIdentifier.matches("\\d+")) { + summary.add("daemonPid=" + daemonIdentifier); + Path procRoot = Paths.get("/proc"); + if (Files.isDirectory(procRoot)) { + Path processDir = procRoot.resolve(daemonIdentifier); + summary.add("daemonProcessPresent=" + Files.isDirectory(processDir)); + copyIfExists(processDir.resolve("status"), failureDir.resolve("proc-status"), summary); + copyIfExists(processDir.resolve("limits"), failureDir.resolve("proc-limits"), summary); + Path daemonCgroupMembership = processDir.resolve("cgroup"); + copyIfExists(daemonCgroupMembership, failureDir.resolve("proc-cgroup"), summary); + if (Files.isRegularFile(daemonCgroupMembership)) { + cgroupMembership = daemonCgroupMembership; + } + } else { + summary.add("daemonProcessPresent=unknown"); + } + } + } + + try { System.out.println("=============================================================="); - System.out.println("Gradle Daemon log:\n" + new String(Files.readAllBytes(daemonLog))); + System.out.println( + "Gradle Daemon log:\n" + + new String(Files.readAllBytes(daemonLog), StandardCharsets.UTF_8)); System.out.println("=============================================================="); + } catch (IOException e) { + summary.add("daemonLogReadFailed=" + e); } - throw new RuntimeException(e); + } else { + summary.add("daemonLog=not found"); + } + + summary.add("cgroupMembership=" + cgroupMembership); + Path cgroupDirectory = resolveCgroupV2Directory(cgroupMembership, summary); + summary.add("cgroupDirectory=" + cgroupDirectory); + copyIfExists( + cgroupDirectory.resolve("memory.events"), + failureDir.resolve("cgroup-memory.events"), + summary); + copyIfExists( + cgroupDirectory.resolve("memory.current"), + failureDir.resolve("cgroup-memory.current"), + summary); + copyIfExists( + cgroupDirectory.resolve("memory.peak"), failureDir.resolve("cgroup-memory.peak"), summary); + copyIfExists( + cgroupDirectory.resolve("pids.events"), failureDir.resolve("cgroup-pids.events"), summary); + copyIfExists( + cgroupDirectory.resolve("pids.current"), + failureDir.resolve("cgroup-pids.current"), + summary); + copyIfExists( + cgroupDirectory.resolve("pids.max"), failureDir.resolve("cgroup-pids.max"), summary); + + Files.write(failureDir.resolve("summary.txt"), summary, StandardCharsets.UTF_8); + System.out.println("Gradle daemon diagnostics written to " + failureDir); + } + + private static Path resolveCgroupV2Directory(Path cgroupMembership, List summary) { + Path cgroupRoot = Paths.get("/sys/fs/cgroup"); + if (!Files.isRegularFile(cgroupMembership)) { + return cgroupRoot; + } + + try { + for (String line : Files.readAllLines(cgroupMembership, StandardCharsets.UTF_8)) { + if (line.startsWith("0::")) { + String relativePath = line.substring(3); + if (relativePath.startsWith("/")) { + relativePath = relativePath.substring(1); + } + Path cgroupDirectory = cgroupRoot.resolve(relativePath); + if (Files.isDirectory(cgroupDirectory)) { + return cgroupDirectory; + } + summary.add("cgroupDirectoryNotFound=" + cgroupDirectory); + break; + } + } + } catch (IOException e) { + summary.add("cgroupMembershipReadFailed[" + cgroupMembership + "]=" + e); + } + return cgroupRoot; + } + + private static Path newestDaemonLog(Path daemonLogDir) throws IOException { + if (!Files.isDirectory(daemonLogDir)) { + return null; + } + try (Stream daemonLogs = Files.list(daemonLogDir)) { + return daemonLogs + .filter(path -> DAEMON_LOG_PATTERN.matcher(path.getFileName().toString()).matches()) + .max(Comparator.comparingLong(path -> path.toFile().lastModified())) + .orElse(null); + } + } + + private static void copyIfExists(Path source, Path destination, List summary) { + if (!Files.exists(source)) { + return; + } + try { + Files.copy(source, destination, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + summary.add("copyFailed[" + source + "]=" + e); } }