diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index da0b0162b79..a06660e3f0e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -450,12 +450,16 @@ test_published_artifacts: - source .gitlab/gitlab-utils.sh - gitlab_section_start "collect-reports" "Collecting reports" - .gitlab/collect_reports.sh --destination ./check_reports --move + - .gitlab/collect_results.sh - gitlab_section_end "collect-reports" artifacts: when: always paths: - ./check_reports + - ./results - '.gradle/daemon/*/*.out.log' + reports: + junit: results/*.xml retry: max: 2 when: @@ -516,15 +520,21 @@ muzzle: after_script: - *container_info - *cgroup_info + - *set_datadog_api_keys - source .gitlab/gitlab-utils.sh - gitlab_section_start "collect-reports" "Collecting reports" - .gitlab/collect_reports.sh + - .gitlab/collect_results.sh + - .gitlab/upload_ciapp.sh $CACHE_TYPE - gitlab_section_end "collect-reports" artifacts: when: always paths: - ./reports + - ./results - '.gradle/daemon/*/*.out.log' + reports: + junit: results/*.xml muzzle-dep-report: extends: .gradle_build diff --git a/.gitlab/collect_results.sh b/.gitlab/collect_results.sh index d756ea9951e..0dc3a22f85c 100755 --- a/.gitlab/collect_results.sh +++ b/.gitlab/collect_results.sh @@ -67,15 +67,17 @@ do # E.g. for the example path: tomcat-5.5_forkedTest_TEST-TomcatServletV1ForkedTest.xml AGGREGATED_FILE_NAME=$(echo "$RESULT_XML_FILE" | rev | cut -d "/" -f 1,2,5 | rev | tr "/" "_") echo -n " as $AGGREGATED_FILE_NAME" - cp "$RESULT_XML_FILE" "$TEST_RESULTS_DIR/$AGGREGATED_FILE_NAME" + TARGET_DIR="$TEST_RESULTS_DIR" + mkdir -p "$TARGET_DIR" + cp "$RESULT_XML_FILE" "$TARGET_DIR/$AGGREGATED_FILE_NAME" # Insert file attribute to testcase XML nodes get_source_file - sed -i "/&1) fi -# Extract Java properties from the JVM used to run the tests -JAVA_PROPS=$($JAVA_BIN -XshowSettings:properties -version 2>&1) java_prop() { local PROP_NAME=$1 echo "$JAVA_PROPS" | grep "$PROP_NAME" | head -n1 | cut -d'=' -f2 | xargs @@ -27,11 +35,23 @@ junit_upload() { # Build custom tags array directly from arguments local custom_tags_args=() - # Extract job base name from CI_JOB_NAME (strip matrix suffix) + # Extract job base name from CI_JOB_NAME. + # Handles: + # - matrix suffix format: "job-name: [value, 1/6]" -> "job-name" + # - split suffix format: "job-name 1/6" -> "job-name" local job_base_name="${CI_JOB_NAME%%:*}" + job_base_name="$(echo "$job_base_name" | sed -E 's/[[:space:]]+[0-9]+\/[0-9]+$//')" # Add custom test configuration tags - custom_tags_args+=(--tags "test.configuration.jvm:${TEST_JVM}") + if [ -n "$TEST_JVM" ]; then + custom_tags_args+=(--tags "test.configuration.jvm:${TEST_JVM}") + custom_tags_args+=(--tags "runtime.name:$(java_prop java.runtime.name)") + custom_tags_args+=(--tags "runtime.vendor:$(java_prop java.vendor)") + custom_tags_args+=(--tags "runtime.version:$(java_prop java.version)") + custom_tags_args+=(--tags "os.architecture:$(java_prop os.arch)") + custom_tags_args+=(--tags "os.platform:$(java_prop os.name)") + custom_tags_args+=(--tags "os.version:$(java_prop os.version)") + fi if [ -n "$CI_NODE_INDEX" ] && [ -n "$CI_NODE_TOTAL" ]; then custom_tags_args+=(--tags "test.configuration.split:${CI_NODE_INDEX}/${CI_NODE_TOTAL}") fi @@ -43,12 +63,6 @@ junit_upload() { datadog-ci junit upload --service $SERVICE_NAME \ --logs \ --tags "test.traits:{\"category\":[\"$CACHE_TYPE\"]}" \ - --tags "runtime.name:$(java_prop java.runtime.name)" \ - --tags "runtime.vendor:$(java_prop java.vendor)" \ - --tags "runtime.version:$(java_prop java.version)" \ - --tags "os.architecture:$(java_prop os.arch)" \ - --tags "os.platform:$(java_prop os.name)" \ - --tags "os.version:$(java_prop os.version)" \ --tags "git.repository_url:https://github.com/DataDog/dd-trace-java" \ "${custom_tags_args[@]}" \ ./results diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/MuzzlePlugin.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/MuzzlePlugin.kt index 1b46385ca3e..ae15cc58fcb 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/MuzzlePlugin.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/MuzzlePlugin.kt @@ -120,17 +120,20 @@ class MuzzlePlugin : Plugin { project.afterEvaluate { // use runAfter to set up task finalizers in version order var runAfter: TaskProvider = muzzleTask + val muzzleReportTasks = mutableListOf>() project.extensions.getByType().directives.forEach { directive -> project.logger.debug("configuring {}", directive) if (directive.isCoreJdk) { runAfter = addMuzzleTask(directive, null, project, runAfter, muzzleBootstrap, muzzleTooling) + muzzleReportTasks.add(runAfter) } else { val range = resolveVersionRange(directive, system, session) muzzleDirectiveToArtifacts(directive, range).forEach { runAfter = addMuzzleTask(directive, it, project, runAfter, muzzleBootstrap, muzzleTooling) + muzzleReportTasks.add(runAfter) } if (directive.assertInverse) { @@ -139,6 +142,7 @@ class MuzzlePlugin : Plugin { muzzleDirectiveToArtifacts(inverseDirective, inverseRange).forEach { runAfter = addMuzzleTask(inverseDirective, it, project, runAfter, muzzleBootstrap, muzzleTooling) + muzzleReportTasks.add(runAfter) } } } @@ -146,8 +150,13 @@ class MuzzlePlugin : Plugin { project.logger.info("configured $directive") } + if (muzzleReportTasks.isEmpty() && !project.extensions.getByType().directives.any { it.assertPass }) { + muzzleReportTasks.add(muzzleTask) + } + val timingTask = project.tasks.register("muzzle-end") { startTimeMs.set(startTime) + muzzleResultFiles.from(muzzleReportTasks.map { it.flatMap { task -> task.result } }) } // last muzzle task to run runAfter.configure { diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/tasks/MuzzleEndTask.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/tasks/MuzzleEndTask.kt index daf0ce44b06..47ab18bb9c9 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/tasks/MuzzleEndTask.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/tasks/MuzzleEndTask.kt @@ -1,39 +1,167 @@ package datadog.gradle.plugin.muzzle.tasks import datadog.gradle.plugin.muzzle.pathSlug +import org.gradle.api.file.ConfigurableFileCollection import org.gradle.api.provider.Property import org.gradle.api.tasks.Input +import org.gradle.api.tasks.InputFiles import org.gradle.api.tasks.OutputFile +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity import org.gradle.api.tasks.TaskAction +import java.io.File +import java.io.StringWriter +import javax.xml.stream.XMLOutputFactory abstract class MuzzleEndTask : AbstractMuzzleTask() { @get:Input abstract val startTimeMs: Property + @get:InputFiles + @get:PathSensitive(PathSensitivity.RELATIVE) + abstract val muzzleResultFiles: ConfigurableFileCollection + + @get:OutputFile + val resultsFile = project + .layout + .buildDirectory + .file("test-results/muzzle/TEST-muzzle-${project.pathSlug}.xml") + @get:OutputFile - val resultsFile = project.rootProject + val legacyResultsFile = project.rootProject .layout .buildDirectory .file("${MUZZLE_TEST_RESULTS}/${project.pathSlug}_muzzle/results.xml") @TaskAction fun generatesResultFile() { + val report = buildJUnitReport() + writeReportFile(project.file(resultsFile), renderReportXml(report), "muzzle junit") + writeReportFile(project.file(legacyResultsFile), renderLegacyReportXml(report.durationSeconds), "muzzle legacy") + } + + private fun buildJUnitReport(): MuzzleJUnitReport { val endTimeMs = System.currentTimeMillis() val seconds = (endTimeMs - startTimeMs.get()).toDouble() / 1000.0 - with(project.file(resultsFile)) { - parentFile.mkdirs() - writeText( - """ - - - - - """.trimIndent() - ) - project.logger.info("Wrote muzzle results report to\n $this") + val testCases = muzzleResultFiles.files + .sortedBy { it.name } + .map { resultFile -> + val taskName = resultFile.name.removeSuffix(".txt") + when { + !resultFile.exists() -> { + MuzzleJUnitCase( + name = taskName, + failureMessage = "Muzzle result file missing", + failureText = "Expected ${resultFile.path}" + ) + } + + resultFile.readText() == "PASSING" -> MuzzleJUnitCase(name = taskName) + else -> { + MuzzleJUnitCase( + name = taskName, + failureMessage = "Muzzle validation failed", + failureText = resultFile.readText() + ) + } + } + } + return MuzzleJUnitReport( + suiteName = project.path, + module = project.path, + className = "muzzle.${project.pathSlug}", + durationSeconds = seconds, + testCases = testCases + ) + } + + private fun renderReportXml(report: MuzzleJUnitReport): String { + val output = StringWriter() + val xmlWriter = XMLOutputFactory.newInstance().createXMLStreamWriter(output) + with(xmlWriter) { + try { + writeStartDocument("UTF-8", "1.0") + writeCharacters("\n") + writeStartElement("testsuite") + writeAttribute("name", report.suiteName) + writeAttribute("tests", report.testCases.size.toString()) + writeAttribute("failures", report.failures.toString()) + writeAttribute("errors", "0") + writeAttribute("skipped", "0") + writeAttribute("time", report.durationSeconds.toString()) + writeCharacters("\n") + + writeStartElement("properties") + writeCharacters("\n") + writeEmptyElement("property") + writeAttribute("name", "category") + writeAttribute("value", "muzzle") + writeCharacters("\n") + writeEmptyElement("property") + writeAttribute("name", "module") + writeAttribute("value", report.module) + writeCharacters("\n") + writeEndElement() + writeCharacters("\n") + + report.testCases.forEach { testCase -> + writeStartElement("testcase") + writeAttribute("classname", report.className) + writeAttribute("name", testCase.name) + writeAttribute("time", "0") + if (testCase.failureMessage != null) { + writeCharacters("\n") + writeStartElement("failure") + writeAttribute("message", testCase.failureMessage) + writeCharacters(testCase.failureText ?: "") + writeEndElement() + writeCharacters("\n") + } + writeEndElement() + writeCharacters("\n") + } + writeEndElement() + writeEndDocument() + flush() + } finally { + close() + } } + return output.toString() + } + + private fun writeReportFile(file: File, xml: String, label: String) { + file.parentFile.mkdirs() + file.writeText(xml) + project.logger.info("Wrote $label report to\n $file") } + private fun renderLegacyReportXml(durationSeconds: Double): String { + return """ + + + + + """.trimIndent() + } + + private data class MuzzleJUnitReport( + val suiteName: String, + val module: String, + val className: String, + val durationSeconds: Double, + val testCases: List + ) { + val failures: Int + get() = testCases.count { it.failureMessage != null } + } + + private data class MuzzleJUnitCase( + val name: String, + val failureMessage: String? = null, + val failureText: String? = null + ) + companion object { private const val MUZZLE_TEST_RESULTS = "muzzle-test-results" } diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/tasks/MuzzleTask.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/tasks/MuzzleTask.kt index 32231e19c2b..ec3929ba619 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/tasks/MuzzleTask.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/tasks/MuzzleTask.kt @@ -66,10 +66,8 @@ abstract class MuzzleTask @Inject constructor( @get:Optional val muzzleDirective: Property = objects.property() - // This output is only used to make the task cacheable, this is not exposed @get:OutputFile - @get:Optional - protected val result: RegularFileProperty = objects.fileProperty().convention( + val result: RegularFileProperty = objects.fileProperty().convention( project.layout.buildDirectory.file("reports/$name.txt") ) diff --git a/buildSrc/src/test/kotlin/datadog/gradle/plugin/muzzle/tasks/MuzzleEndTaskTest.kt b/buildSrc/src/test/kotlin/datadog/gradle/plugin/muzzle/tasks/MuzzleEndTaskTest.kt new file mode 100644 index 00000000000..e567a1611f4 --- /dev/null +++ b/buildSrc/src/test/kotlin/datadog/gradle/plugin/muzzle/tasks/MuzzleEndTaskTest.kt @@ -0,0 +1,124 @@ +package datadog.gradle.plugin.muzzle.tasks + +import org.gradle.kotlin.dsl.register +import org.gradle.testfixtures.ProjectBuilder +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.io.TempDir +import org.w3c.dom.Document +import org.w3c.dom.Element +import java.nio.file.Path +import javax.xml.parsers.DocumentBuilderFactory +import kotlin.io.path.createDirectories +import kotlin.io.path.readText +import kotlin.io.path.writeText + +class MuzzleEndTaskTest { + + @TempDir + lateinit var tempDir: Path + + private lateinit var junitDoc: Document + private lateinit var legacyDoc: Document + + @BeforeEach + fun setup() { + val rootProject = ProjectBuilder.builder() + .withProjectDir(tempDir.toFile()) + .withName("root") + .build() + + val childProjectDir = tempDir.resolve("lettuce-5.0").createDirectories().toFile() + val project = ProjectBuilder.builder() + .withParent(rootProject) + .withName("lettuce-5.0") + .withProjectDir(childProjectDir) + .build() + + val passReportPath = project.layout.buildDirectory.file("reports/muzzle-pass.txt").get().asFile.toPath().apply { + parent.createDirectories() + writeText("PASSING") + } + + val failReportPath = project.layout.buildDirectory.file("reports/muzzle-fail.txt").get().asFile.toPath().apply { + parent.createDirectories() + writeText("java.lang.IllegalStateException: something is broken") + } + + val task = project.tasks.register("muzzle-end").get().apply { + startTimeMs.set(System.currentTimeMillis() - 1_000) + muzzleResultFiles.from(passReportPath.toFile(), failReportPath.toFile()) + } + + // Pre run the task + task.generatesResultFile() + + val junitReportXml = project.layout.buildDirectory + .file("test-results/muzzle/TEST-muzzle-lettuce-5.0.xml") + .get().asFile.toPath().readText() + junitDoc = parseXml(junitReportXml) + + val legacyReportXml = rootProject.layout.buildDirectory + .file("muzzle-test-results/lettuce-5.0_muzzle/results.xml") + .get().asFile.toPath().readText() + legacyDoc = parseXml(legacyReportXml) + } + + @Test + fun `junit report contains expected testsuite counters`() { + val suite = junitDoc.documentElement + assertEquals("testsuite", suite.tagName) + assertEquals(":lettuce-5.0", suite.getAttribute("name")) + assertEquals("2", suite.getAttribute("tests")) + assertEquals("1", suite.getAttribute("failures")) + assertEquals("0", suite.getAttribute("errors")) + assertEquals("0", suite.getAttribute("skipped")) + } + + @Test + fun `passed testcase has no failure node`() { + val passedTestCase = findTestCaseByName(junitDoc, "muzzle-pass") + assertNotNull(passedTestCase) + assertNull(passedTestCase.getElementsByTagName("failure").item(0)) + } + + @Test + fun `failed testcase contains failure node and message`() { + val failedTestCase = findTestCaseByName(junitDoc, "muzzle-fail") + assertNotNull(failedTestCase) + val failureNode = failedTestCase.getElementsByTagName("failure").item(0) as Element + assertEquals("Muzzle validation failed", failureNode.getAttribute("message")) + assertEquals("java.lang.IllegalStateException: something is broken", failureNode.textContent) + } + + @Test + fun `legacy report keeps historical shape`() { + val legacySuite = legacyDoc.documentElement + assertEquals("testsuite", legacySuite.tagName) + assertEquals("1", legacySuite.getAttribute("tests")) + assertEquals("0", legacySuite.getAttribute("id")) + assertEquals("muzzle-end", legacySuite.getAttribute("name")) + assertEquals(1, legacySuite.getElementsByTagName("testcase").length) + } + + private fun parseXml(xml: String): Document { + val builderFactory = DocumentBuilderFactory.newInstance() + builderFactory.isNamespaceAware = false + builderFactory.isIgnoringComments = true + return builderFactory.newDocumentBuilder().parse(xml.byteInputStream()) + } + + private fun findTestCaseByName(document: Document, name: String): Element { + val testCases = document.getElementsByTagName("testcase") + for (idx in 0 until testCases.length) { + val testCase = testCases.item(idx) as Element + if (testCase.getAttribute("name") == name) { + return testCase + } + } + throw IllegalStateException("Could not find testcase with name '$name'") + } +}