diff --git a/.github/workflows/pre-review.yml b/.github/workflows/pre-review.yml index 6fbdac103a0..6c13d5181fa 100644 --- a/.github/workflows/pre-review.yml +++ b/.github/workflows/pre-review.yml @@ -82,6 +82,26 @@ jobs: cache-disabled: true - name: Gradle Compile run: ./gradlew build -x test -x spotlessCheck + verify-source-metadata: + name: "Verify Dependency Source Metadata" + runs-on: ubuntu-latest + needs: [spotless-checkLicense, gradle-wrapper, repolint] + steps: + - name: Checkout Repo + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.event.pull_request.head.sha || github.ref }} + - name: Set up Java + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + with: + distribution: temurin + java-version: 21 + - name: Setup Gradle + uses: gradle/actions/setup-gradle@39e147cb9de83bb9910b8ef8bd7fff0ee20fcd6f # v6.0.1 + with: + cache-disabled: true + - name: Verify source artifacts recorded in verification-metadata.xml + run: ./gradlew verifySourceArtifacts unitTests: runs-on: besu-research-ubuntu-16 # more cores needs: [spotless-checkLicense, gradle-wrapper, repolint] @@ -126,7 +146,7 @@ jobs: unittests-passed: name: "unittests-passed" runs-on: ubuntu-latest - needs: [compile, unitTests] + needs: [compile, unitTests, verify-source-metadata] permissions: checks: write statuses: write diff --git a/build.gradle b/build.gradle index 6a6cf55d5a6..41c0d670bd9 100644 --- a/build.gradle +++ b/build.gradle @@ -474,66 +474,80 @@ configure(allprojects - project(':platform')) { } } -task resolveSourceArtifacts { - group = 'verification' - description = 'Resolves source artifacts for all configurations so they can be included in dependency verification metadata' - doLast { - def componentIds = new HashSet() - def unresolvedConfigs = 0 - - def collectComponentIds = { configs, String label -> - configs.matching { it.canBeResolved }.each { config -> - try { - config.incoming.resolutionResult.allComponents.each { component -> - componentIds.add(component.id) - } - } catch (Exception e) { - unresolvedConfigs++ - logger.info("Could not resolve ${label} '${config.name}': ${e.message}") +ext.collectExternalComponentIds = { + -> + def componentIds = new HashSet() + def unresolvedConfigs = 0 + + def collectComponentIds = { configs, String label -> + configs.matching { it.canBeResolved }.each { config -> + try { + config.incoming.resolutionResult.allComponents.each { component -> + componentIds.add(component.id) } + } catch (Exception e) { + unresolvedConfigs++ + logger.info("Could not resolve ${label} '${config.name}': ${e.message}") } } + } - allprojects.each { proj -> - collectComponentIds(proj.configurations, "${proj.path} config") - collectComponentIds(proj.buildscript.configurations, "${proj.path} buildscript config") - } + allprojects.each { proj -> + collectComponentIds(proj.configurations, "${proj.path} config") + collectComponentIds(proj.buildscript.configurations, "${proj.path} buildscript config") + } - if (unresolvedConfigs > 0) { - logger.warn("${unresolvedConfigs} configurations could not be resolved (run with --info for details)") - } + if (unresolvedConfigs > 0) { + logger.warn("${unresolvedConfigs} configurations could not be resolved (run with --info for details)") + } - def externalIds = componentIds.findAll { it instanceof org.gradle.api.artifacts.component.ModuleComponentIdentifier } - logger.lifecycle("Resolving sources for ${externalIds.size()} external dependencies...") + return componentIds.findAll { it instanceof org.gradle.api.artifacts.component.ModuleComponentIdentifier } +} - // Populated by resolveSources across both project and buildscript repository calls - def resolvedComponentIds = new HashSet() - def resolveSources = { depHandler, componentIdsToResolve, String label -> - def sizeBefore = resolvedComponentIds.size() - def result = depHandler.createArtifactResolutionQuery() - .forComponents(componentIdsToResolve) - .withArtifacts(JvmLibrary, SourcesArtifact) - .execute() - result.resolvedComponents.each { component -> - component.getArtifacts(SourcesArtifact).each { source -> - if (source instanceof ResolvedArtifactResult) { - source.file // access .file to trigger download and register with verification metadata - resolvedComponentIds.add(component.id) - } else { - logger.warn("Could not download sources for ${component.id}: ${source}") - } +// Retries buildscript repositories (which include the Gradle Plugin Portal) for any +// components whose sources did not resolve against the project repositories. +ext.forEachResolvedSourceArtifact = { Collection externalIds, Closure action -> + def resolvedComponentIds = new HashSet() + + def resolveSources = { depHandler, componentIdsToResolve, String label -> + def sizeBefore = resolvedComponentIds.size() + def result = depHandler.createArtifactResolutionQuery() + .forComponents(componentIdsToResolve) + .withArtifacts(JvmLibrary, SourcesArtifact) + .execute() + result.resolvedComponents.each { component -> + component.getArtifacts(SourcesArtifact).each { source -> + if (source instanceof ResolvedArtifactResult) { + action(component.id, source) + resolvedComponentIds.add(component.id) + } else { + logger.info("Could not resolve sources for ${component.id}: ${source}") } } - def count = resolvedComponentIds.size() - sizeBefore - logger.lifecycle("Resolved sources for ${count} components from ${label}") } + def count = resolvedComponentIds.size() - sizeBefore + logger.lifecycle("Resolved sources for ${count} components from ${label}") + } - resolveSources(dependencies, externalIds, "project repositories") + resolveSources(dependencies, externalIds, "project repositories") - // Retry unresolved sources using buildscript repositories (includes Gradle Plugin Portal) - def unresolvedIds = externalIds.findAll { !resolvedComponentIds.contains(it) } - if (!unresolvedIds.isEmpty()) { - resolveSources(buildscript.dependencies, unresolvedIds, "buildscript repositories") + def unresolvedIds = externalIds.findAll { !resolvedComponentIds.contains(it) } + if (!unresolvedIds.isEmpty()) { + resolveSources(buildscript.dependencies, unresolvedIds, "buildscript repositories") + } + + return resolvedComponentIds +} + +task resolveSourceArtifacts { + group = 'verification' + description = 'Resolves source artifacts for all configurations so they can be included in dependency verification metadata' + doLast { + def externalIds = collectExternalComponentIds() + logger.lifecycle("Resolving sources for ${externalIds.size()} external dependencies...") + + def resolvedComponentIds = forEachResolvedSourceArtifact(externalIds) { id, source -> + source.file // access .file to trigger download and register with verification metadata } def finalUnresolved = externalIds.size() - resolvedComponentIds.size() @@ -544,6 +558,50 @@ task resolveSourceArtifacts { } } +task verifySourceArtifacts { + group = 'verification' + description = 'Fails the build if gradle/verification-metadata.xml is missing source artifact entries for any resolvable dependency' + doLast { + def metadataFile = rootProject.file('gradle/verification-metadata.xml') + def slurper = new groovy.xml.XmlSlurper() + slurper.setFeature('http://xml.org/sax/features/namespaces', false) + def metadata = slurper.parse(metadataFile) + + def componentsWithRecordedSources = new HashSet() + metadata.components.component.each { comp -> + def group = comp.@group.text() + def name = comp.@name.text() + def version = comp.@version.text() + def expected = "${name}-${version}-sources.jar".toString() + if (comp.artifact.any { it.@name.text() == expected }) { + componentsWithRecordedSources.add("${group}:${name}:${version}".toString()) + } + } + + def externalIds = collectExternalComponentIds() + def missing = new TreeSet() + forEachResolvedSourceArtifact(externalIds) { id, source -> + def key = "${id.group}:${id.module}:${id.version}".toString() + if (!componentsWithRecordedSources.contains(key)) { + missing.add(key) + } + } + + if (!missing.isEmpty()) { + def list = missing.collect { " - ${it}" }.join('\n') + throw new GradleException( + "gradle/verification-metadata.xml is missing source artifact entries for ${missing.size()} " + + "${missing.size() == 1 ? 'dependency' : 'dependencies'}:\n" + + "${list}\n\n" + + "Regenerate the metadata by running:\n" + + " ./gradlew --write-verification-metadata sha256 resolveSourceArtifacts\n" + + "and commit the updated gradle/verification-metadata.xml.") + } + + logger.lifecycle("verification-metadata.xml contains source entries for all resolvable external components.") + } +} + task deploy() {} task checkMavenCoordinateCollisions {