CI check to make sure that all libraries have their source code verified

.github/workflows/pre-review.yml

@@ -82,6 +82,26 @@ jobs:
           cache-disabled: true
       - name: Gradle Compile
         run: ./gradlew build -x test -x spotlessCheck
+  verify-source-metadata:
+    name: "Verify Dependency Source Metadata"
+    runs-on: ubuntu-latest
+    needs: [spotless-checkLicense, gradle-wrapper, repolint]
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
+      - name: Set up Java
+        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
+        with:
+          distribution: temurin
+          java-version: 21
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@39e147cb9de83bb9910b8ef8bd7fff0ee20fcd6f # v6.0.1
+        with:
+          cache-disabled: true
+      - name: Verify source artifacts recorded in verification-metadata.xml
+        run: ./gradlew verifySourceArtifacts
   unitTests:
     runs-on: besu-research-ubuntu-16 # more cores
     needs: [spotless-checkLicense, gradle-wrapper, repolint]
@@ -126,7 +146,7 @@ jobs:
   unittests-passed:
     name: "unittests-passed"
     runs-on: ubuntu-latest
-    needs: [compile, unitTests]
+    needs: [compile, unitTests, verify-source-metadata]
     permissions:
       checks: write
       statuses: write

build.gradle

@@ -474,66 +474,80 @@ configure(allprojects - project(':platform')) {
   }
 }
 
-task resolveSourceArtifacts {
-  group = 'verification'
-  description = 'Resolves source artifacts for all configurations so they can be included in dependency verification metadata'
-  doLast {
-    def componentIds = new HashSet()
-    def unresolvedConfigs = 0
-
-    def collectComponentIds = { configs, String label ->
-      configs.matching { it.canBeResolved }.each { config ->
-        try {
-          config.incoming.resolutionResult.allComponents.each { component ->
-            componentIds.add(component.id)
-          }
-        } catch (Exception e) {
-          unresolvedConfigs++
-          logger.info("Could not resolve ${label} '${config.name}': ${e.message}")
+ext.collectExternalComponentIds = {
+  ->
+  def componentIds = new HashSet()
+  def unresolvedConfigs = 0
+
+  def collectComponentIds = { configs, String label ->
+    configs.matching { it.canBeResolved }.each { config ->
+      try {
+        config.incoming.resolutionResult.allComponents.each { component ->
+          componentIds.add(component.id)
         }
+      } catch (Exception e) {
+        unresolvedConfigs++
+        logger.info("Could not resolve ${label} '${config.name}': ${e.message}")
       }
     }
+  }
 
-    allprojects.each { proj ->
-      collectComponentIds(proj.configurations, "${proj.path} config")
-      collectComponentIds(proj.buildscript.configurations, "${proj.path} buildscript config")
-    }
+  allprojects.each { proj ->
+    collectComponentIds(proj.configurations, "${proj.path} config")
+    collectComponentIds(proj.buildscript.configurations, "${proj.path} buildscript config")
+  }
 
-    if (unresolvedConfigs > 0) {
-      logger.warn("${unresolvedConfigs} configurations could not be resolved (run with --info for details)")
-    }
+  if (unresolvedConfigs > 0) {
+    logger.warn("${unresolvedConfigs} configurations could not be resolved (run with --info for details)")
+  }
 
-    def externalIds = componentIds.findAll { it instanceof org.gradle.api.artifacts.component.ModuleComponentIdentifier }
-    logger.lifecycle("Resolving sources for ${externalIds.size()} external dependencies...")
+  return componentIds.findAll { it instanceof org.gradle.api.artifacts.component.ModuleComponentIdentifier }
+}
 
-    // Populated by resolveSources across both project and buildscript repository calls
-    def resolvedComponentIds = new HashSet()
-    def resolveSources = { depHandler, componentIdsToResolve, String label ->
-      def sizeBefore = resolvedComponentIds.size()
-      def result = depHandler.createArtifactResolutionQuery()
-        .forComponents(componentIdsToResolve)
-        .withArtifacts(JvmLibrary, SourcesArtifact)
-        .execute()
-      result.resolvedComponents.each { component ->
-        component.getArtifacts(SourcesArtifact).each { source ->
-          if (source instanceof ResolvedArtifactResult) {
-            source.file // access .file to trigger download and register with verification metadata
-            resolvedComponentIds.add(component.id)
-          } else {
-            logger.warn("Could not download sources for ${component.id}: ${source}")
-          }
+// Retries buildscript repositories (which include the Gradle Plugin Portal) for any
+// components whose sources did not resolve against the project repositories.
+ext.forEachResolvedSourceArtifact = { Collection externalIds, Closure action ->
+  def resolvedComponentIds = new HashSet()
+
+  def resolveSources = { depHandler, componentIdsToResolve, String label ->
+    def sizeBefore = resolvedComponentIds.size()
+    def result = depHandler.createArtifactResolutionQuery()
+      .forComponents(componentIdsToResolve)
+      .withArtifacts(JvmLibrary, SourcesArtifact)
+      .execute()
+    result.resolvedComponents.each { component ->
+      component.getArtifacts(SourcesArtifact).each { source ->
+        if (source instanceof ResolvedArtifactResult) {
+          action(component.id, source)
+          resolvedComponentIds.add(component.id)
+        } else {
+          logger.info("Could not resolve sources for ${component.id}: ${source}")
         }
       }
-      def count = resolvedComponentIds.size() - sizeBefore
-      logger.lifecycle("Resolved sources for ${count} components from ${label}")
     }
+    def count = resolvedComponentIds.size() - sizeBefore
+    logger.lifecycle("Resolved sources for ${count} components from ${label}")
+  }
 
-    resolveSources(dependencies, externalIds, "project repositories")
+  resolveSources(dependencies, externalIds, "project repositories")
 
-    // Retry unresolved sources using buildscript repositories (includes Gradle Plugin Portal)
-    def unresolvedIds = externalIds.findAll { !resolvedComponentIds.contains(it) }
-    if (!unresolvedIds.isEmpty()) {
-      resolveSources(buildscript.dependencies, unresolvedIds, "buildscript repositories")
+  def unresolvedIds = externalIds.findAll { !resolvedComponentIds.contains(it) }
+  if (!unresolvedIds.isEmpty()) {
+    resolveSources(buildscript.dependencies, unresolvedIds, "buildscript repositories")
+  }
+
+  return resolvedComponentIds
+}
+
+task resolveSourceArtifacts {
+  group = 'verification'
+  description = 'Resolves source artifacts for all configurations so they can be included in dependency verification metadata'
+  doLast {
+    def externalIds = collectExternalComponentIds()
+    logger.lifecycle("Resolving sources for ${externalIds.size()} external dependencies...")
+
+    def resolvedComponentIds = forEachResolvedSourceArtifact(externalIds) { id, source ->
+      source.file // access .file to trigger download and register with verification metadata
     }
 
     def finalUnresolved = externalIds.size() - resolvedComponentIds.size()
@@ -544,6 +558,50 @@ task resolveSourceArtifacts {
   }
 }
 
+task verifySourceArtifacts {
+  group = 'verification'
+  description = 'Fails the build if gradle/verification-metadata.xml is missing source artifact entries for any resolvable dependency'
+  doLast {
+    def metadataFile = rootProject.file('gradle/verification-metadata.xml')
+    def slurper = new groovy.xml.XmlSlurper()
+    slurper.setFeature('http://xml.org/sax/features/namespaces', false)
+    def metadata = slurper.parse(metadataFile)
+
+    def componentsWithRecordedSources = new HashSet()
+    metadata.components.component.each { comp ->
+      def group = comp.@group.text()
+      def name = comp.@name.text()
+      def version = comp.@version.text()
+      def expected = "${name}-${version}-sources.jar".toString()
+      if (comp.artifact.any { it.@name.text() == expected }) {
+        componentsWithRecordedSources.add("${group}:${name}:${version}".toString())
+      }
+    }
+
+    def externalIds = collectExternalComponentIds()
+    def missing = new TreeSet()
+    forEachResolvedSourceArtifact(externalIds) { id, source ->
+      def key = "${id.group}:${id.module}:${id.version}".toString()
+      if (!componentsWithRecordedSources.contains(key)) {
+        missing.add(key)
+      }
+    }
+
+    if (!missing.isEmpty()) {
+      def list = missing.collect { "  - ${it}" }.join('\n')
+      throw new GradleException(
+      "gradle/verification-metadata.xml is missing source artifact entries for ${missing.size()} " +
+      "${missing.size() == 1 ? 'dependency' : 'dependencies'}:\n" +
+      "${list}\n\n" +
+      "Regenerate the metadata by running:\n" +
+      "  ./gradlew --write-verification-metadata sha256 resolveSourceArtifacts\n" +
+      "and commit the updated gradle/verification-metadata.xml.")
+    }
+
+    logger.lifecycle("verification-metadata.xml contains source entries for all resolvable external components.")
+  }
+}
+
 task deploy() {}
 
 task checkMavenCoordinateCollisions {