BOM, support ARM libraries (#45655)

XiaofeiCao · web-flow · commit 47f54e8cfb06 · 2025-06-27T13:48:13.000-07:00
* support other groupIds in patch release

patch release

* support BOM generation

* adjustments

* break

* fix

* add resourcemanger packages to patch release

* BomGenerator, hack to only include premium packages

* put the hack at the front to avoid inflating conflictlist

* remove ARM libraries from patch release candidate

* add artifact filter

* nit, rename
diff --git a/eng/bomgenerator/.gitignore b/eng/bomgenerator/.gitignore
@@ -0,0 +1,2 @@
+inputdir/*
+outputdir/*
diff --git a/eng/bomgenerator/generateAzureSDKBOM.ps1 b/eng/bomgenerator/generateAzureSDKBOM.ps1
@@ -14,6 +14,11 @@ $BomPomFilePath = Join-Path $RepoRoot "sdk" "boms" "azure-sdk-bom" $PomFileName
 $EngScriptDir = Join-Path  $EngDir "scripts"
 $BomGeneratorPomFilePath = Join-Path ${PSScriptRoot} $PomFileName
 $NewBomFilePath = Join-Path $OutputDir $PomFileName
+$GroupIds = @("com.azure", "com.azure.resourcemanager")
+# This directory contains artifacts to include organized by {GroupId}.txt for each GroupId.
+# If this file is present for a given GroupId, only those artifacts in this file will be included.
+# Otherwise, no filter will be applied, meaning all artifacts under this GroupId will be included.
+$IncludesDirectoryPath = Join-Path ${PSScriptRoot} "includes"
 
 . (Join-Path $EngScriptDir syncversionclient.ps1)
 
@@ -36,8 +41,7 @@ function UpdateBomProjectElement($OldPomFilePath, $NewPomFilePath) {
 Write-Output "InputDir:$($InputDir)"
 Write-Output "OutputDir:$($OutputDir)"
 Write-Output "Updating version_client.txt file by looking at the packages released to maven."
-# TODO (alzimmer): Handle other group IDs for generating BOMs. https://github.com/Azure/azure-sdk-for-java/issues/44475
-SyncVersionClientFile -GroupId "com.azure"
+SyncVersionClientFile -GroupIds $GroupIds
 Write-Output "Updated version_client.txt file."
 
 New-Item -Path $PSScriptRoot -Name "inputdir" -ItemType "directory" -Force
@@ -50,7 +54,12 @@ if (!(Test-Path -Path $DefaultPomFilePath)) {
   Copy-Item $BomPomFilePath -Destination $InputDir
 }
 
-$cmdoutput = mvn clean install -f $BomGeneratorPomFilePath
+if (Test-Path -Path $IncludesDirectoryPath) {
+  Copy-Item $IncludesDirectoryPath -Destination $InputDir -Recurse -Force
+}
+
+$groupIdString = $GroupIds -join ","
+$cmdoutput = mvn clean install -f $BomGeneratorPomFilePath -DgroupIds="$groupIdString"
 
 if (Test-Path -Path $BomPomFilePath && Test-Path -Path $NewBomFilePath) {
   Copy-Item $NewBomFilePath -Destination $BomPomFilePath -Force
diff --git a/eng/bomgenerator/includes/com.azure.resourcemanager.txt b/eng/bomgenerator/includes/com.azure.resourcemanager.txt
@@ -0,0 +1,24 @@
+azure-resourcemanager
+azure-resourcemanager-appplatform
+azure-resourcemanager-appservice
+azure-resourcemanager-authorization
+azure-resourcemanager-cdn
+azure-resourcemanager-compute
+azure-resourcemanager-containerinstance
+azure-resourcemanager-containerregistry
+azure-resourcemanager-containerservice
+azure-resourcemanager-cosmos
+azure-resourcemanager-dns
+azure-resourcemanager-eventhubs
+azure-resourcemanager-keyvault
+azure-resourcemanager-monitor
+azure-resourcemanager-msi
+azure-resourcemanager-network
+azure-resourcemanager-privatedns
+azure-resourcemanager-resources
+azure-resourcemanager-redis
+azure-resourcemanager-search
+azure-resourcemanager-servicebus
+azure-resourcemanager-sql
+azure-resourcemanager-storage
+azure-resourcemanager-trafficmanager
diff --git a/eng/bomgenerator/pom.xml b/eng/bomgenerator/pom.xml
@@ -11,6 +11,7 @@
   <properties>
     <maven.compiler.source>11</maven.compiler.source>
     <maven.compiler.target>11</maven.compiler.target>
+    <groupIds>com.azure,com.azure.resourcemanager</groupIds>
   </properties>
   <dependencyManagement>
     <dependencies>
@@ -72,6 +73,7 @@
             <argument>-inputdir=${project.basedir}/inputdir</argument>
             <argument>-outputdir=${project.basedir}/outputdir</argument>
             <argument>-mode=generate</argument>
+            <argument>-groupids=${groupIds}</argument>
           </arguments>
           <cleanupDaemonThreads>false</cleanupDaemonThreads>
         </configuration>
diff --git a/eng/bomgenerator/src/main/java/com/azure/tools/bomgenerator/BomGenerator.java b/eng/bomgenerator/src/main/java/com/azure/tools/bomgenerator/BomGenerator.java
@@ -5,6 +5,7 @@
 
 import com.azure.tools.bomgenerator.models.BomDependency;
 import com.azure.tools.bomgenerator.models.BomDependencyManagement;
+import com.azure.tools.bomgenerator.models.BomDependencyNoVersion;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.databind.SerializationFeature;
 import com.fasterxml.jackson.dataformat.xml.XmlMapper;
@@ -38,9 +39,12 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 import static com.azure.tools.bomgenerator.Utils.ANALYZE_MODE;
@@ -69,19 +73,31 @@ public class BomGenerator {
     private String mode;
     private String outputDirectory;
     private String inputDirectory;
+    private final Pattern sdkDependencyPattern;
+    private final Set<String> groupIds;
+    private final DependencyFilter dependencyFilter;
 
     private static Logger logger = LoggerFactory.getLogger(BomGenerator.class);
 
-    BomGenerator(String inputDirectory, String outputDirectory, String mode) throws FileNotFoundException {
+    BomGenerator(String inputDirectory, String outputDirectory, String mode, List<String> groupIds) throws IOException {
         validateNotNullOrEmpty(inputDirectory, "inputDirectory");
         validateNotNullOrEmpty(outputDirectory, "outputDirectory");
 
         this.inputDirectory = inputDirectory;
         this.outputDirectory = outputDirectory;
         this.mode = (mode == null ? GENERATE_MODE : mode);
+        if (groupIds == null || groupIds.isEmpty()) {
+            logger.warn("groupIds null, will use com.azure");
+            this.groupIds = Set.of(BASE_AZURE_GROUPID);
+            this.sdkDependencyPattern = SDK_DEPENDENCY_PATTERN;
+        } else {
+            this.groupIds = new HashSet<>(groupIds);
+            this.sdkDependencyPattern = Pattern.compile(String.format("(%s):(.+);(.+);(.+)", String.join("|", this.groupIds)));
+        }
 
         parseInputs();
         validateInputs();
+        this.dependencyFilter = new DependencyFilter(this.inputDirectory);
 
         Path outputDirPath = Paths.get(outputDirectory);
         outputDirPath.toFile().mkdirs();
@@ -147,7 +163,7 @@ private boolean generate() {
         analyzer.reduce();
         Collection<BomDependency> outputDependencies = analyzer.getBomEligibleDependencies();
 
-        // 2. Create the new tree for the BOM.
+        // 3. Create the new tree for the BOM.
         analyzer = new DependencyAnalyzer(outputDependencies, externalDependencies, this.reportFileName);
         boolean validationFailed = analyzer.validate();
         outputDependencies = analyzer.getBomEligibleDependencies();
@@ -169,7 +185,7 @@ private List<BomDependency> scanVersioningClientFileDependencies() {
             for (String line : Files.readAllLines(Paths.get(inputFileName))) {
                 BomDependency dependency = scanDependency(line);
 
-                if(dependency != null) {
+                if(dependencyFilter.apply(dependency)) {
                     inputDependencies.add(dependency);
                 }
             }
@@ -244,17 +260,18 @@ private List<BomDependency> scanOverriddenDependencies() {
     }
 
     private BomDependency scanDependency(String line) {
-        Matcher matcher = SDK_DEPENDENCY_PATTERN.matcher(line);
+        Matcher matcher = sdkDependencyPattern.matcher(line);
         if (!matcher.matches()) {
             return null;
         }
 
-        if (matcher.groupCount() != 3) {
+        if (matcher.groupCount() != 4) {
             return null;
         }
 
-        String artifactId = matcher.group(1);
-        String version = matcher.group(2);
+        String groupId = matcher.group(1);
+        String artifactId = matcher.group(2);
+        String version = matcher.group(3);
 
         if(version.contains("-")) {
             // This is a non-GA library
@@ -264,11 +281,11 @@ private BomDependency scanDependency(String line) {
         if (EXCLUSION_LIST.contains(artifactId)
             || artifactId.contains(AZURE_PERF_LIBRARY_IDENTIFIER)
             || (artifactId.contains(AZURE_TEST_LIBRARY_IDENTIFIER))) {
-            logger.trace("Skipping dependency {}:{}", BASE_AZURE_GROUPID, artifactId);
+            logger.trace("Skipping dependency {}:{}", groupId, artifactId);
             return null;
         }
 
-        return new BomDependency(BASE_AZURE_GROUPID, artifactId, version);
+        return new BomDependency(groupId, artifactId, version);
     }
 
     private Model readModel() {
@@ -338,8 +355,44 @@ private void writeBom(Collection<BomDependency> bomDependencies) {
         dependencies.sort(new DependencyComparator());
 
         // Remove external dependencies from the BOM.
-        dependencies = dependencies.stream().filter(dependency -> BASE_AZURE_GROUPID.equals(dependency.getGroupId())).collect(Collectors.toList());
+        dependencies = dependencies.stream().filter(dependency -> groupIds.contains(dependency.getGroupId())).collect(Collectors.toList());
         management.setDependencies(dependencies);
         writeModel(this.pomFileName, this.outputFileName, model);
     }
+
+    private static class DependencyFilter {
+        private final Map<String, Set<BomDependencyNoVersion>> groupIdDependenciesMap = new HashMap<>();
+        DependencyFilter(String inputDirectory) throws IOException {
+            Path includesDirectory = Paths.get(inputDirectory, "includes");
+            String fileSuffix = ".txt";
+            Files.list(includesDirectory)
+                .filter(file -> Files.isRegularFile(file) && file.getFileName().toString().endsWith(fileSuffix))
+                .forEach(file -> {
+                    try {
+                        String fileName = file.getFileName().toString();
+                        String groupId = fileName.substring(0, fileName.length() - fileSuffix.length());
+                        Set<BomDependencyNoVersion> dependencies = groupIdDependenciesMap.computeIfAbsent(groupId, ignored -> new HashSet<>());
+                        for (String line : Files.readAllLines(file)) {
+                            if (line != null && !line.trim().isEmpty()) {
+                                dependencies.add(new BomDependencyNoVersion(groupId, line));
+                            }
+                        }
+                    } catch (IOException e) {
+                        throw new RuntimeException(e);
+                    }
+                });
+        }
+
+        boolean apply(BomDependency dependency) {
+            if (dependency == null) {
+                return false;
+            }
+            Set<BomDependencyNoVersion> allowedDependencies = this.groupIdDependenciesMap.get(dependency.getGroupId());
+            // if allowed list is null, no filter will be applied
+            if (allowedDependencies == null) {
+                return true;
+            }
+            return allowedDependencies.contains(Utils.toBomDependencyNoVersion(dependency));
+        }
+    }
 }
diff --git a/eng/bomgenerator/src/main/java/com/azure/tools/bomgenerator/Main.java b/eng/bomgenerator/src/main/java/com/azure/tools/bomgenerator/Main.java
@@ -4,8 +4,13 @@
 package com.azure.tools.bomgenerator;
 
 import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
 import java.util.regex.Matcher;
 import static com.azure.tools.bomgenerator.Utils.ANALYZE_MODE;
+import static com.azure.tools.bomgenerator.Utils.COMMANDLINE_GROUP_IDS;
 import static com.azure.tools.bomgenerator.Utils.COMMANDLINE_INPUTDIRECTORY;
 import static com.azure.tools.bomgenerator.Utils.COMMANDLINE_OUTPUTDIRECTORY;
 import static com.azure.tools.bomgenerator.Utils.COMMANDLINE_MODE;
@@ -25,15 +30,16 @@ public static void main(String[] args) {
             }
 
             System.out.println("Completed successfully.");
-        } catch (FileNotFoundException e) {
+        } catch (IOException e) {
             System.out.println("Error occurred.");
             e.printStackTrace();
             System.exit(1);
         }
     }
 
-    private static BomGenerator parseCommandLine(String[] args) throws FileNotFoundException {
+    private static BomGenerator parseCommandLine(String[] args) throws IOException {
         String inputDir = null, outputDir = null, mode = null;
+        List<String> groupIds = null;
         for (String arg : args) {
             Matcher matcher = COMMANDLINE_REGEX.matcher(arg);
             if (matcher.matches()) {
@@ -57,6 +63,10 @@ private static BomGenerator parseCommandLine(String[] args) throws FileNotFoundE
                             validateValues(argName, argValue, GENERATE_MODE, ANALYZE_MODE);
                             mode = argValue;
                             break;
+
+                        case COMMANDLINE_GROUP_IDS:
+                            groupIds = Arrays.asList(argValue.split(","));
+                            break;
                     }
                 }
 
@@ -65,7 +75,7 @@ private static BomGenerator parseCommandLine(String[] args) throws FileNotFoundE
 
         validateNotNullOrEmpty(inputDir, "inputDir");
         validateNotNullOrEmpty(outputDir, "outputDir");
-        BomGenerator generator = new BomGenerator(inputDir, outputDir, mode);
+        BomGenerator generator = new BomGenerator(inputDir, outputDir, mode, groupIds);
         return generator;
     }
 }
diff --git a/eng/bomgenerator/src/main/java/com/azure/tools/bomgenerator/Utils.java b/eng/bomgenerator/src/main/java/com/azure/tools/bomgenerator/Utils.java
@@ -45,11 +45,12 @@ public class Utils {
     public static final String COMMANDLINE_INPUTDIRECTORY = "inputdir";
     public static final String COMMANDLINE_OUTPUTDIRECTORY = "outputdir";
     public static final String COMMANDLINE_MODE = "mode";
+    public static final String COMMANDLINE_GROUP_IDS = "groupids";
     public static final String ANALYZE_MODE = "analyze";
     public static final String GENERATE_MODE = "generate";
     public  static final Pattern COMMANDLINE_REGEX = Pattern.compile("-(.*)=(.*)");
     public static final List<String> EXCLUSION_LIST = Arrays.asList("azure-spring-data-cosmos", "azure-core-test", "azure-sdk-all", "azure-sdk-parent", "azure-client-sdk-parent");
-    public static final Pattern SDK_DEPENDENCY_PATTERN = Pattern.compile("com.azure:(.+);(.+);(.+)");
+    public static final Pattern SDK_DEPENDENCY_PATTERN = Pattern.compile("(com.azure):(.+);(.+);(.+)");
     // TODO (alzimmer): Handle creation of BOM for v2 libraries. https://github.com/Azure/azure-sdk-for-java/issues/44475
     public static final String BASE_AZURE_GROUPID = "com.azure";
     public static final String AZURE_TEST_LIBRARY_IDENTIFIER = "-test";
diff --git a/eng/scripts/Generate-Patches-For-Automatic-Releases.ps1 b/eng/scripts/Generate-Patches-For-Automatic-Releases.ps1
@@ -54,7 +54,7 @@ try {
 
     # Reset each package to the latest stable release and update CHANGELOG, POM and README for patch release.
     foreach ($packageData in $packagesData) {
-        . "${PSScriptRoot}/generatepatch.ps1" -ArtifactIds $packageData["name"] -ServiceDirectoryName $packageData["ServiceDirectory"] -BranchName $branchName
+        . "${PSScriptRoot}/generatepatch.ps1" -ArtifactIds $packageData["name"] -ServiceDirectoryName $packageData["ServiceDirectory"] -BranchName $branchName -GroupId $packageData["groupId"]
         $libraryList += $packageData["groupId"] + ":" + $packageData["name"] + ","
     }
 
diff --git a/eng/scripts/Update-Artifacts-List-For-Patch-Release.ps1 b/eng/scripts/Update-Artifacts-List-For-Patch-Release.ps1
@@ -81,7 +81,7 @@ foreach ($line in Get-Content "${PSScriptRoot}/../pipelines/patch_release_client
     if (($line) -and !($line.StartsWith("#"))) {
         $libraryId = $line.split(" ")[0]
         $groupId, $artifactId = $libraryId.split(":")
-        $ArtifactInfos[$artifactId] = GetVersionInfoForMavenArtifact -ArtifactId $artifactId
+        $ArtifactInfos[$artifactId] = GetVersionInfoForMavenArtifact -ArtifactId $artifactId -GroupId $groupId
     }
 }
 
diff --git a/eng/scripts/bomhelpers.ps1 b/eng/scripts/bomhelpers.ps1
@@ -7,11 +7,11 @@ class MavenArtifactInfo {
   [String] $LatestGAOrPatchVersion
   [String] $LatestReleasedVersion
 
-  MavenArtifactInfo($ArtifactId, $LatestGAOrPatchVersion, $LatestReleasedVersion) {
+  MavenArtifactInfo($ArtifactId, $LatestGAOrPatchVersion, $LatestReleasedVersion, $GroupId = "com.azure") {
     $this.ArtifactId = $ArtifactId
     $this.LatestGAOrPatchVersion = $LatestGAOrPatchVersion
     $this.LatestReleasedVersion = $LatestReleasedVersion
-    $this.GroupId = 'com.azure'
+    $this.GroupId = $GroupId
   }
 }
 
@@ -42,23 +42,25 @@ function UpdateDependencyOfClientSDK() {
 }
 
 # Get all azure com client artifacts from Maven.
-function GetAllAzComClientArtifactsFromMaven() {
-  $webResponseObj = Invoke-WebRequest -Uri "https://repo1.maven.org/maven2/com/azure"
+function GetAllAzComClientArtifactsFromMaven($GroupId = "com.azure") {
+  $groupPath = $GroupId -replace '\.', '/'
+  $webResponseObj = Invoke-WebRequest -Uri "https://repo1.maven.org/maven2/$groupPath"
   $azureComArtifactIds = $webResponseObj.Links.HRef | Where-Object { ($_ -like 'azure-*') -and ($IgnoreList -notcontains $_) } |  ForEach-Object { $_.substring(0, $_.length - 1) }
   return $azureComArtifactIds | Where-Object { ($_ -like "azure-*") -and !($_ -like "azure-spring") }
 }
 
 # Get version info for an artifact.
-function GetVersionInfoForAnArtifactId([String]$ArtifactId) {
-  $mavenMetadataUrl = "https://repo1.maven.org/maven2/com/azure/$($ArtifactId)/maven-metadata.xml"
+function GetVersionInfoForAnArtifactId([String]$GroupId = "com.azure", [String]$ArtifactId) {
+  $groupPath = $GroupId -replace '\.', '/'
+  $mavenMetadataUrl = "https://repo1.maven.org/maven2/$groupPath/$($ArtifactId)/maven-metadata.xml"
   $webResponseObj = Invoke-WebRequest -Uri $mavenMetadataUrl
   $versions = ([xml]$webResponseObj.Content).metadata.versioning.versions.version
   $semVersions = $versions | ForEach-Object { [AzureEngSemanticVersion]::ParseVersionString($_) }
   $sortedVersions = [AzureEngSemanticVersion]::SortVersions($semVersions)
   $latestReleasedVersion = $sortedVersions[0].RawVersion
   $latestPatchOrGAVersion = $sortedVersions | Where-Object { !($_.IsPrerelease) } | ForEach-Object { $_.RawVersion } | Select-Object -First 1
 
-  $mavenArtifactInfo = [MavenArtifactInfo]::new($ArtifactId, $latestPatchOrGAVersion, $latestReleasedVersion)
+  $mavenArtifactInfo = [MavenArtifactInfo]::new($ArtifactId, $latestPatchOrGAVersion, $latestReleasedVersion, $GroupId)
 
   return $mavenArtifactInfo
 }
@@ -279,7 +281,7 @@ function GeneratePatch($PatchInfo, [string]$BranchName, [string]$RemoteName, [st
 
   if (!$releaseVersion) {
     Write-Output "Computing the latest release version for each of the relevant artifacts from Maven Central."
-    $mavenArtifactInfo = [MavenArtifactInfo](GetVersionInfoForAnArtifactId -ArtifactId $artifactId)
+    $mavenArtifactInfo = [MavenArtifactInfo](GetVersionInfoForAnArtifactId -GroupId $GroupId -ArtifactId $artifactId)
 
     if ($null -eq $mavenArtifactInfo) {
       LogError "Could not find $artifactId on Maven Central."
diff --git a/eng/scripts/patchhelpers.ps1 b/eng/scripts/patchhelpers.ps1
diff --git a/eng/scripts/syncversionclient.ps1 b/eng/scripts/syncversionclient.ps1

Original file line number	Diff line number	Diff line change
`@@ -54,7 +54,7 @@ try {`
`54`	`54`
`55`	`55`	`# Reset each package to the latest stable release and update CHANGELOG, POM and README for patch release.`
`56`	`56`	`foreach ($packageData in $packagesData) {`
`57`		`- . "${PSScriptRoot}/generatepatch.ps1" -ArtifactIds $packageData["name"] -ServiceDirectoryName $packageData["ServiceDirectory"] -BranchName $branchName`
	`57`	`+ . "${PSScriptRoot}/generatepatch.ps1" -ArtifactIds $packageData["name"] -ServiceDirectoryName $packageData["ServiceDirectory"] -BranchName $branchName -GroupId $packageData["groupId"]`
`58`	`58`	`$libraryList += $packageData["groupId"] + ":" + $packageData["name"] + ","`
`59`	`59`	`}`
`60`	`60`
Original file line number	Diff line number	Diff line change
`@@ -81,7 +81,7 @@ foreach ($line in Get-Content "${PSScriptRoot}/../pipelines/patch_release_client`
`81`	`81`	`if (($line) -and !($line.StartsWith("#"))) {`
`82`	`82`	`$libraryId = $line.split(" ")[0]`
`83`	`83`	`$groupId, $artifactId = $libraryId.split(":")`
`84`		`- $ArtifactInfos[$artifactId] = GetVersionInfoForMavenArtifact -ArtifactId $artifactId`
	`84`	`+ $ArtifactInfos[$artifactId] = GetVersionInfoForMavenArtifact -ArtifactId $artifactId -GroupId $groupId`
`85`	`85`	`}`
`86`	`86`	`}`
`87`	`87`