Update pom file version scanner

eng/pipelines/code-quality-reports.yml

@@ -53,7 +53,7 @@ jobs:
         displayName: 'Generate FromSource POM and directories for sparse checkout'
         inputs:
           scriptPath: 'eng/scripts/generate_from_source_pom.py'
-          arguments: '--set-skip-linting-projects SkipLintingProjects --project-list $(ProjectList)'
+          arguments: '--set-skip-linting-projects SkipLintingProjects --artifacts-list $(ArtifactsList) --additional-modules-list $(AdditionalModulesList)'
           workingDirectory: '$(System.DefaultWorkingDirectory)'
 
       - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml

eng/pipelines/latest-jdk.yml

@@ -50,7 +50,7 @@ stages:
             displayName: 'Generate FromSource POM and directories for sparse checkout'
             inputs:
               scriptPath: 'eng/scripts/generate_from_source_pom.py'
-              arguments: '--match-any-version --set-skip-linting-projects SkipLintingProjects --project-list com.azure:azure-core'
+              arguments: '--match-any-version --set-skip-linting-projects SkipLintingProjects --artifacts-list com.azure:azure-core'
               workingDirectory: '$(System.DefaultWorkingDirectory)'
 
           - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml

eng/pipelines/templates/stages/archetype-sdk-client-patch.yml

@@ -92,7 +92,7 @@ stages:
           displayName: 'Generate FromSource POM and directories for sparse checkout'
           inputs:
             scriptPath: 'eng/scripts/generate_from_source_pom.py'
-            arguments: '--project-list $(ProjectList)'
+            arguments: '--artifacts-list $(ArtifactsList) --additional-modules-list $(AdditionalModulesList)'
             workingDirectory: '$(System.DefaultWorkingDirectory)'
 
         - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
@@ -232,7 +232,7 @@ stages:
           displayName: 'Generate FromSource POM and directories for sparse checkout'
           inputs:
             scriptPath: 'eng/scripts/generate_from_source_pom.py'
-            arguments: '--project-list $(ProjectList)'
+            arguments: '--artifacts-list $(ArtifactsList) --additional-modules-list $(AdditionalModulesList)'
             workingDirectory: '$(System.DefaultWorkingDirectory)'
 
         - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml

eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml

@@ -32,17 +32,29 @@ steps:
     $additionalModules = '${{ convertToJson(parameters.AdditionalModules) }}' | ConvertFrom-Json
 
     $projectList = @()
+    $artifactsList = @()
+    $additionalModulesList = @()
     foreach ($artifact in $artifacts) {
       $projectList += "$($artifact.groupId):$($artifact.name)"
+      $artifactsList += "$($artifact.groupId):$($artifact.name)"
     }
     foreach ($artifact in $additionalModules) {
       $projectList += "$($artifact.groupId):$($artifact.name)"
+      $additionalModulesList += "$($artifact.groupId):$($artifact.name)"
     }
-    $projects = $projectList -join ','
 
+    $projects = $projectList -join ','
     Write-Host "ProjectList = $projects"
     Write-Host "##vso[task.setvariable variable=ProjectList;]$projects"
 
+    $artifactsString = $artifactsList -join ','
+    Write-Host "ArtifactsList = $artifactsString"
+    Write-Host "##vso[task.setvariable variable=ArtifactsList;]$artifactsString"
+
+    $additionalModulesString = $additionalModulesList -join ','
+    Write-Host "AdditionalModulesList = $additionalModulesString"
+    Write-Host "##vso[task.setvariable variable=AdditionalModulesList;]$additionalModulesString"
+
     $sha256 = new-object -TypeName System.Security.Cryptography.SHA256Managed
     $utf8 = new-object -TypeName System.Text.UTF8Encoding
 

eng/pipelines/templates/steps/initialize-test-environment.yml

@@ -77,7 +77,7 @@ steps:
     displayName: 'Generate FromSource POM and directories for sparse checkout'
     inputs:
       scriptPath: 'eng/scripts/generate_from_source_pom.py'
-      arguments: '--project-list $(ProjectList)'
+      arguments: '--artifacts-list $(ArtifactsList) --additional-modules-list $(AdditionalModulesList)'
       workingDirectory: '$(System.DefaultWorkingDirectory)'
     condition: and(succeeded(), eq(variables['TestFromSource'], 'true'))
 

eng/scripts/generate_from_source_pom.py

@@ -53,32 +53,37 @@
 sdk_string = "/sdk/"
 
 # Function that creates the aggregate POM.
-def create_from_source_pom(project_list: str, set_skip_linting_projects: str, match_any_version: bool):
-    project_list_identifiers = project_list.split(',')
+def create_from_source_pom(artifacts_list: str, additional_modules_list: str, set_skip_linting_projects: str, match_any_version: bool):
+    artifacts_list_identifiers = artifacts_list.split(',')
+
+    additional_modules_identifiers = []
+    if additional_modules_list is not None:
+        additional_modules_identifiers = additional_modules_list.split(',')
 
     # Get the artifact identifiers from client_versions.txt to act as our source of truth.
     artifact_identifier_to_version = load_client_artifact_identifiers()
 
-    projects = create_projects(project_list_identifiers, artifact_identifier_to_version, match_any_version)
+    projects = create_projects(artifacts_list_identifiers, artifact_identifier_to_version, match_any_version)
 
     dependent_modules: Set[str] = set()
 
     # Resolve all projects, including transitively, that are dependent on the projects in the project list.
-    for project_identifier in project_list_identifiers:
+    for project_identifier in artifacts_list_identifiers:
         dependent_modules = resolve_dependent_project(project_identifier, dependent_modules, projects)
 
     dependency_modules: Set[str] = set()
 
     # Resolve all dependencies of the projects in the project list and of the dependent modules.
-    for project_identifier in project_list_identifiers:
+    for project_identifier in artifacts_list_identifiers:
         dependency_modules = resolve_project_dependencies(project_identifier, dependency_modules, projects)
     for project_identifier in dependent_modules:
         dependency_modules = resolve_project_dependencies(project_identifier, dependency_modules, projects)
 
     source_projects: Set[Project] = set()
 
     # Finally map the project identifiers to projects.
-    add_source_projects(source_projects, project_list_identifiers, projects)
+    add_source_projects(source_projects, artifacts_list_identifiers, projects)
+    add_source_projects(source_projects, additional_modules_identifiers, projects)
     add_source_projects(source_projects, dependent_modules, projects)
     add_source_projects(source_projects, dependency_modules, projects)
 
@@ -105,7 +110,7 @@ def create_from_source_pom(project_list: str, set_skip_linting_projects: str, ma
         sparse_checkout_directories.add(sparse_checkout_directory)
         # The ServiceDirectories list should only ever contain the list of service
         # directories for the project list and nothing else.
-        if p.identifier in project_list_identifiers:
+        if p.identifier in artifacts_list_identifiers:
             # Sparse checkout directories can contain directories that aren't service directories.
             # (aka. /common). Any service directory will start with "/sdk/", everything else is
             # would be attributed to supporting libraries (ex. perf-test-core).
@@ -181,7 +186,7 @@ def load_client_artifact_identifiers() -> Dict[str, ArtifactVersion]:
 
 # Function that creates the Projects within the repository.
 # Projects contain a Maven identifier, module path, parent POM, its dependency Maven identifiers, and Maven identifiers for projects dependent on it.
-def create_projects(project_list_identifiers: list, artifact_identifier_to_version: Dict[str, ArtifactVersion], match_any_version: bool) -> Dict[str, Project]:
+def create_projects(artifacts_list_identifiers: list, artifact_identifier_to_version: Dict[str, ArtifactVersion], match_any_version: bool) -> Dict[str, Project]:
     projects: Dict[str, Project] = {}
 
     for root, _, files in os.walk(root_path):
@@ -198,7 +203,7 @@ def create_projects(project_list_identifiers: list, artifact_identifier_to_versi
 
             # Only parse files that are pom.xml files.
             if (file_name.startswith('pom') and file_name.endswith('.xml')):
-                project = create_project_for_pom(file_path, project_list_identifiers, artifact_identifier_to_version, match_any_version)
+                project = create_project_for_pom(file_path, artifacts_list_identifiers, artifact_identifier_to_version, match_any_version)
                 if project is not None:
                     projects[project.identifier] = project
 
@@ -210,7 +215,7 @@ def create_projects(project_list_identifiers: list, artifact_identifier_to_versi
 
     return projects
 
-def create_project_for_pom(pom_path: str, project_list_identifiers: list, artifact_identifier_to_version: Dict[str, ArtifactVersion], match_any_version: bool):
+def create_project_for_pom(pom_path: str, artifacts_list_identifiers: list, artifact_identifier_to_version: Dict[str, ArtifactVersion], match_any_version: bool):
     if 'eng' in pom_path.split(os.sep):
         return
 
@@ -227,7 +232,7 @@ def create_project_for_pom(pom_path: str, project_list_identifiers: list, artifa
         return Project(project_identifier, directory_path, module_path, parent_pom)
 
     # If the project isn't a track 2 POM skip it and not one of the project list identifiers.
-    if not project_identifier in project_list_identifiers and not is_spring_child_pom(tree_root) and not parent_pom in valid_parents: # Spring pom's parent can be empty.
+    if not project_identifier in artifacts_list_identifiers and not is_spring_child_pom(tree_root) and not parent_pom in valid_parents: # Spring pom's parent can be empty.
         return
 
     project = Project(project_identifier, directory_path, module_path, parent_pom)
@@ -303,14 +308,15 @@ def project_uses_client_parent(project: Project, projects: Dict[str, Project]) -
 
 def main():
     parser = argparse.ArgumentParser(description='Generated an aggregate POM for a From Source run.')
-    parser.add_argument('--project-list', '--pl', type=str)
+    parser.add_argument('--artifacts-list', '--al', type=str)
+    parser.add_argument('--additional-modules-list', '--aml', type=str, required=False, default=None, nargs='?')
     parser.add_argument('--set-skip-linting-projects', type=str)
     parser.add_argument('--match-any-version', action='store_true')
     args = parser.parse_args()
-    if args.project_list == None:
-        raise ValueError('Missing project list.')
+    if args.artifacts_list == None:
+        raise ValueError('Missing artifacts list.')
     start_time = time.time()
-    create_from_source_pom(args.project_list, args.set_skip_linting_projects, args.match_any_version)
+    create_from_source_pom(args.artifacts_list, args.additional_modules_list, args.set_skip_linting_projects, args.match_any_version)
     elapsed_time = time.time() - start_time
 
     print('Effective From Source POM File')

eng/versioning/external_dependencies.txt

@@ -430,7 +430,7 @@ media_com.microsoft.azure:adal4j;1.2.0
 resourcemanager_com.jcraft:jsch;0.1.55
 
 # sdk\storage\azure-storage-blob-cryptography\pom.xml
-storage_com.microsoft.azure:azure-storage;8.4.0
+storage_com.microsoft.azure:azure-storage;8.6.6
 
 # sdk\appconfiguration\azure-spring-cloud-test-appconfiguration-config\pom.xml
 spring_com.microsoft.azure:azure;1.34.0