External dependency version overrides in core, eventhubs, servicebus

eng/pipelines/templates/jobs/live.tests.yml

@@ -21,6 +21,7 @@ parameters:
   TestGoals: $(TestGoals)
   TestOptions: $(TestOptions)
   TestParallelization: '1'
+  TestVersionSupport: $(TestVersionSupport)
 
 jobs:
   - job:
@@ -94,8 +95,10 @@ jobs:
           TestGoals: ${{ parameters.TestGoals }}
           TestOptions: ${{ parameters.TestOptions }}
           TestParallelization: ${{ parameters.TestParallelization }}
+          TestVersionSupport: ${{ parameters.TestVersionSupport }}
           TestEnvVars:
             AZURE_TEST_MODE: ${{ parameters.TestMode }}
+            AZURE_VERSION_OVERRIDE_TESTS: ${{ parameters.TestVersionSupport }}
             ${{ each var in parameters.EnvVars }}:
               ${{ var.key }}: ${{ var.value }}
 

eng/pipelines/templates/stages/archetype-sdk-tests.yml

@@ -59,6 +59,9 @@ parameters:
 - name: TestMode
   type: string
   default: 'LIVE'
+- name: TestVersionSupport
+  type: boolean
+  default: false
 - name: CloudConfig
   type: object
   default:
@@ -135,6 +138,7 @@ stages:
               TestResourceDirectories: ${{ parameters.TestResourceDirectories }}
               TestSetupSteps: ${{ parameters.TestSetupSteps }}
               TestMode: ${{ parameters.TestMode }}
+              TestVersionSupport: ${{ parameters.TestVersionSupport }}
               Artifacts: ${{ parameters.Artifacts }}
               AdditionalModules: ${{ parameters.AdditionalModules }}
               TestResultsFiles: ${{ parameters.TestResultsFiles }}

eng/pipelines/templates/steps/build-and-test.yml

@@ -29,6 +29,9 @@ parameters:
   - name: TestEnvVars
     type: object
     default: {}
+  - name: TestVersionSupport
+    type: boolean
+    default: false
   - name: IsLiveTest
     type: boolean
     default: false
@@ -48,6 +51,8 @@ steps:
       jdkArchitectureOption: 'x64'
       publishJUnitResults: false
       goals: 'install'
+    env: 
+      AZURE_VERSION_OVERRIDE_TESTS: ${{ parameters.TestVersionSupport }}
     condition: and(succeeded(), ne(variables['TestFromSource'], 'true'))
 
   - task: Maven@3

eng/pipelines/templates/steps/initialize-test-environment.yml

@@ -46,6 +46,13 @@ steps:
       arguments: '--update-type library --build-type client --sr'
     condition: and(succeeded(), eq(variables['TestFromSource'], 'true'))
 
+  - task: PythonScript@0
+    displayName: 'Update external dependency versions'
+    inputs:
+      scriptPath: 'eng/versioning/update_versions.py'
+      arguments: '--update-type external_dependency --build-type client --sr --vo $(VersionOverrides)'
+    condition: and(succeeded(), eq(variables['TestVersionSupport'], 'true'))
+
   - template: /eng/pipelines/templates/steps/generate-project-list-and-cache-maven-repository.yml
     parameters:
       Artifacts: ${{ parameters.Artifacts }}

eng/versioning/supported_external_dependency_versions.json

@@ -0,0 +1,55 @@
+{
+  "jackson_2.10":
+  [
+    { "com.fasterxml.jackson.core:jackson-annotations": "2.10.0" },
+    { "com.fasterxml.jackson.core:jackson-annotations": "2.10.0" },
+    { "com.fasterxml.jackson.core:jackson-core": "2.10.0" },
+    { "com.fasterxml.jackson.core:jackson-databind": "2.10.0" },
+    { "com.fasterxml.jackson.dataformat:jackson-dataformat-avro": "2.10.0" },
+    { "com.fasterxml.jackson.dataformat:jackson-dataformat-xml": "2.10.0" },
+    { "com.fasterxml.jackson.datatype:jackson-datatype-jdk8": "2.10.0" },
+    { "com.fasterxml.jackson.datatype:jackson-datatype-jsr310": "2.10.0" },
+    { "com.fasterxml.jackson.module:jackson-module-parameter-names": "2.10.0" },
+    { "com.fasterxml.jackson.module:jackson-module-scala_2.12": "2.10.0" }
+  ],
+  "jackson_2.11":
+  [
+    { "com.fasterxml.jackson.core:jackson-annotations": "2.11.0" },
+    { "com.fasterxml.jackson.core:jackson-annotations": "2.11.0" },
+    { "com.fasterxml.jackson.core:jackson-core": "2.11.0" },
+    { "com.fasterxml.jackson.core:jackson-databind": "2.11.0" },
+    { "com.fasterxml.jackson.dataformat:jackson-dataformat-avro": "2.11.0"},
+    { "com.fasterxml.jackson.dataformat:jackson-dataformat-xml": "2.11.0" },
+    { "com.fasterxml.jackson.datatype:jackson-datatype-jdk8": "2.11.0" },
+    { "com.fasterxml.jackson.datatype:jackson-datatype-jsr310": "2.11.0" },
+    { "com.fasterxml.jackson.module:jackson-module-parameter-names": "2.11.0" },
+    { "com.fasterxml.jackson.module:jackson-module-scala_2.12": "2.11.0" }
+  ],
+  "jackson_2.12":
+  [
+    { "com.fasterxml.jackson.core:jackson-annotations": "2.12.0" },
+    { "com.fasterxml.jackson.core:jackson-annotations": "2.12.0" },
+    { "com.fasterxml.jackson.core:jackson-core": "2.12.0" },
+    { "com.fasterxml.jackson.core:jackson-databind": "2.12.0" },
+    { "com.fasterxml.jackson.dataformat:jackson-dataformat-avro": "2.12.0" },
+    { "com.fasterxml.jackson.dataformat:jackson-dataformat-xml": "2.12.0" },
+    { "com.fasterxml.jackson.datatype:jackson-datatype-jdk8": "2.12.0" },
+    { "com.fasterxml.jackson.datatype:jackson-datatype-jsr310": "2.12.0" },
+    { "com.fasterxml.jackson.module:jackson-module-parameter-names": "2.12.0" },
+    { "com.fasterxml.jackson.module:jackson-module-scala_2.12": "2.12.0" }
+  ],
+  "reactor_2022":
+  [
+    { "io.projectreactor.netty:reactor-netty": "1.1.0-M5" },
+    { "io.projectreactor.netty:reactor-netty-http": "1.1.0-M5" },
+    { "io.projectreactor:reactor-core": "3.5.0-M5" },
+    { "io.projectreactor:reactor-test": "3.5.0-M5" }
+  ],
+  "reactor_2020":
+  [
+    { "io.projectreactor.netty:reactor-netty": "1.0.0" },
+    { "io.projectreactor.netty:reactor-netty-http": "1.0.0" },
+    { "io.projectreactor:reactor-core": "3.4.15" },
+    { "io.projectreactor:reactor-test": "3.4.15" }
+  ]
+}

eng/versioning/update_versions.py

@@ -31,7 +31,7 @@
 
 import argparse
 from datetime import timedelta
-import errno
+import json
 import os
 import re
 import sys
@@ -209,11 +209,28 @@ def load_version_map_from_file(the_file, version_map):
 
             version_map[module.name] = module
 
+def load_version_overrides(the_file, version_map, overrides_name):
+    with open(the_file) as f:
+        data = json.load(f)
+        if overrides_name not in data:
+            raise ValueError('Version override name: {0} is not found in {1}'.format(overrides_name, the_file))
+
+        overrides = data[overrides_name]
+        for override in overrides:
+            if len(override) != 1:
+                raise ValueError('Expected exactly one module, but got: {0}'.format(override))
+
+            for module_name in override:
+                module_str = module_name + ";" + override[module_name]
+                module = CodeModule(module_str)
+                version_map[module.name] = module
+                break
+
 def display_version_info(version_map):
     for value in version_map.values():
         print(value)
 
-def update_versions_all(update_type, build_type, target_file, skip_readme, auto_version_increment, library_array):
+def update_versions_all(update_type, build_type, target_file, skip_readme, auto_version_increment, library_array, version_overrides):
     version_map = {}
     ext_dep_map = {}
     # Load the version and/or external dependency file for the given UpdateType
@@ -229,6 +246,10 @@ def update_versions_all(update_type, build_type, target_file, skip_readme, auto_
         print('external_dependency_file=' + dependency_file)
         load_version_map_from_file(dependency_file, ext_dep_map)
 
+    if version_overrides and not version_overrides.startswith('$'):
+        # Azure DevOps passes '$(VersionOverrides)' when the variable value is not set
+        load_version_overrides("eng/versioning/supported_external_dependency_versions.json", ext_dep_map, version_overrides)
+
     display_version_info(version_map)
     display_version_info(ext_dep_map)
 
@@ -275,6 +296,7 @@ def main():
     parser.add_argument('--auto-version-increment', '--avi', action='store_true', help='If this script is being run after an auto version increment, add changelog entry for new version')
     # Comma separated list artifacts, has to be split into an array. If we're not skipping README updates, only update MD files for entries for the list of libraries passed in
     parser.add_argument('--library-list', '--ll', nargs='?', help='(Optional) Comma seperated list of groupId:artifactId. If updating MD files, only update entries in this list.')
+    parser.add_argument('--version_override', '--vo', nargs='?', help='(Optional) identifier of version update configuratation matching (exactly) first-level identifier in supported_external_dependency_versions.json')
     args = parser.parse_args()
     if args.build_type == BuildType.management:
         raise ValueError('{} is not currently supported.'.format(BuildType.management.name))
@@ -284,7 +306,7 @@ def main():
         library_array = args.library_list.split(',')
     print('library_array length: {0}'.format(len(library_array)))
     print(library_array)
-    update_versions_all(args.update_type, args.build_type, args.target_file, args.skip_readme, args.auto_version_increment, library_array)
+    update_versions_all(args.update_type, args.build_type, args.target_file, args.skip_readme, args.auto_version_increment, library_array, args.version_override)
     elapsed_time = time.time() - start_time
     print('elapsed_time={}'.format(elapsed_time))
     print('Total time for replacement: {}'.format(str(timedelta(seconds=elapsed_time))))

sdk/core/azure-core-tracing-opentelemetry/pom.xml

@@ -145,12 +145,6 @@
       <artifactId>opentelemetry-sdk-extension-autoconfigure</artifactId>
       <version>1.14.0-alpha</version> <!-- {x-version-update;io.opentelemetry:opentelemetry-sdk-extension-autoconfigure;external_dependency} -->
     </dependency>
-    <dependency>
-      <groupId>io.opentelemetry</groupId>
-      <artifactId>opentelemetry-sdk-testing</artifactId>
-      <version>1.14.0</version> <!-- {x-version-update;io.opentelemetry:opentelemetry-sdk-testing;external_dependency} -->
-      <scope>test</scope>
-    </dependency>
   </dependencies>
 
   <build>

sdk/core/azure-core/src/test/java/com/azure/core/util/serializer/SerializationBenchmark.java

@@ -86,6 +86,10 @@ class InnerModelJsonAny {
         }
 
         @JsonAnyGetter
+        public Map<String, String> getAny() {
+            return any;
+        }
+
         @JsonAnySetter
         private Map<String, String> any;
     }

sdk/core/ci.yml

@@ -172,26 +172,6 @@ extends:
     MatrixReplace:
       - TestGoals=(surefire:test)/$1 failsafe:integration-test failsafe:verify
     AdditionalStagesAfterBuild:
-      - stage: JacksonSupportedVersions
-        displayName: 'Jackson Supported Versions'
-        dependsOn: []
-        jobs:
-        - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml
-          parameters:
-            JobTemplatePath: /eng/pipelines/templates/jobs/ci.tests.yml
-            MatrixConfigs:
-              - Name: jackson_supported_version_tests
-                Path: sdk/core/supported-version-matrix.json
-                Selection: sparse
-                NonSparseParameters:
-                  - AZURE_CORE_TEST_SUPPORTED_JACKSON_VERSION
-                GenerateVMJobs: true
-            AdditionalParameters:
-              ServiceDirectory: core
-              Artifacts:
-                - name: azure-core-jackson-tests
-                  groupId: com.azure
-                  safeName: azurecorejacksontests
       - stage: JacksonUnsupportedVersions
         displayName: 'Jackson Unsupported Versions'
         dependsOn: []
@@ -267,6 +247,11 @@ extends:
               safeName: azurejsongson
           MatrixReplace:
             - TestGoals=(surefire:test)/$1 failsafe:integration-test failsafe:verify
+          AdditionalMatrixConfigs:
+            - Name: version_overrides_tests
+              Path: sdk/core/version-overrides-matrix.json
+              Selection: all
+              GenerateVMJobs: true
       - template: /sdk/storage/tests-template.yml
         parameters:
           AdditionalMatrixReplace:

sdk/core/version-overrides-matrix.json

@@ -0,0 +1,18 @@
+{
+  "matrix": {
+    "Agent": {
+      "ubuntu-20.04": { "OSVmImage": "MMSUbuntu20.04", "Pool": "azsdk-pool-mms-ubuntu-2004-general" }
+    },
+    "JavaTestVersion": "1.17",
+    "AZURE_TEST_HTTP_CLIENTS": "netty",
+    "TestGoals": "surefire:test",
+    "VersionOverrides": [
+        "jackson_2.10",
+        "jackson_2.11",
+        "jackson_2.12",
+        "reactor_2020",
+        "reactor_2022"
+    ],
+    "TestVersionSupport": true
+  }
+}

sdk/cosmos/azure-cosmos/src/test/java/com/azure/cosmos/CosmosDiagnosticsTest.java

@@ -42,6 +42,7 @@
 import com.azure.cosmos.models.ThroughputProperties;
 import com.azure.cosmos.rx.TestSuiteBase;
 import com.azure.cosmos.util.CosmosPagedFlux;
+import com.fasterxml.jackson.core.JsonFactory;
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
@@ -1179,7 +1180,7 @@ private void validateTransportRequestTimelineDirect(String diagnostics) {
 
     public void isValidJSON(final String json) {
         try {
-            final JsonParser parser = new ObjectMapper().createParser(json);
+            final JsonParser parser = new JsonFactory().createParser(json);
             while (parser.nextToken() != null) {
             }
         } catch (IOException ex) {

sdk/eventhubs/tests.yml

@@ -18,3 +18,8 @@ stages:
       SupportedClouds: 'Public,UsGov,China'
       EnvVars:
         AZURE_LOG_LEVEL: 2
+      AdditionalMatrixConfigs:
+        - Name: version_overrides_tests
+          Path: sdk/eventhubs/version-overrides-matrix.json
+          Selection: all
+          GenerateVMJobs: true

sdk/eventhubs/version-overrides-matrix.json

@@ -0,0 +1,14 @@
+{
+  "matrix": {
+    "Agent": {
+      "ubuntu-20.04": { "OSVmImage": "MMSUbuntu20.04", "Pool": "azsdk-pool-mms-ubuntu-2004-general" }
+    },
+    "JavaTestVersion": "1.17",
+    "TestGoals": "surefire:test",
+    "VersionOverrides": [
+      "reactor_2020",
+      "reactor_2022"
+    ],
+    "TestVersionSupport": true
+  }
+}

sdk/parents/azure-client-sdk-parent/pom.xml

@@ -1548,5 +1548,27 @@
         </plugins>
       </build>
     </profile>
+
+    <profile>
+      <id>external-dependency-version-overrides</id>
+      <activation>
+        <property>
+          <name>env.AZURE_VERSION_OVERRIDE_TESTS</name>
+        </property>
+      </activation>
+      <repositories>
+        <repository>
+          <id>repository.spring.milestone</id>
+          <name>Spring Milestone Repository</name>
+          <url>https://repo.spring.io/milestone</url>
+          <releases>
+            <enabled>true</enabled>
+          </releases>
+          <snapshots>
+            <enabled>true</enabled>
+          </snapshots>
+        </repository>
+      </repositories>
+    </profile>
   </profiles>
 </project>

sdk/servicebus/tests.yml

@@ -15,3 +15,8 @@ stages:
         AZURE_TENANT_ID: $(SERVICEBUS_TENANT_ID)
         AZURE_CLIENT_SECRET: $(SERVICEBUS_CLIENT_SECRET)
         AZURE_SUBSCRIPTION_ID: $(SERVICEBUS_SUBSCRIPTION_ID)
+      AdditionalMatrixConfigs:
+        - Name: version_overrides_tests
+          Path: sdk/servicebus/version-overrides-matrix.json
+          Selection: all
+          GenerateVMJobs: true

sdk/servicebus/version-overrides-matrix.json

@@ -0,0 +1,15 @@
+{
+  "matrix": {
+    "Agent": {
+      "ubuntu-20.04": { "OSVmImage": "MMSUbuntu20.04", "Pool": "azsdk-pool-mms-ubuntu-2004-general" }
+    },
+    "JavaTestVersion": "1.17",
+    "TestGoals": "surefire:test",
+    "VersionOverrides": [
+      "jackson_2.10",
+      "reactor_2020",
+      "reactor_2022"
+    ],
+    "TestVersionSupport": true
+  }
+}