update confluent dependency version to 7.6.0 (#30638)

* update confluent dependency version to 7.6.0

* make avroSchema field non-transient as the Schema class is now Serializable

* remove now unused org.codehaus.jackson package from GcpApiSurfaceTest

* update expected exception class in PubsubClientTest

* fix AvroGenericRecordToStorageApiProtoTest

* move org.tukaani:xz:1.9 dependency to testImplementation

* bring back the tukaani dep to the avroVersions configurations

* trigger the beam_PostCommitJava_Avro_Versions workflow

.github/trigger_files/beam_PostCommit_Java_Avro_Versions.json

@@ -1,3 +1,3 @@
 {
-  "comment": "Modify this file in a trivial way to cause this test suite to run"
-}
+    "comment": "Modify this file in a trivial way to cause this test suite to run"
+}

buildSrc/build.gradle.kts

@@ -45,7 +45,7 @@ dependencies {
   implementation("com.github.spotbugs.snom:spotbugs-gradle-plugin:5.0.14")
 
   runtimeOnly("com.google.protobuf:protobuf-gradle-plugin:0.8.13")                                         // Enable proto code generation
-  runtimeOnly("com.github.davidmc24.gradle-avro-plugin:gradle-avro-plugin:0.16.0")                         // Enable Avro code generation
+  runtimeOnly("com.github.davidmc24.gradle.plugin:gradle-avro-plugin:1.9.1")                               // Enable Avro code generation
   runtimeOnly("com.diffplug.spotless:spotless-plugin-gradle:5.6.1")                                        // Enable a code formatting plugin
   runtimeOnly("gradle.plugin.com.dorongold.plugins:task-tree:1.5")                                         // Adds a 'taskTree' task to print task dependency tree
   runtimeOnly("gradle.plugin.com.github.johnrengelman:shadow:7.1.1")                                       // Enable shading Java dependencies

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -664,8 +664,8 @@ class BeamModulePlugin implements Plugin<Project> {
         antlr_runtime                               : "org.antlr:antlr4-runtime:4.7",
         args4j                                      : "args4j:args4j:2.33",
         auto_value_annotations                      : "com.google.auto.value:auto-value-annotations:$autovalue_version",
-        avro                                        : "org.apache.avro:avro:1.8.2",
-        avro_tests                                  : "org.apache.avro:avro:1.8.2:tests",
+        avro                                        : "org.apache.avro:avro:1.11.3",
+        avro_tests                                  : "org.apache.avro:avro:1.11.3:tests",
         aws_java_sdk_cloudwatch                     : "com.amazonaws:aws-java-sdk-cloudwatch:$aws_java_sdk_version",
         aws_java_sdk_core                           : "com.amazonaws:aws-java-sdk-core:$aws_java_sdk_version",
         aws_java_sdk_dynamodb                       : "com.amazonaws:aws-java-sdk-dynamodb:$aws_java_sdk_version",
@@ -2448,7 +2448,7 @@ class BeamModulePlugin implements Plugin<Project> {
     // TODO: Decide whether this should be inlined into the one project that relies on it
     // or be left here.
     project.ext.applyAvroNature = {
-      project.apply plugin: "com.commercehub.gradle.plugin.avro"
+      project.apply plugin: "com.github.davidmc24.gradle.plugin.avro"
 
       // add dependency BeamModulePlugin defined custom tasks
       // they are defined only when certain flags are provided (e.g. -Prelease; -Ppublishing, etc)

sdks/java/extensions/avro/build.gradle

@@ -72,6 +72,7 @@ dependencies {
   }
   testImplementation library.java.avro_tests
   testImplementation library.java.junit
+  testImplementation "org.tukaani:xz:1.9" // marked as optional in avro
   testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow")
   testRuntimeOnly library.java.slf4j_jdk14
   avroVersions.each { k,v ->

sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroUtils.java

@@ -630,7 +630,7 @@ public static SimpleFunction<Row, byte[]> getRowToAvroBytesFunction(Schema beamS
   }
 
   private static class RowToAvroBytesFn extends SimpleFunction<Row, byte[]> {
-    private final transient org.apache.avro.Schema avroSchema;
+    private final org.apache.avro.Schema avroSchema;
     private final AvroCoder<GenericRecord> coder;
 
     RowToAvroBytesFn(Schema beamSchema) {

sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java

@@ -262,8 +262,7 @@ private static TableFieldSchema fieldDescriptorFromAvroField(Schema.Field field)
             elementType.getType() != Schema.Type.UNION,
             "Multiple non-null union types are not supported.");
         TableFieldSchema unionFieldSchema =
-            fieldDescriptorFromAvroField(
-                new Schema.Field(field.name(), elementType, field.doc(), field.defaultVal()));
+            fieldDescriptorFromAvroField(new Schema.Field(field.name(), elementType, field.doc()));
         builder =
             builder
                 .setType(unionFieldSchema.getType())

sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java

@@ -124,7 +124,6 @@ public void testGcpApiSurface() throws Exception {
             classesInPackage("javax"),
             classesInPackage("org.apache.avro"),
             classesInPackage("org.apache.beam"),
-            classesInPackage("org.codehaus.jackson"),
             classesInPackage("org.joda.time"),
             classesInPackage("org.threeten.bp"),
             classesInPackage("com.google.gson"));

sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProtoTest.java

@@ -301,7 +301,10 @@ enum TestEnum {
               .set(
                   "decimalValue",
                   new Conversions.DecimalConversion()
-                      .toBytes(bd, Schema.create(Schema.Type.NULL), LogicalTypes.decimal(1, 1)))
+                      .toBytes(
+                          bd,
+                          Schema.create(Schema.Type.NULL),
+                          LogicalTypes.decimal(bd.precision(), bd.scale())))
               .set("dateValue", now)
               .set("timestampMicrosValue", now.getMillis() * 1000)
               .set("timestampMicrosAsInstantValue", now)

sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtilsTest.java

@@ -145,7 +145,7 @@ public void testConvertGenericRecordToTableRow() throws Exception {
       }
       // After a Field is added to a Schema, it is assigned a position, so we can't simply reuse
       // the existing Field.
-      avroFields.add(new Schema.Field(field.name(), schema, field.doc(), field.defaultValue()));
+      avroFields.add(new Schema.Field(field.name(), schema, field.doc(), field.defaultVal()));
     }
     Schema avroSchema = Schema.createRecord(avroFields);
 

sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubClientTest.java

@@ -205,7 +205,7 @@ public void fromPubsubSchema() {
 
     assertThrows(
         "null definition should throw an exception",
-        NullPointerException.class,
+        SchemaParseException.class,
         () ->
             PubsubClient.fromPubsubSchema(
                 com.google.pubsub.v1.Schema.newBuilder().setType(Schema.Type.AVRO).build()));

sdks/java/io/kafka/build.gradle

@@ -29,7 +29,10 @@ provideIntegrationTestingDependencies()
 enableJavaPerformanceTesting()
 
 description = "Apache Beam :: SDKs :: Java :: IO :: Kafka"
-ext.summary = "Library to read Kafka topics."
+ext {
+    summary = "Library to read Kafka topics."
+    confluentVersion = "7.6.0"
+}
 
 def kafkaVersions = [
     '01103': "0.11.0.3",
@@ -68,14 +71,14 @@ dependencies {
   implementation library.java.jackson_annotations
   implementation library.java.jackson_databind
   implementation "org.springframework:spring-expression:5.3.27"
-  implementation ("io.confluent:kafka-avro-serializer:5.3.2") {
+  implementation ("io.confluent:kafka-avro-serializer:${confluentVersion}") {
     // zookeeper depends on "spotbugs-annotations:3.1.9" which clashes with current
     // "spotbugs-annotations:3.1.12" used in Beam. Not required.
     exclude group: "org.apache.zookeeper", module: "zookeeper"
     // "kafka-clients" has to be provided since user can use its own version.
     exclude group: "org.apache.kafka", module: "kafka-clients"
   }
-  implementation ("io.confluent:kafka-schema-registry-client:5.3.2") {
+  implementation ("io.confluent:kafka-schema-registry-client:${confluentVersion}") {
     // It depends on "spotbugs-annotations:3.1.9" which clashes with current
     // "spotbugs-annotations:3.1.12" used in Beam. Not required.
     exclude group: "org.apache.zookeeper", module: "zookeeper"