diff --git a/.github/trigger_files/beam_PostCommit_Java_Avro_Versions.json b/.github/trigger_files/beam_PostCommit_Java_Avro_Versions.json index 08c2e40784a9..a03c067d2c4e 100644 --- a/.github/trigger_files/beam_PostCommit_Java_Avro_Versions.json +++ b/.github/trigger_files/beam_PostCommit_Java_Avro_Versions.json @@ -1,3 +1,3 @@ { - "comment": "Modify this file in a trivial way to cause this test suite to run" -} \ No newline at end of file + "comment": "Modify this file in a trivial way to cause this test suite to run" +} diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index bcb2f898f99d..cd8aed6d3a67 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -45,7 +45,7 @@ dependencies { implementation("com.github.spotbugs.snom:spotbugs-gradle-plugin:5.0.14") runtimeOnly("com.google.protobuf:protobuf-gradle-plugin:0.8.13") // Enable proto code generation - runtimeOnly("com.github.davidmc24.gradle-avro-plugin:gradle-avro-plugin:0.16.0") // Enable Avro code generation + runtimeOnly("com.github.davidmc24.gradle.plugin:gradle-avro-plugin:1.9.1") // Enable Avro code generation runtimeOnly("com.diffplug.spotless:spotless-plugin-gradle:5.6.1") // Enable a code formatting plugin runtimeOnly("gradle.plugin.com.dorongold.plugins:task-tree:1.5") // Adds a 'taskTree' task to print task dependency tree runtimeOnly("gradle.plugin.com.github.johnrengelman:shadow:7.1.1") // Enable shading Java dependencies diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy index c5d7ed8d571e..a03b1940e4a5 100644 --- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy +++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy @@ -664,8 +664,8 @@ class BeamModulePlugin implements Plugin { antlr_runtime : "org.antlr:antlr4-runtime:4.7", args4j : "args4j:args4j:2.33", auto_value_annotations : "com.google.auto.value:auto-value-annotations:$autovalue_version", - avro : "org.apache.avro:avro:1.8.2", - avro_tests : "org.apache.avro:avro:1.8.2:tests", + avro : "org.apache.avro:avro:1.11.3", + avro_tests : "org.apache.avro:avro:1.11.3:tests", aws_java_sdk_cloudwatch : "com.amazonaws:aws-java-sdk-cloudwatch:$aws_java_sdk_version", aws_java_sdk_core : "com.amazonaws:aws-java-sdk-core:$aws_java_sdk_version", aws_java_sdk_dynamodb : "com.amazonaws:aws-java-sdk-dynamodb:$aws_java_sdk_version", @@ -2448,7 +2448,7 @@ class BeamModulePlugin implements Plugin { // TODO: Decide whether this should be inlined into the one project that relies on it // or be left here. project.ext.applyAvroNature = { - project.apply plugin: "com.commercehub.gradle.plugin.avro" + project.apply plugin: "com.github.davidmc24.gradle.plugin.avro" // add dependency BeamModulePlugin defined custom tasks // they are defined only when certain flags are provided (e.g. -Prelease; -Ppublishing, etc) diff --git a/sdks/java/extensions/avro/build.gradle b/sdks/java/extensions/avro/build.gradle index ecd3ce867c83..6bf1dfc3f8eb 100644 --- a/sdks/java/extensions/avro/build.gradle +++ b/sdks/java/extensions/avro/build.gradle @@ -72,6 +72,7 @@ dependencies { } testImplementation library.java.avro_tests testImplementation library.java.junit + testImplementation "org.tukaani:xz:1.9" // marked as optional in avro testRuntimeOnly project(path: ":runners:direct-java", configuration: "shadow") testRuntimeOnly library.java.slf4j_jdk14 avroVersions.each { k,v -> diff --git a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroUtils.java b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroUtils.java index 50be0e6b3414..7622132c7e27 100644 --- a/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroUtils.java +++ b/sdks/java/extensions/avro/src/main/java/org/apache/beam/sdk/extensions/avro/schemas/utils/AvroUtils.java @@ -630,7 +630,7 @@ public static SimpleFunction getRowToAvroBytesFunction(Schema beamS } private static class RowToAvroBytesFn extends SimpleFunction { - private final transient org.apache.avro.Schema avroSchema; + private final org.apache.avro.Schema avroSchema; private final AvroCoder coder; RowToAvroBytesFn(Schema beamSchema) { diff --git a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java index fc153d6c97d5..7141869b2283 100644 --- a/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java +++ b/sdks/java/io/google-cloud-platform/src/main/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProto.java @@ -262,8 +262,7 @@ private static TableFieldSchema fieldDescriptorFromAvroField(Schema.Field field) elementType.getType() != Schema.Type.UNION, "Multiple non-null union types are not supported."); TableFieldSchema unionFieldSchema = - fieldDescriptorFromAvroField( - new Schema.Field(field.name(), elementType, field.doc(), field.defaultVal())); + fieldDescriptorFromAvroField(new Schema.Field(field.name(), elementType, field.doc())); builder = builder .setType(unionFieldSchema.getType()) diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java index e32f45a43823..84eb12982856 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/GcpApiSurfaceTest.java @@ -124,7 +124,6 @@ public void testGcpApiSurface() throws Exception { classesInPackage("javax"), classesInPackage("org.apache.avro"), classesInPackage("org.apache.beam"), - classesInPackage("org.codehaus.jackson"), classesInPackage("org.joda.time"), classesInPackage("org.threeten.bp"), classesInPackage("com.google.gson")); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProtoTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProtoTest.java index f2f5cb692628..ac73a0122d8b 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProtoTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/AvroGenericRecordToStorageApiProtoTest.java @@ -301,7 +301,10 @@ enum TestEnum { .set( "decimalValue", new Conversions.DecimalConversion() - .toBytes(bd, Schema.create(Schema.Type.NULL), LogicalTypes.decimal(1, 1))) + .toBytes( + bd, + Schema.create(Schema.Type.NULL), + LogicalTypes.decimal(bd.precision(), bd.scale()))) .set("dateValue", now) .set("timestampMicrosValue", now.getMillis() * 1000) .set("timestampMicrosAsInstantValue", now) diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtilsTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtilsTest.java index 0bdecb31a316..c87888134c8a 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtilsTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/bigquery/BigQueryAvroUtilsTest.java @@ -145,7 +145,7 @@ public void testConvertGenericRecordToTableRow() throws Exception { } // After a Field is added to a Schema, it is assigned a position, so we can't simply reuse // the existing Field. - avroFields.add(new Schema.Field(field.name(), schema, field.doc(), field.defaultValue())); + avroFields.add(new Schema.Field(field.name(), schema, field.doc(), field.defaultVal())); } Schema avroSchema = Schema.createRecord(avroFields); diff --git a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubClientTest.java b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubClientTest.java index 6461af10a220..895ed35bfb12 100644 --- a/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubClientTest.java +++ b/sdks/java/io/google-cloud-platform/src/test/java/org/apache/beam/sdk/io/gcp/pubsub/PubsubClientTest.java @@ -205,7 +205,7 @@ public void fromPubsubSchema() { assertThrows( "null definition should throw an exception", - NullPointerException.class, + SchemaParseException.class, () -> PubsubClient.fromPubsubSchema( com.google.pubsub.v1.Schema.newBuilder().setType(Schema.Type.AVRO).build())); diff --git a/sdks/java/io/kafka/build.gradle b/sdks/java/io/kafka/build.gradle index 16f3a39b2073..269ddb3f5eb2 100644 --- a/sdks/java/io/kafka/build.gradle +++ b/sdks/java/io/kafka/build.gradle @@ -29,7 +29,10 @@ provideIntegrationTestingDependencies() enableJavaPerformanceTesting() description = "Apache Beam :: SDKs :: Java :: IO :: Kafka" -ext.summary = "Library to read Kafka topics." +ext { + summary = "Library to read Kafka topics." + confluentVersion = "7.6.0" +} def kafkaVersions = [ '01103': "0.11.0.3", @@ -68,14 +71,14 @@ dependencies { implementation library.java.jackson_annotations implementation library.java.jackson_databind implementation "org.springframework:spring-expression:5.3.27" - implementation ("io.confluent:kafka-avro-serializer:5.3.2") { + implementation ("io.confluent:kafka-avro-serializer:${confluentVersion}") { // zookeeper depends on "spotbugs-annotations:3.1.9" which clashes with current // "spotbugs-annotations:3.1.12" used in Beam. Not required. exclude group: "org.apache.zookeeper", module: "zookeeper" // "kafka-clients" has to be provided since user can use its own version. exclude group: "org.apache.kafka", module: "kafka-clients" } - implementation ("io.confluent:kafka-schema-registry-client:5.3.2") { + implementation ("io.confluent:kafka-schema-registry-client:${confluentVersion}") { // It depends on "spotbugs-annotations:3.1.9" which clashes with current // "spotbugs-annotations:3.1.12" used in Beam. Not required. exclude group: "org.apache.zookeeper", module: "zookeeper"