From d6f240e8649622f8098725d3ee79609f18c9e248 Mon Sep 17 00:00:00 2001 From: Masahiro Sakamoto Date: Mon, 6 Jan 2020 13:06:05 +0900 Subject: [PATCH] Upgrade Avro to 1.9.1 (#5938) ### Motivation Currently, Pulsar uses Avro 1.8.2, a version released two years ago. The latest version of Avro is 1.9.1, which uses FasterXML's Jackson 2.x instead of Codehaus's Jackson 1.x. Jackson is prone to security issues, so we should not keep using older versions. https://blog.godatadriven.com/apache-avro-1-9-release ### Modifications Avro 1.9 has some major changes: - The library used to handle logical datetime values has changed from Joda-Time to JSR-310 (https://github.com/apache/avro/pull/631) - Namespaces no longer include "$" when generating schemas containing inner classes using ReflectData (https://github.com/apache/avro/pull/283) - Validation of default values has been enabled (https://github.com/apache/avro/pull/288). This results in a validation error when parsing the following schema: ```json { "name": "fieldName", "type": [ "null", "string" ], "default": "defaultValue" } ``` The default value of a nullable field must be null (cf. https://issues.apache.org/jira/browse/AVRO-1803), and the default value of the field as above is actually null. However, this PR disables the validation in order to maintain the traditional behavior. --- .../server/src/assemble/LICENSE.bin.txt | 12 +---- pom.xml | 2 +- .../AvroSchemaBasedCompatibilityCheck.java | 2 + .../schema/JsonSchemaCompatibilityCheck.java | 1 + .../validator/StructSchemaDataValidator.java | 1 + .../BaseAvroSchemaCompatibilityTest.java | 24 +++++----- .../service/schema/SchemaServiceTest.java | 6 +-- .../pulsar/client/impl/MessageParserTest.java | 3 +- pulsar-client-go/pulsar/schemaDef_test.go | 2 +- pulsar-client/pom.xml | 6 --- .../pulsar/client/impl/schema/AvroSchema.java | 45 +++++++------------ .../client/impl/schema/StructSchema.java | 24 +++++++++- .../client/impl/schema/AvroSchemaTest.java | 21 +++++---- .../client/impl/schema/JSONSchemaTest.java | 1 + .../impl/schema/ProtobufSchemaTest.java | 2 +- .../client/impl/schema/SchemaBuilderTest.java | 8 ++-- .../client/impl/schema/SchemaInfoTest.java | 8 ++-- .../client/impl/schema/SchemaTestUtils.java | 14 +++--- pulsar-flink/pom.xml | 6 +++ pulsar-proxy/pom.xml | 5 +++ .../proxy/server/ParserProxyHandler.java | 3 +- pulsar-sql/presto-distribution/LICENSE | 12 ++--- .../sql/presto/PulsarConnectorUtils.java | 1 + site2/docs/io-quickstart.md | 2 +- tests/integration/pom.xml | 6 --- .../tests/integration/schema/SchemaTest.java | 9 ++-- .../tests/integration/schema/Schemas.java | 9 ++-- 27 files changed, 116 insertions(+), 119 deletions(-) diff --git a/distribution/server/src/assemble/LICENSE.bin.txt b/distribution/server/src/assemble/LICENSE.bin.txt index a091bd3fcdc41..9a57a48791066 100644 --- a/distribution/server/src/assemble/LICENSE.bin.txt +++ b/distribution/server/src/assemble/LICENSE.bin.txt @@ -312,8 +312,6 @@ The Apache Software License, Version 2.0 * JCommander -- com.beust-jcommander-1.48.jar * High Performance Primitive Collections for Java -- com.carrotsearch-hppc-0.7.3.jar * Jackson - - org.codehaus.jackson-jackson-core-asl-1.9.13.jar - - org.codehaus.jackson-jackson-mapper-asl-1.9.13.jar - com.fasterxml.jackson.core-jackson-annotations-2.10.1.jar - com.fasterxml.jackson.core-jackson-core-2.10.1.jar - com.fasterxml.jackson.core-jackson-databind-2.10.1.jar @@ -454,13 +452,11 @@ The Apache Software License, Version 2.0 * OpenCensus - io.opencensus-opencensus-api-0.18.0.jar - io.opencensus-opencensus-contrib-grpc-metrics-0.18.0.jar - * Paranamer - - com.thoughtworks.paranamer-paranamer-2.7.jar * Jodah - net.jodah-typetools-0.5.0.jar * Apache Avro - - org.apache.avro-avro-1.8.2.jar - - org.apache.avro-avro-protobuf-1.8.2.jar + - org.apache.avro-avro-1.9.1.jar + - org.apache.avro-avro-protobuf-1.9.1.jar * Apache Curator - org.apache.curator-curator-client-4.0.1.jar - org.apache.curator-curator-framework-4.0.1.jar @@ -568,10 +564,6 @@ Eclipse Public License 1.0 -- licenses/LICENSE-AspectJ.txt - org.aspectj-aspectjrt-1.9.2.jar - org.aspectj-aspectjweaver-1.9.2.jar -Public Domain - * XZ for Java -- licenses/LICENSE-xz.txt - - org.tukaani-xz-1.5.jar - Public Domain (CC0) -- licenses/LICENSE-CC0.txt * Reactive Streams -- org.reactivestreams-reactive-streams-1.0.2.jar diff --git a/pom.xml b/pom.xml index 1c2391baf01f2..094b8097ce927 100644 --- a/pom.xml +++ b/pom.xml @@ -183,7 +183,7 @@ flexible messaging model and an intuitive client API. 2.3.0 5.1.1 1.11.297 - 1.8.2 + 1.9.1 2.10.1 2.1.1 3.8.11.2 diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java index 9b8d4a10d6a06..5249fc617b6e1 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/AvroSchemaBasedCompatibilityCheck.java @@ -52,9 +52,11 @@ public void checkCompatible(Iterable from, SchemaData to, SchemaComp try { for (SchemaData schemaData : from) { Schema.Parser parser = new Schema.Parser(); + parser.setValidateDefaults(false); fromList.addFirst(parser.parse(new String(schemaData.getData(), UTF_8))); } Schema.Parser parser = new Schema.Parser(); + parser.setValidateDefaults(false); Schema toSchema = parser.parse(new String(to.getData(), UTF_8)); SchemaValidator schemaValidator = createSchemaValidator(strategy); schemaValidator.validate(toSchema, fromList); diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheck.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheck.java index 8f4f1398446b2..0e9f839e211d8 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheck.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/JsonSchemaCompatibilityCheck.java @@ -99,6 +99,7 @@ private boolean isAvroSchema(SchemaData schemaData) { try { Schema.Parser fromParser = new Schema.Parser(); + fromParser.setValidateDefaults(false); Schema fromSchema = fromParser.parse(new String(schemaData.getData(), UTF_8)); return true; } catch (SchemaParseException e) { diff --git a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/validator/StructSchemaDataValidator.java b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/validator/StructSchemaDataValidator.java index 1eeed83a00081..2212e50be176f 100644 --- a/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/validator/StructSchemaDataValidator.java +++ b/pulsar-broker/src/main/java/org/apache/pulsar/broker/service/schema/validator/StructSchemaDataValidator.java @@ -49,6 +49,7 @@ public void validate(SchemaData schemaData) throws InvalidSchemaDataException { try { Schema.Parser avroSchemaParser = new Schema.Parser(); + avroSchemaParser.setValidateDefaults(false); avroSchemaParser.parse(new String(data, UTF_8)); } catch (SchemaParseException e) { if (schemaData.getType() == SchemaType.JSON) { diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/BaseAvroSchemaCompatibilityTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/BaseAvroSchemaCompatibilityTest.java index 70881d69449d9..7b1987ad09fe4 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/BaseAvroSchemaCompatibilityTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/BaseAvroSchemaCompatibilityTest.java @@ -33,49 +33,49 @@ public abstract class BaseAvroSchemaCompatibilityTest { private static final String schemaJson1 = "{\"type\":\"record\",\"name\":\"DefaultTest\",\"namespace\":\"org.apache.pulsar.broker.service.schema" + - ".AvroSchemaCompatibilityCheckTest$\",\"fields\":[{\"name\":\"field1\",\"type\":\"string\"}]}"; + ".AvroSchemaCompatibilityCheckTest\",\"fields\":[{\"name\":\"field1\",\"type\":\"string\"}]}"; private static final SchemaData schemaData1 = getSchemaData(schemaJson1); private static final String schemaJson2 = "{\"type\":\"record\",\"name\":\"DefaultTest\",\"namespace\":\"org.apache.pulsar.broker.service.schema" + - ".AvroSchemaCompatibilityCheckTest$\",\"fields\":[{\"name\":\"field1\",\"type\":\"string\"}," + + ".AvroSchemaCompatibilityCheckTest\",\"fields\":[{\"name\":\"field1\",\"type\":\"string\"}," + "{\"name\":\"field2\",\"type\":\"string\",\"default\":\"foo\"}]}"; private static final SchemaData schemaData2 = getSchemaData(schemaJson2); private static final String schemaJson3 = "{\"type\":\"record\",\"name\":\"DefaultTest\",\"namespace\":\"org" + - ".apache.pulsar.broker.service.schema.AvroSchemaCompatibilityCheckTest$\"," + + ".apache.pulsar.broker.service.schema.AvroSchemaCompatibilityCheckTest\"," + "\"fields\":[{\"name\":\"field1\",\"type\":\"string\"},{\"name\":\"field2\",\"type\":\"string\"}]}"; private static final SchemaData schemaData3 = getSchemaData(schemaJson3); private static final String schemaJson4 = "{\"type\":\"record\",\"name\":\"DefaultTest\",\"namespace\":\"org.apache.pulsar.broker.service.schema" + - ".AvroSchemaCompatibilityCheckTest$\",\"fields\":[{\"name\":\"field1_v2\",\"type\":\"string\"," + + ".AvroSchemaCompatibilityCheckTest\",\"fields\":[{\"name\":\"field1_v2\",\"type\":\"string\"," + "\"aliases\":[\"field1\"]}]}"; private static final SchemaData schemaData4 = getSchemaData(schemaJson4); private static final String schemaJson5 = "{\"type\":\"record\",\"name\":\"DefaultTest\",\"namespace\":\"org.apache.pulsar.broker.service.schema" + - ".AvroSchemaCompatibilityCheckTest$\",\"fields\":[{\"name\":\"field1\",\"type\":[\"null\"," + + ".AvroSchemaCompatibilityCheckTest\",\"fields\":[{\"name\":\"field1\",\"type\":[\"null\"," + "\"string\"]}]}"; private static final SchemaData schemaData5 = getSchemaData(schemaJson5); private static final String schemaJson6 = "{\"type\":\"record\",\"name\":\"DefaultTest\",\"namespace\":\"org.apache.pulsar.broker.service.schema" + - ".AvroSchemaCompatibilityCheckTest$\",\"fields\":[{\"name\":\"field1\",\"type\":[\"null\"," + + ".AvroSchemaCompatibilityCheckTest\",\"fields\":[{\"name\":\"field1\",\"type\":[\"null\"," + "\"string\",\"int\"]}]}"; private static final SchemaData schemaData6 = getSchemaData(schemaJson6); private static final String schemaJson7 = "{\"type\":\"record\",\"name\":\"DefaultTest\",\"namespace\":\"org.apache.pulsar.broker.service.schema" + - ".AvroSchemaCompatibilityCheckTest$\",\"fields\":[{\"name\":\"field1\",\"type\":\"string\"}," + + ".AvroSchemaCompatibilityCheckTest\",\"fields\":[{\"name\":\"field1\",\"type\":\"string\"}," + "{\"name\":\"field2\",\"type\":\"string\",\"default\":\"foo\"},{\"name\":\"field3\"," + "\"type\":\"string\",\"default\":\"bar\"}]}"; private static final SchemaData schemaData7 = getSchemaData(schemaJson7); private static final String schemaJson8 = "{\"type\":\"record\",\"name\":\"DefaultTest\",\"namespace\":\"org.apache.pulsar.broker.service.schema" + - ".AvroSchemaCompatibilityCheckTest$\",\"fields\":[{\"name\":\"field1\",\"type\":\"string\"}," + + ".AvroSchemaCompatibilityCheckTest\",\"fields\":[{\"name\":\"field1\",\"type\":\"string\"}," + "{\"name\":\"field2\",\"type\":\"string\"}]}"; private static final SchemaData schemaData8 = getSchemaData(schemaJson8); @@ -96,10 +96,10 @@ public void testBackwardCompatibility() { Assert.assertFalse(schemaCompatibilityCheck.isCompatible(schemaData1, schemaData3, SchemaCompatibilityStrategy.BACKWARD), "adding a field without default is NOT backwards compatible"); - // Modifying a field name is not backwards compatible - Assert.assertFalse(schemaCompatibilityCheck.isCompatible(schemaData1, schemaData4, - SchemaCompatibilityStrategy.BACKWARD), - "Modifying a field name is not backwards compatible"); + // Modifying a field name with an alias is backwards compatible + Assert.assertTrue(schemaCompatibilityCheck.isCompatible(schemaData1, schemaData4, + SchemaCompatibilityStrategy.BACKWARD), + "Modifying a field name with an alias is backwards compatible"); // evolving field to a union is backwards compatible Assert.assertTrue(schemaCompatibilityCheck.isCompatible(schemaData1, schemaData5, SchemaCompatibilityStrategy.BACKWARD), diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java index e18fa839dec7b..ab7e910b51146 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/broker/service/schema/SchemaServiceTest.java @@ -293,18 +293,18 @@ public void dontReAddExistingSchemaInMiddle() throws Exception { public void checkIsCompatible() throws Exception { String schemaJson1 = "{\"type\":\"record\",\"name\":\"DefaultTest\",\"namespace\":\"org.apache.pulsar.broker.service.schema" + - ".AvroSchemaCompatibilityCheckTest$\",\"fields\":[{\"name\":\"field1\",\"type\":\"string\"}]}"; + ".AvroSchemaCompatibilityCheckTest\",\"fields\":[{\"name\":\"field1\",\"type\":\"string\"}]}"; SchemaData schemaData1 = getSchemaData(schemaJson1); String schemaJson2 = "{\"type\":\"record\",\"name\":\"DefaultTest\",\"namespace\":\"org.apache.pulsar.broker.service.schema" + - ".AvroSchemaCompatibilityCheckTest$\",\"fields\":[{\"name\":\"field1\",\"type\":\"string\"}," + + ".AvroSchemaCompatibilityCheckTest\",\"fields\":[{\"name\":\"field1\",\"type\":\"string\"}," + "{\"name\":\"field2\",\"type\":\"string\",\"default\":\"foo\"}]}"; SchemaData schemaData2 = getSchemaData(schemaJson2); String schemaJson3 = "{\"type\":\"record\",\"name\":\"DefaultTest\",\"namespace\":\"org.apache.pulsar.broker.service.schema" + - ".AvroSchemaCompatibilityCheckTest$\",\"fields\":[{\"name\":\"field1\",\"type\":\"string\"}," + + ".AvroSchemaCompatibilityCheckTest\",\"fields\":[{\"name\":\"field1\",\"type\":\"string\"}," + "{\"name\":\"field2\",\"type\":\"string\"}]}"; SchemaData schemaData3 = getSchemaData(schemaJson3); diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageParserTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageParserTest.java index f64a0a925e739..87278a4d0745b 100644 --- a/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageParserTest.java +++ b/pulsar-broker/src/test/java/org/apache/pulsar/client/impl/MessageParserTest.java @@ -20,6 +20,7 @@ import static org.testng.Assert.assertEquals; +import com.google.common.collect.Lists; import com.google.common.collect.Sets; import io.netty.buffer.Unpooled; @@ -44,8 +45,6 @@ import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; -import avro.shaded.com.google.common.collect.Lists; - public class MessageParserTest extends MockedPulsarServiceBaseTest { @BeforeMethod diff --git a/pulsar-client-go/pulsar/schemaDef_test.go b/pulsar-client-go/pulsar/schemaDef_test.go index 5fb68032147a4..8a06080f99ff3 100644 --- a/pulsar-client-go/pulsar/schemaDef_test.go +++ b/pulsar-client-go/pulsar/schemaDef_test.go @@ -51,7 +51,7 @@ func TestSchemaDef(t *testing.T) { _, err = initAvroCodec(errSchemaDef4) assert.NotNil(t, err) - errSchemaDef5 := "{\"type\":\"record\",\"name\":\"Example\",\"namespace\":\"operation.createJsonConsumer$\"," + + errSchemaDef5 := "{\"type\":\"record\",\"name\":\"Example\",\"namespace\":\"operation.createJsonConsumer\"," + "\"fields\":[{\"name\":\"ID\",\"type\":\"byte\"},{\"name\":\"Name\",\"type\":\":[\"null\",\"string\"],\"default\":null\"}]}" _, err = initAvroCodec(errSchemaDef5) assert.NotNil(t, err) diff --git a/pulsar-client/pom.xml b/pulsar-client/pom.xml index f0f63d5306f98..e3eab8b3cb649 100644 --- a/pulsar-client/pom.xml +++ b/pulsar-client/pom.xml @@ -127,12 +127,6 @@ jackson-module-jsonSchema - - joda-time - joda-time - provided - - net.jcip jcip-annotations diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java index 9eddee5a4bd3f..f4d130f5ba03b 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/AvroSchema.java @@ -41,38 +41,27 @@ public class AvroSchema extends StructSchema { private static final Logger LOG = LoggerFactory.getLogger(AvroSchema.class); - // the aim to fix avro's bug -// https://issues.apache.org/jira/browse/AVRO-1891 bug address explain -// fix the avro logical type read and write + // the aim to fix avro's bug + // https://issues.apache.org/jira/browse/AVRO-1891 bug address explain + // fix the avro logical type read and write static { - try { - ReflectData reflectDataAllowNull = ReflectData.AllowNull.get(); + ReflectData reflectDataAllowNull = ReflectData.AllowNull.get(); - reflectDataAllowNull.addLogicalTypeConversion(new Conversions.DecimalConversion()); - reflectDataAllowNull.addLogicalTypeConversion(new TimeConversions.DateConversion()); - reflectDataAllowNull.addLogicalTypeConversion(new TimeConversions.LossyTimeMicrosConversion()); - reflectDataAllowNull.addLogicalTypeConversion(new TimeConversions.LossyTimestampMicrosConversion()); - reflectDataAllowNull.addLogicalTypeConversion(new TimeConversions.TimeMicrosConversion()); - reflectDataAllowNull.addLogicalTypeConversion(new TimeConversions.TimestampMicrosConversion()); - reflectDataAllowNull.addLogicalTypeConversion(new TimeConversions.TimestampConversion()); - reflectDataAllowNull.addLogicalTypeConversion(new TimeConversions.TimeConversion()); + reflectDataAllowNull.addLogicalTypeConversion(new Conversions.DecimalConversion()); + reflectDataAllowNull.addLogicalTypeConversion(new TimeConversions.DateConversion()); + reflectDataAllowNull.addLogicalTypeConversion(new TimeConversions.TimeMillisConversion()); + reflectDataAllowNull.addLogicalTypeConversion(new TimeConversions.TimeMicrosConversion()); + reflectDataAllowNull.addLogicalTypeConversion(new TimeConversions.TimestampMillisConversion()); + reflectDataAllowNull.addLogicalTypeConversion(new TimeConversions.TimestampMicrosConversion()); - ReflectData reflectDataNotAllowNull = ReflectData.get(); + ReflectData reflectDataNotAllowNull = ReflectData.get(); - reflectDataNotAllowNull.addLogicalTypeConversion(new Conversions.DecimalConversion()); - reflectDataNotAllowNull.addLogicalTypeConversion(new TimeConversions.DateConversion()); - reflectDataNotAllowNull.addLogicalTypeConversion(new TimeConversions.TimestampConversion()); - reflectDataNotAllowNull.addLogicalTypeConversion(new TimeConversions.LossyTimeMicrosConversion()); - reflectDataNotAllowNull.addLogicalTypeConversion(new TimeConversions.LossyTimestampMicrosConversion()); - reflectDataNotAllowNull.addLogicalTypeConversion(new TimeConversions.TimeMicrosConversion()); - reflectDataNotAllowNull.addLogicalTypeConversion(new TimeConversions.TimestampMicrosConversion()); - reflectDataNotAllowNull.addLogicalTypeConversion(new TimeConversions.TimeConversion()); - } catch (Throwable t) { - if (LOG.isDebugEnabled()) { - LOG.debug("Avro logical types are not available. If you are going to use avro logical types, " + - "you can include `joda-time` in your dependency."); - } - } + reflectDataNotAllowNull.addLogicalTypeConversion(new Conversions.DecimalConversion()); + reflectDataNotAllowNull.addLogicalTypeConversion(new TimeConversions.DateConversion()); + reflectDataNotAllowNull.addLogicalTypeConversion(new TimeConversions.TimeMillisConversion()); + reflectDataNotAllowNull.addLogicalTypeConversion(new TimeConversions.TimeMicrosConversion()); + reflectDataNotAllowNull.addLogicalTypeConversion(new TimeConversions.TimestampMillisConversion()); + reflectDataNotAllowNull.addLogicalTypeConversion(new TimeConversions.TimestampMicrosConversion()); } private AvroSchema(SchemaInfo schemaInfo) { diff --git a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/StructSchema.java b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/StructSchema.java index ede229cb74e63..1302d7c84985f 100644 --- a/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/StructSchema.java +++ b/pulsar-client/src/main/java/org/apache/pulsar/client/impl/schema/StructSchema.java @@ -20,6 +20,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; +import java.lang.reflect.Field; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; @@ -29,6 +30,7 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufInputStream; import org.apache.avro.AvroTypeException; +import org.apache.avro.Schema; import org.apache.avro.Schema.Parser; import org.apache.avro.reflect.ReflectData; import org.apache.commons.codec.binary.Hex; @@ -133,7 +135,26 @@ protected static org.apache.avro.Schema createAvroSchema(SchemaDefinition schema if (StringUtils.isNotBlank(schemaDefinition.getJsonDef())) { return parseAvroSchema(schemaDefinition.getJsonDef()); } else if (pojo != null) { - return schemaDefinition.getAlwaysAllowNull() ? ReflectData.AllowNull.get().getSchema(pojo) : ReflectData.get().getSchema(pojo); + ThreadLocal validateDefaults = null; + + try { + Field validateDefaultsField = Schema.class.getDeclaredField("VALIDATE_DEFAULTS"); + validateDefaultsField.setAccessible(true); + validateDefaults = (ThreadLocal) validateDefaultsField.get(null); + } catch (NoSuchFieldException | IllegalAccessException e) { + throw new RuntimeException("Cannot disable validation of default values", e); + } + + final boolean savedValidateDefaults = validateDefaults.get(); + + try { + // Disable validation of default values for compatibility + validateDefaults.set(false); + return schemaDefinition.getAlwaysAllowNull() ? ReflectData.AllowNull.get().getSchema(pojo) + : ReflectData.get().getSchema(pojo); + } finally { + validateDefaults.set(savedValidateDefaults); + } } else { throw new RuntimeException("Schema definition must specify pojo class or schema json definition"); } @@ -141,6 +162,7 @@ protected static org.apache.avro.Schema createAvroSchema(SchemaDefinition schema protected static org.apache.avro.Schema parseAvroSchema(String schemaJson) { final Parser parser = new Parser(); + parser.setValidateDefaults(false); return parser.parse(schemaJson); } diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java index 62acef4ea3d5a..caeb8ac8393a6 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/AvroSchemaTest.java @@ -27,8 +27,10 @@ import static org.testng.Assert.fail; import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; import java.util.Arrays; -import java.util.Date; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufAllocator; @@ -51,10 +53,6 @@ import org.apache.pulsar.client.impl.schema.SchemaTestUtils.Foo; import org.apache.pulsar.common.schema.SchemaInfo; import org.apache.pulsar.common.schema.SchemaType; -import org.joda.time.DateTime; -import org.joda.time.LocalDate; -import org.joda.time.LocalTime; -import org.joda.time.chrono.ISOChronology; import org.testng.Assert; import org.testng.annotations.Test; @@ -90,7 +88,7 @@ private static class SchemaLogicalType{ @org.apache.avro.reflect.AvroSchema("{\"type\":\"int\",\"logicalType\":\"date\"}") LocalDate date; @org.apache.avro.reflect.AvroSchema("{\"type\":\"long\",\"logicalType\":\"timestamp-millis\"}") - DateTime timestampMillis; + Instant timestampMillis; @org.apache.avro.reflect.AvroSchema("{\"type\":\"int\",\"logicalType\":\"time-millis\"}") LocalTime timeMillis; @org.apache.avro.reflect.AvroSchema("{\"type\":\"long\",\"logicalType\":\"timestamp-micros\"}") @@ -117,7 +115,7 @@ public void testSchemaDefinition() throws SchemaValidationException { validator.validate( schema1, Arrays.asList( - new Schema.Parser().parse(schemaDef2) + new Schema.Parser().setValidateDefaults(false).parse(schemaDef2) ) ); fail("Should fail on validating incompatible schemas"); @@ -171,6 +169,7 @@ public void testAllowNullSchema() { AvroSchema avroSchema = AvroSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); assertEquals(avroSchema.getSchemaInfo().getType(), SchemaType.AVRO); Schema.Parser parser = new Schema.Parser(); + parser.setValidateDefaults(false); String schemaJson = new String(avroSchema.getSchemaInfo().getSchema()); assertEquals(schemaJson, SCHEMA_AVRO_ALLOW_NULL); Schema schema = parser.parse(schemaJson); @@ -250,7 +249,7 @@ public void testLogicalType() { SchemaLogicalType schemaLogicalType = new SchemaLogicalType(); schemaLogicalType.setTimestampMicros(System.currentTimeMillis()*1000); - schemaLogicalType.setTimestampMillis(new DateTime("2019-03-26T04:39:58.469Z", ISOChronology.getInstanceUTC())); + schemaLogicalType.setTimestampMillis(Instant.parse("2019-03-26T04:39:58.469Z")); schemaLogicalType.setDecimal(new BigDecimal("12.34")); schemaLogicalType.setDate(LocalDate.now()); schemaLogicalType.setTimeMicros(System.currentTimeMillis()*1000); @@ -292,9 +291,9 @@ public void testDateAndTimestamp() { NasaMission nasaMission = NasaMission.newBuilder() .setId(1001) .setName("one") - .setCreateYear(new LocalDate(new Date().getTime())) - .setCreateTime(new LocalTime(new Date().getTime())) - .setCreateTimestamp(new DateTime(new Date().getTime())) + .setCreateYear(LocalDate.now()) + .setCreateTime(LocalTime.now()) + .setCreateTimestamp(Instant.now()) .build(); byte[] bytes = avroSchema.encode(nasaMission); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java index f28011324a84d..c303ce9293329 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/JSONSchemaTest.java @@ -71,6 +71,7 @@ public void testAllowNullSchema() { JSONSchema jsonSchema = JSONSchema.of(SchemaDefinition.builder().withPojo(Foo.class).build()); Assert.assertEquals(jsonSchema.getSchemaInfo().getType(), SchemaType.JSON); Schema.Parser parser = new Schema.Parser(); + parser.setValidateDefaults(false); String schemaJson = new String(jsonSchema.getSchemaInfo().getSchema()); Assert.assertEquals(schemaJson, SCHEMA_JSON_ALLOW_NULL); Schema schema = parser.parse(schemaJson); diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/ProtobufSchemaTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/ProtobufSchemaTest.java index fc03412d72d31..abcfaf2ce16e6 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/ProtobufSchemaTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/ProtobufSchemaTest.java @@ -38,7 +38,7 @@ public class ProtobufSchemaTest { private static final String NAME = "foo"; private static final String EXPECTED_SCHEMA_JSON = "{\"type\":\"record\",\"name\":\"TestMessage\"," + - "\"namespace\":\"org.apache.pulsar.client.schema.proto.Test$\",\"fields\":[{\"name\":\"stringField\"," + + "\"namespace\":\"org.apache.pulsar.client.schema.proto.Test\",\"fields\":[{\"name\":\"stringField\"," + "\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"},\"default\":\"\"}," + "{\"name\":\"doubleField\",\"type\":\"double\",\"default\":0},{\"name\":\"intField\",\"type\":\"int\"," + "\"default\":0},{\"name\":\"testEnum\",\"type\":{\"type\":\"enum\",\"name\":\"TestEnum\"," + diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaBuilderTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaBuilderTest.java index 9ce3d5c2f5d44..72e9eafe6b226 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaBuilderTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaBuilderTest.java @@ -85,7 +85,7 @@ private static class People2 { @Test public void testAllOptionalFieldsSchema() { RecordSchemaBuilder recordSchemaBuilder = - SchemaBuilder.record("org.apache.pulsar.client.impl.schema.SchemaBuilderTest$.AllOptionalFields"); + SchemaBuilder.record("org.apache.pulsar.client.impl.schema.SchemaBuilderTest.AllOptionalFields"); recordSchemaBuilder.field("intField") .type(SchemaType.INT32).optional(); recordSchemaBuilder.field("longField") @@ -118,7 +118,7 @@ public void testAllOptionalFieldsSchema() { @Test public void testAllPrimitiveFieldsSchema() { RecordSchemaBuilder recordSchemaBuilder = - SchemaBuilder.record("org.apache.pulsar.client.impl.schema.SchemaBuilderTest$.AllPrimitiveFields"); + SchemaBuilder.record("org.apache.pulsar.client.impl.schema.SchemaBuilderTest.AllPrimitiveFields"); recordSchemaBuilder.field("intField") .type(SchemaType.INT32); recordSchemaBuilder.field("longField") @@ -149,7 +149,7 @@ public void testAllPrimitiveFieldsSchema() { @Test public void testGenericRecordBuilderByFieldName() { RecordSchemaBuilder recordSchemaBuilder = - SchemaBuilder.record("org.apache.pulsar.client.impl.schema.SchemaBuilderTest$.AllPrimitiveFields"); + SchemaBuilder.record("org.apache.pulsar.client.impl.schema.SchemaBuilderTest.AllPrimitiveFields"); recordSchemaBuilder.field("intField") .type(SchemaType.INT32); recordSchemaBuilder.field("longField") @@ -188,7 +188,7 @@ public void testGenericRecordBuilderByFieldName() { @Test public void testGenericRecordBuilderByIndex() { RecordSchemaBuilder recordSchemaBuilder = - SchemaBuilder.record("org.apache.pulsar.client.impl.schema.SchemaBuilderTest$.AllPrimitiveFields"); + SchemaBuilder.record("org.apache.pulsar.client.impl.schema.SchemaBuilderTest.AllPrimitiveFields"); recordSchemaBuilder.field("intField") .type(SchemaType.INT32); recordSchemaBuilder.field("longField") diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaInfoTest.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaInfoTest.java index bacf22be99e44..f74b3c9075f4d 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaInfoTest.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaInfoTest.java @@ -50,7 +50,7 @@ public class SchemaInfoTest { + " \"schema\": {\n" + " \"type\": \"record\",\n" + " \"name\": \"Bar\",\n" - + " \"namespace\": \"org.apache.pulsar.client.impl.schema.SchemaTestUtils$\",\n" + + " \"namespace\": \"org.apache.pulsar.client.impl.schema.SchemaTestUtils\",\n" + " \"fields\": [\n" + " {\n" + " \"name\": \"field1\",\n" @@ -72,7 +72,7 @@ public class SchemaInfoTest { + " \"schema\": {\n" + " \"type\": \"record\",\n" + " \"name\": \"Foo\",\n" - + " \"namespace\": \"org.apache.pulsar.client.impl.schema.SchemaTestUtils$\",\n" + + " \"namespace\": \"org.apache.pulsar.client.impl.schema.SchemaTestUtils\",\n" + " \"fields\": [\n" + " {\n" + " \"name\": \"field1\",\n" @@ -146,7 +146,7 @@ public class SchemaInfoTest { + " \"schema\": {\n" + " \"type\": \"record\",\n" + " \"name\": \"Foo\",\n" - + " \"namespace\": \"org.apache.pulsar.client.impl.schema.SchemaTestUtils$\",\n" + + " \"namespace\": \"org.apache.pulsar.client.impl.schema.SchemaTestUtils\",\n" + " \"fields\": [\n" + " {\n" + " \"name\": \"field1\",\n" @@ -216,7 +216,7 @@ public class SchemaInfoTest { + " \"schema\": {\n" + " \"type\": \"record\",\n" + " \"name\": \"Bar\",\n" - + " \"namespace\": \"org.apache.pulsar.client.impl.schema.SchemaTestUtils$\",\n" + + " \"namespace\": \"org.apache.pulsar.client.impl.schema.SchemaTestUtils\",\n" + " \"fields\": [\n" + " {\n" + " \"name\": \"field1\",\n" diff --git a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaTestUtils.java b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaTestUtils.java index 9f7ed211cc2a5..478dc10e2bdc1 100644 --- a/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaTestUtils.java +++ b/pulsar-client/src/test/java/org/apache/pulsar/client/impl/schema/SchemaTestUtils.java @@ -104,34 +104,34 @@ public static class DerivedDerivedFoo extends DerivedFoo { } public static final String SCHEMA_AVRO_NOT_ALLOW_NULL - = "{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils$\",\"fields\":[{\"name\":\"field1\",\"type\":[\"null\",\"string\"]," + + = "{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils\",\"fields\":[{\"name\":\"field1\",\"type\":[\"null\",\"string\"]," + "\"default\":null},{\"name\":\"field2\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field3\",\"type\":\"int\"},{\"name\":\"field4\",\"type\":[\"null\",{\"type\":" + "\"record\",\"name\":\"Bar\",\"fields\":[{\"name\":\"field1\",\"type\":\"boolean\"}]}],\"default\":null},{\"name\":\"color\",\"type\":[\"null\",{\"type\":\"enum\",\"name\":\"Color\"," + "\"symbols\":[\"RED\",\"BLUE\"]}],\"default\":null},{\"name\":\"fieldUnableNull\",\"type\":\"string\",\"default\":\"defaultValue\"}]}"; - public static final String SCHEMA_AVRO_ALLOW_NULL = "{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils$\",\"fields\":[{\"name\":\"field1\"," + + public static final String SCHEMA_AVRO_ALLOW_NULL = "{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils\",\"fields\":[{\"name\":\"field1\"," + "\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field2\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field3\",\"type\":\"int\"},{\"name\":\"field4\",\"type\":[\"" + "null\",{\"type\":\"record\",\"name\":\"Bar\",\"fields\":[{\"name\":\"field1\",\"type\":\"boolean\"}]}],\"default\":null},{\"name\":\"color\",\"type\":[\"null\",{\"type\":\"enum\",\"name\":\"Color\"" + ",\"symbols\":[\"RED\",\"BLUE\"]}],\"default\":null},{\"name\":\"fieldUnableNull\",\"type\":[\"null\",\"string\"],\"default\":\"defaultValue\"}]}"; public static final String SCHEMA_JSON_NOT_ALLOW_NULL - = "{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils$\",\"fields\":[{\"name\":\"field1\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\"" + + = "{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils\",\"fields\":[{\"name\":\"field1\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\"" + ":\"field2\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field3\",\"type\":\"int\"},{\"name\":\"field4\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"Bar\",\"fields\":[{\"name\":\"" + "field1\",\"type\":\"boolean\"}]}],\"default\":null},{\"name\":\"color\",\"type\":[\"null\",{\"type\":\"enum\",\"name\":\"Color\",\"symbols\":[\"RED\",\"BLUE\"]}],\"default\":null},{\"name\":\"fieldUnableNull\"," + "\"type\":\"string\",\"default\":\"defaultValue\"}]}"; public static final String SCHEMA_JSON_ALLOW_NULL - = "{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils$\",\"fields\":[{\"name\":\"field1\",\"type\":[\"null\",\"string\"],\"default\":null}," + + = "{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils\",\"fields\":[{\"name\":\"field1\",\"type\":[\"null\",\"string\"],\"default\":null}," + "{\"name\":\"field2\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field3\",\"type\":\"int\"},{\"name\":\"field4\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"Bar\",\"fields\":" + "[{\"name\":\"field1\",\"type\":\"boolean\"}]}],\"default\":null},{\"name\":\"color\",\"type\":[\"null\",{\"type\":\"enum\",\"name\":\"Color\",\"symbols\":[\"RED\",\"BLUE\"]}],\"default\":null},{\"name\":" + "\"fieldUnableNull\",\"type\":[\"null\",\"string\"],\"default\":\"defaultValue\"}]}"; - public static final String KEY_VALUE_SCHEMA_INFO_INCLUDE_PRIMITIVE = "{\"key\":{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils$\",\"fields\":[{\"name\":\"" + + public static final String KEY_VALUE_SCHEMA_INFO_INCLUDE_PRIMITIVE = "{\"key\":{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils\",\"fields\":[{\"name\":\"" + "field1\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field2\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field3\",\"type\":\"int\"},{\"name\":\"field4\",\"type\":[\"null\"," + "{\"type\":\"record\",\"name\":\"Bar\",\"fields\":[{\"name\":\"field1\",\"type\":\"boolean\"}]}],\"default\":null},{\"name\":\"color\",\"type\":[\"null\",{\"type\":\"enum\",\"name\":\"Color\",\"symbols\":[\"RED\"" + ",\"BLUE\"]}],\"default\":null},{\"name\":\"fieldUnableNull\",\"type\":[\"null\",\"string\"],\"default\":\"defaultValue\"}]},\"value\":\"\"}"; - public static final String KEY_VALUE_SCHEMA_INFO_NOT_INCLUDE_PRIMITIVE = "{\"key\":{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils$\",\"fields\":[{\"name\":\"field1\"" + + public static final String KEY_VALUE_SCHEMA_INFO_NOT_INCLUDE_PRIMITIVE = "{\"key\":{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils\",\"fields\":[{\"name\":\"field1\"" + ",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field2\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field3\",\"type\":\"int\"},{\"name\":\"field4\",\"type\":[\"null\",{\"type\":\"record\"" + ",\"name\":\"Bar\",\"fields\":[{\"name\":\"field1\",\"type\":\"boolean\"}]}],\"default\":null},{\"name\":\"color\",\"type\":[\"null\",{\"type\":\"enum\",\"name\":\"Color\",\"symbols\":[\"RED\",\"BLUE\"]}],\"default\":null}," + - "{\"name\":\"fieldUnableNull\",\"type\":[\"null\",\"string\"],\"default\":\"defaultValue\"}]},\"value\":{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils$\",\"fields\":" + + "{\"name\":\"fieldUnableNull\",\"type\":[\"null\",\"string\"],\"default\":\"defaultValue\"}]},\"value\":{\"type\":\"record\",\"name\":\"Foo\",\"namespace\":\"org.apache.pulsar.client.impl.schema.SchemaTestUtils\",\"fields\":" + "[{\"name\":\"field1\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field2\",\"type\":[\"null\",\"string\"],\"default\":null},{\"name\":\"field3\",\"type\":\"int\"},{\"name\":\"field4\",\"type\":[\"null\"," + "{\"type\":\"record\",\"name\":\"Bar\",\"fields\":[{\"name\":\"field1\",\"type\":\"boolean\"}]}],\"default\":null},{\"name\":\"color\",\"type\":[\"null\",{\"type\":\"enum\",\"name\":\"Color\",\"symbols\":[\"RED\",\"BLUE\"]}]," + "\"default\":null},{\"name\":\"fieldUnableNull\",\"type\":[\"null\",\"string\"],\"default\":\"defaultValue\"}]}}"; diff --git a/pulsar-flink/pom.xml b/pulsar-flink/pom.xml index b44338fee6468..8280ca6280e52 100644 --- a/pulsar-flink/pom.xml +++ b/pulsar-flink/pom.xml @@ -61,6 +61,12 @@ ${flink.version} + + org.apache.avro + avro + ${avro.version} + + ${project.groupId} pulsar-client diff --git a/pulsar-proxy/pom.xml b/pulsar-proxy/pom.xml index e7788fa9832c1..3320749649ed6 100644 --- a/pulsar-proxy/pom.xml +++ b/pulsar-proxy/pom.xml @@ -89,6 +89,11 @@ jersey-media-json-jackson + + com.google.guava + guava + + com.fasterxml.jackson.jaxrs jackson-jaxrs-json-provider diff --git a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ParserProxyHandler.java b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ParserProxyHandler.java index 8b4fe64545347..cd04b8e62012e 100644 --- a/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ParserProxyHandler.java +++ b/pulsar-proxy/src/main/java/org/apache/pulsar/proxy/server/ParserProxyHandler.java @@ -19,8 +19,7 @@ package org.apache.pulsar.proxy.server; - -import avro.shaded.com.google.common.collect.Lists; +import com.google.common.collect.Lists; import io.netty.buffer.ByteBuf; import io.netty.buffer.ByteBufUtil; import io.netty.buffer.Unpooled; diff --git a/pulsar-sql/presto-distribution/LICENSE b/pulsar-sql/presto-distribution/LICENSE index 601dba5b50f92..d130836de8d0f 100644 --- a/pulsar-sql/presto-distribution/LICENSE +++ b/pulsar-sql/presto-distribution/LICENSE @@ -209,6 +209,7 @@ The Apache Software License, Version 2.0 * Jackson - jackson-annotations-2.8.11.jar + - jackson-core-2.8.11.jar - jackson-databind-2.8.11.4.jar - jackson-dataformat-smile-2.8.11.jar - jackson-datatype-guava-2.8.11.jar @@ -217,9 +218,6 @@ The Apache Software License, Version 2.0 - jackson-datatype-jdk8-2.8.11.jar - jackson-datatype-joda-2.8.11.jar - jackson-datatype-jsr310-2.8.11.jar - - jackson-core-2.8.11.jar - - jackson-core-asl-1.9.13.jar - - jackson-mapper-asl-1.9.13.jar - jackson-dataformat-yaml-2.8.11.jar * Guava - guava-25.1-jre.jar @@ -355,8 +353,8 @@ The Apache Software License, Version 2.0 * Apache XBean :: Reflect - xbean-reflect-3.4.jar * Avro - - avro-1.8.2.jar - - avro-protobuf-1.8.2.jar + - avro-1.9.1.jar + - avro-protobuf-1.9.1.jar * Javax - javax.inject-1.jar - javax.inject-1.jar @@ -475,8 +473,6 @@ BSD License - asm-util-6.0.jar * JLine - jline-2.14.6.jar - * ParaNamer Core - - paranamer-2.7.jar * Zstd-jni - zstd-jni-1.3.7-3.jar @@ -545,8 +541,6 @@ Public Domain (CC0) -- licenses/LICENSE-CC0.txt - HdrHistogram-2.1.9.jar * AOP Alliance - aopalliance-1.0.jar - * XZ For Java - - xz-1.5.jar * Reactive Streams - reactive-streams-1.0.2.jar diff --git a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorUtils.java b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorUtils.java index 58794c089b570..03a3a58b080a7 100644 --- a/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorUtils.java +++ b/pulsar-sql/presto-pulsar/src/main/java/org/apache/pulsar/sql/presto/PulsarConnectorUtils.java @@ -34,6 +34,7 @@ public class PulsarConnectorUtils { public static Schema parseSchema(String schemaJson) { Schema.Parser parser = new Schema.Parser(); + parser.setValidateDefaults(false); return parser.parse(schemaJson); } diff --git a/site2/docs/io-quickstart.md b/site2/docs/io-quickstart.md index eb2787e956aec..6fe87b03ed05d 100644 --- a/site2/docs/io-quickstart.md +++ b/site2/docs/io-quickstart.md @@ -504,7 +504,7 @@ In this section, you need to configure a JDBC sink connector. > #### Tip > - > For more information about AVRO, see [Apache Avro](https://avro.apache.org/docs/1.8.2/). + > For more information about AVRO, see [Apache Avro](https://avro.apache.org/docs/1.9.1/). 3. Upload a schema to a topic. diff --git a/tests/integration/pom.xml b/tests/integration/pom.xml index 717eceafb15fa..43bcfa4c805af 100644 --- a/tests/integration/pom.xml +++ b/tests/integration/pom.xml @@ -131,12 +131,6 @@ elasticsearch-rest-high-level-client - - joda-time - joda-time - test - - com.facebook.presto presto-jdbc diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaTest.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaTest.java index 89e9291af0965..70a469290d848 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaTest.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/SchemaTest.java @@ -35,14 +35,13 @@ import org.apache.pulsar.tests.integration.schema.Schemas.Student; import org.apache.pulsar.tests.integration.schema.Schemas.AvroLogicalType; import org.apache.pulsar.tests.integration.suites.PulsarTestSuite; -import org.joda.time.DateTime; -import org.joda.time.LocalDate; -import org.joda.time.LocalTime; -import org.joda.time.chrono.ISOChronology; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; import java.util.ArrayList; import java.util.List; @@ -187,7 +186,7 @@ public void testAvroLogicalType() throws Exception { AvroLogicalType messageForSend = AvroLogicalType.builder() .decimal(new BigDecimal("12.34")) .timestampMicros(System.currentTimeMillis() * 1000) - .timestampMillis(new DateTime("2019-03-26T04:39:58.469Z", ISOChronology.getInstanceUTC())) + .timestampMillis(Instant.parse("2019-03-26T04:39:58.469Z")) .timeMillis(LocalTime.now()) .timeMicros(System.currentTimeMillis() * 1000) .date(LocalDate.now()) diff --git a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/Schemas.java b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/Schemas.java index 8158e107c6b33..9841bcfc9b6bb 100644 --- a/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/Schemas.java +++ b/tests/integration/src/test/java/org/apache/pulsar/tests/integration/schema/Schemas.java @@ -42,11 +42,10 @@ import org.apache.avro.reflect.AvroDefault; -import org.joda.time.DateTime; -import org.joda.time.LocalDate; -import org.joda.time.LocalTime; - import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalTime; /** * Keep a list of schemas for testing. @@ -121,7 +120,7 @@ public static class AvroLogicalType{ @org.apache.avro.reflect.AvroSchema("{\"type\":\"int\",\"logicalType\":\"date\"}") LocalDate date; @org.apache.avro.reflect.AvroSchema("{\"type\":\"long\",\"logicalType\":\"timestamp-millis\"}") - DateTime timestampMillis; + Instant timestampMillis; @org.apache.avro.reflect.AvroSchema("{\"type\":\"int\",\"logicalType\":\"time-millis\"}") LocalTime timeMillis; @org.apache.avro.reflect.AvroSchema("{\"type\":\"long\",\"logicalType\":\"timestamp-micros\"}")