From 5ea76f6eaca6b607cede850612ef50307553fbf4 Mon Sep 17 00:00:00 2001 From: Claire McGinty Date: Tue, 19 Aug 2025 12:45:11 -0400 Subject: [PATCH 1/3] Generate clearer exception for Avro schema translation failures --- .../spotify/dbeam/avro/JdbcAvroSchema.java | 72 +++++++++++-------- 1 file changed, 41 insertions(+), 31 deletions(-) diff --git a/dbeam-core/src/main/java/com/spotify/dbeam/avro/JdbcAvroSchema.java b/dbeam-core/src/main/java/com/spotify/dbeam/avro/JdbcAvroSchema.java index 9d2eefee..4a8a7db1 100644 --- a/dbeam-core/src/main/java/com/spotify/dbeam/avro/JdbcAvroSchema.java +++ b/dbeam-core/src/main/java/com/spotify/dbeam/avro/JdbcAvroSchema.java @@ -163,42 +163,52 @@ private static SchemaBuilder.FieldAssembler createAvroFields( final String typeName = JDBCType.valueOf(columnType).getName(); final String columnClassName = meta.getColumnClassName(i); final String columnTypeName = meta.getColumnTypeName(i); - SchemaBuilder.FieldBuilder field = - builder - .name(normalizeForAvro(columnName)) - .doc(String.format("From sqlType %d %s (%s)", columnType, typeName, columnClassName)) - .prop("columnName", columnName) - .prop("sqlCode", String.valueOf(columnType)) - .prop("typeName", typeName) - .prop("columnClassName", columnClassName); - if (columnTypeName != null) { - field = field.prop("columnTypeName", columnTypeName); - } + try { + SchemaBuilder.FieldBuilder field = + builder + .name(normalizeForAvro(columnName)) + .doc(String.format( + "From sqlType %d %s (%s)", columnType, typeName, columnClassName)) + .prop("columnName", columnName) + .prop("sqlCode", String.valueOf(columnType)) + .prop("typeName", typeName) + .prop("columnClassName", columnClassName); - final SchemaBuilder.BaseTypeBuilder< - SchemaBuilder.UnionAccumulator>> - fieldSchemaBuilder = field.type().unionOf().nullBuilder().endNull().and(); + if (columnTypeName != null) { + field = field.prop("columnTypeName", columnTypeName); + } - Array arrayInstance = - resultSet.isFirst() && columnType == ARRAY - && arrayMode.equals(ArrayHandlingMode.TypedMetaFromFirstRow) - ? resultSet.getArray(i) : null; + final SchemaBuilder.BaseTypeBuilder< + SchemaBuilder.UnionAccumulator>> + fieldSchemaBuilder = field.type().unionOf().nullBuilder().endNull().and(); - final SchemaBuilder.UnionAccumulator> schemaFieldAssembler = - buildAvroFieldType( - columnName, - columnType, - arrayInstance, - meta.getPrecision(i), - columnClassName, - columnTypeName, - useLogicalTypes, - arrayMode, - nullableArrayItems, - fieldSchemaBuilder); + Array arrayInstance = + resultSet.isFirst() && columnType == ARRAY + && arrayMode.equals(ArrayHandlingMode.TypedMetaFromFirstRow) + ? resultSet.getArray(i) : null; + + final SchemaBuilder.UnionAccumulator< + SchemaBuilder.NullDefault> schemaFieldAssembler = + buildAvroFieldType( + columnName, + columnType, + arrayInstance, + meta.getPrecision(i), + columnClassName, + columnTypeName, + useLogicalTypes, + arrayMode, + nullableArrayItems, + fieldSchemaBuilder); - schemaFieldAssembler.endUnion().nullDefault(); + schemaFieldAssembler.endUnion().nullDefault(); + } catch (Exception e) { + throw new RuntimeException(String.format( + "Failed to build Avro schema for sqlType %d %s [%s, %s]", + columnType, typeName, columnClassName, columnTypeName + ), e); + } } return builder; } From 7193e6658642472ed434388c5b8f8ebf6c6cca87 Mon Sep 17 00:00:00 2001 From: Claire McGinty Date: Tue, 19 Aug 2025 16:50:04 -0400 Subject: [PATCH 2/3] fix test --- .../java/com/spotify/dbeam/avro/JdbcAvroSchemaTest.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dbeam-core/src/test/java/com/spotify/dbeam/avro/JdbcAvroSchemaTest.java b/dbeam-core/src/test/java/com/spotify/dbeam/avro/JdbcAvroSchemaTest.java index f981c16f..e2592430 100644 --- a/dbeam-core/src/test/java/com/spotify/dbeam/avro/JdbcAvroSchemaTest.java +++ b/dbeam-core/src/test/java/com/spotify/dbeam/avro/JdbcAvroSchemaTest.java @@ -127,22 +127,22 @@ public void shouldThrowOnNonSupportedTypes() throws SQLException { final ResultSet resultSet = buildMockResultSet(Types.STRUCT); RuntimeException thrown = Assert.assertThrows(RuntimeException.class, () -> createAvroSchemaForSingleField(resultSet, false)); - Assert.assertEquals("STRUCT type is not supported", thrown.getMessage()); + Assert.assertEquals("STRUCT type is not supported", thrown.getCause().getMessage()); final ResultSet resultSet2 = buildMockResultSet(Types.REF); RuntimeException thrown2 = Assert.assertThrows(RuntimeException.class, () -> createAvroSchemaForSingleField(resultSet2, false)); - Assert.assertEquals("REF and REF_CURSOR type are not supported", thrown2.getMessage()); + Assert.assertEquals("REF and REF_CURSOR type are not supported", thrown2.getCause().getMessage()); final ResultSet resultSet3 = buildMockResultSet(Types.REF_CURSOR); RuntimeException thrown3 = Assert.assertThrows(RuntimeException.class, () -> createAvroSchemaForSingleField(resultSet3, false)); - Assert.assertEquals("REF and REF_CURSOR type are not supported", thrown3.getMessage()); + Assert.assertEquals("REF and REF_CURSOR type are not supported", thrown3.getCause().getMessage()); final ResultSet resultSet4 = buildMockResultSet(Types.DATALINK); RuntimeException thrown4 = Assert.assertThrows(RuntimeException.class, () -> createAvroSchemaForSingleField(resultSet4, false)); - Assert.assertEquals("DATALINK type is not supported", thrown4.getMessage()); + Assert.assertEquals("DATALINK type is not supported", thrown4.getCause().getMessage()); } @Test From b2fc9c1ca4c2495db328eaf0974ab23ae07ebbe3 Mon Sep 17 00:00:00 2001 From: Claire McGinty Date: Tue, 19 Aug 2025 16:59:02 -0400 Subject: [PATCH 3/3] fix test/checkstyle --- .../java/com/spotify/dbeam/avro/JdbcAvroSchemaTest.java | 6 ++++-- .../java/com/spotify/dbeam/avro/PostgresJdbcAvroTest.java | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/dbeam-core/src/test/java/com/spotify/dbeam/avro/JdbcAvroSchemaTest.java b/dbeam-core/src/test/java/com/spotify/dbeam/avro/JdbcAvroSchemaTest.java index e2592430..26906354 100644 --- a/dbeam-core/src/test/java/com/spotify/dbeam/avro/JdbcAvroSchemaTest.java +++ b/dbeam-core/src/test/java/com/spotify/dbeam/avro/JdbcAvroSchemaTest.java @@ -132,12 +132,14 @@ public void shouldThrowOnNonSupportedTypes() throws SQLException { final ResultSet resultSet2 = buildMockResultSet(Types.REF); RuntimeException thrown2 = Assert.assertThrows(RuntimeException.class, () -> createAvroSchemaForSingleField(resultSet2, false)); - Assert.assertEquals("REF and REF_CURSOR type are not supported", thrown2.getCause().getMessage()); + Assert.assertEquals( + "REF and REF_CURSOR type are not supported", thrown2.getCause().getMessage()); final ResultSet resultSet3 = buildMockResultSet(Types.REF_CURSOR); RuntimeException thrown3 = Assert.assertThrows(RuntimeException.class, () -> createAvroSchemaForSingleField(resultSet3, false)); - Assert.assertEquals("REF and REF_CURSOR type are not supported", thrown3.getCause().getMessage()); + Assert.assertEquals( + "REF and REF_CURSOR type are not supported", thrown3.getCause().getMessage()); final ResultSet resultSet4 = buildMockResultSet(Types.DATALINK); RuntimeException thrown4 = Assert.assertThrows(RuntimeException.class, diff --git a/dbeam-core/src/test/java/com/spotify/dbeam/avro/PostgresJdbcAvroTest.java b/dbeam-core/src/test/java/com/spotify/dbeam/avro/PostgresJdbcAvroTest.java index 5f1d8239..47c73b19 100644 --- a/dbeam-core/src/test/java/com/spotify/dbeam/avro/PostgresJdbcAvroTest.java +++ b/dbeam-core/src/test/java/com/spotify/dbeam/avro/PostgresJdbcAvroTest.java @@ -344,7 +344,7 @@ public void shouldThrowOnInvalidArrayColumnTypeName() throws SQLException { () -> JdbcAvroSchema.createAvroSchema(resultSet, "ns", "conn_url", Optional.empty(), "doc", true, arrayMode, nullableArrayItems)); Assert.assertEquals("columnName=array_field_text columnTypeName=text should start with '_'", - thrown.getMessage()); + thrown.getCause().getMessage()); } @Test @@ -366,6 +366,6 @@ public void shouldThrowOnNotSupportedArrayColumnTypeName() throws SQLException { Optional.empty(), "doc", true, arrayMode, nullableArrayItems)); Assert.assertEquals( "columnName=array_field_text Postgres type 'not_supported' is not supported", - thrown.getMessage()); + thrown.getCause().getMessage()); } }