Bug/26 charset conversion (#33)

* updated version, pk fix, artifact references and added documentation
* #26: Enabled to use MySQL database with character set `latin1` and characters not strictly ASCII.
* updated to latest PR of VSCJDBC
* removed repository maven.exasol.com
* upgraded to exasol-testcontainers 6.4.0
* excluded vulnerablity
Co-authored-by: Christoph Pirkl <[email protected]>

Author: ckunki

doc/changes/changelog.md

@@ -0,0 +1,32 @@
+# Virtual Schema for MySQL 4.1.0, released 2022-12-05
+
+Code name: Configurable datatype detection
+
+## Summary
+
+Virtual-schema-common-jdbc version 10.0.0 introduced enhanced detection for data types of result sets.
+
+Unfortunately with the new algorithm compatibility problems with the source database can happen under the following circumstances:
+
+* data type `CHAR` or `VARCHAR`
+* 8-bit character sets with encodings like `latin1` or `ISO-8859-1`
+* characters being not strictly ASCII, e.g. German umlaut "Ü"
+
+The current release therefore uses an updated version of `virtual-schema-common-jdbc` with an additional adapter property to configure the data type detection.
+
+For details please see [adapter Properties for JDBC-Based Virtual Schemas](https://github.com/exasol/virtual-schema-common-jdbc/blob/main/README.md#adapter-properties-for-jdbc-based-virtual-schemas).
+
+## Bugfixes
+
+* #26: Enabled to use MySQL database with character set `latin1` and characters not strictly ASCII.
+
+## Dependency Updates
+
+### Compile Dependency Updates
+
+* Updated `com.exasol:virtual-schema-common-jdbc:10.0.1` to `10.1.0`
+
+### Test Dependency Updates
+
+* Updated `com.exasol:exasol-testcontainers:6.3.1` to `6.4.0`
+* Updated `com.exasol:virtual-schema-common-jdbc:10.0.1` to `10.1.0`

doc/changes/changes_4.1.0.md

@@ -28,7 +28,7 @@ You need to specify the following settings when adding the JDBC driver via EXAOp
 | Port (optional)  | default 3306                                                                              |
 
 IMPORTANT: Currently you have to **Disable Security Manager** for the driver if you want to connect to MySQL using Virtual Schemas.
-It is necessary because JDBC driver requires a JAVA permission which we do not grant by default.  
+It is necessary because JDBC driver requires a JAVA permission which we do not grant by default.
 
 ## Uploading the JDBC Driver to BucketFS
 
@@ -50,10 +50,11 @@ CREATE SCHEMA SCHEMA_FOR_VS_SCRIPT;
 The SQL statement below creates the adapter script, defines the Java class that serves as entry point and tells the UDF framework where to find the libraries (JAR files) for Virtual Schema and JDBC database driver.
 
 ```sql
+--/
 CREATE OR REPLACE JAVA ADAPTER SCRIPT SCHEMA_FOR_VS_SCRIPT.ADAPTER_SCRIPT_MYSQL AS
     %scriptclass com.exasol.adapter.RequestDispatcher;
-    %jar /buckets/<BFS service>/<bucket>/virtual-schema-dist-10.0.1-mysql-4.0.1.jar;
-    %jar /buckets/<BFS service>/<bucket>/mysql-connector-j-<version>.jar;
+    %jar /buckets/<BFS service>/<bucket>/virtual-schema-dist-10.1.0-mysql-4.1.0.jar;
+    %jar /buckets/<BFS service>/<bucket>/mysql-connector-java-<version>.jar;
 /
 ;
 ```
@@ -81,6 +82,8 @@ CREATE VIRTUAL SCHEMA <virtual schema name>
     CATALOG_NAME = '<database name>';
 ```
 
+See also [Adapter Properties for JDBC-Based Virtual Schemas](https://github.com/exasol/virtual-schema-common-jdbc#adapter-properties-for-jdbc-based-virtual-schemas).
+
 ## Data Types Conversion
 
 | MySQL Data Type | Supported | Converted Exasol Data Type| Known limitations                                          |
@@ -115,8 +118,8 @@ CREATE VIRTUAL SCHEMA <virtual schema name>
 | VARCHAR         |  ✓        | VARCHAR                   |                                                            |
 | YEAR            |  ✓        | DATE                      |                                                            |
 
-* The tested versions of MySQL Connector JDBC Driver return the column's size depending on the charset and its collation. 
-As the real data in a MySQL table can sometimes exceed the size that we get from the JDBC driver, we set the size for all TEXT columns to 65535 characters.  
+* The tested versions of MySQL Connector JDBC Driver return the column's size depending on the charset and its collation.
+As the real data in a MySQL table can sometimes exceed the size that we get from the JDBC driver, we set the size for all TEXT columns to 65535 characters.
 
 If you need to use currently unsupported data types or find a way around known limitations, please, create a github issue in the [VS repository](https://github.com/exasol/virtual-schemas/issues).
 

doc/user_guide/mysql_user_guide.md

@@ -2,23 +2,14 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <artifactId>mysql-virtual-schema</artifactId>
-    <version>4.0.1</version>
+    <version>4.1.0</version>
     <name>Virtual Schema for MySQL</name>
     <description>Virtual Schema for MySQL</description>
     <url>https://github.com/exasol/mysql-virtual-schema/</url>
     <properties>
-        <vscjdbc.version>10.0.1</vscjdbc.version>
+        <vscjdbc.version>10.1.0</vscjdbc.version>
         <org.testcontainers.version>1.17.6</org.testcontainers.version>
     </properties>
-    <repositories>
-        <repository>
-            <id>maven.exasol.com</id>
-            <url>https://maven.exasol.com/artifactory/exasol-releases</url>
-            <snapshots>
-                <enabled>false</enabled>
-            </snapshots>
-        </repository>
-    </repositories>
     <dependencies>
         <dependency>
             <groupId>com.exasol</groupId>
@@ -55,7 +46,7 @@
         <dependency>
             <groupId>com.exasol</groupId>
             <artifactId>exasol-testcontainers</artifactId>
-            <version>6.3.1</version>
+            <version>6.4.0</version>
             <scope>test</scope>
         </dependency>
         <dependency>
@@ -155,7 +146,13 @@
                     <excludeVulnerabilityIds>
                         <!-- False positive in snakeyaml. According to https://bitbucket.org/snakeyaml/snakeyaml/issues/531/stackoverflow-oss-fuzz-47081
                              this is already fixed in 1.32. -->
-                        <exclude>CVE-2022-38752</exclude>
+                        <exclude>xCVE-2022-38752</exclude>
+                        <!-- Exclude vulnerabilities found in transitive dependency
+                             to org.yaml:snakeyaml:jar:1.33
+                             required by virtual-schema-shared-integration-tests
+                             as there is no update available currently -->
+                        <exclude>CVE-2022-1471</exclude>
+                        <exclude>CVE-2022-40150</exclude>
                     </excludeVulnerabilityIds>
                 </configuration>
             </plugin>
@@ -171,7 +168,7 @@
     <parent>
         <artifactId>mysql-virtual-schema-generated-parent</artifactId>
         <groupId>com.exasol</groupId>
-        <version>4.0.1</version>
+        <version>4.1.0</version>
         <relativePath>pk_generated_parent.pom</relativePath>
     </parent>
 </project>

pk_generated_parent.pom

@@ -3,7 +3,7 @@
 import java.nio.file.Path;
 
 public final class IntegrationTestConstants {
-    public static final String VIRTUAL_SCHEMAS_JAR_NAME_AND_VERSION = "virtual-schema-dist-10.0.1-mysql-4.0.1.jar";
+    public static final String VIRTUAL_SCHEMAS_JAR_NAME_AND_VERSION = "virtual-schema-dist-10.1.0-mysql-4.1.0.jar";
     public static final String EXASOL_DOCKER_IMAGE_REFERENCE = "7.1.14";
     public static final String MYSQL_DOCKER_IMAGE_REFERENCE = "mysql:8.0.30";
     public static final Path PATH_TO_VIRTUAL_SCHEMAS_JAR = Path.of("target", VIRTUAL_SCHEMAS_JAR_NAME_AND_VERSION);

pom.xml

@@ -4,21 +4,26 @@
 import static com.exasol.matcher.ResultSetMatcher.matchesResultSet;
 import static com.exasol.matcher.ResultSetStructureMatcher.table;
 import static org.hamcrest.MatcherAssert.assertThat;
-import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
-import static org.junit.jupiter.api.Assertions.fail;
+import static org.hamcrest.Matchers.matchesRegex;
+import static org.junit.jupiter.api.Assertions.*;
 
 import java.io.IOException;
 import java.sql.*;
-import java.util.Collections;
-import java.util.List;
+import java.util.*;
 import java.util.stream.Collectors;
 
 import org.hamcrest.Matcher;
+import org.junit.Assume;
 import org.junit.jupiter.api.*;
 import org.testcontainers.junit.jupiter.Testcontainers;
 
+import com.exasol.adapter.dialects.DataTypeDetection;
+import com.exasol.adapter.dialects.mysql.charset.ColumnInspector;
+import com.exasol.adapter.dialects.mysql.charset.Version;
+import com.exasol.containers.ExasolDockerImageReference;
 import com.exasol.dbbuilder.dialects.*;
 import com.exasol.dbbuilder.dialects.exasol.VirtualSchema;
+import com.exasol.dbbuilder.dialects.mysql.MySQLIdentifier;
 import com.exasol.dbbuilder.dialects.mysql.MySqlSchema;
 import com.exasol.matcher.TypeMatchMode;
 
@@ -90,13 +95,11 @@ private ResultSet getExpectedResultSet(final List<String> expectedColumns, final
                 + String.join(", ", expectedColumns) + ")");
         statement.execute("INSERT INTO " + qualifiedExpectedTableName + " VALUES" + expectedValues);
         return statement.executeQuery("SELECT * FROM " + qualifiedExpectedTableName);
-
     }
 
     private ResultSet getActualResultSet(final String query) throws SQLException {
         final Statement statement = SETUP.getExasolStatement();
         return statement.executeQuery(query);
-
     }
 
     private static void createMySqlSimpleTable(final Schema mySqlSchema) {
@@ -145,6 +148,70 @@ private static void createMySqlStringTable(final Schema mySqlSchema) {
         table.insert(null, null, "aaaaa", "a", "blob", "text", null, null, null, null, null, null, null, null);
     }
 
+    @Test
+    void importDataTypesFromResultSet() throws SQLException {
+        Assume.assumeTrue(runCharsetTest());
+        final String query = setupCharacterSet(DataTypeDetection.Strategy.FROM_RESULT_SET);
+        final ResultSet actual = getActualResultSet(query);
+        final ResultSet expected = getExpectedResultSet(List.of("c1 CHAR(1) UTF8", "c2 CHAR(1) UTF8"), //
+                List.of(SPECIAL_CHAR_QUOTED + ", " + SPECIAL_CHAR_QUOTED));
+        assertThat(actual, matchesResultSet(expected));
+    }
+
+    @Test
+    void importDataTypesExasolCalculated() throws SQLException {
+        Assume.assumeTrue(runCharsetTest());
+        final String query = setupCharacterSet(DataTypeDetection.Strategy.EXASOL_CALCULATED);
+        final Exception exception = assertThrows(SQLException.class, () -> getActualResultSet(query));
+        assertThat(exception.getMessage(),
+                matchesRegex("ETL-3009: .*Charset conversion from 'UTF-8' to 'ASCII' failed.*"));
+    }
+
+    private boolean runCharsetTest() {
+        final ExasolDockerImageReference dockerImage = SETUP.getExasolContainer().getDockerImageReference();
+        if (!dockerImage.hasMajor() || !dockerImage.hasMinor() || !dockerImage.hasFix()) {
+            return false;
+        }
+        final Version version = Version.of(dockerImage.getMajor(), dockerImage.getMinor(), dockerImage.getFixVersion());
+        if ((dockerImage.getMajor() == 7) && version.isGreaterOrEqualThan(Version.parse("7.1.14"))) {
+            return true;
+        }
+        if ((dockerImage.getMajor() == 8) && version.isGreaterOrEqualThan(Version.parse("8.6.0"))) {
+            return true;
+        }
+        return false;
+    }
+
+    private String setupCharacterSet(final DataTypeDetection.Strategy strategy) throws SQLException {
+        final String tableName = MYSQL_SOURCE_TABLE;
+        createMySqlTableWithCharacterSet(MYSQL_SOURCE_SCHEMA, tableName, "latin1");
+        this.virtualSchema = SETUP.createVirtualSchema( //
+                Map.of(DataTypeDetection.STRATEGY_PROPERTY, strategy.name()), //
+                MYSQL_SOURCE_SCHEMA);
+        final ColumnInspector inspector = SETUP.getColumnInspector(MYSQL_SOURCE_SCHEMA);
+        inspector.describeFromMetadata(MYSQL_SOURCE_SCHEMA, MYSQL_SOURCE_TABLE);
+        final String query = String.format("select * from %s.%s", MYSQL_SOURCE_SCHEMA, MYSQL_SOURCE_TABLE);
+        inspector.describeFromQuery(MYSQL_SOURCE_SCHEMA, query);
+        return "SELECT * FROM " + this.virtualSchema.getName() + "." + tableName;
+    }
+
+    private static final char[] GERMAN_UMLAUT = { 0xDC };
+    private static final String SPECIAL_CHAR = new String(GERMAN_UMLAUT);
+    private static final String SPECIAL_CHAR_QUOTED = "'" + SPECIAL_CHAR + "'";
+
+    private void createMySqlTableWithCharacterSet(final String schemaName, final String tableName,
+            final String characterSet) {
+        this.sourceSchema = getSchemaWithCharacterSet(schemaName, "latin1");
+        final String mySqlEnum = "ENUM('A', " + SPECIAL_CHAR_QUOTED + ")";
+        final Table table = this.sourceSchema.createTable(tableName, List.of("c1", "c2"),
+                List.of("CHAR(1)", mySqlEnum));
+        table.insert(SPECIAL_CHAR, SPECIAL_CHAR);
+    }
+
+    private static MySqlSchema getSchemaWithCharacterSet(final String schemaName, final String characterSet) {
+        return new MySqlSchema(SETUP.getTableWriterWithCharacterSet(characterSet), MySQLIdentifier.of(schemaName));
+    }
+
     @Test
     void testSelectAll() throws SQLException {
         final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE;

src/test/java/com/exasol/adapter/dialects/mysql/IntegrationTestConstants.java

@@ -26,6 +26,7 @@
 import org.mockito.junit.jupiter.MockitoExtension;
 
 import com.exasol.adapter.AdapterProperties;
+import com.exasol.adapter.dialects.DataTypeDetection;
 import com.exasol.adapter.dialects.SqlDialect;
 import com.exasol.adapter.jdbc.ConnectionFactory;
 import com.exasol.adapter.jdbc.RemoteMetadataReaderException;
@@ -139,7 +140,8 @@ void testCreateRemoteMetadataReaderConnectionFails() throws SQLException {
     void testGetSupportedProperties() {
         assertThat(this.dialect.getSupportedProperties(),
                 containsInAnyOrder(CONNECTION_NAME_PROPERTY, TABLE_FILTER_PROPERTY, CATALOG_NAME_PROPERTY,
-                        EXCLUDED_CAPABILITIES_PROPERTY, DEBUG_ADDRESS_PROPERTY, LOG_LEVEL_PROPERTY));
+                        EXCLUDED_CAPABILITIES_PROPERTY, DEBUG_ADDRESS_PROPERTY, LOG_LEVEL_PROPERTY,
+                        DataTypeDetection.STRATEGY_PROPERTY));
     }
 
     @Test

src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java

@@ -6,18 +6,22 @@
 import java.io.Closeable;
 import java.io.FileNotFoundException;
 import java.sql.*;
-import java.util.*;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.TimeoutException;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
 import org.testcontainers.containers.MySQLContainer;
 
+import com.exasol.adapter.dialects.mysql.charset.ColumnInspector;
+import com.exasol.adapter.dialects.mysql.charset.TableWriterWithCharacterSet;
 import com.exasol.bucketfs.Bucket;
 import com.exasol.bucketfs.BucketAccessException;
 import com.exasol.containers.ExasolContainer;
 import com.exasol.containers.ExasolService;
 import com.exasol.dbbuilder.dialects.exasol.*;
+import com.exasol.dbbuilder.dialects.mysql.MySqlImmediateDatabaseObjectWriter;
 import com.exasol.dbbuilder.dialects.mysql.MySqlObjectFactory;
 import com.exasol.errorreporting.ExaError;
 import com.exasol.udfdebugging.UdfTestSetup;
@@ -154,13 +158,12 @@ public VirtualSchema createVirtualSchema(final Map<String, String> additionalPro
                 .properties(properties).build();
     }
 
-    Map<String, String> debugProperties() {
-        final String debugAddress = System.getProperty("com.exasol.virtualschema.debug.address");
-        if (debugAddress == null) {
-            return Collections.emptyMap();
-        }
-        final String logLevel = System.getProperty("com.exasol.virtualschema.debug.level");
-        return Map.of("DEBUG_ADDRESS", debugAddress, "LOG_LEVEL", (logLevel != null ? logLevel : "ALL"));
+    public MySqlImmediateDatabaseObjectWriter getTableWriterWithCharacterSet(final String characterSet) {
+        return new TableWriterWithCharacterSet(this.mySqlConnection, characterSet);
+    }
+
+    public ColumnInspector getColumnInspector(final String catalogName) {
+        return ColumnInspector.from(this.mySqlConnection, catalogName);
     }
 
     @Override