From 611684d9d8557bfb92fed3b1b0642e00f98c0b8d Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Tue, 31 Oct 2023 09:59:23 +0100 Subject: [PATCH 01/17] add v8 to ci --- .github/workflows/ci-build.yml | 9 +++- .github/workflows/dependencies_check.yml | 23 +++++++-- dependencies.md | 48 ++++++++----------- doc/changes/changelog.md | 1 + doc/changes/changes_4.1.4.md | 24 ++++++++++ doc/user_guide/mysql_user_guide.md | 2 +- pk_generated_parent.pom | 16 +++---- pom.xml | 6 +-- .../mysql/IntegrationTestConstants.java | 2 +- 9 files changed, 83 insertions(+), 48 deletions(-) create mode 100644 doc/changes/changes_4.1.4.md diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 47d57c5..4088dc1 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -10,8 +10,12 @@ jobs: build: runs-on: ubuntu-20.04 # UDFs fail with "VM error: Internal error: VM crashed" on ubuntu-latest concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.docker_db_version }} cancel-in-progress: true + strategy: + fail-fast: false + matrix: + docker_db_version: ["7.1.24", "8.23.0"] steps: - name: Checkout the repository uses: actions/checkout@v4 @@ -37,7 +41,8 @@ jobs: run: | JAVA_HOME=$JAVA_HOME_11_X64 mvn --batch-mode clean verify \ -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ - -DtrimStackTrace=false + -DtrimStackTrace=false \ + -Dcom.exasol.dockerdb.image=${{ matrix.docker_db_version }} - name: Publish Test Report uses: scacap/action-surefire-report@v1 if: ${{ always() && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' }} diff --git a/.github/workflows/dependencies_check.yml b/.github/workflows/dependencies_check.yml index 3059964..4b6eadf 100644 --- a/.github/workflows/dependencies_check.yml +++ b/.github/workflows/dependencies_check.yml @@ -1,12 +1,15 @@ -name: Dependencies Check +name: Report Security Issues for Repository on: + workflow_dispatch: schedule: - cron: "0 2 * * *" jobs: - build: + report_security_issues: runs-on: ubuntu-latest + permissions: + issues: write steps: - uses: actions/checkout@v4 @@ -16,5 +19,17 @@ jobs: distribution: "temurin" java-version: 11 cache: "maven" - - name: Checking dependencies for vulnerabilities - run: mvn --batch-mode org.sonatype.ossindex.maven:ossindex-maven-plugin:audit -f pom.xml + + - name: Generate ossindex report + run: | + mvn org.sonatype.ossindex.maven:ossindex-maven-plugin:audit \ + org.sonatype.ossindex.maven:ossindex-maven-plugin:audit-aggregate \ + -Dossindex.reportFile=$(pwd)/ossindex-report.json \ + -Dossindex.fail=false + + - name: Report Security Issues + uses: exasol/python-toolbox/.github/actions/security-issues@main + with: + format: "maven" + command: "cat ossindex-report.json" + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/dependencies.md b/dependencies.md index 4bc43ac..b22cb24 100644 --- a/dependencies.md +++ b/dependencies.md @@ -29,30 +29,25 @@ ## Plugin Dependencies -| Dependency | License | -| ------------------------------------------------------- | ---------------------------------------------- | -| [SonarQube Scanner for Maven][27] | [GNU LGPL 3][28] | -| [Apache Maven Compiler Plugin][29] | [Apache-2.0][30] | -| [Apache Maven Enforcer Plugin][31] | [Apache-2.0][30] | -| [Maven Flatten Plugin][32] | [Apache Software Licenese][30] | -| [org.sonatype.ossindex.maven:ossindex-maven-plugin][33] | [ASL2][34] | -| [Maven Surefire Plugin][35] | [Apache-2.0][30] | -| [Versions Maven Plugin][36] | [Apache License, Version 2.0][30] | -| [duplicate-finder-maven-plugin Maven Mojo][37] | [Apache License 2.0][38] | -| [Apache Maven Assembly Plugin][39] | [Apache-2.0][30] | -| [Apache Maven JAR Plugin][40] | [Apache License, Version 2.0][30] | -| [Artifact reference checker and unifier][41] | [MIT License][42] | -| [Apache Maven Dependency Plugin][43] | [Apache-2.0][30] | -| [Project keeper maven plugin][44] | [The MIT License][45] | -| [Maven Failsafe Plugin][46] | [Apache-2.0][30] | -| [JaCoCo :: Maven Plugin][47] | [Eclipse Public License 2.0][26] | -| [error-code-crawler-maven-plugin][48] | [MIT License][49] | -| [Reproducible Build Maven Plugin][50] | [Apache 2.0][34] | -| [Maven Clean Plugin][51] | [The Apache Software License, Version 2.0][34] | -| [Maven Resources Plugin][52] | [The Apache Software License, Version 2.0][34] | -| [Maven Install Plugin][53] | [The Apache Software License, Version 2.0][34] | -| [Maven Deploy Plugin][54] | [The Apache Software License, Version 2.0][34] | -| [Maven Site Plugin 3][55] | [The Apache Software License, Version 2.0][34] | +| Dependency | License | +| ------------------------------------------------------- | --------------------------------- | +| [SonarQube Scanner for Maven][27] | [GNU LGPL 3][28] | +| [Apache Maven Compiler Plugin][29] | [Apache-2.0][30] | +| [Apache Maven Enforcer Plugin][31] | [Apache-2.0][30] | +| [Maven Flatten Plugin][32] | [Apache Software Licenese][30] | +| [org.sonatype.ossindex.maven:ossindex-maven-plugin][33] | [ASL2][34] | +| [Maven Surefire Plugin][35] | [Apache-2.0][30] | +| [Versions Maven Plugin][36] | [Apache License, Version 2.0][30] | +| [duplicate-finder-maven-plugin Maven Mojo][37] | [Apache License 2.0][38] | +| [Apache Maven Assembly Plugin][39] | [Apache-2.0][30] | +| [Apache Maven JAR Plugin][40] | [Apache License, Version 2.0][30] | +| [Artifact reference checker and unifier][41] | [MIT License][42] | +| [Apache Maven Dependency Plugin][43] | [Apache-2.0][30] | +| [Project keeper maven plugin][44] | [The MIT License][45] | +| [Maven Failsafe Plugin][46] | [Apache-2.0][30] | +| [JaCoCo :: Maven Plugin][47] | [Eclipse Public License 2.0][26] | +| [error-code-crawler-maven-plugin][48] | [MIT License][49] | +| [Reproducible Build Maven Plugin][50] | [Apache 2.0][34] | [0]: https://github.com/exasol/virtual-schema-common-jdbc/ [1]: https://github.com/exasol/virtual-schema-common-jdbc/blob/main/LICENSE @@ -105,8 +100,3 @@ [48]: https://github.com/exasol/error-code-crawler-maven-plugin/ [49]: https://github.com/exasol/error-code-crawler-maven-plugin/blob/main/LICENSE [50]: http://zlika.github.io/reproducible-build-maven-plugin -[51]: http://maven.apache.org/plugins/maven-clean-plugin/ -[52]: http://maven.apache.org/plugins/maven-resources-plugin/ -[53]: http://maven.apache.org/plugins/maven-install-plugin/ -[54]: http://maven.apache.org/plugins/maven-deploy-plugin/ -[55]: http://maven.apache.org/plugins/maven-site-plugin/ diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index 338c50d..36649d6 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,5 +1,6 @@ # Changes +* [4.1.4](changes_4.1.4.md) * [4.1.3](changes_4.1.3.md) * [4.1.2](changes_4.1.2.md) * [4.1.1](changes_4.1.1.md) diff --git a/doc/changes/changes_4.1.4.md b/doc/changes/changes_4.1.4.md new file mode 100644 index 0000000..ea93a48 --- /dev/null +++ b/doc/changes/changes_4.1.4.md @@ -0,0 +1,24 @@ +# Virtual Schema for MySQL 4.1.4, released 2023-??-?? + +Code name: + +## Summary + +## Features + +* ISSUE_NUMBER: description + +## Dependency Updates + +### Test Dependency Updates + +* Updated `org.jacoco:org.jacoco.agent:0.8.10` to `0.8.11` + +### Plugin Dependency Updates + +* Updated `com.exasol:error-code-crawler-maven-plugin:1.3.0` to `1.3.1` +* Updated `com.exasol:project-keeper-maven-plugin:2.9.12` to `2.9.15` +* Updated `org.apache.maven.plugins:maven-enforcer-plugin:3.4.0` to `3.4.1` +* Updated `org.codehaus.mojo:versions-maven-plugin:2.16.0` to `2.16.1` +* Updated `org.jacoco:jacoco-maven-plugin:0.8.10` to `0.8.11` +* Updated `org.sonarsource.scanner.maven:sonar-maven-plugin:3.9.1.2184` to `3.10.0.2594` diff --git a/doc/user_guide/mysql_user_guide.md b/doc/user_guide/mysql_user_guide.md index 5ddec5f..af01435 100644 --- a/doc/user_guide/mysql_user_guide.md +++ b/doc/user_guide/mysql_user_guide.md @@ -43,7 +43,7 @@ The SQL statement below creates the adapter script, defines the Java class that --/ CREATE OR REPLACE JAVA ADAPTER SCRIPT SCHEMA_FOR_VS_SCRIPT.ADAPTER_SCRIPT_MYSQL AS %scriptclass com.exasol.adapter.RequestDispatcher; - %jar /buckets/bfsdefault/default/virtual-schema-dist-11.0.2-mysql-4.1.3.jar; + %jar /buckets/bfsdefault/default/virtual-schema-dist-11.0.2-mysql-4.1.4.jar; %jar /buckets/bfsdefault/default/mysql-connector-java-.jar; / ; diff --git a/pk_generated_parent.pom b/pk_generated_parent.pom index 5c78646..ef30a35 100644 --- a/pk_generated_parent.pom +++ b/pk_generated_parent.pom @@ -3,7 +3,7 @@ 4.0.0 com.exasol mysql-virtual-schema-generated-parent - 4.1.3 + 4.1.4 pom UTF-8 @@ -35,7 +35,7 @@ org.jacoco org.jacoco.agent - 0.8.10 + 0.8.11 test runtime @@ -45,7 +45,7 @@ org.sonarsource.scanner.maven sonar-maven-plugin - 3.9.1.2184 + 3.10.0.2594 org.apache.maven.plugins @@ -59,7 +59,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.4.0 + 3.4.1 enforce-maven @@ -69,7 +69,7 @@ - [3.8.7,3.9.0) + 3.6.3 @@ -129,7 +129,7 @@ org.codehaus.mojo versions-maven-plugin - 2.16.0 + 2.16.1 display-updates @@ -265,7 +265,7 @@ org.jacoco jacoco-maven-plugin - 0.8.10 + 0.8.11 prepare-agent @@ -312,7 +312,7 @@ com.exasol error-code-crawler-maven-plugin - 1.3.0 + 1.3.1 verify diff --git a/pom.xml b/pom.xml index 3187e2d..07e9e21 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ 4.0.0 mysql-virtual-schema - 4.1.3 + 4.1.4 Virtual Schema for MySQL Virtual Schema for MySQL https://github.com/exasol/mysql-virtual-schema/ @@ -136,7 +136,7 @@ com.exasol project-keeper-maven-plugin - 2.9.12 + 2.9.15 @@ -157,7 +157,7 @@ mysql-virtual-schema-generated-parent com.exasol - 4.1.3 + 4.1.4 pk_generated_parent.pom diff --git a/src/test/java/com/exasol/adapter/dialects/mysql/IntegrationTestConstants.java b/src/test/java/com/exasol/adapter/dialects/mysql/IntegrationTestConstants.java index 1b8a4dd..da22d7e 100644 --- a/src/test/java/com/exasol/adapter/dialects/mysql/IntegrationTestConstants.java +++ b/src/test/java/com/exasol/adapter/dialects/mysql/IntegrationTestConstants.java @@ -3,7 +3,7 @@ import java.nio.file.Path; public final class IntegrationTestConstants { - public static final String VIRTUAL_SCHEMAS_JAR_NAME_AND_VERSION = "virtual-schema-dist-11.0.2-mysql-4.1.3.jar"; + public static final String VIRTUAL_SCHEMAS_JAR_NAME_AND_VERSION = "virtual-schema-dist-11.0.2-mysql-4.1.4.jar"; public static final String MYSQL_DOCKER_IMAGE_REFERENCE = "mysql:8.1.0"; public static final Path PATH_TO_VIRTUAL_SCHEMAS_JAR = Path.of("target", VIRTUAL_SCHEMAS_JAR_NAME_AND_VERSION); public static final String SCHEMA_EXASOL = "SCHEMA_EXASOL"; From 3a336ba74c16b7a4def3162ae4f2559097292f36 Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Wed, 8 Nov 2023 14:56:53 +0100 Subject: [PATCH 02/17] bump default version to v8 --- .../dialects/mysql/MySQLVirtualSchemaIntegrationTestSetup.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/com/exasol/adapter/dialects/mysql/MySQLVirtualSchemaIntegrationTestSetup.java b/src/test/java/com/exasol/adapter/dialects/mysql/MySQLVirtualSchemaIntegrationTestSetup.java index 697e400..4b3dad7 100644 --- a/src/test/java/com/exasol/adapter/dialects/mysql/MySQLVirtualSchemaIntegrationTestSetup.java +++ b/src/test/java/com/exasol/adapter/dialects/mysql/MySQLVirtualSchemaIntegrationTestSetup.java @@ -40,7 +40,7 @@ public class MySQLVirtualSchemaIntegrationTestSetup implements Closeable { private final Statement mySqlStatement; private final MySQLContainer mySqlContainer = new MySQLContainer<>(MYSQL_DOCKER_IMAGE_REFERENCE) .withUsername("root").withPassword(""); - private final ExasolContainer> exasolContainer = new ExasolContainer<>() + private final ExasolContainer> exasolContainer = new ExasolContainer<>("8.23.0") .withRequiredServices(ExasolService.BUCKETFS, ExasolService.UDF).withReuse(true); private final Connection exasolConection; private final Statement exasolStatement; From ef52b903aab5018fb666f7ffe1704ca12195900b Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Thu, 30 Nov 2023 15:04:17 +0100 Subject: [PATCH 03/17] - updated mysql version - fixed jvm arguments being passed on - renamed and refactored methods of 'importDataTypesExasolCalculated' --- dependencies.md | 2 +- doc/changes/changes_4.1.4.md | 8 +- pk_generated_parent.pom | 17 +- pom.xml | 4 +- .../mysql/IntegrationTestConstants.java | 2 +- .../dialects/mysql/MySQLSqlDialectIT.java | 1268 +++++++++-------- ...ySQLVirtualSchemaIntegrationTestSetup.java | 23 +- 7 files changed, 686 insertions(+), 638 deletions(-) diff --git a/dependencies.md b/dependencies.md index b22cb24..5004e8f 100644 --- a/dependencies.md +++ b/dependencies.md @@ -43,7 +43,7 @@ | [Apache Maven JAR Plugin][40] | [Apache License, Version 2.0][30] | | [Artifact reference checker and unifier][41] | [MIT License][42] | | [Apache Maven Dependency Plugin][43] | [Apache-2.0][30] | -| [Project keeper maven plugin][44] | [The MIT License][45] | +| [Project Keeper Maven plugin][44] | [The MIT License][45] | | [Maven Failsafe Plugin][46] | [Apache-2.0][30] | | [JaCoCo :: Maven Plugin][47] | [Eclipse Public License 2.0][26] | | [error-code-crawler-maven-plugin][48] | [MIT License][49] | diff --git a/doc/changes/changes_4.1.4.md b/doc/changes/changes_4.1.4.md index ea93a48..b9b3c30 100644 --- a/doc/changes/changes_4.1.4.md +++ b/doc/changes/changes_4.1.4.md @@ -12,13 +12,17 @@ Code name: ### Test Dependency Updates +* Updated `com.mysql:mysql-connector-j:8.1.0` to `8.2.0` * Updated `org.jacoco:org.jacoco.agent:0.8.10` to `0.8.11` ### Plugin Dependency Updates * Updated `com.exasol:error-code-crawler-maven-plugin:1.3.0` to `1.3.1` -* Updated `com.exasol:project-keeper-maven-plugin:2.9.12` to `2.9.15` +* Updated `com.exasol:project-keeper-maven-plugin:2.9.12` to `2.9.17` +* Updated `org.apache.maven.plugins:maven-dependency-plugin:3.6.0` to `3.6.1` * Updated `org.apache.maven.plugins:maven-enforcer-plugin:3.4.0` to `3.4.1` -* Updated `org.codehaus.mojo:versions-maven-plugin:2.16.0` to `2.16.1` +* Updated `org.apache.maven.plugins:maven-failsafe-plugin:3.1.2` to `3.2.2` +* Updated `org.apache.maven.plugins:maven-surefire-plugin:3.1.2` to `3.2.2` +* Updated `org.codehaus.mojo:versions-maven-plugin:2.16.0` to `2.16.2` * Updated `org.jacoco:jacoco-maven-plugin:0.8.10` to `0.8.11` * Updated `org.sonarsource.scanner.maven:sonar-maven-plugin:3.9.1.2184` to `3.10.0.2594` diff --git a/pk_generated_parent.pom b/pk_generated_parent.pom index ef30a35..cbd4b05 100644 --- a/pk_generated_parent.pom +++ b/pk_generated_parent.pom @@ -54,6 +54,12 @@ ${java.version} ${java.version} + true + + + -Xlint:all,-processing + + @@ -118,7 +124,7 @@ org.apache.maven.plugins maven-surefire-plugin - 3.1.2 + 3.2.2 @@ -129,7 +135,7 @@ org.codehaus.mojo versions-maven-plugin - 2.16.1 + 2.16.2 display-updates @@ -222,7 +228,7 @@ org.apache.maven.plugins maven-dependency-plugin - 3.6.0 + 3.6.1 copy-jacoco @@ -242,10 +248,9 @@ org.apache.maven.plugins maven-failsafe-plugin - 3.1.2 + 3.2.2 - + -Djava.util.logging.config.file=src/test/resources/logging.properties ${argLine} true diff --git a/pom.xml b/pom.xml index 07e9e21..1d5ab68 100644 --- a/pom.xml +++ b/pom.xml @@ -64,7 +64,7 @@ com.mysql mysql-connector-j - 8.1.0 + 8.2.0 test @@ -136,7 +136,7 @@ com.exasol project-keeper-maven-plugin - 2.9.15 + 2.9.17 diff --git a/src/test/java/com/exasol/adapter/dialects/mysql/IntegrationTestConstants.java b/src/test/java/com/exasol/adapter/dialects/mysql/IntegrationTestConstants.java index da22d7e..bf95192 100644 --- a/src/test/java/com/exasol/adapter/dialects/mysql/IntegrationTestConstants.java +++ b/src/test/java/com/exasol/adapter/dialects/mysql/IntegrationTestConstants.java @@ -4,7 +4,7 @@ public final class IntegrationTestConstants { public static final String VIRTUAL_SCHEMAS_JAR_NAME_AND_VERSION = "virtual-schema-dist-11.0.2-mysql-4.1.4.jar"; - public static final String MYSQL_DOCKER_IMAGE_REFERENCE = "mysql:8.1.0"; + public static final String MYSQL_DOCKER_IMAGE_REFERENCE = "mysql:8.2.0"; public static final Path PATH_TO_VIRTUAL_SCHEMAS_JAR = Path.of("target", VIRTUAL_SCHEMAS_JAR_NAME_AND_VERSION); public static final String SCHEMA_EXASOL = "SCHEMA_EXASOL"; public static final String ADAPTER_SCRIPT_EXASOL = "ADAPTER_SCRIPT_EXASOL"; diff --git a/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java b/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java index 4f82abb..3a5e202 100644 --- a/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java +++ b/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java @@ -34,694 +34,734 @@ @Tag("integration") @Testcontainers class MySQLSqlDialectIT { - private static final String MYSQL_SCHEMA = "MYSQL_SCHEMA"; - private static final String MYSQL_SIMPLE_TABLE = "MYSQL_SIMPLE_TABLE"; - private static final String MYSQL_NUMERIC_DATE_DATATYPES_TABLE = "MYSQL_NUMERIC_DATE_TABLE"; - private static final String MYSQL_STRING_DATATYPES_TABLE = "MYSQL_STRING_TABLE"; - private static final MySQLVirtualSchemaIntegrationTestSetup SETUP = new MySQLVirtualSchemaIntegrationTestSetup(); - private static final String MYSQL_SOURCE_SCHEMA = "SOURCE_SCHEMA"; - private static final String MYSQL_SOURCE_TABLE = "SOURCE_TABLE"; - private static String virtualSchemaJdbc; - private MySqlSchema sourceSchema; - private VirtualSchema virtualSchema; - - @BeforeAll - static void beforeAll() throws SQLException { - final MySqlSchema mySqlSchema = SETUP.getMySqlObjectFactory().createSchema(MYSQL_SCHEMA); - createMySqlSimpleTable(mySqlSchema); - createMySqlNumericDateTable(mySqlSchema); - createMySqlStringTable(mySqlSchema); - createTestTablesForJoinTests(SETUP.getMySqlStatement(), mySqlSchema.getName()); - virtualSchemaJdbc = SETUP.createVirtualSchema(Collections.emptyMap(), mySqlSchema.getName()).getName(); - } - - @AfterAll - static void afterAll() throws IOException { - SETUP.close(); - } - - @AfterEach - void afterEach() { - dropAll(this.virtualSchema, this.sourceSchema); - this.virtualSchema = null; - this.sourceSchema = null; - } - - private static void dropAll(final DatabaseObject... databaseObjects) { - for (final DatabaseObject databaseObject : databaseObjects) { - if (databaseObject != null) { - databaseObject.drop(); - } - } - } - - private static void createTestTablesForJoinTests(final Statement statement, final String schemaName) - throws SQLException { - statement.execute("CREATE TABLE " + schemaName + "." + TABLE_JOIN_1 + "(x INT, y VARCHAR(100))"); - statement.execute("INSERT INTO " + schemaName + "." + TABLE_JOIN_1 + " VALUES (1,'aaa')"); - statement.execute("INSERT INTO " + schemaName + "." + TABLE_JOIN_1 + " VALUES (2,'bbb')"); - statement.execute("CREATE TABLE " + schemaName + "." + TABLE_JOIN_2 + "(x INT, y VARCHAR(100))"); - statement.execute("INSERT INTO " + schemaName + "." + TABLE_JOIN_2 + " VALUES (2,'bbb')"); - statement.execute("INSERT INTO " + schemaName + "." + TABLE_JOIN_2 + " VALUES (3,'ccc')"); - } - - private ResultSet getExpectedResultSet(final List expectedColumns, final List expectedRows) - throws SQLException { - final Statement statement = SETUP.getExasolStatement(); - final String expectedValues = expectedRows.stream().map(row -> "(" + row + ")") - .collect(Collectors.joining(",")); - final String qualifiedExpectedTableName = SCHEMA_EXASOL + ".EXPECTED"; - statement.execute("CREATE OR REPLACE TABLE " + qualifiedExpectedTableName + "(" - + String.join(", ", expectedColumns) + ")"); - statement.execute("INSERT INTO " + qualifiedExpectedTableName + " VALUES" + expectedValues); - return statement.executeQuery("SELECT * FROM " + qualifiedExpectedTableName); - } - - private ResultSet getActualResultSet(final String query) throws SQLException { - final Statement statement = SETUP.getExasolStatement(); - return statement.executeQuery(query); - } - - private static void createMySqlSimpleTable(final Schema mySqlSchema) { - final Table table = mySqlSchema.createTable(MYSQL_SIMPLE_TABLE, List.of("int_col", "bool_col", "varchar_col"), - List.of("INT", "BOOLEAN", "VARCHAR(20)")); - table.insert(-100, true, "a"); - table.insert(-1, false, "abbb"); - table.insert(0, true, "b"); - table.insert(10, false, "bbbb"); - table.insert(50, true, "abc"); - table.insert(100, false, "a"); - } - - private static void createMySqlNumericDateTable(final Schema mySqlSchema) { - final Table table = mySqlSchema.createTable(MYSQL_NUMERIC_DATE_DATATYPES_TABLE, - List.of("BiT_Col", "tinyint_col", "BOOL_COL", "smallint_col", "mediumint_col", "int_col", "bigint_col", - "decimal_col", "float_col", "double_col", // - "date_col", "datetime_col", "timestamp_col", "time_col", "year_col"), - List.of("BIT(6)", "TINYINT", "BOOLEAN", "SMALLINT", "MEDIUMINT", "INT", "BIGINT", "DECIMAL(5, 2)", - "FLOAT(7, 4)", "DOUBLE", // - "DATE", "DATETIME", "TIMESTAMP", "TIME", "YEAR")); - table.insert("5", 127, 1, 32767, 8388607, 2147483647, 9223372036854775807L, 999.99, 999.00009, 999.00009, // - "1000-01-01", "1000-01-01 00:00:00", "1970-01-01 00:00:01.000000", "16:59:59.000000", 1901); - table.insert("9", -127, 0, -32768, -8388608, -2147483648, -9223372036854775808L, -999.99, -999.9999, -999.9999, // - "9999-12-31", "9999-12-31 22:59:59", "2037-01-19 03:14:07.999999", "05:34:13.000000", 2155); - table.insert(null, 0, true, 0, 0, 0, 0, 0, 0, 0, // - null, null, null, null, "1901"); - table.insert(null, 0, false, 0, 0, 0, 0, 0, 0, 0, // - null, null, null, null, 69); - } - - private static void createMySqlStringTable(final Schema mySqlSchema) { - final Table table = mySqlSchema.createTable(MYSQL_STRING_DATATYPES_TABLE, - List.of("binary_col", "varbinary_col", "tinyblob_col", "tinytext_col", "blob_col", "text_col", - "mediumblob_col", "mediumtext_col", "longblob_col", "longtext_col", "enum_col", "set_col", - "varchar_col", "char_col"), - List.of(" BINARY(20)", "VARBINARY(20)", "TINYBLOB", "TINYTEXT", "BLOB", "TEXT", "MEDIUMBLOB", - "MEDIUMTEXT", "LONGBLOB", "LONGTEXT", "ENUM('1', '2', '3')", "SET('1')", "VARCHAR(16000)", - "CHAR(255)")); - table.insert("a", "a", "a", "a", "blob", "text", "mediumblob", "mediumtext", "longblob", "longtext", "1", "1", - "ab", "asd24"); - table.insert("a\0", "a\0", "aa", "b", "blob", "text2", "mediumblob2", "mediumtext2", "longblob", "longtext2", - "2", "1", "a", "11111"); - table.insert(null, null, "aaa", "aaaaaaaaaaaaa", "bloooooooooooob", "text3", null, null, null, null, "3", null, - "", ""); - table.insert(null, null, "aaaaa", "a", "blob", "text", null, null, null, null, null, null, null, null); - } - - @Test - void importDataTypesFromResultSet() throws SQLException { - Assume.assumeTrue(runCharsetTest()); - final String query = setupCharacterSet(DataTypeDetection.Strategy.FROM_RESULT_SET); - final ResultSet actual = getActualResultSet(query); - final ResultSet expected = getExpectedResultSet(List.of("c1 CHAR(1) UTF8", "c2 CHAR(1) UTF8"), // - List.of(SPECIAL_CHAR_QUOTED + ", " + SPECIAL_CHAR_QUOTED)); - assertThat(actual, matchesResultSet(expected)); - } - - @Test - void importDataTypesExasolCalculated() throws SQLException { - Assume.assumeTrue(runCharsetTest()); - final String query = setupCharacterSet(DataTypeDetection.Strategy.EXASOL_CALCULATED); - final Exception exception = assertThrows(SQLException.class, () -> getActualResultSet(query)); - assertThat(exception.getMessage(), - matchesRegex("ETL-3009: .*Charset conversion from 'UTF-8' to 'ASCII' failed.*")); - } - - private boolean runCharsetTest() { - final ExasolDockerImageReference dockerImage = SETUP.getExasolContainer().getDockerImageReference(); - if (!dockerImage.hasMajor() || !dockerImage.hasMinor() || !dockerImage.hasFix()) { - return false; - } - final Version version = Version.of(dockerImage.getMajor(), dockerImage.getMinor(), dockerImage.getFixVersion()); - if ((dockerImage.getMajor() == 7) && version.isGreaterOrEqualThan(Version.parse("7.1.14"))) { - return true; - } - if ((dockerImage.getMajor() == 8) && version.isGreaterOrEqualThan(Version.parse("8.6.0"))) { - return true; - } - return false; - } - - private String setupCharacterSet(final DataTypeDetection.Strategy strategy) throws SQLException { - final String tableName = MYSQL_SOURCE_TABLE; - createMySqlTableWithCharacterSet(MYSQL_SOURCE_SCHEMA, tableName, "latin1"); - this.virtualSchema = SETUP.createVirtualSchema( // - Map.of(DataTypeDetection.STRATEGY_PROPERTY, strategy.name()), // - MYSQL_SOURCE_SCHEMA); - final ColumnInspector inspector = SETUP.getColumnInspector(MYSQL_SOURCE_SCHEMA); - inspector.describeFromMetadata(MYSQL_SOURCE_SCHEMA, MYSQL_SOURCE_TABLE); - final String query = String.format("select * from %s.%s", MYSQL_SOURCE_SCHEMA, MYSQL_SOURCE_TABLE); - inspector.describeFromQuery(MYSQL_SOURCE_SCHEMA, query); - return "SELECT * FROM " + this.virtualSchema.getName() + "." + tableName; - } - - private static final char[] GERMAN_UMLAUT = { 0xDC }; - private static final String SPECIAL_CHAR = new String(GERMAN_UMLAUT); - private static final String SPECIAL_CHAR_QUOTED = "'" + SPECIAL_CHAR + "'"; - - private void createMySqlTableWithCharacterSet(final String schemaName, final String tableName, - final String characterSet) { - this.sourceSchema = getSchemaWithCharacterSet(schemaName, "latin1"); - final String mySqlEnum = "ENUM('A', " + SPECIAL_CHAR_QUOTED + ")"; - final Table table = this.sourceSchema.createTable(tableName, List.of("c1", "c2"), - List.of("CHAR(1)", mySqlEnum)); - table.insert(SPECIAL_CHAR, SPECIAL_CHAR); - } - - private static MySqlSchema getSchemaWithCharacterSet(final String schemaName, final String characterSet) { - return new MySqlSchema(SETUP.getTableWriterWithCharacterSet(characterSet), MySQLIdentifier.of(schemaName)); - } - - @Test - void testSelectAll() throws SQLException { - final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE; - final ResultSet actualResultSet = getActualResultSet(query); - final ResultSet expected = getExpectedResultSet(List.of("col1 INT", "col2 BOOLEAN", "col3 VARCHAR(20)"), // - List.of("-100, true, 'a'", // - "-1, false, 'abbb'", // - "0, true, 'b'", // - "10, false, 'bbbb'", // - "50, true, 'abc'", // - "100, false, 'a'")); - assertThat(actualResultSet, matchesResultSet(expected)); - } - - @Test - void testAggregateGroupByColumn() throws SQLException { - final String query = "SELECT \"bool_col\", min(\"int_col\") FROM " + virtualSchemaJdbc + "." - + MYSQL_SIMPLE_TABLE + " GROUP BY \"bool_col\""; - final ResultSet expected = getExpectedResultSet(List.of("bool_col BOOLEAN", "int_col DECIMAL(10,0)"), - List.of("true, -100", // - "false, -1")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testAggregateHaving() throws SQLException { - final String query = "SELECT \"bool_col\", min(\"int_col\") FROM " + virtualSchemaJdbc + "." - + MYSQL_SIMPLE_TABLE + " GROUP BY \"bool_col\" HAVING MIN(\"int_col\") < 0"; - final ResultSet expected = getExpectedResultSet(List.of("bool_col BOOLEAN", "int_col DECIMAL(10,0)"), - List.of("true, -100", // - "false, -1")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - // =, !=, <, <=, >, >= - void testComparisonPredicates() throws SQLException { - final String query = "SELECT \"int_col\", \"int_col\" = 60, \"int_col\" != 60, \"int_col\" < 60, " - + "\"int_col\" <= 60, \"int_col\" > 60, \"int_col\" >= 60 FROM " + virtualSchemaJdbc + "." - + MYSQL_SIMPLE_TABLE + " WHERE \"int_col\" = 0"; - final ResultSet expected = getExpectedResultSet( - List.of("int_col DECIMAL(10,0)", "b1 BOOLEAN", "b2 BOOLEAN", "b3 BOOLEAN", "b4 BOOLEAN", "b5 BOOLEAN", - "b6 BOOLEAN"), // - List.of("0, 0, 1, 1, 1, 0, 0")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - // NOT, AND, OR - void testLogicalPredicates() throws SQLException { - final String query = "SELECT \"int_col\" FROM " + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE - + " WHERE (\"int_col\" < 0 or \"int_col\" > 0) AND NOT (\"int_col\" is null)"; - final ResultSet expected = getExpectedResultSet(List.of("int_col DECIMAL(10,0)"), // - List.of("-100", // - "-1", // - "10", // - "50", // - "100")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - // LIKE, LIKE ESCAPE (not pushed down) - void testLikePredicates() throws SQLException { - final String query = "SELECT \"varchar_col\", \"varchar_col\" LIKE 'a%' ESCAPE 'a' FROM " + virtualSchemaJdbc - + "." + MYSQL_SIMPLE_TABLE + " WHERE (\"varchar_col\" LIKE 'a%')"; - final ResultSet expected = getExpectedResultSet(List.of("varchar_col VARCHAR(10)", "bool_col BOOLEAN"), - List.of("'a', false", // - "'abbb', false", // - "'abc', false", // - "'a', false")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - // BETWEEN, IN, IS NULL, !=NULL(rewritten to "IS NOT NULL") - void testMiscPredicates() throws SQLException { - final String query = "SELECT \"int_col\", \"int_col\" in (56, 61), \"int_col\" is null, \"int_col\" != null" - + " FROM " + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE + " WHERE \"int_col\" between -10 and 51"; - final ResultSet expected = getExpectedResultSet( - List.of("int_col DECIMAL(10,0)", "b1 BOOLEAN", "b2 BOOLEAN", "b3 BOOLEAN"), // - List.of("-1, false, false, false", // - "0, false, false, false", // - "10, false, false, false", // - "50, false, false, false")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testCountSumAggregateFunction() throws SQLException { - final String query = "SELECT COUNT(\"int_col\"), COUNT(*), COUNT(DISTINCT \"int_col\"), SUM(\"int_col\"), " - + "SUM(DISTINCT \"int_col\") FROM " + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE; - final ResultSet expected = getExpectedResultSet( - List.of("a DECIMAL(10,0)", "b DECIMAL(10,0)", "c DECIMAL(10,0)", "d DECIMAL(19,0)", "e DECIMAL(19,0)"), - List.of("6, 6, 6, 59, 59")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testAvgMinMaxAggregateFunction() throws SQLException { - final String query = "SELECT AVG(\"int_col\"), MIN(\"int_col\"), MAX(\"int_col\") FROM " + virtualSchemaJdbc - + "." + MYSQL_SIMPLE_TABLE; - final ResultSet expected = getExpectedResultSet( // - List.of("a DOUBLE", "b DECIMAL(10,0)", "c DECIMAL(10,0)"), // - List.of("9.833300000000001, -100.0000, 100.0000")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testCastedStringFunctions() throws SQLException { - final String query = "SELECT concat(upper(\"varchar_col\"),lower(repeat(\"varchar_col\",2))) FROM " - + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("a VARCHAR(100)"), // - List.of("'Aaa'", // - "'ABBBabbbabbb'", // - "'Bbb'", // - "'BBBBbbbbbbbb'", // - "'ABCabcabc'", // - "'Aaa'")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testRewrittenDivAndModFunctions() throws SQLException { - final String query = "SELECT DIV(\"int_col\",\"int_col\"), mod(\"int_col\",\"int_col\") FROM " - + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("a DECIMAL(10,0)", "b DECIMAL(10,0)"), // - List.of("1, 0", // - "1, 0", // - "null, null", // - "1, 0", // - "1, 0", // - "1, 0")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testRewrittenSubStringFunction() throws SQLException { - final String query = "SELECT substring(\"varchar_col\" FROM 1 FOR 2) FROM " + virtualSchemaJdbc + "." - + MYSQL_SIMPLE_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("a VARCHAR(100)"), // - List.of("'a'", // - "'ab'", // - "'b'", // - "'bb'", // - "'ab'", // - "'a'")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testOrderByLimit() throws SQLException { - final String query = "SELECT \"bool_col\", \"int_col\" FROM " + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE - + " ORDER BY \"int_col\" LIMIT 3"; - final ResultSet expected = getExpectedResultSet(List.of("a BOOLEAN", "b DECIMAL(10,0)"), // - List.of("true, -100", // - "false, -1", // - "true, 0")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testOrderByLimitOffset() throws SQLException { - final String query = "SELECT \"bool_col\", \"int_col\" FROM " + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE - + " ORDER BY \"int_col\" LIMIT 2 OFFSET 1"; - final ResultSet expected = getExpectedResultSet(List.of("a BOOLEAN", "b DECIMAL(10,0)"), // - List.of("false, -1", // - "true, 0")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testLeftShiftScalarFunction() { - createSourceTable(List.of("INT_COL_1", "INT_COL_2"), List.of("INT", "INT"), new Object[][] { { 10, 3 } }); - this.virtualSchema = SETUP.createVirtualSchema(Collections.emptyMap(), MYSQL_SOURCE_SCHEMA); - final String query = "SELECT BIT_LSHIFT(INT_COL_1, INT_COL_2) FROM " + this.virtualSchema.getName() + "." - + MYSQL_SOURCE_TABLE; - assertVsQuery(query, table().row(80).matches(TypeMatchMode.NO_JAVA_TYPE_CHECK)); - } - - private void createSourceTable(final List columnNames, final List types, final Object[][] values) { - this.sourceSchema = SETUP.getMySqlObjectFactory().createSchema(MYSQL_SOURCE_SCHEMA); - final Table table = this.sourceSchema.createTable(MYSQL_SOURCE_TABLE, columnNames, types); - for (final Object[] value : values) { - table.insert(value); - } - } - - private ResultSet query(final String sql) throws SQLException { - return SETUP.getExasolStatement().executeQuery(sql); - } - - @Test - void testRightShiftScalarFunction() { - createSourceTable(List.of("INT_COL_1", "INT_COL_2"), List.of("INT", "INT"), new Object[][] { { 10, 3 } }); - this.virtualSchema = SETUP.createVirtualSchema(Collections.emptyMap(), MYSQL_SOURCE_SCHEMA); - final String query = "SELECT BIT_RSHIFT(INT_COL_1, INT_COL_2) FROM " + this.virtualSchema.getName() + "." - + MYSQL_SOURCE_TABLE; - assertVsQuery(query, table().row(1).matches(TypeMatchMode.NO_JAVA_TYPE_CHECK)); - } - - private void assertVsQuery(final String sql, final Matcher expected) { - try { - assertThat(query(sql), expected); - } catch (final SQLException exception) { - fail("Unable to run assertion query: " + sql + "\nCaused by: " + exception.getMessage()); - } - } - - @Test - void testHourScalarFunction() { - createSourceTable(List.of("TIMESTAMP_COL"), List.of("TIMESTAMP"), new Object[][] { { "2021-02-16 11:48:01" } }); - this.virtualSchema = SETUP.createVirtualSchema(Collections.emptyMap(), MYSQL_SOURCE_SCHEMA); - final String query = "SELECT HOUR(timestamp_col) FROM " + this.virtualSchema.getName() + "." - + MYSQL_SOURCE_TABLE; - assertVsQuery(query, table().row(11).matches(TypeMatchMode.NO_JAVA_TYPE_CHECK)); - } - - @Nested - @DisplayName("Datatype tests") - class DatatypeTest { - @Test - void testBit() throws SQLException { - final String query = "SELECT \"BiT_Col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 BOOLEAN"), // - List.of("true", "true", "false", "false")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + private static final String MYSQL_SCHEMA = "MYSQL_SCHEMA"; + private static final String MYSQL_SIMPLE_TABLE = "MYSQL_SIMPLE_TABLE"; + private static final String MYSQL_NUMERIC_DATE_DATATYPES_TABLE = "MYSQL_NUMERIC_DATE_TABLE"; + private static final String MYSQL_STRING_DATATYPES_TABLE = "MYSQL_STRING_TABLE"; + private static final MySQLVirtualSchemaIntegrationTestSetup SETUP = new MySQLVirtualSchemaIntegrationTestSetup(); + + private static final String MYSQL_SOURCE_SCHEMA = "SOURCE_SCHEMA"; + private static final String MYSQL_SOURCE_TABLE = "SOURCE_TABLE"; + private static String virtualSchemaJdbc; + private MySqlSchema sourceSchema; + private VirtualSchema virtualSchema; + + @BeforeAll + static void beforeAll() throws SQLException { + final MySqlSchema mySqlSchema = SETUP.getMySqlObjectFactory().createSchema(MYSQL_SCHEMA); + createMySqlSimpleTable(mySqlSchema); + createMySqlNumericDateTable(mySqlSchema); + createMySqlStringTable(mySqlSchema); + createTestTablesForJoinTests(SETUP.getMySqlStatement(), mySqlSchema.getName()); + virtualSchemaJdbc = SETUP.createVirtualSchema(Collections.emptyMap(), mySqlSchema.getName()).getName(); + } + + @AfterAll + static void afterAll() throws IOException { + SETUP.close(); + } + + @AfterEach + void afterEach() { + dropAll(this.virtualSchema, this.sourceSchema); + this.virtualSchema = null; + this.sourceSchema = null; + } + + private static void dropAll(final DatabaseObject... databaseObjects) { + for (final DatabaseObject databaseObject : databaseObjects) { + if (databaseObject != null) { + databaseObject.drop(); + } + } + } + + private static void createTestTablesForJoinTests(final Statement statement, final String schemaName) + throws SQLException { + statement.execute("CREATE TABLE " + schemaName + "." + TABLE_JOIN_1 + "(x INT, y VARCHAR(100))"); + statement.execute("INSERT INTO " + schemaName + "." + TABLE_JOIN_1 + " VALUES (1,'aaa')"); + statement.execute("INSERT INTO " + schemaName + "." + TABLE_JOIN_1 + " VALUES (2,'bbb')"); + statement.execute("CREATE TABLE " + schemaName + "." + TABLE_JOIN_2 + "(x INT, y VARCHAR(100))"); + statement.execute("INSERT INTO " + schemaName + "." + TABLE_JOIN_2 + " VALUES (2,'bbb')"); + statement.execute("INSERT INTO " + schemaName + "." + TABLE_JOIN_2 + " VALUES (3,'ccc')"); + } + + private ResultSet getExpectedResultSet(final List expectedColumns, final List expectedRows) + throws SQLException { + final Statement statement = SETUP.getExasolStatement(); + final String expectedValues = expectedRows.stream().map(row -> "(" + row + ")") + .collect(Collectors.joining(",")); + final String qualifiedExpectedTableName = SCHEMA_EXASOL + ".EXPECTED"; + statement.execute("CREATE OR REPLACE TABLE " + qualifiedExpectedTableName + "(" + + String.join(", ", expectedColumns) + ")"); + statement.execute("INSERT INTO " + qualifiedExpectedTableName + " VALUES" + expectedValues); + return statement.executeQuery("SELECT * FROM " + qualifiedExpectedTableName); + } + + private ResultSet getActualResultSet(final String query) throws SQLException { + final Statement statement = SETUP.getExasolStatement(); + return statement.executeQuery(query); + } + + private static void createMySqlSimpleTable(final Schema mySqlSchema) { + final Table table = mySqlSchema.createTable(MYSQL_SIMPLE_TABLE, + List.of("int_col", "bool_col", "varchar_col"), + List.of("INT", "BOOLEAN", "VARCHAR(20)")); + table.insert(-100, true, "a"); + table.insert(-1, false, "abbb"); + table.insert(0, true, "b"); + table.insert(10, false, "bbbb"); + table.insert(50, true, "abc"); + table.insert(100, false, "a"); + } + + private static void createMySqlNumericDateTable(final Schema mySqlSchema) { + final Table table = mySqlSchema.createTable(MYSQL_NUMERIC_DATE_DATATYPES_TABLE, + List.of("BiT_Col", "tinyint_col", "BOOL_COL", "smallint_col", "mediumint_col", + "int_col", "bigint_col", "decimal_col", "float_col", "double_col", // + "date_col", "datetime_col", "timestamp_col", "time_col", "year_col"), + List.of("BIT(6)", "TINYINT", "BOOLEAN", "SMALLINT", "MEDIUMINT", "INT", "BIGINT", + "DECIMAL(5, 2)", "FLOAT(7, 4)", "DOUBLE", // + "DATE", "DATETIME", "TIMESTAMP", "TIME", "YEAR")); + table.insert("5", 127, 1, 32767, 8388607, 2147483647, 9223372036854775807L, 999.99, 999.00009, + 999.00009, // + "1000-01-01", "1000-01-01 00:00:00", "1970-01-01 00:00:01.000000", "16:59:59.000000", + 1901); + table.insert("9", -127, 0, -32768, -8388608, -2147483648, -9223372036854775808L, -999.99, -999.9999, + -999.9999, // + "9999-12-31", "9999-12-31 22:59:59", "2037-01-19 03:14:07.999999", "05:34:13.000000", + 2155); + table.insert(null, 0, true, 0, 0, 0, 0, 0, 0, 0, // + null, null, null, null, "1901"); + table.insert(null, 0, false, 0, 0, 0, 0, 0, 0, 0, // + null, null, null, null, 69); + } + + private static void createMySqlStringTable(final Schema mySqlSchema) { + final Table table = mySqlSchema.createTable(MYSQL_STRING_DATATYPES_TABLE, + List.of("binary_col", "varbinary_col", "tinyblob_col", "tinytext_col", "blob_col", + "text_col", "mediumblob_col", "mediumtext_col", "longblob_col", + "longtext_col", "enum_col", "set_col", "varchar_col", "char_col"), + List.of(" BINARY(20)", "VARBINARY(20)", "TINYBLOB", "TINYTEXT", "BLOB", "TEXT", + "MEDIUMBLOB", "MEDIUMTEXT", "LONGBLOB", "LONGTEXT", + "ENUM('1', '2', '3')", "SET('1')", "VARCHAR(16000)", "CHAR(255)")); + table.insert("a", "a", "a", "a", "blob", "text", "mediumblob", "mediumtext", "longblob", "longtext", + "1", "1", "ab", "asd24"); + table.insert("a\0", "a\0", "aa", "b", "blob", "text2", "mediumblob2", "mediumtext2", "longblob", + "longtext2", "2", "1", "a", "11111"); + table.insert(null, null, "aaa", "aaaaaaaaaaaaa", "bloooooooooooob", "text3", null, null, null, null, + "3", null, "", ""); + table.insert(null, null, "aaaaa", "a", "blob", "text", null, null, null, null, null, null, null, null); } @Test - void testTinyInt() throws SQLException { - final String query = "SELECT \"tinyint_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 DECIMAL(3,0)"), // - List.of("127", "-127", "0", "0")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void importDataTypesFromResultSet() throws SQLException { + Assume.assumeTrue(runCharsetTest()); + final String query = setupMySQLTableWithLatin1AndVirtualSchemaWithStrategy( + DataTypeDetection.Strategy.FROM_RESULT_SET); + final ResultSet actual = getActualResultSet(query); + final ResultSet expected = getExpectedResultSet(List.of("c1 CHAR(1) UTF8", "c2 CHAR(1) UTF8"), // + List.of(SPECIAL_CHAR_QUOTED + ", " + SPECIAL_CHAR_QUOTED)); + assertThat(actual, matchesResultSet(expected)); } @Test - void testBoolean() throws SQLException { - final String query = "SELECT \"BOOL_COL\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 BOOLEAN"), // - List.of("true", "false", "true", "false")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void importDataTypesExasolCalculated() throws SQLException { + Assume.assumeTrue(runCharsetTest()); + final String query = setupMySQLTableWithLatin1AndVirtualSchemaWithStrategy( + DataTypeDetection.Strategy.EXASOL_CALCULATED); + final Exception exception = assertThrows(SQLException.class, () -> getActualResultSet(query)); + assertThat(exception.getMessage(), + matchesRegex("ETL-3009: .*Charset conversion from 'UTF-8' to 'ASCII' failed.*")); } - @Test - void testSmallInt() throws SQLException { - final String query = "SELECT \"smallint_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 DECIMAL(5,0)"), // - List.of("32767", "-32768", "0", "0")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + private boolean runCharsetTest() { + final ExasolDockerImageReference dockerImage = SETUP.getExasolContainer().getDockerImageReference(); + if (!dockerImage.hasMajor() || !dockerImage.hasMinor() || !dockerImage.hasFix()) { + return false; + } + final Version version = Version.of(dockerImage.getMajor(), dockerImage.getMinor(), + dockerImage.getFixVersion()); + if ((dockerImage.getMajor() == 7) && version.isGreaterOrEqualThan(Version.parse("7.1.14"))) { + return true; + } + if ((dockerImage.getMajor() == 8) && version.isGreaterOrEqualThan(Version.parse("8.6.0"))) { + return true; + } + return false; } - @Test - void testMediumInt() throws SQLException { - final String query = "SELECT \"mediumint_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 DECIMAL(7,0)"), // - List.of("8388607", "-8388608", "0", "0")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } + private String setupMySQLTableWithLatin1AndVirtualSchemaWithStrategy(final DataTypeDetection.Strategy strategy) + throws SQLException { + final String tableName = MYSQL_SOURCE_TABLE; + // create SQL schema with latin1 characterset and virtual schema + createMySqlTableContainingCharAndEnumWithCharacterSet(MYSQL_SOURCE_SCHEMA, tableName, "latin1"); - @Test - void testInt() throws SQLException { - final String query = "SELECT \"int_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 DECIMAL(10,0)"), // - List.of("2147483647", "-2147483648", "0", "0")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } + this.virtualSchema = SETUP.createVirtualSchema( // + Map.of(DataTypeDetection.STRATEGY_PROPERTY, strategy.name()), // + MYSQL_SOURCE_SCHEMA); - @Test - void testBigInt() throws SQLException { - final String query = "SELECT \"bigint_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 DECIMAL(19,0)"), // - List.of("9223372036854775807", "-9223372036854775808", "0", "0")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } + final ColumnInspector inspector = SETUP.getColumnInspector(MYSQL_SOURCE_SCHEMA); + inspector.describeFromMetadata(MYSQL_SOURCE_SCHEMA, MYSQL_SOURCE_TABLE); - @Test - void testDecimal() throws SQLException { - final String query = "SELECT \"decimal_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 DECIMAL(5,2)"), // - List.of("999.99", "-999.99", "0", "0")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + final String query = String.format("select * from %s.%s", MYSQL_SOURCE_SCHEMA, MYSQL_SOURCE_TABLE); + inspector.describeFromQuery(MYSQL_SOURCE_SCHEMA, query); + + return selectEverythingFromVirtualSchemaTable(tableName); } - @Test - void testFloat() throws SQLException { - final String query = "SELECT \"float_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 DOUBLE PRECISION"), // - List.of("999.0001", "-999.9999", "0", "0")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + private String selectEverythingFromVirtualSchemaTable(final String tableName) { + return "SELECT * FROM " + this.virtualSchema.getName() + "." + tableName; } - @Test - void testDouble() throws SQLException { - final String query = "SELECT \"double_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 DOUBLE PRECISION"), // - List.of("999.00009", "-999.9999", "0", "0")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + private static final char[] GERMAN_UMLAUT = { 0xDC }; + private static final String SPECIAL_CHAR = new String(GERMAN_UMLAUT); + private static final String SPECIAL_CHAR_QUOTED = "'" + SPECIAL_CHAR + "'"; + + private void createMySqlTableContainingCharAndEnumWithCharacterSet(final String schemaName, + final String tableName, final String characterSet) { + + this.sourceSchema = getSchemaWithCharacterSet(schemaName, "latin1"); + + final String mySqlEnum = "ENUM('A', " + SPECIAL_CHAR_QUOTED + ")"; + + final Table table = this.sourceSchema.createTable(tableName, List.of("c1", "c2"), + List.of("CHAR(1)", mySqlEnum)); + + table.insert(SPECIAL_CHAR, SPECIAL_CHAR); } - @Test - void testDate() throws SQLException { - final String query = "SELECT \"date_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 DATE"), // - List.of("'1000-01-01'", "'9999-12-31'", "null", "null")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + private static MySqlSchema getSchemaWithCharacterSet(final String schemaName, final String characterSet) { + return new MySqlSchema(SETUP.getTableWriterWithCharacterSet(characterSet), + MySQLIdentifier.of(schemaName)); } @Test - void testDatetime() throws SQLException { - final String query = "SELECT \"datetime_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 TIMESTAMP"), // - List.of("'1000-01-01 00:00:00'", "'9999-12-31 22:59:59'", "null", "null")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void testSelectAll() throws SQLException { + final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE; + final ResultSet actualResultSet = getActualResultSet(query); + final ResultSet expected = getExpectedResultSet(List.of("col1 INT", "col2 BOOLEAN", "col3 VARCHAR(20)"), // + List.of("-100, true, 'a'", // + "-1, false, 'abbb'", // + "0, true, 'b'", // + "10, false, 'bbbb'", // + "50, true, 'abc'", // + "100, false, 'a'")); + assertThat(actualResultSet, matchesResultSet(expected)); } @Test - void testTimestamp() throws SQLException { - final String query = "SELECT \"timestamp_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 TIMESTAMP"), // - List.of("'1970-01-01 00:00:01.000000'", "'2037-01-19 03:14:08.0'", "null", "null")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void testAggregateGroupByColumn() throws SQLException { + final String query = "SELECT \"bool_col\", min(\"int_col\") FROM " + virtualSchemaJdbc + "." + + MYSQL_SIMPLE_TABLE + " GROUP BY \"bool_col\""; + final ResultSet expected = getExpectedResultSet(List.of("bool_col BOOLEAN", "int_col DECIMAL(10,0)"), + List.of("true, -100", // + "false, -1")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void testTime() throws SQLException { - final String query = "SELECT \"time_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 TIMESTAMP"), // - List.of("'1970-01-01 16:59:59.000000'", "'1970-01-01 05:34:13.000000'", "null", "null")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void testAggregateHaving() throws SQLException { + final String query = "SELECT \"bool_col\", min(\"int_col\") FROM " + virtualSchemaJdbc + "." + + MYSQL_SIMPLE_TABLE + " GROUP BY \"bool_col\" HAVING MIN(\"int_col\") < 0"; + final ResultSet expected = getExpectedResultSet(List.of("bool_col BOOLEAN", "int_col DECIMAL(10,0)"), + List.of("true, -100", // + "false, -1")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void testYear() throws SQLException { - final String query = "SELECT \"year_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 DATE"), // - List.of("'1901-01-01'", "'2155-01-01'", "'1901-01-01'", "'2069-01-01'")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + // =, !=, <, <=, >, >= + void testComparisonPredicates() throws SQLException { + final String query = "SELECT \"int_col\", \"int_col\" = 60, \"int_col\" != 60, \"int_col\" < 60, " + + "\"int_col\" <= 60, \"int_col\" > 60, \"int_col\" >= 60 FROM " + virtualSchemaJdbc + + "." + MYSQL_SIMPLE_TABLE + " WHERE \"int_col\" = 0"; + final ResultSet expected = getExpectedResultSet( + List.of("int_col DECIMAL(10,0)", "b1 BOOLEAN", "b2 BOOLEAN", "b3 BOOLEAN", "b4 BOOLEAN", + "b5 BOOLEAN", "b6 BOOLEAN"), // + List.of("0, 0, 1, 1, 1, 0, 0")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void testUnsupported() { - final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + MYSQL_STRING_DATATYPES_TABLE; - assertDoesNotThrow(() -> getActualResultSet(query)); + // NOT, AND, OR + void testLogicalPredicates() throws SQLException { + final String query = "SELECT \"int_col\" FROM " + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE + + " WHERE (\"int_col\" < 0 or \"int_col\" > 0) AND NOT (\"int_col\" is null)"; + final ResultSet expected = getExpectedResultSet(List.of("int_col DECIMAL(10,0)"), // + List.of("-100", // + "-1", // + "10", // + "50", // + "100")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void testTinyText() throws SQLException { - final String query = "SELECT \"tinytext_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_STRING_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 VARCHAR(100)"), // - List.of("'a'", "'b'", "'aaaaaaaaaaaaa'", "'a'")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + // LIKE, LIKE ESCAPE (not pushed down) + void testLikePredicates() throws SQLException { + final String query = "SELECT \"varchar_col\", \"varchar_col\" LIKE 'a%' ESCAPE 'a' FROM " + + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE + " WHERE (\"varchar_col\" LIKE 'a%')"; + final ResultSet expected = getExpectedResultSet(List.of("varchar_col VARCHAR(10)", "bool_col BOOLEAN"), + List.of("'a', false", // + "'abbb', false", // + "'abc', false", // + "'a', false")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void testText() throws SQLException { - final String query = "SELECT \"text_col\" FROM " + virtualSchemaJdbc + "." + MYSQL_STRING_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 VARCHAR(65535)"), // - List.of("'text'", "'text2'", "'text3'", "'text'")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + // BETWEEN, IN, IS NULL, !=NULL(rewritten to "IS NOT NULL") + void testMiscPredicates() throws SQLException { + final String query = "SELECT \"int_col\", \"int_col\" in (56, 61), \"int_col\" is null, \"int_col\" != null" + + " FROM " + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE + + " WHERE \"int_col\" between -10 and 51"; + final ResultSet expected = getExpectedResultSet( + List.of("int_col DECIMAL(10,0)", "b1 BOOLEAN", "b2 BOOLEAN", "b3 BOOLEAN"), // + List.of("-1, false, false, false", // + "0, false, false, false", // + "10, false, false, false", // + "50, false, false, false")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void testMediumText() throws SQLException { - final String query = "SELECT \"mediumtext_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_STRING_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 VARCHAR(2000000)"), // - List.of("'mediumtext'", "'mediumtext2'", "NULL", "NULL")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void testCountSumAggregateFunction() throws SQLException { + final String query = "SELECT COUNT(\"int_col\"), COUNT(*), COUNT(DISTINCT \"int_col\"), SUM(\"int_col\"), " + + "SUM(DISTINCT \"int_col\") FROM " + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("a DECIMAL(10,0)", "b DECIMAL(10,0)", + "c DECIMAL(10,0)", "d DECIMAL(19,0)", "e DECIMAL(19,0)"), List.of("6, 6, 6, 59, 59")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void testLongText() throws SQLException { - final String query = "SELECT \"longtext_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_STRING_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 VARCHAR(2000000)"), // - List.of("'longtext'", "'longtext2'", "NULL", "NULL")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void testAvgMinMaxAggregateFunction() throws SQLException { + final String query = "SELECT AVG(\"int_col\"), MIN(\"int_col\"), MAX(\"int_col\") FROM " + + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE; + final ResultSet expected = getExpectedResultSet( // + List.of("a DOUBLE", "b DECIMAL(10,0)", "c DECIMAL(10,0)"), // + List.of("9.833300000000001, -100.0000, 100.0000")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void testEnum() throws SQLException { - final String query = "SELECT \"enum_col\" FROM " + virtualSchemaJdbc + "." + MYSQL_STRING_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 CHAR"), // - List.of("'1'", "'2'", "'3'", "NULL")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void testCastedStringFunctions() throws SQLException { + final String query = "SELECT concat(upper(\"varchar_col\"),lower(repeat(\"varchar_col\",2))) FROM " + + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("a VARCHAR(100)"), // + List.of("'Aaa'", // + "'ABBBabbbabbb'", // + "'Bbb'", // + "'BBBBbbbbbbbb'", // + "'ABCabcabc'", // + "'Aaa'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void testSet() throws SQLException { - final String query = "SELECT \"set_col\" FROM " + virtualSchemaJdbc + "." + MYSQL_STRING_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 CHAR"), // - List.of("'1'", "'1'", "NULL", "NULL")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void testRewrittenDivAndModFunctions() throws SQLException { + final String query = "SELECT DIV(\"int_col\",\"int_col\"), mod(\"int_col\",\"int_col\") FROM " + + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("a DECIMAL(10,0)", "b DECIMAL(10,0)"), // + List.of("1, 0", // + "1, 0", // + "null, null", // + "1, 0", // + "1, 0", // + "1, 0")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void testVarchar() throws SQLException { - final String query = "SELECT \"varchar_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_STRING_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 VARCHAR(100)"), // - List.of("'ab'", "'a'", "''", "NULL")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void testRewrittenSubStringFunction() throws SQLException { + final String query = "SELECT substring(\"varchar_col\" FROM 1 FOR 2) FROM " + virtualSchemaJdbc + "." + + MYSQL_SIMPLE_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("a VARCHAR(100)"), // + List.of("'a'", // + "'ab'", // + "'b'", // + "'bb'", // + "'ab'", // + "'a'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void testChar() throws SQLException { - final String query = "SELECT \"char_col\" FROM " + virtualSchemaJdbc + "." + MYSQL_STRING_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 CHAR(255)"), // - List.of("'asd24'", "'11111'", "''", "NULL")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void testOrderByLimit() throws SQLException { + final String query = "SELECT \"bool_col\", \"int_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_SIMPLE_TABLE + " ORDER BY \"int_col\" LIMIT 3"; + final ResultSet expected = getExpectedResultSet(List.of("a BOOLEAN", "b DECIMAL(10,0)"), // + List.of("true, -100", // + "false, -1", // + "true, 0")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } - } - @Nested - @DisplayName("Join test") - class JoinTest { @Test - void testInnerJoin() throws SQLException { - final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 + " a INNER JOIN " - + virtualSchemaJdbc + "." + TABLE_JOIN_2 + " b ON a.\"x\"=b.\"x\""; - final ResultSet expected = getExpectedResultSet( - List.of("x INT", "y VARCHAR(100)", "a INT", "b VARCHAR(100)"), // - List.of("2,'bbb', 2,'bbb'")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void testOrderByLimitOffset() throws SQLException { + final String query = "SELECT \"bool_col\", \"int_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_SIMPLE_TABLE + " ORDER BY \"int_col\" LIMIT 2 OFFSET 1"; + final ResultSet expected = getExpectedResultSet(List.of("a BOOLEAN", "b DECIMAL(10,0)"), // + List.of("false, -1", // + "true, 0")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void testInnerJoinWithProjection() throws SQLException { - final String query = "SELECT b.\"y\" || " + virtualSchemaJdbc + "." + TABLE_JOIN_1 + ".\"y\" FROM " - + virtualSchemaJdbc + "." + TABLE_JOIN_1 + " INNER JOIN " + virtualSchemaJdbc + "." + TABLE_JOIN_2 - + " b ON " + virtualSchemaJdbc + "." + TABLE_JOIN_1 + ".\"x\"=b.\"x\""; - final ResultSet expected = getExpectedResultSet(List.of("y VARCHAR(100)"), // - List.of("'bbbbbb'")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void testLeftShiftScalarFunction() { + createSourceTable(List.of("INT_COL_1", "INT_COL_2"), List.of("INT", "INT"), + new Object[][] { { 10, 3 } }); + this.virtualSchema = SETUP.createVirtualSchema(Collections.emptyMap(), MYSQL_SOURCE_SCHEMA); + final String query = "SELECT BIT_LSHIFT(INT_COL_1, INT_COL_2) FROM " + this.virtualSchema.getName() + + "." + MYSQL_SOURCE_TABLE; + assertVsQuery(query, table().row(80).matches(TypeMatchMode.NO_JAVA_TYPE_CHECK)); } - @Test - void testLeftJoin() throws SQLException { - final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 + " a LEFT OUTER JOIN " - + virtualSchemaJdbc + "." + TABLE_JOIN_2 + " b ON a.\"x\"=b.\"x\" ORDER BY a.\"x\""; - final ResultSet expected = getExpectedResultSet( - List.of("x INT", "y VARCHAR(100)", "a INT", "b VARCHAR(100)"), // - List.of("1, 'aaa', null, null", // - "2, 'bbb', 2, 'bbb'")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + private void createSourceTable(final List columnNames, final List types, + final Object[][] values) { + this.sourceSchema = SETUP.getMySqlObjectFactory().createSchema(MYSQL_SOURCE_SCHEMA); + final Table table = this.sourceSchema.createTable(MYSQL_SOURCE_TABLE, columnNames, types); + for (final Object[] value : values) { + table.insert(value); + } } - @Test - void testRightJoin() throws SQLException { - final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 + " a RIGHT OUTER JOIN " - + virtualSchemaJdbc + "." + TABLE_JOIN_2 + " b ON a.\"x\"=b.\"x\" ORDER BY a.\"x\""; - final ResultSet expected = getExpectedResultSet( - List.of("x INT", "y VARCHAR(100)", "a INT", "b VARCHAR(100)"), // - List.of("null, null, 3, 'ccc'", // - "2, 'bbb', 2, 'bbb'")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + private ResultSet query(final String sql) throws SQLException { + return SETUP.getExasolStatement().executeQuery(sql); } @Test - void testFullOuterJoin() throws SQLException { - final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 + " a FULL OUTER JOIN " - + virtualSchemaJdbc + "." + TABLE_JOIN_2 + " b ON a.\"x\"=b.\"x\" ORDER BY a.\"x\""; - final ResultSet expected = getExpectedResultSet( - List.of("x INT", "y VARCHAR(100)", "a INT", "b VARCHAR(100)"), // - List.of("1, 'aaa', null, null", // - "2, 'bbb', 2, 'bbb'", // - "null, null, 3, 'ccc'")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void testRightShiftScalarFunction() { + createSourceTable(List.of("INT_COL_1", "INT_COL_2"), List.of("INT", "INT"), + new Object[][] { { 10, 3 } }); + this.virtualSchema = SETUP.createVirtualSchema(Collections.emptyMap(), MYSQL_SOURCE_SCHEMA); + final String query = "SELECT BIT_RSHIFT(INT_COL_1, INT_COL_2) FROM " + this.virtualSchema.getName() + + "." + MYSQL_SOURCE_TABLE; + assertVsQuery(query, table().row(1).matches(TypeMatchMode.NO_JAVA_TYPE_CHECK)); } - @Test - void testRightJoinWithComplexCondition() throws SQLException { - final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 + " a RIGHT OUTER JOIN " - + virtualSchemaJdbc + "." + TABLE_JOIN_2 - + " b ON a.\"x\"||a.\"y\"=b.\"x\"||b.\"y\" ORDER BY a.\"x\""; - final ResultSet expected = getExpectedResultSet( - List.of("x INT", "y VARCHAR(100)", "a INT", "b VARCHAR(100)"), // - List.of("null, null, 3, 'ccc'", // - "2, 'bbb', 2, 'bbb'")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + private void assertVsQuery(final String sql, final Matcher expected) { + try { + assertThat(query(sql), expected); + } catch (final SQLException exception) { + fail("Unable to run assertion query: " + sql + "\nCaused by: " + exception.getMessage()); + } } @Test - void testFullOuterJoinWithComplexCondition() throws SQLException { - final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 + " a FULL OUTER JOIN " - + virtualSchemaJdbc + "." + TABLE_JOIN_2 + " b ON a.\"x\"-b.\"x\"=0 ORDER BY a.\"x\""; - final ResultSet expected = getExpectedResultSet( - List.of("x INT", "y VARCHAR(100)", "a INT", "b VARCHAR(100)"), // - List.of("1, 'aaa', null, null", // - "2, 'bbb', 2, 'bbb'", // - "null, null, 3, 'ccc'")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - } + void testHourScalarFunction() { + createSourceTable(List.of("TIMESTAMP_COL"), List.of("TIMESTAMP"), + new Object[][] { { "2021-02-16 11:48:01" } }); + this.virtualSchema = SETUP.createVirtualSchema(Collections.emptyMap(), MYSQL_SOURCE_SCHEMA); + final String query = "SELECT HOUR(timestamp_col) FROM " + this.virtualSchema.getName() + "." + + MYSQL_SOURCE_TABLE; + assertVsQuery(query, table().row(11).matches(TypeMatchMode.NO_JAVA_TYPE_CHECK)); + } + + @Nested + @DisplayName("Datatype tests") + class DatatypeTest { + @Test + void testBit() throws SQLException { + final String query = "SELECT \"BiT_Col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 BOOLEAN"), // + List.of("true", "true", "false", "false")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testTinyInt() throws SQLException { + final String query = "SELECT \"tinyint_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 DECIMAL(3,0)"), // + List.of("127", "-127", "0", "0")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testBoolean() throws SQLException { + final String query = "SELECT \"BOOL_COL\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 BOOLEAN"), // + List.of("true", "false", "true", "false")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testSmallInt() throws SQLException { + final String query = "SELECT \"smallint_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 DECIMAL(5,0)"), // + List.of("32767", "-32768", "0", "0")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testMediumInt() throws SQLException { + final String query = "SELECT \"mediumint_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 DECIMAL(7,0)"), // + List.of("8388607", "-8388608", "0", "0")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testInt() throws SQLException { + final String query = "SELECT \"int_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 DECIMAL(10,0)"), // + List.of("2147483647", "-2147483648", "0", "0")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testBigInt() throws SQLException { + final String query = "SELECT \"bigint_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 DECIMAL(19,0)"), // + List.of("9223372036854775807", "-9223372036854775808", "0", "0")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testDecimal() throws SQLException { + final String query = "SELECT \"decimal_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 DECIMAL(5,2)"), // + List.of("999.99", "-999.99", "0", "0")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testFloat() throws SQLException { + final String query = "SELECT \"float_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 DOUBLE PRECISION"), // + List.of("999.0001", "-999.9999", "0", "0")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testDouble() throws SQLException { + final String query = "SELECT \"double_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 DOUBLE PRECISION"), // + List.of("999.00009", "-999.9999", "0", "0")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testDate() throws SQLException { + final String query = "SELECT \"date_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 DATE"), // + List.of("'1000-01-01'", "'9999-12-31'", "null", "null")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testDatetime() throws SQLException { + final String query = "SELECT \"datetime_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 TIMESTAMP"), // + List.of("'1000-01-01 00:00:00'", "'9999-12-31 22:59:59'", "null", "null")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testTimestamp() throws SQLException { + final String query = "SELECT \"timestamp_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 TIMESTAMP"), // + List.of("'1970-01-01 00:00:01.000000'", "'2037-01-19 03:14:08.0'", "null", + "null")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testTime() throws SQLException { + final String query = "SELECT \"time_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 TIMESTAMP"), // + List.of("'1970-01-01 16:59:59.000000'", "'1970-01-01 05:34:13.000000'", "null", + "null")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testYear() throws SQLException { + final String query = "SELECT \"year_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 DATE"), // + List.of("'1901-01-01'", "'2155-01-01'", "'1901-01-01'", "'2069-01-01'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testUnsupported() { + final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + MYSQL_STRING_DATATYPES_TABLE; + assertDoesNotThrow(() -> getActualResultSet(query)); + } + + @Test + void testTinyText() throws SQLException { + final String query = "SELECT \"tinytext_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_STRING_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 VARCHAR(100)"), // + List.of("'a'", "'b'", "'aaaaaaaaaaaaa'", "'a'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testText() throws SQLException { + final String query = "SELECT \"text_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_STRING_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 VARCHAR(65535)"), // + List.of("'text'", "'text2'", "'text3'", "'text'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testMediumText() throws SQLException { + final String query = "SELECT \"mediumtext_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_STRING_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 VARCHAR(2000000)"), // + List.of("'mediumtext'", "'mediumtext2'", "NULL", "NULL")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testLongText() throws SQLException { + final String query = "SELECT \"longtext_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_STRING_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 VARCHAR(2000000)"), // + List.of("'longtext'", "'longtext2'", "NULL", "NULL")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testEnum() throws SQLException { + final String query = "SELECT \"enum_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_STRING_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 CHAR"), // + List.of("'1'", "'2'", "'3'", "NULL")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testSet() throws SQLException { + final String query = "SELECT \"set_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_STRING_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 CHAR"), // + List.of("'1'", "'1'", "NULL", "NULL")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testVarchar() throws SQLException { + final String query = "SELECT \"varchar_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_STRING_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 VARCHAR(100)"), // + List.of("'ab'", "'a'", "''", "NULL")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testChar() throws SQLException { + final String query = "SELECT \"char_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_STRING_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 CHAR(255)"), // + List.of("'asd24'", "'11111'", "''", "NULL")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + } + + @Nested + @DisplayName("Join test") + class JoinTest { + @Test + void testInnerJoin() throws SQLException { + final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 + + " a INNER JOIN " + virtualSchemaJdbc + "." + TABLE_JOIN_2 + + " b ON a.\"x\"=b.\"x\""; + final ResultSet expected = getExpectedResultSet( + List.of("x INT", "y VARCHAR(100)", "a INT", "b VARCHAR(100)"), // + List.of("2,'bbb', 2,'bbb'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testInnerJoinWithProjection() throws SQLException { + final String query = "SELECT b.\"y\" || " + virtualSchemaJdbc + "." + TABLE_JOIN_1 + + ".\"y\" FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 + " INNER JOIN " + + virtualSchemaJdbc + "." + TABLE_JOIN_2 + " b ON " + virtualSchemaJdbc + "." + + TABLE_JOIN_1 + ".\"x\"=b.\"x\""; + final ResultSet expected = getExpectedResultSet(List.of("y VARCHAR(100)"), // + List.of("'bbbbbb'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testLeftJoin() throws SQLException { + final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 + + " a LEFT OUTER JOIN " + virtualSchemaJdbc + "." + TABLE_JOIN_2 + + " b ON a.\"x\"=b.\"x\" ORDER BY a.\"x\""; + final ResultSet expected = getExpectedResultSet( + List.of("x INT", "y VARCHAR(100)", "a INT", "b VARCHAR(100)"), // + List.of("1, 'aaa', null, null", // + "2, 'bbb', 2, 'bbb'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testRightJoin() throws SQLException { + final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 + + " a RIGHT OUTER JOIN " + virtualSchemaJdbc + "." + TABLE_JOIN_2 + + " b ON a.\"x\"=b.\"x\" ORDER BY a.\"x\""; + final ResultSet expected = getExpectedResultSet( + List.of("x INT", "y VARCHAR(100)", "a INT", "b VARCHAR(100)"), // + List.of("null, null, 3, 'ccc'", // + "2, 'bbb', 2, 'bbb'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testFullOuterJoin() throws SQLException { + final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 + + " a FULL OUTER JOIN " + virtualSchemaJdbc + "." + TABLE_JOIN_2 + + " b ON a.\"x\"=b.\"x\" ORDER BY a.\"x\""; + final ResultSet expected = getExpectedResultSet( + List.of("x INT", "y VARCHAR(100)", "a INT", "b VARCHAR(100)"), // + List.of("1, 'aaa', null, null", // + "2, 'bbb', 2, 'bbb'", // + "null, null, 3, 'ccc'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testRightJoinWithComplexCondition() throws SQLException { + final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 + + " a RIGHT OUTER JOIN " + virtualSchemaJdbc + "." + TABLE_JOIN_2 + + " b ON a.\"x\"||a.\"y\"=b.\"x\"||b.\"y\" ORDER BY a.\"x\""; + final ResultSet expected = getExpectedResultSet( + List.of("x INT", "y VARCHAR(100)", "a INT", "b VARCHAR(100)"), // + List.of("null, null, 3, 'ccc'", // + "2, 'bbb', 2, 'bbb'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testFullOuterJoinWithComplexCondition() throws SQLException { + final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 + + " a FULL OUTER JOIN " + virtualSchemaJdbc + "." + TABLE_JOIN_2 + + " b ON a.\"x\"-b.\"x\"=0 ORDER BY a.\"x\""; + final ResultSet expected = getExpectedResultSet( + List.of("x INT", "y VARCHAR(100)", "a INT", "b VARCHAR(100)"), // + List.of("1, 'aaa', null, null", // + "2, 'bbb', 2, 'bbb'", // + "null, null, 3, 'ccc'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + } } \ No newline at end of file diff --git a/src/test/java/com/exasol/adapter/dialects/mysql/MySQLVirtualSchemaIntegrationTestSetup.java b/src/test/java/com/exasol/adapter/dialects/mysql/MySQLVirtualSchemaIntegrationTestSetup.java index 4b3dad7..caf017f 100644 --- a/src/test/java/com/exasol/adapter/dialects/mysql/MySQLVirtualSchemaIntegrationTestSetup.java +++ b/src/test/java/com/exasol/adapter/dialects/mysql/MySQLVirtualSchemaIntegrationTestSetup.java @@ -36,13 +36,12 @@ public class MySQLVirtualSchemaIntegrationTestSetup implements Closeable { private static final String ADAPTER_SCRIPT_EXASOL = "ADAPTER_SCRIPT_EXASOL"; private static final Logger LOGGER = Logger.getLogger(MySQLVirtualSchemaIntegrationTestSetup.class.getName()); - private static final boolean USE_JACOCO = true; private final Statement mySqlStatement; private final MySQLContainer mySqlContainer = new MySQLContainer<>(MYSQL_DOCKER_IMAGE_REFERENCE) .withUsername("root").withPassword(""); private final ExasolContainer> exasolContainer = new ExasolContainer<>("8.23.0") .withRequiredServices(ExasolService.BUCKETFS, ExasolService.UDF).withReuse(true); - private final Connection exasolConection; + private final Connection exasolConnection; private final Statement exasolStatement; private final AdapterScript adapterScript; private final ConnectionDefinition connectionDefinition; @@ -58,17 +57,17 @@ public class MySQLVirtualSchemaIntegrationTestSetup implements Closeable { final Bucket bucket = this.exasolContainer.getDefaultBucket(); uploadDriverToBucket(bucket); uploadVsJarToBucket(bucket); - this.exasolConection = this.exasolContainer.createConnection(""); - this.exasolStatement = this.exasolConection.createStatement(); + this.exasolConnection = this.exasolContainer.createConnection(""); + this.exasolStatement = this.exasolConnection.createStatement(); + exasolStatement.execute("CONTROL SET TRACE LEVEL NOTICE WITH LOG TIMEOUT 0;"); this.mySqlConnection = this.mySqlContainer.createConnection(""); this.mySqlStatement = this.mySqlConnection.createStatement(); - final ExasolObjectConfiguration.Builder builder = ExasolObjectConfiguration.builder(); - if (USE_JACOCO) { - final UdfTestSetup udfTestSetup = new UdfTestSetup(getTestHostIpFromInsideExasol(), - this.exasolContainer.getDefaultBucket(), this.exasolConection); - builder.withJvmOptions(udfTestSetup.getJvmOptions()); - } - this.exasolFactory = new ExasolObjectFactory(this.exasolContainer.createConnection(""), builder.build()); + + final UdfTestSetup udfTestSetup = new UdfTestSetup(getTestHostIpFromInsideExasol(), + this.exasolContainer.getDefaultBucket(), this.exasolConnection); + this.exasolFactory = new ExasolObjectFactory(this.exasolContainer.createConnection(""), + ExasolObjectConfiguration.builder().withJvmOptions(udfTestSetup.getJvmOptions()).build()); + final ExasolSchema exasolSchema = this.exasolFactory.createSchema(SCHEMA_EXASOL); this.mySqlObjectFactory = new MySqlObjectFactory(this.mySqlConnection); this.adapterScript = createAdapterScript(exasolSchema); @@ -169,7 +168,7 @@ public ColumnInspector getColumnInspector(final String catalogName) { public void close() { try { this.exasolStatement.close(); - this.exasolConection.close(); + this.exasolConnection.close(); this.mySqlStatement.close(); this.mySqlConnection.close(); this.exasolContainer.stop(); From d640fdc3b25c198022f30493d168a542ab255d46 Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Tue, 9 Jan 2024 15:52:27 +0100 Subject: [PATCH 04/17] update to common-jdbc v12 --- .github/workflows/broken_links_checker.yml | 2 + .github/workflows/ci-build-next-java.yml | 8 +- .github/workflows/dependencies_check.yml | 11 ++- .../release_droid_print_quick_checksum.yml | 11 ++- ...ase_droid_upload_github_release_assets.yml | 11 ++- dependencies.md | 76 ++++++++++--------- doc/changes/changelog.md | 1 + doc/changes/changes_5.0.0.md | 34 +++++++++ doc/user_guide/mysql_user_guide.md | 2 +- pk_generated_parent.pom | 30 +++++++- pom.xml | 8 +- .../mysql/MySQLColumnMetadataReader.java | 4 +- .../mysql/IntegrationTestConstants.java | 2 +- .../mysql/MySQLScalarFunctionsIT.java | 10 +-- 14 files changed, 141 insertions(+), 69 deletions(-) create mode 100644 doc/changes/changes_5.0.0.md diff --git a/.github/workflows/broken_links_checker.yml b/.github/workflows/broken_links_checker.yml index 82ec1cd..0fbcad5 100644 --- a/.github/workflows/broken_links_checker.yml +++ b/.github/workflows/broken_links_checker.yml @@ -1,3 +1,5 @@ +# Generated by Project Keeper +# https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/broken_links_checker.yml name: Broken Links Checker on: diff --git a/.github/workflows/ci-build-next-java.yml b/.github/workflows/ci-build-next-java.yml index 7cbab08..e3acdb7 100644 --- a/.github/workflows/ci-build-next-java.yml +++ b/.github/workflows/ci-build-next-java.yml @@ -1,5 +1,6 @@ +# Generated by Project Keeper +# https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/ci-build-next-java.yml name: CI Build next Java - on: push: branches: @@ -18,7 +19,7 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 17 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: "temurin" java-version: 17 @@ -26,8 +27,9 @@ jobs: - name: Run tests and build with Maven run: | mvn --batch-mode --update-snapshots clean package -DtrimStackTrace=false \ + -Djava.version=17 \ -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn - - name: Publish Test Report + - name: Publish Test Report for Java 17 uses: scacap/action-surefire-report@v1 if: ${{ always() && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' }} with: diff --git a/.github/workflows/dependencies_check.yml b/.github/workflows/dependencies_check.yml index 4b6eadf..87b64ba 100644 --- a/.github/workflows/dependencies_check.yml +++ b/.github/workflows/dependencies_check.yml @@ -1,5 +1,6 @@ +# Generated by Project Keeper +# https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/dependencies_check.yml name: Report Security Issues for Repository - on: workflow_dispatch: schedule: @@ -13,11 +14,13 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up JDK 11 - uses: actions/setup-java@v3 + - name: Set up JDKs + uses: actions/setup-java@v4 with: distribution: "temurin" - java-version: 11 + java-version: | + 11 + 17 cache: "maven" - name: Generate ossindex report diff --git a/.github/workflows/release_droid_print_quick_checksum.yml b/.github/workflows/release_droid_print_quick_checksum.yml index aed4444..86979cd 100644 --- a/.github/workflows/release_droid_print_quick_checksum.yml +++ b/.github/workflows/release_droid_print_quick_checksum.yml @@ -1,5 +1,6 @@ +# Generated by Project Keeper +# https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/release_droid_print_quick_checksum.yml name: Release Droid - Print Quick Checksum - on: workflow_dispatch: @@ -11,11 +12,13 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Set up JDK 11 - uses: actions/setup-java@v3 + - name: Set up JDKs + uses: actions/setup-java@v4 with: distribution: "temurin" - java-version: 11 + java-version: | + 11 + 17 cache: "maven" - name: Build with Maven skipping tests run: mvn --batch-mode clean verify -DskipTests diff --git a/.github/workflows/release_droid_upload_github_release_assets.yml b/.github/workflows/release_droid_upload_github_release_assets.yml index 7ae8bbb..b19f7cf 100644 --- a/.github/workflows/release_droid_upload_github_release_assets.yml +++ b/.github/workflows/release_droid_upload_github_release_assets.yml @@ -1,5 +1,6 @@ +# Generated by Project Keeper +# https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/release_droid_upload_github_release_assets.yml name: Release Droid - Upload GitHub Release Assets - on: workflow_dispatch: inputs: @@ -15,11 +16,13 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Set up JDK 11 - uses: actions/setup-java@v3 + - name: Set up JDKs + uses: actions/setup-java@v4 with: distribution: "temurin" - java-version: 11 + java-version: | + 11 + 17 cache: "maven" - name: Build with Maven skipping tests run: mvn --batch-mode clean verify -DskipTests diff --git a/dependencies.md b/dependencies.md index 5004e8f..fdca9b0 100644 --- a/dependencies.md +++ b/dependencies.md @@ -32,22 +32,23 @@ | Dependency | License | | ------------------------------------------------------- | --------------------------------- | | [SonarQube Scanner for Maven][27] | [GNU LGPL 3][28] | -| [Apache Maven Compiler Plugin][29] | [Apache-2.0][30] | -| [Apache Maven Enforcer Plugin][31] | [Apache-2.0][30] | -| [Maven Flatten Plugin][32] | [Apache Software Licenese][30] | -| [org.sonatype.ossindex.maven:ossindex-maven-plugin][33] | [ASL2][34] | -| [Maven Surefire Plugin][35] | [Apache-2.0][30] | -| [Versions Maven Plugin][36] | [Apache License, Version 2.0][30] | -| [duplicate-finder-maven-plugin Maven Mojo][37] | [Apache License 2.0][38] | -| [Apache Maven Assembly Plugin][39] | [Apache-2.0][30] | -| [Apache Maven JAR Plugin][40] | [Apache License, Version 2.0][30] | -| [Artifact reference checker and unifier][41] | [MIT License][42] | -| [Apache Maven Dependency Plugin][43] | [Apache-2.0][30] | -| [Project Keeper Maven plugin][44] | [The MIT License][45] | -| [Maven Failsafe Plugin][46] | [Apache-2.0][30] | -| [JaCoCo :: Maven Plugin][47] | [Eclipse Public License 2.0][26] | -| [error-code-crawler-maven-plugin][48] | [MIT License][49] | -| [Reproducible Build Maven Plugin][50] | [Apache 2.0][34] | +| [Apache Maven Toolchains Plugin][29] | [Apache License, Version 2.0][30] | +| [Apache Maven Compiler Plugin][31] | [Apache-2.0][30] | +| [Apache Maven Enforcer Plugin][32] | [Apache-2.0][30] | +| [Maven Flatten Plugin][33] | [Apache Software Licenese][30] | +| [org.sonatype.ossindex.maven:ossindex-maven-plugin][34] | [ASL2][35] | +| [Maven Surefire Plugin][36] | [Apache-2.0][30] | +| [Versions Maven Plugin][37] | [Apache License, Version 2.0][30] | +| [duplicate-finder-maven-plugin Maven Mojo][38] | [Apache License 2.0][39] | +| [Apache Maven Assembly Plugin][40] | [Apache-2.0][30] | +| [Apache Maven JAR Plugin][41] | [Apache License, Version 2.0][30] | +| [Artifact reference checker and unifier][42] | [MIT License][43] | +| [Apache Maven Dependency Plugin][44] | [Apache-2.0][30] | +| [Project Keeper Maven plugin][45] | [The MIT License][46] | +| [Maven Failsafe Plugin][47] | [Apache-2.0][30] | +| [JaCoCo :: Maven Plugin][48] | [Eclipse Public License 2.0][26] | +| [error-code-crawler-maven-plugin][49] | [MIT License][50] | +| [Reproducible Build Maven Plugin][51] | [Apache 2.0][35] | [0]: https://github.com/exasol/virtual-schema-common-jdbc/ [1]: https://github.com/exasol/virtual-schema-common-jdbc/blob/main/LICENSE @@ -78,25 +79,26 @@ [26]: https://www.eclipse.org/legal/epl-2.0/ [27]: http://sonarsource.github.io/sonar-scanner-maven/ [28]: http://www.gnu.org/licenses/lgpl.txt -[29]: https://maven.apache.org/plugins/maven-compiler-plugin/ +[29]: https://maven.apache.org/plugins/maven-toolchains-plugin/ [30]: https://www.apache.org/licenses/LICENSE-2.0.txt -[31]: https://maven.apache.org/enforcer/maven-enforcer-plugin/ -[32]: https://www.mojohaus.org/flatten-maven-plugin/ -[33]: https://sonatype.github.io/ossindex-maven/maven-plugin/ -[34]: http://www.apache.org/licenses/LICENSE-2.0.txt -[35]: https://maven.apache.org/surefire/maven-surefire-plugin/ -[36]: https://www.mojohaus.org/versions/versions-maven-plugin/ -[37]: https://basepom.github.io/duplicate-finder-maven-plugin -[38]: http://www.apache.org/licenses/LICENSE-2.0.html -[39]: https://maven.apache.org/plugins/maven-assembly-plugin/ -[40]: https://maven.apache.org/plugins/maven-jar-plugin/ -[41]: https://github.com/exasol/artifact-reference-checker-maven-plugin/ -[42]: https://github.com/exasol/artifact-reference-checker-maven-plugin/blob/main/LICENSE -[43]: https://maven.apache.org/plugins/maven-dependency-plugin/ -[44]: https://github.com/exasol/project-keeper/ -[45]: https://github.com/exasol/project-keeper/blob/main/LICENSE -[46]: https://maven.apache.org/surefire/maven-failsafe-plugin/ -[47]: https://www.jacoco.org/jacoco/trunk/doc/maven.html -[48]: https://github.com/exasol/error-code-crawler-maven-plugin/ -[49]: https://github.com/exasol/error-code-crawler-maven-plugin/blob/main/LICENSE -[50]: http://zlika.github.io/reproducible-build-maven-plugin +[31]: https://maven.apache.org/plugins/maven-compiler-plugin/ +[32]: https://maven.apache.org/enforcer/maven-enforcer-plugin/ +[33]: https://www.mojohaus.org/flatten-maven-plugin/ +[34]: https://sonatype.github.io/ossindex-maven/maven-plugin/ +[35]: http://www.apache.org/licenses/LICENSE-2.0.txt +[36]: https://maven.apache.org/surefire/maven-surefire-plugin/ +[37]: https://www.mojohaus.org/versions/versions-maven-plugin/ +[38]: https://basepom.github.io/duplicate-finder-maven-plugin +[39]: http://www.apache.org/licenses/LICENSE-2.0.html +[40]: https://maven.apache.org/plugins/maven-assembly-plugin/ +[41]: https://maven.apache.org/plugins/maven-jar-plugin/ +[42]: https://github.com/exasol/artifact-reference-checker-maven-plugin/ +[43]: https://github.com/exasol/artifact-reference-checker-maven-plugin/blob/main/LICENSE +[44]: https://maven.apache.org/plugins/maven-dependency-plugin/ +[45]: https://github.com/exasol/project-keeper/ +[46]: https://github.com/exasol/project-keeper/blob/main/LICENSE +[47]: https://maven.apache.org/surefire/maven-failsafe-plugin/ +[48]: https://www.jacoco.org/jacoco/trunk/doc/maven.html +[49]: https://github.com/exasol/error-code-crawler-maven-plugin/ +[50]: https://github.com/exasol/error-code-crawler-maven-plugin/blob/main/LICENSE +[51]: http://zlika.github.io/reproducible-build-maven-plugin diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index 36649d6..cd146e2 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,5 +1,6 @@ # Changes +* [5.0.0](changes_5.0.0.md) * [4.1.4](changes_4.1.4.md) * [4.1.3](changes_4.1.3.md) * [4.1.2](changes_4.1.2.md) diff --git a/doc/changes/changes_5.0.0.md b/doc/changes/changes_5.0.0.md new file mode 100644 index 0000000..cd84848 --- /dev/null +++ b/doc/changes/changes_5.0.0.md @@ -0,0 +1,34 @@ +# Virtual Schema for MySQL 5.0.0, released 2024-??-?? + +Code name: + +## Summary + +## Features + +* ISSUE_NUMBER: description + +## Dependency Updates + +### Compile Dependency Updates + +* Updated `com.exasol:virtual-schema-common-jdbc:11.0.2` to `12.0.0` + +### Test Dependency Updates + +* Updated `com.exasol:virtual-schema-common-jdbc:11.0.2` to `12.0.0` +* Updated `com.mysql:mysql-connector-j:8.1.0` to `8.2.0` +* Updated `org.jacoco:org.jacoco.agent:0.8.10` to `0.8.11` + +### Plugin Dependency Updates + +* Updated `com.exasol:error-code-crawler-maven-plugin:1.3.0` to `1.3.1` +* Updated `com.exasol:project-keeper-maven-plugin:2.9.12` to `3.0.0` +* Updated `org.apache.maven.plugins:maven-dependency-plugin:3.6.0` to `3.6.1` +* Updated `org.apache.maven.plugins:maven-enforcer-plugin:3.4.0` to `3.4.1` +* Updated `org.apache.maven.plugins:maven-failsafe-plugin:3.1.2` to `3.2.3` +* Updated `org.apache.maven.plugins:maven-surefire-plugin:3.1.2` to `3.2.3` +* Added `org.apache.maven.plugins:maven-toolchains-plugin:3.1.0` +* Updated `org.codehaus.mojo:versions-maven-plugin:2.16.0` to `2.16.2` +* Updated `org.jacoco:jacoco-maven-plugin:0.8.10` to `0.8.11` +* Updated `org.sonarsource.scanner.maven:sonar-maven-plugin:3.9.1.2184` to `3.10.0.2594` diff --git a/doc/user_guide/mysql_user_guide.md b/doc/user_guide/mysql_user_guide.md index af01435..d5db1fe 100644 --- a/doc/user_guide/mysql_user_guide.md +++ b/doc/user_guide/mysql_user_guide.md @@ -43,7 +43,7 @@ The SQL statement below creates the adapter script, defines the Java class that --/ CREATE OR REPLACE JAVA ADAPTER SCRIPT SCHEMA_FOR_VS_SCRIPT.ADAPTER_SCRIPT_MYSQL AS %scriptclass com.exasol.adapter.RequestDispatcher; - %jar /buckets/bfsdefault/default/virtual-schema-dist-11.0.2-mysql-4.1.4.jar; + %jar /buckets/bfsdefault/default/virtual-schema-dist-12.0.0-mysql-5.0.0.jar; %jar /buckets/bfsdefault/default/mysql-connector-java-.jar; / ; diff --git a/pk_generated_parent.pom b/pk_generated_parent.pom index cbd4b05..0f85b08 100644 --- a/pk_generated_parent.pom +++ b/pk_generated_parent.pom @@ -3,12 +3,14 @@ 4.0.0 com.exasol mysql-virtual-schema-generated-parent - 4.1.4 + 5.0.0 pom UTF-8 UTF-8 11 + exasol + https://sonarcloud.io @@ -47,6 +49,25 @@ sonar-maven-plugin 3.10.0.2594 + + org.apache.maven.plugins + maven-toolchains-plugin + 3.1.0 + + + + toolchain + + + + + + + ${java.version} + + + + org.apache.maven.plugins maven-compiler-plugin @@ -77,6 +98,9 @@ 3.6.3 + + 17 + @@ -124,7 +148,7 @@ org.apache.maven.plugins maven-surefire-plugin - 3.2.2 + 3.2.3 @@ -248,7 +272,7 @@ org.apache.maven.plugins maven-failsafe-plugin - 3.2.2 + 3.2.3 -Djava.util.logging.config.file=src/test/resources/logging.properties ${argLine} diff --git a/pom.xml b/pom.xml index 1d5ab68..cd89e8f 100644 --- a/pom.xml +++ b/pom.xml @@ -2,12 +2,12 @@ 4.0.0 mysql-virtual-schema - 4.1.4 + 5.0.0 Virtual Schema for MySQL Virtual Schema for MySQL https://github.com/exasol/mysql-virtual-schema/ - 11.0.2 + 12.0.0 1.19.0 @@ -136,7 +136,7 @@ com.exasol project-keeper-maven-plugin - 2.9.17 + 3.0.0 @@ -157,7 +157,7 @@ mysql-virtual-schema-generated-parent com.exasol - 4.1.4 + 5.0.0 pk_generated_parent.pom diff --git a/src/main/java/com/exasol/adapter/dialects/mysql/MySQLColumnMetadataReader.java b/src/main/java/com/exasol/adapter/dialects/mysql/MySQLColumnMetadataReader.java index a1768a6..a5362ef 100644 --- a/src/main/java/com/exasol/adapter/dialects/mysql/MySQLColumnMetadataReader.java +++ b/src/main/java/com/exasol/adapter/dialects/mysql/MySQLColumnMetadataReader.java @@ -44,9 +44,7 @@ public DataType mapJdbcType(final JDBCTypeDescription jdbcTypeDescription) { private DataType convertVarChar(final JDBCTypeDescription jdbcTypeDescription) { final int size = getVarcharSize(jdbcTypeDescription); - final int octetLength = jdbcTypeDescription.getByteSize(); - final DataType.ExaCharset charset = (octetLength == size) ? DataType.ExaCharset.ASCII - : DataType.ExaCharset.UTF8; + final DataType.ExaCharset charset = DataType.ExaCharset.UTF8; if (size <= DataType.MAX_EXASOL_VARCHAR_SIZE) { final int precision = getVarcharPrecision(size); return DataType.createVarChar(precision, charset); diff --git a/src/test/java/com/exasol/adapter/dialects/mysql/IntegrationTestConstants.java b/src/test/java/com/exasol/adapter/dialects/mysql/IntegrationTestConstants.java index bf95192..bdf5ce1 100644 --- a/src/test/java/com/exasol/adapter/dialects/mysql/IntegrationTestConstants.java +++ b/src/test/java/com/exasol/adapter/dialects/mysql/IntegrationTestConstants.java @@ -3,7 +3,7 @@ import java.nio.file.Path; public final class IntegrationTestConstants { - public static final String VIRTUAL_SCHEMAS_JAR_NAME_AND_VERSION = "virtual-schema-dist-11.0.2-mysql-4.1.4.jar"; + public static final String VIRTUAL_SCHEMAS_JAR_NAME_AND_VERSION = "virtual-schema-dist-12.0.0-mysql-5.0.0.jar"; public static final String MYSQL_DOCKER_IMAGE_REFERENCE = "mysql:8.2.0"; public static final Path PATH_TO_VIRTUAL_SCHEMAS_JAR = Path.of("target", VIRTUAL_SCHEMAS_JAR_NAME_AND_VERSION); public static final String SCHEMA_EXASOL = "SCHEMA_EXASOL"; diff --git a/src/test/java/com/exasol/adapter/dialects/mysql/MySQLScalarFunctionsIT.java b/src/test/java/com/exasol/adapter/dialects/mysql/MySQLScalarFunctionsIT.java index a2bea17..d160d6f 100644 --- a/src/test/java/com/exasol/adapter/dialects/mysql/MySQLScalarFunctionsIT.java +++ b/src/test/java/com/exasol/adapter/dialects/mysql/MySQLScalarFunctionsIT.java @@ -77,7 +77,7 @@ public VirtualSchemaTestSetupProvider getVirtualSchemaTestSetupProvider() { } final Table table = tableBuilder.build(); for (final List row : tableRequest.getRows()) { - List newRow = preprocessValuesForMySql(row); + final List newRow = preprocessValuesForMySql(row); table.insert(newRow.toArray()); } } @@ -87,11 +87,11 @@ public VirtualSchemaTestSetupProvider getVirtualSchemaTestSetupProvider() { }; } - private List preprocessValuesForMySql(List row) { - List newRow = new ArrayList<>(row.size()); - for (Object col : row) { + private List preprocessValuesForMySql(final List row) { + final List newRow = new ArrayList<>(row.size()); + for (final Object col : row) { if (col instanceof Timestamp) { - Timestamp timestamp = (Timestamp) col; + final Timestamp timestamp = (Timestamp) col; final String formatted = timestamp.toInstant().atOffset(ZoneOffset.UTC) .format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")); newRow.add(formatted); From 9b8e9d03bddc473d9e1789708ceb09d02c6cd57b Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Thu, 11 Jan 2024 13:42:43 +0100 Subject: [PATCH 05/17] adapt tests to changes in vs-common-jdbc --- .../dialects/mysql/MySQLSqlDialectIT.java | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java b/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java index 3a5e202..13fa9e2 100644 --- a/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java +++ b/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java @@ -4,7 +4,7 @@ import static com.exasol.matcher.ResultSetMatcher.matchesResultSet; import static com.exasol.matcher.ResultSetStructureMatcher.table; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.matchesRegex; +import static org.hamcrest.Matchers.containsString; import static org.junit.jupiter.api.Assertions.*; import java.io.IOException; @@ -159,20 +159,16 @@ void importDataTypesFromResultSet() throws SQLException { Assume.assumeTrue(runCharsetTest()); final String query = setupMySQLTableWithLatin1AndVirtualSchemaWithStrategy( DataTypeDetection.Strategy.FROM_RESULT_SET); - final ResultSet actual = getActualResultSet(query); - final ResultSet expected = getExpectedResultSet(List.of("c1 CHAR(1) UTF8", "c2 CHAR(1) UTF8"), // - List.of(SPECIAL_CHAR_QUOTED + ", " + SPECIAL_CHAR_QUOTED)); - assertThat(actual, matchesResultSet(expected)); + // final ResultSet actual = getActualResultSet(query); + final Exception exception = assertThrows(SQLDataException.class, () -> getActualResultSet(query)); + assertThat(exception.getMessage(), containsString("E-VSCJDBC-46")); } @Test void importDataTypesExasolCalculated() throws SQLException { Assume.assumeTrue(runCharsetTest()); - final String query = setupMySQLTableWithLatin1AndVirtualSchemaWithStrategy( - DataTypeDetection.Strategy.EXASOL_CALCULATED); - final Exception exception = assertThrows(SQLException.class, () -> getActualResultSet(query)); - assertThat(exception.getMessage(), - matchesRegex("ETL-3009: .*Charset conversion from 'UTF-8' to 'ASCII' failed.*")); + assertDoesNotThrow(() -> setupMySQLTableWithLatin1AndVirtualSchemaWithStrategy( + DataTypeDetection.Strategy.EXASOL_CALCULATED)); } private boolean runCharsetTest() { From 1b0b731369064b23d9825cd435b9141bf0b019b3 Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Thu, 11 Jan 2024 14:11:47 +0100 Subject: [PATCH 06/17] adapt tests to changes in vs-common-jdbc (after fix in datatype validator) --- .../adapter/dialects/mysql/MySQLSqlDialectIT.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java b/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java index 13fa9e2..e90ccfb 100644 --- a/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java +++ b/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java @@ -157,11 +157,13 @@ private static void createMySqlStringTable(final Schema mySqlSchema) { @Test void importDataTypesFromResultSet() throws SQLException { Assume.assumeTrue(runCharsetTest()); - final String query = setupMySQLTableWithLatin1AndVirtualSchemaWithStrategy( - DataTypeDetection.Strategy.FROM_RESULT_SET); + // final ResultSet actual = getActualResultSet(query); - final Exception exception = assertThrows(SQLDataException.class, () -> getActualResultSet(query)); - assertThat(exception.getMessage(), containsString("E-VSCJDBC-46")); + final Exception exception = assertThrows(DatabaseObjectException.class, () -> { + setupMySQLTableWithLatin1AndVirtualSchemaWithStrategy( + DataTypeDetection.Strategy.FROM_RESULT_SET); + }); + assertThat(exception.getMessage(), containsString("E-VSCJDBC-47")); } @Test From 5c9aff3c57d8956af59e73a53de95da88b9ab61d Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Fri, 12 Jan 2024 13:51:37 +0100 Subject: [PATCH 07/17] altered tests for Exasol v8 and added older tests for v7 (strict datatypes check on and off respectively) --- .../dialects/mysql/MySQLSqlDialectIT.java | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java b/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java index e90ccfb..8a077dd 100644 --- a/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java +++ b/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java @@ -5,7 +5,9 @@ import static com.exasol.matcher.ResultSetStructureMatcher.table; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.matchesRegex; import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assumptions.assumeTrue; import java.io.IOException; import java.sql.*; @@ -154,8 +156,50 @@ private static void createMySqlStringTable(final Schema mySqlSchema) { table.insert(null, null, "aaaaa", "a", "blob", "text", null, null, null, null, null, null, null, null); } + static void assumeExasol8OrHigher() { + assumeTrue(isExasol8OrHigher(), "is Exasol version 8 or higher"); + } + + static void assumeExasol7OrLower() { + assumeTrue(isExasol7OrLower(), "is Exasol version 7 or lower"); + } + + static boolean isExasol8OrHigher() { + final ExasolDockerImageReference imageReference = SETUP.getExasolContainer().getDockerImageReference(); + return imageReference.hasMajor() && (imageReference.getMajor() >= 8); + } + + static boolean isExasol7OrLower() { + final ExasolDockerImageReference imageReference = SETUP.getExasolContainer().getDockerImageReference(); + return imageReference.hasMajor() && (imageReference.getMajor() <= 7); + } + + @Test + void importDataTypesFromResultSetV7() throws SQLException { + assumeExasol7OrLower(); + Assume.assumeTrue(runCharsetTest()); + final String query = setupMySQLTableWithLatin1AndVirtualSchemaWithStrategy( + DataTypeDetection.Strategy.FROM_RESULT_SET); + final ResultSet actual = getActualResultSet(query); + final ResultSet expected = getExpectedResultSet(List.of("c1 CHAR(1) UTF8", "c2 CHAR(1) UTF8"), // + List.of(SPECIAL_CHAR_QUOTED + ", " + SPECIAL_CHAR_QUOTED)); + assertThat(actual, matchesResultSet(expected)); + } + + @Test + void importDataTypesExasolCalculatedV7() throws SQLException { + assumeExasol7OrLower(); + Assume.assumeTrue(runCharsetTest()); + final String query = setupMySQLTableWithLatin1AndVirtualSchemaWithStrategy( + DataTypeDetection.Strategy.EXASOL_CALCULATED); + final Exception exception = assertThrows(SQLException.class, () -> getActualResultSet(query)); + assertThat(exception.getMessage(), + matchesRegex("ETL-3009: .*Charset conversion from 'UTF-8' to 'ASCII' failed.*")); + } + @Test void importDataTypesFromResultSet() throws SQLException { + assumeExasol8OrHigher(); Assume.assumeTrue(runCharsetTest()); // final ResultSet actual = getActualResultSet(query); @@ -168,6 +212,7 @@ void importDataTypesFromResultSet() throws SQLException { @Test void importDataTypesExasolCalculated() throws SQLException { + assumeExasol8OrHigher(); Assume.assumeTrue(runCharsetTest()); assertDoesNotThrow(() -> setupMySQLTableWithLatin1AndVirtualSchemaWithStrategy( DataTypeDetection.Strategy.EXASOL_CALCULATED)); From 21e41e9f21c1204204f7cacbaeecfcf082b3ee62 Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Fri, 12 Jan 2024 14:00:28 +0100 Subject: [PATCH 08/17] fix ci build error: remove ci build from PK excludes list + run pk fix --- .github/workflows/ci-build.yml | 31 +++++++++---------- ...elease_droid_prepare_original_checksum.yml | 18 ++++++++--- .project-keeper.yml | 3 +- 3 files changed, 29 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 4088dc1..603424d 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -1,5 +1,6 @@ +# Generated by Project Keeper +# https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/ci-build.yml name: CI Build - on: push: branches: @@ -8,26 +9,27 @@ on: jobs: build: - runs-on: ubuntu-20.04 # UDFs fail with "VM error: Internal error: VM crashed" on ubuntu-latest + runs-on: ubuntu-latest concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.docker_db_version }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true - strategy: - fail-fast: false - matrix: - docker_db_version: ["7.1.24", "8.23.0"] steps: + - name: Free Disk Space + if: ${{ false }} + run: | + sudo rm -rf /usr/local/lib/android + sudo rm -rf /usr/share/dotnet - name: Checkout the repository uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Set up JDK 11 & 17 - uses: actions/setup-java@v3 + - name: Set up JDKs + uses: actions/setup-java@v4 with: distribution: "temurin" java-version: | - 17 11 + 17 cache: "maven" - name: Cache SonarCloud packages uses: actions/cache@v3 @@ -39,10 +41,9 @@ jobs: run: echo 'testcontainers.reuse.enable=true' > "$HOME/.testcontainers.properties" - name: Run tests and build with Maven run: | - JAVA_HOME=$JAVA_HOME_11_X64 mvn --batch-mode clean verify \ + mvn --batch-mode clean verify \ -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ - -DtrimStackTrace=false \ - -Dcom.exasol.dockerdb.image=${{ matrix.docker_db_version }} + -DtrimStackTrace=false - name: Publish Test Report uses: scacap/action-surefire-report@v1 if: ${{ always() && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' }} @@ -51,11 +52,9 @@ jobs: - name: Sonar analysis if: ${{ env.SONAR_TOKEN != null }} run: | - JAVA_HOME=$JAVA_HOME_17_X64 mvn --batch-mode org.sonarsource.scanner.maven:sonar-maven-plugin:sonar \ + mvn --batch-mode org.sonarsource.scanner.maven:sonar-maven-plugin:sonar \ -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ -DtrimStackTrace=false \ - -Dsonar.organization=exasol \ - -Dsonar.host.url=https://sonarcloud.io \ -Dsonar.token=$SONAR_TOKEN env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release_droid_prepare_original_checksum.yml b/.github/workflows/release_droid_prepare_original_checksum.yml index e7381af..4f58aab 100644 --- a/.github/workflows/release_droid_prepare_original_checksum.yml +++ b/.github/workflows/release_droid_prepare_original_checksum.yml @@ -1,21 +1,29 @@ +# Generated by Project Keeper +# https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/release_droid_prepare_original_checksum.yml name: Release Droid - Prepare Original Checksum - on: workflow_dispatch: jobs: build: - runs-on: ubuntu-20.04 # UDFs fail with "VM error: Internal error: VM crashed" on ubuntu-latest + runs-on: ubuntu-latest steps: + - name: Free Disk Space + if: ${{ false }} + run: | + sudo rm -rf /usr/local/lib/android + sudo rm -rf /usr/share/dotnet - name: Checkout the repository uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Set up JDK 11 - uses: actions/setup-java@v3 + - name: Set up JDKs + uses: actions/setup-java@v4 with: distribution: "temurin" - java-version: 11 + java-version: | + 11 + 17 cache: "maven" - name: Enable testcontainer reuse run: echo 'testcontainers.reuse.enable=true' > "$HOME/.testcontainers.properties" diff --git a/.project-keeper.yml b/.project-keeper.yml index 557933f..74a6c43 100644 --- a/.project-keeper.yml +++ b/.project-keeper.yml @@ -6,7 +6,6 @@ sources: - udf_coverage - jar_artifact excludes: - - "E-PK-CORE-18: Outdated content: '.github/workflows/ci-build.yml'" - - "E-PK-CORE-18: Outdated content: '.github/workflows/release_droid_prepare_original_checksum.yml'" + linkReplacements: - https://developers.google.com/protocol-buffers/protobuf-java/|https://developers.google.com/protocol-buffers From db300953b8e38b84c0f2090cd17213ed20630bd8 Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Fri, 12 Jan 2024 14:13:15 +0100 Subject: [PATCH 09/17] add exasol version build matrix via .project-keeper.yml + pk fix --- .github/workflows/ci-build.yml | 29 +++++++++++++++++++++++------ .project-keeper.yml | 6 ++++-- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 603424d..49b9e1d 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -1,5 +1,5 @@ # Generated by Project Keeper -# https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/ci-build.yml +# https://github.com/exasol/project-keeper/blob/main/project-keeper/src/main/resources/templates/.github/workflows/ci-build-db-version-matrix.yml name: CI Build on: push: @@ -8,11 +8,17 @@ on: pull_request: jobs: - build: + matrix-build: runs-on: ubuntu-latest concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.exasol_db_version }} cancel-in-progress: true + strategy: + fail-fast: false + matrix: + exasol_db_version: ["7.1.25", "8.24.0"] + env: + DEFAULT_EXASOL_DB_VERSION: "7.1.25" steps: - name: Free Disk Space if: ${{ false }} @@ -43,14 +49,19 @@ jobs: run: | mvn --batch-mode clean verify \ -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ - -DtrimStackTrace=false - - name: Publish Test Report + -DtrimStackTrace=false \ + -Dcom.exasol.dockerdb.image=${{ matrix.exasol_db_version }} + env: + # Set additional environment variable as in scala projects the scalatest plugin does not forward + # the system property -Dcom.exasol.dockerdb.image to the test's implementation. + EXASOL_DB_VERSION: ${{ matrix.exasol_db_version }} + - name: Publish Test Report for Exasol ${{ matrix.exasol_db_version }} uses: scacap/action-surefire-report@v1 if: ${{ always() && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]' }} with: github_token: ${{ secrets.GITHUB_TOKEN }} - name: Sonar analysis - if: ${{ env.SONAR_TOKEN != null }} + if: ${{ env.SONAR_TOKEN != null && matrix.exasol_db_version == env.DEFAULT_EXASOL_DB_VERSION }} run: | mvn --batch-mode org.sonarsource.scanner.maven:sonar-maven-plugin:sonar \ -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \ @@ -59,3 +70,9 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + + build: + needs: matrix-build + runs-on: ubuntu-latest + steps: + - run: echo "Build successful" diff --git a/.project-keeper.yml b/.project-keeper.yml index 74a6c43..d2a0b8f 100644 --- a/.project-keeper.yml +++ b/.project-keeper.yml @@ -5,7 +5,9 @@ sources: - integration_tests - udf_coverage - jar_artifact -excludes: - +build: + exasolDbVersions: + - "7.1.25" + - "8.24.0" linkReplacements: - https://developers.google.com/protocol-buffers/protobuf-java/|https://developers.google.com/protocol-buffers From e6d9ae220651243210415563670cc8ad69f93d2b Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Fri, 12 Jan 2024 14:26:58 +0100 Subject: [PATCH 10/17] use older version of ubuntu as ci runner --- .github/workflows/ci-build.yml | 2 +- .github/workflows/release_droid_prepare_original_checksum.yml | 2 +- .project-keeper.yml | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 49b9e1d..76adff4 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -9,7 +9,7 @@ on: jobs: matrix-build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.exasol_db_version }} cancel-in-progress: true diff --git a/.github/workflows/release_droid_prepare_original_checksum.yml b/.github/workflows/release_droid_prepare_original_checksum.yml index 4f58aab..413274b 100644 --- a/.github/workflows/release_droid_prepare_original_checksum.yml +++ b/.github/workflows/release_droid_prepare_original_checksum.yml @@ -6,7 +6,7 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - name: Free Disk Space if: ${{ false }} diff --git a/.project-keeper.yml b/.project-keeper.yml index d2a0b8f..02be779 100644 --- a/.project-keeper.yml +++ b/.project-keeper.yml @@ -6,6 +6,8 @@ sources: - udf_coverage - jar_artifact build: + runnerOs: ubuntu-20.04 + freeDiskSpace: false exasolDbVersions: - "7.1.25" - "8.24.0" From ba073f058d76d5cb63916f2fe8269f7656726707 Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Fri, 12 Jan 2024 15:45:21 +0100 Subject: [PATCH 11/17] fix tests for v7 / tests for v7 & 8 should be identical since behaviour is altered --- .../dialects/mysql/MySQLSqlDialectIT.java | 45 ------------------- ...ySQLVirtualSchemaIntegrationTestSetup.java | 2 +- 2 files changed, 1 insertion(+), 46 deletions(-) diff --git a/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java b/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java index 8a077dd..e90ccfb 100644 --- a/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java +++ b/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java @@ -5,9 +5,7 @@ import static com.exasol.matcher.ResultSetStructureMatcher.table; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.matchesRegex; import static org.junit.jupiter.api.Assertions.*; -import static org.junit.jupiter.api.Assumptions.assumeTrue; import java.io.IOException; import java.sql.*; @@ -156,50 +154,8 @@ private static void createMySqlStringTable(final Schema mySqlSchema) { table.insert(null, null, "aaaaa", "a", "blob", "text", null, null, null, null, null, null, null, null); } - static void assumeExasol8OrHigher() { - assumeTrue(isExasol8OrHigher(), "is Exasol version 8 or higher"); - } - - static void assumeExasol7OrLower() { - assumeTrue(isExasol7OrLower(), "is Exasol version 7 or lower"); - } - - static boolean isExasol8OrHigher() { - final ExasolDockerImageReference imageReference = SETUP.getExasolContainer().getDockerImageReference(); - return imageReference.hasMajor() && (imageReference.getMajor() >= 8); - } - - static boolean isExasol7OrLower() { - final ExasolDockerImageReference imageReference = SETUP.getExasolContainer().getDockerImageReference(); - return imageReference.hasMajor() && (imageReference.getMajor() <= 7); - } - - @Test - void importDataTypesFromResultSetV7() throws SQLException { - assumeExasol7OrLower(); - Assume.assumeTrue(runCharsetTest()); - final String query = setupMySQLTableWithLatin1AndVirtualSchemaWithStrategy( - DataTypeDetection.Strategy.FROM_RESULT_SET); - final ResultSet actual = getActualResultSet(query); - final ResultSet expected = getExpectedResultSet(List.of("c1 CHAR(1) UTF8", "c2 CHAR(1) UTF8"), // - List.of(SPECIAL_CHAR_QUOTED + ", " + SPECIAL_CHAR_QUOTED)); - assertThat(actual, matchesResultSet(expected)); - } - - @Test - void importDataTypesExasolCalculatedV7() throws SQLException { - assumeExasol7OrLower(); - Assume.assumeTrue(runCharsetTest()); - final String query = setupMySQLTableWithLatin1AndVirtualSchemaWithStrategy( - DataTypeDetection.Strategy.EXASOL_CALCULATED); - final Exception exception = assertThrows(SQLException.class, () -> getActualResultSet(query)); - assertThat(exception.getMessage(), - matchesRegex("ETL-3009: .*Charset conversion from 'UTF-8' to 'ASCII' failed.*")); - } - @Test void importDataTypesFromResultSet() throws SQLException { - assumeExasol8OrHigher(); Assume.assumeTrue(runCharsetTest()); // final ResultSet actual = getActualResultSet(query); @@ -212,7 +168,6 @@ void importDataTypesFromResultSet() throws SQLException { @Test void importDataTypesExasolCalculated() throws SQLException { - assumeExasol8OrHigher(); Assume.assumeTrue(runCharsetTest()); assertDoesNotThrow(() -> setupMySQLTableWithLatin1AndVirtualSchemaWithStrategy( DataTypeDetection.Strategy.EXASOL_CALCULATED)); diff --git a/src/test/java/com/exasol/adapter/dialects/mysql/MySQLVirtualSchemaIntegrationTestSetup.java b/src/test/java/com/exasol/adapter/dialects/mysql/MySQLVirtualSchemaIntegrationTestSetup.java index caf017f..7baea1a 100644 --- a/src/test/java/com/exasol/adapter/dialects/mysql/MySQLVirtualSchemaIntegrationTestSetup.java +++ b/src/test/java/com/exasol/adapter/dialects/mysql/MySQLVirtualSchemaIntegrationTestSetup.java @@ -59,7 +59,7 @@ public class MySQLVirtualSchemaIntegrationTestSetup implements Closeable { uploadVsJarToBucket(bucket); this.exasolConnection = this.exasolContainer.createConnection(""); this.exasolStatement = this.exasolConnection.createStatement(); - exasolStatement.execute("CONTROL SET TRACE LEVEL NOTICE WITH LOG TIMEOUT 0;"); + // exasolStatement.execute("CONTROL SET TRACE LEVEL NOTICE WITH LOG TIMEOUT 0;"); this.mySqlConnection = this.mySqlContainer.createConnection(""); this.mySqlStatement = this.mySqlConnection.createStatement(); From 82b1b17a801933e9f5f50fae6495518cdd9a8a07 Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Mon, 15 Jan 2024 13:18:57 +0100 Subject: [PATCH 12/17] updated changes doc for release --- doc/changes/changes_5.0.0.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/doc/changes/changes_5.0.0.md b/doc/changes/changes_5.0.0.md index cd84848..6e2dcd4 100644 --- a/doc/changes/changes_5.0.0.md +++ b/doc/changes/changes_5.0.0.md @@ -1,12 +1,16 @@ -# Virtual Schema for MySQL 5.0.0, released 2024-??-?? +# Virtual Schema for MySQL 5.0.0, released 2024-01-15 -Code name: +Code name: Changes related to Exasol V8 / importing text datatypes. ## Summary -## Features +The behaviour when it comes to character sets is now simplified, +The target char set is now always UTF-8. +The IMPORT_DATA_TYPES property (and value FROM_RESULT_SET) are now deprecated (change in vs-common-jdbc). -* ISSUE_NUMBER: description +## Refactoring + +* #37: Update tests to V8 VSMYSQL / Update to vsjdbc 12.0.0 ## Dependency Updates From 870f978da0aaea4e83e23ff3b6b7cc4fc49d9586 Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Mon, 15 Jan 2024 13:23:20 +0100 Subject: [PATCH 13/17] removed unneeded version document --- doc/changes/changes_4.1.4.md | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 doc/changes/changes_4.1.4.md diff --git a/doc/changes/changes_4.1.4.md b/doc/changes/changes_4.1.4.md deleted file mode 100644 index b9b3c30..0000000 --- a/doc/changes/changes_4.1.4.md +++ /dev/null @@ -1,28 +0,0 @@ -# Virtual Schema for MySQL 4.1.4, released 2023-??-?? - -Code name: - -## Summary - -## Features - -* ISSUE_NUMBER: description - -## Dependency Updates - -### Test Dependency Updates - -* Updated `com.mysql:mysql-connector-j:8.1.0` to `8.2.0` -* Updated `org.jacoco:org.jacoco.agent:0.8.10` to `0.8.11` - -### Plugin Dependency Updates - -* Updated `com.exasol:error-code-crawler-maven-plugin:1.3.0` to `1.3.1` -* Updated `com.exasol:project-keeper-maven-plugin:2.9.12` to `2.9.17` -* Updated `org.apache.maven.plugins:maven-dependency-plugin:3.6.0` to `3.6.1` -* Updated `org.apache.maven.plugins:maven-enforcer-plugin:3.4.0` to `3.4.1` -* Updated `org.apache.maven.plugins:maven-failsafe-plugin:3.1.2` to `3.2.2` -* Updated `org.apache.maven.plugins:maven-surefire-plugin:3.1.2` to `3.2.2` -* Updated `org.codehaus.mojo:versions-maven-plugin:2.16.0` to `2.16.2` -* Updated `org.jacoco:jacoco-maven-plugin:0.8.10` to `0.8.11` -* Updated `org.sonarsource.scanner.maven:sonar-maven-plugin:3.9.1.2184` to `3.10.0.2594` From 2d44292425608eb2932c59e958af46f01db0f6d3 Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Wed, 17 Jan 2024 17:06:47 +0100 Subject: [PATCH 14/17] PR feedback changes --- .gitattributes | 3 +- doc/changes/changelog.md | 1 - doc/changes/changes_5.0.0.md | 2 +- .../dialects/mysql/MySQLSqlDialectIT.java | 1278 ++++++++--------- ...ySQLVirtualSchemaIntegrationTestSetup.java | 1 - 5 files changed, 629 insertions(+), 656 deletions(-) diff --git a/.gitattributes b/.gitattributes index f1e6c1c..c67f857 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,10 +2,11 @@ pk_generated_parent.pom linguist-genera dependencies.md linguist-generated=true doc/changes/changelog.md linguist-generated=true .github/workflows/broken_links_checker.yml linguist-generated=true +.github/workflows/ci-build.yml linguist-generated=true .github/workflows/ci-build-next-java.yml linguist-generated=true .github/workflows/dependencies_check.yml linguist-generated=true .github/workflows/release_droid_print_quick_checksum.yml linguist-generated=true .github/workflows/release_droid_upload_github_release_assets.yml linguist-generated=true - +.github/workflows/release_droid_prepare_original_checksum.yml linguist-generated=true .settings/org.eclipse.jdt.core.prefs linguist-generated=true .settings/org.eclipse.jdt.ui.prefs linguist-generated=true diff --git a/doc/changes/changelog.md b/doc/changes/changelog.md index cd146e2..d5038e4 100644 --- a/doc/changes/changelog.md +++ b/doc/changes/changelog.md @@ -1,7 +1,6 @@ # Changes * [5.0.0](changes_5.0.0.md) -* [4.1.4](changes_4.1.4.md) * [4.1.3](changes_4.1.3.md) * [4.1.2](changes_4.1.2.md) * [4.1.1](changes_4.1.1.md) diff --git a/doc/changes/changes_5.0.0.md b/doc/changes/changes_5.0.0.md index 6e2dcd4..a527c45 100644 --- a/doc/changes/changes_5.0.0.md +++ b/doc/changes/changes_5.0.0.md @@ -1,6 +1,6 @@ # Virtual Schema for MySQL 5.0.0, released 2024-01-15 -Code name: Changes related to Exasol V8 / importing text datatypes. +Code name: Char set is always `utf-8`, deprecated IMPORT_DATA_TYPES `from_result_set` value . ## Summary diff --git a/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java b/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java index e90ccfb..4504957 100644 --- a/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java +++ b/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java @@ -34,732 +34,706 @@ @Tag("integration") @Testcontainers class MySQLSqlDialectIT { - private static final String MYSQL_SCHEMA = "MYSQL_SCHEMA"; - private static final String MYSQL_SIMPLE_TABLE = "MYSQL_SIMPLE_TABLE"; - private static final String MYSQL_NUMERIC_DATE_DATATYPES_TABLE = "MYSQL_NUMERIC_DATE_TABLE"; - private static final String MYSQL_STRING_DATATYPES_TABLE = "MYSQL_STRING_TABLE"; - private static final MySQLVirtualSchemaIntegrationTestSetup SETUP = new MySQLVirtualSchemaIntegrationTestSetup(); - - private static final String MYSQL_SOURCE_SCHEMA = "SOURCE_SCHEMA"; - private static final String MYSQL_SOURCE_TABLE = "SOURCE_TABLE"; - private static String virtualSchemaJdbc; - private MySqlSchema sourceSchema; - private VirtualSchema virtualSchema; - - @BeforeAll - static void beforeAll() throws SQLException { - final MySqlSchema mySqlSchema = SETUP.getMySqlObjectFactory().createSchema(MYSQL_SCHEMA); - createMySqlSimpleTable(mySqlSchema); - createMySqlNumericDateTable(mySqlSchema); - createMySqlStringTable(mySqlSchema); - createTestTablesForJoinTests(SETUP.getMySqlStatement(), mySqlSchema.getName()); - virtualSchemaJdbc = SETUP.createVirtualSchema(Collections.emptyMap(), mySqlSchema.getName()).getName(); - } - - @AfterAll - static void afterAll() throws IOException { - SETUP.close(); - } - - @AfterEach - void afterEach() { - dropAll(this.virtualSchema, this.sourceSchema); - this.virtualSchema = null; - this.sourceSchema = null; - } - - private static void dropAll(final DatabaseObject... databaseObjects) { - for (final DatabaseObject databaseObject : databaseObjects) { - if (databaseObject != null) { - databaseObject.drop(); - } - } - } - - private static void createTestTablesForJoinTests(final Statement statement, final String schemaName) - throws SQLException { - statement.execute("CREATE TABLE " + schemaName + "." + TABLE_JOIN_1 + "(x INT, y VARCHAR(100))"); - statement.execute("INSERT INTO " + schemaName + "." + TABLE_JOIN_1 + " VALUES (1,'aaa')"); - statement.execute("INSERT INTO " + schemaName + "." + TABLE_JOIN_1 + " VALUES (2,'bbb')"); - statement.execute("CREATE TABLE " + schemaName + "." + TABLE_JOIN_2 + "(x INT, y VARCHAR(100))"); - statement.execute("INSERT INTO " + schemaName + "." + TABLE_JOIN_2 + " VALUES (2,'bbb')"); - statement.execute("INSERT INTO " + schemaName + "." + TABLE_JOIN_2 + " VALUES (3,'ccc')"); - } - - private ResultSet getExpectedResultSet(final List expectedColumns, final List expectedRows) - throws SQLException { - final Statement statement = SETUP.getExasolStatement(); - final String expectedValues = expectedRows.stream().map(row -> "(" + row + ")") - .collect(Collectors.joining(",")); - final String qualifiedExpectedTableName = SCHEMA_EXASOL + ".EXPECTED"; - statement.execute("CREATE OR REPLACE TABLE " + qualifiedExpectedTableName + "(" - + String.join(", ", expectedColumns) + ")"); - statement.execute("INSERT INTO " + qualifiedExpectedTableName + " VALUES" + expectedValues); - return statement.executeQuery("SELECT * FROM " + qualifiedExpectedTableName); - } - - private ResultSet getActualResultSet(final String query) throws SQLException { - final Statement statement = SETUP.getExasolStatement(); - return statement.executeQuery(query); - } - - private static void createMySqlSimpleTable(final Schema mySqlSchema) { - final Table table = mySqlSchema.createTable(MYSQL_SIMPLE_TABLE, - List.of("int_col", "bool_col", "varchar_col"), - List.of("INT", "BOOLEAN", "VARCHAR(20)")); - table.insert(-100, true, "a"); - table.insert(-1, false, "abbb"); - table.insert(0, true, "b"); - table.insert(10, false, "bbbb"); - table.insert(50, true, "abc"); - table.insert(100, false, "a"); - } - - private static void createMySqlNumericDateTable(final Schema mySqlSchema) { - final Table table = mySqlSchema.createTable(MYSQL_NUMERIC_DATE_DATATYPES_TABLE, - List.of("BiT_Col", "tinyint_col", "BOOL_COL", "smallint_col", "mediumint_col", - "int_col", "bigint_col", "decimal_col", "float_col", "double_col", // - "date_col", "datetime_col", "timestamp_col", "time_col", "year_col"), - List.of("BIT(6)", "TINYINT", "BOOLEAN", "SMALLINT", "MEDIUMINT", "INT", "BIGINT", - "DECIMAL(5, 2)", "FLOAT(7, 4)", "DOUBLE", // - "DATE", "DATETIME", "TIMESTAMP", "TIME", "YEAR")); - table.insert("5", 127, 1, 32767, 8388607, 2147483647, 9223372036854775807L, 999.99, 999.00009, - 999.00009, // - "1000-01-01", "1000-01-01 00:00:00", "1970-01-01 00:00:01.000000", "16:59:59.000000", - 1901); - table.insert("9", -127, 0, -32768, -8388608, -2147483648, -9223372036854775808L, -999.99, -999.9999, - -999.9999, // - "9999-12-31", "9999-12-31 22:59:59", "2037-01-19 03:14:07.999999", "05:34:13.000000", - 2155); - table.insert(null, 0, true, 0, 0, 0, 0, 0, 0, 0, // - null, null, null, null, "1901"); - table.insert(null, 0, false, 0, 0, 0, 0, 0, 0, 0, // - null, null, null, null, 69); - } - - private static void createMySqlStringTable(final Schema mySqlSchema) { - final Table table = mySqlSchema.createTable(MYSQL_STRING_DATATYPES_TABLE, - List.of("binary_col", "varbinary_col", "tinyblob_col", "tinytext_col", "blob_col", - "text_col", "mediumblob_col", "mediumtext_col", "longblob_col", - "longtext_col", "enum_col", "set_col", "varchar_col", "char_col"), - List.of(" BINARY(20)", "VARBINARY(20)", "TINYBLOB", "TINYTEXT", "BLOB", "TEXT", - "MEDIUMBLOB", "MEDIUMTEXT", "LONGBLOB", "LONGTEXT", - "ENUM('1', '2', '3')", "SET('1')", "VARCHAR(16000)", "CHAR(255)")); - table.insert("a", "a", "a", "a", "blob", "text", "mediumblob", "mediumtext", "longblob", "longtext", - "1", "1", "ab", "asd24"); - table.insert("a\0", "a\0", "aa", "b", "blob", "text2", "mediumblob2", "mediumtext2", "longblob", - "longtext2", "2", "1", "a", "11111"); - table.insert(null, null, "aaa", "aaaaaaaaaaaaa", "bloooooooooooob", "text3", null, null, null, null, - "3", null, "", ""); - table.insert(null, null, "aaaaa", "a", "blob", "text", null, null, null, null, null, null, null, null); + private static final String MYSQL_SCHEMA = "MYSQL_SCHEMA"; + private static final String MYSQL_SIMPLE_TABLE = "MYSQL_SIMPLE_TABLE"; + private static final String MYSQL_NUMERIC_DATE_DATATYPES_TABLE = "MYSQL_NUMERIC_DATE_TABLE"; + private static final String MYSQL_STRING_DATATYPES_TABLE = "MYSQL_STRING_TABLE"; + private static final MySQLVirtualSchemaIntegrationTestSetup SETUP = new MySQLVirtualSchemaIntegrationTestSetup(); + private static final String MYSQL_SOURCE_SCHEMA = "SOURCE_SCHEMA"; + private static final String MYSQL_SOURCE_TABLE = "SOURCE_TABLE"; + private static String virtualSchemaJdbc; + private MySqlSchema sourceSchema; + private VirtualSchema virtualSchema; + + @BeforeAll + static void beforeAll() throws SQLException { + final MySqlSchema mySqlSchema = SETUP.getMySqlObjectFactory().createSchema(MYSQL_SCHEMA); + createMySqlSimpleTable(mySqlSchema); + createMySqlNumericDateTable(mySqlSchema); + createMySqlStringTable(mySqlSchema); + createTestTablesForJoinTests(SETUP.getMySqlStatement(), mySqlSchema.getName()); + virtualSchemaJdbc = SETUP.createVirtualSchema(Collections.emptyMap(), mySqlSchema.getName()).getName(); + } + + @AfterAll + static void afterAll() throws IOException { + SETUP.close(); + } + + @AfterEach + void afterEach() { + dropAll(this.virtualSchema, this.sourceSchema); + this.virtualSchema = null; + this.sourceSchema = null; + } + + private static void dropAll(final DatabaseObject... databaseObjects) { + for (final DatabaseObject databaseObject : databaseObjects) { + if (databaseObject != null) { + databaseObject.drop(); + } + } + } + + private static void createTestTablesForJoinTests(final Statement statement, final String schemaName) + throws SQLException { + statement.execute("CREATE TABLE " + schemaName + "." + TABLE_JOIN_1 + "(x INT, y VARCHAR(100))"); + statement.execute("INSERT INTO " + schemaName + "." + TABLE_JOIN_1 + " VALUES (1,'aaa')"); + statement.execute("INSERT INTO " + schemaName + "." + TABLE_JOIN_1 + " VALUES (2,'bbb')"); + statement.execute("CREATE TABLE " + schemaName + "." + TABLE_JOIN_2 + "(x INT, y VARCHAR(100))"); + statement.execute("INSERT INTO " + schemaName + "." + TABLE_JOIN_2 + " VALUES (2,'bbb')"); + statement.execute("INSERT INTO " + schemaName + "." + TABLE_JOIN_2 + " VALUES (3,'ccc')"); + } + + private ResultSet getExpectedResultSet(final List expectedColumns, final List expectedRows) + throws SQLException { + final Statement statement = SETUP.getExasolStatement(); + final String expectedValues = expectedRows.stream().map(row -> "(" + row + ")") + .collect(Collectors.joining(",")); + final String qualifiedExpectedTableName = SCHEMA_EXASOL + ".EXPECTED"; + statement.execute("CREATE OR REPLACE TABLE " + qualifiedExpectedTableName + "(" + + String.join(", ", expectedColumns) + ")"); + statement.execute("INSERT INTO " + qualifiedExpectedTableName + " VALUES" + expectedValues); + return statement.executeQuery("SELECT * FROM " + qualifiedExpectedTableName); + } + + private ResultSet getActualResultSet(final String query) throws SQLException { + final Statement statement = SETUP.getExasolStatement(); + return statement.executeQuery(query); + } + + private static void createMySqlSimpleTable(final Schema mySqlSchema) { + final Table table = mySqlSchema.createTable(MYSQL_SIMPLE_TABLE, List.of("int_col", "bool_col", "varchar_col"), + List.of("INT", "BOOLEAN", "VARCHAR(20)")); + table.insert(-100, true, "a"); + table.insert(-1, false, "abbb"); + table.insert(0, true, "b"); + table.insert(10, false, "bbbb"); + table.insert(50, true, "abc"); + table.insert(100, false, "a"); + } + + private static void createMySqlNumericDateTable(final Schema mySqlSchema) { + final Table table = mySqlSchema.createTable(MYSQL_NUMERIC_DATE_DATATYPES_TABLE, + List.of("BiT_Col", "tinyint_col", "BOOL_COL", "smallint_col", "mediumint_col", "int_col", "bigint_col", + "decimal_col", "float_col", "double_col", // + "date_col", "datetime_col", "timestamp_col", "time_col", "year_col"), + List.of("BIT(6)", "TINYINT", "BOOLEAN", "SMALLINT", "MEDIUMINT", "INT", "BIGINT", "DECIMAL(5, 2)", + "FLOAT(7, 4)", "DOUBLE", // + "DATE", "DATETIME", "TIMESTAMP", "TIME", "YEAR")); + table.insert("5", 127, 1, 32767, 8388607, 2147483647, 9223372036854775807L, 999.99, 999.00009, 999.00009, // + "1000-01-01", "1000-01-01 00:00:00", "1970-01-01 00:00:01.000000", "16:59:59.000000", 1901); + table.insert("9", -127, 0, -32768, -8388608, -2147483648, -9223372036854775808L, -999.99, -999.9999, -999.9999, // + "9999-12-31", "9999-12-31 22:59:59", "2037-01-19 03:14:07.999999", "05:34:13.000000", 2155); + table.insert(null, 0, true, 0, 0, 0, 0, 0, 0, 0, // + null, null, null, null, "1901"); + table.insert(null, 0, false, 0, 0, 0, 0, 0, 0, 0, // + null, null, null, null, 69); + } + + private static void createMySqlStringTable(final Schema mySqlSchema) { + final Table table = mySqlSchema.createTable(MYSQL_STRING_DATATYPES_TABLE, + List.of("binary_col", "varbinary_col", "tinyblob_col", "tinytext_col", "blob_col", "text_col", + "mediumblob_col", "mediumtext_col", "longblob_col", "longtext_col", "enum_col", "set_col", + "varchar_col", "char_col"), + List.of(" BINARY(20)", "VARBINARY(20)", "TINYBLOB", "TINYTEXT", "BLOB", "TEXT", "MEDIUMBLOB", + "MEDIUMTEXT", "LONGBLOB", "LONGTEXT", "ENUM('1', '2', '3')", "SET('1')", "VARCHAR(16000)", + "CHAR(255)")); + table.insert("a", "a", "a", "a", "blob", "text", "mediumblob", "mediumtext", "longblob", "longtext", "1", "1", + "ab", "asd24"); + table.insert("a\0", "a\0", "aa", "b", "blob", "text2", "mediumblob2", "mediumtext2", "longblob", "longtext2", + "2", "1", "a", "11111"); + table.insert(null, null, "aaa", "aaaaaaaaaaaaa", "bloooooooooooob", "text3", null, null, null, null, "3", null, + "", ""); + table.insert(null, null, "aaaaa", "a", "blob", "text", null, null, null, null, null, null, null, null); + } + + @Test + void importDataTypesFromResultSet() throws SQLException { + Assume.assumeTrue(runCharsetTest()); + + final Exception exception = assertThrows(DatabaseObjectException.class, () -> { + setupMySQLTableWithLatin1AndVirtualSchemaWithStrategy(DataTypeDetection.Strategy.FROM_RESULT_SET); + }); + assertThat(exception.getMessage(), containsString("E-VSCJDBC-47")); + } + + @Test + void importDataTypesExasolCalculated() throws SQLException { + Assume.assumeTrue(runCharsetTest()); + assertDoesNotThrow(() -> setupMySQLTableWithLatin1AndVirtualSchemaWithStrategy( + DataTypeDetection.Strategy.EXASOL_CALCULATED)); + } + + private boolean runCharsetTest() { + final ExasolDockerImageReference dockerImage = SETUP.getExasolContainer().getDockerImageReference(); + if (!dockerImage.hasMajor() || !dockerImage.hasMinor() || !dockerImage.hasFix()) { + return false; + } + final Version version = Version.of(dockerImage.getMajor(), dockerImage.getMinor(), dockerImage.getFixVersion()); + if ((dockerImage.getMajor() == 7) && version.isGreaterOrEqualThan(Version.parse("7.1.14"))) { + return true; + } + if ((dockerImage.getMajor() == 8) && version.isGreaterOrEqualThan(Version.parse("8.6.0"))) { + return true; + } + return false; + } + + private String setupMySQLTableWithLatin1AndVirtualSchemaWithStrategy(final DataTypeDetection.Strategy strategy) + throws SQLException { + final String tableName = MYSQL_SOURCE_TABLE; + // create SQL schema with latin1 characterset and virtual schema + createMySqlTableContainingCharAndEnumWithCharacterSet(MYSQL_SOURCE_SCHEMA, tableName, "latin1"); + + this.virtualSchema = SETUP.createVirtualSchema( // + Map.of(DataTypeDetection.STRATEGY_PROPERTY, strategy.name()), // + MYSQL_SOURCE_SCHEMA); + + final ColumnInspector inspector = SETUP.getColumnInspector(MYSQL_SOURCE_SCHEMA); + inspector.describeFromMetadata(MYSQL_SOURCE_SCHEMA, MYSQL_SOURCE_TABLE); + + final String query = String.format("select * from %s.%s", MYSQL_SOURCE_SCHEMA, MYSQL_SOURCE_TABLE); + inspector.describeFromQuery(MYSQL_SOURCE_SCHEMA, query); + + return selectEverythingFromVirtualSchemaTable(tableName); + } + + private String selectEverythingFromVirtualSchemaTable(final String tableName) { + return "SELECT * FROM " + this.virtualSchema.getName() + "." + tableName; + } + + private static final char[] GERMAN_UMLAUT = { 0xDC }; + private static final String SPECIAL_CHAR = new String(GERMAN_UMLAUT); + private static final String SPECIAL_CHAR_QUOTED = "'" + SPECIAL_CHAR + "'"; + + private void createMySqlTableContainingCharAndEnumWithCharacterSet(final String schemaName, final String tableName, + final String characterSet) { + + this.sourceSchema = getSchemaWithCharacterSet(schemaName, "latin1"); + + final String mySqlEnum = "ENUM('A', " + SPECIAL_CHAR_QUOTED + ")"; + + final Table table = this.sourceSchema.createTable(tableName, List.of("c1", "c2"), + List.of("CHAR(1)", mySqlEnum)); + + table.insert(SPECIAL_CHAR, SPECIAL_CHAR); + } + + private static MySqlSchema getSchemaWithCharacterSet(final String schemaName, final String characterSet) { + return new MySqlSchema(SETUP.getTableWriterWithCharacterSet(characterSet), MySQLIdentifier.of(schemaName)); + } + + @Test + void testSelectAll() throws SQLException { + final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE; + final ResultSet actualResultSet = getActualResultSet(query); + final ResultSet expected = getExpectedResultSet(List.of("col1 INT", "col2 BOOLEAN", "col3 VARCHAR(20)"), // + List.of("-100, true, 'a'", // + "-1, false, 'abbb'", // + "0, true, 'b'", // + "10, false, 'bbbb'", // + "50, true, 'abc'", // + "100, false, 'a'")); + assertThat(actualResultSet, matchesResultSet(expected)); + } + + @Test + void testAggregateGroupByColumn() throws SQLException { + final String query = "SELECT \"bool_col\", min(\"int_col\") FROM " + virtualSchemaJdbc + "." + + MYSQL_SIMPLE_TABLE + " GROUP BY \"bool_col\""; + final ResultSet expected = getExpectedResultSet(List.of("bool_col BOOLEAN", "int_col DECIMAL(10,0)"), + List.of("true, -100", // + "false, -1")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testAggregateHaving() throws SQLException { + final String query = "SELECT \"bool_col\", min(\"int_col\") FROM " + virtualSchemaJdbc + "." + + MYSQL_SIMPLE_TABLE + " GROUP BY \"bool_col\" HAVING MIN(\"int_col\") < 0"; + final ResultSet expected = getExpectedResultSet(List.of("bool_col BOOLEAN", "int_col DECIMAL(10,0)"), + List.of("true, -100", // + "false, -1")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + // =, !=, <, <=, >, >= + void testComparisonPredicates() throws SQLException { + final String query = "SELECT \"int_col\", \"int_col\" = 60, \"int_col\" != 60, \"int_col\" < 60, " + + "\"int_col\" <= 60, \"int_col\" > 60, \"int_col\" >= 60 FROM " + virtualSchemaJdbc + "." + + MYSQL_SIMPLE_TABLE + " WHERE \"int_col\" = 0"; + final ResultSet expected = getExpectedResultSet( + List.of("int_col DECIMAL(10,0)", "b1 BOOLEAN", "b2 BOOLEAN", "b3 BOOLEAN", "b4 BOOLEAN", "b5 BOOLEAN", + "b6 BOOLEAN"), // + List.of("0, 0, 1, 1, 1, 0, 0")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + // NOT, AND, OR + void testLogicalPredicates() throws SQLException { + final String query = "SELECT \"int_col\" FROM " + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE + + " WHERE (\"int_col\" < 0 or \"int_col\" > 0) AND NOT (\"int_col\" is null)"; + final ResultSet expected = getExpectedResultSet(List.of("int_col DECIMAL(10,0)"), // + List.of("-100", // + "-1", // + "10", // + "50", // + "100")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + // LIKE, LIKE ESCAPE (not pushed down) + void testLikePredicates() throws SQLException { + final String query = "SELECT \"varchar_col\", \"varchar_col\" LIKE 'a%' ESCAPE 'a' FROM " + virtualSchemaJdbc + + "." + MYSQL_SIMPLE_TABLE + " WHERE (\"varchar_col\" LIKE 'a%')"; + final ResultSet expected = getExpectedResultSet(List.of("varchar_col VARCHAR(10)", "bool_col BOOLEAN"), + List.of("'a', false", // + "'abbb', false", // + "'abc', false", // + "'a', false")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + // BETWEEN, IN, IS NULL, !=NULL(rewritten to "IS NOT NULL") + void testMiscPredicates() throws SQLException { + final String query = "SELECT \"int_col\", \"int_col\" in (56, 61), \"int_col\" is null, \"int_col\" != null" + + " FROM " + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE + " WHERE \"int_col\" between -10 and 51"; + final ResultSet expected = getExpectedResultSet( + List.of("int_col DECIMAL(10,0)", "b1 BOOLEAN", "b2 BOOLEAN", "b3 BOOLEAN"), // + List.of("-1, false, false, false", // + "0, false, false, false", // + "10, false, false, false", // + "50, false, false, false")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testCountSumAggregateFunction() throws SQLException { + final String query = "SELECT COUNT(\"int_col\"), COUNT(*), COUNT(DISTINCT \"int_col\"), SUM(\"int_col\"), " + + "SUM(DISTINCT \"int_col\") FROM " + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE; + final ResultSet expected = getExpectedResultSet( + List.of("a DECIMAL(10,0)", "b DECIMAL(10,0)", "c DECIMAL(10,0)", "d DECIMAL(19,0)", "e DECIMAL(19,0)"), + List.of("6, 6, 6, 59, 59")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testAvgMinMaxAggregateFunction() throws SQLException { + final String query = "SELECT AVG(\"int_col\"), MIN(\"int_col\"), MAX(\"int_col\") FROM " + virtualSchemaJdbc + + "." + MYSQL_SIMPLE_TABLE; + final ResultSet expected = getExpectedResultSet( // + List.of("a DOUBLE", "b DECIMAL(10,0)", "c DECIMAL(10,0)"), // + List.of("9.833300000000001, -100.0000, 100.0000")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testCastedStringFunctions() throws SQLException { + final String query = "SELECT concat(upper(\"varchar_col\"),lower(repeat(\"varchar_col\",2))) FROM " + + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("a VARCHAR(100)"), // + List.of("'Aaa'", // + "'ABBBabbbabbb'", // + "'Bbb'", // + "'BBBBbbbbbbbb'", // + "'ABCabcabc'", // + "'Aaa'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testRewrittenDivAndModFunctions() throws SQLException { + final String query = "SELECT DIV(\"int_col\",\"int_col\"), mod(\"int_col\",\"int_col\") FROM " + + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("a DECIMAL(10,0)", "b DECIMAL(10,0)"), // + List.of("1, 0", // + "1, 0", // + "null, null", // + "1, 0", // + "1, 0", // + "1, 0")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testRewrittenSubStringFunction() throws SQLException { + final String query = "SELECT substring(\"varchar_col\" FROM 1 FOR 2) FROM " + virtualSchemaJdbc + "." + + MYSQL_SIMPLE_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("a VARCHAR(100)"), // + List.of("'a'", // + "'ab'", // + "'b'", // + "'bb'", // + "'ab'", // + "'a'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testOrderByLimit() throws SQLException { + final String query = "SELECT \"bool_col\", \"int_col\" FROM " + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE + + " ORDER BY \"int_col\" LIMIT 3"; + final ResultSet expected = getExpectedResultSet(List.of("a BOOLEAN", "b DECIMAL(10,0)"), // + List.of("true, -100", // + "false, -1", // + "true, 0")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testOrderByLimitOffset() throws SQLException { + final String query = "SELECT \"bool_col\", \"int_col\" FROM " + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE + + " ORDER BY \"int_col\" LIMIT 2 OFFSET 1"; + final ResultSet expected = getExpectedResultSet(List.of("a BOOLEAN", "b DECIMAL(10,0)"), // + List.of("false, -1", // + "true, 0")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + + @Test + void testLeftShiftScalarFunction() { + createSourceTable(List.of("INT_COL_1", "INT_COL_2"), List.of("INT", "INT"), new Object[][] { { 10, 3 } }); + this.virtualSchema = SETUP.createVirtualSchema(Collections.emptyMap(), MYSQL_SOURCE_SCHEMA); + final String query = "SELECT BIT_LSHIFT(INT_COL_1, INT_COL_2) FROM " + this.virtualSchema.getName() + "." + + MYSQL_SOURCE_TABLE; + assertVsQuery(query, table().row(80).matches(TypeMatchMode.NO_JAVA_TYPE_CHECK)); + } + + private void createSourceTable(final List columnNames, final List types, final Object[][] values) { + this.sourceSchema = SETUP.getMySqlObjectFactory().createSchema(MYSQL_SOURCE_SCHEMA); + final Table table = this.sourceSchema.createTable(MYSQL_SOURCE_TABLE, columnNames, types); + for (final Object[] value : values) { + table.insert(value); + } + } + + private ResultSet query(final String sql) throws SQLException { + return SETUP.getExasolStatement().executeQuery(sql); + } + + @Test + void testRightShiftScalarFunction() { + createSourceTable(List.of("INT_COL_1", "INT_COL_2"), List.of("INT", "INT"), new Object[][] { { 10, 3 } }); + this.virtualSchema = SETUP.createVirtualSchema(Collections.emptyMap(), MYSQL_SOURCE_SCHEMA); + final String query = "SELECT BIT_RSHIFT(INT_COL_1, INT_COL_2) FROM " + this.virtualSchema.getName() + "." + + MYSQL_SOURCE_TABLE; + assertVsQuery(query, table().row(1).matches(TypeMatchMode.NO_JAVA_TYPE_CHECK)); + } + + private void assertVsQuery(final String sql, final Matcher expected) { + try { + assertThat(query(sql), expected); + } catch (final SQLException exception) { + fail("Unable to run assertion query: " + sql + "\nCaused by: " + exception.getMessage()); + } + } + + @Test + void testHourScalarFunction() { + createSourceTable(List.of("TIMESTAMP_COL"), List.of("TIMESTAMP"), new Object[][] { { "2021-02-16 11:48:01" } }); + this.virtualSchema = SETUP.createVirtualSchema(Collections.emptyMap(), MYSQL_SOURCE_SCHEMA); + final String query = "SELECT HOUR(timestamp_col) FROM " + this.virtualSchema.getName() + "." + + MYSQL_SOURCE_TABLE; + assertVsQuery(query, table().row(11).matches(TypeMatchMode.NO_JAVA_TYPE_CHECK)); + } + + @Nested + @DisplayName("Datatype tests") + class DatatypeTest { + @Test + void testBit() throws SQLException { + final String query = "SELECT \"BiT_Col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 BOOLEAN"), // + List.of("true", "true", "false", "false")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void importDataTypesFromResultSet() throws SQLException { - Assume.assumeTrue(runCharsetTest()); - - // final ResultSet actual = getActualResultSet(query); - final Exception exception = assertThrows(DatabaseObjectException.class, () -> { - setupMySQLTableWithLatin1AndVirtualSchemaWithStrategy( - DataTypeDetection.Strategy.FROM_RESULT_SET); - }); - assertThat(exception.getMessage(), containsString("E-VSCJDBC-47")); + void testTinyInt() throws SQLException { + final String query = "SELECT \"tinyint_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 DECIMAL(3,0)"), // + List.of("127", "-127", "0", "0")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void importDataTypesExasolCalculated() throws SQLException { - Assume.assumeTrue(runCharsetTest()); - assertDoesNotThrow(() -> setupMySQLTableWithLatin1AndVirtualSchemaWithStrategy( - DataTypeDetection.Strategy.EXASOL_CALCULATED)); + void testBoolean() throws SQLException { + final String query = "SELECT \"BOOL_COL\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 BOOLEAN"), // + List.of("true", "false", "true", "false")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } - private boolean runCharsetTest() { - final ExasolDockerImageReference dockerImage = SETUP.getExasolContainer().getDockerImageReference(); - if (!dockerImage.hasMajor() || !dockerImage.hasMinor() || !dockerImage.hasFix()) { - return false; - } - final Version version = Version.of(dockerImage.getMajor(), dockerImage.getMinor(), - dockerImage.getFixVersion()); - if ((dockerImage.getMajor() == 7) && version.isGreaterOrEqualThan(Version.parse("7.1.14"))) { - return true; - } - if ((dockerImage.getMajor() == 8) && version.isGreaterOrEqualThan(Version.parse("8.6.0"))) { - return true; - } - return false; + @Test + void testSmallInt() throws SQLException { + final String query = "SELECT \"smallint_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 DECIMAL(5,0)"), // + List.of("32767", "-32768", "0", "0")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } - private String setupMySQLTableWithLatin1AndVirtualSchemaWithStrategy(final DataTypeDetection.Strategy strategy) - throws SQLException { - final String tableName = MYSQL_SOURCE_TABLE; - // create SQL schema with latin1 characterset and virtual schema - createMySqlTableContainingCharAndEnumWithCharacterSet(MYSQL_SOURCE_SCHEMA, tableName, "latin1"); - - this.virtualSchema = SETUP.createVirtualSchema( // - Map.of(DataTypeDetection.STRATEGY_PROPERTY, strategy.name()), // - MYSQL_SOURCE_SCHEMA); - - final ColumnInspector inspector = SETUP.getColumnInspector(MYSQL_SOURCE_SCHEMA); - inspector.describeFromMetadata(MYSQL_SOURCE_SCHEMA, MYSQL_SOURCE_TABLE); - - final String query = String.format("select * from %s.%s", MYSQL_SOURCE_SCHEMA, MYSQL_SOURCE_TABLE); - inspector.describeFromQuery(MYSQL_SOURCE_SCHEMA, query); - - return selectEverythingFromVirtualSchemaTable(tableName); + @Test + void testMediumInt() throws SQLException { + final String query = "SELECT \"mediumint_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 DECIMAL(7,0)"), // + List.of("8388607", "-8388608", "0", "0")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } - private String selectEverythingFromVirtualSchemaTable(final String tableName) { - return "SELECT * FROM " + this.virtualSchema.getName() + "." + tableName; + @Test + void testInt() throws SQLException { + final String query = "SELECT \"int_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 DECIMAL(10,0)"), // + List.of("2147483647", "-2147483648", "0", "0")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } - private static final char[] GERMAN_UMLAUT = { 0xDC }; - private static final String SPECIAL_CHAR = new String(GERMAN_UMLAUT); - private static final String SPECIAL_CHAR_QUOTED = "'" + SPECIAL_CHAR + "'"; - - private void createMySqlTableContainingCharAndEnumWithCharacterSet(final String schemaName, - final String tableName, final String characterSet) { - - this.sourceSchema = getSchemaWithCharacterSet(schemaName, "latin1"); - - final String mySqlEnum = "ENUM('A', " + SPECIAL_CHAR_QUOTED + ")"; + @Test + void testBigInt() throws SQLException { + final String query = "SELECT \"bigint_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 DECIMAL(19,0)"), // + List.of("9223372036854775807", "-9223372036854775808", "0", "0")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } - final Table table = this.sourceSchema.createTable(tableName, List.of("c1", "c2"), - List.of("CHAR(1)", mySqlEnum)); + @Test + void testDecimal() throws SQLException { + final String query = "SELECT \"decimal_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 DECIMAL(5,2)"), // + List.of("999.99", "-999.99", "0", "0")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } - table.insert(SPECIAL_CHAR, SPECIAL_CHAR); + @Test + void testFloat() throws SQLException { + final String query = "SELECT \"float_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 DOUBLE PRECISION"), // + List.of("999.0001", "-999.9999", "0", "0")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } - private static MySqlSchema getSchemaWithCharacterSet(final String schemaName, final String characterSet) { - return new MySqlSchema(SETUP.getTableWriterWithCharacterSet(characterSet), - MySQLIdentifier.of(schemaName)); + @Test + void testDouble() throws SQLException { + final String query = "SELECT \"double_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 DOUBLE PRECISION"), // + List.of("999.00009", "-999.9999", "0", "0")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void testSelectAll() throws SQLException { - final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE; - final ResultSet actualResultSet = getActualResultSet(query); - final ResultSet expected = getExpectedResultSet(List.of("col1 INT", "col2 BOOLEAN", "col3 VARCHAR(20)"), // - List.of("-100, true, 'a'", // - "-1, false, 'abbb'", // - "0, true, 'b'", // - "10, false, 'bbbb'", // - "50, true, 'abc'", // - "100, false, 'a'")); - assertThat(actualResultSet, matchesResultSet(expected)); + void testDate() throws SQLException { + final String query = "SELECT \"date_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 DATE"), // + List.of("'1000-01-01'", "'9999-12-31'", "null", "null")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void testAggregateGroupByColumn() throws SQLException { - final String query = "SELECT \"bool_col\", min(\"int_col\") FROM " + virtualSchemaJdbc + "." - + MYSQL_SIMPLE_TABLE + " GROUP BY \"bool_col\""; - final ResultSet expected = getExpectedResultSet(List.of("bool_col BOOLEAN", "int_col DECIMAL(10,0)"), - List.of("true, -100", // - "false, -1")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void testDatetime() throws SQLException { + final String query = "SELECT \"datetime_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 TIMESTAMP"), // + List.of("'1000-01-01 00:00:00'", "'9999-12-31 22:59:59'", "null", "null")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void testAggregateHaving() throws SQLException { - final String query = "SELECT \"bool_col\", min(\"int_col\") FROM " + virtualSchemaJdbc + "." - + MYSQL_SIMPLE_TABLE + " GROUP BY \"bool_col\" HAVING MIN(\"int_col\") < 0"; - final ResultSet expected = getExpectedResultSet(List.of("bool_col BOOLEAN", "int_col DECIMAL(10,0)"), - List.of("true, -100", // - "false, -1")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void testTimestamp() throws SQLException { + final String query = "SELECT \"timestamp_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 TIMESTAMP"), // + List.of("'1970-01-01 00:00:01.000000'", "'2037-01-19 03:14:08.0'", "null", "null")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - // =, !=, <, <=, >, >= - void testComparisonPredicates() throws SQLException { - final String query = "SELECT \"int_col\", \"int_col\" = 60, \"int_col\" != 60, \"int_col\" < 60, " - + "\"int_col\" <= 60, \"int_col\" > 60, \"int_col\" >= 60 FROM " + virtualSchemaJdbc - + "." + MYSQL_SIMPLE_TABLE + " WHERE \"int_col\" = 0"; - final ResultSet expected = getExpectedResultSet( - List.of("int_col DECIMAL(10,0)", "b1 BOOLEAN", "b2 BOOLEAN", "b3 BOOLEAN", "b4 BOOLEAN", - "b5 BOOLEAN", "b6 BOOLEAN"), // - List.of("0, 0, 1, 1, 1, 0, 0")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void testTime() throws SQLException { + final String query = "SELECT \"time_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 TIMESTAMP"), // + List.of("'1970-01-01 16:59:59.000000'", "'1970-01-01 05:34:13.000000'", "null", "null")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - // NOT, AND, OR - void testLogicalPredicates() throws SQLException { - final String query = "SELECT \"int_col\" FROM " + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE - + " WHERE (\"int_col\" < 0 or \"int_col\" > 0) AND NOT (\"int_col\" is null)"; - final ResultSet expected = getExpectedResultSet(List.of("int_col DECIMAL(10,0)"), // - List.of("-100", // - "-1", // - "10", // - "50", // - "100")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void testYear() throws SQLException { + final String query = "SELECT \"year_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 DATE"), // + List.of("'1901-01-01'", "'2155-01-01'", "'1901-01-01'", "'2069-01-01'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - // LIKE, LIKE ESCAPE (not pushed down) - void testLikePredicates() throws SQLException { - final String query = "SELECT \"varchar_col\", \"varchar_col\" LIKE 'a%' ESCAPE 'a' FROM " - + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE + " WHERE (\"varchar_col\" LIKE 'a%')"; - final ResultSet expected = getExpectedResultSet(List.of("varchar_col VARCHAR(10)", "bool_col BOOLEAN"), - List.of("'a', false", // - "'abbb', false", // - "'abc', false", // - "'a', false")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void testUnsupported() { + final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + MYSQL_STRING_DATATYPES_TABLE; + assertDoesNotThrow(() -> getActualResultSet(query)); } @Test - // BETWEEN, IN, IS NULL, !=NULL(rewritten to "IS NOT NULL") - void testMiscPredicates() throws SQLException { - final String query = "SELECT \"int_col\", \"int_col\" in (56, 61), \"int_col\" is null, \"int_col\" != null" - + " FROM " + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE - + " WHERE \"int_col\" between -10 and 51"; - final ResultSet expected = getExpectedResultSet( - List.of("int_col DECIMAL(10,0)", "b1 BOOLEAN", "b2 BOOLEAN", "b3 BOOLEAN"), // - List.of("-1, false, false, false", // - "0, false, false, false", // - "10, false, false, false", // - "50, false, false, false")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void testTinyText() throws SQLException { + final String query = "SELECT \"tinytext_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_STRING_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 VARCHAR(100)"), // + List.of("'a'", "'b'", "'aaaaaaaaaaaaa'", "'a'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void testCountSumAggregateFunction() throws SQLException { - final String query = "SELECT COUNT(\"int_col\"), COUNT(*), COUNT(DISTINCT \"int_col\"), SUM(\"int_col\"), " - + "SUM(DISTINCT \"int_col\") FROM " + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("a DECIMAL(10,0)", "b DECIMAL(10,0)", - "c DECIMAL(10,0)", "d DECIMAL(19,0)", "e DECIMAL(19,0)"), List.of("6, 6, 6, 59, 59")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void testText() throws SQLException { + final String query = "SELECT \"text_col\" FROM " + virtualSchemaJdbc + "." + MYSQL_STRING_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 VARCHAR(65535)"), // + List.of("'text'", "'text2'", "'text3'", "'text'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void testAvgMinMaxAggregateFunction() throws SQLException { - final String query = "SELECT AVG(\"int_col\"), MIN(\"int_col\"), MAX(\"int_col\") FROM " - + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE; - final ResultSet expected = getExpectedResultSet( // - List.of("a DOUBLE", "b DECIMAL(10,0)", "c DECIMAL(10,0)"), // - List.of("9.833300000000001, -100.0000, 100.0000")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void testMediumText() throws SQLException { + final String query = "SELECT \"mediumtext_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_STRING_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 VARCHAR(2000000)"), // + List.of("'mediumtext'", "'mediumtext2'", "NULL", "NULL")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void testCastedStringFunctions() throws SQLException { - final String query = "SELECT concat(upper(\"varchar_col\"),lower(repeat(\"varchar_col\",2))) FROM " - + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("a VARCHAR(100)"), // - List.of("'Aaa'", // - "'ABBBabbbabbb'", // - "'Bbb'", // - "'BBBBbbbbbbbb'", // - "'ABCabcabc'", // - "'Aaa'")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void testLongText() throws SQLException { + final String query = "SELECT \"longtext_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_STRING_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 VARCHAR(2000000)"), // + List.of("'longtext'", "'longtext2'", "NULL", "NULL")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void testRewrittenDivAndModFunctions() throws SQLException { - final String query = "SELECT DIV(\"int_col\",\"int_col\"), mod(\"int_col\",\"int_col\") FROM " - + virtualSchemaJdbc + "." + MYSQL_SIMPLE_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("a DECIMAL(10,0)", "b DECIMAL(10,0)"), // - List.of("1, 0", // - "1, 0", // - "null, null", // - "1, 0", // - "1, 0", // - "1, 0")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void testEnum() throws SQLException { + final String query = "SELECT \"enum_col\" FROM " + virtualSchemaJdbc + "." + MYSQL_STRING_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 CHAR"), // + List.of("'1'", "'2'", "'3'", "NULL")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void testRewrittenSubStringFunction() throws SQLException { - final String query = "SELECT substring(\"varchar_col\" FROM 1 FOR 2) FROM " + virtualSchemaJdbc + "." - + MYSQL_SIMPLE_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("a VARCHAR(100)"), // - List.of("'a'", // - "'ab'", // - "'b'", // - "'bb'", // - "'ab'", // - "'a'")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void testSet() throws SQLException { + final String query = "SELECT \"set_col\" FROM " + virtualSchemaJdbc + "." + MYSQL_STRING_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 CHAR"), // + List.of("'1'", "'1'", "NULL", "NULL")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void testOrderByLimit() throws SQLException { - final String query = "SELECT \"bool_col\", \"int_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_SIMPLE_TABLE + " ORDER BY \"int_col\" LIMIT 3"; - final ResultSet expected = getExpectedResultSet(List.of("a BOOLEAN", "b DECIMAL(10,0)"), // - List.of("true, -100", // - "false, -1", // - "true, 0")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void testVarchar() throws SQLException { + final String query = "SELECT \"varchar_col\" FROM " + virtualSchemaJdbc + "." + + MYSQL_STRING_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 VARCHAR(100)"), // + List.of("'ab'", "'a'", "''", "NULL")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void testOrderByLimitOffset() throws SQLException { - final String query = "SELECT \"bool_col\", \"int_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_SIMPLE_TABLE + " ORDER BY \"int_col\" LIMIT 2 OFFSET 1"; - final ResultSet expected = getExpectedResultSet(List.of("a BOOLEAN", "b DECIMAL(10,0)"), // - List.of("false, -1", // - "true, 0")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); + void testChar() throws SQLException { + final String query = "SELECT \"char_col\" FROM " + virtualSchemaJdbc + "." + MYSQL_STRING_DATATYPES_TABLE; + final ResultSet expected = getExpectedResultSet(List.of("col1 CHAR(255)"), // + List.of("'asd24'", "'11111'", "''", "NULL")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } + } + @Nested + @DisplayName("Join test") + class JoinTest { @Test - void testLeftShiftScalarFunction() { - createSourceTable(List.of("INT_COL_1", "INT_COL_2"), List.of("INT", "INT"), - new Object[][] { { 10, 3 } }); - this.virtualSchema = SETUP.createVirtualSchema(Collections.emptyMap(), MYSQL_SOURCE_SCHEMA); - final String query = "SELECT BIT_LSHIFT(INT_COL_1, INT_COL_2) FROM " + this.virtualSchema.getName() - + "." + MYSQL_SOURCE_TABLE; - assertVsQuery(query, table().row(80).matches(TypeMatchMode.NO_JAVA_TYPE_CHECK)); + void testInnerJoin() throws SQLException { + final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 + " a INNER JOIN " + + virtualSchemaJdbc + "." + TABLE_JOIN_2 + " b ON a.\"x\"=b.\"x\""; + final ResultSet expected = getExpectedResultSet( + List.of("x INT", "y VARCHAR(100)", "a INT", "b VARCHAR(100)"), // + List.of("2,'bbb', 2,'bbb'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } - private void createSourceTable(final List columnNames, final List types, - final Object[][] values) { - this.sourceSchema = SETUP.getMySqlObjectFactory().createSchema(MYSQL_SOURCE_SCHEMA); - final Table table = this.sourceSchema.createTable(MYSQL_SOURCE_TABLE, columnNames, types); - for (final Object[] value : values) { - table.insert(value); - } + @Test + void testInnerJoinWithProjection() throws SQLException { + final String query = "SELECT b.\"y\" || " + virtualSchemaJdbc + "." + TABLE_JOIN_1 + ".\"y\" FROM " + + virtualSchemaJdbc + "." + TABLE_JOIN_1 + " INNER JOIN " + virtualSchemaJdbc + "." + TABLE_JOIN_2 + + " b ON " + virtualSchemaJdbc + "." + TABLE_JOIN_1 + ".\"x\"=b.\"x\""; + final ResultSet expected = getExpectedResultSet(List.of("y VARCHAR(100)"), // + List.of("'bbbbbb'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } - private ResultSet query(final String sql) throws SQLException { - return SETUP.getExasolStatement().executeQuery(sql); + @Test + void testLeftJoin() throws SQLException { + final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 + " a LEFT OUTER JOIN " + + virtualSchemaJdbc + "." + TABLE_JOIN_2 + " b ON a.\"x\"=b.\"x\" ORDER BY a.\"x\""; + final ResultSet expected = getExpectedResultSet( + List.of("x INT", "y VARCHAR(100)", "a INT", "b VARCHAR(100)"), // + List.of("1, 'aaa', null, null", // + "2, 'bbb', 2, 'bbb'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void testRightShiftScalarFunction() { - createSourceTable(List.of("INT_COL_1", "INT_COL_2"), List.of("INT", "INT"), - new Object[][] { { 10, 3 } }); - this.virtualSchema = SETUP.createVirtualSchema(Collections.emptyMap(), MYSQL_SOURCE_SCHEMA); - final String query = "SELECT BIT_RSHIFT(INT_COL_1, INT_COL_2) FROM " + this.virtualSchema.getName() - + "." + MYSQL_SOURCE_TABLE; - assertVsQuery(query, table().row(1).matches(TypeMatchMode.NO_JAVA_TYPE_CHECK)); + void testRightJoin() throws SQLException { + final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 + " a RIGHT OUTER JOIN " + + virtualSchemaJdbc + "." + TABLE_JOIN_2 + " b ON a.\"x\"=b.\"x\" ORDER BY a.\"x\""; + final ResultSet expected = getExpectedResultSet( + List.of("x INT", "y VARCHAR(100)", "a INT", "b VARCHAR(100)"), // + List.of("null, null, 3, 'ccc'", // + "2, 'bbb', 2, 'bbb'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } - private void assertVsQuery(final String sql, final Matcher expected) { - try { - assertThat(query(sql), expected); - } catch (final SQLException exception) { - fail("Unable to run assertion query: " + sql + "\nCaused by: " + exception.getMessage()); - } + @Test + void testFullOuterJoin() throws SQLException { + final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 + " a FULL OUTER JOIN " + + virtualSchemaJdbc + "." + TABLE_JOIN_2 + " b ON a.\"x\"=b.\"x\" ORDER BY a.\"x\""; + final ResultSet expected = getExpectedResultSet( + List.of("x INT", "y VARCHAR(100)", "a INT", "b VARCHAR(100)"), // + List.of("1, 'aaa', null, null", // + "2, 'bbb', 2, 'bbb'", // + "null, null, 3, 'ccc'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } @Test - void testHourScalarFunction() { - createSourceTable(List.of("TIMESTAMP_COL"), List.of("TIMESTAMP"), - new Object[][] { { "2021-02-16 11:48:01" } }); - this.virtualSchema = SETUP.createVirtualSchema(Collections.emptyMap(), MYSQL_SOURCE_SCHEMA); - final String query = "SELECT HOUR(timestamp_col) FROM " + this.virtualSchema.getName() + "." - + MYSQL_SOURCE_TABLE; - assertVsQuery(query, table().row(11).matches(TypeMatchMode.NO_JAVA_TYPE_CHECK)); - } - - @Nested - @DisplayName("Datatype tests") - class DatatypeTest { - @Test - void testBit() throws SQLException { - final String query = "SELECT \"BiT_Col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 BOOLEAN"), // - List.of("true", "true", "false", "false")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testTinyInt() throws SQLException { - final String query = "SELECT \"tinyint_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 DECIMAL(3,0)"), // - List.of("127", "-127", "0", "0")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testBoolean() throws SQLException { - final String query = "SELECT \"BOOL_COL\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 BOOLEAN"), // - List.of("true", "false", "true", "false")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testSmallInt() throws SQLException { - final String query = "SELECT \"smallint_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 DECIMAL(5,0)"), // - List.of("32767", "-32768", "0", "0")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testMediumInt() throws SQLException { - final String query = "SELECT \"mediumint_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 DECIMAL(7,0)"), // - List.of("8388607", "-8388608", "0", "0")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testInt() throws SQLException { - final String query = "SELECT \"int_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 DECIMAL(10,0)"), // - List.of("2147483647", "-2147483648", "0", "0")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testBigInt() throws SQLException { - final String query = "SELECT \"bigint_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 DECIMAL(19,0)"), // - List.of("9223372036854775807", "-9223372036854775808", "0", "0")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testDecimal() throws SQLException { - final String query = "SELECT \"decimal_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 DECIMAL(5,2)"), // - List.of("999.99", "-999.99", "0", "0")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testFloat() throws SQLException { - final String query = "SELECT \"float_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 DOUBLE PRECISION"), // - List.of("999.0001", "-999.9999", "0", "0")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testDouble() throws SQLException { - final String query = "SELECT \"double_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 DOUBLE PRECISION"), // - List.of("999.00009", "-999.9999", "0", "0")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testDate() throws SQLException { - final String query = "SELECT \"date_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 DATE"), // - List.of("'1000-01-01'", "'9999-12-31'", "null", "null")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testDatetime() throws SQLException { - final String query = "SELECT \"datetime_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 TIMESTAMP"), // - List.of("'1000-01-01 00:00:00'", "'9999-12-31 22:59:59'", "null", "null")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testTimestamp() throws SQLException { - final String query = "SELECT \"timestamp_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 TIMESTAMP"), // - List.of("'1970-01-01 00:00:01.000000'", "'2037-01-19 03:14:08.0'", "null", - "null")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testTime() throws SQLException { - final String query = "SELECT \"time_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 TIMESTAMP"), // - List.of("'1970-01-01 16:59:59.000000'", "'1970-01-01 05:34:13.000000'", "null", - "null")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testYear() throws SQLException { - final String query = "SELECT \"year_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_NUMERIC_DATE_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 DATE"), // - List.of("'1901-01-01'", "'2155-01-01'", "'1901-01-01'", "'2069-01-01'")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testUnsupported() { - final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + MYSQL_STRING_DATATYPES_TABLE; - assertDoesNotThrow(() -> getActualResultSet(query)); - } - - @Test - void testTinyText() throws SQLException { - final String query = "SELECT \"tinytext_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_STRING_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 VARCHAR(100)"), // - List.of("'a'", "'b'", "'aaaaaaaaaaaaa'", "'a'")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testText() throws SQLException { - final String query = "SELECT \"text_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_STRING_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 VARCHAR(65535)"), // - List.of("'text'", "'text2'", "'text3'", "'text'")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testMediumText() throws SQLException { - final String query = "SELECT \"mediumtext_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_STRING_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 VARCHAR(2000000)"), // - List.of("'mediumtext'", "'mediumtext2'", "NULL", "NULL")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testLongText() throws SQLException { - final String query = "SELECT \"longtext_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_STRING_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 VARCHAR(2000000)"), // - List.of("'longtext'", "'longtext2'", "NULL", "NULL")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testEnum() throws SQLException { - final String query = "SELECT \"enum_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_STRING_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 CHAR"), // - List.of("'1'", "'2'", "'3'", "NULL")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testSet() throws SQLException { - final String query = "SELECT \"set_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_STRING_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 CHAR"), // - List.of("'1'", "'1'", "NULL", "NULL")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testVarchar() throws SQLException { - final String query = "SELECT \"varchar_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_STRING_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 VARCHAR(100)"), // - List.of("'ab'", "'a'", "''", "NULL")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testChar() throws SQLException { - final String query = "SELECT \"char_col\" FROM " + virtualSchemaJdbc + "." - + MYSQL_STRING_DATATYPES_TABLE; - final ResultSet expected = getExpectedResultSet(List.of("col1 CHAR(255)"), // - List.of("'asd24'", "'11111'", "''", "NULL")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - } - - @Nested - @DisplayName("Join test") - class JoinTest { - @Test - void testInnerJoin() throws SQLException { - final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 - + " a INNER JOIN " + virtualSchemaJdbc + "." + TABLE_JOIN_2 - + " b ON a.\"x\"=b.\"x\""; - final ResultSet expected = getExpectedResultSet( - List.of("x INT", "y VARCHAR(100)", "a INT", "b VARCHAR(100)"), // - List.of("2,'bbb', 2,'bbb'")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testInnerJoinWithProjection() throws SQLException { - final String query = "SELECT b.\"y\" || " + virtualSchemaJdbc + "." + TABLE_JOIN_1 - + ".\"y\" FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 + " INNER JOIN " - + virtualSchemaJdbc + "." + TABLE_JOIN_2 + " b ON " + virtualSchemaJdbc + "." - + TABLE_JOIN_1 + ".\"x\"=b.\"x\""; - final ResultSet expected = getExpectedResultSet(List.of("y VARCHAR(100)"), // - List.of("'bbbbbb'")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testLeftJoin() throws SQLException { - final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 - + " a LEFT OUTER JOIN " + virtualSchemaJdbc + "." + TABLE_JOIN_2 - + " b ON a.\"x\"=b.\"x\" ORDER BY a.\"x\""; - final ResultSet expected = getExpectedResultSet( - List.of("x INT", "y VARCHAR(100)", "a INT", "b VARCHAR(100)"), // - List.of("1, 'aaa', null, null", // - "2, 'bbb', 2, 'bbb'")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testRightJoin() throws SQLException { - final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 - + " a RIGHT OUTER JOIN " + virtualSchemaJdbc + "." + TABLE_JOIN_2 - + " b ON a.\"x\"=b.\"x\" ORDER BY a.\"x\""; - final ResultSet expected = getExpectedResultSet( - List.of("x INT", "y VARCHAR(100)", "a INT", "b VARCHAR(100)"), // - List.of("null, null, 3, 'ccc'", // - "2, 'bbb', 2, 'bbb'")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testFullOuterJoin() throws SQLException { - final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 - + " a FULL OUTER JOIN " + virtualSchemaJdbc + "." + TABLE_JOIN_2 - + " b ON a.\"x\"=b.\"x\" ORDER BY a.\"x\""; - final ResultSet expected = getExpectedResultSet( - List.of("x INT", "y VARCHAR(100)", "a INT", "b VARCHAR(100)"), // - List.of("1, 'aaa', null, null", // - "2, 'bbb', 2, 'bbb'", // - "null, null, 3, 'ccc'")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testRightJoinWithComplexCondition() throws SQLException { - final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 - + " a RIGHT OUTER JOIN " + virtualSchemaJdbc + "." + TABLE_JOIN_2 - + " b ON a.\"x\"||a.\"y\"=b.\"x\"||b.\"y\" ORDER BY a.\"x\""; - final ResultSet expected = getExpectedResultSet( - List.of("x INT", "y VARCHAR(100)", "a INT", "b VARCHAR(100)"), // - List.of("null, null, 3, 'ccc'", // - "2, 'bbb', 2, 'bbb'")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } - - @Test - void testFullOuterJoinWithComplexCondition() throws SQLException { - final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 - + " a FULL OUTER JOIN " + virtualSchemaJdbc + "." + TABLE_JOIN_2 - + " b ON a.\"x\"-b.\"x\"=0 ORDER BY a.\"x\""; - final ResultSet expected = getExpectedResultSet( - List.of("x INT", "y VARCHAR(100)", "a INT", "b VARCHAR(100)"), // - List.of("1, 'aaa', null, null", // - "2, 'bbb', 2, 'bbb'", // - "null, null, 3, 'ccc'")); - assertThat(getActualResultSet(query), matchesResultSet(expected)); - } + void testRightJoinWithComplexCondition() throws SQLException { + final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 + " a RIGHT OUTER JOIN " + + virtualSchemaJdbc + "." + TABLE_JOIN_2 + + " b ON a.\"x\"||a.\"y\"=b.\"x\"||b.\"y\" ORDER BY a.\"x\""; + final ResultSet expected = getExpectedResultSet( + List.of("x INT", "y VARCHAR(100)", "a INT", "b VARCHAR(100)"), // + List.of("null, null, 3, 'ccc'", // + "2, 'bbb', 2, 'bbb'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); } + + @Test + void testFullOuterJoinWithComplexCondition() throws SQLException { + final String query = "SELECT * FROM " + virtualSchemaJdbc + "." + TABLE_JOIN_1 + " a FULL OUTER JOIN " + + virtualSchemaJdbc + "." + TABLE_JOIN_2 + " b ON a.\"x\"-b.\"x\"=0 ORDER BY a.\"x\""; + final ResultSet expected = getExpectedResultSet( + List.of("x INT", "y VARCHAR(100)", "a INT", "b VARCHAR(100)"), // + List.of("1, 'aaa', null, null", // + "2, 'bbb', 2, 'bbb'", // + "null, null, 3, 'ccc'")); + assertThat(getActualResultSet(query), matchesResultSet(expected)); + } + } } \ No newline at end of file diff --git a/src/test/java/com/exasol/adapter/dialects/mysql/MySQLVirtualSchemaIntegrationTestSetup.java b/src/test/java/com/exasol/adapter/dialects/mysql/MySQLVirtualSchemaIntegrationTestSetup.java index 7baea1a..5d2c001 100644 --- a/src/test/java/com/exasol/adapter/dialects/mysql/MySQLVirtualSchemaIntegrationTestSetup.java +++ b/src/test/java/com/exasol/adapter/dialects/mysql/MySQLVirtualSchemaIntegrationTestSetup.java @@ -59,7 +59,6 @@ public class MySQLVirtualSchemaIntegrationTestSetup implements Closeable { uploadVsJarToBucket(bucket); this.exasolConnection = this.exasolContainer.createConnection(""); this.exasolStatement = this.exasolConnection.createStatement(); - // exasolStatement.execute("CONTROL SET TRACE LEVEL NOTICE WITH LOG TIMEOUT 0;"); this.mySqlConnection = this.mySqlContainer.createConnection(""); this.mySqlStatement = this.mySqlConnection.createStatement(); From 4fb076557202bff8316c2d75ee2f17040fc3ddda Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Wed, 17 Jan 2024 17:13:33 +0100 Subject: [PATCH 15/17] improved summary --- doc/changes/changes_5.0.0.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/changes/changes_5.0.0.md b/doc/changes/changes_5.0.0.md index a527c45..3661e13 100644 --- a/doc/changes/changes_5.0.0.md +++ b/doc/changes/changes_5.0.0.md @@ -1,12 +1,13 @@ # Virtual Schema for MySQL 5.0.0, released 2024-01-15 -Code name: Char set is always `utf-8`, deprecated IMPORT_DATA_TYPES `from_result_set` value . +Code name: Char set is always `utf-8`, deprecated IMPORT_DATA_TYPES `FROM_RESULT_SET` value . ## Summary The behaviour when it comes to character sets is now simplified, The target char set is now always UTF-8. -The IMPORT_DATA_TYPES property (and value FROM_RESULT_SET) are now deprecated (change in vs-common-jdbc). +The `IMPORT_DATA_TYPES` property (and value `FROM_RESULT_SET`) are now deprecated (change in vs-common-jdbc): +An exception will be thrown when users use`FROM_RESULT_SET`. The exception message warns the user that the value is no longer supported and the property itself is also deprecated. ## Refactoring From c27cc5fcd1bee1dc877a4b44407d43b9d4bd409c Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Wed, 17 Jan 2024 17:27:35 +0100 Subject: [PATCH 16/17] change release date --- doc/changes/changes_5.0.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/changes/changes_5.0.0.md b/doc/changes/changes_5.0.0.md index 3661e13..b912f48 100644 --- a/doc/changes/changes_5.0.0.md +++ b/doc/changes/changes_5.0.0.md @@ -1,4 +1,4 @@ -# Virtual Schema for MySQL 5.0.0, released 2024-01-15 +# Virtual Schema for MySQL 5.0.0, released 2024-01-18 Code name: Char set is always `utf-8`, deprecated IMPORT_DATA_TYPES `FROM_RESULT_SET` value . From 5703b23918e42045e1866eec1b112b4b478b42c3 Mon Sep 17 00:00:00 2001 From: Pieterjan Spoelders Date: Thu, 18 Jan 2024 10:10:24 +0100 Subject: [PATCH 17/17] Apply suggestions from code review Co-authored-by: Christoph Pirkl --- doc/changes/changes_5.0.0.md | 4 ++-- .../com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/changes/changes_5.0.0.md b/doc/changes/changes_5.0.0.md index b912f48..f994fb2 100644 --- a/doc/changes/changes_5.0.0.md +++ b/doc/changes/changes_5.0.0.md @@ -1,13 +1,13 @@ # Virtual Schema for MySQL 5.0.0, released 2024-01-18 -Code name: Char set is always `utf-8`, deprecated IMPORT_DATA_TYPES `FROM_RESULT_SET` value . +Code name: Charset is always `utf-8`, deprecated IMPORT_DATA_TYPES `FROM_RESULT_SET` value. ## Summary The behaviour when it comes to character sets is now simplified, The target char set is now always UTF-8. The `IMPORT_DATA_TYPES` property (and value `FROM_RESULT_SET`) are now deprecated (change in vs-common-jdbc): -An exception will be thrown when users use`FROM_RESULT_SET`. The exception message warns the user that the value is no longer supported and the property itself is also deprecated. +An exception will be thrown when users use `FROM_RESULT_SET`. The exception message warns the user that the value is no longer supported and the property itself is also deprecated. ## Refactoring diff --git a/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java b/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java index 4504957..59cd90c 100644 --- a/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java +++ b/src/test/java/com/exasol/adapter/dialects/mysql/MySQLSqlDialectIT.java @@ -183,7 +183,6 @@ private boolean runCharsetTest() { private String setupMySQLTableWithLatin1AndVirtualSchemaWithStrategy(final DataTypeDetection.Strategy strategy) throws SQLException { final String tableName = MYSQL_SOURCE_TABLE; - // create SQL schema with latin1 characterset and virtual schema createMySqlTableContainingCharAndEnumWithCharacterSet(MYSQL_SOURCE_SCHEMA, tableName, "latin1"); this.virtualSchema = SETUP.createVirtualSchema( //