Skip to content

Commit

Permalink
[apache#5631] Add test cases for other supported catalogs in Trino Ca…
Browse files Browse the repository at this point in the history 
…scading (apache#5632)

### What changes were proposed in this pull request?

1. Trino connector: add some external table type test cases in hive
catalog.
2. Cascading connector: Add cases to cover hive,PG,Iceberg catalogs.
3. Some test cases can't pass on the Trino cascading v0.0.1, I move to
"ignored" directory.

### Why are the changes needed?

In trino-cascading-testsets, it only covered MySQL catalog.
We need to add more test cases for other supported catalogs.

Fix: apache#5631 

### Does this PR introduce _any_ user-facing change?

N/A

### How was this patch tested?

```
./gradlew -PskipTests -PtestMode=embedded -PjdkVersion=17 -PskipDockerTests=false :trino-connector:integration-test:test

./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=17 -PskipDockerTests=false :trino-connector:integration-test:test

trino-connector/integration-test/trino-test-tools/run_test.sh

2024-11-20 19:39:05 INFO  [main] TrinoQueryIT:402 - All testers have finished. Total:116, Pass: 116
hive:hive/00001_datatype.sql, PASS, 4.743, 
iceberg:lakehouse-iceberg-hive/00002_partition_sort_order.sql, PASS, 4.042, 
iceberg_mysql:lakehouse-iceberg-mysql/00002_partition_sort_order.sql, PASS, 1.614, 
iceberg_postgres:lakehouse-iceberg-postgres/00002_partition_sort_order.sql, PASS, 3.290, 
mysql:jdbc-mysql/00001_datatype.sql, PASS, 1.753, 
mysql:jdbc-mysql/00002_pushdown.sql, PASS, 1.095, 
mysql:jdbc-mysql/catalog_mysql_cascading_test.sql, PASS, 0.207, 
mysql:tpcds/00000.sql, PASS, 16.305, 
mysql:tpcds/00001.sql, PASS, 1.036, 
mysql:tpcds/00002.sql, PASS, 1.954, 
mysql:tpcds/00003.sql, PASS, 0.795, 
mysql:tpcds/00004.sql, PASS, 3.311, 
mysql:tpcds/00005.sql, PASS, 2.008, 
mysql:tpcds/00006.sql, PASS, 1.282, 
mysql:tpcds/00007.sql, PASS, 1.439, 
....
mysql:tpch/00021.sql, PASS, 1.336, 
mysql:tpch/00022.sql, PASS, 0.536, 
pg:jdbc-postgresql/00001_datatype.sql, PASS, 1.259, 
pg:jdbc-postgresql/00002_pushdown.sql, PASS, 1.583, 
pg:jdbc-postgresql/catalog_pg_cascading_test.sql, PASS, 0.217, 
All testers have finished. Total:116, Pass: 116
```
  • Loading branch information
@danhuawang
danhuawang authored Dec 3, 2024
1 parent f135a03 commit 986af92
Show file tree
Hide file tree
Showing 64 changed files with 1,989 additions and 112 deletions.
1 change: 1 addition & 0 deletions LICENSE
View file
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,7 @@
Trino
./integration-test-common/src/test/java/org/apache/gravitino/integration/test/util/CloseableGroup.java
./trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/hive/SortingColumn.java
./trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00012_format.sql

Apache Arrow
./dev/ci/util_free_space.sh
Expand Down
4 changes: 2 additions & 2 deletions ...est/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ void executeSqlFile(
sqls = removeSqlComments(sqls);

Matcher sqlMatcher =
Pattern.compile("(\\w.*?);", Pattern.DOTALL | Pattern.UNIX_LINES).matcher(sqls);
Pattern.compile("([<\\w].*?);", Pattern.DOTALL | Pattern.UNIX_LINES).matcher(sqls);
while (sqlMatcher.find()) {
String sql = sqlMatcher.group(1);
sql = resolveParameters(sql);
Expand Down Expand Up @@ -221,7 +221,7 @@ void executeSqlFileWithCheckResult(
String testResults = TrinoQueryITBase.readFileToString(resultFileName);

Matcher sqlMatcher =
Pattern.compile("(\\w.*?);", Pattern.DOTALL | Pattern.UNIX_LINES).matcher(sqls);
Pattern.compile("([<\\w].*?);", Pattern.DOTALL | Pattern.UNIX_LINES).matcher(sqls);
Matcher resultMatcher =
Pattern.compile("((\".*?\")\\n{2,})|((\\S.*?)\\n{2,})", Pattern.DOTALL | Pattern.UNIX_LINES)
.matcher(testResults);
Expand Down
2 changes: 1 addition & 1 deletion ...src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ protected static void dropCatalog(String catalogName) {
});

metalake.disableCatalog(catalogName);
metalake.dropCatalog(catalogName);
metalake.dropCatalog(catalogName, true);
LOG.info("Drop catalog \"{}.{}\"", metalakeName, catalogName);
}

Expand Down
25 changes: 25 additions & 0 deletions ...src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,10 @@
import java.time.ZoneId;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import jodd.io.StringOutputStream;
import okhttp3.logging.HttpLoggingInterceptor;
import org.awaitility.Awaitility;
import org.jline.terminal.Terminal;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -65,6 +67,29 @@ private QueryRunner createQueryRunner() throws Exception {
}

String runQuery(String query) {
String retryFlag = "<RETRY_WITH_NOT_EXISTS>";
if (!query.startsWith(retryFlag)) {
return runQueryOnce(query);
} else {
String finalQuery = query.replace(retryFlag, "");
AtomicReference<String> output = new AtomicReference<>("");
Awaitility.await()
.atMost(30, TimeUnit.SECONDS)
.pollInterval(1, TimeUnit.SECONDS)
.until(
() -> {
String result = runQueryOnce(finalQuery);
if (!result.contains("does not exist")) {
output.set(result);
return true;
}
return false;
});
return output.get();
}
}

String runQueryOnce(String query) {
Query queryResult = queryRunner.startQuery(query);
StringOutputStream outputStream = new StringOutputStream();
queryResult.renderOutput(
Expand Down
52 changes: 26 additions & 26 deletions ...ctor/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00008_decimal.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,56 +2,56 @@ CREATE SCHEMA gt_hive.gt_decimal_db1;

USE gt_hive.gt_decimal_db1;

CREATE TABLE test_decimal_bounds (amount DECIMAL(10, 2));
CREATE TABLE tb01 (amount DECIMAL(10, 2));

INSERT INTO test_decimal_bounds VALUES (12345.67), (-9999999.99), (0.01);
INSERT INTO tb01 VALUES (12345.67), (-9999999.99), (0.01);

INSERT INTO test_decimal_bounds VALUES (123456789.00); -- Exceeds precision
INSERT INTO tb01 VALUES (123456789.00); -- Exceeds precision

SELECT * FROM test_decimal_bounds;
SELECT * FROM tb01;

CREATE TABLE test_decimal_aggregation (value DECIMAL(12, 3));
CREATE TABLE tb02 (value DECIMAL(12, 3));

INSERT INTO test_decimal_aggregation VALUES (1234.567), (8901.234), (567.890);
INSERT INTO tb02 VALUES (1234.567), (8901.234), (567.890);

SELECT SUM(value) FROM test_decimal_aggregation;
SELECT SUM(value) FROM tb02;

SELECT AVG(value) FROM test_decimal_aggregation;
SELECT AVG(value) FROM tb02;

CREATE TABLE test_decimal_arithmetic (val1 DECIMAL(5, 2), val2 DECIMAL(4, 1));
CREATE TABLE tb03 (val1 DECIMAL(5, 2), val2 DECIMAL(4, 1));

INSERT INTO test_decimal_arithmetic VALUES (123.45,10.1);
INSERT INTO tb03 VALUES (123.45,10.1);

SELECT val1 + val2 FROM test_decimal_arithmetic;
SELECT val1 + val2 FROM tb03;

SELECT val1 * val2 FROM test_decimal_arithmetic;
SELECT val1 * val2 FROM tb03;

SELECT val1 / val2 FROM test_decimal_arithmetic;
SELECT val1 / val2 FROM tb03;

CREATE TABLE test_decimal_max_min (max_min_val DECIMAL(18, 4));
CREATE TABLE tb04 (max_min_val DECIMAL(18, 4));

INSERT INTO test_decimal_max_min VALUES (99999999999999.9999);
INSERT INTO tb04 VALUES (99999999999999.9999);

INSERT INTO test_decimal_max_min VALUES (-99999999999999.9999);
INSERT INTO tb04 VALUES (-99999999999999.9999);

INSERT INTO test_decimal_max_min VALUES (100000000000000.0000); -- Exceeds max
INSERT INTO tb04 VALUES (100000000000000.0000); -- Exceeds max

SELECT * FROM test_decimal_max_min ORDER BY max_min_val;
SELECT * FROM tb04 ORDER BY max_min_val;

CREATE TABLE test_decimal_nulls (nullable_val DECIMAL(8, 2));
CREATE TABLE tb05 (nullable_val DECIMAL(8, 2));

INSERT INTO test_decimal_nulls VALUES (NULL), (123.45), (NULL);
INSERT INTO tb05 VALUES (NULL), (123.45), (NULL);

SELECT * FROM test_decimal_nulls;
SELECT * FROM tb05;

DROP TABLE gt_hive.gt_decimal_db1.test_decimal_bounds;
DROP TABLE gt_hive.gt_decimal_db1.tb01;

DROP TABLE gt_hive.gt_decimal_db1.test_decimal_aggregation;
DROP TABLE gt_hive.gt_decimal_db1.tb02;

DROP TABLE gt_hive.gt_decimal_db1.test_decimal_arithmetic;
DROP TABLE gt_hive.gt_decimal_db1.tb03;

DROP TABLE gt_hive.gt_decimal_db1.test_decimal_max_min;
DROP TABLE gt_hive.gt_decimal_db1.tb04;

DROP TABLE gt_hive.gt_decimal_db1.test_decimal_nulls;
DROP TABLE gt_hive.gt_decimal_db1.tb05;

DROP SCHEMA gt_hive.gt_decimal_db1;
60 changes: 30 additions & 30 deletions ...nector/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00009_array.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,64 +2,64 @@ CREATE SCHEMA gt_hive.gt_array_db1;

USE gt_hive.gt_array_db1;

CREATE TABLE test_array_basic (int_array ARRAY(INTEGER));
CREATE TABLE tb01 (int_array ARRAY(INTEGER));

INSERT INTO test_array_basic VALUES (ARRAY[1, 2, 3]), (ARRAY[4, 5, NULL, 7]), (ARRAY[]);
INSERT INTO tb01 VALUES (ARRAY[1, 2, 3]), (ARRAY[4, 5, NULL, 7]), (ARRAY[]);

SELECT * FROM test_array_basic;
SELECT * FROM tb01;

SELECT int_array, CARDINALITY(int_array) AS array_length FROM test_array_basic;
SELECT int_array, CARDINALITY(int_array) AS array_length FROM tb01;

CREATE TABLE test_array_access (elements ARRAY(VARCHAR));
CREATE TABLE tb02 (elements ARRAY(VARCHAR));

INSERT INTO test_array_access VALUES (ARRAY['apple', 'banana', 'cherry']);
INSERT INTO tb02 VALUES (ARRAY['apple', 'banana', 'cherry']);

SELECT elements[1] AS first_element, elements[2] AS second_element FROM test_array_access;
SELECT elements[1] AS first_element, elements[2] AS second_element FROM tb02;

SELECT * FROM test_array_basic WHERE contains(int_array, 2);
SELECT * FROM tb01 WHERE contains(int_array, 2);

CREATE TABLE test_array_concat (array1 ARRAY(INTEGER), array2 ARRAY(INTEGER));
CREATE TABLE tb03 (array1 ARRAY(INTEGER), array2 ARRAY(INTEGER));

INSERT INTO test_array_concat VALUES (ARRAY[1, 2, 3], ARRAY[4, 5]);
INSERT INTO tb03 VALUES (ARRAY[1, 2, 3], ARRAY[4, 5]);

SELECT array1, array2, CONCAT(array1, array2) AS concatenated_array FROM test_array_concat;
SELECT array1, array2, CONCAT(array1, array2) AS concatenated_array FROM tb03;

CREATE TABLE test_array_sort (unsorted_array ARRAY(INTEGER));
CREATE TABLE tb04 (unsorted_array ARRAY(INTEGER));

INSERT INTO test_array_sort VALUES (ARRAY[3, 1, 2]), (ARRAY[9, 7, 8]);
INSERT INTO tb04 VALUES (ARRAY[3, 1, 2]), (ARRAY[9, 7, 8]);

SELECT unsorted_array, array_sort(unsorted_array) AS sorted_array FROM test_array_sort;
SELECT unsorted_array, array_sort(unsorted_array) AS sorted_array FROM tb04;

CREATE TABLE test_array_nulls (mixed_array ARRAY(INTEGER));
CREATE TABLE tb05 (mixed_array ARRAY(INTEGER));

INSERT INTO test_array_nulls VALUES (ARRAY[1, NULL, 3]), (ARRAY[NULL, NULL]);
INSERT INTO tb05 VALUES (ARRAY[1, NULL, 3]), (ARRAY[NULL, NULL]);

SELECT mixed_array, CARDINALITY(mixed_array) FROM test_array_nulls;
SELECT mixed_array, CARDINALITY(mixed_array) FROM tb05;

CREATE TABLE test_array_agg (val INTEGER);
CREATE TABLE tb06 (val INTEGER);

INSERT INTO test_array_agg VALUES (1), (2), (3), (4);
INSERT INTO tb06 VALUES (1), (2), (3), (4);

SELECT ARRAY_AGG(val) AS aggregated_array FROM test_array_agg;
SELECT ARRAY_AGG(val) AS tb07 FROM tb06;

CREATE TABLE test_nested_array (nested_array ARRAY(ARRAY(VARCHAR)));
CREATE TABLE tb08 (nested_array ARRAY(ARRAY(VARCHAR)));

INSERT INTO test_nested_array VALUES (ARRAY[ARRAY['a', 'b'], ARRAY['c', 'd']]);
INSERT INTO tb08 VALUES (ARRAY[ARRAY['a', 'b'], ARRAY['c', 'd']]);

SELECT nested_array FROM test_nested_array;
SELECT nested_array FROM tb08;

DROP TABLE gt_hive.gt_array_db1.test_array_basic;
DROP TABLE gt_hive.gt_array_db1.tb01;

DROP TABLE gt_hive.gt_array_db1.test_array_access;
DROP TABLE gt_hive.gt_array_db1.tb02;

DROP TABLE gt_hive.gt_array_db1.test_array_concat;
DROP TABLE gt_hive.gt_array_db1.tb03;

DROP TABLE gt_hive.gt_array_db1.test_array_sort;
DROP TABLE gt_hive.gt_array_db1.tb04;

DROP TABLE gt_hive.gt_array_db1.test_array_nulls;
DROP TABLE gt_hive.gt_array_db1.tb05;

DROP TABLE gt_hive.gt_array_db1.test_array_agg;
DROP TABLE gt_hive.gt_array_db1.tb06;

DROP TABLE gt_hive.gt_array_db1.test_nested_array;
DROP TABLE gt_hive.gt_array_db1.tb08;

DROP SCHEMA gt_hive.gt_array_db1;
44 changes: 22 additions & 22 deletions ...onnector/integration-test/src/test/resources/trino-ci-testset/testsets/hive/00010_map.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,48 +2,48 @@ CREATE SCHEMA gt_hive.gt_map_db1;

USE gt_hive.gt_map_db1;

CREATE TABLE test_map_nulls (string_map MAP(VARCHAR, VARCHAR));
CREATE TABLE tb01 (string_map MAP(VARCHAR, VARCHAR));

INSERT INTO test_map_nulls VALUES (MAP(ARRAY['key1'], ARRAY[NULL]));
INSERT INTO tb01 VALUES (MAP(ARRAY['key1'], ARRAY[NULL]));

INSERT INTO test_map_nulls VALUES (MAP(ARRAY[NULL], ARRAY['value1']));
INSERT INTO tb01 VALUES (MAP(ARRAY[NULL], ARRAY['value1']));

SELECT * FROM test_map_nulls;
SELECT * FROM tb01;

INSERT INTO test_map_nulls VALUES (MAP(ARRAY[], ARRAY[]));
INSERT INTO tb01 VALUES (MAP(ARRAY[], ARRAY[]));

SELECT * FROM test_map_nulls ORDER BY cardinality(string_map);
SELECT * FROM tb01 ORDER BY cardinality(string_map);

INSERT INTO test_map_nulls VALUES (MAP(ARRAY['dup', 'dup'], ARRAY['value1', 'value2']));
INSERT INTO tb01 VALUES (MAP(ARRAY['dup', 'dup'], ARRAY['value1', 'value2']));

CREATE TABLE test_map_types (int_decimal_map MAP(INTEGER, DECIMAL(10, 2)));
CREATE TABLE tb02 (int_decimal_map MAP(INTEGER, DECIMAL(10, 2)));

INSERT INTO test_map_types VALUES (MAP(ARRAY[1, 2147483647], ARRAY[12345.67, 99999.99]));
INSERT INTO tb02 VALUES (MAP(ARRAY[1, 2147483647], ARRAY[12345.67, 99999.99]));

SELECT * FROM test_map_types;
SELECT * FROM tb02;

INSERT INTO test_map_nulls VALUES (MAP(ARRAY['k1', 'k2', 'k3'], ARRAY['v1', 'v2', 'v3']));
INSERT INTO tb01 VALUES (MAP(ARRAY['k1', 'k2', 'k3'], ARRAY['v1', 'v2', 'v3']));

SELECT element_at(string_map, 'k1') AS key1_value, element_at(string_map, 'k3') AS key3_value FROM test_map_nulls ORDER BY key1_value;
SELECT element_at(string_map, 'k1') AS key1_value, element_at(string_map, 'k3') AS key3_value FROM tb01 ORDER BY key1_value;

CREATE TABLE test_map_complex (map_of_arrays MAP(VARCHAR, ARRAY(INTEGER)));
CREATE TABLE tb03 (map_of_arrays MAP(VARCHAR, ARRAY(INTEGER)));

INSERT INTO test_map_complex VALUES (MAP(ARRAY['a', 'b'], ARRAY[ARRAY[1, 2], ARRAY[3, 4, 5]]));
INSERT INTO tb03 VALUES (MAP(ARRAY['a', 'b'], ARRAY[ARRAY[1, 2], ARRAY[3, 4, 5]]));

SELECT * FROM test_map_complex;
SELECT * FROM tb03;

CREATE TABLE test_map_aggregation (map_data MAP(VARCHAR, INTEGER));
CREATE TABLE tb04 (map_data MAP(VARCHAR, INTEGER));

INSERT INTO test_map_aggregation VALUES (MAP(ARRAY['a', 'b'], ARRAY[1, 2])), (MAP(ARRAY['a', 'b'], ARRAY[3, 4]));
INSERT INTO tb04 VALUES (MAP(ARRAY['a', 'b'], ARRAY[1, 2])), (MAP(ARRAY['a', 'b'], ARRAY[3, 4]));

SELECT map_data['a'] AS key_a, SUM(map_data['b']) AS sum_b FROM test_map_aggregation GROUP BY map_data['a'] ORDER BY key_a;
SELECT map_data['a'] AS key_a, SUM(map_data['b']) AS sum_b FROM tb04 GROUP BY map_data['a'] ORDER BY key_a;

DROP TABLE gt_hive.gt_map_db1.test_map_nulls;
DROP TABLE gt_hive.gt_map_db1.tb01;

DROP TABLE gt_hive.gt_map_db1.test_map_types;
DROP TABLE gt_hive.gt_map_db1.tb02;

DROP TABLE gt_hive.gt_map_db1.test_map_complex;
DROP TABLE gt_hive.gt_map_db1.tb03;

DROP TABLE gt_hive.gt_map_db1.test_map_aggregation;
DROP TABLE gt_hive.gt_map_db1.tb04;

DROP SCHEMA gt_hive.gt_map_db1;
Loading

0 comments on commit 986af92

Please sign in to comment.