diff --git a/BaseJdbcClient.java b/BaseJdbcClient.java new file mode 100644 index 000000000000..d595ac4137cb --- /dev/null +++ b/BaseJdbcClient.java @@ -0,0 +1,1053 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.plugin.jdbc; + +import com.google.common.base.CharMatcher; +import com.google.common.base.VerifyException; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedSet; +import io.airlift.log.Logger; +import io.airlift.units.Duration; +import io.prestosql.spi.PrestoException; +import io.prestosql.spi.connector.ColumnHandle; +import io.prestosql.spi.connector.ColumnMetadata; +import io.prestosql.spi.connector.ConnectorSession; +import io.prestosql.spi.connector.ConnectorSplitSource; +import io.prestosql.spi.connector.ConnectorTableMetadata; +import io.prestosql.spi.connector.FixedSplitSource; +import io.prestosql.spi.connector.SchemaTableName; +import io.prestosql.spi.connector.TableNotFoundException; +import io.prestosql.spi.predicate.TupleDomain; +import io.prestosql.spi.statistics.TableStatistics; +import io.prestosql.spi.type.CharType; +import io.prestosql.spi.type.DecimalType; +import io.prestosql.spi.type.Type; +import io.prestosql.spi.type.VarcharType; + +import javax.annotation.Nullable; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Set; +import java.util.UUID; +import java.util.function.BiFunction; +import java.util.function.Function; + +import static com.google.common.base.MoreObjects.firstNonNull; +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Strings.emptyToNull; +import static com.google.common.base.Strings.isNullOrEmpty; +import static com.google.common.base.Verify.verify; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static com.google.common.collect.Iterables.getOnlyElement; +import static io.prestosql.plugin.jdbc.JdbcErrorCode.JDBC_ERROR; +import static io.prestosql.plugin.jdbc.PredicatePushdownController.DISABLE_PUSHDOWN; +import static io.prestosql.plugin.jdbc.StandardColumnMappings.bigintWriteFunction; +import static io.prestosql.plugin.jdbc.StandardColumnMappings.booleanWriteFunction; +import static io.prestosql.plugin.jdbc.StandardColumnMappings.charWriteFunction; +import static io.prestosql.plugin.jdbc.StandardColumnMappings.dateWriteFunction; +import static io.prestosql.plugin.jdbc.StandardColumnMappings.doubleWriteFunction; +import static io.prestosql.plugin.jdbc.StandardColumnMappings.integerWriteFunction; +import static io.prestosql.plugin.jdbc.StandardColumnMappings.jdbcTypeToPrestoType; +import static io.prestosql.plugin.jdbc.StandardColumnMappings.longDecimalWriteFunction; +import static io.prestosql.plugin.jdbc.StandardColumnMappings.realWriteFunction; +import static io.prestosql.plugin.jdbc.StandardColumnMappings.shortDecimalWriteFunction; +import static io.prestosql.plugin.jdbc.StandardColumnMappings.smallintWriteFunction; +import static io.prestosql.plugin.jdbc.StandardColumnMappings.tinyintWriteFunction; +import static io.prestosql.plugin.jdbc.StandardColumnMappings.varbinaryWriteFunction; +import static io.prestosql.plugin.jdbc.StandardColumnMappings.varcharReadFunction; +import static io.prestosql.plugin.jdbc.StandardColumnMappings.varcharWriteFunction; +import static io.prestosql.plugin.jdbc.TypeHandlingJdbcSessionProperties.getUnsupportedTypeHandling; +import static io.prestosql.plugin.jdbc.UnsupportedTypeHandling.CONVERT_TO_VARCHAR; +import static io.prestosql.plugin.jdbc.UnsupportedTypeHandling.IGNORE; +import static io.prestosql.spi.StandardErrorCode.NOT_FOUND; +import static io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED; +import static io.prestosql.spi.type.BigintType.BIGINT; +import static io.prestosql.spi.type.BooleanType.BOOLEAN; +import static io.prestosql.spi.type.DateType.DATE; +import static io.prestosql.spi.type.DoubleType.DOUBLE; +import static io.prestosql.spi.type.IntegerType.INTEGER; +import static io.prestosql.spi.type.RealType.REAL; +import static io.prestosql.spi.type.SmallintType.SMALLINT; +import static io.prestosql.spi.type.TinyintType.TINYINT; +import static io.prestosql.spi.type.VarbinaryType.VARBINARY; +import static io.prestosql.spi.type.VarcharType.createUnboundedVarcharType; +import static java.lang.String.CASE_INSENSITIVE_ORDER; +import static java.lang.String.format; +import static java.lang.String.join; +import static java.sql.DatabaseMetaData.columnNoNulls; +import static java.util.Collections.emptyMap; +import static java.util.Collections.nCopies; +import static java.util.Locale.ENGLISH; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.stream.Collectors.joining; + +public abstract class BaseJdbcClient + implements JdbcClient +{ + private static final Logger log = Logger.get(BaseJdbcClient.class); + + private static final Map WRITE_MAPPINGS = ImmutableMap.builder() + .put(BOOLEAN, WriteMapping.booleanMapping("boolean", booleanWriteFunction())) + .put(BIGINT, WriteMapping.longMapping("bigint", bigintWriteFunction())) + .put(INTEGER, WriteMapping.longMapping("integer", integerWriteFunction())) + .put(SMALLINT, WriteMapping.longMapping("smallint", smallintWriteFunction())) + .put(TINYINT, WriteMapping.longMapping("tinyint", tinyintWriteFunction())) + .put(DOUBLE, WriteMapping.doubleMapping("double precision", doubleWriteFunction())) + .put(REAL, WriteMapping.longMapping("real", realWriteFunction())) + .put(VARBINARY, WriteMapping.sliceMapping("varbinary", varbinaryWriteFunction())) + .put(DATE, WriteMapping.longMapping("date", dateWriteFunction())) + .build(); + + protected final ConnectionFactory connectionFactory; + protected final String identifierQuote; + protected final Set jdbcTypesMappedToVarchar; + protected final boolean caseInsensitiveNameMatching; + protected final Cache> remoteSchemaNames; + protected final Cache> remoteTableNames; + + public BaseJdbcClient(BaseJdbcConfig config, String identifierQuote, ConnectionFactory connectionFactory) + { + this( + identifierQuote, + connectionFactory, + config.getJdbcTypesMappedToVarchar(), + requireNonNull(config, "config is null").isCaseInsensitiveNameMatching(), + config.getCaseInsensitiveNameMatchingCacheTtl()); + } + + public BaseJdbcClient( + String identifierQuote, + ConnectionFactory connectionFactory, + Set jdbcTypesMappedToVarchar, + boolean caseInsensitiveNameMatching, + Duration caseInsensitiveNameMatchingCacheTtl) + { + this.identifierQuote = requireNonNull(identifierQuote, "identifierQuote is null"); + this.connectionFactory = requireNonNull(connectionFactory, "connectionFactory is null"); + this.jdbcTypesMappedToVarchar = ImmutableSortedSet.orderedBy(CASE_INSENSITIVE_ORDER) + .addAll(requireNonNull(jdbcTypesMappedToVarchar, "jdbcTypesMappedToVarchar is null")) + .build(); + requireNonNull(caseInsensitiveNameMatchingCacheTtl, "caseInsensitiveNameMatchingCacheTtl is null"); + + this.caseInsensitiveNameMatching = caseInsensitiveNameMatching; + CacheBuilder remoteNamesCacheBuilder = CacheBuilder.newBuilder() + .expireAfterWrite(caseInsensitiveNameMatchingCacheTtl.toMillis(), MILLISECONDS); + this.remoteSchemaNames = remoteNamesCacheBuilder.build(); + this.remoteTableNames = remoteNamesCacheBuilder.build(); + } + + @Override + public final Set getSchemaNames(ConnectorSession session) + { + try (Connection connection = connectionFactory.openConnection(session)) { + return listSchemas(connection).stream() + .map(schemaName -> schemaName.toLowerCase(ENGLISH)) + .collect(toImmutableSet()); + } + catch (SQLException e) { + throw new PrestoException(JDBC_ERROR, e); + } + } + + protected Collection listSchemas(Connection connection) + { + ImmutableSet.Builder schemaNames = ImmutableSet.builder(); + try (ResultSet resultSet = connection.getMetaData().getSchemas(connection.getCatalog(), null)) { + boolean found = false; + while (resultSet.next()) { + found = true; + String schemaName = resultSet.getString("TABLE_SCHEM"); + // skip internal schemas + if (filterSchema(schemaName)) { + schemaNames.add(schemaName); + } + } + try { + //needed for firebird and mysql + if (!found && connection.getMetaData().getURL() != null && (connection.getMetaData().getURL().startsWith("jdbc:firebirdsql:") || connection.getMetaData().getURL().startsWith("jdbc:mysql:"))) { + schemaNames.add("@noschema@"); + } + } + catch (Exception e) { + //not all drivers have getURL...like SparkThrift + } + return schemaNames.build(); + } + catch (SQLFeatureNotSupportedException e) { + //needed for sqlite + schemaNames.add("@noschema@"); + return schemaNames.build(); + } + catch (SQLException e) { + throw new PrestoException(JDBC_ERROR, e); + } + } + + protected boolean filterSchema(String schemaName) + { + if (schemaName.equalsIgnoreCase("information_schema")) { + return false; + } + return true; + } + + @Override + public List getTableNames(ConnectorSession session, Optional schema) + { + try (Connection connection = connectionFactory.openConnection(session)) { + JdbcIdentity identity = JdbcIdentity.from(session); + Optional remoteSchema = schema.map(schemaName -> toRemoteSchemaName(identity, connection, schemaName)); + if (remoteSchema.isPresent() && !filterSchema(remoteSchema.get())) { + return ImmutableList.of(); + } + + try (ResultSet resultSet = getTables(connection, remoteSchema, Optional.empty())) { + ImmutableList.Builder list = ImmutableList.builder(); + while (resultSet.next()) { + String tableSchema = getTableSchemaName(resultSet); + if (tableSchema == null) { + tableSchema = "@noschema@"; + } + String tableName = resultSet.getString("TABLE_NAME"); + if (filterSchema(tableSchema)) { + list.add(new SchemaTableName(tableSchema, tableName)); + } + } + return list.build(); + } + } + catch (SQLException e) { + throw new PrestoException(JDBC_ERROR, e); + } + } + + @Override + public Optional getTableHandle(ConnectorSession session, SchemaTableName schemaTableName) + { + try (Connection connection = connectionFactory.openConnection(session)) { + JdbcIdentity identity = JdbcIdentity.from(session); + String remoteSchema = toRemoteSchemaName(identity, connection, schemaTableName.getSchemaName()); + String remoteTable = toRemoteTableName(identity, connection, remoteSchema, schemaTableName.getTableName()); + try (ResultSet resultSet = getTables(connection, Optional.of(remoteSchema), Optional.of(remoteTable))) { + List tableHandles = new ArrayList<>(); + while (resultSet.next()) { + tableHandles.add(new JdbcTableHandle(schemaTableName, getRemoteTable(resultSet))); + } + if (tableHandles.isEmpty()) { + return Optional.empty(); + } + if (tableHandles.size() > 1) { + throw new PrestoException(NOT_SUPPORTED, "Multiple tables matched: " + schemaTableName); + } + return Optional.of(getOnlyElement(tableHandles)); + } + } + catch (SQLException e) { + throw new PrestoException(JDBC_ERROR, e); + } + } + + @Override + public List getColumns(ConnectorSession session, JdbcTableHandle tableHandle) + { + try (Connection connection = connectionFactory.openConnection(session); + ResultSet resultSet = getColumns(tableHandle, connection.getMetaData())) { + int allColumns = 0; + List columns = new ArrayList<>(); + while (resultSet.next()) { + // skip if table doesn't match expected + if (!(Objects.equals(tableHandle.getRemoteTableName(), getRemoteTable(resultSet)))) { + continue; + } + allColumns++; + String columnName = resultSet.getString("COLUMN_NAME"); + JdbcTypeHandle typeHandle = new JdbcTypeHandle( + getInteger(resultSet, "DATA_TYPE").orElseThrow(() -> new IllegalStateException("DATA_TYPE is null")), + Optional.ofNullable(resultSet.getString("TYPE_NAME")), + getInteger(resultSet, "COLUMN_SIZE"), + getInteger(resultSet, "DECIMAL_DIGITS"), + Optional.empty(), + Optional.empty()); + Optional columnMapping = toPrestoType(session, connection, typeHandle); + log.debug("Mapping data type of '%s' column '%s': %s mapped to %s", tableHandle.getSchemaTableName(), columnName, typeHandle, columnMapping); + // skip unsupported column types + boolean nullable = (resultSet.getInt("NULLABLE") != columnNoNulls); + // Note: some databases (e.g. SQL Server) do not return column remarks/comment here. + Optional comment = Optional.ofNullable(emptyToNull(resultSet.getString("REMARKS"))); + if (columnMapping.isPresent()) { + columns.add(JdbcColumnHandle.builder() + .setColumnName(columnName) + .setJdbcTypeHandle(typeHandle) + .setColumnType(columnMapping.get().getType()) + .setNullable(nullable) + .setComment(comment) + .build()); + } + if (columnMapping.isEmpty()) { + UnsupportedTypeHandling unsupportedTypeHandling = getUnsupportedTypeHandling(session); + verify( + unsupportedTypeHandling == IGNORE, + "Unsupported type handling is set to %s, but toPrestoType() returned empty for %s", + unsupportedTypeHandling, + typeHandle); + } + } + if (columns.isEmpty()) { + // A table may have no supported columns. In rare cases (e.g. PostgreSQL) a table might have no columns at all. + throw new TableNotFoundException( + tableHandle.getSchemaTableName(), + format("Table '%s' has no supported columns (all %s columns are not supported)", tableHandle.getSchemaTableName(), allColumns)); + } + return ImmutableList.copyOf(columns); + } + catch (SQLException e) { + throw new PrestoException(JDBC_ERROR, e); + } + } + + protected static Optional getInteger(ResultSet resultSet, String columnLabel) + throws SQLException + { + int value = resultSet.getInt(columnLabel); + if (resultSet.wasNull()) { + return Optional.empty(); + } + return Optional.of(value); + } + + protected ResultSet getColumns(JdbcTableHandle tableHandle, DatabaseMetaData metadata) + throws SQLException + { + if (metadata.getSearchStringEscape() == null || metadata.getSearchStringEscape().isEmpty()) { + return metadata.getColumns( + tableHandle.getRemoteTableName().getCatalogName().orElse(null), + tableHandle.getRemoteTableName().getSchemaName().orElse(null), + Optional.of(tableHandle.getRemoteTableName().getTableName()).orElse(null), + null); + } + return metadata.getColumns( + tableHandle.getRemoteTableName().getCatalogName().orElse(null), + escapeNamePattern(tableHandle.getRemoteTableName().getSchemaName(), metadata.getSearchStringEscape()).orElse(null), + escapeNamePattern(Optional.of(tableHandle.getRemoteTableName().getTableName()), metadata.getSearchStringEscape()).orElse(null), + null); + } + + @Override + public Optional toPrestoType(ConnectorSession session, Connection connection, JdbcTypeHandle typeHandle) + { + Optional mapping = getForcedMappingToVarchar(typeHandle); + if (mapping.isPresent()) { + return mapping; + } + Optional connectorMapping = jdbcTypeToPrestoType(typeHandle); + if (connectorMapping.isPresent()) { + return connectorMapping; + } + if (getUnsupportedTypeHandling(session) == CONVERT_TO_VARCHAR) { + return mapToUnboundedVarchar(typeHandle); + } + return Optional.empty(); + } + + @Override + public List getColumnMappings(ConnectorSession session, List typeHandles) + { + try (Connection connection = connectionFactory.openConnection(session)) { + return typeHandles.stream() + .map(typeHandle -> toPrestoType(session, connection, typeHandle) + .orElseThrow(() -> new VerifyException(format("Unsupported type handle %s", typeHandle)))) + .collect(toImmutableList()); + } + catch (SQLException e) { + throw new PrestoException(JDBC_ERROR, e); + } + } + + protected Optional getForcedMappingToVarchar(JdbcTypeHandle typeHandle) + { + if (typeHandle.getJdbcTypeName().isPresent() && jdbcTypesMappedToVarchar.contains(typeHandle.getJdbcTypeName().get())) { + return mapToUnboundedVarchar(typeHandle); + } + return Optional.empty(); + } + + protected static Optional mapToUnboundedVarchar(JdbcTypeHandle typeHandle) + { + VarcharType unboundedVarcharType = createUnboundedVarcharType(); + return Optional.of(ColumnMapping.sliceMapping( + unboundedVarcharType, + varcharReadFunction(unboundedVarcharType), + (statement, index, value) -> { + throw new PrestoException( + NOT_SUPPORTED, + "Underlying type that is mapped to VARCHAR is not supported for INSERT: " + typeHandle.getJdbcTypeName().get()); + }, + DISABLE_PUSHDOWN)); + } + + @Override + public ConnectorSplitSource getSplits(ConnectorSession session, JdbcTableHandle tableHandle) + { + return new FixedSplitSource(ImmutableList.of(new JdbcSplit(Optional.empty()))); + } + + @Override + public Connection getConnection(ConnectorSession session, JdbcSplit split) + throws SQLException + { + Connection connection = connectionFactory.openConnection(session); + try { + connection.setReadOnly(true); + } + catch (SQLException e) { + //Cannot change read-only flag after establishing a connection. Use SQLiteConfig#setReadOnly and SQLiteConfig.createConnection(). + //Enabling read-only mode not supported at org.apache.hive.jdbc.HiveConnection.setReadOnly(HiveConnection.java:1411) + //if (connection.getMetaData().getSearchStringEscape() != null) { + // connection.close(); + // throw e; + //} + } + return connection; + } + + @Override + public PreparedStatement buildSql(ConnectorSession session, Connection connection, JdbcSplit split, JdbcTableHandle table, List columns) + throws SQLException + { + return new QueryBuilder(this).buildSql( + session, + connection, + table.getRemoteTableName(), + table.getGroupingSets(), + columns, + table.getConstraint(), + split.getAdditionalPredicate(), + tryApplyLimit(table.getLimit())); + } + + @Override + public void createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata) + { + try { + createTable(session, tableMetadata, tableMetadata.getTable().getTableName()); + } + catch (SQLException e) { + throw new PrestoException(JDBC_ERROR, e); + } + } + + @Override + public JdbcOutputTableHandle beginCreateTable(ConnectorSession session, ConnectorTableMetadata tableMetadata) + { + try { + return createTable(session, tableMetadata, generateTemporaryTableName()); + } + catch (SQLException e) { + throw new PrestoException(JDBC_ERROR, e); + } + } + + protected JdbcOutputTableHandle createTable(ConnectorSession session, ConnectorTableMetadata tableMetadata, String tableName) + throws SQLException + { + SchemaTableName schemaTableName = tableMetadata.getTable(); + + JdbcIdentity identity = JdbcIdentity.from(session); + if (!getSchemaNames(session).contains(schemaTableName.getSchemaName())) { + throw new PrestoException(NOT_FOUND, "Schema not found: " + schemaTableName.getSchemaName()); + } + + try (Connection connection = connectionFactory.openConnection(session)) { + boolean uppercase = connection.getMetaData().storesUpperCaseIdentifiers(); + String remoteSchema = toRemoteSchemaName(identity, connection, schemaTableName.getSchemaName()); + String remoteTable = toRemoteTableName(identity, connection, remoteSchema, schemaTableName.getTableName()); + if (uppercase) { + tableName = tableName.toUpperCase(ENGLISH); + } + String catalog = connection.getCatalog(); + + ImmutableList.Builder columnNames = ImmutableList.builder(); + ImmutableList.Builder columnTypes = ImmutableList.builder(); + ImmutableList.Builder columnList = ImmutableList.builder(); + for (ColumnMetadata column : tableMetadata.getColumns()) { + String columnName = column.getName(); + if (uppercase) { + columnName = columnName.toUpperCase(ENGLISH); + } + columnNames.add(columnName); + columnTypes.add(column.getType()); + columnList.add(getColumnDefinitionSql(session, column, columnName)); + } + + RemoteTableName remoteTableName = new RemoteTableName(Optional.ofNullable(catalog), Optional.ofNullable(remoteSchema), tableName); + String sql = createTableSql(remoteTableName, columnList.build()); + execute(connection, sql); + + return new JdbcOutputTableHandle( + catalog, + remoteSchema, + remoteTable, + columnNames.build(), + columnTypes.build(), + Optional.empty(), + tableName); + } + } + + protected String createTableSql(RemoteTableName remoteTableName, List columns) + { + return format("CREATE TABLE %s (%s)", quoted(remoteTableName), join(", ", columns)); + } + + protected String getColumnDefinitionSql(ConnectorSession session, ColumnMetadata column, String columnName) + { + StringBuilder sb = new StringBuilder() + .append(quoted(columnName)) + .append(" ") + .append(toWriteMapping(session, column.getType()).getDataType()); + if (!column.isNullable()) { + sb.append(" NOT NULL"); + } + return sb.toString(); + } + + @Override + public JdbcOutputTableHandle beginInsertTable(ConnectorSession session, JdbcTableHandle tableHandle, List columns) + { + SchemaTableName schemaTableName = tableHandle.getSchemaTableName(); + JdbcIdentity identity = JdbcIdentity.from(session); + + try (Connection connection = connectionFactory.openConnection(session)) { + boolean uppercase = connection.getMetaData().storesUpperCaseIdentifiers(); + String remoteSchema = toRemoteSchemaName(identity, connection, schemaTableName.getSchemaName()); + String remoteTable = toRemoteTableName(identity, connection, remoteSchema, schemaTableName.getTableName()); + String tableName = generateTemporaryTableName(); + if (uppercase) { + tableName = tableName.toUpperCase(ENGLISH); + } + String catalog = connection.getCatalog(); + + ImmutableList.Builder columnNames = ImmutableList.builder(); + ImmutableList.Builder columnTypes = ImmutableList.builder(); + ImmutableList.Builder jdbcColumnTypes = ImmutableList.builder(); + for (JdbcColumnHandle column : columns) { + columnNames.add(column.getColumnName()); + columnTypes.add(column.getColumnType()); + jdbcColumnTypes.add(column.getJdbcTypeHandle()); + } + + copyTableSchema(connection, catalog, remoteSchema, remoteTable, tableName, columnNames.build()); + + return new JdbcOutputTableHandle( + catalog, + remoteSchema, + remoteTable, + columnNames.build(), + columnTypes.build(), + Optional.of(jdbcColumnTypes.build()), + tableName); + } + catch (SQLException e) { + throw new PrestoException(JDBC_ERROR, e); + } + } + + protected void copyTableSchema(Connection connection, String catalogName, String schemaName, String tableName, String newTableName, List columnNames) + { + String sql = format( + "CREATE TABLE %s AS SELECT %s FROM %s WHERE 0 = 1", + quoted(catalogName, schemaName, newTableName), + columnNames.stream() + .map(this::quoted) + .collect(joining(", ")), + quoted(catalogName, schemaName, tableName)); + execute(connection, sql); + } + + protected String generateTemporaryTableName() + { + return "tmp_presto_" + UUID.randomUUID().toString().replace("-", ""); + } + + @Override + public void commitCreateTable(ConnectorSession session, JdbcOutputTableHandle handle) + { + renameTable( + session, + handle.getCatalogName(), + handle.getSchemaName(), + handle.getTemporaryTableName(), + new SchemaTableName(handle.getSchemaName(), handle.getTableName())); + } + + @Override + public void renameTable(ConnectorSession session, JdbcTableHandle handle, SchemaTableName newTableName) + { + renameTable(session, handle.getCatalogName(), handle.getSchemaName(), handle.getTableName(), newTableName); + } + + protected void renameTable(ConnectorSession session, String catalogName, String schemaName, String tableName, SchemaTableName newTable) + { + try (Connection connection = connectionFactory.openConnection(session)) { + String newSchemaName = newTable.getSchemaName(); + String newTableName = newTable.getTableName(); + if (connection.getMetaData().storesUpperCaseIdentifiers()) { + newSchemaName = newSchemaName.toUpperCase(ENGLISH); + newTableName = newTableName.toUpperCase(ENGLISH); + } + String sql = format( + "ALTER TABLE %s RENAME TO %s", + quoted(catalogName, schemaName, tableName), + quoted(catalogName, newSchemaName, newTableName)); + execute(connection, sql); + } + catch (SQLException e) { + throw new PrestoException(JDBC_ERROR, e); + } + } + + @Override + public void finishInsertTable(ConnectorSession session, JdbcOutputTableHandle handle) + { + String temporaryTable = quoted(handle.getCatalogName(), handle.getSchemaName(), handle.getTemporaryTableName()); + String targetTable = quoted(handle.getCatalogName(), handle.getSchemaName(), handle.getTableName()); + String columnNames = handle.getColumnNames().stream() + .map(this::quoted) + .collect(joining(", ")); + String insertSql = format("INSERT INTO %s (%s) SELECT %s FROM %s", targetTable, columnNames, columnNames, temporaryTable); + String cleanupSql = "DROP TABLE " + temporaryTable; + + try (Connection connection = getConnection(session, handle)) { + execute(connection, insertSql); + } + catch (SQLException e) { + throw new PrestoException(JDBC_ERROR, e); + } + + try (Connection connection = getConnection(session, handle)) { + execute(connection, cleanupSql); + } + catch (SQLException e) { + log.warn(e, "Failed to cleanup temporary table: %s", temporaryTable); + } + } + + @Override + public void addColumn(ConnectorSession session, JdbcTableHandle handle, ColumnMetadata column) + { + try (Connection connection = connectionFactory.openConnection(session)) { + String columnName = column.getName(); + if (connection.getMetaData().storesUpperCaseIdentifiers()) { + columnName = columnName.toUpperCase(ENGLISH); + } + String sql = format( + "ALTER TABLE %s ADD %s", + quoted(handle.getRemoteTableName()), + getColumnDefinitionSql(session, column, columnName)); + execute(connection, sql); + } + catch (SQLException e) { + throw new PrestoException(JDBC_ERROR, e); + } + } + + @Override + public void renameColumn(ConnectorSession session, JdbcTableHandle handle, JdbcColumnHandle jdbcColumn, String newColumnName) + { + try (Connection connection = connectionFactory.openConnection(session)) { + if (connection.getMetaData().storesUpperCaseIdentifiers()) { + newColumnName = newColumnName.toUpperCase(ENGLISH); + } + String sql = format( + "ALTER TABLE %s RENAME COLUMN %s TO %s", + quoted(handle.getRemoteTableName()), + jdbcColumn.getColumnName(), + newColumnName); + execute(connection, sql); + } + catch (SQLException e) { + throw new PrestoException(JDBC_ERROR, e); + } + } + + @Override + public void dropColumn(ConnectorSession session, JdbcTableHandle handle, JdbcColumnHandle column) + { + String sql = format( + "ALTER TABLE %s DROP COLUMN %s", + quoted(handle.getRemoteTableName()), + column.getColumnName()); + execute(session, sql); + } + + @Override + public void dropTable(ConnectorSession session, JdbcTableHandle handle) + { + String sql = "DROP TABLE " + quoted(handle.getRemoteTableName()); + execute(session, sql); + } + + @Override + public void rollbackCreateTable(ConnectorSession session, JdbcOutputTableHandle handle) + { + dropTable(session, new JdbcTableHandle( + new SchemaTableName(handle.getSchemaName(), handle.getTemporaryTableName()), + handle.getCatalogName(), + handle.getSchemaName(), + handle.getTemporaryTableName())); + } + + @Override + public String buildInsertSql(JdbcOutputTableHandle handle) + { + return format( + "INSERT INTO %s (%s) VALUES (%s)", + quoted(handle.getCatalogName(), handle.getSchemaName(), handle.getTemporaryTableName()), + handle.getColumnNames().stream() + .map(this::quoted) + .collect(joining(", ")), + join(",", nCopies(handle.getColumnNames().size(), "?"))); + } + + @Override + public Connection getConnection(ConnectorSession session, JdbcOutputTableHandle handle) + throws SQLException + { + return connectionFactory.openConnection(session); + } + + @Override + public PreparedStatement getPreparedStatement(Connection connection, String sql) + throws SQLException + { + return connection.prepareStatement(sql); + } + + protected ResultSet getTables(Connection connection, Optional schemaName, Optional tableName) + throws SQLException + { + DatabaseMetaData metadata = connection.getMetaData(); + if (metadata.getSearchStringEscape() == null || metadata.getSearchStringEscape().isEmpty()) { + return metadata.getTables( + connection.getCatalog(), + schemaName.isPresent() ? schemaName.get() : null, + tableName.isPresent() ? tableName.get() : null, + new String[] {"TABLE", "VIEW"}); + } + return metadata.getTables( + connection.getCatalog(), + escapeNamePattern(schemaName, metadata.getSearchStringEscape()).orElse(null), + escapeNamePattern(tableName, metadata.getSearchStringEscape()).orElse(null), + new String[] {"TABLE", "VIEW"}); + } + + protected String getTableSchemaName(ResultSet resultSet) + throws SQLException + { + return resultSet.getString("TABLE_SCHEM"); + } + + protected String toRemoteSchemaName(JdbcIdentity identity, Connection connection, String schemaName) + { + requireNonNull(schemaName, "schemaName is null"); + verify(CharMatcher.forPredicate(Character::isUpperCase).matchesNoneOf(schemaName), "Expected schema name from internal metadata to be lowercase: %s", schemaName); + + if (caseInsensitiveNameMatching) { + try { + Map mapping = remoteSchemaNames.getIfPresent(identity); + if (mapping != null && !mapping.containsKey(schemaName)) { + // This might be a schema that has just been created. Force reload. + mapping = null; + } + if (mapping == null) { + mapping = listSchemasByLowerCase(connection); + remoteSchemaNames.put(identity, mapping); + } + String remoteSchema = mapping.get(schemaName); + if (remoteSchema != null) { + return remoteSchema; + } + } + catch (RuntimeException e) { + throw new PrestoException(JDBC_ERROR, "Failed to find remote schema name: " + firstNonNull(e.getMessage(), e), e); + } + } + + try { + DatabaseMetaData metadata = connection.getMetaData(); + try { + if (metadata.storesUpperCaseIdentifiers()) { + return schemaName.toUpperCase(ENGLISH); + } + } + catch (Exception e) { + //not all drivers have storesUpperCaseIdentifiers...like SparkThrift + } + return schemaName; + } + catch (SQLException e) { + throw new PrestoException(JDBC_ERROR, e); + } + } + + protected Map listSchemasByLowerCase(Connection connection) + { + return listSchemas(connection).stream() + .collect(toImmutableMap(schemaName -> schemaName.toLowerCase(ENGLISH), schemaName -> schemaName)); + } + + protected String toRemoteTableName(JdbcIdentity identity, Connection connection, String remoteSchema, String tableName) + { + requireNonNull(remoteSchema, "remoteSchema is null"); + requireNonNull(tableName, "tableName is null"); + verify(CharMatcher.forPredicate(Character::isUpperCase).matchesNoneOf(tableName), "Expected table name from internal metadata to be lowercase: %s", tableName); + + if (caseInsensitiveNameMatching) { + try { + RemoteTableNameCacheKey cacheKey = new RemoteTableNameCacheKey(identity, remoteSchema); + Map mapping = remoteTableNames.getIfPresent(cacheKey); + if (mapping != null && !mapping.containsKey(tableName)) { + // This might be a table that has just been created. Force reload. + mapping = null; + } + if (mapping == null) { + mapping = listTablesByLowerCase(connection, remoteSchema); + remoteTableNames.put(cacheKey, mapping); + } + String remoteTable = mapping.get(tableName); + if (remoteTable != null) { + return remoteTable; + } + } + catch (RuntimeException e) { + throw new PrestoException(JDBC_ERROR, "Failed to find remote table name: " + firstNonNull(e.getMessage(), e), e); + } + } + + try { + DatabaseMetaData metadata = connection.getMetaData(); + try { + if (metadata.storesUpperCaseIdentifiers()) { + return tableName.toUpperCase(ENGLISH); + } + } + catch (Exception e) { + //not all drivers have storesUpperCaseIdentifiers...like SparkThrift + } + return tableName; + } + catch (SQLException e) { + throw new PrestoException(JDBC_ERROR, e); + } + } + + protected Map listTablesByLowerCase(Connection connection, String remoteSchema) + { + try (ResultSet resultSet = getTables(connection, Optional.of(remoteSchema), Optional.empty())) { + ImmutableMap.Builder map = ImmutableMap.builder(); + while (resultSet.next()) { + String tableName = resultSet.getString("TABLE_NAME"); + map.put(tableName.toLowerCase(ENGLISH), tableName); + } + return map.build(); + } + catch (SQLException e) { + throw new PrestoException(JDBC_ERROR, e); + } + } + + @Override + public TableStatistics getTableStatistics(ConnectorSession session, JdbcTableHandle handle, TupleDomain tupleDomain) + { + return TableStatistics.empty(); + } + + @Override + public void createSchema(ConnectorSession session, String schemaName) + { + execute(session, "CREATE SCHEMA " + quoted(schemaName)); + } + + @Override + public void dropSchema(ConnectorSession session, String schemaName) + { + execute(session, "DROP SCHEMA " + quoted(schemaName)); + } + + protected void execute(ConnectorSession session, String query) + { + try (Connection connection = connectionFactory.openConnection(session)) { + execute(connection, query); + } + catch (SQLException e) { + throw new PrestoException(JDBC_ERROR, e); + } + } + + protected void execute(Connection connection, String query) + { + try (Statement statement = connection.createStatement()) { + log.debug("Execute: %s", query); + statement.execute(query); + } + catch (SQLException e) { + PrestoException exception = new PrestoException(JDBC_ERROR, e); + exception.addSuppressed(new RuntimeException("Query: " + query)); + throw exception; + } + } + + @Override + public WriteMapping toWriteMapping(ConnectorSession session, Type type) + { + if (type instanceof VarcharType) { + VarcharType varcharType = (VarcharType) type; + String dataType; + if (varcharType.isUnbounded()) { + dataType = "varchar"; + } + else { + dataType = "varchar(" + varcharType.getBoundedLength() + ")"; + } + return WriteMapping.sliceMapping(dataType, varcharWriteFunction()); + } + if (type instanceof CharType) { + return WriteMapping.sliceMapping("char(" + ((CharType) type).getLength() + ")", charWriteFunction()); + } + if (type instanceof DecimalType) { + DecimalType decimalType = (DecimalType) type; + String dataType = format("decimal(%s, %s)", decimalType.getPrecision(), decimalType.getScale()); + if (decimalType.isShort()) { + return WriteMapping.longMapping(dataType, shortDecimalWriteFunction(decimalType)); + } + return WriteMapping.sliceMapping(dataType, longDecimalWriteFunction(decimalType)); + } + + WriteMapping writeMapping = WRITE_MAPPINGS.get(type); + if (writeMapping != null) { + return writeMapping; + } + throw new PrestoException(NOT_SUPPORTED, "Unsupported column type: " + type.getDisplayName()); + } + + protected Function tryApplyLimit(OptionalLong limit) + { + if (limit.isEmpty()) { + return Function.identity(); + } + return limitFunction() + .map(limitFunction -> (Function) sql -> limitFunction.apply(sql, limit.getAsLong())) + .orElseGet(Function::identity); + } + + @Override + public boolean supportsLimit() + { + return limitFunction().isPresent(); + } + + protected Optional> limitFunction() + { + return Optional.empty(); + } + + @Override + public boolean isLimitGuaranteed(ConnectorSession session) + { + throw new PrestoException(JDBC_ERROR, "limitFunction() is implemented without isLimitGuaranteed()"); + } + + @Override + public String quoted(String name) + { + name = name.replace(identifierQuote, identifierQuote + identifierQuote); + return identifierQuote + name + identifierQuote; + } + + @Override + public String quoted(RemoteTableName remoteTableName) + { + return quoted( + remoteTableName.getCatalogName().orElse(null), + remoteTableName.getSchemaName().orElse(null), + remoteTableName.getTableName()); + } + + @Override + public Map getTableProperties(ConnectorSession session, JdbcTableHandle tableHandle) + { + return emptyMap(); + } + + protected String quoted(@Nullable String catalog, @Nullable String schema, String table) + { + StringBuilder sb = new StringBuilder(); + //ideally connnection knows that jdbc driver is dremio too, otherwise a dremio catalog in non-dremio db (ie sybase for example) would not work + if (!isNullOrEmpty(catalog) && !catalog.equals("DREMIO")) { + sb.append(quoted(catalog)).append("."); + } + if (!isNullOrEmpty(schema)) { + sb.append(quoted(schema)).append("."); + } + sb.append(quoted(table)); + return sb.toString(); + } + + protected static Optional escapeNamePattern(Optional name, String escape) + { + return name.map(string -> escapeNamePattern(string, escape)); + } + + private static String escapeNamePattern(String name, String escape) + { + requireNonNull(name, "name is null"); + requireNonNull(escape, "escape is null"); + checkArgument(!escape.isEmpty(), "Escape string must not be empty"); + checkArgument(!escape.equals("_"), "Escape string must not be '_'"); + checkArgument(!escape.equals("%"), "Escape string must not be '%'"); + name = name.replace(escape, escape + escape); + name = name.replace("_", escape + "_"); + name = name.replace("%", escape + "%"); + return name; + } + + private static RemoteTableName getRemoteTable(ResultSet resultSet) + throws SQLException + { + return new RemoteTableName( + Optional.ofNullable((resultSet.getString("TABLE_CAT") == null || resultSet.getString("TABLE_CAT").isEmpty()) ? null : resultSet.getString("TABLE_CAT")), + Optional.ofNullable(resultSet.getString("TABLE_SCHEM")), + resultSet.getString("TABLE_NAME")); + } +} diff --git a/BaseJdbcConfig.java b/BaseJdbcConfig.java new file mode 100644 index 000000000000..49f1fce5242c --- /dev/null +++ b/BaseJdbcConfig.java @@ -0,0 +1,130 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.plugin.jdbc; + +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableSet; +import io.airlift.configuration.Config; +import io.airlift.configuration.ConfigDescription; +import io.airlift.units.Duration; +import io.airlift.units.MinDuration; + +import javax.validation.constraints.NotNull; + +import java.util.Set; + +import static com.google.common.base.Strings.nullToEmpty; +import static java.util.concurrent.TimeUnit.MINUTES; + +public class BaseJdbcConfig +{ + private String connectionUrl; + private boolean caseInsensitiveNameMatching; + private Duration caseInsensitiveNameMatchingCacheTtl = new Duration(1, MINUTES); + private Set jdbcTypesMappedToVarchar = ImmutableSet.of(); + private Duration metadataCacheTtl = new Duration(0, MINUTES); + private boolean cacheMissing; + private String driverClass; + + @NotNull + public String getConnectionUrl() + { + return connectionUrl; + } + + @Config("connection-url") + public BaseJdbcConfig setConnectionUrl(String connectionUrl) + { + this.connectionUrl = connectionUrl; + return this; + } + + public String getDriverClass() + { + return driverClass; + } + + @Config("driver-class") + public BaseJdbcConfig setDriverClass(String driverClass) + { + this.driverClass = driverClass; + return this; + } + + public boolean isCaseInsensitiveNameMatching() + { + return caseInsensitiveNameMatching; + } + + @Config("case-insensitive-name-matching") + public BaseJdbcConfig setCaseInsensitiveNameMatching(boolean caseInsensitiveNameMatching) + { + this.caseInsensitiveNameMatching = caseInsensitiveNameMatching; + return this; + } + + @NotNull + @MinDuration("0ms") + public Duration getCaseInsensitiveNameMatchingCacheTtl() + { + return caseInsensitiveNameMatchingCacheTtl; + } + + @Config("case-insensitive-name-matching.cache-ttl") + public BaseJdbcConfig setCaseInsensitiveNameMatchingCacheTtl(Duration caseInsensitiveNameMatchingCacheTtl) + { + this.caseInsensitiveNameMatchingCacheTtl = caseInsensitiveNameMatchingCacheTtl; + return this; + } + + public Set getJdbcTypesMappedToVarchar() + { + return jdbcTypesMappedToVarchar; + } + + @Config("jdbc-types-mapped-to-varchar") + public BaseJdbcConfig setJdbcTypesMappedToVarchar(String jdbcTypesMappedToVarchar) + { + this.jdbcTypesMappedToVarchar = ImmutableSet.copyOf(Splitter.on(",").omitEmptyStrings().trimResults().split(nullToEmpty(jdbcTypesMappedToVarchar))); + return this; + } + + @NotNull + @MinDuration("0ms") + public Duration getMetadataCacheTtl() + { + return metadataCacheTtl; + } + + @Config("metadata.cache-ttl") + @ConfigDescription("Determines how long meta information will be cached") + public BaseJdbcConfig setMetadataCacheTtl(Duration metadataCacheTtl) + { + this.metadataCacheTtl = metadataCacheTtl; + return this; + } + + public boolean isCacheMissing() + { + return cacheMissing; + } + + @Config("metadata.cache-missing") + @ConfigDescription("Determines if missing information will be cached") + public BaseJdbcConfig setCacheMissing(boolean cacheMissing) + { + this.cacheMissing = cacheMissing; + return this; + } +} diff --git a/GenericJdbcClient.java b/GenericJdbcClient.java new file mode 100644 index 000000000000..0388bfe8c1a6 --- /dev/null +++ b/GenericJdbcClient.java @@ -0,0 +1,148 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.plugin.genericjdbc; + +import com.google.common.base.Joiner; +import io.prestosql.plugin.jdbc.BaseJdbcClient; +import io.prestosql.plugin.jdbc.BaseJdbcConfig; +import io.prestosql.plugin.jdbc.ColumnMapping; +import io.prestosql.plugin.jdbc.ConnectionFactory; +import io.prestosql.plugin.jdbc.JdbcTypeHandle; +import io.prestosql.plugin.jdbc.WriteMapping; +import io.prestosql.spi.connector.ConnectorSession; +import io.prestosql.spi.predicate.Domain; +import io.prestosql.spi.type.CharType; +import io.prestosql.spi.type.Type; +import io.prestosql.spi.type.VarcharType; + +import javax.inject.Inject; + +import java.sql.Connection; +import java.util.List; +import java.util.Optional; +import java.util.function.UnaryOperator; + +import static io.prestosql.plugin.jdbc.PredicatePushdownController.DISABLE_PUSHDOWN; +import static io.prestosql.plugin.jdbc.StandardColumnMappings.booleanWriteFunction; +import static io.prestosql.plugin.jdbc.StandardColumnMappings.charWriteFunction; +import static io.prestosql.plugin.jdbc.StandardColumnMappings.varcharWriteFunction; +import static io.prestosql.spi.type.BooleanType.BOOLEAN; +import static java.lang.String.format; +import static java.util.stream.Collectors.joining; + +public class GenericJdbcClient + extends BaseJdbcClient +{ + private static final Joiner DOT_JOINER = Joiner.on("."); + + // Sybase supports 2100 parameters in prepared statement, let's create a space for about 4 big IN predicates + private static final int MAX_LIST_EXPRESSIONS = 500; + + // TODO improve this by calling Domain#simplify + private static final UnaryOperator DISABLE_UNSUPPORTED_PUSHDOWN = domain -> { + if (domain.getValues().getRanges().getRangeCount() <= MAX_LIST_EXPRESSIONS) { + return domain; + } + return Domain.all(domain.getType()); + }; + + //not sure if getIdentifierQuoteString from jdbc driver is accurate instead... + @Inject + public GenericJdbcClient(BaseJdbcConfig config, ConnectionFactory connectionFactory) + { + super(config, (config.getConnectionUrl().startsWith("jdbc:impala") || config.getConnectionUrl().startsWith("jdbc:hive2") || config.getConnectionUrl().startsWith("jdbc:mysql")) ? "`" : "\"", connectionFactory); + } + + protected void copyTableSchema(Connection connection, String catalogName, String schemaName, String tableName, String newTableName, List columnNames) + { + String sql = format( + "SELECT %s INTO %s FROM %s WHERE 0 = 1", + columnNames.stream() + .map(this::quoted) + .collect(joining(", ")), + quoted(catalogName, schemaName, newTableName), + quoted(catalogName, schemaName, tableName)); + execute(connection, sql); + } + + @Override + public Optional toPrestoType(ConnectorSession session, Connection connection, JdbcTypeHandle typeHandle) + { + try { + if (connection.getMetaData().getURL() != null && connection.getMetaData().getURL().startsWith("jdbc:impala") && (typeHandle.toString().contains("jdbcTypeName=ARRAY") || typeHandle.toString().contains("jdbcTypeName=MAP") || typeHandle.toString().contains("jdbcTypeName=STRUCT"))) { + return Optional.empty(); + } + } + catch (Exception e) { + //not all drivers have getURL...like SparkThrift + } + + Optional mapping = getForcedMappingToVarchar(typeHandle); + if (mapping.isPresent()) { + return mapping; + } + // TODO implement proper type mapping + return super.toPrestoType(session, connection, typeHandle) + .map(columnMapping -> new ColumnMapping( + columnMapping.getType(), + columnMapping.getReadFunction(), + columnMapping.getWriteFunction(), + DISABLE_PUSHDOWN)); + } + + @Override + public WriteMapping toWriteMapping(ConnectorSession session, Type type) + { + if (type == BOOLEAN) { + return WriteMapping.booleanMapping("bit", booleanWriteFunction()); + } + + if (type instanceof VarcharType) { + VarcharType varcharType = (VarcharType) type; + String dataType; + if (varcharType.isUnbounded() || varcharType.getBoundedLength() > 4000) { + dataType = "nvarchar(max)"; + } + else { + dataType = "nvarchar(" + varcharType.getBoundedLength() + ")"; + } + return WriteMapping.sliceMapping(dataType, varcharWriteFunction()); + } + + if (type instanceof CharType) { + CharType charType = (CharType) type; + String dataType; + if (charType.getLength() > 4000) { + dataType = "nvarchar(max)"; + } + else { + dataType = "nchar(" + charType.getLength() + ")"; + } + return WriteMapping.sliceMapping(dataType, charWriteFunction()); + } + + // TODO implement proper type mapping + return super.toWriteMapping(session, type); + } + + private static String singleQuote(String... objects) + { + return singleQuote(DOT_JOINER.join(objects)); + } + + private static String singleQuote(String literal) + { + return "\'" + literal + "\'"; + } +} diff --git a/StandardColumnMappings.java b/StandardColumnMappings.java new file mode 100644 index 000000000000..9e9c9345abbf --- /dev/null +++ b/StandardColumnMappings.java @@ -0,0 +1,582 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.plugin.jdbc; + +import com.google.common.base.CharMatcher; +import com.google.common.primitives.Shorts; +import com.google.common.primitives.SignedBytes; +import io.airlift.slice.Slice; +import io.prestosql.spi.type.CharType; +import io.prestosql.spi.type.DecimalType; +import io.prestosql.spi.type.Decimals; +import io.prestosql.spi.type.TimeType; +import io.prestosql.spi.type.TimestampType; +import io.prestosql.spi.type.Type; +import io.prestosql.spi.type.VarcharType; +import org.joda.time.DateTimeZone; +import org.joda.time.chrono.ISOChronology; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.MathContext; +import java.math.RoundingMode; +import java.sql.Date; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.Time; +import java.sql.Timestamp; +import java.sql.Types; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.util.Optional; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Verify.verify; +import static com.google.common.io.BaseEncoding.base16; +import static io.airlift.slice.SliceUtf8.countCodePoints; +import static io.airlift.slice.Slices.utf8Slice; +import static io.airlift.slice.Slices.wrappedBuffer; +import static io.prestosql.plugin.jdbc.PredicatePushdownController.DISABLE_PUSHDOWN; +import static io.prestosql.spi.type.BigintType.BIGINT; +import static io.prestosql.spi.type.BooleanType.BOOLEAN; +import static io.prestosql.spi.type.CharType.createCharType; +import static io.prestosql.spi.type.DateType.DATE; +import static io.prestosql.spi.type.DecimalType.createDecimalType; +import static io.prestosql.spi.type.Decimals.decodeUnscaledValue; +import static io.prestosql.spi.type.Decimals.encodeScaledValue; +import static io.prestosql.spi.type.Decimals.encodeShortScaledValue; +import static io.prestosql.spi.type.DoubleType.DOUBLE; +import static io.prestosql.spi.type.IntegerType.INTEGER; +import static io.prestosql.spi.type.RealType.REAL; +import static io.prestosql.spi.type.SmallintType.SMALLINT; +import static io.prestosql.spi.type.TimeType.TIME; +import static io.prestosql.spi.type.TimestampType.MAX_SHORT_PRECISION; +import static io.prestosql.spi.type.TimestampType.TIMESTAMP_MILLIS; +import static io.prestosql.spi.type.Timestamps.MICROSECONDS_PER_SECOND; +import static io.prestosql.spi.type.Timestamps.NANOSECONDS_PER_DAY; +import static io.prestosql.spi.type.Timestamps.NANOSECONDS_PER_MICROSECOND; +import static io.prestosql.spi.type.Timestamps.PICOSECONDS_PER_DAY; +import static io.prestosql.spi.type.Timestamps.PICOSECONDS_PER_NANOSECOND; +import static io.prestosql.spi.type.Timestamps.round; +import static io.prestosql.spi.type.Timestamps.roundDiv; +import static io.prestosql.spi.type.TinyintType.TINYINT; +import static io.prestosql.spi.type.VarbinaryType.VARBINARY; +import static io.prestosql.spi.type.VarcharType.createUnboundedVarcharType; +import static io.prestosql.spi.type.VarcharType.createVarcharType; +import static java.lang.Float.floatToRawIntBits; +import static java.lang.Float.intBitsToFloat; +import static java.lang.Math.floorDiv; +import static java.lang.Math.floorMod; +import static java.lang.Math.max; +import static java.lang.Math.toIntExact; +import static java.lang.String.format; +import static java.math.RoundingMode.UNNECESSARY; +import static java.time.ZoneOffset.UTC; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.DAYS; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.NANOSECONDS; + +public final class StandardColumnMappings +{ + private StandardColumnMappings() {} + + public static ColumnMapping booleanColumnMapping() + { + return ColumnMapping.booleanMapping(BOOLEAN, ResultSet::getBoolean, booleanWriteFunction()); + } + + public static BooleanWriteFunction booleanWriteFunction() + { + return PreparedStatement::setBoolean; + } + + public static ColumnMapping tinyintColumnMapping() + { + return ColumnMapping.longMapping(TINYINT, ResultSet::getByte, tinyintWriteFunction()); + } + + public static LongWriteFunction tinyintWriteFunction() + { + return (statement, index, value) -> statement.setByte(index, SignedBytes.checkedCast(value)); + } + + public static ColumnMapping smallintColumnMapping() + { + return ColumnMapping.longMapping(SMALLINT, ResultSet::getShort, smallintWriteFunction()); + } + + public static LongWriteFunction smallintWriteFunction() + { + return (statement, index, value) -> statement.setShort(index, Shorts.checkedCast(value)); + } + + public static ColumnMapping integerColumnMapping() + { + return ColumnMapping.longMapping(INTEGER, ResultSet::getInt, integerWriteFunction()); + } + + public static LongWriteFunction integerWriteFunction() + { + return (statement, index, value) -> statement.setInt(index, toIntExact(value)); + } + + public static ColumnMapping bigintColumnMapping() + { + return ColumnMapping.longMapping(BIGINT, ResultSet::getLong, bigintWriteFunction()); + } + + public static LongWriteFunction bigintWriteFunction() + { + return PreparedStatement::setLong; + } + + public static ColumnMapping realColumnMapping() + { + return ColumnMapping.longMapping(REAL, (resultSet, columnIndex) -> floatToRawIntBits(resultSet.getFloat(columnIndex)), realWriteFunction()); + } + + public static LongWriteFunction realWriteFunction() + { + return (statement, index, value) -> statement.setFloat(index, intBitsToFloat(toIntExact(value))); + } + + public static ColumnMapping doubleColumnMapping() + { + return ColumnMapping.doubleMapping(DOUBLE, ResultSet::getDouble, doubleWriteFunction()); + } + + public static DoubleWriteFunction doubleWriteFunction() + { + return PreparedStatement::setDouble; + } + + public static ColumnMapping decimalColumnMapping(DecimalType decimalType) + { + return decimalColumnMapping(decimalType, UNNECESSARY); + } + + public static ColumnMapping decimalColumnMapping(DecimalType decimalType, RoundingMode roundingMode) + { + if (decimalType.isShort()) { + checkArgument(roundingMode == UNNECESSARY, "Round mode is not supported for short decimal, map the type to long decimal instead"); + return ColumnMapping.longMapping( + decimalType, + shortDecimalReadFunction(decimalType), + shortDecimalWriteFunction(decimalType)); + } + return ColumnMapping.sliceMapping( + decimalType, + longDecimalReadFunction(decimalType, roundingMode), + longDecimalWriteFunction(decimalType)); + } + + public static LongReadFunction shortDecimalReadFunction(DecimalType decimalType) + { + return shortDecimalReadFunction(decimalType, UNNECESSARY); + } + + public static LongReadFunction shortDecimalReadFunction(DecimalType decimalType, RoundingMode roundingMode) + { + // JDBC driver can return BigDecimal with lower scale than column's scale when there are trailing zeroes + int scale = requireNonNull(decimalType, "decimalType is null").getScale(); + requireNonNull(roundingMode, "roundingMode is null"); + return (resultSet, columnIndex) -> encodeShortScaledValue(resultSet.getBigDecimal(columnIndex), scale, roundingMode); + } + + public static LongWriteFunction shortDecimalWriteFunction(DecimalType decimalType) + { + requireNonNull(decimalType, "decimalType is null"); + checkArgument(decimalType.isShort()); + return (statement, index, value) -> { + BigInteger unscaledValue = BigInteger.valueOf(value); + BigDecimal bigDecimal = new BigDecimal(unscaledValue, decimalType.getScale(), new MathContext(decimalType.getPrecision())); + statement.setBigDecimal(index, bigDecimal); + }; + } + + public static SliceReadFunction longDecimalReadFunction(DecimalType decimalType) + { + return longDecimalReadFunction(decimalType, UNNECESSARY); + } + + public static SliceReadFunction longDecimalReadFunction(DecimalType decimalType, RoundingMode roundingMode) + { + // JDBC driver can return BigDecimal with lower scale than column's scale when there are trailing zeroes + int scale = requireNonNull(decimalType, "decimalType is null").getScale(); + requireNonNull(roundingMode, "roundingMode is null"); + return (resultSet, columnIndex) -> encodeScaledValue(resultSet.getBigDecimal(columnIndex).setScale(scale, roundingMode)); + } + + public static SliceWriteFunction longDecimalWriteFunction(DecimalType decimalType) + { + requireNonNull(decimalType, "decimalType is null"); + checkArgument(!decimalType.isShort()); + return (statement, index, value) -> { + BigInteger unscaledValue = decodeUnscaledValue(value); + BigDecimal bigDecimal = new BigDecimal(unscaledValue, decimalType.getScale(), new MathContext(decimalType.getPrecision())); + statement.setBigDecimal(index, bigDecimal); + }; + } + + public static ColumnMapping defaultCharColumnMapping(int columnSize) + { + if (columnSize > CharType.MAX_LENGTH) { + return defaultVarcharColumnMapping(columnSize); + } + return charColumnMapping(createCharType(columnSize)); + } + + public static ColumnMapping charColumnMapping(CharType charType) + { + requireNonNull(charType, "charType is null"); + return ColumnMapping.sliceMapping(charType, charReadFunction(charType), charWriteFunction()); + } + + public static SliceReadFunction charReadFunction(CharType charType) + { + requireNonNull(charType, "charType is null"); + return (resultSet, columnIndex) -> { + Slice slice = utf8Slice(CharMatcher.is(' ').trimTrailingFrom(resultSet.getString(columnIndex))); + checkLengthInCodePoints(slice, charType, charType.getLength()); + return slice; + }; + } + + public static SliceWriteFunction charWriteFunction() + { + return (statement, index, value) -> { + statement.setString(index, value.toStringUtf8()); + }; + } + + public static ColumnMapping defaultVarcharColumnMapping(int columnSize) + { + if (columnSize > VarcharType.MAX_LENGTH) { + return varcharColumnMapping(createUnboundedVarcharType()); + } + return varcharColumnMapping(createVarcharType(columnSize)); + } + + public static ColumnMapping varcharColumnMapping(VarcharType varcharType) + { + return ColumnMapping.sliceMapping(varcharType, varcharReadFunction(varcharType), varcharWriteFunction()); + } + + public static SliceReadFunction varcharReadFunction(VarcharType varcharType) + { + requireNonNull(varcharType, "varcharType is null"); + if (varcharType.isUnbounded()) { + return (resultSet, columnIndex) -> utf8Slice(resultSet.getString(columnIndex)); + } + return (resultSet, columnIndex) -> { + Slice slice = utf8Slice(resultSet.getString(columnIndex)); + checkLengthInCodePoints(slice, varcharType, varcharType.getBoundedLength()); + return slice; + }; + } + + private static void checkLengthInCodePoints(Slice value, Type characterDataType, int lengthLimit) + { + // Quick check in bytes + if (value.length() <= lengthLimit) { + return; + } + // Actual check + if (countCodePoints(value) <= lengthLimit) { + return; + } + throw new IllegalStateException(format( + "Illegal value for type %s: '%s' [%s]", + characterDataType, + value.toStringUtf8(), + base16().encode(value.getBytes()))); + } + + public static SliceWriteFunction varcharWriteFunction() + { + return (statement, index, value) -> statement.setString(index, value.toStringUtf8()); + } + + public static ColumnMapping varbinaryColumnMapping() + { + return ColumnMapping.sliceMapping( + VARBINARY, + varbinaryReadFunction(), + varbinaryWriteFunction(), + DISABLE_PUSHDOWN); + } + + public static SliceReadFunction varbinaryReadFunction() + { + return (resultSet, columnIndex) -> wrappedBuffer(resultSet.getBytes(columnIndex)); + } + + public static SliceWriteFunction varbinaryWriteFunction() + { + return (statement, index, value) -> statement.setBytes(index, value.getBytes()); + } + + public static ColumnMapping dateColumnMapping() + { + return ColumnMapping.longMapping( + DATE, + (resultSet, columnIndex) -> { + /* + * JDBC returns a date using a timestamp at midnight in the JVM timezone, or earliest time after that if there was no midnight. + * This works correctly for all dates and zones except when the missing local times 'gap' is 24h. I.e. this fails when JVM time + * zone is Pacific/Apia and date to be returned is 2011-12-30. + * + * `return resultSet.getObject(columnIndex, LocalDate.class).toEpochDay()` avoids these problems but + * is currently known not to work with Redshift (old Postgres connector) and SQL Server. + */ + long localMillis = resultSet.getDate(columnIndex).getTime(); + // Convert it to a ~midnight in UTC. + long utcMillis = ISOChronology.getInstance().getZone().getMillisKeepLocal(DateTimeZone.UTC, localMillis); + // convert to days + return MILLISECONDS.toDays(utcMillis); + }, + dateWriteFunction()); + } + + public static LongWriteFunction dateWriteFunction() + { + return (statement, index, value) -> { + // convert to midnight in default time zone + long millis = DAYS.toMillis(value); + statement.setDate(index, new Date(DateTimeZone.UTC.getMillisKeepLocal(DateTimeZone.getDefault(), millis))); + }; + } + + /** + * @deprecated This method uses {@link java.sql.Time} and the class cannot represent time value when JVM zone had + * forward offset change (a 'gap') at given time on 1970-01-01. If driver only supports {@link LocalTime}, use + * {@link #timeColumnMapping} instead. + */ + @Deprecated + public static ColumnMapping timeColumnMappingUsingSqlTime() + { + return ColumnMapping.longMapping( + TIME, + (resultSet, columnIndex) -> { + Time time = resultSet.getTime(columnIndex); + return (toLocalTime(time).toNanoOfDay() * PICOSECONDS_PER_NANOSECOND) % PICOSECONDS_PER_DAY; + }, + timeWriteFunctionUsingSqlTime()); + } + + private static LocalTime toLocalTime(Time sqlTime) + { + // Time.toLocalTime() does not preserve second fraction + return sqlTime.toLocalTime() + // TODO is the conversion correct if sqlTime.getTime() < 0? + .withNano(toIntExact(MILLISECONDS.toNanos(sqlTime.getTime() % 1000))); + } + + /** + * @deprecated This method uses {@link java.sql.Time} and the class cannot represent time value when JVM zone had + * forward offset change (a 'gap') at given time on 1970-01-01. If driver only supports {@link LocalTime}, use + * {@link #timeWriteFunction} instead. + */ + @Deprecated + public static LongWriteFunction timeWriteFunctionUsingSqlTime() + { + return (statement, index, value) -> statement.setTime(index, toSqlTime(fromPrestoTime(value))); + } + + private static Time toSqlTime(LocalTime localTime) + { + // Time.valueOf does not preserve second fraction + return new Time(Time.valueOf(localTime).getTime() + NANOSECONDS.toMillis(localTime.getNano())); + } + + public static ColumnMapping timeColumnMapping(TimeType timeType) + { + checkArgument(timeType.getPrecision() <= 9, "Unsupported type precision: %s", timeType); + return ColumnMapping.longMapping( + timeType, + (resultSet, columnIndex) -> { + LocalTime time = resultSet.getObject(columnIndex, LocalTime.class); + long nanosOfDay = time.toNanoOfDay(); + verify(nanosOfDay < NANOSECONDS_PER_DAY, "Invalid value of nanosOfDay: %s", nanosOfDay); + long picosOfDay = nanosOfDay * PICOSECONDS_PER_NANOSECOND; + return round(picosOfDay, 12 - timeType.getPrecision()); + }, + timeWriteFunction(timeType.getPrecision())); + } + + public static LongWriteFunction timeWriteFunction(int precision) + { + checkArgument(precision <= 9, "Unsupported precision: %s", precision); + return (statement, index, picosOfDay) -> { + picosOfDay = round(picosOfDay, 12 - precision); + if (picosOfDay == PICOSECONDS_PER_DAY) { + picosOfDay = 0; + } + statement.setObject(index, fromPrestoTime(picosOfDay)); + }; + } + + /** + * @deprecated This method uses {@link java.sql.Timestamp} and the class cannot represent date-time value when JVM zone had + * forward offset change (a 'gap'). This includes regular DST changes (e.g. Europe/Warsaw) and one-time policy changes + * (Asia/Kathmandu's shift by 15 minutes on January 1, 1986, 00:00:00). If driver only supports {@link LocalDateTime}, use + * {@link #timestampColumnMapping} instead. + */ + @Deprecated + public static ColumnMapping timestampColumnMappingUsingSqlTimestamp(TimestampType timestampType) + { + // TODO support higher precision + checkArgument(timestampType.getPrecision() <= MAX_SHORT_PRECISION, "Precision is out of range: %s", timestampType.getPrecision()); + return ColumnMapping.longMapping( + timestampType, + (resultSet, columnIndex) -> { + Timestamp timestamp = resultSet.getTimestamp(columnIndex); + return toPrestoTimestamp(timestampType, timestamp.toLocalDateTime()); + }, + timestampWriteFunctionUsingSqlTimestamp(timestampType)); + } + + @Deprecated + public static ColumnMapping timestampColumnMapping() + { + return timestampColumnMapping(TIMESTAMP_MILLIS); + } + + public static ColumnMapping timestampColumnMapping(TimestampType timestampType) + { + checkArgument(timestampType.getPrecision() <= MAX_SHORT_PRECISION, "Precision is out of range: %s", timestampType.getPrecision()); + return ColumnMapping.longMapping( + timestampType, + timestampReadFunction(timestampType), + timestampWriteFunction(timestampType)); + } + + public static LongReadFunction timestampReadFunction(TimestampType timestampType) + { + checkArgument(timestampType.getPrecision() <= MAX_SHORT_PRECISION, "Precision is out of range: %s", timestampType.getPrecision()); + return (resultSet, columnIndex) -> toPrestoTimestamp(timestampType, resultSet.getObject(columnIndex, LocalDateTime.class)); + } + + /** + * @deprecated This method uses {@link java.sql.Timestamp} and the class cannot represent date-time value when JVM zone had + * forward offset change (a 'gap'). This includes regular DST changes (e.g. Europe/Warsaw) and one-time policy changes + * (Asia/Kathmandu's shift by 15 minutes on January 1, 1986, 00:00:00). If driver only supports {@link LocalDateTime}, use + * {@link #timestampWriteFunction} instead. + */ + @Deprecated + public static LongWriteFunction timestampWriteFunctionUsingSqlTimestamp(TimestampType timestampType) + { + checkArgument(timestampType.getPrecision() <= MAX_SHORT_PRECISION, "Precision is out of range: %s", timestampType.getPrecision()); + return (statement, index, value) -> statement.setTimestamp(index, Timestamp.valueOf(fromPrestoTimestamp(value))); + } + + public static LongWriteFunction timestampWriteFunction(TimestampType timestampType) + { + checkArgument(timestampType.getPrecision() <= MAX_SHORT_PRECISION, "Precision is out of range: %s", timestampType.getPrecision()); + return (statement, index, value) -> statement.setObject(index, fromPrestoTimestamp(value)); + } + + public static long toPrestoTimestamp(TimestampType timestampType, LocalDateTime localDateTime) + { + long precision = timestampType.getPrecision(); + checkArgument(precision <= MAX_SHORT_PRECISION, "Precision is out of range: %s", precision); + Instant instant = localDateTime.atZone(UTC).toInstant(); + return instant.getEpochSecond() * MICROSECONDS_PER_SECOND + roundDiv(instant.getNano(), NANOSECONDS_PER_MICROSECOND); + } + + public static LocalDateTime fromPrestoTimestamp(long epochMicros) + { + long epochSecond = floorDiv(epochMicros, MICROSECONDS_PER_SECOND); + int nanoFraction = floorMod(epochMicros, MICROSECONDS_PER_SECOND) * NANOSECONDS_PER_MICROSECOND; + Instant instant = Instant.ofEpochSecond(epochSecond, nanoFraction); + return LocalDateTime.ofInstant(instant, UTC); + } + + public static LocalTime fromPrestoTime(long value) + { + // value can round up to NANOSECONDS_PER_DAY, so we need to do % to keep it in the desired range + return LocalTime.ofNanoOfDay(roundDiv(value, PICOSECONDS_PER_NANOSECOND) % NANOSECONDS_PER_DAY); + } + + /** + * @deprecated Each connector should provide its own explicit type mapping, along with respective tests. + */ + @Deprecated + public static Optional jdbcTypeToPrestoType(JdbcTypeHandle typeHandle) + { + switch (typeHandle.getJdbcType()) { + case Types.BIT: + case Types.BOOLEAN: + return Optional.of(booleanColumnMapping()); + + case Types.TINYINT: + return Optional.of(tinyintColumnMapping()); + + case Types.SMALLINT: + return Optional.of(smallintColumnMapping()); + + case Types.INTEGER: + return Optional.of(integerColumnMapping()); + + case Types.BIGINT: + return Optional.of(bigintColumnMapping()); + + case Types.REAL: + return Optional.of(realColumnMapping()); + + case Types.FLOAT: + case Types.DOUBLE: + return Optional.of(doubleColumnMapping()); + + case Types.NUMERIC: + case Types.DECIMAL: + int decimalDigits = typeHandle.getDecimalDigits().isEmpty() ? 0 : typeHandle.getRequiredDecimalDigits(); + int precision = typeHandle.getRequiredColumnSize() + max(-decimalDigits, 0); // Map decimal(p, -s) (negative scale) to decimal(p+s, 0). + if (precision > Decimals.MAX_PRECISION) { + return Optional.empty(); + } + return Optional.of(decimalColumnMapping(createDecimalType(precision, max(decimalDigits, 0)))); + + case Types.CHAR: + case Types.NCHAR: + return Optional.of(defaultCharColumnMapping((typeHandle.getColumnSize().isEmpty()) ? Integer.MAX_VALUE : typeHandle.getRequiredColumnSize())); + + case Types.VARCHAR: + case Types.NVARCHAR: + case Types.LONGVARCHAR: + case Types.LONGNVARCHAR: + return Optional.of(defaultVarcharColumnMapping((typeHandle.getColumnSize().isEmpty()) ? Integer.MAX_VALUE : typeHandle.getRequiredColumnSize())); + //firebird has BLOB SUB_TYPE TEXT type with column_size null + + case Types.BINARY: + case Types.VARBINARY: + case Types.LONGVARBINARY: + return Optional.of(varbinaryColumnMapping()); + + case Types.DATE: + return Optional.of(dateColumnMapping()); + + case Types.TIME: + // TODO default to `timeColumnMapping` + return Optional.of(timeColumnMappingUsingSqlTime()); + + case Types.TIMESTAMP: + // TODO default to `timestampColumnMapping` + return Optional.of(timestampColumnMappingUsingSqlTimestamp(TIMESTAMP_MILLIS)); + } + return Optional.empty(); + } +} diff --git a/pom.xml b/pom.xml index e97f94e4260f..8911bb3c8f35 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT pom presto-root @@ -137,6 +137,7 @@ presto-iceberg presto-google-sheets presto-bigquery + presto-genericjdbc @@ -436,6 +437,12 @@ ${project.version} + + io.prestosql + presto-genericjdbc + ${project.version} + + io.prestosql.hadoop hadoop-apache diff --git a/presto-accumulo/pom.xml b/presto-accumulo/pom.xml index 0a239ff0ed55..51392388398d 100644 --- a/presto-accumulo/pom.xml +++ b/presto-accumulo/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-accumulo diff --git a/presto-array/pom.xml b/presto-array/pom.xml index 776b6b5fc8da..ecd22bfff656 100644 --- a/presto-array/pom.xml +++ b/presto-array/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-array diff --git a/presto-atop/pom.xml b/presto-atop/pom.xml index 5e35d3bc7edc..d2b1af8275a5 100644 --- a/presto-atop/pom.xml +++ b/presto-atop/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-atop diff --git a/presto-base-jdbc/pom.xml b/presto-base-jdbc/pom.xml index 75f2037c684a..00bc3e00318b 100644 --- a/presto-base-jdbc/pom.xml +++ b/presto-base-jdbc/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-base-jdbc diff --git a/presto-base-jdbc/src/main/java/io/prestosql/plugin/jdbc/BaseJdbcConfig.java b/presto-base-jdbc/src/main/java/io/prestosql/plugin/jdbc/BaseJdbcConfig.java index 3557656d0275..49f1fce5242c 100644 --- a/presto-base-jdbc/src/main/java/io/prestosql/plugin/jdbc/BaseJdbcConfig.java +++ b/presto-base-jdbc/src/main/java/io/prestosql/plugin/jdbc/BaseJdbcConfig.java @@ -35,6 +35,7 @@ public class BaseJdbcConfig private Set jdbcTypesMappedToVarchar = ImmutableSet.of(); private Duration metadataCacheTtl = new Duration(0, MINUTES); private boolean cacheMissing; + private String driverClass; @NotNull public String getConnectionUrl() @@ -49,6 +50,18 @@ public BaseJdbcConfig setConnectionUrl(String connectionUrl) return this; } + public String getDriverClass() + { + return driverClass; + } + + @Config("driver-class") + public BaseJdbcConfig setDriverClass(String driverClass) + { + this.driverClass = driverClass; + return this; + } + public boolean isCaseInsensitiveNameMatching() { return caseInsensitiveNameMatching; diff --git a/presto-base-jdbc/src/main/java/io/prestosql/plugin/jdbc/JdbcErrorCode.java b/presto-base-jdbc/src/main/java/io/prestosql/plugin/jdbc/JdbcErrorCode.java index d9876d165c1c..e6b02848e46e 100644 --- a/presto-base-jdbc/src/main/java/io/prestosql/plugin/jdbc/JdbcErrorCode.java +++ b/presto-base-jdbc/src/main/java/io/prestosql/plugin/jdbc/JdbcErrorCode.java @@ -18,12 +18,14 @@ import io.prestosql.spi.ErrorType; import static io.prestosql.spi.ErrorType.EXTERNAL; +import static io.prestosql.spi.ErrorType.INTERNAL_ERROR; public enum JdbcErrorCode implements ErrorCodeSupplier { JDBC_ERROR(0, EXTERNAL), - JDBC_NON_TRANSIENT_ERROR(1, EXTERNAL); + JDBC_NON_TRANSIENT_ERROR(1, EXTERNAL), + DRIVER_NOT_FOUND(2, INTERNAL_ERROR); private final ErrorCode errorCode; diff --git a/presto-base-jdbc/src/test/java/io/prestosql/plugin/jdbc/TestBaseJdbcConfig.java b/presto-base-jdbc/src/test/java/io/prestosql/plugin/jdbc/TestBaseJdbcConfig.java index 43e23c34d932..09b667cf530d 100644 --- a/presto-base-jdbc/src/test/java/io/prestosql/plugin/jdbc/TestBaseJdbcConfig.java +++ b/presto-base-jdbc/src/test/java/io/prestosql/plugin/jdbc/TestBaseJdbcConfig.java @@ -38,7 +38,8 @@ public void testDefaults() .setCaseInsensitiveNameMatchingCacheTtl(new Duration(1, MINUTES)) .setJdbcTypesMappedToVarchar("") .setMetadataCacheTtl(Duration.valueOf("0m")) - .setCacheMissing(false)); + .setCacheMissing(false) + .setDriverClass(null)); } @Test @@ -51,6 +52,7 @@ public void testExplicitPropertyMappings() .put("jdbc-types-mapped-to-varchar", "mytype,struct_type1") .put("metadata.cache-ttl", "1s") .put("metadata.cache-missing", "true") + .put("driver-class", "com.sybase.jdbc4.jdbc.SybDriver") .build(); BaseJdbcConfig expected = new BaseJdbcConfig() @@ -59,7 +61,8 @@ public void testExplicitPropertyMappings() .setCaseInsensitiveNameMatchingCacheTtl(new Duration(1, SECONDS)) .setJdbcTypesMappedToVarchar("mytype, struct_type1") .setMetadataCacheTtl(Duration.valueOf("1s")) - .setCacheMissing(true); + .setCacheMissing(true) + .setDriverClass("com.sybase.jdbc4.jdbc.SybDriver"); assertFullMapping(properties, expected); diff --git a/presto-benchmark-driver/pom.xml b/presto-benchmark-driver/pom.xml index 018a8b2068fa..59768e4f4f23 100644 --- a/presto-benchmark-driver/pom.xml +++ b/presto-benchmark-driver/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-benchmark-driver diff --git a/presto-benchmark/pom.xml b/presto-benchmark/pom.xml index 88010d764125..68b0366d40ea 100644 --- a/presto-benchmark/pom.xml +++ b/presto-benchmark/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-benchmark diff --git a/presto-benchto-benchmarks/pom.xml b/presto-benchto-benchmarks/pom.xml index 3aae5f80aa85..36bc1910f615 100644 --- a/presto-benchto-benchmarks/pom.xml +++ b/presto-benchto-benchmarks/pom.xml @@ -4,7 +4,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-benchto-benchmarks diff --git a/presto-bigquery/pom.xml b/presto-bigquery/pom.xml index fcb89eb8c4bf..01bbe911cf24 100644 --- a/presto-bigquery/pom.xml +++ b/presto-bigquery/pom.xml @@ -7,7 +7,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-bigquery diff --git a/presto-blackhole/pom.xml b/presto-blackhole/pom.xml index a202633063ba..d4ac1678e842 100644 --- a/presto-blackhole/pom.xml +++ b/presto-blackhole/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-blackhole diff --git a/presto-cassandra/pom.xml b/presto-cassandra/pom.xml index 57ee801a6133..b027823e5181 100644 --- a/presto-cassandra/pom.xml +++ b/presto-cassandra/pom.xml @@ -4,7 +4,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-cassandra diff --git a/presto-cli/pom.xml b/presto-cli/pom.xml index aa4fc3e78888..d20f5282d860 100644 --- a/presto-cli/pom.xml +++ b/presto-cli/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-cli diff --git a/presto-client/pom.xml b/presto-client/pom.xml index a8cd5134c36c..890b3292685f 100644 --- a/presto-client/pom.xml +++ b/presto-client/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-client diff --git a/presto-docs/pom.xml b/presto-docs/pom.xml index 0ab36facebee..6e81d90243b3 100644 --- a/presto-docs/pom.xml +++ b/presto-docs/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-docs diff --git a/presto-docs/src/main/sphinx/connector.rst b/presto-docs/src/main/sphinx/connector.rst index 84d8aee279be..2651227415fd 100644 --- a/presto-docs/src/main/sphinx/connector.rst +++ b/presto-docs/src/main/sphinx/connector.rst @@ -13,6 +13,7 @@ from different data sources. connector/blackhole connector/cassandra connector/elasticsearch + connector/genericjdbc connector/googlesheets connector/hive connector/hive-gcs-tutorial diff --git a/presto-docs/src/main/sphinx/connector/genericjdbc.rst b/presto-docs/src/main/sphinx/connector/genericjdbc.rst new file mode 100644 index 000000000000..655846e95785 --- /dev/null +++ b/presto-docs/src/main/sphinx/connector/genericjdbc.rst @@ -0,0 +1,73 @@ +======================= +Generic JDBC Connector +======================= + +The Generic JDBC connector allows querying tables in an +external database (using the database's native JDBC driver). This can be used to join data between +different systems like Sybase and Hive, or between two different +Oracle instances. + +Configuration +------------- + +To configure the Generic JDBC connector, create a catalog properties file +in ``etc/catalog`` named, for example, ``sybase.properties``, to +mount the Generic JDBC connector as the ``sybase`` catalog. +Create the file with the following contents, replacing the +connection properties as appropriate for your setup: + +.. code-block:: none + + connector.name=genericjdbc + driver-class= + connection-url=jdbc: + connection-user=root + connection-password=secret + case-insensitive-name-matching=true + +Some examples: + +.. code-block:: none + + connector.name=genericjdbc + driver-class=com.sybase.jdbc4.jdbc.SybDataSource + connection-url=jdbc:sybase:Tds:localhost:8000/MYSYBASE + connection-user=sa + connection-password=myPassword + case-insensitive-name-matching=true + +.. code-block:: none + + connector.name=genericjdbc + driver-class=oracle.jdbc.driver.OracleDriver + connection-url=jdbc:oracle:thin:@localhost:49161:xe + connection-user=system + connection-password=rootpw + case-insensitive-name-matching=true + +Multiple Generic JDBC Databases or Servers +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The Generic JDBC connector can only access a single database within +a SGeneric JDBC server. Thus, if you have multiple Generic JDBC databases, +or want to connect to multiple instances of the Generic JDBC, you must configure +multiple catalogs, one for each instance. + +To add another catalog, simply add another properties file to ``etc/catalog`` +with a different name, making sure it ends in ``.properties``. For example, +if you name the property file ``sales.properties``, Presto creates a +catalog named ``sales`` using the configured connector. + +Specific driver jars +-------------------- + +You must place the specific JAR for your database (ie ojdbc6.jar, jconn4.jar etc) containing the driver-class into the plugin/genericjdbc folder. + +Generic JDBC Connector Limitations +---------------------------------- + +Read-only access +Not all data types can be mapped +Not all databases will work +Performance +Experimental diff --git a/presto-elasticsearch/pom.xml b/presto-elasticsearch/pom.xml index 944f4483c758..2ece5639388a 100644 --- a/presto-elasticsearch/pom.xml +++ b/presto-elasticsearch/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-elasticsearch diff --git a/presto-example-http/pom.xml b/presto-example-http/pom.xml index a484076402a6..88e0da2df555 100644 --- a/presto-example-http/pom.xml +++ b/presto-example-http/pom.xml @@ -4,7 +4,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-example-http diff --git a/presto-genericjdbc/pom.xml b/presto-genericjdbc/pom.xml new file mode 100644 index 000000000000..1e3fede29445 --- /dev/null +++ b/presto-genericjdbc/pom.xml @@ -0,0 +1,99 @@ + + + + io.prestosql + presto-root + 332-SNAPSHOT + + 4.0.0 + + presto-genericjdbc + Presto - Generic JDBC Connector + presto-plugin + + + ${project.parent.basedir} + + + + + io.prestosql + presto-base-jdbc + + + + com.google.guava + guava + + + + io.airlift + configuration + + + + com.google.inject + guice + + + + javax.inject + javax.inject + + + + io.airlift + slice + provided + + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + + io.prestosql + presto-spi + provided + + + + org.openjdk.jol + jol-core + provided + + + + + org.testng + testng + test + + + + io.prestosql + presto-main + test + + + + io.prestosql + presto-tpch + test + + + + io.prestosql + presto-testing + test + + + + io.airlift + testing + test + + + diff --git a/presto-genericjdbc/src/main/java/io/prestosql/plugin/genericjdbc/GenericJdbcClient.java b/presto-genericjdbc/src/main/java/io/prestosql/plugin/genericjdbc/GenericJdbcClient.java new file mode 100644 index 000000000000..9f133496a121 --- /dev/null +++ b/presto-genericjdbc/src/main/java/io/prestosql/plugin/genericjdbc/GenericJdbcClient.java @@ -0,0 +1,128 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.plugin.genericjdbc; + +import com.google.common.base.Joiner; +import io.prestosql.plugin.jdbc.BaseJdbcClient; +import io.prestosql.plugin.jdbc.BaseJdbcConfig; +import io.prestosql.plugin.jdbc.ColumnMapping; +import io.prestosql.plugin.jdbc.ConnectionFactory; +import io.prestosql.plugin.jdbc.JdbcTypeHandle; +import io.prestosql.plugin.jdbc.WriteMapping; +import io.prestosql.spi.connector.ConnectorSession; +import io.prestosql.spi.predicate.Domain; +import io.prestosql.spi.type.CharType; +import io.prestosql.spi.type.Type; +import io.prestosql.spi.type.VarcharType; + +import javax.inject.Inject; + +import java.sql.Connection; +import java.util.Optional; +import java.util.function.UnaryOperator; + +import static io.prestosql.plugin.jdbc.StandardColumnMappings.booleanWriteFunction; +import static io.prestosql.plugin.jdbc.StandardColumnMappings.charWriteFunction; +import static io.prestosql.plugin.jdbc.StandardColumnMappings.varcharWriteFunction; +import static io.prestosql.spi.type.BooleanType.BOOLEAN; +import static io.prestosql.spi.type.Varchars.isVarcharType; + +public class GenericJdbcClient + extends BaseJdbcClient +{ + private static final Joiner DOT_JOINER = Joiner.on("."); + + // Sybase supports 2100 parameters in prepared statement, let's create a space for about 4 big IN predicates + private static final int MAX_LIST_EXPRESSIONS = 500; + + // TODO improve this by calling Domain#simplify + private static final UnaryOperator DISABLE_UNSUPPORTED_PUSHDOWN = domain -> { + if (domain.getValues().getRanges().getRangeCount() <= MAX_LIST_EXPRESSIONS) { + return domain; + } + return Domain.all(domain.getType()); + }; + + @Inject + public GenericJdbcClient(BaseJdbcConfig config, ConnectionFactory connectionFactory) + { + super(config, (config.getConnectionUrl().startsWith("jdbc:impala")) ? "`" : "\"", connectionFactory); + } + + @Override + public Optional toPrestoType(ConnectorSession session, Connection connection, JdbcTypeHandle typeHandle) + { + // TODO this if should only apply to Impala + if (typeHandle.toString().contains("jdbcTypeName=ARRAY") || typeHandle.toString().contains("jdbcTypeName=MAP") || typeHandle.toString().contains("jdbcTypeName=STRUCT")) { + return Optional.empty(); + } + + Optional mapping = getForcedMappingToVarchar(typeHandle); + if (mapping.isPresent()) { + return mapping; + } + // TODO implement proper type mapping + return super.toPrestoType(session, connection, typeHandle) + .map(columnMapping -> new ColumnMapping( + columnMapping.getType(), + columnMapping.getReadFunction(), + columnMapping.getWriteFunction(), + DISABLE_UNSUPPORTED_PUSHDOWN)); + } + + @Override + public WriteMapping toWriteMapping(ConnectorSession session, Type type) + { + if (type == BOOLEAN) { + return WriteMapping.booleanMapping("bit", booleanWriteFunction()); + } + + if (isVarcharType(type)) { + VarcharType varcharType = (VarcharType) type; + String dataType; + if (varcharType.isUnbounded() || varcharType.getBoundedLength() > 4000) { + dataType = "nvarchar(max)"; + } + else { + dataType = "nvarchar(" + varcharType.getBoundedLength() + ")"; + } + return WriteMapping.sliceMapping(dataType, varcharWriteFunction()); + } + + if (type instanceof CharType) { + CharType charType = (CharType) type; + String dataType; + if (charType.getLength() > 4000) { + dataType = "nvarchar(max)"; + } + else { + dataType = "nchar(" + charType.getLength() + ")"; + } + return WriteMapping.sliceMapping(dataType, charWriteFunction()); + } + + // TODO implement proper type mapping + return super.toWriteMapping(session, type); + } + + private static String singleQuote(String... objects) + { + return singleQuote(DOT_JOINER.join(objects)); + } + + private static String singleQuote(String literal) + { + return "\'" + literal + "\'"; + } +} diff --git a/presto-genericjdbc/src/main/java/io/prestosql/plugin/genericjdbc/GenericJdbcClientModule.java b/presto-genericjdbc/src/main/java/io/prestosql/plugin/genericjdbc/GenericJdbcClientModule.java new file mode 100644 index 000000000000..9bfcdf87b0cc --- /dev/null +++ b/presto-genericjdbc/src/main/java/io/prestosql/plugin/genericjdbc/GenericJdbcClientModule.java @@ -0,0 +1,80 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.plugin.genericjdbc; + +import com.google.inject.Binder; +import com.google.inject.Key; +import com.google.inject.Module; +import com.google.inject.Provides; +import com.google.inject.Scopes; +import com.google.inject.Singleton; +import io.prestosql.plugin.jdbc.BaseJdbcConfig; +import io.prestosql.plugin.jdbc.ConnectionFactory; +import io.prestosql.plugin.jdbc.DriverConnectionFactory; +import io.prestosql.plugin.jdbc.ForBaseJdbc; +import io.prestosql.plugin.jdbc.JdbcClient; +import io.prestosql.plugin.jdbc.credential.CredentialProvider; +import io.prestosql.spi.PrestoException; + +import java.lang.reflect.InvocationTargetException; +import java.sql.Driver; + +import static io.airlift.configuration.ConfigBinder.configBinder; +import static io.prestosql.plugin.jdbc.JdbcErrorCode.DRIVER_NOT_FOUND; + +public class GenericJdbcClientModule + implements Module +{ + @Override + public void configure(Binder binder) + { + binder.bind(Key.get(JdbcClient.class, ForBaseJdbc.class)).to(GenericJdbcClient.class) + .in(Scopes.SINGLETON); + configBinder(binder).bindConfig(BaseJdbcConfig.class); + } + + @Provides + @Singleton + @ForBaseJdbc + public ConnectionFactory getConnectionFactory(BaseJdbcConfig config, CredentialProvider credentialProvider) + { + try { + Class.forName(config.getDriverClass()); + } + catch (ClassNotFoundException e) { + throw new PrestoException(DRIVER_NOT_FOUND, config.getDriverClass() + " not found"); + } + try { + try { + return new DriverConnectionFactory((Driver) Class.forName(config.getDriverClass()).getConstructor().newInstance(), config, credentialProvider); + } + catch (InvocationTargetException e) { + e.printStackTrace(); + } + catch (NoSuchMethodException e) { + e.printStackTrace(); + } + } + catch (InstantiationException e) { + e.printStackTrace(); + } + catch (IllegalAccessException e) { + e.printStackTrace(); + } + catch (ClassNotFoundException e) { + e.printStackTrace(); + } + return null; + } +} diff --git a/presto-genericjdbc/src/main/java/io/prestosql/plugin/genericjdbc/GenericJdbcPlugin.java b/presto-genericjdbc/src/main/java/io/prestosql/plugin/genericjdbc/GenericJdbcPlugin.java new file mode 100644 index 000000000000..0c59aef06079 --- /dev/null +++ b/presto-genericjdbc/src/main/java/io/prestosql/plugin/genericjdbc/GenericJdbcPlugin.java @@ -0,0 +1,25 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.plugin.genericjdbc; + +import io.prestosql.plugin.jdbc.JdbcPlugin; + +public class GenericJdbcPlugin + extends JdbcPlugin +{ + public GenericJdbcPlugin() + { + super("genericjdbc", new GenericJdbcClientModule()); + } +} diff --git a/presto-geospatial-toolkit/pom.xml b/presto-geospatial-toolkit/pom.xml index c9656ae93084..7e83cb895365 100644 --- a/presto-geospatial-toolkit/pom.xml +++ b/presto-geospatial-toolkit/pom.xml @@ -4,7 +4,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-geospatial-toolkit diff --git a/presto-geospatial/pom.xml b/presto-geospatial/pom.xml index 7cd676b3cd28..f199c3d64d59 100644 --- a/presto-geospatial/pom.xml +++ b/presto-geospatial/pom.xml @@ -4,7 +4,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-geospatial diff --git a/presto-google-sheets/pom.xml b/presto-google-sheets/pom.xml index 48dbb96a8688..14d5c04c556e 100644 --- a/presto-google-sheets/pom.xml +++ b/presto-google-sheets/pom.xml @@ -4,7 +4,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-google-sheets diff --git a/presto-hive-hadoop2/pom.xml b/presto-hive-hadoop2/pom.xml index f613d23ec600..15d9ae3d50c3 100644 --- a/presto-hive-hadoop2/pom.xml +++ b/presto-hive-hadoop2/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-hive-hadoop2 diff --git a/presto-hive/pom.xml b/presto-hive/pom.xml index 2a0d5bd612bc..ff581aa1a12a 100644 --- a/presto-hive/pom.xml +++ b/presto-hive/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-hive diff --git a/presto-iceberg/pom.xml b/presto-iceberg/pom.xml index 635ae262a332..9027c81055e9 100644 --- a/presto-iceberg/pom.xml +++ b/presto-iceberg/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-iceberg diff --git a/presto-jdbc/pom.xml b/presto-jdbc/pom.xml index 3b2ffd785335..c64bcb514d24 100644 --- a/presto-jdbc/pom.xml +++ b/presto-jdbc/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-jdbc diff --git a/presto-jmx/pom.xml b/presto-jmx/pom.xml index 343a115c2c54..abc934836697 100644 --- a/presto-jmx/pom.xml +++ b/presto-jmx/pom.xml @@ -4,7 +4,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-jmx diff --git a/presto-kafka/pom.xml b/presto-kafka/pom.xml index 7c2fe4454628..b53b3bcb3b00 100644 --- a/presto-kafka/pom.xml +++ b/presto-kafka/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-kafka diff --git a/presto-kinesis/pom.xml b/presto-kinesis/pom.xml index 711cf3e23ee5..24b8dbe1298f 100644 --- a/presto-kinesis/pom.xml +++ b/presto-kinesis/pom.xml @@ -4,7 +4,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-kinesis diff --git a/presto-kudu/pom.xml b/presto-kudu/pom.xml index ff7d337cbc81..3d86827484bc 100644 --- a/presto-kudu/pom.xml +++ b/presto-kudu/pom.xml @@ -4,7 +4,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-kudu diff --git a/presto-local-file/pom.xml b/presto-local-file/pom.xml index 147f1cc08a43..7b82edf7f002 100644 --- a/presto-local-file/pom.xml +++ b/presto-local-file/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-local-file diff --git a/presto-main/etc/catalog/genericjdbc.properties b/presto-main/etc/catalog/genericjdbc.properties new file mode 100644 index 000000000000..7aa76ddfc9ec --- /dev/null +++ b/presto-main/etc/catalog/genericjdbc.properties @@ -0,0 +1,6 @@ +connector.name=genericjdbc +driver-class=com.sybase.jdbc4.jdbc.SybDriver +connection-url=jdbc:sybase:Tds:localhost:8000/MYSYBASE +connection-user=sa +connection-password=myPassword +case-insensitive-name-matching=true diff --git a/presto-main/etc/config.properties b/presto-main/etc/config.properties index 5a30f5dca713..a2bacfb5b17c 100644 --- a/presto-main/etc/config.properties +++ b/presto-main/etc/config.properties @@ -43,6 +43,7 @@ plugin.bundles=\ ../presto-mysql/pom.xml,\ ../presto-memsql/pom.xml,\ ../presto-sqlserver/pom.xml, \ + ../presto-genericjdbc/pom.xml, \ ../presto-postgresql/pom.xml, \ ../presto-thrift/pom.xml, \ ../presto-tpcds/pom.xml, \ diff --git a/presto-main/pom.xml b/presto-main/pom.xml index 56768efbd8d0..f39cafb2bcea 100644 --- a/presto-main/pom.xml +++ b/presto-main/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-main diff --git a/presto-matching/pom.xml b/presto-matching/pom.xml index 213dbcdbb7c3..c408b30b4fd4 100644 --- a/presto-matching/pom.xml +++ b/presto-matching/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-matching diff --git a/presto-memory-context/pom.xml b/presto-memory-context/pom.xml index 3a4cd6b6e550..142f9f5d5e10 100644 --- a/presto-memory-context/pom.xml +++ b/presto-memory-context/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-memory-context diff --git a/presto-memory/pom.xml b/presto-memory/pom.xml index bef8c5b66db8..3ed4a825a011 100644 --- a/presto-memory/pom.xml +++ b/presto-memory/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-memory diff --git a/presto-memsql/pom.xml b/presto-memsql/pom.xml index 7ebc12572fe5..0b060cc456d4 100644 --- a/presto-memsql/pom.xml +++ b/presto-memsql/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-memsql diff --git a/presto-ml/pom.xml b/presto-ml/pom.xml index 3967a75ef7d4..e8f3b1ef48c6 100644 --- a/presto-ml/pom.xml +++ b/presto-ml/pom.xml @@ -4,7 +4,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-ml diff --git a/presto-mongodb/pom.xml b/presto-mongodb/pom.xml index 28d499962027..01d932751f65 100644 --- a/presto-mongodb/pom.xml +++ b/presto-mongodb/pom.xml @@ -4,7 +4,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-mongodb diff --git a/presto-mysql/pom.xml b/presto-mysql/pom.xml index 3ec1c1887f26..c82b8713fa5d 100644 --- a/presto-mysql/pom.xml +++ b/presto-mysql/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-mysql diff --git a/presto-noop/pom.xml b/presto-noop/pom.xml index 866a12a4ce3b..15685c39a0f8 100644 --- a/presto-noop/pom.xml +++ b/presto-noop/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-noop diff --git a/presto-orc/pom.xml b/presto-orc/pom.xml index 5e9158471cf6..dee02c603bae 100644 --- a/presto-orc/pom.xml +++ b/presto-orc/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-orc diff --git a/presto-parquet/pom.xml b/presto-parquet/pom.xml index 2c5c993764aa..e61350bed258 100644 --- a/presto-parquet/pom.xml +++ b/presto-parquet/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-parquet diff --git a/presto-parser/pom.xml b/presto-parser/pom.xml index 71184624e966..854fa06dc15d 100644 --- a/presto-parser/pom.xml +++ b/presto-parser/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-parser diff --git a/presto-password-authenticators/pom.xml b/presto-password-authenticators/pom.xml index dd6b967ca03f..be11c658aaf2 100644 --- a/presto-password-authenticators/pom.xml +++ b/presto-password-authenticators/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-password-authenticators diff --git a/presto-phoenix/pom.xml b/presto-phoenix/pom.xml index 2a1f09869098..5d6f902c6a35 100644 --- a/presto-phoenix/pom.xml +++ b/presto-phoenix/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-phoenix diff --git a/presto-plugin-toolkit/pom.xml b/presto-plugin-toolkit/pom.xml index 315435123c3a..c421585bc30e 100644 --- a/presto-plugin-toolkit/pom.xml +++ b/presto-plugin-toolkit/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-plugin-toolkit diff --git a/presto-postgresql/pom.xml b/presto-postgresql/pom.xml index 6bf472b6a682..032816549d4e 100644 --- a/presto-postgresql/pom.xml +++ b/presto-postgresql/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-postgresql diff --git a/presto-product-tests-launcher/pom.xml b/presto-product-tests-launcher/pom.xml index b93683af62f4..adaf643640db 100644 --- a/presto-product-tests-launcher/pom.xml +++ b/presto-product-tests-launcher/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-product-tests-launcher diff --git a/presto-product-tests/pom.xml b/presto-product-tests/pom.xml index 166090df78e7..6eb036c69fe3 100644 --- a/presto-product-tests/pom.xml +++ b/presto-product-tests/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-product-tests diff --git a/presto-proxy/pom.xml b/presto-proxy/pom.xml index 1f640c23ee05..c62c7e69d6bb 100644 --- a/presto-proxy/pom.xml +++ b/presto-proxy/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-proxy diff --git a/presto-raptor-legacy/pom.xml b/presto-raptor-legacy/pom.xml index a4b8aec6163d..56b33c7b3afd 100644 --- a/presto-raptor-legacy/pom.xml +++ b/presto-raptor-legacy/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-raptor-legacy diff --git a/presto-rcfile/pom.xml b/presto-rcfile/pom.xml index 80257360d9c8..1dd9fd3dd61d 100644 --- a/presto-rcfile/pom.xml +++ b/presto-rcfile/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-rcfile diff --git a/presto-record-decoder/pom.xml b/presto-record-decoder/pom.xml index 1acadd30a34a..1ccdf5f1b296 100644 --- a/presto-record-decoder/pom.xml +++ b/presto-record-decoder/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-record-decoder diff --git a/presto-redis/pom.xml b/presto-redis/pom.xml index be979ae2716f..3df40083db9a 100644 --- a/presto-redis/pom.xml +++ b/presto-redis/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-redis diff --git a/presto-redshift/pom.xml b/presto-redshift/pom.xml index 59ea26e943ef..1a8412cad8ab 100644 --- a/presto-redshift/pom.xml +++ b/presto-redshift/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-redshift diff --git a/presto-resource-group-managers/pom.xml b/presto-resource-group-managers/pom.xml index 73124c5e346b..5fe30c81914e 100644 --- a/presto-resource-group-managers/pom.xml +++ b/presto-resource-group-managers/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-resource-group-managers diff --git a/presto-server-rpm/pom.xml b/presto-server-rpm/pom.xml index a0bd26354791..3a8d7c467487 100644 --- a/presto-server-rpm/pom.xml +++ b/presto-server-rpm/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-server-rpm diff --git a/presto-server/pom.xml b/presto-server/pom.xml index 5924f3c95389..261594b98070 100644 --- a/presto-server/pom.xml +++ b/presto-server/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-server diff --git a/presto-server/src/main/provisio/presto.xml b/presto-server/src/main/provisio/presto.xml index 88ecf2e5b222..f7a84aad47fe 100644 --- a/presto-server/src/main/provisio/presto.xml +++ b/presto-server/src/main/provisio/presto.xml @@ -229,4 +229,10 @@ + + + + + + diff --git a/presto-session-property-managers/pom.xml b/presto-session-property-managers/pom.xml index 1d65f5f5a14d..f0e35b4ad1af 100644 --- a/presto-session-property-managers/pom.xml +++ b/presto-session-property-managers/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-session-property-managers diff --git a/presto-spi/pom.xml b/presto-spi/pom.xml index 9473ac932212..61b7d2cf2571 100644 --- a/presto-spi/pom.xml +++ b/presto-spi/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-spi diff --git a/presto-sqlserver/pom.xml b/presto-sqlserver/pom.xml index a5f2aacbfd14..1092aea2d116 100644 --- a/presto-sqlserver/pom.xml +++ b/presto-sqlserver/pom.xml @@ -3,7 +3,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT 4.0.0 diff --git a/presto-teradata-functions/pom.xml b/presto-teradata-functions/pom.xml index 8a1f79cfef4e..1ee5ed8d5965 100644 --- a/presto-teradata-functions/pom.xml +++ b/presto-teradata-functions/pom.xml @@ -4,7 +4,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-teradata-functions diff --git a/presto-testing-server-launcher/pom.xml b/presto-testing-server-launcher/pom.xml index 918a349db7f9..881fdf932cc8 100644 --- a/presto-testing-server-launcher/pom.xml +++ b/presto-testing-server-launcher/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-testing-server-launcher diff --git a/presto-testing/pom.xml b/presto-testing/pom.xml index 5a8c6daca850..524b930cc64c 100644 --- a/presto-testing/pom.xml +++ b/presto-testing/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-testing diff --git a/presto-tests/pom.xml b/presto-tests/pom.xml index 1c01b9766edf..5056972755c7 100644 --- a/presto-tests/pom.xml +++ b/presto-tests/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-tests diff --git a/presto-thrift-api/pom.xml b/presto-thrift-api/pom.xml index 39e56035068e..4f4332a65619 100644 --- a/presto-thrift-api/pom.xml +++ b/presto-thrift-api/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-thrift-api diff --git a/presto-thrift-testing-server/pom.xml b/presto-thrift-testing-server/pom.xml index 5f1fb3d2018a..85d0f0380913 100644 --- a/presto-thrift-testing-server/pom.xml +++ b/presto-thrift-testing-server/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-thrift-testing-server diff --git a/presto-thrift/pom.xml b/presto-thrift/pom.xml index 9a210ce5c9eb..cbf0d152967a 100644 --- a/presto-thrift/pom.xml +++ b/presto-thrift/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-thrift diff --git a/presto-tpcds/pom.xml b/presto-tpcds/pom.xml index 24f4baee2317..6f17360943ab 100644 --- a/presto-tpcds/pom.xml +++ b/presto-tpcds/pom.xml @@ -4,7 +4,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-tpcds diff --git a/presto-tpch/pom.xml b/presto-tpch/pom.xml index 886aa9bc66a5..dc051f7aff16 100644 --- a/presto-tpch/pom.xml +++ b/presto-tpch/pom.xml @@ -4,7 +4,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-tpch diff --git a/presto-verifier/pom.xml b/presto-verifier/pom.xml index 3a670ae5962c..9b6e6d189873 100644 --- a/presto-verifier/pom.xml +++ b/presto-verifier/pom.xml @@ -5,7 +5,7 @@ io.prestosql presto-root - 331-SNAPSHOT + 332-SNAPSHOT presto-verifier