ARROW-1780 - JDBC Adapter to convert Relational Data objects to Arrow Data Format Vector Objects

dev/release/rat_exclude_files.txt

@@ -74,3 +74,5 @@ c_glib/doc/reference/gtk-doc.make
 *.svg
 *.devhelp2
 *.scss
+*.yml
+*.properties

java/adapter/jdbc/pom.xml

@@ -0,0 +1,76 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+  license agreements. See the NOTICE file distributed with this work for additional
+  information regarding copyright ownership. The ASF licenses this file to
+  You under the Apache License, Version 2.0 (the "License"); you may not use
+  this file except in compliance with the License. You may obtain a copy of
+  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+  by applicable law or agreed to in writing, software distributed under the
+  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+  OF ANY KIND, either express or implied. See the License for the specific
+  language governing permissions and limitations under the License. -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+
+    <modelVersion>4.0.0</modelVersion>
+    <groupId>org.apache.arrow.adapter.jdbc</groupId>
+    <artifactId>arrow-jdbc</artifactId>
+    <packaging>jar</packaging>
+    <version>0.10.0-SNAPSHOT</version>
+    <name>Arrow JDBC Adapter</name>
+    <url>http://maven.apache.org</url>
+    <dependencies>
+
+        <!-- https://mvnrepository.com/artifact/org.apache.arrow/arrow-memory -->
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-memory</artifactId>
+            <version>0.9.0-SNAPSHOT</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/org.apache.arrow/arrow-vector -->
+        <dependency>
+            <groupId>org.apache.arrow</groupId>
+            <artifactId>arrow-vector</artifactId>
+            <version>0.9.0-SNAPSHOT</version>
+        </dependency>
+
+        <!-- Test -->
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+            <version>4.11</version>
+            <scope>test</scope>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/com.h2database/h2 -->
+        <dependency>
+            <groupId>com.h2database</groupId>
+            <artifactId>h2</artifactId>
+            <version>1.4.196</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.dataformat</groupId>
+            <artifactId>jackson-dataformat-yaml</artifactId>
+            <version>2.3.0</version>
+            <scope>test</scope>
+        </dependency>
+
+
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <configuration>
+                    <systemPropertyVariables>
+                        <user.timezone>UTC</user.timezone>
+                    </systemPropertyVariables>
+                </configuration>
+
+            </plugin>
+        </plugins>
+    </build>
+</project>

java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/ArrowDataFetcher.java

@@ -0,0 +1,107 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.adapter.jdbc;
+
+import org.apache.arrow.vector.VectorSchemaRoot;
+
+import java.sql.Connection;
+
+/**
+ * Class to fetch data from a given database table where user can specify columns to fetch
+ * along with limit and offset parameters.
+ *
+ * The object of this class is returned by invoking method jdbcArrowDataFetcher(Connection connection, String tableName)
+ * from {@link JdbcToArrow} class. Caller can use this object to fetch data repetitively based on the
+ * data fetch requirement and can implement pagination like functionality.
+ *
+ * This class doesn't hold any open connections to database but simply executes the "select" query everytime with
+ * the necessary limit and offset parameters.
+ *
+ * @since 0.10.0
+ * @see JdbcToArrow
+ */
+public class ArrowDataFetcher {
+
+    private static final String all_columns_query = "select * from %s limit %d offset %d";
+    private static final String custom_columns_query = "select %s from %s limit %d offset %d";
+    private Connection connection;
+    private String tableName;
+
+    /**
+     * Constructor
+     * @param connection
+     * @param tableName
+     */
+    public ArrowDataFetcher(Connection connection, String tableName) {
+        this.connection = connection;
+        this.tableName = tableName;
+    }
+
+    /**
+     * Fetch the data from underlying table with the given limit and offset and for passed column names.
+     *
+     * @param offset
+     * @param limit
+     * @param columns
+     * @return
+     * @throws Exception
+     */
+    public VectorSchemaRoot fetch(int offset, int limit, String... columns) throws Exception {
+        assert columns != null && columns.length > 0 : "columns can't be empty!";
+        assert limit > 0 : "limit needs to be greater that 0";
+        assert offset >= 0 : "offset needs to be greater than or equal to 0";
+
+        return JdbcToArrow.sqlToArrow(connection,
+                String.format(custom_columns_query, commaSeparatedQueryColumns(columns),
+                        tableName, limit, offset));
+    }
+
+    /**
+     * Fetch the data from underlying table with the given limit and offset and for all the columns.
+     * @param offset
+     * @param limit
+     * @return
+     * @throws Exception
+     */
+    public VectorSchemaRoot fetch(int offset, int limit) throws Exception {
+        assert limit > 0 : "limit needs to be greater that 0";
+        assert offset >= 0 : "offset needs to be greater than or equal to 0";
+
+        return JdbcToArrow.sqlToArrow(connection, String.format(all_columns_query, tableName, limit, offset));
+    }
+
+    public static String commaSeparatedQueryColumns(String... columns) {
+        assert columns != null && columns.length > 0 : "columns can't be empty!";
+
+        StringBuilder columnBuilder = new StringBuilder();
+        boolean insertComma = false;
+        for (String s: columns) {
+            if (insertComma) {
+                columnBuilder.append(',');
+            }
+            columnBuilder.append(' ').append(s);
+            insertComma = true;
+        }
+        columnBuilder.append(' ');
+        return columnBuilder.toString();
+    }
+
+}
+
+

java/adapter/jdbc/src/main/java/org/apache/arrow/adapter/jdbc/JdbcToArrow.java

@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.adapter.jdbc;
+
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
+
+import java.sql.*;
+
+/**
+ * Utility class to convert JDBC objects to columnar Arrow format objects.
+ *
+ * This utility uses following data mapping to map JDBC/SQL datatype to Arrow data types.
+ *
+ * CHAR	--> ArrowType.Utf8
+ * NCHAR	--> ArrowType.Utf8
+ * VARCHAR --> ArrowType.Utf8
+ * NVARCHAR --> ArrowType.Utf8
+ * LONGVARCHAR --> ArrowType.Utf8
+ * LONGNVARCHAR --> ArrowType.Utf8
+ * NUMERIC --> ArrowType.Decimal(precision, scale)
+ * DECIMAL --> ArrowType.Decimal(precision, scale)
+ * BIT --> ArrowType.Bool
+ * TINYINT --> ArrowType.Int(8, signed)
+ * SMALLINT --> ArrowType.Int(16, signed)
+ * INTEGER --> ArrowType.Int(32, signed)
+ * BIGINT --> ArrowType.Int(64, signed)
+ * REAL --> ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)
+ * FLOAT --> ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE)
+ * DOUBLE --> ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE)
+ * BINARY --> ArrowType.Binary
+ * VARBINARY --> ArrowType.Binary
+ * LONGVARBINARY --> ArrowType.Binary
+ * DATE --> ArrowType.Date(DateUnit.MILLISECOND)
+ * TIME --> ArrowType.Time(TimeUnit.MILLISECOND, 32)
+ * TIMESTAMP --> ArrowType.Timestamp(TimeUnit.MILLISECOND, timezone=null)
+ * CLOB --> ArrowType.Utf8
+ * BLOB --> ArrowType.Binary
+ *
+ * @since 0.10.0
+ * @see ArrowDataFetcher
+ */
+public class JdbcToArrow {
+
+    /**
+     * For the given SQL query, execute and fetch the data from Relational DB and convert it to Arrow objects.
+     *
+     * @param connection Database connection to be used. This method will not close the passed connection object. Since hte caller has passed
+     *                   the connection object it's the responsibility of the caller to close or return the connection to the pool.
+     * @param query The DB Query to fetch the data.
+     * @return
+     * @throws SQLException Propagate any SQL Exceptions to the caller after closing any resources opened such as ResultSet and Statment objects.
+     */
+    public static VectorSchemaRoot sqlToArrow(Connection connection, String query) throws Exception {
+
+        assert connection != null: "JDBC conncetion object can not be null";
+        assert query != null && query.length() > 0: "SQL query can not be null or empty";
+
+        RootAllocator rootAllocator = new RootAllocator(Integer.MAX_VALUE);
+
+        Statement stmt = null;
+        ResultSet rs = null;
+        try {
+            stmt = connection.createStatement();
+            rs = stmt.executeQuery(query);
+            ResultSetMetaData rsmd = rs.getMetaData();
+            VectorSchemaRoot root = VectorSchemaRoot.create(
+                    JdbcToArrowUtils.jdbcToArrowSchema(rsmd), rootAllocator);
+            JdbcToArrowUtils.jdbcToArrowVectors(rs, root);
+            return root;
+        } catch (Exception exc) {
+            // just throw it out after logging
+            throw exc;
+        } finally {
+            if (rs != null) {
+                rs.close();
+            }
+            if (stmt != null) {
+                stmt.close(); // test
+            }
+        }
+    }
+
+    /**
+     * This method returns ArrowDataFetcher Object that can be used to fetch and iterate on the data in the given
+     * database table.
+     *
+     * @param connection - Database connection Object
+     * @param tableName - Table name from which records will be fetched
+     *
+     * @return ArrowDataFetcher - Instance of ArrowDataFetcher which can be used to get Arrow Vector obejcts by calling its functionality
+     */
+    public static ArrowDataFetcher jdbcArrowDataFetcher(Connection connection, String tableName) {
+        assert connection != null: "JDBC conncetion object can not be null";
+        assert tableName != null && tableName.length() > 0: "Table name can not be null or empty";
+
+        return new ArrowDataFetcher(connection, tableName);
+    }
+
+}