use spark connect

Author: kevinjqliu

.gitignore

@@ -37,13 +37,6 @@ coverage.xml
 bin/
 .vscode/
 
-# Hive/metastore files
-metastore_db/
-
-# Spark/metastore files
-spark-warehouse/
-derby.log
-
 # Python stuff
 .mypy_cache/
 htmlcov

Makefile

@@ -18,7 +18,7 @@
 # Configuration Variables
 # ========================
 
-PYTEST_ARGS ?= -vvv -s -x  # Override with e.g. PYTEST_ARGS="-vv --tb=short"
+PYTEST_ARGS ?= -v -x  # Override with e.g. PYTEST_ARGS="-vv --tb=short"
 COVERAGE ?= 0      # Set COVERAGE=1 to enable coverage: make test COVERAGE=1
 COVERAGE_FAIL_UNDER ?= 85  # Minimum coverage % to pass: make coverage-report COVERAGE_FAIL_UNDER=70
 KEEP_COMPOSE ?= 0  # Set KEEP_COMPOSE=1 to keep containers after integration tests
@@ -93,7 +93,7 @@ lint: ## Run code linters via pre-commit
 ##@ Testing
 
 test: ## Run all unit tests (excluding integration)
-	$(TEST_RUNNER) pytest tests/ -m "(unmarked or parametrize) and not integration" $(PYTEST_ARGS)
+	$(TEST_RUNNER) pytest tests/io/test_pyarrow.py -m "(unmarked or parametrize) and not integration" $(PYTEST_ARGS)
 
 test-integration: test-integration-setup test-integration-exec test-integration-cleanup ## Run integration tests
 

dev/Dockerfile

@@ -36,25 +36,38 @@ ENV PYTHONPATH=$SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9.7-src.zip:$
 RUN mkdir -p ${HADOOP_HOME} && mkdir -p ${SPARK_HOME} && mkdir -p /home/iceberg/spark-events
 WORKDIR ${SPARK_HOME}
 
-# Remember to also update `tests/conftest`'s spark setting
 ENV SPARK_VERSION=3.5.6
-ENV ICEBERG_SPARK_RUNTIME_VERSION=3.5_2.12
-ENV ICEBERG_VERSION=1.9.1
+ENV SCALA_VERSION=2.12
+ENV ICEBERG_SPARK_RUNTIME_VERSION=3.5_${SCALA_VERSION}
+ENV ICEBERG_VERSION=1.9.2
 ENV PYICEBERG_VERSION=0.10.0
+ENV HADOOP_VERSION=3.3.4
+ENV AWS_SDK_VERSION=1.12.753
 
-RUN curl --retry 5 -s -C - https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \
+RUN curl --retry 5 -s -C - https://downloads.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop3.tgz -o spark-${SPARK_VERSION}-bin-hadoop3.tgz \
  && tar xzf spark-${SPARK_VERSION}-bin-hadoop3.tgz --directory /opt/spark --strip-components 1 \
  && rm -rf spark-${SPARK_VERSION}-bin-hadoop3.tgz
 
+# Download Spark Connect server JAR
+RUN curl --retry 5 -s -L https://repo1.maven.org/maven2/org/apache/spark/spark-connect_${SCALA_VERSION}/${SPARK_VERSION}/spark-connect_${SCALA_VERSION}-${SPARK_VERSION}.jar \
+      -Lo /opt/spark/jars/spark-connect_${SCALA_VERSION}-${SPARK_VERSION}.jar
+
 # Download iceberg spark runtime
 RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar \
       -Lo /opt/spark/jars/iceberg-spark-runtime-${ICEBERG_SPARK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar
 
-
 # Download AWS bundle
 RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar \
       -Lo /opt/spark/jars/iceberg-aws-bundle-${ICEBERG_VERSION}.jar
 
+# Download hadoop-aws (required for S3 support)
+RUN curl --retry 5 -s https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar \
+      -Lo /opt/spark/jars/hadoop-aws-${HADOOP_VERSION}.jar
+
+# Download AWS SDK bundle
+RUN curl --retry 5 -s https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/${AWS_SDK_VERSION}/aws-java-sdk-bundle-${AWS_SDK_VERSION}.jar \
+      -Lo /opt/spark/jars/aws-java-sdk-bundle-${AWS_SDK_VERSION}.jar
+
 COPY spark-defaults.conf /opt/spark/conf
 ENV PATH="/opt/spark/sbin:/opt/spark/bin:${PATH}"
 

dev/docker-compose-integration.yml

@@ -31,8 +31,8 @@ services:
       - AWS_SECRET_ACCESS_KEY=password
       - AWS_REGION=us-east-1
     ports:
-      - 8888:8888
-      - 8080:8080
+      - 15002:15002 # Spark Connect
+      - 4040:4040 # Spark UI
     links:
       - rest:rest
       - hive:hive

dev/entrypoint.sh

@@ -18,8 +18,6 @@
 # under the License.
 #
 
-start-master.sh -p 7077
-start-worker.sh spark://spark-iceberg:7077
-start-history-server.sh
+start-connect-server.sh
 
 tail -f /dev/null

dev/spark-defaults.conf

@@ -16,20 +16,35 @@
 #
 
 spark.sql.extensions                   org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
+
+# Configure Iceberg REST catalog
 spark.sql.catalog.rest                 org.apache.iceberg.spark.SparkCatalog
 spark.sql.catalog.rest.type            rest
 spark.sql.catalog.rest.uri             http://rest:8181
 spark.sql.catalog.rest.io-impl         org.apache.iceberg.aws.s3.S3FileIO
 spark.sql.catalog.rest.warehouse       s3://warehouse/rest/
 spark.sql.catalog.rest.s3.endpoint     http://minio:9000
+spark.sql.catalog.rest.cache-enabled   false
+
+# Configure Iceberg Hive catalog
 spark.sql.catalog.hive                 org.apache.iceberg.spark.SparkCatalog
 spark.sql.catalog.hive.type            hive
 spark.sql.catalog.hive.uri             thrift://hive:9083
 spark.sql.catalog.hive.io-impl         org.apache.iceberg.aws.s3.S3FileIO
 spark.sql.catalog.hive.warehouse       s3://warehouse/hive/
 spark.sql.catalog.hive.s3.endpoint     http://minio:9000
+
+# Configure Spark's default session catalog (spark_catalog) to use Iceberg backed by the Hive Metastore
+spark.sql.catalog.spark_catalog        org.apache.iceberg.spark.SparkSessionCatalog
+spark.sql.catalog.spark_catalog.type   hive
+spark.sql.catalog.spark_catalog.uri    thrift://hive:9083
+spark.hadoop.fs.s3a.endpoint           http://minio:9000
+spark.sql.catalogImplementation        hive
+spark.sql.warehouse.dir                s3a://warehouse/hive/
+
 spark.sql.defaultCatalog               rest
+
+# Configure Spark UI and event logging
+spark.ui.enabled                       true
 spark.eventLog.enabled                 true
 spark.eventLog.dir                     /home/iceberg/spark-events
-spark.history.fs.logDirectory          /home/iceberg/spark-events
-spark.sql.catalogImplementation        in-memory