diff --git a/.github/workflows/docker-GA-images.yml b/.github/workflows/docker-GA-images.yml index 3c2629e13789..a1169f71d8f7 100644 --- a/.github/workflows/docker-GA-images.yml +++ b/.github/workflows/docker-GA-images.yml @@ -112,6 +112,63 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 + - name: Build Hive Image locally + uses: docker/build-push-action@v4 + with: + context: ./packaging/src/docker/ + file: ./packaging/src/docker/Dockerfile + push: false + load: true + tags: hive:test + build-args: + | + HIVE_VERSION=${{ env.HIVE_VERSION }} + HADOOP_VERSION=${{ env.HADOOP_VERSION }} + TEZ_VERSION=${{ env.TEZ_VERSION }} + BUILD_ENV=${{ env.BUILD_ENV }} + + - name: Build Standalone Metastore Image locally + uses: docker/build-push-action@v4 + with: + context: ./standalone-metastore/packaging/src/docker/ + file: ./standalone-metastore/packaging/src/docker/Dockerfile + push: false + load: true + tags: hive:standalone-metastore-test + build-args: + | + HIVE_VERSION=${{ env.HIVE_VERSION }} + HADOOP_VERSION=${{ env.HADOOP_VERSION }} + BUILD_ENV=${{ env.BUILD_ENV }} + + - name: Create k8s cluster + uses: helm/kind-action@v1 + + - name: Set up Helm + uses: azure/setup-helm@v4 + + - name: Load images + run: kind load docker-image hive:test hive:standalone-metastore-test --name chart-testing + + - name: Deploy Hive + run: | + ./itests/test-docker/scripts/setup.sh + + - name: Run JUnit + # On workflow_dispatch, the checked-out source code and libraries could be inconsistent with the specified Hive release + if: github.event_name != 'workflow_dispatch' + run: | + mvn --batch-mode test -Pitests -Pdocker-integration-tests -pl itests/test-docker + + - name: Test HiveServer2 + run: | + ./itests/test-docker/scripts/test_hive_server2.sh + + - name: Clean up k8s + run: | + ./itests/test-docker/scripts/teardown.sh + kind delete cluster --name chart-testing + - name: Build and push Hive Image to docker hub uses: docker/build-push-action@v4 with: @@ -141,8 +198,16 @@ jobs: HADOOP_VERSION=${{ env.HADOOP_VERSION }} BUILD_ENV=${{ env.BUILD_ENV }} - - name: Dump disk usage on failure - if: failure() + - name: Dump resource space at the end + run: | + df -h + free -h + + - name: Dump resource usage on failure + if: failure() || cancelled() run: | df -h du -xh / --max-depth=3 2>/dev/null | sort -h | tail -50 + free -h + ps aux --sort -rss | head -n 30 + kubectl get pods diff --git a/itests/hive-iceberg/pom.xml b/itests/hive-iceberg/pom.xml index 70bdb9bc95a1..5e661cc65e93 100644 --- a/itests/hive-iceberg/pom.xml +++ b/itests/hive-iceberg/pom.xml @@ -27,7 +27,6 @@ ../.. UTF-8 false - 1.10.0 diff --git a/itests/pom.xml b/itests/pom.xml index 669364573db6..f89b4a8e134d 100644 --- a/itests/pom.xml +++ b/itests/pom.xml @@ -26,6 +26,8 @@ .. 2.32.0 + 1.10.0 + 3.27.3 custom-serde @@ -44,6 +46,7 @@ qtest-druid qtest-kudu qtest-iceberg + test-docker hive-iceberg @@ -69,6 +72,11 @@ hive-metastore ${project.version} + + org.apache.hive + hive-standalone-metastore-client + ${standalone-metastore.version} + org.apache.hive hive-standalone-metastore-common @@ -497,6 +505,12 @@ hive-iceberg-handler ${project.version} + + org.assertj + assertj-core + ${assertj.version} + test + diff --git a/itests/test-docker/helm/ozone/values.yaml b/itests/test-docker/helm/ozone/values.yaml new file mode 100644 index 000000000000..49423acba627 --- /dev/null +++ b/itests/test-docker/helm/ozone/values.yaml @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +datanode: + replicas: 1 +env: +# GitHub Actions provide tiny spaces +- name: OZONE-SITE.XML_hdds.datanode.volume.min.free.space + value: "256MB" +- name: OZONE-SITE.XML_hdds.scm.safemode.enabled + value: "false" +- name: OZONE-SITE.XML_ozone.scm.container.size + value: 128MB +- name: OZONE-SITE.XML_ozone.scm.block.size + value: 32MB +- name: OZONE-SITE.XML_ozone.server.default.replication + value: "1" diff --git a/itests/test-docker/k8s/beeline/deployment.yaml b/itests/test-docker/k8s/beeline/deployment.yaml new file mode 100644 index 000000000000..363a39c0eca0 --- /dev/null +++ b/itests/test-docker/k8s/beeline/deployment.yaml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: beeline + name: beeline +spec: + replicas: 1 + selector: + matchLabels: + app: beeline + template: + metadata: + labels: + app: beeline + spec: + containers: + - name: loop + image: hive:test + command: + - timeout + args: + - infinity + - sleep + - infinity + env: + - name: TERM + value: dumb diff --git a/itests/test-docker/k8s/hadoop/configmap.yaml b/itests/test-docker/k8s/hadoop/configmap.yaml new file mode 100644 index 000000000000..f88a326a4c76 --- /dev/null +++ b/itests/test-docker/k8s/hadoop/configmap.yaml @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: hadoop-config +data: + core-site.xml: | + + + fs.s3a.endpoint + http://ozone-s3g-rest:9878 + + + fs.s3a.path.style.access + true + + + fs.s3a.change.detection.version.required + false + + + fs.s3a.change.detection.mode + none + + + fs.s3a.access.key + hadoop + + + fs.s3a.secret.key + dummy + + diff --git a/itests/test-docker/k8s/hive-metastore/configmap.yaml b/itests/test-docker/k8s/hive-metastore/configmap.yaml new file mode 100644 index 000000000000..72d0ce44762e --- /dev/null +++ b/itests/test-docker/k8s/hive-metastore/configmap.yaml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: hive-metastore-config +data: + metastore-site.xml: | + + + metastore.warehouse.dir + s3a://test/test-warehouse + + + hive.metastore.warehouse.external.dir + s3a://test/test-warehouse + + + metastore.event.db.notification.api.auth + false + + + metastore.catalog.servlet.port + 9001 + + + metastore.catalog.servlet.auth + none + + diff --git a/itests/test-docker/k8s/hive-metastore/deployment.yaml b/itests/test-docker/k8s/hive-metastore/deployment.yaml new file mode 100644 index 000000000000..cee75d06e0ec --- /dev/null +++ b/itests/test-docker/k8s/hive-metastore/deployment.yaml @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: hive-metastore + name: hive-metastore +spec: + replicas: 1 + selector: + matchLabels: + app: hive-metastore + template: + metadata: + labels: + app: hive-metastore + spec: + initContainers: + - name: download-aws-sdk + image: ubuntu:26.04 + command: + - /bin/bash + args: + - -c + - | + apt-get update + apt-get install -y --no-install-recommends ca-certificates wget + wget https://repo1.maven.org/maven2/software/amazon/awssdk/bundle/2.26.19/bundle-2.26.19.jar -P /tmp/ext-jars + volumeMounts: + - name: ext-jars + mountPath: /tmp/ext-jars + containers: + - name: hive-metastore + image: hive:standalone-metastore-test + env: + - name: HADOOP_OPTIONAL_TOOLS + value: hadoop-aws + - name: HIVE_CUSTOM_CONF_DIR + value: /etc/hive/conf + readinessProbe: + tcpSocket: + port: 9083 + volumeMounts: + - name: ext-jars + mountPath: /tmp/ext-jars + - name: hadoop-config + mountPath: /etc/hive/conf/core-site.xml + subPath: core-site.xml + - name: hive-metastore-config + mountPath: /etc/hive/conf/metastore-site.xml + subPath: metastore-site.xml + volumes: + - name: ext-jars + emptyDir: {} + - name: hadoop-config + configMap: + name: hadoop-config + - name: hive-metastore-config + configMap: + name: hive-metastore-config diff --git a/itests/test-docker/k8s/hive-metastore/service.yaml b/itests/test-docker/k8s/hive-metastore/service.yaml new file mode 100644 index 000000000000..a20052c4ba90 --- /dev/null +++ b/itests/test-docker/k8s/hive-metastore/service.yaml @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: hive-metastore + labels: + app: hive-metastore +spec: + ports: + - name: thrift + port: 9083 + - name: rest + port: 9001 + selector: + app: hive-metastore diff --git a/itests/test-docker/k8s/hive/configmap.yaml b/itests/test-docker/k8s/hive/configmap.yaml new file mode 100644 index 000000000000..7829d80edf00 --- /dev/null +++ b/itests/test-docker/k8s/hive/configmap.yaml @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: hive-config +data: + hive-site.xml: | + + + hive.metastore.uris + thrift://hive-metastore:9083 + + + hive.server2.enable.doAs + false + + + hive.tez.exec.inplace.progress + false + + + hive.tez.exec.print.summary + true + + + hive.exec.scratchdir + /opt/hive/scratch_dir + + + hive.user.install.directory + /opt/hive/install_dir + + + tez.runtime.optimize.local.fetch + true + + + hive.exec.submit.local.task.via.child + false + + + hive.compactor.worker.threads + 1 + + + mapreduce.framework.name + local + + + tez.local.mode + true + + + hive.metastore.warehouse.dir + s3a://test/test-warehouse + + + hive.metastore.event.db.notification.api.auth + false + + diff --git a/itests/test-docker/k8s/hive/deployment.yaml b/itests/test-docker/k8s/hive/deployment.yaml new file mode 100644 index 000000000000..81fb105233f3 --- /dev/null +++ b/itests/test-docker/k8s/hive/deployment.yaml @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: hive + name: hive +spec: + replicas: 1 + selector: + matchLabels: + app: hive + template: + metadata: + labels: + app: hive + spec: + initContainers: + - name: download-aws-sdk + image: ubuntu:26.04 + command: + - /bin/bash + args: + - -c + - | + apt-get update + apt-get install -y --no-install-recommends ca-certificates wget + wget https://repo1.maven.org/maven2/software/amazon/awssdk/bundle/2.26.19/bundle-2.26.19.jar -P /tmp/ext-jars + volumeMounts: + - name: ext-jars + mountPath: /tmp/ext-jars + containers: + - name: hive + image: hive:test + env: + - name: HADOOP_OPTIONAL_TOOLS + value: hadoop-aws + - name: AWS_ACCESS_KEY_ID + value: hadoop + - name: AWS_SECRET_ACCESS_KEY + value: dummy + - name: IS_RESUME + value: "true" + - name: SERVICE_NAME + value: hiveserver2 + - name: HIVE_CUSTOM_CONF_DIR + value: /etc/hive/conf + readinessProbe: + tcpSocket: + port: 10000 + volumeMounts: + - name: ext-jars + mountPath: /tmp/ext-jars + - name: hadoop-config + mountPath: /etc/hive/conf/core-site.xml + subPath: core-site.xml + - name: hive-config + mountPath: /etc/hive/conf/hive-site.xml + subPath: hive-site.xml + volumes: + - name: ext-jars + emptyDir: {} + - name: hadoop-config + configMap: + name: hadoop-config + - name: hive-config + configMap: + name: hive-config diff --git a/itests/test-docker/k8s/hive/service.yaml b/itests/test-docker/k8s/hive/service.yaml new file mode 100644 index 000000000000..60b7b67428b1 --- /dev/null +++ b/itests/test-docker/k8s/hive/service.yaml @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Service +metadata: + name: hive + labels: + app: hive +spec: + ports: + - port: 10000 + selector: + app: hive diff --git a/itests/test-docker/pom.xml b/itests/test-docker/pom.xml new file mode 100644 index 000000000000..3efaac184206 --- /dev/null +++ b/itests/test-docker/pom.xml @@ -0,0 +1,99 @@ + + + + 4.0.0 + + org.apache.hive + hive-it + 4.3.0-SNAPSHOT + ../pom.xml + + test-docker + jar + Hive Integration - Testing Docker images + + ../.. + true + + + + org.apache.hadoop + hadoop-aws + test + + + org.apache.hadoop + hadoop-common + test + + + org.apache.iceberg + iceberg-api + ${iceberg.version} + tests + test + + + org.apache.iceberg + iceberg-core + ${iceberg.version} + test + + + org.apache.iceberg + iceberg-core + ${iceberg.version} + tests + test + + + org.apache.iceberg + iceberg-open-api + ${iceberg.version} + test-fixtures + test + + + org.assertj + assertj-core + test + + + org.junit.jupiter + junit-jupiter + ${junit.jupiter.version} + test + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + ${skip.docker.integration.tests} + + + + + + + docker-integration-tests + + false + + + + diff --git a/itests/test-docker/scripts/setup.sh b/itests/test-docker/scripts/setup.sh new file mode 100755 index 000000000000..9068670e4f41 --- /dev/null +++ b/itests/test-docker/scripts/setup.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eux + +BUCKET=/s3v/test + +wait_and_monitor_deployment() { + local app="$1" + until kubectl rollout status deployment "$app" --timeout=30s; do + kubectl get deployment "$app" + kubectl get pods -l app="$app" + kubectl describe deployment "$app" + done +} + +helm repo add ozone https://apache.github.io/ozone-helm-charts/ +helm install ozone ozone/ozone --version 0.2.0 --values itests/test-docker/helm/ozone/values.yaml --wait +# Wait for a while because Ozone's Helm chart does not have readiness probes... +sleep 10 +if kubectl exec statefulset/ozone-om -- ozone sh bucket info "$BUCKET" >/dev/null 2>&1; then + echo "Bucket already exists. Skipping." +else + echo "Bucket does not exist. Creating..." + kubectl exec statefulset/ozone-om -- ozone sh bucket create "$BUCKET" +fi + +base_dir=$(dirname "$(cd "$(dirname "$0")" || exit; pwd)") + +kubectl apply -f "$base_dir/k8s/*" + +wait_and_monitor_deployment hive-metastore +wait_and_monitor_deployment hive +wait_and_monitor_deployment beeline + +mkdir -p "$base_dir/target" +nohup kubectl port-forward service/hive-metastore 9083 > "$base_dir/target/hive-metastore-thrift.log" 2>&1 & +echo $! > "$base_dir/target/hive-metastore-thrift.pid" +nohup kubectl port-forward service/hive-metastore 9001 > "$base_dir/target/hive-metastore-rest.log" 2>&1 & +echo $! > "$base_dir/target/hive-metastore-rest.pid" +nohup kubectl port-forward service/ozone-s3g-rest 9878 > "$base_dir/target/ozone-s3g.log" & +echo $! > "$base_dir/target/ozone-s3g.pid" diff --git a/itests/test-docker/scripts/teardown.sh b/itests/test-docker/scripts/teardown.sh new file mode 100755 index 000000000000..ac73b652e7bf --- /dev/null +++ b/itests/test-docker/scripts/teardown.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eux + +base_dir=$(dirname "$(cd "$(dirname "$0")" || exit; pwd)") + +if [ -f "$base_dir/target/hive-metastore-thrift.pid" ]; then + kill $(cat "$base_dir/target/hive-metastore-thrift.pid") +fi +if [ -f "$base_dir/target/hive-metastore-rest.pid" ]; then + kill $(cat "$base_dir/target/hive-metastore-rest.pid") +fi +if [ -f "$base_dir/target/ozone-s3g.pid" ]; then + kill $(cat "$base_dir/target/ozone-s3g.pid") +fi + +kubectl delete -f "$base_dir/k8s/*" +helm uninstall ozone diff --git a/itests/test-docker/scripts/test_hive_server2.sh b/itests/test-docker/scripts/test_hive_server2.sh new file mode 100755 index 000000000000..18561fb62f54 --- /dev/null +++ b/itests/test-docker/scripts/test_hive_server2.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eux + +kubectl exec deployment/beeline -- beeline -u 'jdbc:hive2://hive:10000/' \ + -e 'create table if not exists test_hive_server2 (id int, name string) stored by iceberg' +kubectl exec deployment/beeline -- beeline -u 'jdbc:hive2://hive:10000/' \ + -e "insert into test_hive_server2 values (1, 'aaa'), (2, 'bbb')" +kubectl exec deployment/beeline -- beeline -u 'jdbc:hive2://hive:10000/' \ + -e 'select * from test_hive_server2' diff --git a/itests/test-docker/src/test/java/org/apache/iceberg/rest/HiveIcebergRESTCatalogClient.java b/itests/test-docker/src/test/java/org/apache/iceberg/rest/HiveIcebergRESTCatalogClient.java new file mode 100644 index 000000000000..48667576d2f3 --- /dev/null +++ b/itests/test-docker/src/test/java/org/apache/iceberg/rest/HiveIcebergRESTCatalogClient.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.iceberg.rest; + +import java.nio.file.Files; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.catalog.Namespace; + +class HiveIcebergRESTCatalogClient { + private static final Namespace DEFAULT_NS = Namespace.of("default"); + + private final RESTCatalog restCatalog; + private final Configuration conf; + + HiveIcebergRESTCatalogClient() throws Exception { + this(Collections.emptyMap()); + } + + HiveIcebergRESTCatalogClient(Map additionalProperties) throws Exception { + var properties = new HashMap<>(additionalProperties); + properties.put(CatalogProperties.URI, "http://localhost:9001/iceberg"); + restCatalog = RCKUtils.initCatalogClient(properties); + conf = new Configuration(false); + conf.set("fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem"); + conf.set("fs.s3a.endpoint", "http://localhost:9878"); + conf.set("fs.s3a.path.style.access", "true"); + conf.set("fs.s3a.change.detection.version.required", "false"); + conf.set("fs.s3a.change.detection.mode", "none"); + conf.set("fs.s3a.access.key", "hadoop"); + conf.set("fs.s3a.secret.key", "dummy"); + conf.set("hadoop.tmp.dir", Files.createTempDirectory("hive-docker-test").toString()); + } + + RESTCatalog getRestCatalog() { + return restCatalog; + } + + void cleanupWarehouse() throws Exception { + restCatalog.listNamespaces().stream().filter(namespace -> !DEFAULT_NS.equals(namespace)).forEach(namespace -> { + restCatalog.listTables(namespace).forEach(restCatalog::dropTable); + restCatalog.listViews(namespace).forEach(restCatalog::dropView); + restCatalog.dropNamespace(namespace); + }); + // Delete the DB directories explicitly since HiveCatalog#dropNamespace does not wipe them out + var warehouseRoot = new Path("s3a://test/test-warehouse"); + var fs = warehouseRoot.getFileSystem(conf); + if (!fs.exists(warehouseRoot)) { + return; + } + for (FileStatus fileStatus : fs.listStatus(warehouseRoot)) { + fs.delete(fileStatus.getPath(), true); + } + } + + void close() throws Exception { + cleanupWarehouse(); + restCatalog.close(); + } +} diff --git a/itests/test-docker/src/test/java/org/apache/iceberg/rest/HiveIcebergRESTCatalogTests.java b/itests/test-docker/src/test/java/org/apache/iceberg/rest/HiveIcebergRESTCatalogTests.java new file mode 100644 index 000000000000..680ef9b5758f --- /dev/null +++ b/itests/test-docker/src/test/java/org/apache/iceberg/rest/HiveIcebergRESTCatalogTests.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.iceberg.rest; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Map; +import org.apache.iceberg.catalog.CatalogTests; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; + +class HiveIcebergRESTCatalogTests extends CatalogTests { + private static HiveIcebergRESTCatalogClient client; + + @BeforeAll + static void beforeClass() throws Exception { + client = new HiveIcebergRESTCatalogClient(); + + assertThat(client.getRestCatalog().listNamespaces()) + .withFailMessage("Namespaces list should not contain: %s", RCKUtils.TEST_NAMESPACES) + .doesNotContainAnyElementsOf(RCKUtils.TEST_NAMESPACES); + } + + @BeforeEach + void before() throws Exception { + client.cleanupWarehouse(); + } + + @AfterAll + static void afterClass() throws Exception { + client.close(); + } + + @Override + protected RESTCatalog catalog() { + return client.getRestCatalog(); + } + + @Override + protected RESTCatalog initCatalog(String catalogName, Map additionalProperties) { + try { + return new HiveIcebergRESTCatalogClient(additionalProperties).getRestCatalog(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + protected boolean requiresNamespaceCreate() { + return true; + } + + @Override + protected boolean supportsNamesWithSlashes() { + return false; + } + + @Override + protected boolean supportsServerSideRetry() { + return true; + } +} diff --git a/itests/test-docker/src/test/java/org/apache/iceberg/rest/HiveIcebergRESTViewCatalogTests.java b/itests/test-docker/src/test/java/org/apache/iceberg/rest/HiveIcebergRESTViewCatalogTests.java new file mode 100644 index 000000000000..20041bdbca9a --- /dev/null +++ b/itests/test-docker/src/test/java/org/apache/iceberg/rest/HiveIcebergRESTViewCatalogTests.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.iceberg.rest; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.apache.iceberg.view.ViewCatalogTests; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; + +class HiveIcebergRESTViewCatalogTests extends ViewCatalogTests { + private static HiveIcebergRESTCatalogClient client; + + @BeforeAll + static void beforeClass() throws Exception { + client = new HiveIcebergRESTCatalogClient(); + + assertThat(client.getRestCatalog().listNamespaces()) + .withFailMessage("Namespaces list should not contain: %s", RCKUtils.TEST_NAMESPACES) + .doesNotContainAnyElementsOf(RCKUtils.TEST_NAMESPACES); + } + + @BeforeEach + void before() throws Exception { + client.cleanupWarehouse(); + } + + @AfterAll + static void afterClass() throws Exception { + client.close(); + } + + @Override + protected RESTCatalog catalog() { + return client.getRestCatalog(); + } + + @Override + protected RESTCatalog tableCatalog() { + return client.getRestCatalog(); + } + + @Override + protected boolean requiresNamespaceCreate() { + return true; + } + + @Override + protected boolean supportsServerSideRetry() { + return true; + } + + @Override + public void completeCreateView() { + // This test case requires https://github.com/apache/iceberg/pull/14653 + } + + @Override + public void createAndReplaceViewWithLocation() { + // This test case requires https://github.com/apache/iceberg/pull/14653 + } + + @Override + public void createViewWithCustomMetadataLocation() { + // This test case requires https://github.com/apache/iceberg/pull/14653 + } + + @Override + public void updateViewLocation() { + // This test case requires https://github.com/apache/iceberg/pull/14653 + } +}