diff --git a/bin/test.sh b/bin/test.sh index 73e04459..21f6873a 100755 --- a/bin/test.sh +++ b/bin/test.sh @@ -61,6 +61,16 @@ function check_gpdb() { environment_compose exec gpdb su gpadmin -l -c "pg_isready" } +function check_spark() { + environment_compose exec spark curl -f http://localhost:10213 +} + +function run_spark_tests() { + environment_compose exec spark beeline -u jdbc:hive2://localhost:10213 -e 'SELECT 1;' && + environment_compose exec spark beeline -u jdbc:hive2://localhost:10213 -e 'SHOW DATABASES;' && + true +} + function check_health() { local service=$1 test "$(environment_compose ps --format json | jq -er --arg name "$service" '.[] | select(.Service == $name) | .Health')" == "healthy" @@ -200,7 +210,14 @@ for ARCH in "${platforms[@]}"; do elif [[ ${ENVIRONMENT} == *"openldap"* ]]; then retry check_openldap elif [[ ${ENVIRONMENT} == *"spark"* ]]; then - retry check_health spark + # wait until Spark is started + retry check_spark + + # run tests + set -x + set +e + sleep 10 + run_spark_tests else echo >&2 "ERROR: no test defined for ${ENVIRONMENT}" cleanup diff --git a/testing/spark3-delta/Dockerfile b/testing/spark3-delta/Dockerfile index 466d4439..7eb1ff91 100644 --- a/testing/spark3-delta/Dockerfile +++ b/testing/spark3-delta/Dockerfile @@ -45,8 +45,6 @@ ENV PATH="${SPARK_HOME}/bin:${PATH}" EXPOSE 10213 -HEALTHCHECK --interval=10s --timeout=5s --start-period=10s \ - CMD curl -f http://127.0.0.1:10213/ CMD spark-submit \ --master "local[*]" \ --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 \ diff --git a/testing/spark3-hudi/Dockerfile b/testing/spark3-hudi/Dockerfile index eb1f3783..9b88917c 100644 --- a/testing/spark3-hudi/Dockerfile +++ b/testing/spark3-hudi/Dockerfile @@ -44,8 +44,6 @@ ENV PATH="${SPARK_HOME}/bin:${PATH}" EXPOSE 10213 -HEALTHCHECK --interval=10s --timeout=5s --start-period=10s \ - CMD curl -f http://127.0.0.1:10213/ CMD spark-submit \ --master "local[*]" \ --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 \ diff --git a/testing/spark3-iceberg/Dockerfile b/testing/spark3-iceberg/Dockerfile index 2f6436d0..86250206 100644 --- a/testing/spark3-iceberg/Dockerfile +++ b/testing/spark3-iceberg/Dockerfile @@ -43,8 +43,6 @@ ENV PATH="${SPARK_HOME}/bin:${PATH}" # add hive user needed in interactions with the Apache Hive environment RUN useradd -ms /bin/bash hive -HEALTHCHECK --interval=10s --timeout=5s --start-period=10s \ - CMD curl -f http://127.0.0.1:10213/ CMD spark-submit \ --master "local[*]" \ --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 \