[COST-5551] utilize AWS Glue Data Catalog for PR smoke tests (#5461)

project-koku · Jan 28, 2025 · f87e8b3 · f87e8b3
1 parent 162a221
commit f87e8b3
Show file tree

Hide file tree

Showing 5 changed files with 15 additions and 4 deletions.
diff --git a/.env.example b/.env.example
@@ -42,6 +42,8 @@ AWS_RESOURCE_NAME=YOUR_COST_MANAGEMENT_AWS_ARN
 SCHEMA_SUFFIX="" # if DEVELOPMENT=True, this can be left empty and will default to $USER; otherwise, set this value to something unique
 TRINO_SCHEMA_PREFIX=""
 TRINO_S3A_OR_S3=s3
+HIVE_PROPERTIES_FILE=hive.properties
+GLUE_PROPERTIES_FILE=glue.properties
 
 AWS_CATALOG_ID=589173575009
 MINIO_ENDPOINT=http://koku-minio:9000

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -42,7 +42,7 @@ pipeline {
 
         IQE_PLUGINS="cost_management"
         BUILD_URL="https://ci.ext.devshift.net/job/koku-pipeline-pr-check-main/${BUILD_NUMBER}/"
-        IQE_ENV_VARS="JOB_NAME=koku-ci-jenkins,BUILD_NUMBER=${BUILD_NUMBER},BUILD_URL=${BUILD_URL}"
+        IQE_ENV_VARS="JOB_NAME=koku-ci-jenkins,BUILD_NUMBER=${BUILD_NUMBER},BUILD_URL=${BUILD_URL},SCHEMA_SUFFIX=_${IMAGE_TAG}"
         IQE_PARALLEL_ENABLED="false"
 
         GITHUB_API_ROOT='https://api.github.com/repos/project-koku/koku'

diff --git a/ci/functions.sh b/ci/functions.sh
@@ -101,7 +101,7 @@ function run_smoke_tests_stage() {
     bonfire deploy \
         ${APP_NAME} \
         --ref-env insights-production \
-        --set-template-ref ${APP_NAME}/${COMPONENT_NAME}=${ghprbActualCommit} \
+        --set-template-ref ${COMPONENT_NAME}=${ghprbActualCommit} \
         --set-image-tag ${IMAGE}=${IMAGE_TAG} \
         --namespace ${NAMESPACE} \
         ${COMPONENTS_ARG} \
@@ -112,6 +112,10 @@ function run_smoke_tests_stage() {
         --set-parameter koku/DBM_IMAGE_TAG=${IMAGE_TAG} \
         --set-parameter koku/DBM_INVOCATION=${DBM_INVOCATION} \
         --set-parameter koku/IMAGE=${IMAGE} \
+        --set-parameter koku/SCHEMA_SUFFIX=_${IMAGE_TAG} \
+        --set-parameter koku/TRINO_S3A_OR_S3=s3 \
+        --set-parameter trino/HIVE_PROPERTIES_FILE=glue.properties \
+        --set-parameter trino/GLUE_PROPERTIES_FILE=hive.properties \
         --no-single-replicas \
         --source=appsre \
         --timeout 600

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -681,7 +681,8 @@ services:
       - ./dev/containers/trino/etc/config.properties:/etc/trino/config.properties
       - ./dev/containers/trino/etc/jvm.config:/etc/trino/jvm.config
       - ./dev/containers/trino/etc/log.properties:/etc/trino/log.properties
-      - ./dev/containers/trino/etc/catalog/hive.properties:/etc/trino/catalog/hive.properties
+      - ./dev/containers/trino/etc/catalog/hive.properties:/etc/trino/catalog/${HIVE_PROPERTIES_FILE-hive.properties}
+      - ./dev/containers/trino/etc/catalog/glue.properties:/etc/trino/catalog/${GLUE_PROPERTIES_FILE-glue.properties}
       - ./dev/containers/trino/etc/catalog/postgres.properties:/etc/trino/catalog/postgres.properties
       - ./dev/containers/trino/data:/data/trino/data
       - ./dev/containers/trino/logs:/data/trino/logs

diff --git a/smoke_test.sh b/smoke_test.sh
@@ -10,7 +10,7 @@ IQE_MARKER_EXPRESSION="cost_smoke"
 IQE_FILTER_EXPRESSION=""
 IQE_CJI_TIMEOUT="5h"
 IQE_PARALLEL_ENABLED="false"
-IQE_ENV_VARS="JOB_NAME=${JOB_NAME},BUILD_NUMBER=${BUILD_NUMBER}"
+IQE_ENV_VARS="JOB_NAME=${JOB_NAME},BUILD_NUMBER=${BUILD_NUMBER},SCHEMA_SUFFIX=_${IMAGE_TAG}"
 
 # Get bonfire helper scripts
 CICD_URL="https://raw.githubusercontent.com/RedHatInsights/cicd-tools/main"
@@ -42,6 +42,10 @@ bonfire deploy \
     --optional-deps-method hybrid \
     --no-single-replicas \
     --set-parameter rbac/MIN_REPLICAS=1 \
+    --set-parameter koku/SCHEMA_SUFFIX=_${IMAGE_TAG} \
+    --set-parameter koku/TRINO_S3A_OR_S3=s3 \
+    --set-parameter trino/HIVE_PROPERTIES_FILE=glue.properties \
+    --set-parameter trino/GLUE_PROPERTIES_FILE=hive.properties \
     ${COMPONENTS_ARG} \
     ${COMPONENTS_RESOURCES_ARG} \
     ${EXTRA_DEPLOY_ARGS}