From ecc1b3e58f966c87149d8db68dc63031c37aac6d Mon Sep 17 00:00:00 2001 From: Pankaj Koti Date: Wed, 13 Nov 2024 17:36:52 +0530 Subject: [PATCH 1/5] Test k8s tests --- cosmos/operators/airflow_async.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cosmos/operators/airflow_async.py b/cosmos/operators/airflow_async.py index a7f30a3308..4ea39f34bf 100644 --- a/cosmos/operators/airflow_async.py +++ b/cosmos/operators/airflow_async.py @@ -59,6 +59,7 @@ class DbtSourceAirflowAsyncOperator(DbtBaseAirflowAsyncOperator, DbtSourceLocalO pass +# Test class DbtRunAirflowAsyncOperator(BigQueryInsertJobOperator): # type: ignore template_fields: Sequence[str] = ( From 32cff3afef99fef72f66e6103fd47f6e00b6c97f Mon Sep 17 00:00:00 2001 From: Pankaj Koti Date: Wed, 13 Nov 2024 17:54:32 +0530 Subject: [PATCH 2/5] Add status check for Postgres pod --- scripts/test/kubernetes-setup.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh index 90021f5448..1c1576f94b 100644 --- a/scripts/test/kubernetes-setup.sh +++ b/scripts/test/kubernetes-setup.sh @@ -26,6 +26,13 @@ kind load docker-image dbt-jaffle-shop:1.0.0 # The output is filtered to get the first pod's name POD_NAME=$(kubectl get pods -n default -l app=postgres -o jsonpath='{.items[0].metadata.name}') +# Wait for the PostgreSQL pod to be in the 'Running' and 'Ready' state +echo "Waiting for PostgreSQL pod to be ready..." +while [ "$(kubectl get pod "$POD_NAME" -n default -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" != "True" ]; do + echo "Pod $POD_NAME is not ready yet. Waiting..." + sleep 5 +done + # Print the name of the PostgreSQL pod echo "$POD_NAME" From f6d2f7a372a58dc18e5f09d89a76e5f5d8101fdc Mon Sep 17 00:00:00 2001 From: Pankaj Koti Date: Wed, 13 Nov 2024 18:11:40 +0530 Subject: [PATCH 3/5] Check for error status of pod & exit early --- scripts/test/kubernetes-setup.sh | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh index 1c1576f94b..9360a8e469 100644 --- a/scripts/test/kubernetes-setup.sh +++ b/scripts/test/kubernetes-setup.sh @@ -28,11 +28,25 @@ POD_NAME=$(kubectl get pods -n default -l app=postgres -o jsonpath='{.items[0].m # Wait for the PostgreSQL pod to be in the 'Running' and 'Ready' state echo "Waiting for PostgreSQL pod to be ready..." -while [ "$(kubectl get pod "$POD_NAME" -n default -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" != "True" ]; do - echo "Pod $POD_NAME is not ready yet. Waiting..." - sleep 5 +while true; do + POD_STATUS=$(kubectl get pod "$POD_NAME" -n default -o jsonpath='{.status.phase}') + + if [ "$POD_STATUS" = "Running" ] && [ "$(kubectl get pod "$POD_NAME" -n default -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" = "True" ]; then + echo "PostgreSQL pod is up and running!" + break + elif [ "$POD_STATUS" = "Error" ]; then + echo "Error: PostgreSQL pod failed to start. Exiting..." + kubectl describe pod "$POD_NAME" -n default # Show details for debugging + exit 1 + else + echo "Pod $POD_NAME is not ready yet (status: $POD_STATUS). Waiting..." + sleep 5 + fi done - +#while [ "$(kubectl get pod "$POD_NAME" -n default -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" != "True" ]; do +# echo "Pod $POD_NAME is not ready yet. Waiting..." +# sleep 5 +#done # Print the name of the PostgreSQL pod echo "$POD_NAME" From 3d859323a34f52bef91b044a2263b1cb2d0c76cf Mon Sep 17 00:00:00 2001 From: Pankaj Koti Date: Wed, 13 Nov 2024 18:37:27 +0530 Subject: [PATCH 4/5] Update cosmos/operators/airflow_async.py --- cosmos/operators/airflow_async.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cosmos/operators/airflow_async.py b/cosmos/operators/airflow_async.py index 4ea39f34bf..a7f30a3308 100644 --- a/cosmos/operators/airflow_async.py +++ b/cosmos/operators/airflow_async.py @@ -59,7 +59,6 @@ class DbtSourceAirflowAsyncOperator(DbtBaseAirflowAsyncOperator, DbtSourceLocalO pass -# Test class DbtRunAirflowAsyncOperator(BigQueryInsertJobOperator): # type: ignore template_fields: Sequence[str] = ( From a54df5f6c76368cfd8323e2a4145f3ab33323d3b Mon Sep 17 00:00:00 2001 From: Pankaj Koti Date: Wed, 13 Nov 2024 18:37:48 +0530 Subject: [PATCH 5/5] Update scripts/test/kubernetes-setup.sh --- scripts/test/kubernetes-setup.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh index 9360a8e469..fb2324c047 100644 --- a/scripts/test/kubernetes-setup.sh +++ b/scripts/test/kubernetes-setup.sh @@ -43,10 +43,6 @@ while true; do sleep 5 fi done -#while [ "$(kubectl get pod "$POD_NAME" -n default -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}')" != "True" ]; do -# echo "Pod $POD_NAME is not ready yet. Waiting..." -# sleep 5 -#done # Print the name of the PostgreSQL pod echo "$POD_NAME"