Merge pull request #604 from microsoft/psl-dk-e2epipeline

fix: added e2e testing in validate non waf pipeline and fixed WAF deployment pipeline

Author: Roopan-Microsoft

.github/workflows/deploy-waf.yml

@@ -10,6 +10,10 @@ on:
 jobs:
   deploy:
     runs-on: ubuntu-latest
+    env:
+      GPT_MIN_CAPACITY: 1
+      O4_MINI_MIN_CAPACITY: 1
+      GPT41_MINI_MIN_CAPACITY: 1
     steps:
       - name: Checkout Code
         uses: actions/checkout@v3
@@ -21,7 +25,9 @@ jobs:
           export AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }}
           export AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }}
           export AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}"
-          export GPT_MIN_CAPACITY="150"
+          export GPT_MIN_CAPACITY="1"
+          export O4_MINI_MIN_CAPACITY="1"
+          export GPT41_MINI_MIN_CAPACITY="1"
           export AZURE_REGIONS="${{ vars.AZURE_REGIONS }}"
 
           chmod +x infra/scripts/checkquota.sh
@@ -116,7 +122,9 @@ jobs:
               solutionName=${{ env.SOLUTION_PREFIX }} \
               location="${{ env.AZURE_LOCATION }}" \
               azureAiServiceLocation='${{ env.AZURE_LOCATION }}' \
-              gptModelCapacity=5 \
+              gptModelCapacity=${{ env.GPT_MIN_CAPACITY }} \
+              gpt4_1ModelCapacity=${{ env.GPT41_MINI_MIN_CAPACITY }} \
+              gptReasoningModelCapacity=${{ env.O4_MINI_MIN_CAPACITY }} \
               enableTelemetry=true \
               enableMonitoring=true \
               enablePrivateNetworking=true \

.github/workflows/deploy.yml

@@ -13,7 +13,9 @@ on:
     - cron: "0 11,23 * * *" # Runs at 11:00 AM and 11:00 PM GMT
   workflow_dispatch: #Allow manual triggering
 env:
-  GPT_MIN_CAPACITY: 1
+  GPT_MIN_CAPACITY: 150
+  O4_MINI_MIN_CAPACITY: 50
+  GPT41_MINI_MIN_CAPACITY: 50
   BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
 
 jobs:
@@ -36,7 +38,9 @@ jobs:
           export AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }}
           export AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }}
           export AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}"
-          export GPT_MIN_CAPACITY="1"
+          export GPT_MIN_CAPACITY="150"
+          export O4_MINI_MIN_CAPACITY="50"
+          export GPT41_MINI_MIN_CAPACITY="50"
           export AZURE_REGIONS="${{ vars.AZURE_REGIONS }}"
 
           chmod +x infra/scripts/checkquota.sh
@@ -122,7 +126,7 @@ jobs:
           elif [[ "${{ env.BRANCH_NAME }}" == "hotfix" ]]; then
             IMAGE_TAG="hotfix"
           else
-            IMAGE_TAG="latest"
+            IMAGE_TAG="latest_v3"
           fi
 
           # Generate current timestamp in desired format: YYYY-MM-DDTHH:MM:SS.SSSSSSSZ
@@ -135,12 +139,12 @@ jobs:
               solutionName=${{ env.SOLUTION_PREFIX }} \
               location="${{ env.AZURE_LOCATION }}" \
               gptModelDeploymentType="GlobalStandard" \
-              gptModelName="gpt-4o" \
-              gptModelVersion="2024-08-06" \
+              gptModelName="gpt-4.1-mini" \
+              gptModelVersion="2025-04-14" \
               backendContainerImageTag="${IMAGE_TAG}" \
               frontendContainerImageTag="${IMAGE_TAG}" \
               azureAiServiceLocation='${{ env.AZURE_LOCATION }}' \
-              gptModelCapacity=1 \
+              gptModelCapacity=50 \
               createdBy="Pipeline" \
               tags="{'SecurityControl':'Ignore','Purpose':'Deploying and Cleaning Up Resources for Validation','CreatedDate':'$current_date'}" \
             --output json
@@ -159,9 +163,10 @@ jobs:
       - name: Get Container App Backend URL
         id: get_backend_url
         run: |
+          # Get specifically the backend container app (not the MCP container app)
           CONTAINER_APP_NAME=$(az containerapp list \
             --resource-group ${{ env.RESOURCE_GROUP_NAME }} \
-            --query "[0].name" -o tsv)
+            --query "[?starts_with(name, 'ca-') && !contains(name, 'mcp')].name" -o tsv)
 
           MACAE_URL_API=$(az containerapp show \
             --name "$CONTAINER_APP_NAME" \
@@ -171,6 +176,38 @@ jobs:
           echo "MACAE_URL_API=https://${MACAE_URL_API}" >> $GITHUB_OUTPUT
           echo "CONTAINER_APP=${CONTAINER_APP_NAME}" >> $GITHUB_OUTPUT
 
+      - name: Run Post deployment scripts
+        run: |
+          set -e
+          az account set --subscription "${{ secrets.AZURE_SUBSCRIPTION_ID }}"
+
+          echo "Running post-deployment script..."
+          
+          # Extract required resource names from the deployment
+          STORAGE_ACCOUNT=$(az resource list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --resource-type "Microsoft.Storage/storageAccounts" --query "[0].name" -o tsv)
+          AI_SEARCH=$(az resource list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --resource-type "Microsoft.Search/searchServices" --query "[0].name" -o tsv)
+          
+          echo "Found Storage Account: $STORAGE_ACCOUNT"
+          echo "Found AI Search Service: $AI_SEARCH"
+          echo "Backend URL: ${{ steps.get_backend_url.outputs.MACAE_URL_API }}"
+          
+          # Run upload team config script with parameters
+          bash infra/scripts/upload_team_config.sh \
+            "${{ steps.get_backend_url.outputs.MACAE_URL_API }}" \
+            "data/agent_teams" \
+            "${{ secrets.AZURE_SUBSCRIPTION_ID }}"
+          
+          # Run process sample data script with parameters
+          bash infra/scripts/process_sample_data.sh \
+            "$STORAGE_ACCOUNT" \
+            "sample-dataset" \
+            "$AI_SEARCH" \
+            "sample-dataset-index" \
+            "${{ env.RESOURCE_GROUP_NAME }}" \
+            "${{ secrets.AZURE_SUBSCRIPTION_ID }}"
+
+          echo "=== Post-Deployment Script Completed Successfully ==="
+
       - name: Set Deployment Status
         id: deployment_status
         if: always()
@@ -181,9 +218,20 @@ jobs:
             echo "SUCCESS=false" >> $GITHUB_OUTPUT
           fi
 
+  e2e-test:
+    needs: deploy
+    if: needs.deploy.outputs.DEPLOYMENT_SUCCESS == 'true'
+    uses: ./.github/workflows/test-automation.yml
+    with:
+      MACAE_WEB_URL: ${{ needs.deploy.outputs.WEBAPP_URL }}
+      MACAE_URL_API: ${{ needs.deploy.outputs.MACAE_URL_API }}
+      MACAE_RG: ${{ needs.deploy.outputs.RESOURCE_GROUP_NAME }}
+      MACAE_CONTAINER_APP: ${{ needs.deploy.outputs.CONTAINER_APP }}
+    secrets: inherit
+
   cleanup-deployment:
     if: always() && needs.deploy.outputs.RESOURCE_GROUP_NAME != ''
-    needs: [deploy]
+    needs: [deploy, e2e-test]
     runs-on: ubuntu-latest
     env:
       RESOURCE_GROUP_NAME: ${{ needs.deploy.outputs.RESOURCE_GROUP_NAME }}
@@ -317,7 +365,7 @@ jobs:
           echo "Resource purging completed successfully"
 
       - name: Send Notification on Failure
-        if: failure()
+        
         run: |
           RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
 

.github/workflows/test-automation.yml

@@ -31,7 +31,7 @@ jobs:
       MACAE_URL_API: ${{ inputs.MACAE_URL_API }}
       MACAE_RG: ${{ inputs.MACAE_RG }}
       MACAE_CONTAINER_APP: ${{ inputs.MACAE_CONTAINER_APP }}
-      accelerator_name: "MACAE"
+      accelerator_name: "MACAE v3"
 
     steps:
       - name: Checkout repository

infra/scripts/checkquota.sh

@@ -5,6 +5,8 @@ IFS=', ' read -ra REGIONS <<< "$AZURE_REGIONS"
 
 SUBSCRIPTION_ID="${AZURE_SUBSCRIPTION_ID}"
 GPT_MIN_CAPACITY="${GPT_MIN_CAPACITY}"
+O4_MINI_MIN_CAPACITY="${O4_MINI_MIN_CAPACITY}"
+GPT41_MINI_MIN_CAPACITY="${GPT41_MINI_MIN_CAPACITY}"
 AZURE_CLIENT_ID="${AZURE_CLIENT_ID}"
 AZURE_TENANT_ID="${AZURE_TENANT_ID}"
 AZURE_CLIENT_SECRET="${AZURE_CLIENT_SECRET}"
@@ -17,8 +19,10 @@ if ! az login --service-principal -u "$AZURE_CLIENT_ID" -p "$AZURE_CLIENT_SECRET
 fi
 
 echo "🔄 Validating required environment variables..."
-if [[ -z "$SUBSCRIPTION_ID" || -z "$GPT_MIN_CAPACITY" || -z "$REGIONS" ]]; then
+if [[ -z "$SUBSCRIPTION_ID" || -z "$REGIONS" ]]; then
     echo "❌ ERROR: Missing required environment variables."
+    echo "Required: AZURE_SUBSCRIPTION_ID, AZURE_REGIONS"
+    echo "Optional: O4_MINI_MIN_CAPACITY (default: 50), GPT41_MINI_MIN_CAPACITY (default: 50)"
     exit 1
 fi
 
@@ -31,7 +35,9 @@ echo "✅ Azure subscription set successfully."
 
 # Define models and their minimum required capacities
 declare -A MIN_CAPACITY=(
-    ["OpenAI.GlobalStandard.gpt-4o"]=$GPT_MIN_CAPACITY
+    ["OpenAI.GlobalStandard.o4-mini"]="${O4_MINI_MIN_CAPACITY}"
+    ["OpenAI.GlobalStandard.gpt4.1"]="${GPT_MIN_CAPACITY}"
+    ["OpenAI.GlobalStandard.gpt4.1-mini"]="${GPT41_MINI_MIN_CAPACITY}"
 )
 
 VALID_REGION=""

infra/scripts/upload_team_config.py

@@ -39,7 +39,7 @@ def check_team_exists(backend_url, team_id, user_principal_id):
 
 backend_url = sys.argv[1]
 directory_path = sys.argv[2]
-user_principal_id = sys.argv[3] if len(sys.argv) > 3 else "00000000-0000-0000-0000-000000000000"
+user_principal_id = sys.argv[3] if len(sys.argv) > 3 and sys.argv[3].strip() != "" else "00000000-0000-0000-0000-000000000000"
 
 # Convert to absolute path if provided as relative
 directory_path = os.path.abspath(directory_path)
@@ -80,6 +80,7 @@ def check_team_exists(backend_url, team_id, user_principal_id):
                 headers = {
                     'x-ms-client-principal-id': user_principal_id
                 }
+    
                 params = {
                     'team_id': team_id
                 }