Merge pull request #21 from cassiebreviu/main

Added Support and Intent workflows, added gh action for support and chat flows, change runtime to automatic in notebooks, update req versions

.github/workflows/deploy-chat-pf-online-endpoint-pipeline.yml

@@ -0,0 +1,71 @@
+name: Deploy Contoso-Chat Promptflow
+
+on:
+  workflow_dispatch:
+  workflow_run:
+    workflows: ["run-chat-eval-pf-pipeline"]
+    branches: [main]
+    types:
+      - completed
+
+
+env: 
+  GROUP: ${{secrets.GROUP}}
+  WORKSPACE: ${{secrets.WORKSPACE}}
+  SUBSCRIPTION: ${{secrets.SUBSCRIPTION}}
+  RUN_NAME: contoso_chat
+  EVAL_RUN_NAME: contoso_chat_eval
+  ENDPOINT_NAME: contoso_chat
+  DEPLOYMENT_NAME: blue
+  KEY_VAULT_NAME: ${{secrets.KEY_VAULT_NAME}}
+
+jobs:
+  create-endpoint-and-deploy-pf:
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} 
+    steps:
+    - name: Check out repo
+      uses: actions/checkout@v2
+    - name: Install az ml extension
+      run: az extension add -n ml -y
+    - name: Azure login
+      uses: azure/login@v1
+      with:
+        creds: ${{secrets.AZURE_CREDENTIALS}}
+    - name: List current directory
+      run: ls
+    - name: Set default subscription
+      run: |
+         az account set -s ${{env.SUBSCRIPTION}}
+    - name: Create Hash
+      run: echo "HASH=$(echo -n $RANDOM | sha1sum | cut -c 1-6)" >> "$GITHUB_ENV"
+    - name: Create unique endpoint name   
+      run: echo "ENDPOINT_NAME=$(echo 'contoso-chat-'$HASH)" >> "$GITHUB_ENV"
+    - name: Display endpoint name
+      run: echo "Endpoint name is:" ${{env.ENDPOINT_NAME}}
+    - name: Setup endpoint
+      run: az ml online-endpoint create --file deployment/chat-endpoint.yaml  --name ${{env.ENDPOINT_NAME}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}}
+    - name: Update deployment PRT_CONFIG variable
+      run: |
+        PRT_CONFIG_OVERRIDE=deployment.subscription_id=${{ env.SUBSCRIPTION }},deployment.resource_group=${{ env.GROUP }},deployment.workspace_name=${{ env.WORKSPACE }},deployment.endpoint_name=${{ env.ENDPOINT_NAME }},deployment.deployment_name=${{ env.DEPLOYMENT_NAME }}
+        sed -i "s/PRT_CONFIG_OVERRIDE:.*/PRT_CONFIG_OVERRIDE: $PRT_CONFIG_OVERRIDE/g" deployment/chat-deployment.yaml
+    - name: Setup deployment
+      run: az ml online-deployment create --file deployment/chat-deployment.yaml --endpoint-name ${{env.ENDPOINT_NAME}} --all-traffic -g ${{env.GROUP}} -w ${{env.WORKSPACE}}
+    - name: Check the status of the endpoint
+      run: az ml online-endpoint show -n ${{env.ENDPOINT_NAME}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}}
+    - name: Check the status of the deployment
+      run: az ml online-deployment get-logs --name contoso-chat --endpoint-name ${{env.ENDPOINT_NAME}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}}
+    - name: Read endpoint principal
+      run: |
+        az ml online-endpoint show -n ${{env.ENDPOINT_NAME}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}} > endpoint.json
+        jq -r '.identity.principal_id' endpoint.json > principal.txt
+        echo "Principal is: $(cat principal.txt)"
+    - name: Assign Permission to Endpoint Principal
+      run: |
+          echo "assigning permissions to Principal to AzureML workspace.."
+          az role assignment create --assignee $(cat principal.txt) --role "AzureML Data Scientist" --scope "/subscriptions/${{ env.SUBSCRIPTION }}/resourcegroups/${{env.GROUP}}/providers/Microsoft.MachineLearningServices/workspaces/${{env.WORKSPACE}}"
+          az role assignment create --assignee $(cat principal.txt) --role "Azure Machine Learning Workspace Connection Secrets Reader" --scope "/subscriptions/${{ env.SUBSCRIPTION }}/resourcegroups/${{env.GROUP}}/providers/Microsoft.MachineLearningServices/workspaces/${{env.WORKSPACE}}/onlineEndpoints/${{env.ENDPOINT_NAME}}"
+          
+          echo "assigning permissions to Principal to Key vault.."
+          az keyvault set-policy --name ${{secrets.KEY_VAULT_NAME}} --resource-group ${{env.GROUP}} --object-id $(cat principal.txt) --secret-permissions get list
+    

.github/workflows/deploy-support-pf-online-endpoint-pipeline.yml

@@ -0,0 +1,71 @@
+name: Deploy Contoso-Support Promptflow
+
+on:
+  workflow_dispatch:
+  workflow_run:
+    workflows: ["run-support-eval-pf-pipeline"]
+    branches: [main]
+    types:
+      - completed
+
+
+env: 
+  GROUP: ${{secrets.GROUP}}
+  WORKSPACE: ${{secrets.WORKSPACE}}
+  SUBSCRIPTION: ${{secrets.SUBSCRIPTION}}
+  RUN_NAME: contoso_support
+  EVAL_RUN_NAME: contoso_support_eval
+  ENDPOINT_NAME: contoso_support
+  DEPLOYMENT_NAME: blue
+  KEY_VAULT_NAME: ${{secrets.KEY_VAULT_NAME}}
+
+jobs:
+  create-endpoint-and-deploy-pf:
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} 
+    steps:
+    - name: Check out repo
+      uses: actions/checkout@v2
+    - name: Install az ml extension
+      run: az extension add -n ml -y
+    - name: Azure login
+      uses: azure/login@v1
+      with:
+        creds: ${{secrets.AZURE_CREDENTIALS}}
+    - name: List current directory
+      run: ls
+    - name: Set default subscription
+      run: |
+         az account set -s ${{env.SUBSCRIPTION}}
+    - name: Create Hash
+      run: echo "HASH=$(echo -n $RANDOM | sha1sum | cut -c 1-6)" >> "$GITHUB_ENV"
+    - name: Create unique endpoint name   
+      run: echo "ENDPOINT_NAME=$(echo 'contoso-support-'$HASH)" >> "$GITHUB_ENV"
+    - name: Display endpoint name
+      run: echo "Endpoint name is:" ${{env.ENDPOINT_NAME}}
+    - name: Setup endpoint
+      run: az ml online-endpoint create --file deployment/support-endpoint.yaml  --name ${{env.ENDPOINT_NAME}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}}
+    - name: Update deployment PRT_CONFIG variable
+      run: |
+        PRT_CONFIG_OVERRIDE=deployment.subscription_id=${{ env.SUBSCRIPTION }},deployment.resource_group=${{ env.GROUP }},deployment.workspace_name=${{ env.WORKSPACE }},deployment.endpoint_name=${{ env.ENDPOINT_NAME }},deployment.deployment_name=${{ env.DEPLOYMENT_NAME }}
+        sed -i "s/PRT_CONFIG_OVERRIDE:.*/PRT_CONFIG_OVERRIDE: $PRT_CONFIG_OVERRIDE/g" deployment/support-deployment.yaml
+    - name: Setup deployment
+      run: az ml online-deployment create --file deployment/support-deployment.yaml --endpoint-name ${{env.ENDPOINT_NAME}} --all-traffic -g ${{env.GROUP}} -w ${{env.WORKSPACE}}
+    - name: Check the status of the endpoint
+      run: az ml online-endpoint show -n ${{env.ENDPOINT_NAME}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}}
+    - name: Check the status of the deployment
+      run: az ml online-deployment get-logs --name contoso-support --endpoint-name ${{env.ENDPOINT_NAME}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}}
+    - name: Read endpoint principal
+      run: |
+        az ml online-endpoint show -n ${{env.ENDPOINT_NAME}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}} > endpoint.json
+        jq -r '.identity.principal_id' endpoint.json > principal.txt
+        echo "Principal is: $(cat principal.txt)"
+    - name: Assign Permission to Endpoint Principal
+      run: |
+          echo "assigning permissions to Principal to AzureML workspace.."
+          az role assignment create --assignee $(cat principal.txt) --role "AzureML Data Scientist" --scope "/subscriptions/${{ env.SUBSCRIPTION }}/resourcegroups/${{env.GROUP}}/providers/Microsoft.MachineLearningServices/workspaces/${{env.WORKSPACE}}"
+          az role assignment create --assignee $(cat principal.txt) --role "Azure Machine Learning Workspace Connection Secrets Reader" --scope "/subscriptions/${{ env.SUBSCRIPTION }}/resourcegroups/${{env.GROUP}}/providers/Microsoft.MachineLearningServices/workspaces/${{env.WORKSPACE}}/onlineEndpoints/${{env.ENDPOINT_NAME}}"
+          
+          echo "assigning permissions to Principal to Key vault.."
+          az keyvault set-policy --name ${{secrets.KEY_VAULT_NAME}} --resource-group ${{env.GROUP}} --object-id $(cat principal.txt) --secret-permissions get list
+    

.github/workflows/run-chat-eval-pf-pipeline.yml

@@ -0,0 +1,84 @@
+name: Test and Evaulate Contoso-Chat with Promptflow
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ main ]
+
+env: 
+  GROUP: ${{secrets.GROUP}}
+  WORKSPACE: ${{secrets.WORKSPACE}}
+  SUBSCRIPTION: ${{secrets.SUBSCRIPTION}}
+  RUN_NAME: contoso_chat
+  EVAL_RUN_NAME: contoso_chat_eval
+
+jobs:
+  login-runpf-evalpf-assertpf-registermodel:
+    runs-on: ubuntu-latest 
+    steps:
+    - name: Check out repo
+      uses: actions/checkout@v2
+    - name: Install az ml extension
+      run: az extension add -n ml -y
+    - name: Azure login
+      uses: azure/login@v1
+      with:
+        creds: ${{secrets.AZURE_CREDENTIALS}}
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.11.4'
+    - name: List current directory
+      run: ls
+    - name: Install promptflow
+      run: pip install -r ../contoso-chat/requirements.txt
+    - name: Install promptflow again
+      run: pip install -r ../contoso-chat/requirements.txt
+    - name: Run promptflow
+      run: |
+        pfazure run create -f contoso-chat/run.yml --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}} --stream > run_info.txt
+        cat run_info.txt
+    - name: List current directory
+      run: ls
+    - name: Set run name
+      run: |
+        echo "RUN_NAME=$(python deployment/llmops-helper/parse_run_output.py run_info.txt)" >> "$GITHUB_ENV"
+    - name: Show the current run name
+      run: echo "Run name is:" ${{env.RUN_NAME}}
+    - name: Show promptflow results
+      run: pfazure run show-details --name ${{env.RUN_NAME}} --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}}
+    - name: Run promptflow evaluations
+      run: |
+        pfazure run create -f contoso-chat/run_evaluation_multi.yml --run ${{env.RUN_NAME}} --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}} --stream > eval_info.txt 
+        cat eval_info.txt
+    - name: Get eval run name
+      run: echo "EVAL_RUN_NAME=$(python deployment/llmops-helper/parse_run_output.py eval_info.txt)" >> "$GITHUB_ENV"
+    - name: Show the current eval run name
+      run: echo "Eval run name is:" ${{env.EVAL_RUN_NAME}}
+    - name: Show promptflow details
+      run: pfazure run show-details --name ${{env.EVAL_RUN_NAME}} --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}}
+    - name: Show promptflow metrics
+      run: |
+        pfazure run show-metrics --name ${{env.EVAL_RUN_NAME}} --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}} > eval_result.json 
+        cat eval_result.json
+    - name: List current directory
+      run: ls
+    - name: Get assert eval results
+      id: jobMetricAssert
+      run: |
+            # NOTE The number after the file is the threshold score to pass the assertion.
+            export ASSERT=$(python deployment/llmops-helper/assert.py eval_result.json 3) # NOTE <file>.json is the file name and decimal is the threshold for the assertion
+            echo "::debug::Assert has returned the following value: $ASSERT"
+            # assert.py will return True or False, but bash expects lowercase.
+            if ${ASSERT,,} ; then
+              echo "::debug::Prompt flow run met the quality bar and can be deployed."
+              echo "::set-output name=result::true"
+            else
+              echo "::warning::Prompt flow run didn't meet quality bar."
+              echo "::set-output name=result::false"
+            fi
+    - name: Show the assert result
+      run: echo "Assert result is:" ${{ steps.jobMetricAssert.outputs.result }}
+    - name: Register promptflow model
+      if: ${{ steps.jobMetricAssert.outputs.result == 'true' }}
+      run: az ml model create --file deployment/chat-model.yaml  -g ${{env.GROUP}} -w ${{env.WORKSPACE}}

.github/workflows/run-support-eval-pf-pipeline.yml

@@ -0,0 +1,84 @@
+name: Test and Evaulate Contoso-Support Promptflow
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ main ]
+
+env: 
+  GROUP: ${{secrets.GROUP}}
+  WORKSPACE: ${{secrets.WORKSPACE}}
+  SUBSCRIPTION: ${{secrets.SUBSCRIPTION}}
+  RUN_NAME: contoso_support
+  EVAL_RUN_NAME: contoso_support_eval
+
+jobs:
+  login-runpf-evalpf-assertpf-registermodel:
+    runs-on: ubuntu-latest 
+    steps:
+    - name: Check out repo
+      uses: actions/checkout@v2
+    - name: Install az ml extension
+      run: az extension add -n ml -y
+    - name: Azure login
+      uses: azure/login@v1
+      with:
+        creds: ${{secrets.AZURE_CREDENTIALS}}
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.11.4'
+    - name: List current directory
+      run: ls
+    - name: Install promptflow
+      run: pip install -r contoso-support/requirements.txt
+    - name: Install promptflow again
+      run: pip install -r contoso-support/requirements.txt
+    - name: Run promptflow
+      run: |
+        pfazure run create -f contoso-support/run.yml --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}} --stream > run_info.txt
+        cat run_info.txt
+    - name: List current directory
+      run: ls
+    - name: Set run name
+      run: |
+        echo "RUN_NAME=$(python deployment/llmops-helper/parse_run_output.py run_info.txt)" >> "$GITHUB_ENV"
+    - name: Show the current run name
+      run: echo "Run name is:" ${{env.RUN_NAME}}
+    - name: Show promptflow results
+      run: pfazure run show-details --name ${{env.RUN_NAME}} --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}}
+    - name: Run promptflow evaluations
+      run: |
+        pfazure run create -f contoso-support/run_evaluation_multi.yml --run ${{env.RUN_NAME}} --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}} --stream > eval_info.txt 
+        cat eval_info.txt
+    - name: Get eval run name
+      run: echo "EVAL_RUN_NAME=$(python deployment/llmops-helper/parse_run_output.py eval_info.txt)" >> "$GITHUB_ENV"
+    - name: Show the current eval run name
+      run: echo "Eval run name is:" ${{env.EVAL_RUN_NAME}}
+    - name: Show promptflow details
+      run: pfazure run show-details --name ${{env.EVAL_RUN_NAME}} --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}}
+    - name: Show promptflow metrics
+      run: |
+        pfazure run show-metrics --name ${{env.EVAL_RUN_NAME}} --subscription ${{env.SUBSCRIPTION}} -g ${{env.GROUP}} -w ${{env.WORKSPACE}} > eval_result.json 
+        cat eval_result.json
+    - name: List current directory
+      run: ls
+    - name: Get assert eval results
+      id: jobMetricAssert
+      run: |
+            # NOTE The number after the file is the threshold score to pass the assertion.
+            export ASSERT=$(python deployment/llmops-helper/assert.py eval_result.json 3) # NOTE <file>.json is the file name and decimal is the threshold for the assertion
+            echo "::debug::Assert has returned the following value: $ASSERT"
+            # assert.py will return True or False, but bash expects lowercase.
+            if ${ASSERT,,} ; then
+              echo "::debug::Prompt flow run met the quality bar and can be deployed."
+              echo "::set-output name=result::true"
+            else
+              echo "::warning::Prompt flow run didn't meet quality bar."
+              echo "::set-output name=result::false"
+            fi
+    - name: Show the assert result
+      run: echo "Assert result is:" ${{ steps.jobMetricAssert.outputs.result }}
+    - name: Register promptflow model
+      if: ${{ steps.jobMetricAssert.outputs.result == 'true' }}
+      run: az ml model create --file deployment/support-model.yaml  -g ${{env.GROUP}} -w ${{env.WORKSPACE}}