Add live tests to release, main and new live-test workflows

.github/workflows/live-test.yml

@@ -0,0 +1,42 @@
+# This workflow is triggered by the user and runs live tests on the codebase.
+name: Live Test
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types:
+      - labeled
+
+jobs:
+  test:
+    name: Live Test
+    runs-on: ubuntu-latest
+    if: github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'live test')
+    environment: Live Testing
+    env:
+      version_suffix_args: ${{ format('--version-suffix="alpha.{0}"', github.run_number) }}
+    steps:
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: '8.x'
+
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Run live tests
+        run: dotnet test
+          --configuration Release
+          --filter="TestCategory!=Smoke&TestCategory!=Images&TestCategory!=Moderations&TestCategory!=Manual"
+          --logger "trx;LogFilePrefix=live"
+          --results-directory ${{github.workspace}}/artifacts/test-results
+          ${{ env.version_suffix_args }}
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v2
+        if: ${{ !cancelled() }}
+        with:
+          name: test-artifacts
+          path: ${{github.workspace}}/artifacts

.github/workflows/main.yml

@@ -1,4 +1,6 @@
-name: Build and Test
+# This worflow is triggered on push to main, pull request, or by manual invocation.
+# It builds and unit tests the codebase.
+name: Build and Unit Test
 
 on:
   workflow_dispatch:
@@ -10,51 +12,35 @@ on:
 
 jobs:
   build: # Test, pack and publish the Open AI nuget package as a build artifact 
+    name: Build
     runs-on: ubuntu-latest
     env:
       version_suffix_args: ${{ format('--version-suffix="alpha.{0}"', github.run_number) }}
     steps:
       - name: Setup .NET
-        uses: actions/setup-dotnet@v1
+        uses: actions/setup-dotnet@v3
         with:
-          dotnet-version: 8.x
+          dotnet-version: '8.x'
 
       - name: Checkout code
         uses: actions/checkout@v2
 
-      - name: Build and Pack
+      - name: Build and pack
         run: dotnet pack
           --configuration Release
           --output "${{github.workspace}}/artifacts/packages"
           ${{ env.version_suffix_args }}
 
-      - name: Test
+      - name: Run unit tests
         run: dotnet test
           --configuration Release
-          --filter="TestCategory~${{ github.event_name == 'pull_request' && 'Offline' || 'Online' }}|TestCategory~Smoke"
-          --logger "trx;LogFileName=${{github.workspace}}/artifacts/test-results/full.trx"
-        env:
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          --filter="TestCategory=Smoke&TestCategory!=Manual"
+          --logger "trx;LogFilePrefix=smoke"
+          --results-directory ${{github.workspace}}/artifacts/test-results
 
       - name: Upload artifact
         uses: actions/upload-artifact@v2
+        if: ${{ !cancelled() }}
         with:
           name: build-artifacts
           path: ${{github.workspace}}/artifacts
-
-      - name: NuGet Autenticate
-        if: github.event_name != 'pull_request'
-        run: dotnet nuget add source
-          "https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json"
-          --name "github"
-          --username ${{ github.actor }}
-          --password ${{ secrets.GITHUB_TOKEN }}
-          --store-password-in-clear-text
-
-      - name: Publish
-        if: github.event_name != 'pull_request'
-        run: dotnet nuget push
-          ${{github.workspace}}/artifacts/packages/*.nupkg
-          --source "github"
-          --api-key ${{ secrets.GITHUB_TOKEN }}
-          --skip-duplicate

.github/workflows/release.yml

@@ -1,53 +1,104 @@
+# This workflow is triggered by new releases and on a daily schedule.
+# It builds, unit tests, live tests and published the Open AI nuget package.
+# For daily runs, the package is published to the GitHub package registry.
+# For releases, the package is published to the NuGet package registry.
 name: Release package
 
 on:
   release:
     types: [published]
+  schedule:
+    # run every day at 00:00
+    - cron: '0 0 * * *'
 
 jobs:
-  deploy:
+  build:
+    name: Build
     runs-on: ubuntu-latest
+    environment: Live Testing
+    env:
+      version_suffix_args: ${{ github.event_name == 'schedule' && format('--version-suffix="alpha.{0}"', github.run_number) || '' }}
     permissions:
       packages: write
       contents: write
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-dotnet@v3
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v3
         with:
-          dotnet-version: '8.x' # SDK Version to use.
+          dotnet-version: '8.x'
+
+      - name: Checkout code
+        uses: actions/checkout@v2
 
       # Pack the client nuget package and include url back to the repository and release tag
       - name: Build and Pack
         run: dotnet pack
           --configuration Release
-          --output "${{github.workspace}}/artifacts/packages"
+          --output "${{ github.workspace }}/artifacts/packages"
           /p:PackageProjectUrl="${{ github.server_url }}/${{ github.repository }}/tree/${{ github.event.release.tag_name }}"
           /p:PackageReleaseNotes="${{ github.server_url }}/${{ github.repository }}/blob/${{ github.event.release.tag_name }}/CHANGELOG.md"
+          ${{ env.version_suffix_args }}
 
-      - name: Test
+      - name: Unit Test
+        run: dotnet test
+          --configuration Release
+          --filter="TestCategory=Smoke&TestCategory!=Manual"
+          --logger "trx;LogFileName=${{ github.workspace }}/artifacts/test-results/smoke.trx"
+          ${{ env.version_suffix_args }}
+
+      - name: Run Live Tests
         run: dotnet test
           --configuration Release
-          --filter="TestCategory~Online"
-          --logger "trx;LogFileName=${{github.workspace}}/artifacts/test-results/full.trx"
+          --filter="TestCategory!=Smoke&TestCategory!=Images&TestCategory!=Moderations&TestCategory!=Manual"
+          --logger "trx;LogFilePrefix=live"
+          --results-directory ${{ github.workspace }}/artifacts/test-results
+          ${{ env.version_suffix_args }}
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
 
+      - name: Upload artifact
+        uses: actions/upload-artifact@v2
+        if: ${{ !cancelled() }}
+        with:
+          name: build-artifacts
+          path: ${{ github.workspace }}/artifacts
+
+  deploy:
+    name: Publish Package
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: write
+    steps:
+      - uses: actions/download-artifact@v2
+
       # Append the nuget package to the github release that triggered this workflow
       - name: Upload release asset
+        if: github.event_name == 'release'
         run: gh release upload ${{ github.event.release.tag_name }}
-          ${{github.workspace}}/artifacts/packages/*.*nupkg
+          ${{ github.workspace }}/build-artifacts/packages/*.*nupkg
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Upload artifact
-        uses: actions/upload-artifact@v2
-        with:
-          name: build-artifacts
-          path: ${{github.workspace}}/artifacts
+      - name: NuGet authenticate
+        run: dotnet nuget add source
+          "https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json"
+          --name "github"
+          --username ${{ github.actor }}
+          --password ${{ secrets.GITHUB_TOKEN }}
+          --store-password-in-clear-text
+
+      - name: Publish package to local feed
+        run: dotnet nuget push
+            ${{github.workspace}}/build-artifacts/packages/*.nupkg
+            --source github
+            --skip-duplicate
 
-      - name: Publish
+      - name: Publish package to nuget.org
+        if: github.event_name == 'release'
         run: dotnet nuget push
-          ${{github.workspace}}/artifacts/packages/*.nupkg
-          --source https://api.nuget.org/v3/index.json
-          --api-key ${{ secrets.NUGET_API_KEY }}
-          --skip-duplicate
+            ${{github.workspace}}/build-artifacts/packages/*.nupkg
+            --source https://api.nuget.org/v3/index.json
+            --api-key ${{ secrets.NUGET_API_KEY }}
+            --skip-duplicate