From db2e0b2a56b41e7bc23345e1f2c2be3127ac621a Mon Sep 17 00:00:00 2001 From: Summer Warren Date: Thu, 4 Dec 2025 23:46:44 -0800 Subject: [PATCH 1/3] Add script for tool description eval workflow --- .../Invoke-ToolDescriptionEvaluator.ps1 | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 eng/common/scripts/Invoke-ToolDescriptionEvaluator.ps1 diff --git a/eng/common/scripts/Invoke-ToolDescriptionEvaluator.ps1 b/eng/common/scripts/Invoke-ToolDescriptionEvaluator.ps1 new file mode 100644 index 000000000000..2e0bd40873cf --- /dev/null +++ b/eng/common/scripts/Invoke-ToolDescriptionEvaluator.ps1 @@ -0,0 +1,99 @@ +<# +.SYNOPSIS +Evaluates tool and prompt descriptions and produces a Markdown results report using Tool Description Evaluator from Mcp repo. + +.DESCRIPTION +This script builds and runs the ToolDescriptionEvaluator (.NET) against a set of tool and prompt definitions. +It restores and compiles the evaluator, executes it with the provided JSON inputs, and emits a `results.md` +report. The script supports configuration via parameters and environment variables for the embedding model +service used during evaluation. + +.PARAMETER EvaluatorPath +The path to the evaluator project root (or its `src` directory) that will be restored, built, and executed. + +.PARAMETER ToolsFilePath +The path to the JSON file containing tool definitions to be evaluated. + +.PARAMETER PromptsFilePath +The path to the JSON file containing prompt definitions to be evaluated. + +.PARAMETER outputFilePath +The target folder where the generated `results.md` will be moved after the evaluator runs. + +.PARAMETER AoaiEndpoint +The full endpoint URL for the embedding model (e.g., Azure OpenAI embeddings) used by the evaluator. + +.PARAMETER TextEmbeddingApiKey +The API key used to authenticate with the embedding endpoint. Prefer providing this via a secure +secret store or environment variable rather than hard-coding. + +.NOTES +- The evaluator emits `results.md` in the evaluator folder; this script moves it to `outputFilePath`. +- Requires .NET SDK available on PATH. +- Set-StrictMode is enabled. + +.EXAMPLE +.\Run-Evaluator.ps1 ` + -EvaluatorPath "C:\work\mcp\eng\tools\ToolDescriptionEvaluator\src" ` + -ToolsFilePath "C:\work\azure-sdk-tools\tools\azsdk-cli\azure-sdk-tools.json" ` + -PromptsFilePath "C:\work\azure-sdk-tools\tools\azsdk-cli\azure-sdk-prompts.json" ` + -outputFilePath "C:\work\azure-sdk-tools\tools\azsdk-cli" ` + -AoaiEndpoint "https:///openai/deployments/text-embedding-3-large/embeddings?api-version=2023-05-15" ` + -TextEmbeddingApiKey (Get-Secret -Name 'TextEmbeddingApiKey') + +Runs the evaluator with the specified tools and prompts files, then moves the generated results to the output path. +#> +param ( + [Parameter(Mandatory = $true)] + [string] $EvaluatorPath, + + [Parameter(Mandatory = $true)] + [string] $ToolsFilePath, + + [Parameter(Mandatory = $true)] + [string] $PromptsFilePath, + + [Parameter(Mandatory = $true)] + [string] $outputFilePath, + + # Environment Variables + [Parameter(Mandatory = $true)] + [string] $AoaiEndpoint, + + [Parameter(Mandatory = $true)] + [string] $TextEmbeddingApiKey +) + +Set-StrictMode -Version 3 + +# Build & run the evaluator +Write-Host "Changing directory to evaluator: $EvaluatorPath" +Push-Location $EvaluatorPath +try { + $env:AOAI_ENDPOINT = $AoaiEndpoint + $env:TEXT_EMBEDDING_API_KEY = $TextEmbeddingApiKey + + Write-Host "Restoring packages..." + dotnet restore + + Write-Host "Building (Release)..." + dotnet build --configuration Release + + Write-Host "Running Tool..." + dotnet run -- --tools-file "$ToolsFilePath" --prompts-file "$PromptsFilePath" +} +finally { + Pop-Location +} + +# The tool emits results.md in the evaluator folder +$generatedName = 'results.md' +$EvaluatorRoot = Split-Path $EvaluatorPath -Parent +$generatedPath = Join-Path -Path $EvaluatorRoot -ChildPath $generatedName +if (-not (Test-Path -Path $generatedPath -PathType Leaf)) { + throw "Expected output file not found: $generatedPath" +} + +Write-Host "Moving Results File: $generatedPath -> $outputFilePath" +Move-Item -Path $generatedPath -Destination $outputFilePath -Force +Write-Host "Successfully moved results file to $outputFilePath" \ No newline at end of file From 364372c71f59540de09705b2e15fcb42c677e98e Mon Sep 17 00:00:00 2001 From: Summer Warren Date: Wed, 10 Dec 2025 11:59:08 -0800 Subject: [PATCH 2/3] Fix param casing --- .../scripts/Invoke-ToolDescriptionEvaluator.ps1 | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/eng/common/scripts/Invoke-ToolDescriptionEvaluator.ps1 b/eng/common/scripts/Invoke-ToolDescriptionEvaluator.ps1 index 2e0bd40873cf..dd9f1d137d33 100644 --- a/eng/common/scripts/Invoke-ToolDescriptionEvaluator.ps1 +++ b/eng/common/scripts/Invoke-ToolDescriptionEvaluator.ps1 @@ -17,7 +17,7 @@ The path to the JSON file containing tool definitions to be evaluated. .PARAMETER PromptsFilePath The path to the JSON file containing prompt definitions to be evaluated. -.PARAMETER outputFilePath +.PARAMETER OutputFilePath The target folder where the generated `results.md` will be moved after the evaluator runs. .PARAMETER AoaiEndpoint @@ -28,7 +28,7 @@ The API key used to authenticate with the embedding endpoint. Prefer providing t secret store or environment variable rather than hard-coding. .NOTES -- The evaluator emits `results.md` in the evaluator folder; this script moves it to `outputFilePath`. +- The evaluator emits `results.md` in the evaluator folder; this script moves it to `OutputFilePath`. - Requires .NET SDK available on PATH. - Set-StrictMode is enabled. @@ -37,7 +37,7 @@ secret store or environment variable rather than hard-coding. -EvaluatorPath "C:\work\mcp\eng\tools\ToolDescriptionEvaluator\src" ` -ToolsFilePath "C:\work\azure-sdk-tools\tools\azsdk-cli\azure-sdk-tools.json" ` -PromptsFilePath "C:\work\azure-sdk-tools\tools\azsdk-cli\azure-sdk-prompts.json" ` - -outputFilePath "C:\work\azure-sdk-tools\tools\azsdk-cli" ` + -OutputFilePath "C:\work\azure-sdk-tools\tools\azsdk-cli" ` -AoaiEndpoint "https:///openai/deployments/text-embedding-3-large/embeddings?api-version=2023-05-15" ` -TextEmbeddingApiKey (Get-Secret -Name 'TextEmbeddingApiKey') @@ -54,7 +54,7 @@ param ( [string] $PromptsFilePath, [Parameter(Mandatory = $true)] - [string] $outputFilePath, + [string] $OutputFilePath, # Environment Variables [Parameter(Mandatory = $true)] @@ -94,6 +94,6 @@ if (-not (Test-Path -Path $generatedPath -PathType Leaf)) { throw "Expected output file not found: $generatedPath" } -Write-Host "Moving Results File: $generatedPath -> $outputFilePath" -Move-Item -Path $generatedPath -Destination $outputFilePath -Force -Write-Host "Successfully moved results file to $outputFilePath" \ No newline at end of file +Write-Host "Moving Results File: $generatedPath -> $OutputFilePath" +Move-Item -Path $generatedPath -Destination $OutputFilePath -Force +Write-Host "Successfully moved results file to $OutputFilePath" \ No newline at end of file From 264cd461fbc39306de8382a26fa40b9c9a0ac00e Mon Sep 17 00:00:00 2001 From: Summer Warren Date: Wed, 10 Dec 2025 16:27:23 -0800 Subject: [PATCH 3/3] respond to comments --- .../Invoke-ToolDescriptionEvaluator.ps1 | 75 +++++++++++-------- 1 file changed, 44 insertions(+), 31 deletions(-) diff --git a/eng/common/scripts/Invoke-ToolDescriptionEvaluator.ps1 b/eng/common/scripts/Invoke-ToolDescriptionEvaluator.ps1 index dd9f1d137d33..6029917adcfd 100644 --- a/eng/common/scripts/Invoke-ToolDescriptionEvaluator.ps1 +++ b/eng/common/scripts/Invoke-ToolDescriptionEvaluator.ps1 @@ -1,47 +1,50 @@ <# .SYNOPSIS -Evaluates tool and prompt descriptions and produces a Markdown results report using Tool Description Evaluator from Mcp repo. + Evaluates tool and prompt descriptions and produces a Markdown results report using Tool Description Evaluator from Mcp repo. .DESCRIPTION -This script builds and runs the ToolDescriptionEvaluator (.NET) against a set of tool and prompt definitions. -It restores and compiles the evaluator, executes it with the provided JSON inputs, and emits a `results.md` -report. The script supports configuration via parameters and environment variables for the embedding model -service used during evaluation. + This script builds and runs the ToolDescriptionEvaluator (.NET) against a set of tool and prompt definitions. + It restores and compiles the evaluator, executes it with the provided JSON inputs, and emits a `results.md` + report. The script supports configuration via parameters and environment variables for the embedding model + service used during evaluation. + +.LINK + https://github.com/microsoft/mcp/tree/main/eng/tools/ToolDescriptionEvaluator .PARAMETER EvaluatorPath -The path to the evaluator project root (or its `src` directory) that will be restored, built, and executed. + The path to the evaluator project root (or its `src` directory) that will be restored, built, and executed. .PARAMETER ToolsFilePath -The path to the JSON file containing tool definitions to be evaluated. + The path to the JSON file containing tool definitions to be evaluated. .PARAMETER PromptsFilePath -The path to the JSON file containing prompt definitions to be evaluated. + The path to the JSON file containing prompt definitions to be evaluated. .PARAMETER OutputFilePath -The target folder where the generated `results.md` will be moved after the evaluator runs. + The target file path where the generated `results.md` will be moved after the evaluator runs. .PARAMETER AoaiEndpoint -The full endpoint URL for the embedding model (e.g., Azure OpenAI embeddings) used by the evaluator. + The full endpoint URL for the embedding model (e.g., Azure OpenAI embeddings) used by the evaluator. .PARAMETER TextEmbeddingApiKey -The API key used to authenticate with the embedding endpoint. Prefer providing this via a secure -secret store or environment variable rather than hard-coding. + The API key used to authenticate with the embedding endpoint. Prefer providing this via a secure + secret store or environment variable rather than hard-coding. .NOTES -- The evaluator emits `results.md` in the evaluator folder; this script moves it to `OutputFilePath`. -- Requires .NET SDK available on PATH. -- Set-StrictMode is enabled. + - The evaluator emits `results.md` in the evaluator folder; this script moves it to `OutputFilePath`. + - Requires .NET SDK available on PATH. + - Set-StrictMode is enabled. .EXAMPLE -.\Run-Evaluator.ps1 ` - -EvaluatorPath "C:\work\mcp\eng\tools\ToolDescriptionEvaluator\src" ` - -ToolsFilePath "C:\work\azure-sdk-tools\tools\azsdk-cli\azure-sdk-tools.json" ` - -PromptsFilePath "C:\work\azure-sdk-tools\tools\azsdk-cli\azure-sdk-prompts.json" ` - -OutputFilePath "C:\work\azure-sdk-tools\tools\azsdk-cli" ` - -AoaiEndpoint "https:///openai/deployments/text-embedding-3-large/embeddings?api-version=2023-05-15" ` - -TextEmbeddingApiKey (Get-Secret -Name 'TextEmbeddingApiKey') - -Runs the evaluator with the specified tools and prompts files, then moves the generated results to the output path. + .\Invoke-ToolDescriptionEvaluator.ps1 ` + -EvaluatorPath "C:\work\mcp\eng\tools\ToolDescriptionEvaluator\src" ` + -ToolsFilePath "C:\work\azure-sdk-tools\tools\azsdk-cli\azure-sdk-tools.json" ` + -PromptsFilePath "C:\work\azure-sdk-tools\tools\azsdk-cli\azure-sdk-prompts.json" ` + -OutputFilePath "C:\work\azure-sdk-tools\tools\azsdk-cli" ` + -AoaiEndpoint "https:///openai/deployments/text-embedding-3-large/embeddings?api-version=2023-05-15" ` + -TextEmbeddingApiKey (Get-Secret -Name 'TextEmbeddingApiKey') + + Runs the evaluator with the specified tools and prompts files, then moves the generated results to the output path. #> param ( [Parameter(Mandatory = $true)] @@ -66,6 +69,20 @@ param ( Set-StrictMode -Version 3 +# Validate input paths +$pathsToCheck = @{ + "EvaluatorPath" = $EvaluatorPath + "ToolsFilePath" = $ToolsFilePath + "PromptsFilePath" = $PromptsFilePath + "OutputFilePath" = $OutputFilePath +} + +foreach ($p in $pathsToCheck.GetEnumerator()) { + if (-not (Test-Path -Path $p.Value)) { + throw "Path does not exist for parameter '$($p.Key)': $($p.Value)" + } +} + # Build & run the evaluator Write-Host "Changing directory to evaluator: $EvaluatorPath" Push-Location $EvaluatorPath @@ -73,17 +90,13 @@ try { $env:AOAI_ENDPOINT = $AoaiEndpoint $env:TEXT_EMBEDDING_API_KEY = $TextEmbeddingApiKey - Write-Host "Restoring packages..." - dotnet restore - - Write-Host "Building (Release)..." - dotnet build --configuration Release - Write-Host "Running Tool..." - dotnet run -- --tools-file "$ToolsFilePath" --prompts-file "$PromptsFilePath" + dotnet run --configuration Release -- --tools-file "$ToolsFilePath" --prompts-file "$PromptsFilePath" } finally { Pop-Location + Remove-Item Env:\AOAI_ENDPOINT -ErrorAction SilentlyContinue + Remove-Item Env:\TEXT_EMBEDDING_API_KEY -ErrorAction SilentlyContinue } # The tool emits results.md in the evaluator folder