diff --git a/documentation/samplefiles/Microsoft.YourServiceName/stable/YYYY-MM-DD/YourServiceCat.json b/documentation/samplefiles/Microsoft.YourServiceName/stable/YYYY-MM-DD/YourServiceCat.json new file mode 100644 index 000000000000..bf0990d76bb3 --- /dev/null +++ b/documentation/samplefiles/Microsoft.YourServiceName/stable/YYYY-MM-DD/YourServiceCat.json @@ -0,0 +1,234 @@ +{ + "swagger": "2.0", + "info": { + "version": "2019-12-01", + "title": "YourServiceName", + "description": "Description of the new service", + "x-ms-code-generation-settings": { + "name": "YourServiceNameClient" + } + }, + "host": "management.azure.com", + "schemes": [ + "https" + ], + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "security": [ + { + "azure_auth": [ + "user_impersonation" + ] + } + ], + "securityDefinitions": { + "azure_auth": { + "type": "oauth2", + "authorizationUrl": "https://login.microsoftonline.com/common/oauth2/authorize", + "flow": "implicit", + "description": "Azure Active Directory OAuth2 Flow", + "scopes": { + "user_impersonation": "impersonate your user account" + } + } + }, + "paths": { + "/subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.YourServiceName/cats/{catName}": { + "get": { + "operationId": "Cats_Get", + "x-ms-examples": { + "GetCat": "./examples/CatsGet.json" + }, + "description": "This is a sample get operation, please see guidelines in azure-rest-api-specs repository for more info", + "parameters": [ + { + "$ref": "#/parameters/SubscriptionIdParameter" + }, + { + "$ref": "#/parameters/ResourceGroupNameParameter" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/CatNameParameter" + } + ], + "responses": { + "200": { + "description": "Describe the result of a successful operation.", + "schema": { + "$ref": "#/definitions/Cat" + } + }, + "default": { + "description": "Error response describing why the operation failed.", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + } + }, + "put": { + "operationId": "Cats_CreateOrUpdate", + "x-ms-examples": { + "UpdateCat": "./examples/CatsCreateOrUpdate.json" + }, + "description": "This is a sample get operation, please see guidelines in azure-rest-api-specs repository for more info", + "parameters": [ + { + "$ref": "#/parameters/SubscriptionIdParameter" + }, + { + "$ref": "#/parameters/ResourceGroupNameParameter" + }, + { + "$ref": "#/parameters/ApiVersionParameter" + }, + { + "$ref": "#/parameters/CatNameParameter", + }, + { + "name": "parameters", + "in": "body", + "description": "The parameters for creating or updating a cat.", + "required": true, + "schema": { + "$ref": "#/definitions/Cat" + } + } + ], + "responses": { + "200": { + "description": "Describe the result of a successful operation.", + "schema": { + "$ref": "#/definitions/Cat" + } + }, + "default": { + "description": "Error response describing why the operation failed.", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + } + } + } + }, + "definitions": { + "Result": { + "description": "Sample result definition", + "properties": { + "sampleProperty": { + "type": "string", + "description": "Sample property of type string" + } + } + }, + "ErrorResponse": { + "description": "Error response.", + "properties": { + "error": { + "$ref": "#/definitions/ErrorDefinition", + "description": "The error details." + } + } + }, + "ErrorDefinition": { + "description": "Error definition.", + "properties": { + "code": { + "description": "Service specific error code which serves as the substatus for the HTTP error code.", + "type": "string", + "readOnly": true + }, + "message": { + "description": "Description of the error.", + "type": "string", + "readOnly": true + }, + "details": { + "description": "Internal error details.", + "type": "array", + "items": { + "$ref": "#/definitions/ErrorDefinition" + }, + "readOnly": true + } + } + }, + "Cat": { + "description": "Sample cat model", + "properties": { + "id": { + "type": "string", + "description": "The id of the cat" + }, + "location": { + "type": "string", + "description": "The location of the cat" + }, + "name": { + "type": "string", + "description": "The name of the cat" + }, + "type": { + "type": "string", + "description": "The type of the cat" + }, + "properties": { + "$ref": "#/definitions/CatProperties" + } + } + }, + "CatProperties": { + "description": "Sample cat properties model", + "properties": { + "fullName": { + "type": "string", + "description": "The full name of the cat" + }, + "mateId": { + "type": "string", + "description": "The id of the cat's mate" + } + }, + "required": ["name"] + } + }, + "parameters": { + "SubscriptionIdParameter": { + "name": "subscriptionId", + "in": "path", + "required": true, + "type": "string", + "description": "The Azure subscription ID. This is a GUID-formatted string (e.g. 00000000-0000-0000-0000-000000000000)" + }, + "ResourceGroupNameParameter": { + "name": "resourceGroupName", + "in": "path", + "required": true, + "type": "string", + "description": "The name of the resource group.", + "x-ms-parameter-location": "method" + }, + "ApiVersionParameter": { + "name": "api-version", + "in": "query", + "required": true, + "type": "string", + "description": "The API version to be used with the HTTP request." + }, + "CatNameParameter": { + "name": "catName", + "in": "path", + "required": true, + "type": "string", + "description": "The name of the cat." + } + } +} diff --git a/documentation/samplefiles/Microsoft.YourServiceName/stable/YYYY-MM-DD/examples/CatsCreateOrUpdate.json b/documentation/samplefiles/Microsoft.YourServiceName/stable/YYYY-MM-DD/examples/CatsCreateOrUpdate.json new file mode 100644 index 000000000000..3faed7a0bcd5 --- /dev/null +++ b/documentation/samplefiles/Microsoft.YourServiceName/stable/YYYY-MM-DD/examples/CatsCreateOrUpdate.json @@ -0,0 +1,27 @@ +{ + "parameters": { + "accountName": "sampleacct", + "resourceGroupName": "YourServiceNameClient", + "api-version": "2019-12-01", + "subscriptionId": "subid", + "catName": "Rococo", + "parameters": { + "properties": { + "fullName": "Rococo Sue" + } + } + }, + "responses": { + "200": { + "body": { + "name": "Rococo", + "location": "westus", + "id": "/subscriptions/subid/resourceGroups/YourServiceNameClient/providers/Microsoft.YourServiceName/cats/Rococo", + "type": "Microsoft.YourServiceName/cats", + "properties": { + "fullName": "Rococo Sue" + } + } + } + } +} diff --git a/documentation/samplefiles/Microsoft.YourServiceName/stable/YYYY-MM-DD/examples/CatsGet.json b/documentation/samplefiles/Microsoft.YourServiceName/stable/YYYY-MM-DD/examples/CatsGet.json new file mode 100644 index 000000000000..fef4b532b247 --- /dev/null +++ b/documentation/samplefiles/Microsoft.YourServiceName/stable/YYYY-MM-DD/examples/CatsGet.json @@ -0,0 +1,19 @@ +{ + "parameters": { + "accountName": "sampleacct", + "resourceGroupName": "YourServiceNameClient", + "api-version": "2019-12-01", + "subscriptionId": "subid", + "catName": "Rococo" + }, + "responses": { + "200": { + "body": { + "name": "Rococo", + "location": "westus", + "id": "/subscriptions/subid/resourceGroups/YourServiceNameClient/providers/Microsoft.YourServiceName/cats/Rococo", + "type": "Microsoft.YourServiceName/cats" + } + } + } +} diff --git a/documentation/samplefiles/Microsoft.YourServiceName/stable/YYYY-MM-DD/examples/OperationGroupGet.json b/documentation/samplefiles/Microsoft.YourServiceName/stable/YYYY-MM-DD/examples/OperationGroupGet.json index a192d62cb610..f41dba65ede5 100644 --- a/documentation/samplefiles/Microsoft.YourServiceName/stable/YYYY-MM-DD/examples/OperationGroupGet.json +++ b/documentation/samplefiles/Microsoft.YourServiceName/stable/YYYY-MM-DD/examples/OperationGroupGet.json @@ -8,7 +8,6 @@ "responses": { "200": { "body": { - "name": "sampleacct", "location": "uswest", "id": "/subscriptions/subid/resourceGroups/YourServiceNameClient/providers/Microsoft.YourServiceName/batchAccounts/sampleacct", "type": "Microsoft.YourServiceName/YourServiceName" diff --git a/documentation/samplefiles/Microsoft.YourServiceName/stable/YYYY-MM-DD/scenarios/testYourService.yaml b/documentation/samplefiles/Microsoft.YourServiceName/stable/YYYY-MM-DD/scenarios/testYourService.yaml new file mode 100644 index 000000000000..5fbd5b9a6547 --- /dev/null +++ b/documentation/samplefiles/Microsoft.YourServiceName/stable/YYYY-MM-DD/scenarios/testYourService.yaml @@ -0,0 +1,47 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/documentation/test-scenario/references/v1.0/schema.json + +scope: ResourceGroup +testScenarios: + - description: Microsoft.YourService Test Operation Get + steps: + - step: Test Operation Get + exampleFile: ../examples/OperationGroupGet.json + + - description: Microsoft.YourService Test Cat Create and Update + steps: + - step: Create cat Rococo + exampleFile: ../examples/CatsCreateOrUpdate.json + resourceName: cat0 + + - step: Get cat Rococo + exampleFile: ../examples/CatsGet.json + + - step: Update full name + resourceName: cat0 + resourceUpdate: + - replace: /properties/fullName + value: Rococo the Cat + + - description: Microsoft.YourService Test Cat mate link + steps: + - step: Create cat Rococo + exampleFile: ../examples/CatsCreateOrUpdate.json + resourceName: cat0 + + - step: Create cat Baroque + exampleFile: ../examples/CatsCreateOrUpdate.json + resourceName: cat1 + variables: + catName: Baroque + requestUpdate: + - replace: /parameters/properties/fullName + value: Baroque the Cat + outputVariables: + cat1Id: + fromResponse: /id + + - step: Link cat Baroque to cat Rococo + resourceName: cat0 + resourceUpdate: + - add: /properties/mateId + value: $(cat1Id) diff --git a/documentation/samplefiles/readme.go.md b/documentation/samplefiles/readme.go.md index f755026a6888..2bef6f16af36 100644 --- a/documentation/samplefiles/readme.go.md +++ b/documentation/samplefiles/readme.go.md @@ -2,10 +2,25 @@ These settings apply only when `--go` is specified on the command line. -```yaml $(go) && $(track2) -azure-arm: true -license-header: MICROSOFT_MIT_NO_VERSION -module-name: sdk/[[ServiceName]]/arm[[ServiceName]] -module: github.com/Azure/azure-sdk-for-go/$(module-name) -output-folder: $(go-sdk-folder)/$(module-name) +```yaml $(go) +go: + license-header: MICROSOFT_MIT_NO_VERSION + namespace: [[ServiceName]] + clear-output-folder: true +``` + +### Go multi-api + +``` yaml $(go) && $(multiapi) +batch: + - tag: package-[[Version]][[-ReleaseState]] +``` + +### Tag: package-[[Version]][[-ReleaseState]] and go + +These settings apply only when `--tag=package-[[Version]][[-ReleaseState]] --go` is specified on the command line. +Please also specify `--go-sdk-folder=`. + +```yaml $(tag) == 'package-[[Version]][[-ReleaseState]]' && $(go) +output-folder: $(go-sdk-folder)/services[[/ReleaseState]]/$(namespace)/mgmt/[[Version]]/$(namespace) ``` diff --git a/documentation/samplefiles/samplereadme.md b/documentation/samplefiles/samplereadme.md index 87547f82a67b..bce1228cf02e 100644 --- a/documentation/samplefiles/samplereadme.md +++ b/documentation/samplefiles/samplereadme.md @@ -51,7 +51,7 @@ This is not used by Autorest itself. swagger-to-sdk: - repo: azure-sdk-for-python-track2 - repo: azure-sdk-for-java - - repo: azure-sdk-for-go-track2 + - repo: azure-sdk-for-go - repo: azure-sdk-for-js - repo: azure-resource-manager-schemas - repo: azure-cli-extensions diff --git a/documentation/test-scenario/how-to/QuickStart.md b/documentation/test-scenario/how-to/QuickStart.md new file mode 100644 index 000000000000..1057dcfa8f40 --- /dev/null +++ b/documentation/test-scenario/how-to/QuickStart.md @@ -0,0 +1,111 @@ + + +# API test quick start + +## Install + +`oav` is an open-source powerful tool for swagger validation, example generation, and API testing. GitHub: https://github.com/Azure/oav. + +```sh +npm install -g oav@latest +``` +### OAV Features +- Very easy to use and run. +- Support postman collection format. Debug easily. +- Request response validation. `oav` implement a powerful validation algorithm and help developer to detect service issue in the early phase. +- Validation result report. After each run test scenario, developer will get a validation report which contains detect issue in api test. +- Integrate everywhere. Easily integrate with azure-pipeline, cloud-test. + + +## Create AAD app + +To run API test, first please prepare an AAD app which is used for provisioning Azure resource. Please grant subscription contributor permission to this AAD app. + +For how to create AAD app, please follow this doc https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal + +## Authoring steps + +We will write test scenario file for SignalR service as an example. + +#### 1. Write your first test scenario file + +First, create a folder `scenarios` under the api version folder. All test scenario files under the `scenarios` folder should bind with the api version. + +![folder-structure](./folder-structure.png) + +Now write your basic test scenario. For more detail about test scenario file format, please refer to +[Test Scenario Definition Reference](../references/TestDefinitionReference.md). + +```yaml +# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-rest-api-specs/main/documentation/test-scenario/references/v1.0/schema.json + +contentVersion: 1.0.0 +scope: ResourceGroup +testScenarios: + - description: Microsoft.SignalRService/signalR SignalR_CreateOrUpdate + steps: + - step: SignalR_CreateOrUpdate + exampleFile: ../examples/SignalR_CreateOrUpdate.json + - step: SignalR_Delete + exampleFile: ../examples/SignalR_Delete.json +``` + +#### 2. create your env file + +The `env.json` file contains required test scenario variables such as, subscriptionId, AAD applicationId, AAD applicationSecret. + +```json +{ + "subscriptionId": "", + "location": "westus", + "tenantId": "", + "client_id": "", + "client_secret": "" +} +``` + +#### 3. Run api test + +```sh +oav run /home/user/azure-rest-api-specs/specification/signalr/resource-manager/Microsoft.SignalRService/preview/2020-07-01-preview/scenarios/signalR.yaml -e env.json +``` + +#### 4. Debug with postman + +Sometimes the command `oav run` may fail due to non 2xx HTTP status code. Now you need to debug the test scenario with postman. + +When run `run`, it automatically generate postman collection and postman env in `generated////` folder. Here is the generated file folder structure. The `collection.json` and `env.json` is generated postman collection file and environment file. `202105120922-5c3x5` is current runId. For each run command it will generated unique runId. + +``` +generated +└── Microsoft.SignalRService + └── 2020-07-01-preview + └── signalR + └── 202105120922-5c3x5 + ├── signalR_0 + │ ├── collection.json + │ └── env.json + | |__ report.json + └── signalR_0.json +``` + +Postman is a widely used GUI API testing tool. And you could use Postman import the generated postman collection and env for your local debug. + +![import-postman-collection](./import-postman-collection.png) + +After you import postman collection, you will get such requests. Now you could debug API test with postman locally. + +![postman-collection-signalr](./postman-collection-signalr.PNG) + +#### 5. manual update example value + +After debug with postman, you need to rewrite back all the updated values and run `oav run -e ` again. The result should be successful. + +## Feedback + +If you have any question, feel free to send email to vscswagger@microsoft.com diff --git a/documentation/test-scenario/how-to/armTemplate.png b/documentation/test-scenario/how-to/armTemplate.png new file mode 100644 index 000000000000..b0e87914ce73 Binary files /dev/null and b/documentation/test-scenario/how-to/armTemplate.png differ diff --git a/documentation/test-scenario/how-to/folder-structure.png b/documentation/test-scenario/how-to/folder-structure.png new file mode 100755 index 000000000000..f0c66c2137b0 Binary files /dev/null and b/documentation/test-scenario/how-to/folder-structure.png differ diff --git a/documentation/test-scenario/how-to/genTestScenario.gif b/documentation/test-scenario/how-to/genTestScenario.gif new file mode 100644 index 000000000000..b1e3bfae623f Binary files /dev/null and b/documentation/test-scenario/how-to/genTestScenario.gif differ diff --git a/documentation/test-scenario/how-to/generateABasicTestScenario.md b/documentation/test-scenario/how-to/generateABasicTestScenario.md new file mode 100644 index 000000000000..e1302cdb59db --- /dev/null +++ b/documentation/test-scenario/how-to/generateABasicTestScenario.md @@ -0,0 +1,57 @@ +# Generate a basic test scenario file + +## Prerequisite + +We use `oav` tools to generate basic test scenario. `oav` analyze swagger file and use swagger example as test scenario steps. So first, you need to install the latest oav. + +## Introduction + +`oav` support rule based test scenario file generation. We use this command to generate test scenario file. + +`oav generate-static-test-scenario --readme --tag --rules ` + +- readme: swagger readme file. +- tag: which tag to generate. oav will analyze swagger file under the tag and generate test scenario. +- rules: Currently support two types. `resource-put-delete`, `operations-list`. Default: `resource-put-delete` + - `resource-put-delete`: generate resource put and delete test scenario. + - `operations-list`: generate operations list test scenario. `operations-list` is the simplest API which must be defined in swagger. + +Example: + +![](./genTestScenario.gif) + +This command will load and analyze swagger and generate a basic test scenario file (`resource-put-delete`). + +Result: the output contains two files + +- test-scenarios/signalR.yaml: The test scenario file. +- readme.test.md: The entry for SDK test generation + +The generated test scenario file: The generated test scenario file contains two steps. Create signalR and delete it. It's a basic test scenario and developer can add more step based on the basic test scenario file. + +``` +scope: ResourceGroup +testScenarios: + - description: Microsoft.SignalRService/signalR SignalR_CreateOrUpdate + steps: + - step: SignalR_CreateOrUpdate + exampleFile: ../examples/SignalR_CreateOrUpdate.json + - step: SignalR_Delete + exampleFile: ../examples/SignalR_Delete.json +``` + +If you pass rule option `operations-list`, you will get such test scenario file. + +``` +scope: ResourceGroup +testScenarios: + - description: operationsList + steps: + - step: operationsList + exampleFile: ../examples/Operations_List.json + +``` + +## Reference + +- [oav](https://github.com/Azure/oav/tree/develop) diff --git a/documentation/test-scenario/how-to/import-postman-collection.png b/documentation/test-scenario/how-to/import-postman-collection.png new file mode 100644 index 000000000000..6c5e73700482 Binary files /dev/null and b/documentation/test-scenario/how-to/import-postman-collection.png differ diff --git a/documentation/test-scenario/how-to/postman-collection-signalr.PNG b/documentation/test-scenario/how-to/postman-collection-signalr.PNG new file mode 100644 index 000000000000..97203ec50900 Binary files /dev/null and b/documentation/test-scenario/how-to/postman-collection-signalr.PNG differ diff --git a/documentation/test-scenario/how-to/runApiTest.gif b/documentation/test-scenario/how-to/runApiTest.gif new file mode 100644 index 000000000000..a35fb91cc457 Binary files /dev/null and b/documentation/test-scenario/how-to/runApiTest.gif differ diff --git a/documentation/test-scenario/how-to/testScenarioWithARMTemplate.md b/documentation/test-scenario/how-to/testScenarioWithARMTemplate.md new file mode 100644 index 000000000000..7f6524b7c359 --- /dev/null +++ b/documentation/test-scenario/how-to/testScenarioWithARMTemplate.md @@ -0,0 +1,104 @@ +# Test scenario integrate with armTemplate + +## Background + +In some cases, we need to do some more complex operations before creating a resource. For example, + +- Create Azure SignalR service with a global unique name. Using ARMTemplate to generate a random unique string. +- Create VM with a storage account. Using ARMTemplate to provision storage account and passing the storage account resourceId as VM creation parameter. + +## Examples + +Here is an example about `generate unique resource name for signalR service` + +#### Generate unique resource name + +We use `armTemplate output` to overwrite `resourceName` variable and following `createResource` step will use this variable. Below is generate unique name armTemplate. This armTemplate output `resourceName` variables, so test scenario following step will using the output variable. + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "resourceNamePrefix": { + "type": "string", + "defaultValue": "signalr-" + } + }, + "variables": { + "resourceName": "[concat(parameters('resourceNamePrefix'), uniqueString(resourceGroup().id))]" + }, + "resources": [], + "outputs": { + "resourceName": { + "type": "string", + "value": "[variables('resourceName')]" + } + } +} +``` + +After we have this armTemplate, we could define current test scenario file. We defined `resourceName` variable globally. `./generate_unique_string.json` is armTemplate. + +`SignalR_CreateOrUpdate.json` + +```json +{ + "parameters": { + "parameters": { + "tag": { + "key1": "tag1" + }, + "properties":{ + ... + } + }, + "api-version": "2020-07-01-preview", + "subscriptionId": "00000000-0000-0000-0000-000000000000", + "resourceGroupName": "myResourceGroup", + "resourceName": "mySignalRService123xx" + }, + "responses": { + "200": { + "body": { + ... + } + }, + "201": { + "body": { + ... + } + }, + "202": {} + } +} +``` + +> NOTE: the example file `../examples/SignalR_CreateOrUpdate` has the same parameter name `resourceName`. So it will be automatically overwrite in runner. + +```yaml +scope: ResourceGroup +variables: + resourceName: "" +testScenarios: + - description: Microsoft.SignalRService/signalR CRUD + steps: + - step: Generate_Unique_string + armTemplateDeployment: ./generate_unique_string.json + - step: SignalR_checknameAvailability + exampleFile: ../examples/SignalR_CheckNameAvailability.json + - step: SignalR_CreateOrUpdate + exampleFile: ../examples/SignalR_CreateOrUpdate.json + - step: SignalR_ListKey + exampleFile: ../examples/SignalR_ListKeys.json + - step: SignalR_Delete + exampleFile: ../examples/SignalR_Delete.json +``` + +**Result**: + +![](./armTemplate.png) + +## Reference + +- [ARMTemplate deployment script](https://docs.microsoft.com/en-us/azure/azure-resource-manager/templates/deployment-script-template) diff --git a/documentation/test-scenario/readme.md b/documentation/test-scenario/readme.md new file mode 100644 index 000000000000..1103f0bcf7fa --- /dev/null +++ b/documentation/test-scenario/readme.md @@ -0,0 +1,26 @@ +# Test Scenario Documentation + +Test Scenario is a YAML file defining RESTful API usage scenarios of your service with a series of API calls. Test scenario can be used for service functional test, API quality validation and SDK/CLIs test generation. + +_**Caution**: This project is in early preview phase, hence breaking changes should be expected._ +## Features +- Simple to use: Intuitive test step definition based on Swagger examples and raw REST call. +- ARM Template integration: Support creating external Azure resources with ARM Template and executing Azure Powershell or Azure CLI scripts with ARM Template deployment script. +- Implementation independent: [oav](https://github.com/Azure/oav) is the default test scenario runner, and more runners will be supported, like SDKs in different languages. + +### Demo gif + +![demo](./how-to/runApiTest.gif) + +## Quick start + +- [Example: Write and run your first test scenario file](./how-to/QuickStart.md) +- [Example: Generate a basic test scenario file](./how-to/generateABasicTestScenario.md) +- [Example: use armTemplate to generate unique resourceName](./how-to/testScenarioWithARMTemplate.md) +- [Test scenario file sample](../samplefiles/Microsoft.YourServiceName/stable/YYYY-MM-DD/scenarios/testYourService.yaml) + +## References +- [Test Scenario Definition Reference](./references/TestDefinitionReference.md) +- [Test Scenario Variable Definition Reference](./references/Variables.md) +- [Test Scenario Runner Reference](./references/Runner.md) +- [Test Scenario Schema Reference](./references/v1.0/schema.json) diff --git a/documentation/test-scenario/references/ErrorCodeReference.md b/documentation/test-scenario/references/ErrorCodeReference.md new file mode 100644 index 000000000000..5863185800d1 --- /dev/null +++ b/documentation/test-scenario/references/ErrorCodeReference.md @@ -0,0 +1,121 @@ + + +# API Test Error Code + +OAV api testing define some rules and check whether actual service response match with example. Example is very important for downstream SDK test code generation, code samples and docs. The goal to detect example quality issues and check service behaviors consistent with example. + +## INCORRECT_PROVISIONING_STATE + +If service return 200 status code,it means the operation is succeed and finished. So the provisioning state should be one of the terminal states ["succeeded", "failed", "canceled", "ready", "created", "deleted"]. + +The provisioning state is very important for downstream terraform or cli to manage resource status. For more details about provisioning state, please refer to this [ARM RPC provisioning state](https://github.com/Azure/azure-resource-manager-rpc/blob/master/v1.0/async-api-reference.md#provisioningstate-property) + +## RESPONSE_MISSING_VALUE + +**Error message**: The response value is missing. Path: {}. Expected: {}. Actual: undefined + +The example has defined response value, but actually the server doesn't return that value. + +Example: + +```diff +{ + "properties":{ + "targetType":"blobNfs", + "junctions":[ + { + "namespacePath":"/blobnfs" + } + ], + "blobNfs":{ + "target":"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/scgroup/providers/Microsoft.Storage/storageAccounts/blofnfs/blobServices/default/containers/blobnfs", +- "usageModel":"WRITE_WORKLOAD_15" + } + } +} +``` + +## RESPONSE_ADDITIONAL_VALUE + +**Error message** Return additional response value. Path: {}. Expected: undefined. Actual: {} + +The example doesn't define the response value, but service actually return this value. + +Example: + +```diff +{ + "properties":{ + "targetType":"blobNfs", + "junctions":[ + { + "namespacePath":"/blobnfs" + } + ], + "blobNfs":{ + "target":"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/scgroup/providers/Microsoft.Storage/storageAccounts/blofnfs/blobServices/default/containers/blobnfs", + "usageModel":"WRITE_WORKLOAD_15" ++ "enableFeature": true + } + } +} +``` + +## RESPONSE_INCONSISTENT_VALUE + +**Error message** The actual response value is different from example. Path: {}. Expected: {}. Actual: {} + +The service returned value is different from example value. + +Example: + +```diff +{ + "properties":{ + "targetType":"blobNfs", + "junctions":[ + { + "namespacePath":"/blobnfs" + } + ], + "blobNfs":{ + "target":"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/scgroup/providers/Microsoft.Storage/storageAccounts/blofnfs/blobServices/default/containers/blobnfs", +- "usageModel":"WRITE_WORKLOAD_15" ++ "usageModel":"WORK_LOAD_14" + } + } +} +``` + +## ROUNDTRIP_INCONSISTENT_PROPERTY + +**Error message** The property's value in the response is different from what was set in the request. Path: {}. Request: {}. Response: {} + +Example: The sku in request parameters is `default`, but actual return is `standard`. + +```diff +{ + "parameters":{ + "properties":{ + "name":"myService", + "SKU":"default" + } + }, + "responses":{ + "200":{ + "properties":{ + "name":"myService", ++ "SKU":"standard" +- "SKU":"default" + + } + } + } +} + +``` diff --git a/documentation/test-scenario/references/Runner.md b/documentation/test-scenario/references/Runner.md new file mode 100644 index 000000000000..d7d553043b6c --- /dev/null +++ b/documentation/test-scenario/references/Runner.md @@ -0,0 +1,130 @@ +# Runner Behavior + +This document explains the expected behavior of runner. The word "runner" here references to any customer that consume the test scenario, including the runner that send out requests defined by test scenario, code generator that generate code that executes steps defined by test scenario, and any other consumer that need to understand the content of test scenario. + +## Load Test Scenario via OAV + +You could load the test scenario file via oav. It would be resolved as a simple object. + +```typescript + const readmeMd: string = + "/home/username/azure-rest-api-specs/specification/containerservice/resource-manager/readme.md"; + const argv = { + ["try-require"]: "readme.test.md", + tag: "package-2020-12", + }; + + // Get input-file config in readme.md + const autorestConfig = await getAutorestConfig(argv, readmeMd); + const swaggerFilePaths: string[] = autorestConfig["input-file"]; + const fileRoot = dirname(readmeMd); + + console.log("input-file:"); + console.log(swaggerFilePaths); + + // Create the loader from OAV + const loader = TestResourceLoader.create({ + useJsonParser: false, + checkUnderFileRoot: false, + fileRoot, + swaggerFilePaths, + }); + + // Load the test scenario file. File list could also be specified in readme.test.md + const testDef = await loader.load( + "Microsoft.ContainerService/stable/2020-12-01/test-scenarios/containerService.yaml" + ); + + console.log(testDef.testScenarios[0].steps); + + // Setup initial variable env + const env = new VariableEnv(); + env.setBatch({ + subscriptionId: "__your_subs_id_", + location: "westus", + SSH_PUBLIC_KEY: "__public_key_ssh__", + }); + + // Reference runner implementation in OAV. You need to implement your own runner. + const runner = new TestScenarioRunner({ + jsonLoader: loader.jsonLoader, + env, + client: new TestScenarioRestClient(getDefaultAzureCredential(), {}), + }); + + try { + for (const scenario of testDef.testScenarios) { + await runner.executeScenario(scenario); + } + } catch (e) { + console.log(e.message, e.stack); + } finally { + console.timeLog("TestLoad"); + await runner.cleanAllTestScope(); + } +``` + +After the test scenario is loaded, the test step will be slightly different from the file content. Every rest step will have the following resolved fields: + +- requestParameters + - Type: `object`, map of resolved parameter name and value. +- responseExpected + - Type: `any` + - The expected response body from the request. + +## Procedure of runner + +### Input + +Let's assume the following things as input of runner: + +- Test scenario definition that loaded via OAV. +- The scenario id of the test scenario. Test scenario definition file could contains multiple test scenarios, runner need to run one of them. +- Extra environment variables that required by test scenario, defined in `requiredVariables`. + +### Scope + +It's the `scope` field defined at top level of test scenario file. Now only `ResourceGroup` is supported, it means that the test scenario will: + +- Run under specified resource group. +- Runner would manage the resource group, it could create the resource group (defined by variable `subscriptionId`, `resourceGroup`, `location`) or use predefined resource group, it could also delete the resource group after the test scenario is done. Runner itself is responsible for managing the resource group, the behavior is not defined by this spec. +- Runner would run all the arm template deployment under the specified resource group. + +The scope is a convention, however it would not be enforced by test scenario. User could override the variable `resourceGroup` in any step to run that step in another resource group for example. + +### Variables + +See [Variables](./Variables.md) for variable spec. The runner must follow the variable definition in test scenario. Runner do not need to care about the variable conventions as it's already resolved by OAV. The runner must: + +- Load variables layer by layer as defined in the variable spec. +- Resolve variables like `$(variableName)` step by step in: + - requestParameter + - responseExpected (if it's used by runner) + - armTemplate payload + +### Procedure + +- Load definition via OAV, load required variables' value (runner need to specify how to load it). +- Manage the test scope, runner could create/reuse the scope as user defined in input. +- Run top level prepareSteps if it has not run. it's a list of steps defined in test scenario. +- Pass the variables from prepareSteps to the following main steps. +- For each steps defined in the test scenario array: + - If `type` field is `restCall`: + - Replace variables in `requestParameters`. + - Fill the request via parameter definition in swagger and parameter value in `requestParameters`. + - Send out the request. + - If the request is long running request, runner need to poll for the response. + - If the response's long running poll's final call is operation status, and the step itself is resource PUT/PATCH/DELETE, runner could run another GET against the resource to check. For DELETE, the final GET is expected to return 404. For PUT/PATCH, the final GET is expected to return 200, and it should represent the final response of the step. + - Check if the response status code is the same as the expected `statusCode` field defined in step. Optional. + - Check if the response body is the same as the expected `responseExpected` field defined in step. Optional. + - If `outputVariables` is defined, runner need to extract and define the variable from specified path in response body. + - If `type` field is `armTemplateDeployment`: + - Use the convention to replace arm template parameters if the parameter name matches the variable name and the parameter type is string. + - Send arm template deployment request under the resource group. + - Wait for the deployment to finish. + - Get the output variables from the deployment. Define the variables from the output variables. + - Else `type` is unsupported in runner. + +### Compare the response with expectedResponse + +It's hard for service team to make sure every field in expectedResponse is the same as the response, so here test scenario suggest to compare properties that are not `readOnly` and are not `x-ms-secret`. The detail should be defined by the runner, not this spec. diff --git a/documentation/test-scenario/references/TestDefinitionReference.md b/documentation/test-scenario/references/TestDefinitionReference.md new file mode 100644 index 000000000000..731d3127ccc6 --- /dev/null +++ b/documentation/test-scenario/references/TestDefinitionReference.md @@ -0,0 +1,431 @@ +# Test Definition Reference + +## Test Definition File + +See [Test Definition File Schema](./v1.0/schema.json#L1) + +File should be in format of yaml. + +**Example:** +```yaml +scope: ResourceGroup +requiredVariables: + - subscriptionId +variables: + publicIpAddressName: pubipdns +prepareSteps: + - step: prepare_resources + armTemplateDeployment: ./dep-something.json +testScenarios: + - description: test_network_public_ip + steps: + - step: Create_publicIPAddresses_pubipdns + resourceName: publicIPAddresses_pubipdns + exampleFile: ../examples/Create_publicIPAddresses_pubipdns_Generated.json + operationId: PublicIPAddresses_CreateOrUpdate + variables: + publicIpAddressName: pubipdns +``` + +**Fields:** +- **scope** + - **Type:** Required, Enum + - **Enum:** ResourceGroup + - Now only "ResourceGroup" is supported. + - **ResourceGroup:** All of the following test scenario and steps should be under some resourceGroup. It means: + - The consumer (test scenario runner or anything consumes test scenario) SHOULD maintain the resource group itself. Usually it requires user to input the subscriptionId/location, then it creates the resource group before test running, and deletes the resource group after running + - The consumer SHOULD set the following variables: + - **subscriptionId** + - **resourceGroupName** + - **location** + - For details of how variables works please see [Variables](./Variables.md) +- **variables** + - **Type:** Optional, Map of strings + - See [Variables](./Variables.md) +- **requiredVariables** + - **Type:** Optional, Array of string + - Variables that must be defined by user. By default, **subscriptionId** and **location** are required. +- **prepareSteps** + - **Type:** Optional, Array of [Test Step](#test-step) + - Steps that should run before every test scenario steps. +- **testScenarios** + - **Type:** Required, Array of [Test Scenario](#test-scenario) + +## Test Scenario + +See [Test Scenario Schema](./v1.0/schema.json#L331). + +It defines one test scenario that could go through on its own. + +**Example:** +```yaml +description: test_network_public_ip +shareTestScope: true +steps: + - step: Create_publicIPAddresses_pubipdns + resourceName: publicIPAddresses_pubipdns + exampleFile: ../examples/Create_publicIPAddresses_pubipdns_Generated.json + operationId: PublicIPAddresses_CreateOrUpdate +variables: + publicIpAddressName: pubipdns +``` + +**Fields:** +- **description** + - **Type:** Required, String + - Description for this test scenario. +- **shareTestScope** + - **Type:** Optional, Boolean or String + - **Default:** true + - Describe how the testScope (ResourceGroup if scope is ResourceGroup) could be shared with other tests. If it's true or it's the same string setting for different test scenario, then they share the same test scope, which means: + - These tests will run under the same test scope (e.g. ResourceGroup). They may launch in parallel. + - **prepareSteps** will only run once in the testScope. The variables will be shared. + - By default all the test scenario in one test definition file will be launched in the same test scope. If shareTestScope is false then it will not share anything with other test scenarios in the same file. +- **variables** + - **Type:** Optional, Map of strings + - See [Variables](./Variables.md) +- **steps** + - **Type:** Required, Array of [Test Step](#test-step) + - Steps in this test scenario + +## Test Step + +See [Test Step Schema](./v1.0/schema.json#L50). + +Defines one test step in test scenario. + +Should be one of the following: +- [Test Step](#test-step) +- [Test Step ARM Template Deployment](#test-step-arm-template-deployment) +- [Test Step Rest Call](#test-step-rest-call) + - [Rest Call](#rest-call) + - [Rest Call by ResourceName Tracking and Update](#rest-call-by-resourcename-tracking-and-update) + +All of the above definitions share the following fields: +- **variables** + - **Type:** Optional, Map of Strings + - See [Variables](./Variables.md) +- **step** + - **Type:** Required, String + - Step name. Must be unique in the same file. + +## Test Step ARM Template Deployment + +See [Test Step ARM Template Deployment Schema](./v1.0/schema.json#L78). + +Step to deploy ARM template to the test scope. Template parameters and outputs will also interact with variables automatically, see [Variables](./Variables.md). + +**Example:** +```yaml +step: prepare_resources +armTemplateDeployment: ./dep-storage-account.json +armTemplateParameters: ./dep-storage-account-params.json +``` + +**Fields:** +- **armTemplateDeployment** + - **Type:** Required, String + - Path to ARM template json file. See [ARM Template](https://docs.microsoft.com/azure/templates/). +- **armTemplateParameters** + - **Type:** Optional, String + - Path to ARM template parameter file. See [ARM Template Parameter File](https://docs.microsoft.com/azure/azure-resource-manager/templates/parameter-files). + +## Test Step Rest Call + +See [Test Step Rest Call Schema](./v1.0/schema.json#L97) + +Step to run a swagger operation defined rest call. This may not be just one http call. + +- If the operation is a long running operation (LRO), then follow the LRO polling strategy. + - Response statusCode must be 200 if the LRO succeeded, no matter what code the initial response is. + - If the LRO is PUT/PATCH, the runner should automatically insert a GET after the polling to verify the resource update result. +- If the operation is DELETE, then after the operation, the runner should automatically insert a GET to verify resource cannot be found. + +Rest call step could be defined either by an example file, or by resourceName tracking and update. + +Rest call will have computed **requestParameter** and **responseExpected** after parsing and loading: +- **requestParameter** + +### Rest Call + +**Example:** +```yaml +step: Create_publicIPAddresses_pubipdns +resourceName: publicIPAddresses_pubipdns +exampleFile: ../examples/Create_publicIPAddresses_pubipdns_Generated.json +operationId: PublicIPAddresses_CreateOrUpdate +statusCode: 200 +``` + +**Fields:** +- **exampleFile** + - **Type:** Optional, String + - Path to example file. Should be in format of "x-ms-example" files. +- **operationId** + - **Type:** Optional, String + - OperationId defined in swagger operation. It could be skipped if the example file is referenced by only one operation so we could detect the operationId. +- **statusCode:** + - **Type:** Optional, Number + - **Default:** 200 + - Expected response code. + - For LRO it must be 200 to indicate succeeded result, and must be 400 to indicate failed result. +- **requestUpdate** + - **Type:** Optional, Array of [JsonPatchOp](#jsonpatchop) + - Updates that applied to the requestParameters before sending it. +- **responseUpdate** + - **Type:** Optional, Array of [JsonPatchOp](#jsonpatchop) + - Updates that applied to the responseExpected. +- **outputVariables** + - **Type:** Optional, Map from variable name to object with property: + - **fromResponse** + - **Type:** Required, String + - Path to the response field to be used as variable. + +### Rest Call by ResourceName Tracking and Update + +**Example** +```yaml +- step: Create_publicIPAddresses_pubipdns + resourceName: publicIPAddresses_pubipdns + exampleFile: ../examples/Create_publicIPAddresses_pubipdns_Generated.json + operationId: PublicIPAddresses_CreateOrUpdate + statusCode: 200 + +- step: Update_publicIPAddresses + resourceName: publicIPAddresses_pubipdns + resourceUpdate: + - replace: /properties/location + value: westus +``` + +Different steps with the same resourceName will be tracked by the test scenario. It knows that you are trying to update the same resource. You can use the first request with example to specify the request and resource id, then the following step with the same resourceName will use the same resource id to update the resource. For the + +**Fields:** +- **resourceName** + - **Type:** Required, String + - The user-defined resource name of the resource to be tracked. It's only used as a name of that resource and do not need to be same as the actual resource name. +- **resourceUpdate** + - **Type:** Optional, Array of [JsonPatchOp](#jsonpatchop) + - Array of changes to be applied to the resource. + +resourceUpdate will help to automate compute the request body and the expected response body. The algorithm will be: + +- Get the expected response body from previous step with same `resourceName`, or from current step with example loaded. +- For each change in `resourceUpdate`, apply the change to the expected response body, mark as `computedAllProperties`. +- Let new request body parameter value to be: `computedAllProperties` without `readOnly` fields and `x-ms-mutability` fields that don't contains `update`. +- Let new response expected to be: `computedAllProperties` without `x-ms-secrets` fields and `x-ms-mutability` fields that don't contain `read`. +- Let the operationId to be: resource PUT operationId. + +### JsonPatchOp + +JsonPatchOp is used to define the update operation on json. You could add, remove, replace, move, copy and merge on json path. +All the json path used in JsonPatchOp is in format of [JsonPointer](https://datatracker.ietf.org/doc/html/rfc6901). + + - [JsonPatchOp](#jsonpatchop) + - [JsonPatchOpAdd](#jsonpatchopadd) + - [JsonPatchOpRemove](#jsonpatchopremove) + - [JsonPatchOpReplace](#jsonpatchopreplace) + - [JsonPatchOpMove](#jsonpatchopmove) + - [JsonPatchOpCopy](#jsonpatchopcopy) + - [JsonPatchOpMerge](#jsonpatchopmerge) +#### JsonPatchOpAdd + +**Example** +```yaml +add: /properties/items +value: 1 +``` + +**Fields:** +- **add** + - **Type:** Required, JsonPointer +- **value** + - **Type:** Required, Any + +Add json property at specified path. +1. If any segment of path does not exist, then it will be created. +2. If any value already exists on the path, then it will be overwritten. +3. If the parent of the destination is array, then the value will be inserted at the specified index. + +**Example of add** +``` +apply: +- add: /properties/location + value: "eastus" + +on data: +- { "properties": { } } + +result: +- { "properties": { "location": "eastus" } } } + +--- +apply: +- add: /properties/items/1 + value: 4 + +on data: +- { "properties": { "items": [1, 2, 3] } } + +result: +- { "properties": { "items": [1, 4, 2, 3] } } +``` + +#### JsonPatchOpRemove + +**Example** +```yaml +remove: /properties/items/1 +``` + +**Fields:** +- **remove** + - **Type:** Required, JsonPointer + +Remove element at specified path. +1. If any segment of path does not exist, then error will be thrown. +2. If parent of the specified path is array, then the element will be removed from the array. + +**Example of remove** +``` +apply: +- remove: /properties/items + +on data: +- { "properties": { "items": [1, 2, 3] } } + +result: +- { "properties": { } } + +--- +apply: +- remove: /properties/items/1 + +on data: +- { "properties": { "items": [1, 2, 3] } } + +result: +- { "properties": { "items": [1, 3] } } +``` + +#### JsonPatchOpReplace + +**Example** +```yaml +replace: /properties/items +value: 1 +``` + +**Fields:** +- **replace** + - **Type:** Required, JsonPointer +- **value** + - **Type:** Required, Any + +Replace json property at specified path. +1. If any segment of path does not exist, error will be thrown. +2. If any value already exists on the path, then it will be overwritten. + +**Example of replace** +``` +apply: +- replace: /properties/location + value: "eastus" + +on data: +- { "properties": { "location": "westus" } } + +result: +- { "properties": { "location": "eastus" } } } +``` + +#### JsonPatchOpMove + +**Example** +```yaml +move: /properties/items +path: /properties/items2 +``` + +**Fields:** +- **move** + - **Type:** Required, JsonPointer +- **path** + - **Type:** Required, JsonPointer + +Move json property at specified path to another path. It works as a combination of remove followed by add. Array index is also supported and works as add/remove does. + +**Example of move** +``` +apply: +- move: /properties/items + path: /properties/items2 + +on data: +- { "properties": { "items": [1, 2, 3] } } + +result: +- { "properties": { "items2": [1, 2, 3] } } +``` +#### JsonPatchOpCopy + +**Example** +```yaml +copy: /properties/items +path: /properties/items2 +``` + +**Fields:** +- **copy** + - **Type:** Required, JsonPointer +- **path** + - **Type:** Required, JsonPointer + +Copy json property at specified path to another path. Array index is also supported and works as add/remove does. + +**Example of copy** +``` +apply: +- copy: /properties/items + path: /properties/items2 + +on data: +- { "properties": { "items": [1, 2, 3] } } + +result: +- { "properties": { "items": [1, 2, 3] }, "items2": [1, 2, 3] } } +``` +#### JsonPatchOpMerge + +**Example** +```yaml +merge: /properties/item +value: + a: 1 + b: 2 +``` + +**Fields:** +- **merge** + - **Type:** Required, JsonPointer +- **value** + - **Type:** Required, Object + +Merge values into the object at specified path. +1. Property value at the specified path must be an object. +2. Properties with same key will be overwritten. + +**Example of merge** +``` +apply: +- merge: /properties + value: + a: 1 + b: 2 + +on data: +- { "properties": { "b": 0, "c": 0} } + +result: +- { "properties": { "a": 1, "b": 2, "c": 0 } } +``` \ No newline at end of file diff --git a/documentation/test-scenario/references/Variables.md b/documentation/test-scenario/references/Variables.md new file mode 100644 index 000000000000..6c9329e57fc9 --- /dev/null +++ b/documentation/test-scenario/references/Variables.md @@ -0,0 +1,140 @@ +# Variables in test scenario + +## Variable definition and replacement + +Variables could be defined in different level of test scenario: +- Test Definition level variable definition +- Test Scenario level variable definition +- Test Step level variable definition + +Variable could be referenced by `$(variableName)`. Currently variable type must be string. + +For example, in the following test scenario: + +```yaml +variables: + resourceName: level-1 + +test-scenarios: +- definition: Create some resource + variables: + resourceName: level-2 + steps: + - step: Create resource + variables: + resourceName: level-3 + exampleFile: ../examples/ResourceCreate.json +``` + +if in `../examples/ResourceCreate.json` we have `$(resourceName)` in some string, it would be replaced with `level-3`. + +Variables could also be defined on test running. For example you could set `subscriptionId` or `resourceGroupName` on the global scope. How to set global env is based on the test scenario consumer. + +Variables could be replaced recursively. For example if we have the following variables: +```yaml +variables: + resourceName: abc + resourceId: Microsoft.Contoso/$(resourceName) +``` + +Then `$(resourceId)` would be resolved to `Microsoft.Contoso/abc`. + +Variable resolving is limited to at most 100 times for certain string. + +## Convention: parameter name in example + +In one rest call step, if we have any parameter, for example it's named `param`, in the step requestParameters, with value `paramValue`, and test scenario has variable `param`, then by default, test scenario loader will: +- replace the parameter value of `param` to `$(param)` so that it will reference the variable value. +- replace `paramValue` to `$(param)` in every string in request body (in requestParameter) and in responseExpected. + +For example, for the following step with loaded requestParameter and responseExpected +```yaml +requestParameters: + subscriptionId: 00000000-0000-0000-0000-000000000000 + resourceGroupName: testGroup + resourceName: abc + api-version: 2021-01-01 + parameters: + properties: + a: abc +responseExpected: + id: /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/testGroup/providers/Microsoft.Contoso/SomeResource/abc + name: abc + type: Microsoft.Contoso/SomeResource + properties: + a: abc +``` + +And when we are on this step, we already have `subscriptionId`, `resourceGroupName` and `resourceName` in variables, then the step will be replaced with: + +```yaml +requestParameters: + subscriptionId: $(subscriptionId) + resourceGroupName: $(resourceGroupName) + resourceName: $(resourceName) + api-version: 2021-01-01 + parameters: + properties: + a: abc +responseExpected: + id: /subscriptions/$(subscriptionId)/resourceGroups/$(resourceGroupName)/providers/Microsoft.Contoso/SomeResource/$(resourceName) + name: $(resourceName) + type: Microsoft.Contoso/SomeResource + properties: + a: $(resourceName) +``` + +With this convention, you could control most of the parameters with variables. + +## Convention: location + +In one rest call step, if we have variable `location` (exact match) in the test scenario, and we have `location` as top level property defined in request body (`requestParameters[bodyParamName]`) and response body (responseExpected), then the top level location property will be replaced with variable value of location. + +For example, +```yaml +requestParameters: + parameters: + id: someId + location: westus +responseExpected: + id: someId + location: westus +``` + +When we have `location` variable defined, this step will be transformed to: + +```yaml +requestParameters: + parameters: + id: someId + location: $(location) +responseExpected: + id: someId + location: $(location) +``` + +## Convention: Arm Template Deployment + +When you deploy arm template in test scenario, you could define template parameters and outputs. By default if the parameter name matches the variable exists and the parameter type is string, then the parameter value would use the variable value. If the template has output which is string type, the variables will be set with output values. + +For example, given the following arm template: + +```json +{ + "parameters": { + "userName": { + "type": "string" + } + }, + "resources": [ + ], + "outputs": { + "nameResult": { + "type": "string", + "value": "[concat('prefix/', parameters['userName'])]" + } + } +} +``` + +If we have variable `userName` defined with `abc`, then we will have variable `nameResult` defined with value `prefix/abc` so that following steps in the test scenario could use variable `nameResult`. diff --git a/documentation/test-scenario/references/v1.0/schema.json b/documentation/test-scenario/references/v1.0/schema.json new file mode 100644 index 000000000000..8d31455e0eb5 --- /dev/null +++ b/documentation/test-scenario/references/v1.0/schema.json @@ -0,0 +1,374 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/VariableScope" + } + ], + "properties": { + "scope": { + "type": "string", + "enum": [ + "ResourceGroup" + ] + }, + "requiredVariables": { + "type": "array", + "items": { + "type": "string" + } + }, + "prepareSteps": { + "type": "array", + "items": { + "$ref": "#/definitions/TestStep" + } + }, + "testScenarios": { + "type": "array", + "items": { + "$ref": "#/definitions/TestScenario" + } + } + }, + "required": [ + "testScenarios" + ], + "definitions": { + "Name": { + "type": "string", + "pattern": "^[a-zA-Z0-9_-]+$" + }, + "JsonPointer": { + "type": "string", + "description": "String syntax for identifying a specific value within JSON document", + "pattern": "^(/(([^/~])|(~[01]))*)*$" + }, + "VariableScope": { + "type": "object", + "properties": { + "variables": { + "type": "object", + "additionalProperties": true + } + } + }, + "TestStep": { + "oneOf": [ + { + "$ref": "#/definitions/TestStepRestCall" + }, + { + "$ref": "#/definitions/TestStepArmTemplateDeployment" + }, + { + "$ref": "#/definitions/TestStepRawCall" + } + ] + }, + "TestStepBase": { + "properties": { + "step": { + "$ref": "#/definitions/Name" + }, + "type": { + "type": "string" + }, + "outputVariables": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "fromResponse": { + "type": "string" + } + } + } + } + }, + "required":[ + "step" + ] + }, + "TestStepArmTemplateDeployment": { + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/TestStepBase" + } + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "armTemplate" + ] + }, + "armTemplateDeployment": { + "type": "string" + }, + "armTemplateParameters": { + "type": "string" + } + }, + "required": [ + "armTemplateDeployment" + ] + }, + "TestStepRestCall": { + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/VariableScope" + }, + { + "$ref": "#/definitions/TestStepBase" + } + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "exampleFile" + ] + }, + "resourceName": { + "type": "string" + }, + "exampleFile": { + "type": "string" + }, + "resourceUpdate": { + "type": "array", + "items": { + "$ref": "#/definitions/JsonPatchOp" + }, + "minItems": 1 + }, + "requestUpdate": { + "type": "array", + "items": { + "$ref": "#/definitions/JsonPatchOp" + } + }, + "responseUpdate": { + "type": "array", + "items": { + "$ref": "#/definitions/JsonPatchOp" + } + }, + "operationId": { + "type": "string" + }, + "statusCode": { + "type": "number" + } + } + }, + "TestStepRawCall": { + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/VariableScope" + }, + { + "$ref": "#/definitions/TestStepBase" + } + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "rawCall" + ] + }, + "method": { + "type": "string", + "enum": [ + "GET", + "PUT", + "PATCH", + "POST", + "DELETE", + "OPTIONS", + "HEAD" + ] + }, + "url": { + "type": "string" + }, + "requestHeaders": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "requestBody": { + "type": "string" + }, + "statusCode": { + "type": "number" + }, + "responseExpected": { + "type": "string" + } + }, + "required": [ + "method", + "url", + "requestHeaders", + "requestBody" + ] + }, + "JsonPatchOp": { + "type": "object", + "oneOf": [ + { + "$ref": "#/definitions/JsonPatchOpAdd" + }, + { + "$ref": "#/definitions/JsonPatchOpRemove" + }, + { + "$ref": "#/definitions/JsonPatchOpReplace" + }, + { + "$ref": "#/definitions/JsonPatchOpCopy" + }, + { + "$ref": "#/definitions/JsonPatchOpMove" + }, + { + "$ref": "#/definitions/JsonPatchOpMerge" + } + ] + }, + "JsonPatchOpAdd": { + "type": "object", + "required": [ + "add", + "value" + ], + "properties": { + "add": { + "$ref": "#/definitions/JsonPointer" + }, + "value": {} + }, + "additionalProperties": false + }, + "JsonPatchOpRemove": { + "type": "object", + "required": [ + "remove" + ], + "properties": { + "remove": { + "$ref": "#/definitions/JsonPointer" + } + }, + "additionalProperties": false + }, + "JsonPatchOpReplace": { + "type": "object", + "required": [ + "replace", + "value" + ], + "properties": { + "replace": { + "$ref": "#/definitions/JsonPointer" + }, + "value": {} + }, + "additionalProperties": false + }, + "JsonPatchOpCopy": { + "type": "object", + "required": [ + "copy", + "path" + ], + "properties": { + "copy": { + "$ref": "#/definitions/JsonPointer" + }, + "path": { + "$ref": "#/definitions/JsonPointer" + } + }, + "additionalProperties": false + }, + "JsonPatchOpMove": { + "type": "object", + "required": [ + "move", + "path" + ], + "properties": { + "move": { + "$ref": "#/definitions/JsonPointer" + }, + "path": { + "$ref": "#/definitions/JsonPointer" + } + }, + "additionalProperties": false + }, + "JsonPatchOpMerge": { + "type": "object", + "required": [ + "merge", + "value" + ], + "properties": { + "merge": { + "$ref": "#/definitions/JsonPointer" + }, + "value": { + "type": "object", + "additionalProperties": true + } + }, + "additionalProperties": false + }, + "TestScenario": { + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/VariableScope" + } + ], + "properties": { + "scenario": { + "$ref": "#/definitions/Name", + "description": "Name of scenario" + }, + "description": { + "type": "string" + }, + "requiredVariables": { + "type": "array", + "items": { + "type": "string" + } + }, + "steps": { + "type": "array", + "items": { + "$ref": "#/definitions/TestStep" + } + }, + "dependsOn": { + "$ref": "#/definitions/Name", + "description": "Name of scenario that is depended on" + } + }, + "required": [ + "description", + "steps" + ] + } + } +}