From 84476bf5863739ae10325d91ba21fb05d0de20d3 Mon Sep 17 00:00:00 2001 From: Microsoft GitHub User Date: Wed, 26 Apr 2017 15:15:59 -0700 Subject: [PATCH 001/167] Initial commit --- .gitignore | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000000..72364f99fe4b --- /dev/null +++ b/.gitignore @@ -0,0 +1,89 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# IPython Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# dotenv +.env + +# virtualenv +venv/ +ENV/ + +# Spyder project settings +.spyderproject + +# Rope project settings +.ropeproject From 0eebaad6f8665e070c1ac9cc0962ea56dedcfe14 Mon Sep 17 00:00:00 2001 From: Microsoft Open Source Date: Wed, 26 Apr 2017 15:16:05 -0700 Subject: [PATCH 002/167] Initial commit --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000000..4b1ad51b2f0e --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE From c1c35c53182640b28087fedfb716110e4f6f1ebd Mon Sep 17 00:00:00 2001 From: Microsoft Open Source Date: Wed, 26 Apr 2017 15:16:06 -0700 Subject: [PATCH 003/167] Initial commit --- README.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 000000000000..8624b3d25cd6 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# Contributing + +This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. From bb5a0264b9cc9f9b515167591e5a329bd448aabb Mon Sep 17 00:00:00 2001 From: Troy Dai Date: Thu, 11 May 2017 15:05:58 -0700 Subject: [PATCH 004/167] wip: initial work - incomplete --- requirements.txt | 1 + setup.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 requirements.txt create mode 100644 setup.py diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 000000000000..02c0bd6a71ed --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +setuptools-markdown \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 000000000000..734de1c12e3f --- /dev/null +++ b/setup.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python + +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os.path +from setuptools import setup + + +VERSION = "0.1.0+dev" + + +CLASSIFIERS = [ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'License :: OSI Approved :: MIT License', +] + + +DEPENDENCIES = [ + 'setuptools-markdown' +] + +with open('README.rst', 'r', encoding='utf-8') as f: + README = f.read() + +with open('HISTORY.rst', 'r', encoding='utf-8') as f: + HISTORY = f.read() + +setup( + name='azure-devtools', + version=VERSION, + description='Microsoft Azure Developing Tools for SDK', + long_description_markdown_file='README.md' + license='MIT', + author='Microsoft Corporation', + author_email='azpycli@microsoft.com', + url='https://github.com/Azure/azure-cli', + zip_safe=False, + classifiers=CLASSIFIERS, + packages=[ + 'azure', + 'azure.devtools', + 'azure.devtools.automationsdk' + ], + install_requires=DEPENDENCIES, + cmdclass=cmdclass +) From e3dbb7a936bae8cf1a2f63d97ff24b0130eaddd9 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Fri, 12 May 2017 16:12:38 -0700 Subject: [PATCH 005/167] Copy code from https://github.com/Azure/azure-cli/tree/master/src/azure-cli-testsdk --- doc/recording_vcr_tests.md | 111 +++++ doc/scenario_base_tests.md | 179 ++++++++ src/scenario_tests/__init__.py | 17 + src/scenario_tests/base.py | 318 +++++++++++++ src/scenario_tests/checkers.py | 77 ++++ src/scenario_tests/const.py | 14 + src/scenario_tests/decorators.py | 20 + src/scenario_tests/exceptions.py | 25 + src/scenario_tests/patches.py | 80 ++++ src/scenario_tests/preparers.py | 256 +++++++++++ src/scenario_tests/recording_processors.py | 156 +++++++ src/scenario_tests/utilities.py | 36 ++ src/scenario_tests/vcr_test_base.py | 506 +++++++++++++++++++++ 13 files changed, 1795 insertions(+) create mode 100644 doc/recording_vcr_tests.md create mode 100644 doc/scenario_base_tests.md create mode 100644 src/scenario_tests/__init__.py create mode 100644 src/scenario_tests/base.py create mode 100644 src/scenario_tests/checkers.py create mode 100644 src/scenario_tests/const.py create mode 100644 src/scenario_tests/decorators.py create mode 100644 src/scenario_tests/exceptions.py create mode 100644 src/scenario_tests/patches.py create mode 100644 src/scenario_tests/preparers.py create mode 100644 src/scenario_tests/recording_processors.py create mode 100644 src/scenario_tests/utilities.py create mode 100644 src/scenario_tests/vcr_test_base.py diff --git a/doc/recording_vcr_tests.md b/doc/recording_vcr_tests.md new file mode 100644 index 000000000000..a2425a369fb9 --- /dev/null +++ b/doc/recording_vcr_tests.md @@ -0,0 +1,111 @@ +Recording Command Tests with VCR.py +======================================== + +Azure CLI uses the VCR.py library to record the HTTP messages exchanged during a program run and play them back at a later time, making it useful for creating command level scenario tests. These tests can be replayed at a later time without any network activity, allowing us to detect regressions in the handling of parameters and in the compatability between AzureCLI and the PythonSDK. + +## Overview + +Each command module has a `tests` folder with a file called: `test__commands.py`. This is where you will define tests. + +Tests all derive from the `VCRTestBase` class found in `azure.cli.core.test_utils.vcr_test_base`. This class exposes the VCR tests using the standard Python `unittest` framework and allows the tests to be discovered by and debugged in Visual Studio. + +The majority of tests however inherit from the `ResourceGroupVCRTestBase` class as this handles creating and tearing down the test resource group automatically, helping to ensure that tests can be recorded and cleaned up without manual creation or deletion of resources. + +After adding your test, run it. The test driver will automatically detect the test is unrecorded and record the HTTP requests and responses in a cassette .yaml file. If the test succeeds, the cassette will be preserved and future playthroughs of the test will come from the cassette. + +If the tests are run on TravisCI, any tests which cannot be replayed will automatically fail. + +## Authoring Tests + +To create a new test, simply create a class in the `test__commands.py` file with the following structure: + +```Python + +class MyTestClass(ResourceGroupVCRTestBase): # or VCRTestBase in special circumstances + + def __init__(self, test_method): + # TODO: replace MyTestClass with your class name + super(MyTestClass, self).__init__(__file__, test_method, debug=False, run_live=False, skip_setup=False, skip_teardown=False) + + def test_my_test_class(self): # TODO: rename to 'test_' + self.execute() + + def body(self): + # TODO: insert your test logic here + + def set_up(self): + super(MyTestClass, self).set_up() # if you need custom logic, be sure to call the base class version first + # TODO: Optional setup logic here (will not be replayed on playback) + + + def tear_down(self): + # TODO: Optional tear down logic here (will not be replayed on playback) + super(MyTestClass, self).tear_down() # if you need custom logic, call the base class version last +``` + +The `debug`, `run_live`, `skip_setup` and `skip_teardown` parameters in the `__init__` method are shown with their defaults and can be omitted. `debug` is the equivalent of specifying `debug=True` for all calls to `cmd` in the test (see below). Specifying `run_live=True` will cause the test to always be run with actual HTTP requests, ignoring VCR entirely. `skip_setup` and `skip_teardown` can be useful during test creation to avoid repeatedly creating and deleting resource groups. + +The `set_up` and `tear_down` methods are optional and can be omitted. For the ResourceGroupVCRTestBase these have default implementations which set up a test resource group and tear it down after the recording completes. Any commands used in these methods are only executed during a live or recorded test. These sections are skipped during playback so your test body should not rely any logic within these methods. + +A number of helper methods are available for structuring your script tests. + +#### cmd(command_string, checks=None, allowed_exceptions=None, debug=False) + +This method executes a given command and returns the output. If the output is in JSON format, the method will return the results as a JSON object for easier manuipulation. + +The `debug` parameter can be specified as `True` on a single call to `cmd` or in the init of the test class. Turning this on will print the command string, the results and, if a failure occurred, the failure. + +The `allowed_exceptions` parameter allows you to specify one or more (as a list) exception messages that will allow the test to still pass. Exception types are not used because the CLI wraps many types of errors in a `CLIError`. There are some tests where a specific exception is intended. Add the exception message to this list to allow the test to continue successfully in the presence of this message. + +The `checks` parameter allows you to specify one or more (as a list) checks to automatically validate the output. A number of Check objects exist for this purpose. You can create your own as long as they implement the compare method (see existing checks for examples): + +#### JMESPathCheck(query, expected_result) + +Use the JMESPathCheck object to validate the result using any valid JMESPath query. This is useful for checking that the JSON result has fields you were expecting, arrays of certain lengths etc. See www.jmespath.org for guidance on writing JMESPath queries. + +##### Usage +``` +JMESPathCheck(query, expected_result) +``` +- `query` - JMESPath query as a string. +- `expected_result` - The expected result from the JMESPath query (see [jmespath.search()](https://github.com/jmespath/jmespath.py#api)) + +##### Example + +The example below shows how you can use a JMESPath query to validate the values from a command. +When calling `test(command_string, checks)` you can pass in just one JMESPathComparator or a list of JMESPathComparators. + +```Python +self.cmd('vm list-ip-addresses --resource-group myResourceGroup', checks=[ + JMESPathCheck('length(@)', 1), + JMESPathCheck('[0].virtualMachine.name', 'myVMName') +]) +``` +#### NoneCheck() + +Use this to verify that the output contains nothing. Note that this is different from `checks=None` which will skip any validation. + +#### StringCheck(expected_result) + +Matches string output to expected. + +#### BooleanCheck(expected_result) + +Compares truthy responses (True, 'true', 1, etc.) to a Boolean True or False. + +#### set_env(variable_name, value) + +This method is a wrapper around `os.environ` and simply sets an environment variable to the specified value. + +#### pop_env(variable_name) + +Another wrapper around `os.environ` this pops the value of the indicated environment variable. + +## Test Issues + +Here are some common issues that occur when authoring tests that you should be aware of. + +- **Non-deterministic results**: If you find that a test will pass on some playbacks but fail on others, there are a couple possible things to check: + 1. check if your command makes use of concurrency. + 2. check your parameter aliasing (particularly if it complains that a required parameter is missing that you know is there) +- **Paths**: When including paths in your tests as parameter values, always wrap them in double quotes. While this isn't necessary when running from the command line (depending on your shell environment) it will likely cause issues with the test framework. diff --git a/doc/scenario_base_tests.md b/doc/scenario_base_tests.md new file mode 100644 index 000000000000..b35a9bf46dbc --- /dev/null +++ b/doc/scenario_base_tests.md @@ -0,0 +1,179 @@ +# How to write ScenarioTest based VCR test + +The `ScenarioTest` class is introduced in pull request [#2393](https://github.com/Azure/azure-cli/pull/2393). It is the preferred base class for and VCR based test cases from now on. The `ScenarioTest` class is designed to be a better and easier test harness for authoring scenario based VCR test. + +### Sample 1. Basic fixture +```Python +from azure.cli.testsdk import ScenarioTest + +class StorageAccountTests(ScenarioTest): + def test_list_storage_account(self): + self.cmd('az storage account list') +``` +Note: + +1. When the test is run without recording file, the test will be run under live mode. A recording file will be created at `recording/.yaml` +2. Wrap the command in `self.cmd` method. It will assert the exit code of the command to be zero. +3. All the functions and classes your need for writing tests are included in `azure.cli.testsdk` namespace. It is recommanded __not__ to refrenced to the sub-namespace to avoid breaking changes. + +### Sample 2. Validate the return value in JSON +``` Python +class StorageAccountTests(ScenarioTest): + def test_list_storage_account(self): + accounts_list = self.cmd('az storage account list').get_output_in_json() + assert len(accounts_list) > 0 +``` +Note: + +1. The return value of `self.cmd` is an instance of class `ExecutionResult`. It has the exit code and stdout as its properties. +2. `get_output_in_json` deserialize the output to a JSON object + +Tip: + +1. Don't make any rigid assertions based on any assumptions which may not stand in a live test environment. + + +### Sample 3. Validate the return JSON value using JMESPath +``` Python +from azure.cli.testsdk import ScenarioTest, JMESPathCheck + +class StorageAccountTests(ScenarioTest): + def test_list_storage_account(self): + self.cmd('az account list-locations', + checks=[JMESPathCheck("[?name=='westus'].displayName | [0]", 'West US')]) +``` +Note: + +1. What is JMESPath? [JMESPath is a query language for JSON](http://jmespath.org/) +2. If a command is return value in JSON, multiple JMESPath based check can be added to the checks list to validate the result. +3. In addition to the `JMESPatchCheck`, there are other checks list `NoneCheck` which validate the output is `None`. The check mechanism is extensible. Any callable accept `ExecutionResult` can act as a check. + + +### Sample 4. Prepare a resource group for a test +``` Python +from azure.cli.testsdk import ScenarioTest, JMESPathCheck, ResourceGroupPreparer + +class StorageAccountTests(ScenarioTest): + @ResourceGroupPreparer() + def test_create_storage_account(self, resource_group): + self.cmd('az group show -n {}'.format(resource_group), checks=[ + JMESPathCheck('name', resource_group), + JMESPathCheck('properties.provisioningState', 'Succeeded') + ]) +``` +Note: + +1. The preparers are executed in before each test in the test class when `setUp` is executed. The resource will be cleaned up after testing. +2. The resource group name is injected to the test method as a parameter. By default 'ResourceGroupPreparer' set the value to 'resource_group' parameter. The target parameter can be customized (see following samples). +3. The resource group will be deleted in async for performance reason. + + +### Sample 5. Get more from ResourceGroupPreparer +``` Python +class StorageAccountTests(ScenarioTest): + @ResourceGroupPreparer(parameter_name='group_name', parameter_name_for_location='group_location') + def test_create_storage_account(self, group_name, group_location): + self.cmd('az group show -n {}'.format(group_name), checks=[ + JMESPathCheck('name', group_name), + JMESPathCheck('location', group_location), + JMESPathCheck('properties.provisioningState', 'Succeeded') + ]) +``` +Note: + +1. In addition to the name, the location of the resource group can be also injected into the test method. +2. Both parameters' names can be customized. +3. The test method parameter accepting the location value is optional. The test harness will inspect the method signature and decide if the value will be added to the keyworded arguments. + + +### Sample 6. Random name and name mapping +``` Python +class StorageAccountTests(ScenarioTest): + @ResourceGroupPreparer(parameter_name_for_location='location') + def test_create_storage_account(self, resource_group, location): + name = self.create_random_name(prefix='cli', length=24) + self.cmd('az storage account create -n {} -g {} --sku {} -l {}'.format( + name, resource_group, 'Standard_LRS', location)) + self.cmd('az storage account show -n {} -g {}'.format(name, resource_group), checks=[ + JMESPathCheck('name', name), + JMESPathCheck('location', location), + JMESPathCheck('sku.name', 'Standard_LRS'), + JMESPathCheck('kind', 'Storage') + ]) +``` +Note: + +One of the most important features of `ScenarioTest` is names managements. For the tests to be able to run in a live environment and avoid name collision a strong name randomization is required. On the other hand, for the tests to be recorded and replay, the naming mechanism must be repeatable during playback mode. The `self.create_randome_name` assist the test to achieve the goal. + +The method will create a random name during recording, and when it is called during playback, it returns a name (internally it is called moniker) based on the sequence of the name request. The order won't change once the test is written. Peak into the recording file, you find no random name. For example, note the names like 'clitest.rg000001', they aren't the names of the resources which are actually created in Azure. They're placed before the requests are persisted. +``` Yaml +- request: + body: '{"location": "westus", "tags": {"use": "az-test"}}' + headers: + Accept: [application/json] + Accept-Encoding: ['gzip, deflate'] + CommandName: [group create] + Connection: [keep-alive] + Content-Length: ['50'] + Content-Type: [application/json; charset=utf-8] + User-Agent: [python/3.5.2 (Darwin-16.4.0-x86_64-i386-64bit) requests/2.9.1 msrest/0.4.6 + msrest_azure/0.4.7 resourcemanagementclient/0.30.2 Azure-SDK-For-Python + AZURECLI/2.0.0+dev] + accept-language: [en-US] + method: PUT + uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clitest.rg000001?api-version=2016-09-01 + response: + body: {string: '{"id":"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/clitest.rg000001","name":"clitest.rg000001","location":"westus","tags":{"use":"az-test"},"properties":{"provisioningState":"Succeeded"}}'} + headers: + cache-control: [no-cache] + content-length: ['326'] + content-type: [application/json; charset=utf-8] + date: ['Fri, 10 Mar 2017 17:59:58 GMT'] + expires: ['-1'] + pragma: [no-cache] + strict-transport-security: [max-age=31536000; includeSubDomains] + x-ms-ratelimit-remaining-subscription-writes: ['1199'] + status: {code: 201, message: Created} +``` + +In short, for the names of any Azure resources used in the tests, always use the `self.create_random_name` to generate its value. Also make sure the correct length is given to the method because different resource have different limitation of the name length. The method will always try to create the longest name possible to fully randomize the name. + + +### Sample 7. Prepare storage account for tests +``` Python +from azure.cli.testsdk import ScenarioTest, ResourceGroupPreparer, StorageAccountPreparer + +class StorageAccountTests(ScenarioTest): + @ResourceGroupPreparer() + @StorageAccountPreparer() + def test_list_storage_accounts(self, storage_account): + accounts = self.cmd('az storage account list').get_output_in_json() + search = [account for account in accounts if account['name'] == storage_account] + assert len(search) == 1 +``` +Note: + +1. Like `ResourceGroupPreparer` you can use `StorageAccountPreparer` to prepare a disposable storage account for the test. The account is deleted along with the resource group. +2. To create a storage account a resource group is required. Therefore `ResourceGroupPrepare` is needed to place above the `StorageAccountPreparer`. The preparers designed to be executed from top to bottom. (The core implementaiton of preparer is in the[AbstractPreparer](https://github.com/Azure/azure-cli/blob/master/src/azure-cli-testsdk/azure/cli/testsdk/preparers.py#L25)) +3. The preparers communicate among them by adding values to the `kwargs` of the decorated methods. Therefore the `StorageAccountPreparer` uses the resource group created in preceding `ResourceGroupPreparer`. +4. The `StorageAccountPreparer` can be further customized: +``` Python +@StorageAccountPreparer(sku='Standard_LRS', location='southcentralus', parameter_name='storage') +``` + +### Sampel 8. Prepare multiple storage accounts for tests +``` Python +class StorageAccountTests(ScenarioTest): + @ResourceGroupPreparer() + @StorageAccountPreparer(parameter_name='account_1') + @StorageAccountPreparer(parameter_name='account_2') + def test_list_storage_accounts(self, account_1, account_2): + accounts_list = self.cmd('az storage account list').get_output_in_json() + assert len(accounts_list) >= 2 + assert next(acc for acc in accounts_list if acc['name'] == account_1) + assert next(acc for acc in accounts_list if acc['name'] == account_2) +``` +Note: + +1. Two storage accounts name should be assigned to different function parameters. +2. The resource group name is not required in test so the function doesn't have to declare a parameter to accept the name. However it doesn't mean that the resource group is not created. Its name is in the keyworded parameter dictionary for all the preparer to consume. It is removed before the test function is actually invoked. diff --git a/src/scenario_tests/__init__.py b/src/scenario_tests/__init__.py new file mode 100644 index 000000000000..31a4f1c0e6b6 --- /dev/null +++ b/src/scenario_tests/__init__.py @@ -0,0 +1,17 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from .base import ScenarioTest, LiveTest +from .preparers import (StorageAccountPreparer, ResourceGroupPreparer, + RoleBasedServicePrincipalPreparer, KeyVaultPreparer) +from .exceptions import CliTestError +from .checkers import JMESPathCheck, JMESPathCheckExists, NoneCheck, StringCheck, StringContainCheck +from .decorators import live_only, record_only +from .utilities import get_sha1_hash + +__all__ = ['ScenarioTest', 'LiveTest', 'ResourceGroupPreparer', 'StorageAccountPreparer', + 'RoleBasedServicePrincipalPreparer', 'CliTestError', 'JMESPathCheck', 'JMESPathCheckExists', 'NoneCheck', + 'live_only', 'record_only', 'StringCheck', 'StringContainCheck', 'get_sha1_hash', 'KeyVaultPreparer'] +__version__ = '0.1.0+dev' diff --git a/src/scenario_tests/base.py b/src/scenario_tests/base.py new file mode 100644 index 000000000000..dfc843d4a6d2 --- /dev/null +++ b/src/scenario_tests/base.py @@ -0,0 +1,318 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from __future__ import print_function +import datetime +import unittest +import os +import inspect +import subprocess +import json +import shlex +import tempfile +import shutil +import logging +import six +import vcr + +from .patches import (patch_load_cached_subscriptions, patch_main_exception_handler, + patch_retrieve_token_for_user, patch_long_run_operation_delay, + patch_time_sleep_api) +from .exceptions import CliExecutionError +from .const import (ENV_LIVE_TEST, ENV_SKIP_ASSERT, ENV_TEST_DIAGNOSE, MOCKED_SUBSCRIPTION_ID) +from .recording_processors import (SubscriptionRecordingProcessor, OAuthRequestResponsesFilter, + GeneralNameReplacer, LargeRequestBodyProcessor, + LargeResponseBodyProcessor, LargeResponseBodyReplacer, + DeploymentNameReplacer) +from .utilities import create_random_name +from .decorators import live_only + +logger = logging.getLogger('azuer.cli.testsdk') + + +class IntegrationTestBase(unittest.TestCase): + def __init__(self, method_name): + super(IntegrationTestBase, self).__init__(method_name) + self.diagnose = os.environ.get(ENV_TEST_DIAGNOSE, None) == 'True' + + def cmd(self, command, checks=None, expect_failure=False): + if self.diagnose: + begin = datetime.datetime.now() + print('\nExecuting command: {}'.format(command)) + + result = execute(command, expect_failure=expect_failure) + + if self.diagnose: + duration = datetime.datetime.now() - begin + print('\nCommand accomplished in {} s. Exit code {}.\n{}'.format( + duration.total_seconds(), result.exit_code, result.output)) + + return result.assert_with_checks(checks) + + def create_random_name(self, prefix, length): # pylint: disable=no-self-use + return create_random_name(prefix=prefix, length=length) + + def create_temp_file(self, size_kb, full_random=False): + """ + Create a temporary file for testing. The test harness will delete the file during tearing + down. + """ + fd, path = tempfile.mkstemp() + os.close(fd) + self.addCleanup(lambda: os.remove(path)) + + with open(path, mode='r+b') as f: + if full_random: + chunk = os.urandom(1024) + else: + chunk = bytearray([0] * 1024) + for _ in range(size_kb): + f.write(chunk) + + return path + + def create_temp_dir(self): + """ + Create a temporary directory for testing. The test harness will delete the directory during + tearing down. + """ + temp_dir = tempfile.mkdtemp() + self.addCleanup(lambda: shutil.rmtree(temp_dir, ignore_errors=True)) + + return temp_dir + + @classmethod + def set_env(cls, key, val): + os.environ[key] = val + + @classmethod + def pop_env(cls, key): + return os.environ.pop(key, None) + + +@live_only() +class LiveTest(IntegrationTestBase): + pass + + +class ScenarioTest(IntegrationTestBase): # pylint: disable=too-many-instance-attributes + FILTER_HEADERS = [ + 'authorization', + 'client-request-id', + 'x-ms-client-request-id', + 'x-ms-correlation-request-id', + 'x-ms-ratelimit-remaining-subscription-reads', + 'x-ms-request-id', + 'x-ms-routing-request-id', + 'x-ms-gateway-service-instanceid', + 'x-ms-ratelimit-remaining-tenant-reads', + 'x-ms-served-by', + ] + + def __init__(self, method_name): + super(ScenarioTest, self).__init__(method_name) + self.name_replacer = GeneralNameReplacer() + self.recording_processors = [SubscriptionRecordingProcessor(MOCKED_SUBSCRIPTION_ID), + OAuthRequestResponsesFilter(), + LargeRequestBodyProcessor(), + LargeResponseBodyProcessor(), + DeploymentNameReplacer(), + self.name_replacer] + self.replay_processors = [LargeResponseBodyReplacer(), DeploymentNameReplacer()] + + test_file_path = inspect.getfile(self.__class__) + recordings_dir = os.path.join(os.path.dirname(test_file_path), 'recordings') + live_test = os.environ.get(ENV_LIVE_TEST, None) == 'True' + + self.vcr = vcr.VCR( + cassette_library_dir=recordings_dir, + before_record_request=self._process_request_recording, + before_record_response=self._process_response_recording, + decode_compressed_response=True, + record_mode='once' if not live_test else 'all', + filter_headers=self.FILTER_HEADERS + ) + self.vcr.register_matcher('query', self._custom_request_query_matcher) + + self.recording_file = os.path.join(recordings_dir, '{}.yaml'.format(method_name)) + if live_test and os.path.exists(self.recording_file): + os.remove(self.recording_file) + + self.in_recording = live_test or not os.path.exists(self.recording_file) + self.test_resources_count = 0 + self.original_env = os.environ.copy() + + def setUp(self): + super(ScenarioTest, self).setUp() + + # set up cassette + cm = self.vcr.use_cassette(self.recording_file) + self.cassette = cm.__enter__() + self.addCleanup(cm.__exit__) + + # set up mock patches + patch_main_exception_handler(self) + + if not self.in_recording: + patch_time_sleep_api(self) + patch_long_run_operation_delay(self) + patch_load_cached_subscriptions(self) + patch_retrieve_token_for_user(self) + + def tearDown(self): + os.environ = self.original_env + + def create_random_name(self, prefix, length): + self.test_resources_count += 1 + moniker = '{}{:06}'.format(prefix, self.test_resources_count) + + if self.in_recording: + name = create_random_name(prefix, length) + self.name_replacer.register_name_pair(name, moniker) + return name + else: + return moniker + + def _process_request_recording(self, request): + if self.in_recording: + for processor in self.recording_processors: + request = processor.process_request(request) + if not request: + break + else: + for processor in self.replay_processors: + request = processor.process_request(request) + if not request: + break + + return request + + def _process_response_recording(self, response): + if self.in_recording: + # make header name lower case and filter unwanted headers + headers = {} + for key in response['headers']: + if key.lower() not in self.FILTER_HEADERS: + headers[key.lower()] = response['headers'][key] + response['headers'] = headers + + body = response['body']['string'] + if body and not isinstance(body, six.string_types): + response['body']['string'] = body.decode('utf-8') + + for processor in self.recording_processors: + response = processor.process_response(response) + if not response: + break + else: + for processor in self.replay_processors: + response = processor.process_response(response) + if not response: + break + + return response + + @classmethod + def _custom_request_query_matcher(cls, r1, r2): + """ Ensure method, path, and query parameters match. """ + from six.moves.urllib_parse import urlparse, parse_qs # pylint: disable=import-error + + url1 = urlparse(r1.uri) + url2 = urlparse(r2.uri) + + q1 = parse_qs(url1.query) + q2 = parse_qs(url2.query) + shared_keys = set(q1.keys()).intersection(set(q2.keys())) + + if len(shared_keys) != len(q1) or len(shared_keys) != len(q2): + return False + + for key in shared_keys: + if q1[key][0].lower() != q2[key][0].lower(): + return False + + return True + + +class ExecutionResult(object): # pylint: disable=too-few-public-methods + def __init__(self, command, expect_failure=False, in_process=True): + logger.info('Execute command %s', command) + if in_process: + self._in_process_execute(command) + else: + self._out_of_process_execute(command) + + if expect_failure and self.exit_code == 0: + raise AssertionError('The command is expected to fail but it doesn\'.') + elif not expect_failure and self.exit_code != 0: + raise AssertionError('The command failed. Exit code: {}'.format(self.exit_code)) + + self.json_value = None + self.skip_assert = os.environ.get(ENV_SKIP_ASSERT, None) == 'True' + + def assert_with_checks(self, *args): + checks = [] + for each in args: + if isinstance(each, list): + checks.extend(each) + elif callable(each): + checks.append(each) + + logger.info('Checkers to be executed %s', len(checks)) + + if not self.skip_assert: + for c in checks: + c(self) + + return self + + def get_output_in_json(self): + if not self.json_value: + self.json_value = json.loads(self.output) + + if self.json_value is None: + raise AssertionError('The command output cannot be parsed in json.') + + return self.json_value + + def _in_process_execute(self, command): + # from azure.cli import as cli_main + from azure.cli.main import main as cli_main + from six import StringIO + from vcr.errors import CannotOverwriteExistingCassetteException + + if command.startswith('az '): + command = command[3:] + + output_buffer = StringIO() + try: + # issue: stderr cannot be redirect in this form, as a result some failure information + # is lost when command fails. + self.exit_code = cli_main(shlex.split(command), file=output_buffer) or 0 + self.output = output_buffer.getvalue() + except CannotOverwriteExistingCassetteException as ex: + raise AssertionError(ex) + except CliExecutionError as ex: + if ex.exception: + raise ex.exception + else: + raise ex + except Exception as ex: # pylint: disable=broad-except + self.exit_code = 1 + self.output = output_buffer.getvalue() + self.process_error = ex + finally: + output_buffer.close() + + def _out_of_process_execute(self, command): + try: + self.output = subprocess.check_output(shlex.split(command)).decode('utf-8') + self.exit_code = 0 + except subprocess.CalledProcessError as error: + self.exit_code, self.output = error.returncode, error.output.decode('utf-8') + self.process_error = error + + +execute = ExecutionResult diff --git a/src/scenario_tests/checkers.py b/src/scenario_tests/checkers.py new file mode 100644 index 000000000000..09044a911143 --- /dev/null +++ b/src/scenario_tests/checkers.py @@ -0,0 +1,77 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import collections +import jmespath +from .exceptions import JMESPathCheckAssertionError + + +class JMESPathCheck(object): # pylint: disable=too-few-public-methods + def __init__(self, query, expected_result): + self._query = query + self._expected_result = expected_result + + def __call__(self, execution_result): + json_value = execution_result.get_output_in_json() + actual_result = jmespath.search(self._query, json_value, + jmespath.Options(collections.OrderedDict)) + if not actual_result == self._expected_result: + if actual_result: + raise JMESPathCheckAssertionError(self._query, self._expected_result, actual_result, + execution_result.output) + else: + raise JMESPathCheckAssertionError(self._query, self._expected_result, 'None', + execution_result.output) + + +class JMESPathCheckExists(object): # pylint: disable=too-few-public-methods + def __init__(self, query): + self._query = query + + def __call__(self, execution_result): + json_value = execution_result.get_output_in_json() + actual_result = jmespath.search(self._query, json_value, + jmespath.Options(collections.OrderedDict)) + if not actual_result: + raise JMESPathCheckAssertionError(self._query, 'some value', actual_result, + execution_result.output) + + +class NoneCheck(object): # pylint: disable=too-few-public-methods + def __call__(self, execution_result): # pylint: disable=no-self-use + none_strings = ['[]', '{}', 'false'] + try: + data = execution_result.output.strip() + assert not data or data in none_strings + except AssertionError: + raise AssertionError("Actual value '{}' != Expected value falsy (None, '', []) or " + "string in {}".format(data, none_strings)) + + +class StringCheck(object): # pylint: disable=too-few-public-methods + def __init__(self, expected_result): + self.expected_result = expected_result + + def __call__(self, execution_result): + try: + result = execution_result.output.strip().strip('"') + assert result == self.expected_result + except AssertionError: + raise AssertionError( + "Actual value '{}' != Expected value {}".format(result, self.expected_result)) + + +class StringContainCheck(object): # pylint: disable=too-few-public-methods + def __init__(self, expected_result): + self.expected_result = expected_result + + def __call__(self, execution_result): + try: + result = execution_result.output.strip('"') + assert self.expected_result in result + except AssertionError: + raise AssertionError( + "Actual value '{}' doesn't contain Expected value {}".format(result, + self.expected_result)) diff --git a/src/scenario_tests/const.py b/src/scenario_tests/const.py new file mode 100644 index 000000000000..624bac5f84c6 --- /dev/null +++ b/src/scenario_tests/const.py @@ -0,0 +1,14 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +# Replaced mock values +MOCKED_SUBSCRIPTION_ID = '00000000-0000-0000-0000-000000000000' +MOCKED_TENANT_ID = '00000000-0000-0000-0000-000000000000' + +# Configuration environment variable +ENV_COMMAND_COVERAGE = 'AZURE_CLI_TEST_COMMAND_COVERAGE' +ENV_LIVE_TEST = 'AZURE_CLI_TEST_RUN_LIVE' +ENV_SKIP_ASSERT = 'AZURE_CLI_TEST_SKIP_ASSERT' +ENV_TEST_DIAGNOSE = 'AZURE_CLI_TEST_DIAGNOSE' diff --git a/src/scenario_tests/decorators.py b/src/scenario_tests/decorators.py new file mode 100644 index 000000000000..43be70932d15 --- /dev/null +++ b/src/scenario_tests/decorators.py @@ -0,0 +1,20 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +import unittest +from .const import ENV_LIVE_TEST + + +def live_only(): + return unittest.skipUnless( + os.environ.get(ENV_LIVE_TEST, False), + 'This is a live only test. A live test will bypass all vcrpy components.') + + +def record_only(): + return unittest.skipUnless( + not os.environ.get(ENV_LIVE_TEST, False), + 'This test is excluded from being run live. To force a recording, please remove the recording file.') diff --git a/src/scenario_tests/exceptions.py b/src/scenario_tests/exceptions.py new file mode 100644 index 000000000000..f490a6d8e4f6 --- /dev/null +++ b/src/scenario_tests/exceptions.py @@ -0,0 +1,25 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + + +class CliTestError(Exception): + def __init__(self, error_message): + message = 'An error caused by the CLI test harness failed the test: {}' + super(CliTestError, self).__init__(message.format(error_message)) + + +class CliExecutionError(Exception): + def __init__(self, exception): + self.exception = exception + message = 'The CLI throws exception {} during execution and fails the command.' + super(CliExecutionError, self).__init__(message.format(exception.__class__.__name__, + exception)) + + +class JMESPathCheckAssertionError(AssertionError): + def __init__(self, query, expected, actual, json_data): + message = "Query '{}' doesn't yield expected value '{}', instead the actual value " \ + "is '{}'. Data: \n{}\n".format(query, expected, actual, json_data) + super(JMESPathCheckAssertionError, self).__init__(message) diff --git a/src/scenario_tests/patches.py b/src/scenario_tests/patches.py new file mode 100644 index 000000000000..54a351aa3df8 --- /dev/null +++ b/src/scenario_tests/patches.py @@ -0,0 +1,80 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +from .exceptions import CliExecutionError, CliTestError +from .const import MOCKED_SUBSCRIPTION_ID, MOCKED_TENANT_ID + + +def patch_main_exception_handler(unit_test): + from vcr.errors import CannotOverwriteExistingCassetteException + + def _handle_main_exception(ex): + if isinstance(ex, CannotOverwriteExistingCassetteException): + # This exception usually caused by a no match HTTP request. This is a product error + # that is caused by change of SDK invocation. + raise ex + + raise CliExecutionError(ex) + + _mock_in_unit_test(unit_test, 'azure.cli.main.handle_exception', _handle_main_exception) + + +def patch_load_cached_subscriptions(unit_test): + def _handle_load_cached_subscription(*args, **kwargs): # pylint: disable=unused-argument + + return [{ + "id": MOCKED_SUBSCRIPTION_ID, + "user": { + "name": "example@example.com", + "type": "user" + }, + "state": "Enabled", + "name": "Example", + "tenantId": MOCKED_TENANT_ID, + "isDefault": True}] + + _mock_in_unit_test(unit_test, + 'azure.cli.core._profile.Profile.load_cached_subscriptions', + _handle_load_cached_subscription) + + +def patch_retrieve_token_for_user(unit_test): + def _retrieve_token_for_user(*args, **kwargs): # pylint: disable=unused-argument + return 'Bearer', 'top-secret-token-for-you' + + _mock_in_unit_test(unit_test, + 'azure.cli.core._profile.CredsCache.retrieve_token_for_user', + _retrieve_token_for_user) + + +def patch_long_run_operation_delay(unit_test): + def _shortcut_long_run_operation(*args, **kwargs): # pylint: disable=unused-argument + return + + _mock_in_unit_test(unit_test, + 'msrestazure.azure_operation.AzureOperationPoller._delay', + _shortcut_long_run_operation) + _mock_in_unit_test(unit_test, + 'azure.cli.core.commands.LongRunningOperation._delay', + _shortcut_long_run_operation) + + +def patch_time_sleep_api(unit_test): + def _time_sleep_skip(*_): + return + + _mock_in_unit_test(unit_test, 'time.sleep', _time_sleep_skip) + + +def _mock_in_unit_test(unit_test, target, replacement): + import mock + import unittest + + if not isinstance(unit_test, unittest.TestCase): + raise CliTestError('The patch_main_exception_handler can be only used in unit test') + + mp = mock.patch(target, replacement) + mp.__enter__() + unit_test.addCleanup(mp.__exit__) diff --git a/src/scenario_tests/preparers.py b/src/scenario_tests/preparers.py new file mode 100644 index 000000000000..7255349c6f46 --- /dev/null +++ b/src/scenario_tests/preparers.py @@ -0,0 +1,256 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import inspect +import functools +import os + +from .base import ScenarioTest, execute +from .exceptions import CliTestError +from .utilities import create_random_name +from .recording_processors import RecordingProcessor + + +# Core Utility + +class AbstractPreparer(object): + def __init__(self, name_prefix, name_len): + self.name_prefix = name_prefix + self.name_len = name_len + self.resource_moniker = None + self.resource_random_name = None + self.test_class_instance = None + self.live_test = False + + def __call__(self, fn): + def _preparer_wrapper(test_class_instance, **kwargs): + self.live_test = not isinstance(test_class_instance, ScenarioTest) + self.test_class_instance = test_class_instance + + if self.live_test or test_class_instance.in_recording: + resource_name = self.random_name + if not self.live_test and isinstance(self, RecordingProcessor): + test_class_instance.recording_processors.append(self) + else: + resource_name = self.moniker + + parameter_update = self.create_resource(resource_name, **kwargs) + test_class_instance.addCleanup(lambda: self.remove_resource(resource_name, **kwargs)) + + if parameter_update: + kwargs.update(parameter_update) + + if not is_preparer_func(fn): + # the next function is the actual test function. the kwargs need to be trimmed so + # that parameters which are not required will not be passed to it. + args, _, kw, _ = inspect.getargspec(fn) # pylint: disable=deprecated-method + if kw is None: + args = set(args) + for key in [k for k in kwargs.keys() if k not in args]: + del kwargs[key] + + fn(test_class_instance, **kwargs) + + setattr(_preparer_wrapper, '__is_preparer', True) + functools.update_wrapper(_preparer_wrapper, fn) + return _preparer_wrapper + + @property + def moniker(self): + if not self.resource_moniker: + self.test_class_instance.test_resources_count += 1 + self.resource_moniker = '{}{:06}'.format(self.name_prefix, + self.test_class_instance.test_resources_count) + return self.resource_moniker + + @property + def random_name(self): + if not self.resource_random_name: + self.resource_random_name = create_random_name(self.name_prefix, self.name_len) + return self.resource_random_name + + def create_resource(self, name, **kwargs): # pylint: disable=unused-argument,no-self-use + return {} + + def remove_resource(self, name, **kwargs): # pylint: disable=unused-argument + pass + + +# TODO: replaced by GeneralNameReplacer +class SingleValueReplacer(RecordingProcessor): + # pylint: disable=no-member + def process_request(self, request): + from six.moves.urllib_parse import quote_plus # pylint: disable=import-error + if self.random_name in request.uri: + request.uri = request.uri.replace(self.random_name, self.moniker) + elif quote_plus(self.random_name) in request.uri: + request.uri = request.uri.replace(quote_plus(self.random_name), + quote_plus(self.moniker)) + + if request.body: + body = str(request.body) + if self.random_name in body: + request.body = body.replace(self.random_name, self.moniker) + + return request + + def process_response(self, response): + if response['body']['string']: + response['body']['string'] = response['body']['string'].replace(self.random_name, + self.moniker) + + self.replace_header(response, 'location', self.random_name, self.moniker) + self.replace_header(response, 'azure-asyncoperation', self.random_name, self.moniker) + + return response + + +# Resource Group Preparer and its shorthand decorator + +class ResourceGroupPreparer(AbstractPreparer, SingleValueReplacer): + def __init__(self, name_prefix='clitest.rg', + parameter_name='resource_group', + parameter_name_for_location='resource_group_location', location='westus', + dev_setting_name='AZURE_CLI_TEST_DEV_RESOURCE_GROUP_NAME', + dev_setting_location='AZURE_CLI_TEST_DEV_RESOURCE_GROUP_LOCATION', + random_name_length=75): + super(ResourceGroupPreparer, self).__init__(name_prefix, random_name_length) + self.location = location + self.parameter_name = parameter_name + self.parameter_name_for_location = parameter_name_for_location + + self.dev_setting_name = os.environ.get(dev_setting_name, None) + self.dev_setting_location = os.environ.get(dev_setting_location, location) + + def create_resource(self, name, **kwargs): + if self.dev_setting_name: + return {self.parameter_name: self.dev_setting_name, + self.parameter_name_for_location: self.dev_setting_location} + else: + template = 'az group create --location {} --name {} --tag use=az-test' + execute(template.format(self.location, name)) + return {self.parameter_name: name, self.parameter_name_for_location: self.location} + + def remove_resource(self, name, **kwargs): + if not self.dev_setting_name: + execute('az group delete --name {} --yes --no-wait'.format(name)) + + +# Storage Account Preparer and its shorthand decorator + +class StorageAccountPreparer(AbstractPreparer, SingleValueReplacer): + def __init__(self, + name_prefix='clitest', sku='Standard_LRS', location='westus', + parameter_name='storage_account', resource_group_parameter_name='resource_group', + skip_delete=True, dev_setting_name='AZURE_CLI_TEST_DEV_STORAGE_ACCOUNT_NAME'): + super(StorageAccountPreparer, self).__init__(name_prefix, 24) + self.location = location + self.sku = sku + self.resource_group_parameter_name = resource_group_parameter_name + self.skip_delete = skip_delete + self.parameter_name = parameter_name + + self.dev_setting_name = os.environ.get(dev_setting_name, None) + + def create_resource(self, name, **kwargs): + group = self._get_resource_group(**kwargs) + + if not self.dev_setting_name: + template = 'az storage account create -n {} -g {} -l {} --sku {}' + execute(template.format(name, group, self.location, self.sku)) + else: + name = self.dev_setting_name + + account_key = execute('storage account keys list -n {} -g {} --query "[0].value" -otsv' + .format(name, group)).output + return {self.parameter_name: name, self.parameter_name + '_info': (name, account_key)} + + def remove_resource(self, name, **kwargs): + if not self.skip_delete and not self.dev_setting_name: + group = self._get_resource_group(**kwargs) + execute('az storage account delete -n {} -g {} --yes'.format(name, group)) + + def _get_resource_group(self, **kwargs): + try: + return kwargs.get(self.resource_group_parameter_name) + except KeyError: + template = 'To create a storage account a resource group is required. Please add ' \ + 'decorator @{} in front of this storage account preparer.' + raise CliTestError(template.format(ResourceGroupPreparer.__name__)) + + +# KeyVault Preparer and its shorthand decorator + +class KeyVaultPreparer(AbstractPreparer, SingleValueReplacer): + def __init__(self, # pylint: disable=too-many-arguments + name_prefix='clitest', sku='standard', location='westus', + parameter_name='key_vault', resource_group_parameter_name='resource_group', + skip_delete=True, dev_setting_name='AZURE_CLI_TEST_DEV_KEY_VAULT_NAME'): + super(KeyVaultPreparer, self).__init__(name_prefix, 24) + self.location = location + self.sku = sku + self.resource_group_parameter_name = resource_group_parameter_name + self.skip_delete = skip_delete + self.parameter_name = parameter_name + + self.dev_setting_name = os.environ.get(dev_setting_name, None) + + def create_resource(self, name, **kwargs): + if not self.dev_setting_name: + group = self._get_resource_group(**kwargs) + template = 'az keyvault create -n {} -g {} -l {} --sku {}' + execute(template.format(name, group, self.location, self.sku)) + return {self.parameter_name: name} + else: + return {self.parameter_name: self.dev_setting_name} + + def remove_resource(self, name, **kwargs): + if not self.skip_delete and not self.dev_setting_name: + group = self._get_resource_group(**kwargs) + execute('az keyvault delete -n {} -g {} --yes'.format(name, group)) + + def _get_resource_group(self, **kwargs): + try: + return kwargs.get(self.resource_group_parameter_name) + except KeyError: + template = 'To create a KeyVault a resource group is required. Please add ' \ + 'decorator @{} in front of this KeyVault preparer.' + raise CliTestError(template.format(KeyVaultPreparer.__name__)) + + +# Role based access control service principal preparer + +class RoleBasedServicePrincipalPreparer(AbstractPreparer, SingleValueReplacer): + def __init__(self, name_prefix='http://clitest', + skip_assignment=True, parameter_name='sp_name', parameter_password='sp_password', + dev_setting_sp_name='AZURE_CLI_TEST_DEV_SP_NAME', + dev_setting_sp_password='AZURE_CLI_TEST_DEV_SP_PASSWORD'): + super(RoleBasedServicePrincipalPreparer, self).__init__(name_prefix, 24) + self.skip_assignment = skip_assignment + self.result = {} + self.parameter_name = parameter_name + self.parameter_password = parameter_password + self.dev_setting_sp_name = os.environ.get(dev_setting_sp_name, None) + self.dev_setting_sp_password = os.environ.get(dev_setting_sp_password, None) + + def create_resource(self, name, **kwargs): + if not self.dev_setting_sp_name: + command = 'az ad sp create-for-rbac -n {}{}' \ + .format(name, ' --skip-assignment' if self.skip_assignment else '') + self.result = execute(command).get_output_in_json() + return {self.parameter_name: name, self.parameter_password: self.result['password']} + else: + return {self.parameter_name: self.dev_setting_sp_name, + self.parameter_password: self.dev_setting_sp_password} + + def remove_resource(self, name, **kwargs): + if not self.dev_setting_sp_name: + execute('az ad sp delete --id {}'.format(self.result['appId'])) + + +# Utility + +def is_preparer_func(fn): + return getattr(fn, '__is_preparer', False) diff --git a/src/scenario_tests/recording_processors.py b/src/scenario_tests/recording_processors.py new file mode 100644 index 000000000000..7e33b689d7d5 --- /dev/null +++ b/src/scenario_tests/recording_processors.py @@ -0,0 +1,156 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + + +class RecordingProcessor(object): + def process_request(self, request): # pylint: disable=no-self-use + return request + + def process_response(self, response): # pylint: disable=no-self-use + return response + + @classmethod + def replace_header(cls, entity, header, old, new): + cls.replace_header_fn(entity, header, lambda v: v.replace(old, new)) + + @classmethod + def replace_header_fn(cls, entity, header, replace_fn): + try: + header = header.lower() + values = entity['headers'][header] + entity['headers'][header] = [replace_fn(v) for v in values] + except KeyError: + pass + + +class SubscriptionRecordingProcessor(RecordingProcessor): + def __init__(self, replacement): + self._replacement = replacement + + def process_request(self, request): + request.uri = self._replace_subscription_id(request.uri) + return request + + def process_response(self, response): + if response['body']['string']: + response['body']['string'] = self._replace_subscription_id(response['body']['string']) + + self.replace_header_fn(response, 'location', self._replace_subscription_id) + self.replace_header_fn(response, 'azure-asyncoperation', self._replace_subscription_id) + + return response + + def _replace_subscription_id(self, val): + import re + # subscription presents in all api call + retval = re.sub('/subscriptions/([^/]+)/', + '/subscriptions/{}/'.format(self._replacement), + val) + + # subscription is also used in graph call + retval = re.sub('https://graph.windows.net/([^/]+)/', + 'https://graph.windows.net/{}/'.format(self._replacement), + retval) + return retval + + +class LargeRequestBodyProcessor(RecordingProcessor): + def __init__(self, max_request_body=128): + self._max_request_body = max_request_body + + def process_request(self, request): + if request.body and len(request.body) > self._max_request_body * 1024: + request.body = '!!! The request body has been omitted from the recording because its ' \ + 'size {} is larger than {}KB. !!!'.format(len(request.body), + self._max_request_body) + + return request + + +class LargeResponseBodyProcessor(RecordingProcessor): + control_flag = '' + + def __init__(self, max_response_body=128): + self._max_response_body = max_response_body + + def process_response(self, response): + length = len(response['body']['string'] or '') + if length > self._max_response_body * 1024: + response['body']['string'] = \ + "!!! The response body has been omitted from the recording because it is larger " \ + "than {} KB. It will be replaced with blank content of {} bytes while replay. " \ + "{}{}".format(self._max_response_body, length, self.control_flag, length) + + return response + + +class LargeResponseBodyReplacer(RecordingProcessor): + def process_response(self, response): + import six + body = response['body']['string'] + + # backward compatibility. under 2.7 response body is unicode, under 3.5 response body is + # bytes. when set the value back, the same type must be used. + body_is_string = isinstance(body, six.string_types) + + content_in_string = (response['body']['string'] or b'').decode('utf-8') + index = content_in_string.find(LargeResponseBodyProcessor.control_flag) + + if index > -1: + length = int(content_in_string[index + len(LargeResponseBodyProcessor.control_flag):]) + if body_is_string: + response['body']['string'] = '0' * length + else: + response['body']['string'] = bytes([0] * length) + + return response + + +class OAuthRequestResponsesFilter(RecordingProcessor): + """Remove oauth authentication requests and responses from recording.""" + + def process_request(self, request): + # filter request like: + # GET https://login.microsoftonline.com/72f988bf-86f1-41af-91ab-2d7cd011db47/oauth2/token + import re + if not re.match('https://login.microsoftonline.com/([^/]+)/oauth2/token', request.uri): + return request + + +class DeploymentNameReplacer(RecordingProcessor): + """Replace the random deployment name with a fixed mock name.""" + def process_request(self, request): + import re + request.uri = re.sub('/deployments/([^/?]+)', '/deployments/mock-deployment', request.uri) + return request + + +class GeneralNameReplacer(RecordingProcessor): + def __init__(self): + self.names_name = [] + + def register_name_pair(self, old, new): + self.names_name.append((old, new)) + + def process_request(self, request): + for old, new in self.names_name: + request.uri = request.uri.replace(old, new) + + if request.body: + body = str(request.body) + if old in body: + request.body = body.replace(old, new) + + return request + + def process_response(self, response): + for old, new in self.names_name: + if response['body']['string']: + response['body']['string'] = response['body']['string'].replace(old, new) + + self.replace_header(response, 'location', old, new) + self.replace_header(response, 'azure-asyncoperation', old, new) + + return response diff --git a/src/scenario_tests/utilities.py b/src/scenario_tests/utilities.py new file mode 100644 index 000000000000..947f6fe2e393 --- /dev/null +++ b/src/scenario_tests/utilities.py @@ -0,0 +1,36 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import hashlib +import math +import os +import base64 + + +def create_random_name(prefix='clitest', length=24): + if len(prefix) > length: + raise 'The length of the prefix must not be longer than random name length' + + padding_size = length - len(prefix) + if padding_size < 4: + raise 'The randomized part of the name is shorter than 4, which may not be able to offer ' \ + 'enough randomness' + + random_bytes = os.urandom(int(math.ceil(float(padding_size) / 8) * 5)) + random_padding = base64.b32encode(random_bytes)[:padding_size] + + return str(prefix + random_padding.decode().lower()) + + +def get_sha1_hash(file_path): + sha1 = hashlib.sha256() + with open(file_path, 'rb') as f: + while True: + data = f.read(65536) + if not data: + break + sha1.update(data) + + return sha1.hexdigest() diff --git a/src/scenario_tests/vcr_test_base.py b/src/scenario_tests/vcr_test_base.py new file mode 100644 index 000000000000..9749fee81418 --- /dev/null +++ b/src/scenario_tests/vcr_test_base.py @@ -0,0 +1,506 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +# pylint: disable=wrong-import-order + +from __future__ import print_function + +import collections +import json +import os +import re +import shlex +import sys +import tempfile +import traceback +from random import choice +from string import digits, ascii_lowercase + +from six.moves.urllib.parse import urlparse, parse_qs # pylint: disable=import-error + +import unittest +try: + import unittest.mock as mock +except ImportError: + import mock + +import vcr +import jmespath +from six import StringIO + +# TODO Should not depend on azure.cli.main package here. +# Will be ok if this test file is not part of azure.cli.core.utils +from azure.cli.main import main as cli_main + +from azure.cli.core import __version__ as core_version +import azure.cli.core._debug as _debug +from azure.cli.core._profile import Profile, CLOUD +from azure.cli.core.util import CLIError + +LIVE_TEST_CONTROL_ENV = 'AZURE_CLI_TEST_RUN_LIVE' +COMMAND_COVERAGE_CONTROL_ENV = 'AZURE_CLI_TEST_COMMAND_COVERAGE' +MOCKED_SUBSCRIPTION_ID = '00000000-0000-0000-0000-000000000000' +MOCKED_TENANT_ID = '00000000-0000-0000-0000-000000000000' +MOCKED_STORAGE_ACCOUNT = 'dummystorage' + + +# MOCK METHODS + +# Workaround until https://github.com/kevin1024/vcrpy/issues/293 is fixed. +vcr_connection_request = vcr.stubs.VCRConnection.request + + +def patch_vcr_connection_request(*args, **kwargs): + kwargs.pop('encode_chunked', None) + vcr_connection_request(*args, **kwargs) + + +vcr.stubs.VCRConnection.request = patch_vcr_connection_request + + +def _mock_get_mgmt_service_client(client_type, subscription_bound=True, subscription_id=None, + api_version=None, base_url_bound=None, **kwargs): + # version of _get_mgmt_service_client to use when recording or playing tests + profile = Profile() + cred, subscription_id, _ = profile.get_login_credentials(subscription_id=subscription_id) + client_kwargs = {} + + if base_url_bound: + client_kwargs = {'base_url': CLOUD.endpoints.resource_manager} + if api_version: + client_kwargs['api_version'] = api_version + if kwargs: + client_kwargs.update(kwargs) + + if subscription_bound: + client = client_type(cred, subscription_id, **client_kwargs) + else: + client = client_type(cred, **client_kwargs) + + client = _debug.change_ssl_cert_verification(client) + + client.config.add_user_agent("AZURECLI/TEST/{}".format(core_version)) + + return (client, subscription_id) + + +def _mock_generate_deployment_name(namespace): + if not namespace.deployment_name: + namespace.deployment_name = 'mock-deployment' + + +def _mock_handle_exceptions(ex): + raise ex + + +def _mock_subscriptions(self): # pylint: disable=unused-argument + return [{ + "id": MOCKED_SUBSCRIPTION_ID, + "user": { + "name": "example@example.com", + "type": "user" + }, + "state": "Enabled", + "name": "Example", + "tenantId": MOCKED_TENANT_ID, + "isDefault": True}] + + +def _mock_user_access_token(_, _1, _2, _3): # pylint: disable=unused-argument + return ('Bearer', 'top-secret-token-for-you') + + +def _mock_operation_delay(_): + # don't run time.sleep() + return + + +# TEST CHECKS + + +class JMESPathCheckAssertionError(AssertionError): + def __init__(self, comparator, actual_result, json_data): + message = "Actual value '{}' != Expected value '{}'. ".format( + actual_result, + comparator.expected_result) + message += "Query '{}' used on json data '{}'".format(comparator.query, json_data) + super(JMESPathCheckAssertionError, self).__init__(message) + + +class JMESPathCheck(object): # pylint: disable=too-few-public-methods + + def __init__(self, query, expected_result): + self.query = query + self.expected_result = expected_result + + def compare(self, json_data): + actual_result = _search_result_by_jmespath(json_data, self.query) + if not actual_result == self.expected_result: + raise JMESPathCheckAssertionError(self, actual_result, json_data) + + +class JMESPathPatternCheck(object): # pylint: disable=too-few-public-methods + + def __init__(self, query, expected_result): + self.query = query + self.expected_result = expected_result + + def compare(self, json_data): + actual_result = _search_result_by_jmespath(json_data, self.query) + if not re.match(self.expected_result, str(actual_result), re.IGNORECASE): + raise JMESPathCheckAssertionError(self, actual_result, json_data) + + +class BooleanCheck(object): # pylint: disable=too-few-public-methods + + def __init__(self, expected_result): + self.expected_result = expected_result + + def compare(self, data): + result = str(str(data).lower() in ['yes', 'true', '1']) + try: + assert result == str(self.expected_result) + except AssertionError: + raise AssertionError("Actual value '{}' != Expected value {}".format( + result, self.expected_result)) + + +class NoneCheck(object): # pylint: disable=too-few-public-methods + + def __init__(self): + pass + + def compare(self, data): # pylint: disable=no-self-use + none_strings = ['[]', '{}', 'false'] + try: + assert not data or data in none_strings + except AssertionError: + raise AssertionError("Actual value '{}' != Expected value falsy (None, '', []) or " + "string in {}".format(data, none_strings)) + + +class StringCheck(object): # pylint: disable=too-few-public-methods + + def __init__(self, expected_result): + self.expected_result = expected_result + + def compare(self, data): + try: + result = data.replace('"', '') + assert result == self.expected_result + except AssertionError: + raise AssertionError("Actual value '{}' != Expected value {}".format( + data, self.expected_result)) + + +# HELPER METHODS + + +def _scrub_deployment_name(uri): + return re.sub('/deployments/([^/?]+)', '/deployments/mock-deployment', uri) + + +def _scrub_service_principal_name(uri): + return re.sub('userPrincipalName%20eq%20%27(.+)%27', + 'userPrincipalName%20eq%20%27example%40example.com%27', uri) + + +def _search_result_by_jmespath(json_data, query): + if not json_data: + json_data = '{}' + json_val = json.loads(json_data) + return jmespath.search( + query, + json_val, + jmespath.Options(collections.OrderedDict)) + + +def _custom_request_matcher(r1, r2): + """ Ensure method, path, and query parameters match. """ + if r1.method != r2.method: + return False + + url1 = urlparse(r1.uri) + url2 = urlparse(r2.uri) + + if url1.path != url2.path: + return False + + q1 = parse_qs(url1.query) + q2 = parse_qs(url2.query) + shared_keys = set(q1.keys()).intersection(set(q2.keys())) + + if len(shared_keys) != len(q1) or len(shared_keys) != len(q2): + return False + + for key in shared_keys: + if q1[key][0].lower() != q2[key][0].lower(): + return False + + return True + + +# MAIN CLASS + + +class VCRTestBase(unittest.TestCase): # pylint: disable=too-many-instance-attributes + + FILTER_HEADERS = [ + 'authorization', + 'client-request-id', + 'x-ms-client-request-id', + 'x-ms-correlation-request-id', + 'x-ms-ratelimit-remaining-subscription-reads', + 'x-ms-request-id', + 'x-ms-routing-request-id', + 'x-ms-gateway-service-instanceid', + 'x-ms-ratelimit-remaining-tenant-reads', + 'x-ms-served-by', + ] + + def __init__(self, test_file, test_name, run_live=False, debug=False, debug_vcr=False, + skip_setup=False, skip_teardown=False): + super(VCRTestBase, self).__init__(test_name) + self.test_name = test_name + self.recording_dir = os.path.join(os.path.dirname(test_file), 'recordings') + self.cassette_path = os.path.join(self.recording_dir, '{}.yaml'.format(test_name)) + self.playback = os.path.isfile(self.cassette_path) + + if os.environ.get(LIVE_TEST_CONTROL_ENV, None) == 'True': + self.run_live = True + else: + self.run_live = run_live + + self.skip_setup = skip_setup + self.skip_teardown = skip_teardown + self.success = False + self.exception = None + self.track_commands = os.environ.get(COMMAND_COVERAGE_CONTROL_ENV, None) + self._debug = debug + + if not self.playback and ('--buffer' in sys.argv) and not run_live: + self.exception = CLIError('No recorded result provided for {}.'.format(self.test_name)) + + if debug_vcr: + import logging + logging.basicConfig() + vcr_log = logging.getLogger('vcr') + vcr_log.setLevel(logging.INFO) + self.my_vcr = vcr.VCR( + cassette_library_dir=self.recording_dir, + before_record_request=self._before_record_request, + before_record_response=self._before_record_response, + decode_compressed_response=True + ) + self.my_vcr.register_matcher('custom', _custom_request_matcher) + self.my_vcr.match_on = ['custom'] + + def _track_executed_commands(self, command): + if self.track_commands: + with open(self.track_commands, 'a+') as f: + f.write(' '.join(command)) + f.write('\n') + + def _before_record_request(self, request): # pylint: disable=no-self-use + # scrub subscription from the uri + request.uri = re.sub('/subscriptions/([^/]+)/', + '/subscriptions/{}/'.format(MOCKED_SUBSCRIPTION_ID), request.uri) + # scrub jobId from uri, required for ADLA + request.uri = re.sub('/Jobs/([^/]+)', + '/Jobs/{}'.format(MOCKED_SUBSCRIPTION_ID), request.uri) + request.uri = re.sub('/graph.windows.net/([^/]+)/', + '/graph.windows.net/{}/'.format(MOCKED_TENANT_ID), request.uri) + request.uri = re.sub('/sig=([^/]+)&', '/sig=0000&', request.uri) + request.uri = _scrub_deployment_name(request.uri) + request.uri = _scrub_service_principal_name(request.uri) + + # replace random storage account name with dummy name + request.uri = re.sub(r'(vcrstorage[\d]+)', MOCKED_STORAGE_ACCOUNT, request.uri) + # prevents URI mismatch between Python 2 and 3 if request URI has extra / chars + request.uri = re.sub('//', '/', request.uri) + request.uri = re.sub('/', '//', request.uri, count=1) + # do not record requests sent for token refresh' + if (request.body and 'grant-type=refresh_token' in str(request.body)) or \ + ('/oauth2/token' in request.uri): + request = None + return request + + def _before_record_response(self, response): # pylint: disable=no-self-use + for key in VCRTestBase.FILTER_HEADERS: + if key in response['headers']: + del response['headers'][key] + + def _scrub_body_parameters(value): + value = re.sub('/subscriptions/([^/]+)/', + '/subscriptions/{}/'.format(MOCKED_SUBSCRIPTION_ID), value) + value = re.sub('\"jobId\": \"([^/]+)\"', + '\"jobId\": \"{}\"'.format(MOCKED_SUBSCRIPTION_ID), value) + return value + + for key in response['body']: + value = response['body'][key].decode('utf-8') + value = _scrub_body_parameters(value) + try: + response['body'][key] = bytes(value, 'utf-8') + except TypeError: + response['body'][key] = value.encode('utf-8') + + return response + + @mock.patch('azure.cli.main.handle_exception', _mock_handle_exceptions) + @mock.patch('azure.cli.core.commands.client_factory._get_mgmt_service_client', + _mock_get_mgmt_service_client) # pylint: disable=line-too-long + def _execute_live_or_recording(self): + # pylint: disable=no-member + try: + set_up = getattr(self, "set_up", None) + if callable(set_up) and not self.skip_setup: + self.set_up() + + if self.run_live: + self.body() + else: + with self.my_vcr.use_cassette(self.cassette_path): + self.body() + self.success = True + except Exception as ex: + raise ex + finally: + tear_down = getattr(self, "tear_down", None) + if callable(tear_down) and not self.skip_teardown: + self.tear_down() + + @mock.patch('azure.cli.core._profile.Profile.load_cached_subscriptions', _mock_subscriptions) + @mock.patch('azure.cli.core._profile.CredsCache.retrieve_token_for_user', + _mock_user_access_token) # pylint: disable=line-too-long + @mock.patch('azure.cli.main.handle_exception', _mock_handle_exceptions) + @mock.patch('azure.cli.core.commands.client_factory._get_mgmt_service_client', + _mock_get_mgmt_service_client) # pylint: disable=line-too-long + @mock.patch('msrestazure.azure_operation.AzureOperationPoller._delay', _mock_operation_delay) + @mock.patch('time.sleep', _mock_operation_delay) + @mock.patch('azure.cli.core.commands.LongRunningOperation._delay', _mock_operation_delay) + @mock.patch('azure.cli.core.commands.validators.generate_deployment_name', + _mock_generate_deployment_name) + def _execute_playback(self): + # pylint: disable=no-member + with self.my_vcr.use_cassette(self.cassette_path): + self.body() + self.success = True + + def _post_recording_scrub(self): + """ Perform post-recording cleanup on the YAML file that can't be accomplished with the + VCR recording hooks. """ + src_path = self.cassette_path + rg_name = getattr(self, 'resource_group', None) + rg_original = getattr(self, 'resource_group_original', None) + + t = tempfile.NamedTemporaryFile('r+') + with open(src_path, 'r') as f: + for line in f: + # scrub resource group names + if rg_name != rg_original: + line = line.replace(rg_name, rg_original) + # omit bearer tokens + if 'authorization:' not in line.lower(): + t.write(line) + t.seek(0) + with open(src_path, 'w') as f: + for line in t: + f.write(line) + t.close() + + # COMMAND METHODS + + def cmd(self, command, checks=None, allowed_exceptions=None, + debug=False): # pylint: disable=no-self-use + allowed_exceptions = allowed_exceptions or [] + if not isinstance(allowed_exceptions, list): + allowed_exceptions = [allowed_exceptions] + + if self._debug or debug: + print('\n\tRUNNING: {}'.format(command)) + command_list = shlex.split(command) + output = StringIO() + try: + cli_main(command_list, file=output) + except Exception as ex: # pylint: disable=broad-except + ex_msg = str(ex) + if not next((x for x in allowed_exceptions if x in ex_msg), None): + raise ex + self._track_executed_commands(command_list) + result = output.getvalue().strip() + output.close() + + if self._debug or debug: + print('\tRESULT: {}\n'.format(result)) + + if checks: + checks = [checks] if not isinstance(checks, list) else checks + for check in checks: + check.compare(result) + + if '-o' in command_list and 'tsv' in command_list: + return result + else: + try: + result = result or '{}' + return json.loads(result) + except Exception: # pylint: disable=broad-except + return result + + def set_env(self, key, val): # pylint: disable=no-self-use + os.environ[key] = val + + def pop_env(self, key): # pylint: disable=no-self-use + return os.environ.pop(key, None) + + def execute(self): + ''' Method to actually start execution of the test. Must be called from the test_ + method of the test class. ''' + try: + if self.run_live: + print('RUN LIVE: {}'.format(self.test_name)) + self._execute_live_or_recording() + elif self.playback: + print('PLAYBACK: {}'.format(self.test_name)) + self._execute_playback() + else: + print('RECORDING: {}'.format(self.test_name)) + self._execute_live_or_recording() + except Exception as ex: + traceback.print_exc() + raise ex + finally: + if not self.success and not self.playback and os.path.isfile(self.cassette_path): + print('DISCARDING RECORDING: {}'.format(self.cassette_path)) + os.remove(self.cassette_path) + elif self.success and not self.playback and os.path.isfile(self.cassette_path): + try: + self._post_recording_scrub() + except Exception: # pylint: disable=broad-except + os.remove(self.cassette_path) + + +class ResourceGroupVCRTestBase(VCRTestBase): + + def __init__(self, test_file, test_name, resource_group='vcr_resource_group', run_live=False, + debug=False, debug_vcr=False, skip_setup=False, skip_teardown=False, + random_tag_format=None): + super(ResourceGroupVCRTestBase, self).__init__(test_file, test_name, run_live=run_live, + debug=debug, debug_vcr=debug_vcr, + skip_setup=skip_setup, + skip_teardown=skip_teardown) + self.resource_group_original = resource_group + random_tag = (random_tag_format or '_{}_').format( + ''.join((choice(ascii_lowercase + digits) for _ in range(4)))) + self.resource_group = '{}{}'.format(resource_group, '' if self.playback else random_tag) + self.location = 'westus' + + def set_up(self): + self.cmd('group create --location {} --name {} --tags use=az-test'.format( + self.location, self.resource_group)) + + def tear_down(self): + self.cmd('group delete --name {} --no-wait --yes'.format(self.resource_group)) From 74a489a9e6b34a1cab9e4de4a95196b58d2620cd Mon Sep 17 00:00:00 2001 From: valrus Date: Sat, 13 May 2017 13:15:38 -0700 Subject: [PATCH 006/167] Add vcrpy version from azure-cli --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 02c0bd6a71ed..d73dd13e85d4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -setuptools-markdown \ No newline at end of file +setuptools-markdown +vcrpy==1.10.3 From f622aea346da817e339cf315bd19028917f8738a Mon Sep 17 00:00:00 2001 From: valrus Date: Sun, 14 May 2017 19:06:04 -0700 Subject: [PATCH 007/167] Add VSCode ignores --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 72364f99fe4b..71b85dc0709c 100644 --- a/.gitignore +++ b/.gitignore @@ -87,3 +87,6 @@ ENV/ # Rope project settings .ropeproject + +# Editor caches +.vscode From 1da5dfbaa01ecaf4d3250415634047a77ac3331f Mon Sep 17 00:00:00 2001 From: valrus Date: Sun, 14 May 2017 19:06:14 -0700 Subject: [PATCH 008/167] Add Azure imports --- requirements.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d73dd13e85d4..0ca245a8979f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,5 @@ setuptools-markdown -vcrpy==1.10.3 +azure-mgmt-keyvault~=0.31.0 +azure-mgmt-resource~=1.0.0rc1 +azure-mgmt-storage~=1.0.0rc1 +vcrpy==1.10.3 \ No newline at end of file From 0c4e7078f22f8695a6d8df45277141247bd2b27f Mon Sep 17 00:00:00 2001 From: valrus Date: Sun, 14 May 2017 19:06:28 -0700 Subject: [PATCH 009/167] Start converting CLI commands to SDK calls --- src/scenario_tests/preparers.py | 45 +++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/src/scenario_tests/preparers.py b/src/scenario_tests/preparers.py index 7255349c6f46..a4a9b353de8f 100644 --- a/src/scenario_tests/preparers.py +++ b/src/scenario_tests/preparers.py @@ -7,6 +7,9 @@ import functools import os +from azure.mgmt.resource.resources import ResourceManagementClient +from azure.mgmt.storage import StorageManagementClient + from .base import ScenarioTest, execute from .exceptions import CliTestError from .utilities import create_random_name @@ -124,18 +127,30 @@ def __init__(self, name_prefix='clitest.rg', self.dev_setting_name = os.environ.get(dev_setting_name, None) self.dev_setting_location = os.environ.get(dev_setting_location, location) + # This should work with 'az login' + self.client = ResourceManagementClient() + def create_resource(self, name, **kwargs): if self.dev_setting_name: return {self.parameter_name: self.dev_setting_name, self.parameter_name_for_location: self.dev_setting_location} else: - template = 'az group create --location {} --name {} --tag use=az-test' - execute(template.format(self.location, name)) + # template = 'az group create --location {} --name {} --tag use=az-test' + self.client.resource_groups.create_or_update( + name, + { + 'location': self.location, + 'tags': { + 'use': 'az-test', + } + } + ) return {self.parameter_name: name, self.parameter_name_for_location: self.location} def remove_resource(self, name, **kwargs): if not self.dev_setting_name: - execute('az group delete --name {} --yes --no-wait'.format(name)) + # execute('az group delete --name {} --yes --no-wait'.format(name)) + self.client.resource_groups.delete(name) # Storage Account Preparer and its shorthand decorator @@ -153,24 +168,38 @@ def __init__(self, self.parameter_name = parameter_name self.dev_setting_name = os.environ.get(dev_setting_name, None) + self.client = StorageManagementClient() def create_resource(self, name, **kwargs): group = self._get_resource_group(**kwargs) if not self.dev_setting_name: - template = 'az storage account create -n {} -g {} -l {} --sku {}' - execute(template.format(name, group, self.location, self.sku)) + # template = 'az storage account create -n {} -g {} -l {} --sku {}' + # execute(template.format(name, group, self.location, self.sku)) + storage_async_operation = self.client.storage_accounts.create( + group, + name, + { + 'sku': self.sku, + 'location': self.location, + } + ) + storage_async_operation.wait() else: name = self.dev_setting_name - account_key = execute('storage account keys list -n {} -g {} --query "[0].value" -otsv' - .format(name, group)).output + # account_key = execute('storage account keys list -n {} -g {} --query "[0].value" -otsv' + # .format(name, group)).output + # What is -otsv? + storage_keys = self.client.storage_accounts.list_keys(group, name) + account_key = storage_keys[0].value return {self.parameter_name: name, self.parameter_name + '_info': (name, account_key)} def remove_resource(self, name, **kwargs): if not self.skip_delete and not self.dev_setting_name: group = self._get_resource_group(**kwargs) - execute('az storage account delete -n {} -g {} --yes'.format(name, group)) + # execute('az storage account delete -n {} -g {} --yes'.format(name, group)) + self.storage_client.storage_accounts.delete(group, name) def _get_resource_group(self, **kwargs): try: From c7c46f6d9c3819b1c760a941d9980f3710698aab Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Tue, 16 May 2017 09:58:21 -0700 Subject: [PATCH 010/167] Minor proofreading --- doc/scenario_base_tests.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/scenario_base_tests.md b/doc/scenario_base_tests.md index b35a9bf46dbc..1df6400d4145 100644 --- a/doc/scenario_base_tests.md +++ b/doc/scenario_base_tests.md @@ -103,9 +103,9 @@ class StorageAccountTests(ScenarioTest): ``` Note: -One of the most important features of `ScenarioTest` is names managements. For the tests to be able to run in a live environment and avoid name collision a strong name randomization is required. On the other hand, for the tests to be recorded and replay, the naming mechanism must be repeatable during playback mode. The `self.create_randome_name` assist the test to achieve the goal. +One of the most important features of `ScenarioTest` is name management. For the tests to be able to run in a live environment and avoid name collision a strong name randomization is required. On the other hand, for the tests to be recorded and replay, the naming mechanism must be repeatable during playback mode. The `self.create_random_name` method helps the test achieve the goal. -The method will create a random name during recording, and when it is called during playback, it returns a name (internally it is called moniker) based on the sequence of the name request. The order won't change once the test is written. Peak into the recording file, you find no random name. For example, note the names like 'clitest.rg000001', they aren't the names of the resources which are actually created in Azure. They're placed before the requests are persisted. +The method will create a random name during recording, and when it is called during playback, it returns a name (internally it is called moniker) based on the sequence of the name request. The order won't change once the test is written. Peek into the recording file, you find no random name. For example, note the names like 'clitest.rg000001', they aren't the names of the resources which are actually created in Azure. They're placed before the requests are persisted. ``` Yaml - request: body: '{"location": "westus", "tags": {"use": "az-test"}}' @@ -136,7 +136,7 @@ The method will create a random name during recording, and when it is called dur status: {code: 201, message: Created} ``` -In short, for the names of any Azure resources used in the tests, always use the `self.create_random_name` to generate its value. Also make sure the correct length is given to the method because different resource have different limitation of the name length. The method will always try to create the longest name possible to fully randomize the name. +In short, for the names of any Azure resources used in the tests, always use `self.create_random_name` to generate its value. Also make sure the correct length is given to the method because different resource have different limitation of the name length. The method will always try to create the longest name possible to fully randomize the name. ### Sample 7. Prepare storage account for tests @@ -161,7 +161,7 @@ Note: @StorageAccountPreparer(sku='Standard_LRS', location='southcentralus', parameter_name='storage') ``` -### Sampel 8. Prepare multiple storage accounts for tests +### Sample 8. Prepare multiple storage accounts for tests ``` Python class StorageAccountTests(ScenarioTest): @ResourceGroupPreparer() From fe0f28acc26e9aec1954e30ee853d2b74ee03ae7 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Tue, 16 May 2017 09:58:42 -0700 Subject: [PATCH 011/167] Add required common and mock modules --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 0ca245a8979f..ef49036d9303 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,7 @@ setuptools-markdown +azure-common~=1.1.6 azure-mgmt-keyvault~=0.31.0 azure-mgmt-resource~=1.0.0rc1 azure-mgmt-storage~=1.0.0rc1 +mock vcrpy==1.10.3 \ No newline at end of file From 43b349c294fece01f260bea94615fb4a04bd7213 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Tue, 16 May 2017 09:59:09 -0700 Subject: [PATCH 012/167] Add deps, update pkgs and url --- setup.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 734de1c12e3f..3af1c494f480 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,13 @@ DEPENDENCIES = [ - 'setuptools-markdown' + 'azure-common~=1.1.6', + 'azure-mgmt-keyvault~=0.31.0', + 'azure-mgmt-resource~=1.0.0rc1', + 'azure-mgmt-storage~=1.0.0rc1', + 'mock', + 'setuptools-markdown', + 'vcrpy==1.10.3', ] with open('README.rst', 'r', encoding='utf-8') as f: @@ -39,19 +45,17 @@ setup( name='azure-devtools', version=VERSION, - description='Microsoft Azure Developing Tools for SDK', - long_description_markdown_file='README.md' + description='Microsoft Azure Development Tools for SDK', + long_description_markdown_file='README.md', license='MIT', author='Microsoft Corporation', author_email='azpycli@microsoft.com', - url='https://github.com/Azure/azure-cli', + url='https://github.com/Azure/azure-python-devtools', zip_safe=False, classifiers=CLASSIFIERS, packages=[ - 'azure', - 'azure.devtools', - 'azure.devtools.automationsdk' + 'azure_devtools', + 'azure_devtools.scenario_tests' ], install_requires=DEPENDENCIES, - cmdclass=cmdclass ) From 42c93d7ea6ce9292d3a456dfb823eaf0e4cedce1 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Tue, 16 May 2017 09:59:36 -0700 Subject: [PATCH 013/167] Package structure --- src/azure_devtools/__init__.py | 0 .../scenario_tests/__init__.py | 0 .../scenario_tests/base.py | 0 .../scenario_tests/checkers.py | 0 .../scenario_tests/const.py | 0 .../scenario_tests/decorators.py | 0 .../scenario_tests/exceptions.py | 0 .../scenario_tests/patches.py | 0 .../scenario_tests/preparers.py | 49 ++++++++++++++----- .../scenario_tests/recording_processors.py | 0 .../scenario_tests/utilities.py | 0 .../scenario_tests/vcr_test_base.py | 0 12 files changed, 38 insertions(+), 11 deletions(-) create mode 100644 src/azure_devtools/__init__.py rename src/{ => azure_devtools}/scenario_tests/__init__.py (100%) rename src/{ => azure_devtools}/scenario_tests/base.py (100%) rename src/{ => azure_devtools}/scenario_tests/checkers.py (100%) rename src/{ => azure_devtools}/scenario_tests/const.py (100%) rename src/{ => azure_devtools}/scenario_tests/decorators.py (100%) rename src/{ => azure_devtools}/scenario_tests/exceptions.py (100%) rename src/{ => azure_devtools}/scenario_tests/patches.py (100%) rename src/{ => azure_devtools}/scenario_tests/preparers.py (86%) rename src/{ => azure_devtools}/scenario_tests/recording_processors.py (100%) rename src/{ => azure_devtools}/scenario_tests/utilities.py (100%) rename src/{ => azure_devtools}/scenario_tests/vcr_test_base.py (100%) diff --git a/src/azure_devtools/__init__.py b/src/azure_devtools/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/src/scenario_tests/__init__.py b/src/azure_devtools/scenario_tests/__init__.py similarity index 100% rename from src/scenario_tests/__init__.py rename to src/azure_devtools/scenario_tests/__init__.py diff --git a/src/scenario_tests/base.py b/src/azure_devtools/scenario_tests/base.py similarity index 100% rename from src/scenario_tests/base.py rename to src/azure_devtools/scenario_tests/base.py diff --git a/src/scenario_tests/checkers.py b/src/azure_devtools/scenario_tests/checkers.py similarity index 100% rename from src/scenario_tests/checkers.py rename to src/azure_devtools/scenario_tests/checkers.py diff --git a/src/scenario_tests/const.py b/src/azure_devtools/scenario_tests/const.py similarity index 100% rename from src/scenario_tests/const.py rename to src/azure_devtools/scenario_tests/const.py diff --git a/src/scenario_tests/decorators.py b/src/azure_devtools/scenario_tests/decorators.py similarity index 100% rename from src/scenario_tests/decorators.py rename to src/azure_devtools/scenario_tests/decorators.py diff --git a/src/scenario_tests/exceptions.py b/src/azure_devtools/scenario_tests/exceptions.py similarity index 100% rename from src/scenario_tests/exceptions.py rename to src/azure_devtools/scenario_tests/exceptions.py diff --git a/src/scenario_tests/patches.py b/src/azure_devtools/scenario_tests/patches.py similarity index 100% rename from src/scenario_tests/patches.py rename to src/azure_devtools/scenario_tests/patches.py diff --git a/src/scenario_tests/preparers.py b/src/azure_devtools/scenario_tests/preparers.py similarity index 86% rename from src/scenario_tests/preparers.py rename to src/azure_devtools/scenario_tests/preparers.py index a4a9b353de8f..f18a27d49df6 100644 --- a/src/scenario_tests/preparers.py +++ b/src/azure_devtools/scenario_tests/preparers.py @@ -6,7 +6,12 @@ import inspect import functools import os +import uuid +from azure.common.client_factory import get_client_from_cli_profile +from azure.graphrbac import GraphRbacManagementClient +from azure.mgmt.authorization import AuthorizationManagementClient +from azure.mgmt.graphrbac.models import PasswordCredential from azure.mgmt.resource.resources import ResourceManagementClient from azure.mgmt.storage import StorageManagementClient @@ -128,7 +133,7 @@ def __init__(self, name_prefix='clitest.rg', self.dev_setting_location = os.environ.get(dev_setting_location, location) # This should work with 'az login' - self.client = ResourceManagementClient() + self.client = get_client_from_cli_profile(ResourceManagementClient) def create_resource(self, name, **kwargs): if self.dev_setting_name: @@ -168,7 +173,7 @@ def __init__(self, self.parameter_name = parameter_name self.dev_setting_name = os.environ.get(dev_setting_name, None) - self.client = StorageManagementClient() + self.client = get_client_from_cli_profile(StorageManagementClient) def create_resource(self, name, **kwargs): group = self._get_resource_group(**kwargs) @@ -229,8 +234,8 @@ def __init__(self, # pylint: disable=too-many-arguments def create_resource(self, name, **kwargs): if not self.dev_setting_name: group = self._get_resource_group(**kwargs) - template = 'az keyvault create -n {} -g {} -l {} --sku {}' - execute(template.format(name, group, self.location, self.sku)) + # template = 'az keyvault create -n {} -g {} -l {} --sku {}' + # execute(template.format(name, group, self.location, self.sku)) return {self.parameter_name: name} else: return {self.parameter_name: self.dev_setting_name} @@ -238,7 +243,7 @@ def create_resource(self, name, **kwargs): def remove_resource(self, name, **kwargs): if not self.skip_delete and not self.dev_setting_name: group = self._get_resource_group(**kwargs) - execute('az keyvault delete -n {} -g {} --yes'.format(name, group)) + # execute('az keyvault delete -n {} -g {} --yes'.format(name, group)) def _get_resource_group(self, **kwargs): try: @@ -258,25 +263,47 @@ def __init__(self, name_prefix='http://clitest', dev_setting_sp_password='AZURE_CLI_TEST_DEV_SP_PASSWORD'): super(RoleBasedServicePrincipalPreparer, self).__init__(name_prefix, 24) self.skip_assignment = skip_assignment - self.result = {} + self.result = None self.parameter_name = parameter_name self.parameter_password = parameter_password self.dev_setting_sp_name = os.environ.get(dev_setting_sp_name, None) self.dev_setting_sp_password = os.environ.get(dev_setting_sp_password, None) + self.graph_client = get_client_from_cli_profile(GraphRbacManagementClient) + def create_resource(self, name, **kwargs): if not self.dev_setting_sp_name: - command = 'az ad sp create-for-rbac -n {}{}' \ - .format(name, ' --skip-assignment' if self.skip_assignment else '') - self.result = execute(command).get_output_in_json() - return {self.parameter_name: name, self.parameter_password: self.result['password']} + # command = 'az ad sp create-for-rbac -n {}{}' \ + # .format(name, ' --skip-assignment' if self.skip_assignment else '') + # self.result = execute(command).get_output_in_json() + password = uuid.uuid4() + app = self.graph_client.applications.create( + { + 'available_to_other_tenants': False, + 'display_name': name, + 'identifier_uris': [name], + 'password_creds': [ + PasswordCredential( + value=password, + ) + ] + } + ) + self.result = self.graph_client.service_principals.create( + { + 'app_id': app.app_id, + 'account_enabled': True, + } + ) + return {self.parameter_name: name, self.parameter_password: password} else: return {self.parameter_name: self.dev_setting_sp_name, self.parameter_password: self.dev_setting_sp_password} def remove_resource(self, name, **kwargs): if not self.dev_setting_sp_name: - execute('az ad sp delete --id {}'.format(self.result['appId'])) + # execute('az ad sp delete --id {}'.format(self.result['appId'])) + self.graph_client.service_principals.delete(self.result.object_id) # Utility diff --git a/src/scenario_tests/recording_processors.py b/src/azure_devtools/scenario_tests/recording_processors.py similarity index 100% rename from src/scenario_tests/recording_processors.py rename to src/azure_devtools/scenario_tests/recording_processors.py diff --git a/src/scenario_tests/utilities.py b/src/azure_devtools/scenario_tests/utilities.py similarity index 100% rename from src/scenario_tests/utilities.py rename to src/azure_devtools/scenario_tests/utilities.py diff --git a/src/scenario_tests/vcr_test_base.py b/src/azure_devtools/scenario_tests/vcr_test_base.py similarity index 100% rename from src/scenario_tests/vcr_test_base.py rename to src/azure_devtools/scenario_tests/vcr_test_base.py From 26aed99f7c4933de0824c013ea64cad99640cf46 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Wed, 24 May 2017 14:46:57 -0700 Subject: [PATCH 014/167] Add more editor ignores --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 71b85dc0709c..009b55465f7a 100644 --- a/.gitignore +++ b/.gitignore @@ -90,3 +90,6 @@ ENV/ # Editor caches .vscode +.vs +*.pyproj +*.sln From 13aa7665309615048e43025eb0d09c992b9c0897 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Wed, 24 May 2017 14:47:29 -0700 Subject: [PATCH 015/167] Remove azure deps and nonexistent HISTORY ref --- requirements.txt | 4 +--- setup.py | 10 +++------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/requirements.txt b/requirements.txt index ef49036d9303..9685227ce0f4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,5 @@ setuptools-markdown azure-common~=1.1.6 -azure-mgmt-keyvault~=0.31.0 -azure-mgmt-resource~=1.0.0rc1 -azure-mgmt-storage~=1.0.0rc1 +jmespath mock vcrpy==1.10.3 \ No newline at end of file diff --git a/setup.py b/setup.py index 3af1c494f480..e01183c97be7 100644 --- a/setup.py +++ b/setup.py @@ -28,20 +28,15 @@ DEPENDENCIES = [ 'azure-common~=1.1.6', - 'azure-mgmt-keyvault~=0.31.0', - 'azure-mgmt-resource~=1.0.0rc1', - 'azure-mgmt-storage~=1.0.0rc1', + 'jmespath', 'mock', 'setuptools-markdown', 'vcrpy==1.10.3', ] -with open('README.rst', 'r', encoding='utf-8') as f: +with open('README.md', 'r', encoding='utf-8') as f: README = f.read() -with open('HISTORY.rst', 'r', encoding='utf-8') as f: - HISTORY = f.read() - setup( name='azure-devtools', version=VERSION, @@ -57,5 +52,6 @@ 'azure_devtools', 'azure_devtools.scenario_tests' ], + package_dir={'': 'src'}, install_requires=DEPENDENCIES, ) From b85300bdf874cf71ea71276db4e6340d892decd1 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Wed, 24 May 2017 14:47:57 -0700 Subject: [PATCH 016/167] Remove Azure mgmt dependencies --- src/azure_devtools/scenario_tests/__init__.py | 9 +- src/azure_devtools/scenario_tests/base.py | 2 +- .../scenario_tests/exceptions.py | 6 +- src/azure_devtools/scenario_tests/patches.py | 4 +- .../scenario_tests/preparers.py | 197 ------------------ 5 files changed, 9 insertions(+), 209 deletions(-) diff --git a/src/azure_devtools/scenario_tests/__init__.py b/src/azure_devtools/scenario_tests/__init__.py index 31a4f1c0e6b6..963727f2d29d 100644 --- a/src/azure_devtools/scenario_tests/__init__.py +++ b/src/azure_devtools/scenario_tests/__init__.py @@ -4,14 +4,11 @@ # -------------------------------------------------------------------------------------------- from .base import ScenarioTest, LiveTest -from .preparers import (StorageAccountPreparer, ResourceGroupPreparer, - RoleBasedServicePrincipalPreparer, KeyVaultPreparer) -from .exceptions import CliTestError +from .exceptions import AzureTestError from .checkers import JMESPathCheck, JMESPathCheckExists, NoneCheck, StringCheck, StringContainCheck from .decorators import live_only, record_only from .utilities import get_sha1_hash -__all__ = ['ScenarioTest', 'LiveTest', 'ResourceGroupPreparer', 'StorageAccountPreparer', - 'RoleBasedServicePrincipalPreparer', 'CliTestError', 'JMESPathCheck', 'JMESPathCheckExists', 'NoneCheck', - 'live_only', 'record_only', 'StringCheck', 'StringContainCheck', 'get_sha1_hash', 'KeyVaultPreparer'] +__all__ = ['ScenarioTest', 'LiveTest', 'AzureTestError', 'JMESPathCheck', 'JMESPathCheckExists', 'NoneCheck', + 'live_only', 'record_only', 'StringCheck', 'StringContainCheck', 'get_sha1_hash'] __version__ = '0.1.0+dev' diff --git a/src/azure_devtools/scenario_tests/base.py b/src/azure_devtools/scenario_tests/base.py index dfc843d4a6d2..d3577afcacbd 100644 --- a/src/azure_devtools/scenario_tests/base.py +++ b/src/azure_devtools/scenario_tests/base.py @@ -29,7 +29,7 @@ from .utilities import create_random_name from .decorators import live_only -logger = logging.getLogger('azuer.cli.testsdk') +logger = logging.getLogger('azure.cli.testsdk') class IntegrationTestBase(unittest.TestCase): diff --git a/src/azure_devtools/scenario_tests/exceptions.py b/src/azure_devtools/scenario_tests/exceptions.py index f490a6d8e4f6..c8914a2aa1b6 100644 --- a/src/azure_devtools/scenario_tests/exceptions.py +++ b/src/azure_devtools/scenario_tests/exceptions.py @@ -4,10 +4,10 @@ # -------------------------------------------------------------------------------------------- -class CliTestError(Exception): +class AzureTestError(Exception): def __init__(self, error_message): - message = 'An error caused by the CLI test harness failed the test: {}' - super(CliTestError, self).__init__(message.format(error_message)) + message = 'An error caused by the Azure test harness failed the test: {}' + super(AzureTestError, self).__init__(message.format(error_message)) class CliExecutionError(Exception): diff --git a/src/azure_devtools/scenario_tests/patches.py b/src/azure_devtools/scenario_tests/patches.py index 54a351aa3df8..00dac5bc16ca 100644 --- a/src/azure_devtools/scenario_tests/patches.py +++ b/src/azure_devtools/scenario_tests/patches.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -from .exceptions import CliExecutionError, CliTestError +from .exceptions import CliExecutionError, AzureTestError from .const import MOCKED_SUBSCRIPTION_ID, MOCKED_TENANT_ID @@ -73,7 +73,7 @@ def _mock_in_unit_test(unit_test, target, replacement): import unittest if not isinstance(unit_test, unittest.TestCase): - raise CliTestError('The patch_main_exception_handler can be only used in unit test') + raise AzureTestError('The patch_main_exception_handler can be only used in unit test') mp = mock.patch(target, replacement) mp.__enter__() diff --git a/src/azure_devtools/scenario_tests/preparers.py b/src/azure_devtools/scenario_tests/preparers.py index f18a27d49df6..51dc3f7b6ca9 100644 --- a/src/azure_devtools/scenario_tests/preparers.py +++ b/src/azure_devtools/scenario_tests/preparers.py @@ -9,14 +9,8 @@ import uuid from azure.common.client_factory import get_client_from_cli_profile -from azure.graphrbac import GraphRbacManagementClient -from azure.mgmt.authorization import AuthorizationManagementClient -from azure.mgmt.graphrbac.models import PasswordCredential -from azure.mgmt.resource.resources import ResourceManagementClient -from azure.mgmt.storage import StorageManagementClient from .base import ScenarioTest, execute -from .exceptions import CliTestError from .utilities import create_random_name from .recording_processors import RecordingProcessor @@ -115,197 +109,6 @@ def process_response(self, response): return response -# Resource Group Preparer and its shorthand decorator - -class ResourceGroupPreparer(AbstractPreparer, SingleValueReplacer): - def __init__(self, name_prefix='clitest.rg', - parameter_name='resource_group', - parameter_name_for_location='resource_group_location', location='westus', - dev_setting_name='AZURE_CLI_TEST_DEV_RESOURCE_GROUP_NAME', - dev_setting_location='AZURE_CLI_TEST_DEV_RESOURCE_GROUP_LOCATION', - random_name_length=75): - super(ResourceGroupPreparer, self).__init__(name_prefix, random_name_length) - self.location = location - self.parameter_name = parameter_name - self.parameter_name_for_location = parameter_name_for_location - - self.dev_setting_name = os.environ.get(dev_setting_name, None) - self.dev_setting_location = os.environ.get(dev_setting_location, location) - - # This should work with 'az login' - self.client = get_client_from_cli_profile(ResourceManagementClient) - - def create_resource(self, name, **kwargs): - if self.dev_setting_name: - return {self.parameter_name: self.dev_setting_name, - self.parameter_name_for_location: self.dev_setting_location} - else: - # template = 'az group create --location {} --name {} --tag use=az-test' - self.client.resource_groups.create_or_update( - name, - { - 'location': self.location, - 'tags': { - 'use': 'az-test', - } - } - ) - return {self.parameter_name: name, self.parameter_name_for_location: self.location} - - def remove_resource(self, name, **kwargs): - if not self.dev_setting_name: - # execute('az group delete --name {} --yes --no-wait'.format(name)) - self.client.resource_groups.delete(name) - - -# Storage Account Preparer and its shorthand decorator - -class StorageAccountPreparer(AbstractPreparer, SingleValueReplacer): - def __init__(self, - name_prefix='clitest', sku='Standard_LRS', location='westus', - parameter_name='storage_account', resource_group_parameter_name='resource_group', - skip_delete=True, dev_setting_name='AZURE_CLI_TEST_DEV_STORAGE_ACCOUNT_NAME'): - super(StorageAccountPreparer, self).__init__(name_prefix, 24) - self.location = location - self.sku = sku - self.resource_group_parameter_name = resource_group_parameter_name - self.skip_delete = skip_delete - self.parameter_name = parameter_name - - self.dev_setting_name = os.environ.get(dev_setting_name, None) - self.client = get_client_from_cli_profile(StorageManagementClient) - - def create_resource(self, name, **kwargs): - group = self._get_resource_group(**kwargs) - - if not self.dev_setting_name: - # template = 'az storage account create -n {} -g {} -l {} --sku {}' - # execute(template.format(name, group, self.location, self.sku)) - storage_async_operation = self.client.storage_accounts.create( - group, - name, - { - 'sku': self.sku, - 'location': self.location, - } - ) - storage_async_operation.wait() - else: - name = self.dev_setting_name - - # account_key = execute('storage account keys list -n {} -g {} --query "[0].value" -otsv' - # .format(name, group)).output - # What is -otsv? - storage_keys = self.client.storage_accounts.list_keys(group, name) - account_key = storage_keys[0].value - return {self.parameter_name: name, self.parameter_name + '_info': (name, account_key)} - - def remove_resource(self, name, **kwargs): - if not self.skip_delete and not self.dev_setting_name: - group = self._get_resource_group(**kwargs) - # execute('az storage account delete -n {} -g {} --yes'.format(name, group)) - self.storage_client.storage_accounts.delete(group, name) - - def _get_resource_group(self, **kwargs): - try: - return kwargs.get(self.resource_group_parameter_name) - except KeyError: - template = 'To create a storage account a resource group is required. Please add ' \ - 'decorator @{} in front of this storage account preparer.' - raise CliTestError(template.format(ResourceGroupPreparer.__name__)) - - -# KeyVault Preparer and its shorthand decorator - -class KeyVaultPreparer(AbstractPreparer, SingleValueReplacer): - def __init__(self, # pylint: disable=too-many-arguments - name_prefix='clitest', sku='standard', location='westus', - parameter_name='key_vault', resource_group_parameter_name='resource_group', - skip_delete=True, dev_setting_name='AZURE_CLI_TEST_DEV_KEY_VAULT_NAME'): - super(KeyVaultPreparer, self).__init__(name_prefix, 24) - self.location = location - self.sku = sku - self.resource_group_parameter_name = resource_group_parameter_name - self.skip_delete = skip_delete - self.parameter_name = parameter_name - - self.dev_setting_name = os.environ.get(dev_setting_name, None) - - def create_resource(self, name, **kwargs): - if not self.dev_setting_name: - group = self._get_resource_group(**kwargs) - # template = 'az keyvault create -n {} -g {} -l {} --sku {}' - # execute(template.format(name, group, self.location, self.sku)) - return {self.parameter_name: name} - else: - return {self.parameter_name: self.dev_setting_name} - - def remove_resource(self, name, **kwargs): - if not self.skip_delete and not self.dev_setting_name: - group = self._get_resource_group(**kwargs) - # execute('az keyvault delete -n {} -g {} --yes'.format(name, group)) - - def _get_resource_group(self, **kwargs): - try: - return kwargs.get(self.resource_group_parameter_name) - except KeyError: - template = 'To create a KeyVault a resource group is required. Please add ' \ - 'decorator @{} in front of this KeyVault preparer.' - raise CliTestError(template.format(KeyVaultPreparer.__name__)) - - -# Role based access control service principal preparer - -class RoleBasedServicePrincipalPreparer(AbstractPreparer, SingleValueReplacer): - def __init__(self, name_prefix='http://clitest', - skip_assignment=True, parameter_name='sp_name', parameter_password='sp_password', - dev_setting_sp_name='AZURE_CLI_TEST_DEV_SP_NAME', - dev_setting_sp_password='AZURE_CLI_TEST_DEV_SP_PASSWORD'): - super(RoleBasedServicePrincipalPreparer, self).__init__(name_prefix, 24) - self.skip_assignment = skip_assignment - self.result = None - self.parameter_name = parameter_name - self.parameter_password = parameter_password - self.dev_setting_sp_name = os.environ.get(dev_setting_sp_name, None) - self.dev_setting_sp_password = os.environ.get(dev_setting_sp_password, None) - - self.graph_client = get_client_from_cli_profile(GraphRbacManagementClient) - - def create_resource(self, name, **kwargs): - if not self.dev_setting_sp_name: - # command = 'az ad sp create-for-rbac -n {}{}' \ - # .format(name, ' --skip-assignment' if self.skip_assignment else '') - # self.result = execute(command).get_output_in_json() - password = uuid.uuid4() - app = self.graph_client.applications.create( - { - 'available_to_other_tenants': False, - 'display_name': name, - 'identifier_uris': [name], - 'password_creds': [ - PasswordCredential( - value=password, - ) - ] - } - ) - self.result = self.graph_client.service_principals.create( - { - 'app_id': app.app_id, - 'account_enabled': True, - } - ) - return {self.parameter_name: name, self.parameter_password: password} - else: - return {self.parameter_name: self.dev_setting_sp_name, - self.parameter_password: self.dev_setting_sp_password} - - def remove_resource(self, name, **kwargs): - if not self.dev_setting_sp_name: - # execute('az ad sp delete --id {}'.format(self.result['appId'])) - self.graph_client.service_principals.delete(self.result.object_id) - - # Utility def is_preparer_func(fn): From 62920a7efe055d7bd3432e683f070671ae908f4b Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Wed, 24 May 2017 14:52:27 -0700 Subject: [PATCH 017/167] Remove azure.common dependency --- requirements.txt | 1 - setup.py | 1 - src/azure_devtools/scenario_tests/preparers.py | 2 -- 3 files changed, 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9685227ce0f4..2284264e28f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ setuptools-markdown -azure-common~=1.1.6 jmespath mock vcrpy==1.10.3 \ No newline at end of file diff --git a/setup.py b/setup.py index e01183c97be7..bd8d7b8c1e7e 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,6 @@ DEPENDENCIES = [ - 'azure-common~=1.1.6', 'jmespath', 'mock', 'setuptools-markdown', diff --git a/src/azure_devtools/scenario_tests/preparers.py b/src/azure_devtools/scenario_tests/preparers.py index 51dc3f7b6ca9..4211a073cef7 100644 --- a/src/azure_devtools/scenario_tests/preparers.py +++ b/src/azure_devtools/scenario_tests/preparers.py @@ -8,8 +8,6 @@ import os import uuid -from azure.common.client_factory import get_client_from_cli_profile - from .base import ScenarioTest, execute from .utilities import create_random_name from .recording_processors import RecordingProcessor From 6a7e5972bc0949f43f9e3d9351757d34e08ec1a8 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Wed, 24 May 2017 14:57:19 -0700 Subject: [PATCH 018/167] Add dependency on six --- requirements.txt | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 2284264e28f8..c82e88a3a136 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ setuptools-markdown jmespath mock +six vcrpy==1.10.3 \ No newline at end of file diff --git a/setup.py b/setup.py index bd8d7b8c1e7e..4039c57d0e26 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,7 @@ 'jmespath', 'mock', 'setuptools-markdown', + 'six', 'vcrpy==1.10.3', ] From 672cc041e0b68e887b2f216ed3032a8a1bbf8754 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Fri, 26 May 2017 09:40:21 -0700 Subject: [PATCH 019/167] Remove CLI-specific stuff --- src/azure_devtools/scenario_tests/base.py | 118 ++-------------------- 1 file changed, 10 insertions(+), 108 deletions(-) diff --git a/src/azure_devtools/scenario_tests/base.py b/src/azure_devtools/scenario_tests/base.py index d3577afcacbd..514c73a9ea51 100644 --- a/src/azure_devtools/scenario_tests/base.py +++ b/src/azure_devtools/scenario_tests/base.py @@ -17,10 +17,6 @@ import six import vcr -from .patches import (patch_load_cached_subscriptions, patch_main_exception_handler, - patch_retrieve_token_for_user, patch_long_run_operation_delay, - patch_time_sleep_api) -from .exceptions import CliExecutionError from .const import (ENV_LIVE_TEST, ENV_SKIP_ASSERT, ENV_TEST_DIAGNOSE, MOCKED_SUBSCRIPTION_ID) from .recording_processors import (SubscriptionRecordingProcessor, OAuthRequestResponsesFilter, GeneralNameReplacer, LargeRequestBodyProcessor, @@ -29,7 +25,7 @@ from .utilities import create_random_name from .decorators import live_only -logger = logging.getLogger('azure.cli.testsdk') +logger = logging.getLogger('azure.devtools.testsdk') class IntegrationTestBase(unittest.TestCase): @@ -37,20 +33,6 @@ def __init__(self, method_name): super(IntegrationTestBase, self).__init__(method_name) self.diagnose = os.environ.get(ENV_TEST_DIAGNOSE, None) == 'True' - def cmd(self, command, checks=None, expect_failure=False): - if self.diagnose: - begin = datetime.datetime.now() - print('\nExecuting command: {}'.format(command)) - - result = execute(command, expect_failure=expect_failure) - - if self.diagnose: - duration = datetime.datetime.now() - begin - print('\nCommand accomplished in {} s. Exit code {}.\n{}'.format( - duration.total_seconds(), result.exit_code, result.output)) - - return result.assert_with_checks(checks) - def create_random_name(self, prefix, length): # pylint: disable=no-self-use return create_random_name(prefix=prefix, length=length) @@ -122,6 +104,9 @@ def __init__(self, method_name): self.name_replacer] self.replay_processors = [LargeResponseBodyReplacer(), DeploymentNameReplacer()] + self.recording_patches = [] + self.replay_patches = [] + test_file_path = inspect.getfile(self.__class__) recordings_dir = os.path.join(os.path.dirname(test_file_path), 'recordings') live_test = os.environ.get(ENV_LIVE_TEST, None) == 'True' @@ -153,13 +138,12 @@ def setUp(self): self.addCleanup(cm.__exit__) # set up mock patches - patch_main_exception_handler(self) - - if not self.in_recording: - patch_time_sleep_api(self) - patch_long_run_operation_delay(self) - patch_load_cached_subscriptions(self) - patch_retrieve_token_for_user(self) + if self.in_recording: + for patch in self.recording_patches: + patch(self) + else: + for patch in self.replay_patches: + patch(self) def tearDown(self): os.environ = self.original_env @@ -234,85 +218,3 @@ def _custom_request_query_matcher(cls, r1, r2): return False return True - - -class ExecutionResult(object): # pylint: disable=too-few-public-methods - def __init__(self, command, expect_failure=False, in_process=True): - logger.info('Execute command %s', command) - if in_process: - self._in_process_execute(command) - else: - self._out_of_process_execute(command) - - if expect_failure and self.exit_code == 0: - raise AssertionError('The command is expected to fail but it doesn\'.') - elif not expect_failure and self.exit_code != 0: - raise AssertionError('The command failed. Exit code: {}'.format(self.exit_code)) - - self.json_value = None - self.skip_assert = os.environ.get(ENV_SKIP_ASSERT, None) == 'True' - - def assert_with_checks(self, *args): - checks = [] - for each in args: - if isinstance(each, list): - checks.extend(each) - elif callable(each): - checks.append(each) - - logger.info('Checkers to be executed %s', len(checks)) - - if not self.skip_assert: - for c in checks: - c(self) - - return self - - def get_output_in_json(self): - if not self.json_value: - self.json_value = json.loads(self.output) - - if self.json_value is None: - raise AssertionError('The command output cannot be parsed in json.') - - return self.json_value - - def _in_process_execute(self, command): - # from azure.cli import as cli_main - from azure.cli.main import main as cli_main - from six import StringIO - from vcr.errors import CannotOverwriteExistingCassetteException - - if command.startswith('az '): - command = command[3:] - - output_buffer = StringIO() - try: - # issue: stderr cannot be redirect in this form, as a result some failure information - # is lost when command fails. - self.exit_code = cli_main(shlex.split(command), file=output_buffer) or 0 - self.output = output_buffer.getvalue() - except CannotOverwriteExistingCassetteException as ex: - raise AssertionError(ex) - except CliExecutionError as ex: - if ex.exception: - raise ex.exception - else: - raise ex - except Exception as ex: # pylint: disable=broad-except - self.exit_code = 1 - self.output = output_buffer.getvalue() - self.process_error = ex - finally: - output_buffer.close() - - def _out_of_process_execute(self, command): - try: - self.output = subprocess.check_output(shlex.split(command)).decode('utf-8') - self.exit_code = 0 - except subprocess.CalledProcessError as error: - self.exit_code, self.output = error.returncode, error.output.decode('utf-8') - self.process_error = error - - -execute = ExecutionResult From 7d96614856ebe07dc2601a345bb5c61ff17160d9 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Fri, 26 May 2017 09:40:52 -0700 Subject: [PATCH 020/167] Remove CLI exception --- src/azure_devtools/scenario_tests/exceptions.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/azure_devtools/scenario_tests/exceptions.py b/src/azure_devtools/scenario_tests/exceptions.py index c8914a2aa1b6..66da6e9e2c2e 100644 --- a/src/azure_devtools/scenario_tests/exceptions.py +++ b/src/azure_devtools/scenario_tests/exceptions.py @@ -10,14 +10,6 @@ def __init__(self, error_message): super(AzureTestError, self).__init__(message.format(error_message)) -class CliExecutionError(Exception): - def __init__(self, exception): - self.exception = exception - message = 'The CLI throws exception {} during execution and fails the command.' - super(CliExecutionError, self).__init__(message.format(exception.__class__.__name__, - exception)) - - class JMESPathCheckAssertionError(AssertionError): def __init__(self, query, expected, actual, json_data): message = "Query '{}' doesn't yield expected value '{}', instead the actual value " \ From 41c77e87a33e37c3270fbe9586180342341ae86b Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Fri, 26 May 2017 10:45:49 -0700 Subject: [PATCH 021/167] Remove most CLI-centric patches --- src/azure_devtools/scenario_tests/patches.py | 58 +------------------- 1 file changed, 2 insertions(+), 56 deletions(-) diff --git a/src/azure_devtools/scenario_tests/patches.py b/src/azure_devtools/scenario_tests/patches.py index 00dac5bc16ca..8639cd871f97 100644 --- a/src/azure_devtools/scenario_tests/patches.py +++ b/src/azure_devtools/scenario_tests/patches.py @@ -3,64 +3,10 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -from .exceptions import CliExecutionError, AzureTestError +from .exceptions import AzureTestError from .const import MOCKED_SUBSCRIPTION_ID, MOCKED_TENANT_ID -def patch_main_exception_handler(unit_test): - from vcr.errors import CannotOverwriteExistingCassetteException - - def _handle_main_exception(ex): - if isinstance(ex, CannotOverwriteExistingCassetteException): - # This exception usually caused by a no match HTTP request. This is a product error - # that is caused by change of SDK invocation. - raise ex - - raise CliExecutionError(ex) - - _mock_in_unit_test(unit_test, 'azure.cli.main.handle_exception', _handle_main_exception) - - -def patch_load_cached_subscriptions(unit_test): - def _handle_load_cached_subscription(*args, **kwargs): # pylint: disable=unused-argument - - return [{ - "id": MOCKED_SUBSCRIPTION_ID, - "user": { - "name": "example@example.com", - "type": "user" - }, - "state": "Enabled", - "name": "Example", - "tenantId": MOCKED_TENANT_ID, - "isDefault": True}] - - _mock_in_unit_test(unit_test, - 'azure.cli.core._profile.Profile.load_cached_subscriptions', - _handle_load_cached_subscription) - - -def patch_retrieve_token_for_user(unit_test): - def _retrieve_token_for_user(*args, **kwargs): # pylint: disable=unused-argument - return 'Bearer', 'top-secret-token-for-you' - - _mock_in_unit_test(unit_test, - 'azure.cli.core._profile.CredsCache.retrieve_token_for_user', - _retrieve_token_for_user) - - -def patch_long_run_operation_delay(unit_test): - def _shortcut_long_run_operation(*args, **kwargs): # pylint: disable=unused-argument - return - - _mock_in_unit_test(unit_test, - 'msrestazure.azure_operation.AzureOperationPoller._delay', - _shortcut_long_run_operation) - _mock_in_unit_test(unit_test, - 'azure.cli.core.commands.LongRunningOperation._delay', - _shortcut_long_run_operation) - - def patch_time_sleep_api(unit_test): def _time_sleep_skip(*_): return @@ -73,7 +19,7 @@ def _mock_in_unit_test(unit_test, target, replacement): import unittest if not isinstance(unit_test, unittest.TestCase): - raise AzureTestError('The patch_main_exception_handler can be only used in unit test') + raise AzureTestError('Patches can be only called from a unit test') mp = mock.patch(target, replacement) mp.__enter__() From e2d96cca18aa64f72498a9d0cffcfb4188b060ab Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Fri, 26 May 2017 10:54:51 -0700 Subject: [PATCH 022/167] Remove older CLI-centric base class --- .../scenario_tests/vcr_test_base.py | 506 ------------------ 1 file changed, 506 deletions(-) delete mode 100644 src/azure_devtools/scenario_tests/vcr_test_base.py diff --git a/src/azure_devtools/scenario_tests/vcr_test_base.py b/src/azure_devtools/scenario_tests/vcr_test_base.py deleted file mode 100644 index 9749fee81418..000000000000 --- a/src/azure_devtools/scenario_tests/vcr_test_base.py +++ /dev/null @@ -1,506 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -# pylint: disable=wrong-import-order - -from __future__ import print_function - -import collections -import json -import os -import re -import shlex -import sys -import tempfile -import traceback -from random import choice -from string import digits, ascii_lowercase - -from six.moves.urllib.parse import urlparse, parse_qs # pylint: disable=import-error - -import unittest -try: - import unittest.mock as mock -except ImportError: - import mock - -import vcr -import jmespath -from six import StringIO - -# TODO Should not depend on azure.cli.main package here. -# Will be ok if this test file is not part of azure.cli.core.utils -from azure.cli.main import main as cli_main - -from azure.cli.core import __version__ as core_version -import azure.cli.core._debug as _debug -from azure.cli.core._profile import Profile, CLOUD -from azure.cli.core.util import CLIError - -LIVE_TEST_CONTROL_ENV = 'AZURE_CLI_TEST_RUN_LIVE' -COMMAND_COVERAGE_CONTROL_ENV = 'AZURE_CLI_TEST_COMMAND_COVERAGE' -MOCKED_SUBSCRIPTION_ID = '00000000-0000-0000-0000-000000000000' -MOCKED_TENANT_ID = '00000000-0000-0000-0000-000000000000' -MOCKED_STORAGE_ACCOUNT = 'dummystorage' - - -# MOCK METHODS - -# Workaround until https://github.com/kevin1024/vcrpy/issues/293 is fixed. -vcr_connection_request = vcr.stubs.VCRConnection.request - - -def patch_vcr_connection_request(*args, **kwargs): - kwargs.pop('encode_chunked', None) - vcr_connection_request(*args, **kwargs) - - -vcr.stubs.VCRConnection.request = patch_vcr_connection_request - - -def _mock_get_mgmt_service_client(client_type, subscription_bound=True, subscription_id=None, - api_version=None, base_url_bound=None, **kwargs): - # version of _get_mgmt_service_client to use when recording or playing tests - profile = Profile() - cred, subscription_id, _ = profile.get_login_credentials(subscription_id=subscription_id) - client_kwargs = {} - - if base_url_bound: - client_kwargs = {'base_url': CLOUD.endpoints.resource_manager} - if api_version: - client_kwargs['api_version'] = api_version - if kwargs: - client_kwargs.update(kwargs) - - if subscription_bound: - client = client_type(cred, subscription_id, **client_kwargs) - else: - client = client_type(cred, **client_kwargs) - - client = _debug.change_ssl_cert_verification(client) - - client.config.add_user_agent("AZURECLI/TEST/{}".format(core_version)) - - return (client, subscription_id) - - -def _mock_generate_deployment_name(namespace): - if not namespace.deployment_name: - namespace.deployment_name = 'mock-deployment' - - -def _mock_handle_exceptions(ex): - raise ex - - -def _mock_subscriptions(self): # pylint: disable=unused-argument - return [{ - "id": MOCKED_SUBSCRIPTION_ID, - "user": { - "name": "example@example.com", - "type": "user" - }, - "state": "Enabled", - "name": "Example", - "tenantId": MOCKED_TENANT_ID, - "isDefault": True}] - - -def _mock_user_access_token(_, _1, _2, _3): # pylint: disable=unused-argument - return ('Bearer', 'top-secret-token-for-you') - - -def _mock_operation_delay(_): - # don't run time.sleep() - return - - -# TEST CHECKS - - -class JMESPathCheckAssertionError(AssertionError): - def __init__(self, comparator, actual_result, json_data): - message = "Actual value '{}' != Expected value '{}'. ".format( - actual_result, - comparator.expected_result) - message += "Query '{}' used on json data '{}'".format(comparator.query, json_data) - super(JMESPathCheckAssertionError, self).__init__(message) - - -class JMESPathCheck(object): # pylint: disable=too-few-public-methods - - def __init__(self, query, expected_result): - self.query = query - self.expected_result = expected_result - - def compare(self, json_data): - actual_result = _search_result_by_jmespath(json_data, self.query) - if not actual_result == self.expected_result: - raise JMESPathCheckAssertionError(self, actual_result, json_data) - - -class JMESPathPatternCheck(object): # pylint: disable=too-few-public-methods - - def __init__(self, query, expected_result): - self.query = query - self.expected_result = expected_result - - def compare(self, json_data): - actual_result = _search_result_by_jmespath(json_data, self.query) - if not re.match(self.expected_result, str(actual_result), re.IGNORECASE): - raise JMESPathCheckAssertionError(self, actual_result, json_data) - - -class BooleanCheck(object): # pylint: disable=too-few-public-methods - - def __init__(self, expected_result): - self.expected_result = expected_result - - def compare(self, data): - result = str(str(data).lower() in ['yes', 'true', '1']) - try: - assert result == str(self.expected_result) - except AssertionError: - raise AssertionError("Actual value '{}' != Expected value {}".format( - result, self.expected_result)) - - -class NoneCheck(object): # pylint: disable=too-few-public-methods - - def __init__(self): - pass - - def compare(self, data): # pylint: disable=no-self-use - none_strings = ['[]', '{}', 'false'] - try: - assert not data or data in none_strings - except AssertionError: - raise AssertionError("Actual value '{}' != Expected value falsy (None, '', []) or " - "string in {}".format(data, none_strings)) - - -class StringCheck(object): # pylint: disable=too-few-public-methods - - def __init__(self, expected_result): - self.expected_result = expected_result - - def compare(self, data): - try: - result = data.replace('"', '') - assert result == self.expected_result - except AssertionError: - raise AssertionError("Actual value '{}' != Expected value {}".format( - data, self.expected_result)) - - -# HELPER METHODS - - -def _scrub_deployment_name(uri): - return re.sub('/deployments/([^/?]+)', '/deployments/mock-deployment', uri) - - -def _scrub_service_principal_name(uri): - return re.sub('userPrincipalName%20eq%20%27(.+)%27', - 'userPrincipalName%20eq%20%27example%40example.com%27', uri) - - -def _search_result_by_jmespath(json_data, query): - if not json_data: - json_data = '{}' - json_val = json.loads(json_data) - return jmespath.search( - query, - json_val, - jmespath.Options(collections.OrderedDict)) - - -def _custom_request_matcher(r1, r2): - """ Ensure method, path, and query parameters match. """ - if r1.method != r2.method: - return False - - url1 = urlparse(r1.uri) - url2 = urlparse(r2.uri) - - if url1.path != url2.path: - return False - - q1 = parse_qs(url1.query) - q2 = parse_qs(url2.query) - shared_keys = set(q1.keys()).intersection(set(q2.keys())) - - if len(shared_keys) != len(q1) or len(shared_keys) != len(q2): - return False - - for key in shared_keys: - if q1[key][0].lower() != q2[key][0].lower(): - return False - - return True - - -# MAIN CLASS - - -class VCRTestBase(unittest.TestCase): # pylint: disable=too-many-instance-attributes - - FILTER_HEADERS = [ - 'authorization', - 'client-request-id', - 'x-ms-client-request-id', - 'x-ms-correlation-request-id', - 'x-ms-ratelimit-remaining-subscription-reads', - 'x-ms-request-id', - 'x-ms-routing-request-id', - 'x-ms-gateway-service-instanceid', - 'x-ms-ratelimit-remaining-tenant-reads', - 'x-ms-served-by', - ] - - def __init__(self, test_file, test_name, run_live=False, debug=False, debug_vcr=False, - skip_setup=False, skip_teardown=False): - super(VCRTestBase, self).__init__(test_name) - self.test_name = test_name - self.recording_dir = os.path.join(os.path.dirname(test_file), 'recordings') - self.cassette_path = os.path.join(self.recording_dir, '{}.yaml'.format(test_name)) - self.playback = os.path.isfile(self.cassette_path) - - if os.environ.get(LIVE_TEST_CONTROL_ENV, None) == 'True': - self.run_live = True - else: - self.run_live = run_live - - self.skip_setup = skip_setup - self.skip_teardown = skip_teardown - self.success = False - self.exception = None - self.track_commands = os.environ.get(COMMAND_COVERAGE_CONTROL_ENV, None) - self._debug = debug - - if not self.playback and ('--buffer' in sys.argv) and not run_live: - self.exception = CLIError('No recorded result provided for {}.'.format(self.test_name)) - - if debug_vcr: - import logging - logging.basicConfig() - vcr_log = logging.getLogger('vcr') - vcr_log.setLevel(logging.INFO) - self.my_vcr = vcr.VCR( - cassette_library_dir=self.recording_dir, - before_record_request=self._before_record_request, - before_record_response=self._before_record_response, - decode_compressed_response=True - ) - self.my_vcr.register_matcher('custom', _custom_request_matcher) - self.my_vcr.match_on = ['custom'] - - def _track_executed_commands(self, command): - if self.track_commands: - with open(self.track_commands, 'a+') as f: - f.write(' '.join(command)) - f.write('\n') - - def _before_record_request(self, request): # pylint: disable=no-self-use - # scrub subscription from the uri - request.uri = re.sub('/subscriptions/([^/]+)/', - '/subscriptions/{}/'.format(MOCKED_SUBSCRIPTION_ID), request.uri) - # scrub jobId from uri, required for ADLA - request.uri = re.sub('/Jobs/([^/]+)', - '/Jobs/{}'.format(MOCKED_SUBSCRIPTION_ID), request.uri) - request.uri = re.sub('/graph.windows.net/([^/]+)/', - '/graph.windows.net/{}/'.format(MOCKED_TENANT_ID), request.uri) - request.uri = re.sub('/sig=([^/]+)&', '/sig=0000&', request.uri) - request.uri = _scrub_deployment_name(request.uri) - request.uri = _scrub_service_principal_name(request.uri) - - # replace random storage account name with dummy name - request.uri = re.sub(r'(vcrstorage[\d]+)', MOCKED_STORAGE_ACCOUNT, request.uri) - # prevents URI mismatch between Python 2 and 3 if request URI has extra / chars - request.uri = re.sub('//', '/', request.uri) - request.uri = re.sub('/', '//', request.uri, count=1) - # do not record requests sent for token refresh' - if (request.body and 'grant-type=refresh_token' in str(request.body)) or \ - ('/oauth2/token' in request.uri): - request = None - return request - - def _before_record_response(self, response): # pylint: disable=no-self-use - for key in VCRTestBase.FILTER_HEADERS: - if key in response['headers']: - del response['headers'][key] - - def _scrub_body_parameters(value): - value = re.sub('/subscriptions/([^/]+)/', - '/subscriptions/{}/'.format(MOCKED_SUBSCRIPTION_ID), value) - value = re.sub('\"jobId\": \"([^/]+)\"', - '\"jobId\": \"{}\"'.format(MOCKED_SUBSCRIPTION_ID), value) - return value - - for key in response['body']: - value = response['body'][key].decode('utf-8') - value = _scrub_body_parameters(value) - try: - response['body'][key] = bytes(value, 'utf-8') - except TypeError: - response['body'][key] = value.encode('utf-8') - - return response - - @mock.patch('azure.cli.main.handle_exception', _mock_handle_exceptions) - @mock.patch('azure.cli.core.commands.client_factory._get_mgmt_service_client', - _mock_get_mgmt_service_client) # pylint: disable=line-too-long - def _execute_live_or_recording(self): - # pylint: disable=no-member - try: - set_up = getattr(self, "set_up", None) - if callable(set_up) and not self.skip_setup: - self.set_up() - - if self.run_live: - self.body() - else: - with self.my_vcr.use_cassette(self.cassette_path): - self.body() - self.success = True - except Exception as ex: - raise ex - finally: - tear_down = getattr(self, "tear_down", None) - if callable(tear_down) and not self.skip_teardown: - self.tear_down() - - @mock.patch('azure.cli.core._profile.Profile.load_cached_subscriptions', _mock_subscriptions) - @mock.patch('azure.cli.core._profile.CredsCache.retrieve_token_for_user', - _mock_user_access_token) # pylint: disable=line-too-long - @mock.patch('azure.cli.main.handle_exception', _mock_handle_exceptions) - @mock.patch('azure.cli.core.commands.client_factory._get_mgmt_service_client', - _mock_get_mgmt_service_client) # pylint: disable=line-too-long - @mock.patch('msrestazure.azure_operation.AzureOperationPoller._delay', _mock_operation_delay) - @mock.patch('time.sleep', _mock_operation_delay) - @mock.patch('azure.cli.core.commands.LongRunningOperation._delay', _mock_operation_delay) - @mock.patch('azure.cli.core.commands.validators.generate_deployment_name', - _mock_generate_deployment_name) - def _execute_playback(self): - # pylint: disable=no-member - with self.my_vcr.use_cassette(self.cassette_path): - self.body() - self.success = True - - def _post_recording_scrub(self): - """ Perform post-recording cleanup on the YAML file that can't be accomplished with the - VCR recording hooks. """ - src_path = self.cassette_path - rg_name = getattr(self, 'resource_group', None) - rg_original = getattr(self, 'resource_group_original', None) - - t = tempfile.NamedTemporaryFile('r+') - with open(src_path, 'r') as f: - for line in f: - # scrub resource group names - if rg_name != rg_original: - line = line.replace(rg_name, rg_original) - # omit bearer tokens - if 'authorization:' not in line.lower(): - t.write(line) - t.seek(0) - with open(src_path, 'w') as f: - for line in t: - f.write(line) - t.close() - - # COMMAND METHODS - - def cmd(self, command, checks=None, allowed_exceptions=None, - debug=False): # pylint: disable=no-self-use - allowed_exceptions = allowed_exceptions or [] - if not isinstance(allowed_exceptions, list): - allowed_exceptions = [allowed_exceptions] - - if self._debug or debug: - print('\n\tRUNNING: {}'.format(command)) - command_list = shlex.split(command) - output = StringIO() - try: - cli_main(command_list, file=output) - except Exception as ex: # pylint: disable=broad-except - ex_msg = str(ex) - if not next((x for x in allowed_exceptions if x in ex_msg), None): - raise ex - self._track_executed_commands(command_list) - result = output.getvalue().strip() - output.close() - - if self._debug or debug: - print('\tRESULT: {}\n'.format(result)) - - if checks: - checks = [checks] if not isinstance(checks, list) else checks - for check in checks: - check.compare(result) - - if '-o' in command_list and 'tsv' in command_list: - return result - else: - try: - result = result or '{}' - return json.loads(result) - except Exception: # pylint: disable=broad-except - return result - - def set_env(self, key, val): # pylint: disable=no-self-use - os.environ[key] = val - - def pop_env(self, key): # pylint: disable=no-self-use - return os.environ.pop(key, None) - - def execute(self): - ''' Method to actually start execution of the test. Must be called from the test_ - method of the test class. ''' - try: - if self.run_live: - print('RUN LIVE: {}'.format(self.test_name)) - self._execute_live_or_recording() - elif self.playback: - print('PLAYBACK: {}'.format(self.test_name)) - self._execute_playback() - else: - print('RECORDING: {}'.format(self.test_name)) - self._execute_live_or_recording() - except Exception as ex: - traceback.print_exc() - raise ex - finally: - if not self.success and not self.playback and os.path.isfile(self.cassette_path): - print('DISCARDING RECORDING: {}'.format(self.cassette_path)) - os.remove(self.cassette_path) - elif self.success and not self.playback and os.path.isfile(self.cassette_path): - try: - self._post_recording_scrub() - except Exception: # pylint: disable=broad-except - os.remove(self.cassette_path) - - -class ResourceGroupVCRTestBase(VCRTestBase): - - def __init__(self, test_file, test_name, resource_group='vcr_resource_group', run_live=False, - debug=False, debug_vcr=False, skip_setup=False, skip_teardown=False, - random_tag_format=None): - super(ResourceGroupVCRTestBase, self).__init__(test_file, test_name, run_live=run_live, - debug=debug, debug_vcr=debug_vcr, - skip_setup=skip_setup, - skip_teardown=skip_teardown) - self.resource_group_original = resource_group - random_tag = (random_tag_format or '_{}_').format( - ''.join((choice(ascii_lowercase + digits) for _ in range(4)))) - self.resource_group = '{}{}'.format(resource_group, '' if self.playback else random_tag) - self.location = 'westus' - - def set_up(self): - self.cmd('group create --location {} --name {} --tags use=az-test'.format( - self.location, self.resource_group)) - - def tear_down(self): - self.cmd('group delete --name {} --no-wait --yes'.format(self.resource_group)) From 27dfe7c3a81752169689467b6d449b239fa07cec Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Fri, 26 May 2017 11:03:08 -0700 Subject: [PATCH 023/167] Add sleep patch back in --- src/azure_devtools/scenario_tests/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/azure_devtools/scenario_tests/base.py b/src/azure_devtools/scenario_tests/base.py index 514c73a9ea51..ea3bd53936e1 100644 --- a/src/azure_devtools/scenario_tests/base.py +++ b/src/azure_devtools/scenario_tests/base.py @@ -18,6 +18,7 @@ import vcr from .const import (ENV_LIVE_TEST, ENV_SKIP_ASSERT, ENV_TEST_DIAGNOSE, MOCKED_SUBSCRIPTION_ID) +from .patches import patch_time_sleep_api from .recording_processors import (SubscriptionRecordingProcessor, OAuthRequestResponsesFilter, GeneralNameReplacer, LargeRequestBodyProcessor, LargeResponseBodyProcessor, LargeResponseBodyReplacer, @@ -105,7 +106,7 @@ def __init__(self, method_name): self.replay_processors = [LargeResponseBodyReplacer(), DeploymentNameReplacer()] self.recording_patches = [] - self.replay_patches = [] + self.replay_patches = [patch_time_sleep_api] test_file_path = inspect.getfile(self.__class__) recordings_dir = os.path.join(os.path.dirname(test_file_path), 'recordings') From efc0759921f6211c892351f74a813733a0971f26 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Fri, 26 May 2017 13:49:47 -0700 Subject: [PATCH 024/167] First steps toward a unified config --- requirements.txt | 1 + src/azure_devtools/scenario_tests/config.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 src/azure_devtools/scenario_tests/config.py diff --git a/requirements.txt b/requirements.txt index c82e88a3a136..427e9342eafb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ setuptools-markdown +ConfigArgParse jmespath mock six diff --git a/src/azure_devtools/scenario_tests/config.py b/src/azure_devtools/scenario_tests/config.py new file mode 100644 index 000000000000..bb4d47cb9bcb --- /dev/null +++ b/src/azure_devtools/scenario_tests/config.py @@ -0,0 +1,20 @@ +import configargparse + + +class TestConfig(object): + def __init__(self, parent_parsers=None, config_file=None): + parent_parsers = parent_parsers or [] + self.parser = configargparse.ArgumentParser(parents=parent_parsers) + self.parser.add_argument( + '-c', '--config', is_config_file=True, default=config_file, + help='Path to a configuration file in YAML format.' + ) + self.parser.add_argument( + '-m', '--mode', choices=['playback', 'live', 'record'], default='playback', + env_var='AZURE_TESTS_RECORDING_MODE', + help='Test recording mode.' + ) + self.args = configargparse.Namespace() + + def parse_args(self): + self.args = self.parser.parse_args() \ No newline at end of file From d362aefa107208b0ac99a028d6c3a863d6737b57 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Fri, 26 May 2017 15:31:01 -0700 Subject: [PATCH 025/167] Config and no-record headers Add config object to ScenarioTest Add fake header for deactivating recording Align cmdline options with vcr.py options --- src/azure_devtools/scenario_tests/base.py | 11 +++++++++-- src/azure_devtools/scenario_tests/config.py | 10 +++++----- src/azure_devtools/scenario_tests/const.py | 3 +++ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/src/azure_devtools/scenario_tests/base.py b/src/azure_devtools/scenario_tests/base.py index ea3bd53936e1..d78548fb22f3 100644 --- a/src/azure_devtools/scenario_tests/base.py +++ b/src/azure_devtools/scenario_tests/base.py @@ -17,7 +17,9 @@ import six import vcr -from .const import (ENV_LIVE_TEST, ENV_SKIP_ASSERT, ENV_TEST_DIAGNOSE, MOCKED_SUBSCRIPTION_ID) +from .config import TestConfig +from .const import (ENV_LIVE_TEST, ENV_SKIP_ASSERT, ENV_TEST_DIAGNOSE, MOCKED_SUBSCRIPTION_ID, + DUMMY_HEADER_DEACTIVATE_VCR_RECORDING) from .patches import patch_time_sleep_api from .recording_processors import (SubscriptionRecordingProcessor, OAuthRequestResponsesFilter, GeneralNameReplacer, LargeRequestBodyProcessor, @@ -108,6 +110,8 @@ def __init__(self, method_name): self.recording_patches = [] self.replay_patches = [patch_time_sleep_api] + self.config = TestConfig() + test_file_path = inspect.getfile(self.__class__) recordings_dir = os.path.join(os.path.dirname(test_file_path), 'recordings') live_test = os.environ.get(ENV_LIVE_TEST, None) == 'True' @@ -117,7 +121,7 @@ def __init__(self, method_name): before_record_request=self._process_request_recording, before_record_response=self._process_response_recording, decode_compressed_response=True, - record_mode='once' if not live_test else 'all', + record_mode=config.record_mode, filter_headers=self.FILTER_HEADERS ) self.vcr.register_matcher('query', self._custom_request_query_matcher) @@ -179,6 +183,9 @@ def _process_response_recording(self, response): # make header name lower case and filter unwanted headers headers = {} for key in response['headers']: + if key.lower() == DUMMY_HEADER_DEACTIVATE_VCR_RECORDING: + # Disable recording + return None if key.lower() not in self.FILTER_HEADERS: headers[key.lower()] = response['headers'][key] response['headers'] = headers diff --git a/src/azure_devtools/scenario_tests/config.py b/src/azure_devtools/scenario_tests/config.py index bb4d47cb9bcb..0e173d5cb86e 100644 --- a/src/azure_devtools/scenario_tests/config.py +++ b/src/azure_devtools/scenario_tests/config.py @@ -10,11 +10,11 @@ def __init__(self, parent_parsers=None, config_file=None): help='Path to a configuration file in YAML format.' ) self.parser.add_argument( - '-m', '--mode', choices=['playback', 'live', 'record'], default='playback', - env_var='AZURE_TESTS_RECORDING_MODE', + '-m', '--record-mode', choices=['once', 'all'], default='once', + env_var='AZURE_TESTS_RECORD_MODE', help='Test recording mode.' ) - self.args = configargparse.Namespace() + self.args = self.parser.parse_args() - def parse_args(self): - self.args = self.parser.parse_args() \ No newline at end of file + def record_mode(self): + return self.args.mode \ No newline at end of file diff --git a/src/azure_devtools/scenario_tests/const.py b/src/azure_devtools/scenario_tests/const.py index 624bac5f84c6..997461371d55 100644 --- a/src/azure_devtools/scenario_tests/const.py +++ b/src/azure_devtools/scenario_tests/const.py @@ -12,3 +12,6 @@ ENV_LIVE_TEST = 'AZURE_CLI_TEST_RUN_LIVE' ENV_SKIP_ASSERT = 'AZURE_CLI_TEST_SKIP_ASSERT' ENV_TEST_DIAGNOSE = 'AZURE_CLI_TEST_DIAGNOSE' + +# Special header to turn off recording +DUMMY_HEADER_DEACTIVATE_VCR_RECORDING = 'x-ms-deactivate-vcr-recording' \ No newline at end of file From b57c1e21217269f026109308f8d9dee61a55ab95 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Fri, 26 May 2017 15:42:39 -0700 Subject: [PATCH 026/167] Move recording-deactivation header detection to right place --- src/azure_devtools/scenario_tests/base.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/azure_devtools/scenario_tests/base.py b/src/azure_devtools/scenario_tests/base.py index d78548fb22f3..a6090c5ca00d 100644 --- a/src/azure_devtools/scenario_tests/base.py +++ b/src/azure_devtools/scenario_tests/base.py @@ -165,6 +165,10 @@ def create_random_name(self, prefix, length): return moniker def _process_request_recording(self, request): + if DUMMY_HEADER_DEACTIVATE_VCR_RECORDING in request.headers: + # Disable recording + return None + if self.in_recording: for processor in self.recording_processors: request = processor.process_request(request) @@ -183,9 +187,6 @@ def _process_response_recording(self, response): # make header name lower case and filter unwanted headers headers = {} for key in response['headers']: - if key.lower() == DUMMY_HEADER_DEACTIVATE_VCR_RECORDING: - # Disable recording - return None if key.lower() not in self.FILTER_HEADERS: headers[key.lower()] = response['headers'][key] response['headers'] = headers From eba76c28bd43838329f40da3e587f39e87ea6ea3 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Tue, 30 May 2017 12:55:09 -0700 Subject: [PATCH 027/167] Add way to set default config file from scenario test --- src/azure_devtools/scenario_tests/base.py | 8 +++++--- src/azure_devtools/scenario_tests/config.py | 11 +++++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/azure_devtools/scenario_tests/base.py b/src/azure_devtools/scenario_tests/base.py index a6090c5ca00d..4423d27881fe 100644 --- a/src/azure_devtools/scenario_tests/base.py +++ b/src/azure_devtools/scenario_tests/base.py @@ -17,7 +17,7 @@ import six import vcr -from .config import TestConfig +from .config import TestConfig, RecordMode from .const import (ENV_LIVE_TEST, ENV_SKIP_ASSERT, ENV_TEST_DIAGNOSE, MOCKED_SUBSCRIPTION_ID, DUMMY_HEADER_DEACTIVATE_VCR_RECORDING) from .patches import patch_time_sleep_api @@ -96,6 +96,8 @@ class ScenarioTest(IntegrationTestBase): # pylint: disable=too-many-instance-at 'x-ms-served-by', ] + TEST_CONFIG_FILE = None + def __init__(self, method_name): super(ScenarioTest, self).__init__(method_name) self.name_replacer = GeneralNameReplacer() @@ -110,11 +112,11 @@ def __init__(self, method_name): self.recording_patches = [] self.replay_patches = [patch_time_sleep_api] - self.config = TestConfig() + self.config = TestConfig(config_file=self.TEST_CONFIG_FILE) test_file_path = inspect.getfile(self.__class__) recordings_dir = os.path.join(os.path.dirname(test_file_path), 'recordings') - live_test = os.environ.get(ENV_LIVE_TEST, None) == 'True' + live_test = self.config.record_mode == RecordMode.all self.vcr = vcr.VCR( cassette_library_dir=recordings_dir, diff --git a/src/azure_devtools/scenario_tests/config.py b/src/azure_devtools/scenario_tests/config.py index 0e173d5cb86e..46604613eb0a 100644 --- a/src/azure_devtools/scenario_tests/config.py +++ b/src/azure_devtools/scenario_tests/config.py @@ -1,6 +1,11 @@ import configargparse +class RecordMode(object): + once = 'once' + all = 'all' + + class TestConfig(object): def __init__(self, parent_parsers=None, config_file=None): parent_parsers = parent_parsers or [] @@ -10,11 +15,13 @@ def __init__(self, parent_parsers=None, config_file=None): help='Path to a configuration file in YAML format.' ) self.parser.add_argument( - '-m', '--record-mode', choices=['once', 'all'], default='once', + '-m', '--record-mode', choices=[RecordMode.once, RecordMode.all], + default=RecordMode.once, env_var='AZURE_TESTS_RECORD_MODE', help='Test recording mode.' ) - self.args = self.parser.parse_args() + self.args = self.parser.parse_args([]) + @property def record_mode(self): return self.args.mode \ No newline at end of file From e6f703f4996c0e54be29fbf0759dddf6f6ed400b Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Wed, 31 May 2017 14:16:17 -0700 Subject: [PATCH 028/167] Remove 'CLI' from env vars --- src/azure_devtools/scenario_tests/const.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/azure_devtools/scenario_tests/const.py b/src/azure_devtools/scenario_tests/const.py index 997461371d55..18d0ba18c4ff 100644 --- a/src/azure_devtools/scenario_tests/const.py +++ b/src/azure_devtools/scenario_tests/const.py @@ -8,10 +8,7 @@ MOCKED_TENANT_ID = '00000000-0000-0000-0000-000000000000' # Configuration environment variable -ENV_COMMAND_COVERAGE = 'AZURE_CLI_TEST_COMMAND_COVERAGE' -ENV_LIVE_TEST = 'AZURE_CLI_TEST_RUN_LIVE' -ENV_SKIP_ASSERT = 'AZURE_CLI_TEST_SKIP_ASSERT' -ENV_TEST_DIAGNOSE = 'AZURE_CLI_TEST_DIAGNOSE' - -# Special header to turn off recording -DUMMY_HEADER_DEACTIVATE_VCR_RECORDING = 'x-ms-deactivate-vcr-recording' \ No newline at end of file +ENV_COMMAND_COVERAGE = 'AZURE_TEST_COMMAND_COVERAGE' +ENV_LIVE_TEST = 'AZURE_TEST_RUN_LIVE' +ENV_SKIP_ASSERT = 'AZURE_TEST_SKIP_ASSERT' +ENV_TEST_DIAGNOSE = 'AZURE_TEST_DIAGNOSE' \ No newline at end of file From 55597e34de7d7fb659979d82839e63b77d6b2c87 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Wed, 31 May 2017 14:16:48 -0700 Subject: [PATCH 029/167] Reinstate patch_long_run_operation_delay --- src/azure_devtools/scenario_tests/patches.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/azure_devtools/scenario_tests/patches.py b/src/azure_devtools/scenario_tests/patches.py index 8639cd871f97..7ca1edf68dc7 100644 --- a/src/azure_devtools/scenario_tests/patches.py +++ b/src/azure_devtools/scenario_tests/patches.py @@ -14,6 +14,15 @@ def _time_sleep_skip(*_): _mock_in_unit_test(unit_test, 'time.sleep', _time_sleep_skip) +def patch_long_run_operation_delay(unit_test): + def _shortcut_long_run_operation(*args, **kwargs): # pylint: disable=unused-argument + return + + _mock_in_unit_test(unit_test, + 'msrestazure.azure_operation.AzureOperationPoller._delay', + _shortcut_long_run_operation) + + def _mock_in_unit_test(unit_test, target, replacement): import mock import unittest From 32cc32fc5b361855aeac9347daaa1150c3e57b42 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Wed, 31 May 2017 14:17:27 -0700 Subject: [PATCH 030/167] New record disabling mechanism --- .../scenario_tests/preparers.py | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/src/azure_devtools/scenario_tests/preparers.py b/src/azure_devtools/scenario_tests/preparers.py index 4211a073cef7..59ebe2e80a6f 100644 --- a/src/azure_devtools/scenario_tests/preparers.py +++ b/src/azure_devtools/scenario_tests/preparers.py @@ -3,6 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- +import contextlib import inspect import functools import os @@ -16,13 +17,14 @@ # Core Utility class AbstractPreparer(object): - def __init__(self, name_prefix, name_len): + def __init__(self, name_prefix, name_len, disable_recording=False): self.name_prefix = name_prefix self.name_len = name_len self.resource_moniker = None self.resource_random_name = None self.test_class_instance = None self.live_test = False + self.disable_recording = False def __call__(self, fn): def _preparer_wrapper(test_class_instance, **kwargs): @@ -36,8 +38,14 @@ def _preparer_wrapper(test_class_instance, **kwargs): else: resource_name = self.moniker - parameter_update = self.create_resource(resource_name, **kwargs) - test_class_instance.addCleanup(lambda: self.remove_resource(resource_name, **kwargs)) + with self.override_disable_recording(): + parameter_update = self.create_resource( + resource_name, + **kwargs + ) + test_class_instance.addCleanup( + lambda: self.remove_resource_with_record_override(resource_name, **kwargs) + ) if parameter_update: kwargs.update(parameter_update) @@ -57,6 +65,16 @@ def _preparer_wrapper(test_class_instance, **kwargs): functools.update_wrapper(_preparer_wrapper, fn) return _preparer_wrapper + @contextlib.contextmanager + def override_disable_recording(self): + if self.test_class_instance.hasattr('disable_recording'): + orig_enabled = self.test_class_instance.disable_recording + self.test_class_instance.disable_recording = self.disable_recording + yield + self.test_class_instance.disable_recording = orig_enabled + else: + yield + @property def moniker(self): if not self.resource_moniker: @@ -77,6 +95,10 @@ def create_resource(self, name, **kwargs): # pylint: disable=unused-argument,no def remove_resource(self, name, **kwargs): # pylint: disable=unused-argument pass + def remove_resource_with_record_override(self, name, **kwargs): + with self.override_disable_recording(): + self.remove_resource(name, **kwargs) + # TODO: replaced by GeneralNameReplacer class SingleValueReplacer(RecordingProcessor): From 4d9dae2f1ae0c20fe1b140a954f507f7d3140e56 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Wed, 31 May 2017 14:21:53 -0700 Subject: [PATCH 031/167] Modify config options for better backwards compatibility --- src/azure_devtools/scenario_tests/base.py | 25 ++++++++++----------- src/azure_devtools/scenario_tests/config.py | 12 ++++------ 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/src/azure_devtools/scenario_tests/base.py b/src/azure_devtools/scenario_tests/base.py index 4423d27881fe..c1700b47c592 100644 --- a/src/azure_devtools/scenario_tests/base.py +++ b/src/azure_devtools/scenario_tests/base.py @@ -18,9 +18,8 @@ import vcr from .config import TestConfig, RecordMode -from .const import (ENV_LIVE_TEST, ENV_SKIP_ASSERT, ENV_TEST_DIAGNOSE, MOCKED_SUBSCRIPTION_ID, - DUMMY_HEADER_DEACTIVATE_VCR_RECORDING) -from .patches import patch_time_sleep_api +from .const import ENV_LIVE_TEST, ENV_SKIP_ASSERT, ENV_TEST_DIAGNOSE, MOCKED_SUBSCRIPTION_ID +from .patches import patch_time_sleep_api, patch_long_run_operation_delay from .recording_processors import (SubscriptionRecordingProcessor, OAuthRequestResponsesFilter, GeneralNameReplacer, LargeRequestBodyProcessor, LargeResponseBodyProcessor, LargeResponseBodyReplacer, @@ -96,9 +95,7 @@ class ScenarioTest(IntegrationTestBase): # pylint: disable=too-many-instance-at 'x-ms-served-by', ] - TEST_CONFIG_FILE = None - - def __init__(self, method_name): + def __init__(self, method_name, config_file=None): super(ScenarioTest, self).__init__(method_name) self.name_replacer = GeneralNameReplacer() self.recording_processors = [SubscriptionRecordingProcessor(MOCKED_SUBSCRIPTION_ID), @@ -110,13 +107,16 @@ def __init__(self, method_name): self.replay_processors = [LargeResponseBodyReplacer(), DeploymentNameReplacer()] self.recording_patches = [] - self.replay_patches = [patch_time_sleep_api] + self.replay_patches = [patch_time_sleep_api, + patch_long_run_operation_delay] + + self.config = TestConfig(config_file=config_file) - self.config = TestConfig(config_file=self.TEST_CONFIG_FILE) + self.disable_recording = False test_file_path = inspect.getfile(self.__class__) recordings_dir = os.path.join(os.path.dirname(test_file_path), 'recordings') - live_test = self.config.record_mode == RecordMode.all + self.is_live = not self.config.record_mode self.vcr = vcr.VCR( cassette_library_dir=recordings_dir, @@ -129,10 +129,10 @@ def __init__(self, method_name): self.vcr.register_matcher('query', self._custom_request_query_matcher) self.recording_file = os.path.join(recordings_dir, '{}.yaml'.format(method_name)) - if live_test and os.path.exists(self.recording_file): + if self.is_live and os.path.exists(self.recording_file): os.remove(self.recording_file) - self.in_recording = live_test or not os.path.exists(self.recording_file) + self.in_recording = self.is_live or not os.path.exists(self.recording_file) self.test_resources_count = 0 self.original_env = os.environ.copy() @@ -167,8 +167,7 @@ def create_random_name(self, prefix, length): return moniker def _process_request_recording(self, request): - if DUMMY_HEADER_DEACTIVATE_VCR_RECORDING in request.headers: - # Disable recording + if self.disable_recording: return None if self.in_recording: diff --git a/src/azure_devtools/scenario_tests/config.py b/src/azure_devtools/scenario_tests/config.py index 46604613eb0a..4e5b93540f73 100644 --- a/src/azure_devtools/scenario_tests/config.py +++ b/src/azure_devtools/scenario_tests/config.py @@ -1,9 +1,6 @@ import configargparse - -class RecordMode(object): - once = 'once' - all = 'all' +from config import ENV_LIVE_TEST class TestConfig(object): @@ -15,10 +12,9 @@ def __init__(self, parent_parsers=None, config_file=None): help='Path to a configuration file in YAML format.' ) self.parser.add_argument( - '-m', '--record-mode', choices=[RecordMode.once, RecordMode.all], - default=RecordMode.once, - env_var='AZURE_TESTS_RECORD_MODE', - help='Test recording mode.' + '-m', '--record-mode', action='store_true', dest='record_mode', + env_var=ENV_LIVE_TEST, + help='Activate "live" recording mode for tests.' ) self.args = self.parser.parse_args([]) From 8ee868e72d612cc229a59cea31679681a7fe0711 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Mon, 5 Jun 2017 09:07:51 -0700 Subject: [PATCH 032/167] Ignore more VS stuff --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 009b55465f7a..ee458ab88ad8 100644 --- a/.gitignore +++ b/.gitignore @@ -92,4 +92,5 @@ ENV/ .vscode .vs *.pyproj +*.pyproj.user *.sln From afaede195f478782f52f4e4aca145fc85e2ca38d Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Mon, 5 Jun 2017 09:18:56 -0700 Subject: [PATCH 033/167] Remove jmespath dependency, update vcrpy --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 427e9342eafb..585e914fd0ab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ setuptools-markdown ConfigArgParse -jmespath mock six -vcrpy==1.10.3 \ No newline at end of file +vcrpy==1.11.1 \ No newline at end of file From b90edaf69f4f9af34809e26bd770ca190511ca1a Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Mon, 5 Jun 2017 09:19:20 -0700 Subject: [PATCH 034/167] Remove checkers --- src/azure_devtools/scenario_tests/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/azure_devtools/scenario_tests/__init__.py b/src/azure_devtools/scenario_tests/__init__.py index 963727f2d29d..f13786717590 100644 --- a/src/azure_devtools/scenario_tests/__init__.py +++ b/src/azure_devtools/scenario_tests/__init__.py @@ -5,10 +5,8 @@ from .base import ScenarioTest, LiveTest from .exceptions import AzureTestError -from .checkers import JMESPathCheck, JMESPathCheckExists, NoneCheck, StringCheck, StringContainCheck from .decorators import live_only, record_only from .utilities import get_sha1_hash -__all__ = ['ScenarioTest', 'LiveTest', 'AzureTestError', 'JMESPathCheck', 'JMESPathCheckExists', 'NoneCheck', - 'live_only', 'record_only', 'StringCheck', 'StringContainCheck', 'get_sha1_hash'] +__all__ = ['ScenarioTest', 'LiveTest', 'AzureTestError', 'live_only', 'record_only', 'get_sha1_hash'] __version__ = '0.1.0+dev' From 1014d3358156c51d5196d1df224aed65a6727bf2 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Mon, 5 Jun 2017 09:26:13 -0700 Subject: [PATCH 035/167] Remove unuseds, make patches/processors kwargs --- src/azure_devtools/scenario_tests/base.py | 58 +++++++++-------------- 1 file changed, 22 insertions(+), 36 deletions(-) diff --git a/src/azure_devtools/scenario_tests/base.py b/src/azure_devtools/scenario_tests/base.py index c1700b47c592..f0e874deff86 100644 --- a/src/azure_devtools/scenario_tests/base.py +++ b/src/azure_devtools/scenario_tests/base.py @@ -17,17 +17,12 @@ import six import vcr -from .config import TestConfig, RecordMode +from .config import TestConfig from .const import ENV_LIVE_TEST, ENV_SKIP_ASSERT, ENV_TEST_DIAGNOSE, MOCKED_SUBSCRIPTION_ID -from .patches import patch_time_sleep_api, patch_long_run_operation_delay -from .recording_processors import (SubscriptionRecordingProcessor, OAuthRequestResponsesFilter, - GeneralNameReplacer, LargeRequestBodyProcessor, - LargeResponseBodyProcessor, LargeResponseBodyReplacer, - DeploymentNameReplacer) from .utilities import create_random_name from .decorators import live_only -logger = logging.getLogger('azure.devtools.testsdk') +logger = logging.getLogger('azure_devtools.scenario_tests') class IntegrationTestBase(unittest.TestCase): @@ -85,6 +80,7 @@ class ScenarioTest(IntegrationTestBase): # pylint: disable=too-many-instance-at FILTER_HEADERS = [ 'authorization', 'client-request-id', + 'retry-after', 'x-ms-client-request-id', 'x-ms-correlation-request-id', 'x-ms-ratelimit-remaining-subscription-reads', @@ -95,40 +91,41 @@ class ScenarioTest(IntegrationTestBase): # pylint: disable=too-many-instance-at 'x-ms-served-by', ] - def __init__(self, method_name, config_file=None): + def __init__(self, method_name, config_file=None, + recording_dir=None, recording_name=None, + recording_processors=None, replay_processors=None, + recording_patches=None, replay_patches=None): super(ScenarioTest, self).__init__(method_name) - self.name_replacer = GeneralNameReplacer() - self.recording_processors = [SubscriptionRecordingProcessor(MOCKED_SUBSCRIPTION_ID), - OAuthRequestResponsesFilter(), - LargeRequestBodyProcessor(), - LargeResponseBodyProcessor(), - DeploymentNameReplacer(), - self.name_replacer] - self.replay_processors = [LargeResponseBodyReplacer(), DeploymentNameReplacer()] - - self.recording_patches = [] - self.replay_patches = [patch_time_sleep_api, - patch_long_run_operation_delay] + + self.recording_processors = recording_processors or [] + self.replay_processors = replay_processors or [] + + self.recording_patches = recording_patches or [] + self.replay_patches = replay_patches or [] self.config = TestConfig(config_file=config_file) self.disable_recording = False test_file_path = inspect.getfile(self.__class__) - recordings_dir = os.path.join(os.path.dirname(test_file_path), 'recordings') - self.is_live = not self.config.record_mode + recording_dir = recording_dir or os.path.join(os.path.dirname(test_file_path), + 'recordings') + self.is_live = self.config.record_mode self.vcr = vcr.VCR( - cassette_library_dir=recordings_dir, + cassette_library_dir=recording_dir, before_record_request=self._process_request_recording, before_record_response=self._process_response_recording, decode_compressed_response=True, - record_mode=config.record_mode, + record_mode=self.config.record_mode, filter_headers=self.FILTER_HEADERS ) self.vcr.register_matcher('query', self._custom_request_query_matcher) - self.recording_file = os.path.join(recordings_dir, '{}.yaml'.format(method_name)) + self.recording_file = os.path.join( + recording_dir, + '{}.yaml'.format(recording_name or method_name) + ) if self.is_live and os.path.exists(self.recording_file): os.remove(self.recording_file) @@ -155,17 +152,6 @@ def setUp(self): def tearDown(self): os.environ = self.original_env - def create_random_name(self, prefix, length): - self.test_resources_count += 1 - moniker = '{}{:06}'.format(prefix, self.test_resources_count) - - if self.in_recording: - name = create_random_name(prefix, length) - self.name_replacer.register_name_pair(name, moniker) - return name - else: - return moniker - def _process_request_recording(self, request): if self.disable_recording: return None From 7fee146aa50d5b977a5fc04b9fe0de5a283831e3 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Mon, 5 Jun 2017 09:27:40 -0700 Subject: [PATCH 036/167] Remove checkers, used only for CLI --- src/azure_devtools/scenario_tests/checkers.py | 77 ------------------- 1 file changed, 77 deletions(-) delete mode 100644 src/azure_devtools/scenario_tests/checkers.py diff --git a/src/azure_devtools/scenario_tests/checkers.py b/src/azure_devtools/scenario_tests/checkers.py deleted file mode 100644 index 09044a911143..000000000000 --- a/src/azure_devtools/scenario_tests/checkers.py +++ /dev/null @@ -1,77 +0,0 @@ -# -------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for license information. -# -------------------------------------------------------------------------------------------- - -import collections -import jmespath -from .exceptions import JMESPathCheckAssertionError - - -class JMESPathCheck(object): # pylint: disable=too-few-public-methods - def __init__(self, query, expected_result): - self._query = query - self._expected_result = expected_result - - def __call__(self, execution_result): - json_value = execution_result.get_output_in_json() - actual_result = jmespath.search(self._query, json_value, - jmespath.Options(collections.OrderedDict)) - if not actual_result == self._expected_result: - if actual_result: - raise JMESPathCheckAssertionError(self._query, self._expected_result, actual_result, - execution_result.output) - else: - raise JMESPathCheckAssertionError(self._query, self._expected_result, 'None', - execution_result.output) - - -class JMESPathCheckExists(object): # pylint: disable=too-few-public-methods - def __init__(self, query): - self._query = query - - def __call__(self, execution_result): - json_value = execution_result.get_output_in_json() - actual_result = jmespath.search(self._query, json_value, - jmespath.Options(collections.OrderedDict)) - if not actual_result: - raise JMESPathCheckAssertionError(self._query, 'some value', actual_result, - execution_result.output) - - -class NoneCheck(object): # pylint: disable=too-few-public-methods - def __call__(self, execution_result): # pylint: disable=no-self-use - none_strings = ['[]', '{}', 'false'] - try: - data = execution_result.output.strip() - assert not data or data in none_strings - except AssertionError: - raise AssertionError("Actual value '{}' != Expected value falsy (None, '', []) or " - "string in {}".format(data, none_strings)) - - -class StringCheck(object): # pylint: disable=too-few-public-methods - def __init__(self, expected_result): - self.expected_result = expected_result - - def __call__(self, execution_result): - try: - result = execution_result.output.strip().strip('"') - assert result == self.expected_result - except AssertionError: - raise AssertionError( - "Actual value '{}' != Expected value {}".format(result, self.expected_result)) - - -class StringContainCheck(object): # pylint: disable=too-few-public-methods - def __init__(self, expected_result): - self.expected_result = expected_result - - def __call__(self, execution_result): - try: - result = execution_result.output.strip('"') - assert self.expected_result in result - except AssertionError: - raise AssertionError( - "Actual value '{}' doesn't contain Expected value {}".format(result, - self.expected_result)) From 009c6fb7de26d12ce1473f65c35682fa28d2aef7 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Mon, 5 Jun 2017 09:28:13 -0700 Subject: [PATCH 037/167] Update import and instance var name --- src/azure_devtools/scenario_tests/config.py | 4 ++-- src/azure_devtools/scenario_tests/exceptions.py | 7 ------- src/azure_devtools/scenario_tests/preparers.py | 4 ++-- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/src/azure_devtools/scenario_tests/config.py b/src/azure_devtools/scenario_tests/config.py index 4e5b93540f73..cb945a75d938 100644 --- a/src/azure_devtools/scenario_tests/config.py +++ b/src/azure_devtools/scenario_tests/config.py @@ -1,6 +1,6 @@ import configargparse -from config import ENV_LIVE_TEST +from .const import ENV_LIVE_TEST class TestConfig(object): @@ -20,4 +20,4 @@ def __init__(self, parent_parsers=None, config_file=None): @property def record_mode(self): - return self.args.mode \ No newline at end of file + return self.args.record_mode \ No newline at end of file diff --git a/src/azure_devtools/scenario_tests/exceptions.py b/src/azure_devtools/scenario_tests/exceptions.py index 66da6e9e2c2e..bdebae0b44e8 100644 --- a/src/azure_devtools/scenario_tests/exceptions.py +++ b/src/azure_devtools/scenario_tests/exceptions.py @@ -8,10 +8,3 @@ class AzureTestError(Exception): def __init__(self, error_message): message = 'An error caused by the Azure test harness failed the test: {}' super(AzureTestError, self).__init__(message.format(error_message)) - - -class JMESPathCheckAssertionError(AssertionError): - def __init__(self, query, expected, actual, json_data): - message = "Query '{}' doesn't yield expected value '{}', instead the actual value " \ - "is '{}'. Data: \n{}\n".format(query, expected, actual, json_data) - super(JMESPathCheckAssertionError, self).__init__(message) diff --git a/src/azure_devtools/scenario_tests/preparers.py b/src/azure_devtools/scenario_tests/preparers.py index 59ebe2e80a6f..dda8d43e8d53 100644 --- a/src/azure_devtools/scenario_tests/preparers.py +++ b/src/azure_devtools/scenario_tests/preparers.py @@ -9,7 +9,7 @@ import os import uuid -from .base import ScenarioTest, execute +from .base import ScenarioTest from .utilities import create_random_name from .recording_processors import RecordingProcessor @@ -67,7 +67,7 @@ def _preparer_wrapper(test_class_instance, **kwargs): @contextlib.contextmanager def override_disable_recording(self): - if self.test_class_instance.hasattr('disable_recording'): + if hasattr(self.test_class_instance, 'disable_recording'): orig_enabled = self.test_class_instance.disable_recording self.test_class_instance.disable_recording = self.disable_recording yield From 651c9afedc14d1cc61c38ed5f9369be6e3c7621a Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Mon, 5 Jun 2017 09:29:46 -0700 Subject: [PATCH 038/167] Remove unused imports --- src/azure_devtools/scenario_tests/base.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/azure_devtools/scenario_tests/base.py b/src/azure_devtools/scenario_tests/base.py index f0e874deff86..777bb93f4f89 100644 --- a/src/azure_devtools/scenario_tests/base.py +++ b/src/azure_devtools/scenario_tests/base.py @@ -4,13 +4,9 @@ # -------------------------------------------------------------------------------------------- from __future__ import print_function -import datetime import unittest import os import inspect -import subprocess -import json -import shlex import tempfile import shutil import logging @@ -18,7 +14,7 @@ import vcr from .config import TestConfig -from .const import ENV_LIVE_TEST, ENV_SKIP_ASSERT, ENV_TEST_DIAGNOSE, MOCKED_SUBSCRIPTION_ID +from .const import ENV_TEST_DIAGNOSE from .utilities import create_random_name from .decorators import live_only From fc32c79f3559b83e59d90aafb41509a05dcbacc1 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Mon, 5 Jun 2017 10:02:30 -0700 Subject: [PATCH 039/167] line organization --- src/azure_devtools/scenario_tests/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/azure_devtools/scenario_tests/__init__.py b/src/azure_devtools/scenario_tests/__init__.py index f13786717590..83ff5ab378ef 100644 --- a/src/azure_devtools/scenario_tests/__init__.py +++ b/src/azure_devtools/scenario_tests/__init__.py @@ -8,5 +8,6 @@ from .decorators import live_only, record_only from .utilities import get_sha1_hash -__all__ = ['ScenarioTest', 'LiveTest', 'AzureTestError', 'live_only', 'record_only', 'get_sha1_hash'] +__all__ = ['ScenarioTest', 'LiveTest', 'AzureTestError', 'get_sha1_hash', + 'live_only', 'record_only'] __version__ = '0.1.0+dev' From 1f74aed56262a579d4cce691e5db26d7dce38dbb Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Mon, 5 Jun 2017 11:46:14 -0700 Subject: [PATCH 040/167] Update version number and vcrpy version --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 4039c57d0e26..93c3e8d60d1d 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ from setuptools import setup -VERSION = "0.1.0+dev" +VERSION = "0.1.0" CLASSIFIERS = [ @@ -31,7 +31,7 @@ 'mock', 'setuptools-markdown', 'six', - 'vcrpy==1.10.3', + 'vcrpy==1.11.1', ] with open('README.md', 'r', encoding='utf-8') as f: From 2344faf63335e6d216d191942f830d1fb82846fa Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Mon, 5 Jun 2017 12:39:27 -0700 Subject: [PATCH 041/167] Add a little more to README, add setup.cfg --- README.md | 7 +++++++ setup.cfg | 2 ++ 2 files changed, 9 insertions(+) create mode 100644 setup.cfg diff --git a/README.md b/README.md index 8624b3d25cd6..9d60718e351d 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,10 @@ +# Development tools for Python-based Azure tools + +This package contains tools to aid in developing Python-based Azure code. +Currently it includes `scenario_tests`, +a testing framework to handle much of the busywork +associated with testing code that interacts with Azure. + # Contributing This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000000..224a77957f5d --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +description-file = README.md \ No newline at end of file From 4a0d0841f036fb4b9f2ad1cec060d9aef5014ff8 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Mon, 5 Jun 2017 13:58:35 -0700 Subject: [PATCH 042/167] Add unit tests for TestConfig --- src/azure_devtools/scenario_tests/config.py | 4 +-- .../scenario_tests/tests/test_config.py | 29 +++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 src/azure_devtools/scenario_tests/tests/test_config.py diff --git a/src/azure_devtools/scenario_tests/config.py b/src/azure_devtools/scenario_tests/config.py index cb945a75d938..14481caf838a 100644 --- a/src/azure_devtools/scenario_tests/config.py +++ b/src/azure_devtools/scenario_tests/config.py @@ -12,7 +12,7 @@ def __init__(self, parent_parsers=None, config_file=None): help='Path to a configuration file in YAML format.' ) self.parser.add_argument( - '-m', '--record-mode', action='store_true', dest='record_mode', + '-l', '--live-mode', action='store_true', dest='live_mode', env_var=ENV_LIVE_TEST, help='Activate "live" recording mode for tests.' ) @@ -20,4 +20,4 @@ def __init__(self, parent_parsers=None, config_file=None): @property def record_mode(self): - return self.args.record_mode \ No newline at end of file + return self.args.live_mode \ No newline at end of file diff --git a/src/azure_devtools/scenario_tests/tests/test_config.py b/src/azure_devtools/scenario_tests/tests/test_config.py new file mode 100644 index 000000000000..bc59cee93f89 --- /dev/null +++ b/src/azure_devtools/scenario_tests/tests/test_config.py @@ -0,0 +1,29 @@ +import argparse +import os +import tempfile +import unittest + +import mock + + +from azure_devtools.scenario_tests.const import ENV_LIVE_TEST +from azure_devtools.scenario_tests.config import TestConfig + + +class TestScenarioConfig(unittest.TestCase): + def setUp(self): + with tempfile.NamedTemporaryFile(mode='w', delete=False) as cfgfile: + cfgfile.write('live-mode: yes') + self.cfgfile = cfgfile.name + + def tearDown(self): + os.remove(self.cfgfile) + + def test_env_var(self): + with mock.patch.dict('os.environ', {ENV_LIVE_TEST: 'yes'}): + config = TestConfig() + self.assertTrue(config.record_mode) + + def test_config_file(self): + config = TestConfig(config_file=self.cfgfile) + self.assertTrue(config.record_mode) From 6d95db8925852341f86eb84e26abdf435d141565 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Mon, 5 Jun 2017 14:33:36 -0700 Subject: [PATCH 043/167] Add travis.yml and update dependencies --- .travis.yml | 13 +++++++++++++ requirements.txt | 1 + setup.py | 3 +-- 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000000..661e7b472000 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,13 @@ +sudo: false +language: python +python: + - "2.7" + - "3.3" + - "3.4" + - "3.5" + - "3.6" +install: + - pip install -r requirements.txt + - pip install -e . +script: + - nosetests \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 585e914fd0ab..f0e5cffdb1f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ setuptools-markdown ConfigArgParse mock +nose six vcrpy==1.11.1 \ No newline at end of file diff --git a/setup.py b/setup.py index 93c3e8d60d1d..77727e76b421 100644 --- a/setup.py +++ b/setup.py @@ -27,8 +27,7 @@ DEPENDENCIES = [ - 'jmespath', - 'mock', + 'ConfigArgParse', 'setuptools-markdown', 'six', 'vcrpy==1.11.1', From 12be54907c8662b1dc130f4bdf483ace2abaa09f Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Mon, 5 Jun 2017 14:39:37 -0700 Subject: [PATCH 044/167] Update author email --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 77727e76b421..89b01a9c1ebb 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,7 @@ long_description_markdown_file='README.md', license='MIT', author='Microsoft Corporation', - author_email='azpycli@microsoft.com', + author_email='ptvshelp@microsoft.com', url='https://github.com/Azure/azure-python-devtools', zip_safe=False, classifiers=CLASSIFIERS, From b9bdc9e4fbbfac6ac09e6b74da89b0ed8b5fb03f Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Mon, 5 Jun 2017 14:49:29 -0700 Subject: [PATCH 045/167] Dummy push to try to trigger a Travis build --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9d60718e351d..4ce17c694e91 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ Currently it includes `scenario_tests`, a testing framework to handle much of the busywork associated with testing code that interacts with Azure. + # Contributing This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. From c32e8b316056593732ace57bd5ccffdd32b77c03 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Mon, 5 Jun 2017 14:54:45 -0700 Subject: [PATCH 046/167] Another dummy commit for Travis --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 4ce17c694e91..9d60718e351d 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,6 @@ Currently it includes `scenario_tests`, a testing framework to handle much of the busywork associated with testing code that interacts with Azure. - # Contributing This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. From 29ed04d81f1b042a8dba1615d035a3f4b7782013 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Mon, 5 Jun 2017 15:02:43 -0700 Subject: [PATCH 047/167] Switch to README.rst --- README.md | 10 ---------- README.rst | 21 +++++++++++++++++++++ setup.cfg | 2 -- 3 files changed, 21 insertions(+), 12 deletions(-) delete mode 100644 README.md create mode 100644 README.rst delete mode 100644 setup.cfg diff --git a/README.md b/README.md deleted file mode 100644 index 9d60718e351d..000000000000 --- a/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Development tools for Python-based Azure tools - -This package contains tools to aid in developing Python-based Azure code. -Currently it includes `scenario_tests`, -a testing framework to handle much of the busywork -associated with testing code that interacts with Azure. - -# Contributing - -This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. diff --git a/README.rst b/README.rst new file mode 100644 index 000000000000..2ebee1a84586 --- /dev/null +++ b/README.rst @@ -0,0 +1,21 @@ +.. image:: https://travis-ci.org/Azure/azure-sdk-for-python.svg?branch=master + :target: https://travis-ci.org/Azure/azure-sdk-for-python + +Development tools for Python-based Azure tools +============================================== + +This package contains tools to aid in developing Python-based Azure code. +Currently it includes ``scenario_tests``, +a testing framework to handle much of the busywork +associated with testing code that interacts with Azure. + +Contributing +============ + +This project has adopted the +`Microsoft Open Source Code of Conduct `__. +For more information see the +`Code of Conduct FAQ `__ +or contact +`opencode@microsoft.com `__ +with any additional questions or comments. diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 224a77957f5d..000000000000 --- a/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -description-file = README.md \ No newline at end of file From 548c59c9812ac7c8ad60bcbd01e0d641c40de0ac Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Mon, 5 Jun 2017 15:05:02 -0700 Subject: [PATCH 048/167] Update Travis tag pointer --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 2ebee1a84586..1b9a94984ffa 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -.. image:: https://travis-ci.org/Azure/azure-sdk-for-python.svg?branch=master - :target: https://travis-ci.org/Azure/azure-sdk-for-python +.. image:: https://travis-ci.org/Azure/azure-python-devtools.svg?branch=master + :target: https://travis-ci.org/Azure/azure-python-devtools Development tools for Python-based Azure tools ============================================== From 475872a39ad092eb37fb0675c3e9b3c424ed8e9f Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Mon, 5 Jun 2017 15:07:18 -0700 Subject: [PATCH 049/167] Use io to get encoding arg when reading README --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 89b01a9c1ebb..9be2a3c02df3 100644 --- a/setup.py +++ b/setup.py @@ -5,6 +5,7 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- +import io import os.path from setuptools import setup @@ -33,7 +34,7 @@ 'vcrpy==1.11.1', ] -with open('README.md', 'r', encoding='utf-8') as f: +with io.open('README.rst', 'r', encoding='utf-8') as f: README = f.read() setup( From c67f4b1050273ee1c69cbd245d523431dfc380ae Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Tue, 6 Jun 2017 09:52:06 -0700 Subject: [PATCH 050/167] Remove unnecessary import --- src/azure_devtools/scenario_tests/patches.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/azure_devtools/scenario_tests/patches.py b/src/azure_devtools/scenario_tests/patches.py index 7ca1edf68dc7..7127799c36b1 100644 --- a/src/azure_devtools/scenario_tests/patches.py +++ b/src/azure_devtools/scenario_tests/patches.py @@ -4,7 +4,6 @@ # -------------------------------------------------------------------------------------------- from .exceptions import AzureTestError -from .const import MOCKED_SUBSCRIPTION_ID, MOCKED_TENANT_ID def patch_time_sleep_api(unit_test): From 7ccce1a0eec178d816beaf2c299cd50a9c150935 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Tue, 6 Jun 2017 09:52:55 -0700 Subject: [PATCH 051/167] Expose most everything in top-level namespace --- src/azure_devtools/scenario_tests/__init__.py | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/azure_devtools/scenario_tests/__init__.py b/src/azure_devtools/scenario_tests/__init__.py index 83ff5ab378ef..363ee4c921b1 100644 --- a/src/azure_devtools/scenario_tests/__init__.py +++ b/src/azure_devtools/scenario_tests/__init__.py @@ -3,11 +3,25 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -from .base import ScenarioTest, LiveTest +from .base import IntegrationTestBase, ScenarioTest, LiveTest from .exceptions import AzureTestError from .decorators import live_only, record_only -from .utilities import get_sha1_hash +from .patches import patch_time_sleep_api, patch_long_run_operation_delay +from .preparers import AbstractPreparer, SingleValueReplacer +from .recording_processors import ( + RecordingProcessor, SubscriptionRecordingProcessor, + LargeRequestBodyProcessor, LargeResponseBodyProcessor, LargeResponseBodyReplacer, + OAuthRequestResponsesFilter, DeploymentNameReplacer, GeneralNameReplacer, +) +from .utilities import create_random_name, get_sha1_hash -__all__ = ['ScenarioTest', 'LiveTest', 'AzureTestError', 'get_sha1_hash', - 'live_only', 'record_only'] +__all__ = ['IntegrationTestBase', 'ScenarioTest', 'LiveTest', + 'AzureTestError', + 'patch_time_sleep_api', 'patch_long_run_operation_delay', + 'AbstractPreparer', 'SingleValueReplacer', + 'RecordingProcessor', 'SubscriptionRecordingProcessor', + 'LargeRequestBodyProcessor', 'LargeResponseBodyProcessor', 'LargeResponseBodyReplacer', + 'OAuthRequestResponsesFilter', 'DeploymentNameReplacer', 'GeneralNameReplacer', + 'live_only', 'record_only', + 'create_random_name', 'get_sha1_hash'] __version__ = '0.1.0+dev' From ac70d4f07e7d1b1ad0686581603c85ff68be75ea Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Tue, 6 Jun 2017 10:16:25 -0700 Subject: [PATCH 052/167] ScenarioTest -> ReplayableTest; update version --- src/azure_devtools/scenario_tests/__init__.py | 6 +++--- src/azure_devtools/scenario_tests/base.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/azure_devtools/scenario_tests/__init__.py b/src/azure_devtools/scenario_tests/__init__.py index 363ee4c921b1..c064257a8ab9 100644 --- a/src/azure_devtools/scenario_tests/__init__.py +++ b/src/azure_devtools/scenario_tests/__init__.py @@ -3,7 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -from .base import IntegrationTestBase, ScenarioTest, LiveTest +from .base import IntegrationTestBase, ReplayableTest, LiveTest from .exceptions import AzureTestError from .decorators import live_only, record_only from .patches import patch_time_sleep_api, patch_long_run_operation_delay @@ -15,7 +15,7 @@ ) from .utilities import create_random_name, get_sha1_hash -__all__ = ['IntegrationTestBase', 'ScenarioTest', 'LiveTest', +__all__ = ['IntegrationTestBase', 'ReplayableTest', 'LiveTest', 'AzureTestError', 'patch_time_sleep_api', 'patch_long_run_operation_delay', 'AbstractPreparer', 'SingleValueReplacer', @@ -24,4 +24,4 @@ 'OAuthRequestResponsesFilter', 'DeploymentNameReplacer', 'GeneralNameReplacer', 'live_only', 'record_only', 'create_random_name', 'get_sha1_hash'] -__version__ = '0.1.0+dev' +__version__ = '0.2.0' diff --git a/src/azure_devtools/scenario_tests/base.py b/src/azure_devtools/scenario_tests/base.py index 777bb93f4f89..c63d5ce220f9 100644 --- a/src/azure_devtools/scenario_tests/base.py +++ b/src/azure_devtools/scenario_tests/base.py @@ -72,7 +72,7 @@ class LiveTest(IntegrationTestBase): pass -class ScenarioTest(IntegrationTestBase): # pylint: disable=too-many-instance-attributes +class ReplayableTest(IntegrationTestBase): # pylint: disable=too-many-instance-attributes FILTER_HEADERS = [ 'authorization', 'client-request-id', From e1ad6a47db13cf1fd70d740be48a8f5595fad74a Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Tue, 6 Jun 2017 10:17:02 -0700 Subject: [PATCH 053/167] Update version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9be2a3c02df3..0aa278170b62 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ from setuptools import setup -VERSION = "0.1.0" +VERSION = "0.2.0" CLASSIFIERS = [ From 29f1dc3de8b6456259667709990bc8cadf041dc4 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Tue, 6 Jun 2017 10:17:21 -0700 Subject: [PATCH 054/167] Remove unused os.path import --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 0aa278170b62..290160f7157a 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,6 @@ # -------------------------------------------------------------------------------------------- import io -import os.path from setuptools import setup From 05862f46ca29c8bf63481b8582c1a8ac46d77dca Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Tue, 6 Jun 2017 11:13:35 -0700 Subject: [PATCH 055/167] Fix super call --- src/azure_devtools/scenario_tests/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/azure_devtools/scenario_tests/base.py b/src/azure_devtools/scenario_tests/base.py index c63d5ce220f9..084a28c9b970 100644 --- a/src/azure_devtools/scenario_tests/base.py +++ b/src/azure_devtools/scenario_tests/base.py @@ -91,7 +91,7 @@ def __init__(self, method_name, config_file=None, recording_dir=None, recording_name=None, recording_processors=None, replay_processors=None, recording_patches=None, replay_patches=None): - super(ScenarioTest, self).__init__(method_name) + super(ReplayableTest, self).__init__(method_name) self.recording_processors = recording_processors or [] self.replay_processors = replay_processors or [] From d83e9f2daa3daf43f4a4a97b7607983dbf328d89 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Tue, 6 Jun 2017 11:22:31 -0700 Subject: [PATCH 056/167] Fix super call in setUp --- src/azure_devtools/scenario_tests/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/azure_devtools/scenario_tests/base.py b/src/azure_devtools/scenario_tests/base.py index 084a28c9b970..409e57778d12 100644 --- a/src/azure_devtools/scenario_tests/base.py +++ b/src/azure_devtools/scenario_tests/base.py @@ -130,7 +130,7 @@ def __init__(self, method_name, config_file=None, self.original_env = os.environ.copy() def setUp(self): - super(ScenarioTest, self).setUp() + super(ReplayableTest, self).setUp() # set up cassette cm = self.vcr.use_cassette(self.recording_file) From 850f2364fed055f9abfb5e654477c546de9edf4d Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Tue, 6 Jun 2017 11:22:52 -0700 Subject: [PATCH 057/167] Update ReplayableTest import --- src/azure_devtools/scenario_tests/preparers.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/azure_devtools/scenario_tests/preparers.py b/src/azure_devtools/scenario_tests/preparers.py index dda8d43e8d53..8989138f670d 100644 --- a/src/azure_devtools/scenario_tests/preparers.py +++ b/src/azure_devtools/scenario_tests/preparers.py @@ -6,10 +6,8 @@ import contextlib import inspect import functools -import os -import uuid -from .base import ScenarioTest +from .base import ReplayableTest from .utilities import create_random_name from .recording_processors import RecordingProcessor @@ -28,7 +26,7 @@ def __init__(self, name_prefix, name_len, disable_recording=False): def __call__(self, fn): def _preparer_wrapper(test_class_instance, **kwargs): - self.live_test = not isinstance(test_class_instance, ScenarioTest) + self.live_test = not isinstance(test_class_instance, ReplayableTest) self.test_class_instance = test_class_instance if self.live_test or test_class_instance.in_recording: From 6a37a03b464348b31ce040ce75cb23d0010fd5ec Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Tue, 6 Jun 2017 14:11:22 -0700 Subject: [PATCH 058/167] Use earlier vcrpy, update version num --- requirements.txt | 2 +- setup.py | 4 ++-- src/azure_devtools/scenario_tests/__init__.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index f0e5cffdb1f8..6fdb045736d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ ConfigArgParse mock nose six -vcrpy==1.11.1 \ No newline at end of file +vcrpy==1.10.3 \ No newline at end of file diff --git a/setup.py b/setup.py index 290160f7157a..97db8f3d9045 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ from setuptools import setup -VERSION = "0.2.0" +VERSION = "0.2.1" CLASSIFIERS = [ @@ -30,7 +30,7 @@ 'ConfigArgParse', 'setuptools-markdown', 'six', - 'vcrpy==1.11.1', + 'vcrpy==1.10.3', ] with io.open('README.rst', 'r', encoding='utf-8') as f: diff --git a/src/azure_devtools/scenario_tests/__init__.py b/src/azure_devtools/scenario_tests/__init__.py index c064257a8ab9..3ee939808998 100644 --- a/src/azure_devtools/scenario_tests/__init__.py +++ b/src/azure_devtools/scenario_tests/__init__.py @@ -24,4 +24,4 @@ 'OAuthRequestResponsesFilter', 'DeploymentNameReplacer', 'GeneralNameReplacer', 'live_only', 'record_only', 'create_random_name', 'get_sha1_hash'] -__version__ = '0.2.0' +__version__ = '0.2.1' From 517e96e03cbdd074fb547b3018ef2b98b47fb3b7 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Wed, 7 Jun 2017 11:22:35 -0700 Subject: [PATCH 059/167] Don't fix vcrpy version --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6fdb045736d5..b14b05e33aa6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,4 @@ ConfigArgParse mock nose six -vcrpy==1.10.3 \ No newline at end of file +vcrpy \ No newline at end of file diff --git a/setup.py b/setup.py index 97db8f3d9045..d55d9cfee484 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ 'ConfigArgParse', 'setuptools-markdown', 'six', - 'vcrpy==1.10.3', + 'vcrpy', ] with io.open('README.rst', 'r', encoding='utf-8') as f: From 510c2fbd8055913fcd9eec8eadf354b21261e85f Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Wed, 7 Jun 2017 12:31:15 -0700 Subject: [PATCH 060/167] Drop "cli" from default random name prefix --- src/azure_devtools/scenario_tests/utilities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/azure_devtools/scenario_tests/utilities.py b/src/azure_devtools/scenario_tests/utilities.py index 947f6fe2e393..e025c7c7c61e 100644 --- a/src/azure_devtools/scenario_tests/utilities.py +++ b/src/azure_devtools/scenario_tests/utilities.py @@ -9,7 +9,7 @@ import base64 -def create_random_name(prefix='clitest', length=24): +def create_random_name(prefix='aztest', length=24): if len(prefix) > length: raise 'The length of the prefix must not be longer than random name length' From 285a3961581a66757421724a3514603b7588a953 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Wed, 7 Jun 2017 13:42:45 -0700 Subject: [PATCH 061/167] Add deployment to Travis --- .travis.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 661e7b472000..7e8425fc65c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,4 +10,13 @@ install: - pip install -r requirements.txt - pip install -e . script: - - nosetests \ No newline at end of file + - nosetests +deploy: + provider: pypi + user: Laurent.Mazuel + skip_upload_docs: true + # password: use $PYPI_PASSWORD + distributions: "sdist bdist_wheel" + on: + tags: true + python: '3.6' \ No newline at end of file From c6991309fa07a84a54129917692750f2099f4b1d Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Wed, 7 Jun 2017 13:46:06 -0700 Subject: [PATCH 062/167] Specify universal wheel --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 setup.cfg diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000000..3480374bc2f2 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 \ No newline at end of file From 19ad50248dd1bd85c1e3b04d7cbdbde18783fdc3 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Wed, 7 Jun 2017 14:29:57 -0700 Subject: [PATCH 063/167] Bump version to 0.2.2 for testing CI release process --- setup.py | 2 +- src/azure_devtools/scenario_tests/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index d55d9cfee484..961e0b8689a1 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ from setuptools import setup -VERSION = "0.2.1" +VERSION = "0.2.2" CLASSIFIERS = [ diff --git a/src/azure_devtools/scenario_tests/__init__.py b/src/azure_devtools/scenario_tests/__init__.py index 3ee939808998..c931bc49ac55 100644 --- a/src/azure_devtools/scenario_tests/__init__.py +++ b/src/azure_devtools/scenario_tests/__init__.py @@ -24,4 +24,4 @@ 'OAuthRequestResponsesFilter', 'DeploymentNameReplacer', 'GeneralNameReplacer', 'live_only', 'record_only', 'create_random_name', 'get_sha1_hash'] -__version__ = '0.2.1' +__version__ = '0.2.2' From 4533032bb1966cfa9bd6f0712bc39f31004e9882 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Wed, 7 Jun 2017 15:00:08 -0700 Subject: [PATCH 064/167] Let preparer subclasses override create_random_name --- src/azure_devtools/scenario_tests/preparers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/azure_devtools/scenario_tests/preparers.py b/src/azure_devtools/scenario_tests/preparers.py index 8989138f670d..d77e25ed348f 100644 --- a/src/azure_devtools/scenario_tests/preparers.py +++ b/src/azure_devtools/scenario_tests/preparers.py @@ -81,10 +81,13 @@ def moniker(self): self.test_class_instance.test_resources_count) return self.resource_moniker + def create_random_name(self): + return create_random_name(self.name_prefix, self.name_len) + @property def random_name(self): if not self.resource_random_name: - self.resource_random_name = create_random_name(self.name_prefix, self.name_len) + self.resource_random_name = self.create_random_name() return self.resource_random_name def create_resource(self, name, **kwargs): # pylint: disable=unused-argument,no-self-use From aeeebe6afa8f25e0c2db567847385d297213e170 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Thu, 8 Jun 2017 08:08:46 -0700 Subject: [PATCH 065/167] De-"privatize" mock_in_unit_test and expose it --- src/azure_devtools/scenario_tests/__init__.py | 4 ++-- src/azure_devtools/scenario_tests/patches.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/azure_devtools/scenario_tests/__init__.py b/src/azure_devtools/scenario_tests/__init__.py index c931bc49ac55..6807ba341b70 100644 --- a/src/azure_devtools/scenario_tests/__init__.py +++ b/src/azure_devtools/scenario_tests/__init__.py @@ -6,7 +6,7 @@ from .base import IntegrationTestBase, ReplayableTest, LiveTest from .exceptions import AzureTestError from .decorators import live_only, record_only -from .patches import patch_time_sleep_api, patch_long_run_operation_delay +from .patches import mock_in_unit_test, patch_time_sleep_api, patch_long_run_operation_delay from .preparers import AbstractPreparer, SingleValueReplacer from .recording_processors import ( RecordingProcessor, SubscriptionRecordingProcessor, @@ -17,7 +17,7 @@ __all__ = ['IntegrationTestBase', 'ReplayableTest', 'LiveTest', 'AzureTestError', - 'patch_time_sleep_api', 'patch_long_run_operation_delay', + 'mock_in_unit_test', 'patch_time_sleep_api', 'patch_long_run_operation_delay', 'AbstractPreparer', 'SingleValueReplacer', 'RecordingProcessor', 'SubscriptionRecordingProcessor', 'LargeRequestBodyProcessor', 'LargeResponseBodyProcessor', 'LargeResponseBodyReplacer', diff --git a/src/azure_devtools/scenario_tests/patches.py b/src/azure_devtools/scenario_tests/patches.py index 7127799c36b1..674b785c0aa9 100644 --- a/src/azure_devtools/scenario_tests/patches.py +++ b/src/azure_devtools/scenario_tests/patches.py @@ -10,19 +10,19 @@ def patch_time_sleep_api(unit_test): def _time_sleep_skip(*_): return - _mock_in_unit_test(unit_test, 'time.sleep', _time_sleep_skip) + mock_in_unit_test(unit_test, 'time.sleep', _time_sleep_skip) def patch_long_run_operation_delay(unit_test): def _shortcut_long_run_operation(*args, **kwargs): # pylint: disable=unused-argument return - _mock_in_unit_test(unit_test, - 'msrestazure.azure_operation.AzureOperationPoller._delay', - _shortcut_long_run_operation) + mock_in_unit_test(unit_test, + 'msrestazure.azure_operation.AzureOperationPoller._delay', + _shortcut_long_run_operation) -def _mock_in_unit_test(unit_test, target, replacement): +def mock_in_unit_test(unit_test, target, replacement): import mock import unittest From 9f3d8ac08fa66a2cffd9d02e1d3d2e6f02b25039 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Thu, 8 Jun 2017 08:15:57 -0700 Subject: [PATCH 066/167] Bump version to 0.3.0 --- setup.py | 2 +- src/azure_devtools/scenario_tests/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 961e0b8689a1..12e98ebe1888 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ from setuptools import setup -VERSION = "0.2.2" +VERSION = "0.3.0" CLASSIFIERS = [ diff --git a/src/azure_devtools/scenario_tests/__init__.py b/src/azure_devtools/scenario_tests/__init__.py index 6807ba341b70..1355b1b5189c 100644 --- a/src/azure_devtools/scenario_tests/__init__.py +++ b/src/azure_devtools/scenario_tests/__init__.py @@ -24,4 +24,4 @@ 'OAuthRequestResponsesFilter', 'DeploymentNameReplacer', 'GeneralNameReplacer', 'live_only', 'record_only', 'create_random_name', 'get_sha1_hash'] -__version__ = '0.2.2' +__version__ = '0.3.0' From 8d396bfa4e72bde7457adebe4a304c26126bc086 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Thu, 8 Jun 2017 09:00:38 -0700 Subject: [PATCH 067/167] Set disable_recording from kwarg --- src/azure_devtools/scenario_tests/preparers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/azure_devtools/scenario_tests/preparers.py b/src/azure_devtools/scenario_tests/preparers.py index d77e25ed348f..16d0ea0e3f61 100644 --- a/src/azure_devtools/scenario_tests/preparers.py +++ b/src/azure_devtools/scenario_tests/preparers.py @@ -22,7 +22,7 @@ def __init__(self, name_prefix, name_len, disable_recording=False): self.resource_random_name = None self.test_class_instance = None self.live_test = False - self.disable_recording = False + self.disable_recording = disable_recording def __call__(self, fn): def _preparer_wrapper(test_class_instance, **kwargs): From b38e87eef666e8a007d12d3c29b8b7746f981a37 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Thu, 8 Jun 2017 09:06:25 -0700 Subject: [PATCH 068/167] Don't treat mock_in_unit_test as a unittest --- src/azure_devtools/scenario_tests/patches.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/azure_devtools/scenario_tests/patches.py b/src/azure_devtools/scenario_tests/patches.py index 674b785c0aa9..bc0260876384 100644 --- a/src/azure_devtools/scenario_tests/patches.py +++ b/src/azure_devtools/scenario_tests/patches.py @@ -3,6 +3,8 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- +import unittest + from .exceptions import AzureTestError @@ -22,6 +24,7 @@ def _shortcut_long_run_operation(*args, **kwargs): # pylint: disable=unused-arg _shortcut_long_run_operation) +@unittest.skip("this is a helper, not a unit test") def mock_in_unit_test(unit_test, target, replacement): import mock import unittest From 741c5213871e7a124def21b5b4f623ab07ce10c8 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Wed, 14 Jun 2017 14:33:20 -0700 Subject: [PATCH 069/167] Remove skip for mock_in_unit_test, specify test dir --- setup.cfg | 5 ++++- src/azure_devtools/scenario_tests/patches.py | 3 --- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index 3480374bc2f2..c7f0bee1cc24 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,5 @@ [bdist_wheel] -universal=1 \ No newline at end of file +universal=1 + +[nosetests] +tests=src/azure_devtools/scenario_tests/tests \ No newline at end of file diff --git a/src/azure_devtools/scenario_tests/patches.py b/src/azure_devtools/scenario_tests/patches.py index bc0260876384..674b785c0aa9 100644 --- a/src/azure_devtools/scenario_tests/patches.py +++ b/src/azure_devtools/scenario_tests/patches.py @@ -3,8 +3,6 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -import unittest - from .exceptions import AzureTestError @@ -24,7 +22,6 @@ def _shortcut_long_run_operation(*args, **kwargs): # pylint: disable=unused-arg _shortcut_long_run_operation) -@unittest.skip("this is a helper, not a unit test") def mock_in_unit_test(unit_test, target, replacement): import mock import unittest From 01655ba8b5a936595a7c04e2375f61ccf975b6e7 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Thu, 15 Jun 2017 14:02:17 -0700 Subject: [PATCH 070/167] Update version numbers --- setup.py | 2 +- src/azure_devtools/scenario_tests/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 12e98ebe1888..3656ad8c5189 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ from setuptools import setup -VERSION = "0.3.0" +VERSION = "0.4.1" CLASSIFIERS = [ diff --git a/src/azure_devtools/scenario_tests/__init__.py b/src/azure_devtools/scenario_tests/__init__.py index 1355b1b5189c..937fd2c06e66 100644 --- a/src/azure_devtools/scenario_tests/__init__.py +++ b/src/azure_devtools/scenario_tests/__init__.py @@ -24,4 +24,4 @@ 'OAuthRequestResponsesFilter', 'DeploymentNameReplacer', 'GeneralNameReplacer', 'live_only', 'record_only', 'create_random_name', 'get_sha1_hash'] -__version__ = '0.3.0' +__version__ = '0.4.1' From ab50387cdf4987dc554c865cec2c8dd461c5453c Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Wed, 14 Jun 2017 11:04:55 -0700 Subject: [PATCH 071/167] Remove extraneous requirements --- requirements.txt | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index b14b05e33aa6..32a1eecb741d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,4 @@ -setuptools-markdown -ConfigArgParse -mock -nose -six -vcrpy \ No newline at end of file +-e . + +mock;python_version<="2.7" +nose \ No newline at end of file From 64d6479f96c9c2bbc65686273f86e9932b312ee7 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Wed, 14 Jun 2017 11:05:20 -0700 Subject: [PATCH 072/167] Remove outdated README.md stuff --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 3656ad8c5189..32f5a2de0032 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,6 @@ DEPENDENCIES = [ 'ConfigArgParse', - 'setuptools-markdown', 'six', 'vcrpy', ] @@ -40,7 +39,7 @@ name='azure-devtools', version=VERSION, description='Microsoft Azure Development Tools for SDK', - long_description_markdown_file='README.md', + long_description=README, license='MIT', author='Microsoft Corporation', author_email='ptvshelp@microsoft.com', From c33f0a9dfaee6c06ea2f7eb4660f74c2048c0ee4 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Wed, 14 Jun 2017 14:17:43 -0700 Subject: [PATCH 073/167] Update install for new requirements.txt --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7e8425fc65c3..8b8a26011053 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,6 @@ python: - "3.6" install: - pip install -r requirements.txt - - pip install -e . script: - nosetests deploy: From 17c6c6844b199b6e7696d4ba669332a6e5dd564b Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Wed, 14 Jun 2017 14:18:23 -0700 Subject: [PATCH 074/167] Add 3.6-dev to pythons --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 8b8a26011053..26d5bf2c70d8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ python: - "3.4" - "3.5" - "3.6" + - "3.6-dev" install: - pip install -r requirements.txt script: From d07dbc9b715c781ad8f240b9fb71a612b3b66afa Mon Sep 17 00:00:00 2001 From: Troy Dai Date: Sat, 17 Jun 2017 08:43:44 -0700 Subject: [PATCH 075/167] Update code style and run pylint in Travis --- .travis.yml | 1 + pylintrc | 26 +++++++++++++++++++ requirements.txt | 5 ++-- setup.py | 6 ++--- src/azure_devtools/scenario_tests/base.py | 21 +++++---------- src/azure_devtools/scenario_tests/config.py | 9 +++++-- src/azure_devtools/scenario_tests/const.py | 2 +- .../scenario_tests/preparers.py | 3 +-- 8 files changed, 49 insertions(+), 24 deletions(-) create mode 100644 pylintrc diff --git a/.travis.yml b/.travis.yml index 26d5bf2c70d8..750cb6bfb59f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ python: install: - pip install -r requirements.txt script: + - pylint src/azure_devtools - nosetests deploy: provider: pypi diff --git a/pylintrc b/pylintrc new file mode 100644 index 000000000000..9a75374b5076 --- /dev/null +++ b/pylintrc @@ -0,0 +1,26 @@ +[MESSAGES CONTROL] +# For all codes, run 'pylint --list-msgs' or go to 'http://pylint-messages.wikidot.com/all-codes' +# C0111 Missing docstring +# C0103 Invalid %s name "%s" +# I0011 Warning locally suppressed using disable-msg +# W0511 fixme +# R0401 Cyclic import (because of https://github.com/PyCQA/pylint/issues/850) +# R0913 Too many arguments - Due to the nature of the CLI many commands have large arguments set which reflect in large arguments set in corresponding methods. +disable=C0111, C0103 + +[FORMAT] +max-line-length=120 + +[VARIABLES] +# Tells whether we should check for unused import in __init__ files. +init-import=yes + +[DESIGN] +# Maximum number of locals for function / method body +# max-locals=8 +# Maximum number of branch for function / method body +# max-branches=16 + +[SIMILARITIES] +min-similarity-lines=8 + diff --git a/requirements.txt b/requirements.txt index 32a1eecb741d..c41c363ad9b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ -e . -mock;python_version<="2.7" -nose \ No newline at end of file +mock==2.0.0;python_version<="2.7" +nose==1.3.7 +pylint==1.7.1 diff --git a/setup.py b/setup.py index 32f5a2de0032..3827b7a1bd41 100644 --- a/setup.py +++ b/setup.py @@ -27,9 +27,9 @@ DEPENDENCIES = [ - 'ConfigArgParse', - 'six', - 'vcrpy', + 'ConfigArgParse>=0.12.0', + 'six>=1.10.0', + 'vcrpy>=1.11.1', ] with io.open('README.rst', 'r', encoding='utf-8') as f: diff --git a/src/azure_devtools/scenario_tests/base.py b/src/azure_devtools/scenario_tests/base.py index 409e57778d12..6c673f044e1a 100644 --- a/src/azure_devtools/scenario_tests/base.py +++ b/src/azure_devtools/scenario_tests/base.py @@ -18,22 +18,18 @@ from .utilities import create_random_name from .decorators import live_only -logger = logging.getLogger('azure_devtools.scenario_tests') - class IntegrationTestBase(unittest.TestCase): def __init__(self, method_name): super(IntegrationTestBase, self).__init__(method_name) self.diagnose = os.environ.get(ENV_TEST_DIAGNOSE, None) == 'True' + self.logger = logging.getLogger('azure_devtools.scenario_tests') def create_random_name(self, prefix, length): # pylint: disable=no-self-use return create_random_name(prefix=prefix, length=length) def create_temp_file(self, size_kb, full_random=False): - """ - Create a temporary file for testing. The test harness will delete the file during tearing - down. - """ + """ Create a temporary file for testing. The test harness will delete the file during tearing down. """ fd, path = tempfile.mkstemp() os.close(fd) self.addCleanup(lambda: os.remove(path)) @@ -50,8 +46,7 @@ def create_temp_file(self, size_kb, full_random=False): def create_temp_dir(self): """ - Create a temporary directory for testing. The test harness will delete the directory during - tearing down. + Create a temporary directory for testing. The test harness will delete the directory during tearing down. """ temp_dir = tempfile.mkdtemp() self.addCleanup(lambda: shutil.rmtree(temp_dir, ignore_errors=True)) @@ -87,10 +82,9 @@ class ReplayableTest(IntegrationTestBase): # pylint: disable=too-many-instance- 'x-ms-served-by', ] - def __init__(self, method_name, config_file=None, - recording_dir=None, recording_name=None, - recording_processors=None, replay_processors=None, - recording_patches=None, replay_patches=None): + def __init__(self, # pylint: disable=too-many-arguments + method_name, config_file=None, recording_dir=None, recording_name=None, recording_processors=None, + replay_processors=None, recording_patches=None, replay_patches=None): super(ReplayableTest, self).__init__(method_name) self.recording_processors = recording_processors or [] @@ -104,8 +98,7 @@ def __init__(self, method_name, config_file=None, self.disable_recording = False test_file_path = inspect.getfile(self.__class__) - recording_dir = recording_dir or os.path.join(os.path.dirname(test_file_path), - 'recordings') + recording_dir = recording_dir or os.path.join(os.path.dirname(test_file_path), 'recordings') self.is_live = self.config.record_mode self.vcr = vcr.VCR( diff --git a/src/azure_devtools/scenario_tests/config.py b/src/azure_devtools/scenario_tests/config.py index 14481caf838a..ac6ac6f91c0a 100644 --- a/src/azure_devtools/scenario_tests/config.py +++ b/src/azure_devtools/scenario_tests/config.py @@ -1,9 +1,14 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + import configargparse from .const import ENV_LIVE_TEST -class TestConfig(object): +class TestConfig(object): # pylint: disable=too-few-public-methods def __init__(self, parent_parsers=None, config_file=None): parent_parsers = parent_parsers or [] self.parser = configargparse.ArgumentParser(parents=parent_parsers) @@ -20,4 +25,4 @@ def __init__(self, parent_parsers=None, config_file=None): @property def record_mode(self): - return self.args.live_mode \ No newline at end of file + return self.args.live_mode diff --git a/src/azure_devtools/scenario_tests/const.py b/src/azure_devtools/scenario_tests/const.py index 18d0ba18c4ff..e27e7289a021 100644 --- a/src/azure_devtools/scenario_tests/const.py +++ b/src/azure_devtools/scenario_tests/const.py @@ -11,4 +11,4 @@ ENV_COMMAND_COVERAGE = 'AZURE_TEST_COMMAND_COVERAGE' ENV_LIVE_TEST = 'AZURE_TEST_RUN_LIVE' ENV_SKIP_ASSERT = 'AZURE_TEST_SKIP_ASSERT' -ENV_TEST_DIAGNOSE = 'AZURE_TEST_DIAGNOSE' \ No newline at end of file +ENV_TEST_DIAGNOSE = 'AZURE_TEST_DIAGNOSE' diff --git a/src/azure_devtools/scenario_tests/preparers.py b/src/azure_devtools/scenario_tests/preparers.py index 16d0ea0e3f61..eefcb5d16e62 100644 --- a/src/azure_devtools/scenario_tests/preparers.py +++ b/src/azure_devtools/scenario_tests/preparers.py @@ -54,7 +54,7 @@ def _preparer_wrapper(test_class_instance, **kwargs): args, _, kw, _ = inspect.getargspec(fn) # pylint: disable=deprecated-method if kw is None: args = set(args) - for key in [k for k in kwargs.keys() if k not in args]: + for key in [k for k in kwargs if k not in args]: del kwargs[key] fn(test_class_instance, **kwargs) @@ -101,7 +101,6 @@ def remove_resource_with_record_override(self, name, **kwargs): self.remove_resource(name, **kwargs) -# TODO: replaced by GeneralNameReplacer class SingleValueReplacer(RecordingProcessor): # pylint: disable=no-member def process_request(self, request): From 0f4456e26861fff7bf3ce2a92990cb676c4c6477 Mon Sep 17 00:00:00 2001 From: Troy Dai Date: Sat, 17 Jun 2017 10:50:00 -0700 Subject: [PATCH 076/167] Integrate with codecov.io --- .travis.yml | 4 +++- requirements.txt | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 750cb6bfb59f..8cdeec8f9b9b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,9 @@ install: - pip install -r requirements.txt script: - pylint src/azure_devtools - - nosetests + - nosetests --with-coverage --cover-branches +after_success: + - codecov deploy: provider: pypi user: Laurent.Mazuel diff --git a/requirements.txt b/requirements.txt index c41c363ad9b8..6de5c0a7503a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ --e . - +codecov==2.0.9 mock==2.0.0;python_version<="2.7" nose==1.3.7 pylint==1.7.1 + +-e . \ No newline at end of file From b1a9c074072818df2bb6820c32de6c15d48da4d8 Mon Sep 17 00:00:00 2001 From: Troy Dai Date: Sat, 17 Jun 2017 11:49:13 -0700 Subject: [PATCH 077/167] Add test cover IntegrationTestBase Also check in IntelliJ IDEA workspace configuration just in case someone else is using PyCharm --- .gitignore | 4 ++ .idea/azure-python-devtools.iml | 14 ++++++ .../inspectionProfiles/profiles_settings.xml | 7 +++ .idea/misc.xml | 4 ++ .idea/modules.xml | 8 ++++ .idea/vcs.xml | 6 +++ .../tests/test_integration_test_base.py | 48 +++++++++++++++++++ 7 files changed, 91 insertions(+) create mode 100644 .idea/azure-python-devtools.iml create mode 100644 .idea/inspectionProfiles/profiles_settings.xml create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/vcs.xml create mode 100644 src/azure_devtools/scenario_tests/tests/test_integration_test_base.py diff --git a/.gitignore b/.gitignore index ee458ab88ad8..d1f29649e772 100644 --- a/.gitignore +++ b/.gitignore @@ -94,3 +94,7 @@ ENV/ *.pyproj *.pyproj.user *.sln + +# PyCharm / IntelliJ IDEA +.idea/workspace.xml +.idea/tasks.xml diff --git a/.idea/azure-python-devtools.iml b/.idea/azure-python-devtools.iml new file mode 100644 index 000000000000..d1abafd6eada --- /dev/null +++ b/.idea/azure-python-devtools.iml @@ -0,0 +1,14 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 000000000000..c23ecacb3a2e --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,7 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 000000000000..d62449afccb3 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 000000000000..a63cd290bcbd --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 000000000000..94a25f7f4cb4 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/src/azure_devtools/scenario_tests/tests/test_integration_test_base.py b/src/azure_devtools/scenario_tests/tests/test_integration_test_base.py new file mode 100644 index 000000000000..a9e0fb1e0874 --- /dev/null +++ b/src/azure_devtools/scenario_tests/tests/test_integration_test_base.py @@ -0,0 +1,48 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os.path +import unittest +from nose.tools import ok_ +from azure_devtools.scenario_tests.base import IntegrationTestBase + + +class TestIntegrationTestBase(unittest.TestCase): + def test_integration_test_default_constructor(self): + class MockTest(IntegrationTestBase): + def __init__(self): + super(IntegrationTestBase, self).__init__('sample_test') + + def sample_test(self): + pass + + tb = MockTest() + + random_name = tb.create_random_name('example', 90) + self.assertEqual(len(random_name), 90) + self.assertTrue(random_name.startswith('example')) + + random_file = tb.create_temp_file(size_kb=16, full_random=False) + self.addCleanup(lambda: os.remove(random_file)) + self.assertTrue(os.path.isfile(random_file)) + self.assertEqual(os.path.getsize(random_file), 16 * 1024) + self.assertEqual(len(tb._cleanups), 1) + with open(random_file, 'rb') as fq: + # the file is blank + self.assertFalse(any(b for b in fq.read(16 * 1024) if b != '\x00')) + + random_file_2 = tb.create_temp_file(size_kb=8, full_random=True) + self.addCleanup(lambda: os.remove(random_file_2)) + self.assertTrue(os.path.isfile(random_file_2)) + self.assertEqual(os.path.getsize(random_file_2), 8 * 1024) + self.assertEqual(len(tb._cleanups), 2) + with open(random_file_2, 'rb') as fq: + # the file is blank + self.assertTrue(any(b for b in fq.read(8 * 1024) if b != '\x00')) + + random_dir = tb.create_temp_dir() + self.addCleanup(lambda: os.rmdir(random_dir)) + self.assertTrue(os.path.isdir(random_dir)) + self.assertEqual(len(tb._cleanups), 3) From fd14899eb070690b5c06cc19f8bad4e2b669a382 Mon Sep 17 00:00:00 2001 From: Troy Dai Date: Sun, 18 Jun 2017 09:12:22 -0700 Subject: [PATCH 078/167] Add test case for LiveTest constructor --- .../tests/test_integration_test_base.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/azure_devtools/scenario_tests/tests/test_integration_test_base.py b/src/azure_devtools/scenario_tests/tests/test_integration_test_base.py index a9e0fb1e0874..24d57e2e4550 100644 --- a/src/azure_devtools/scenario_tests/tests/test_integration_test_base.py +++ b/src/azure_devtools/scenario_tests/tests/test_integration_test_base.py @@ -5,8 +5,9 @@ import os.path import unittest -from nose.tools import ok_ -from azure_devtools.scenario_tests.base import IntegrationTestBase + +from azure_devtools.scenario_tests.const import ENV_LIVE_TEST +from azure_devtools.scenario_tests.base import IntegrationTestBase, LiveTest class TestIntegrationTestBase(unittest.TestCase): @@ -46,3 +47,13 @@ def sample_test(self): self.addCleanup(lambda: os.rmdir(random_dir)) self.assertTrue(os.path.isdir(random_dir)) self.assertEqual(len(tb._cleanups), 3) + + def test_live_test_default_constructor(self): + class MockTest(LiveTest): + def __init__(self): + super(LiveTest, self).__init__('sample_test') + + def sample_test(self): + self.assertFalse(True) + + self.assertIsNone(MockTest().run(), 'The live test is not skipped as expected') From 6cd35ae843a0937c6fbc771108b93df371cd1356 Mon Sep 17 00:00:00 2001 From: Troy Dai Date: Sun, 18 Jun 2017 12:16:43 -0700 Subject: [PATCH 079/167] Add tests cover create_random_name --- .../scenario_tests/tests/test_utilities.py | 44 +++++++++++++++++++ .../scenario_tests/utilities.py | 6 +-- 2 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 src/azure_devtools/scenario_tests/tests/test_utilities.py diff --git a/src/azure_devtools/scenario_tests/tests/test_utilities.py b/src/azure_devtools/scenario_tests/tests/test_utilities.py new file mode 100644 index 000000000000..ca52e439be2b --- /dev/null +++ b/src/azure_devtools/scenario_tests/tests/test_utilities.py @@ -0,0 +1,44 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import unittest +from azure_devtools.scenario_tests.utilities import create_random_name + + +class TestUtilityFunctions(unittest.TestCase): + def test_create_random_name_default_value(self): + default_generated_name = create_random_name() + self.assertTrue(default_generated_name.startswith('aztest')) + self.assertEqual(24, len(default_generated_name)) + self.assertTrue(isinstance(default_generated_name, str)) + + def test_create_random_name_randomness(self): + self.assertEqual(100, len(set([create_random_name() for _ in range(100)]))) + + def test_create_random_name_customization(self): + customized_name = create_random_name(prefix='pauline', length=61) + self.assertTrue(customized_name.startswith('pauline')) + self.assertEqual(61, len(customized_name)) + self.assertTrue(isinstance(customized_name, str)) + + def test_create_random_name_exception_long_prefix(self): + prefix = 'prefix-too-long' + + with self.assertRaises(ValueError) as cm: + create_random_name(prefix, length=len(prefix)-1) + self.assertEqual(str(cm.exception), 'The length of the prefix must not be longer than random name length') + + self.assertTrue(create_random_name(prefix, length=len(prefix)+4).startswith(prefix)) + + def test_create_random_name_exception_not_enough_space_for_randomness(self): + prefix = 'prefix-too-long' + + for i in range(4): + with self.assertRaises(ValueError) as cm: + create_random_name(prefix, length=len(prefix) + i) + self.assertEqual(str(cm.exception), 'The randomized part of the name is shorter than 4, which may not be ' + 'able to offer enough randomness') + + diff --git a/src/azure_devtools/scenario_tests/utilities.py b/src/azure_devtools/scenario_tests/utilities.py index e025c7c7c61e..9fab0d93db49 100644 --- a/src/azure_devtools/scenario_tests/utilities.py +++ b/src/azure_devtools/scenario_tests/utilities.py @@ -11,12 +11,12 @@ def create_random_name(prefix='aztest', length=24): if len(prefix) > length: - raise 'The length of the prefix must not be longer than random name length' + raise ValueError('The length of the prefix must not be longer than random name length') padding_size = length - len(prefix) if padding_size < 4: - raise 'The randomized part of the name is shorter than 4, which may not be able to offer ' \ - 'enough randomness' + raise ValueError('The randomized part of the name is shorter than 4, which may not be able to offer enough ' + 'randomness') random_bytes = os.urandom(int(math.ceil(float(padding_size) / 8) * 5)) random_padding = base64.b32encode(random_bytes)[:padding_size] From 863f9ee3dc43541d3c702ffa46217ef90ad385df Mon Sep 17 00:00:00 2001 From: Troy Dai Date: Sun, 18 Jun 2017 12:29:37 -0700 Subject: [PATCH 080/167] Add test covering get_sha1_hash --- .../scenario_tests/tests/test_utilities.py | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/src/azure_devtools/scenario_tests/tests/test_utilities.py b/src/azure_devtools/scenario_tests/tests/test_utilities.py index ca52e439be2b..26ceda9edd5d 100644 --- a/src/azure_devtools/scenario_tests/tests/test_utilities.py +++ b/src/azure_devtools/scenario_tests/tests/test_utilities.py @@ -4,7 +4,7 @@ # -------------------------------------------------------------------------------------------- import unittest -from azure_devtools.scenario_tests.utilities import create_random_name +from azure_devtools.scenario_tests.utilities import create_random_name, get_sha1_hash class TestUtilityFunctions(unittest.TestCase): @@ -41,4 +41,42 @@ def test_create_random_name_exception_not_enough_space_for_randomness(self): self.assertEqual(str(cm.exception), 'The randomized part of the name is shorter than 4, which may not be ' 'able to offer enough randomness') + def test_get_sha1_hash(self): + import tempfile + with tempfile.NamedTemporaryFile() as f: + content = b""" +All the world's a stage, +And all the men and women merely players; +They have their exits and their entrances, +And one man in his time plays many parts, +His acts being seven ages. At first, the infant, +Mewling and puking in the nurse's arms. +Then the whining schoolboy, with his satchel +And shining morning face, creeping like snail +Unwillingly to school. And then the lover, +Sighing like furnace, with a woeful ballad +Made to his mistress' eyebrow. Then a soldier, +Full of strange oaths and bearded like the pard, +Jealous in honor, sudden and quick in quarrel, +Seeking the bubble reputation +Even in the cannon's mouth. And then the justice, +In fair round belly with good capon lined, +With eyes severe and beard of formal cut, +Full of wise saws and modern instances; +And so he plays his part. The sixth age shifts +Into the lean and slippered pantaloon, +With spectacles on nose and pouch on side; +His youthful hose, well saved, a world too wide +For his shrunk shank, and his big manly voice, +Turning again toward childish treble, pipes +And whistles in his sound. Last scene of all, +That ends this strange eventful history, +Is second childishness and mere oblivion, +Sans teeth, sans eyes, sans taste, sans everything. +William Shakespeare + """ + f.write(content) + f.seek(0) + hash_value = get_sha1_hash(f.name) + self.assertEqual('6487bbdbd848686338d729e6076da1a795d1ae747642bf906469c6ccd9e642f9', hash_value) From 579f803e8e77dcefef5f0810fc90337c0736b90a Mon Sep 17 00:00:00 2001 From: Troy Dai Date: Sun, 18 Jun 2017 13:09:21 -0700 Subject: [PATCH 081/167] Add test cover RecordingProcessor base class --- .../tests/test_recording_processor.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 src/azure_devtools/scenario_tests/tests/test_recording_processor.py diff --git a/src/azure_devtools/scenario_tests/tests/test_recording_processor.py b/src/azure_devtools/scenario_tests/tests/test_recording_processor.py new file mode 100644 index 000000000000..7f5f42fa35b8 --- /dev/null +++ b/src/azure_devtools/scenario_tests/tests/test_recording_processor.py @@ -0,0 +1,28 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import unittest +from azure_devtools.scenario_tests.recording_processors import RecordingProcessor + + +class TestRecordingProcessors(unittest.TestCase): + def test_recording_processor_base_class(self): + rp = RecordingProcessor() + request_sample = {'url': 'https://www.bing,com', 'headers': {'beta': ['value_1', 'value_2']}} + response_sample = {'body': 'something', 'headers': {'charlie': ['value_3']}} + self.assertIs(request_sample, rp.process_request(request_sample)) # reference equality + self.assertIs(response_sample, rp.process_response(response_sample)) + + rp.replace_header(request_sample, 'beta', 'value_1', 'replaced_1') + self.assertSequenceEqual(request_sample['headers']['beta'], ['replaced_1', 'value_2']) + + rp.replace_header(request_sample, 'Beta', 'replaced_1', 'replaced_2') # case insensitive + self.assertSequenceEqual(request_sample['headers']['beta'], ['replaced_2', 'value_2']) + + rp.replace_header(request_sample, 'alpha', 'replaced_1', 'replaced_2') # ignore KeyError + self.assertSequenceEqual(request_sample['headers']['beta'], ['replaced_2', 'value_2']) + + rp.replace_header_fn(request_sample, 'beta', lambda v: 'customized') + self.assertSequenceEqual(request_sample['headers']['beta'], ['customized', 'customized']) From df25e5bdb718fb647a89cca7260734ce4e5a71ab Mon Sep 17 00:00:00 2001 From: Troy Dai Date: Sun, 18 Jun 2017 14:32:00 -0700 Subject: [PATCH 082/167] Add test covering SubscriptionRecordingProcessor --- .../tests/test_recording_processor.py | 55 ++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/src/azure_devtools/scenario_tests/tests/test_recording_processor.py b/src/azure_devtools/scenario_tests/tests/test_recording_processor.py index 7f5f42fa35b8..2c819bd2ecb9 100644 --- a/src/azure_devtools/scenario_tests/tests/test_recording_processor.py +++ b/src/azure_devtools/scenario_tests/tests/test_recording_processor.py @@ -4,7 +4,14 @@ # -------------------------------------------------------------------------------------------- import unittest -from azure_devtools.scenario_tests.recording_processors import RecordingProcessor +import uuid + +try: + import unittest.mock as mock +except ImportError: + import mock + +from azure_devtools.scenario_tests.recording_processors import RecordingProcessor, SubscriptionRecordingProcessor class TestRecordingProcessors(unittest.TestCase): @@ -26,3 +33,49 @@ def test_recording_processor_base_class(self): rp.replace_header_fn(request_sample, 'beta', lambda v: 'customized') self.assertSequenceEqual(request_sample['headers']['beta'], ['customized', 'customized']) + + def test_subscription_recording_processor_for_request(self): + replaced_subscription_id = str(uuid.uuid4()) + rp = SubscriptionRecordingProcessor(replaced_subscription_id) + + uri_templates = ['https://management.azure.com/subscriptions/{}/providers/Microsoft.ContainerRegistry/' + 'checkNameAvailability?api-version=2017-03-01', + 'https://graph.windows.net/{}/applications?api-version=1.6'] + + for template in uri_templates: + mock_request = mock.Mock() + mock_request.uri = template.format(str(uuid.uuid4())) + + rp.process_request(mock_request) + self.assertEqual(mock_request.uri, template.format(replaced_subscription_id)) + + def test_subscription_recording_processor_for_response(self): + replaced_subscription_id = str(uuid.uuid4()) + rp = SubscriptionRecordingProcessor(replaced_subscription_id) + + uri_templates = ['https://management.azure.com/subscriptions/{}/providers/Microsoft.ContainerRegistry/' + 'checkNameAvailability?api-version=2017-03-01', + 'https://graph.windows.net/{}/applications?api-version=1.6'] + + location_header_template = 'https://graph.windows.net/{}/directoryObjects/' \ + 'f604c53a-aa21-44d5-a41f-c1ef0b5304bd/Microsoft.DirectoryServices.Application' + + asyncoperation_header_template = 'https://management.azure.com/subscriptions/{}/resourceGroups/' \ + 'clitest.rg000001/providers/Microsoft.Sql/servers/clitestserver000002/' \ + 'databases/cliautomationdb01/azureAsyncOperation/' \ + '6ec6196b-fbaa-415f-8c1a-6cb634a96cb2?api-version=2014-04-01-Preview' + + for template in uri_templates: + mock_sub_id = str(uuid.uuid4()) + mock_response = dict({'body': {}}) + mock_response['body']['string'] = template.format(mock_sub_id) + mock_response['headers'] = {'Location': [location_header_template.format(mock_sub_id)], + 'azure-asyncoperation': [asyncoperation_header_template.format(mock_sub_id)]} + rp.process_response(mock_response) + self.assertEqual(mock_response['body']['string'], template.format(replaced_subscription_id)) + + # TODO: Restore after issue https://github.com/Azure/azure-python-devtools/issues/16 is fixed + # self.assertSequenceEqual(mock_response['headers']['Location'], + # [location_header_template.format(replaced_subscription_id)]) + self.assertSequenceEqual(mock_response['headers']['azure-asyncoperation'], + [asyncoperation_header_template.format(replaced_subscription_id)]) From aa64bee1eef9914bca663d57485a51e4c1ea9cd3 Mon Sep 17 00:00:00 2001 From: Troy Dai Date: Sun, 18 Jun 2017 15:39:36 -0700 Subject: [PATCH 083/167] Configure code coverage using configuration file --- .coveagerc | 8 ++++++++ .travis.yml | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 .coveagerc diff --git a/.coveagerc b/.coveagerc new file mode 100644 index 000000000000..72e8e9bb6b87 --- /dev/null +++ b/.coveagerc @@ -0,0 +1,8 @@ +[run] +branch = True + +[paths] +source = src/ + +[report] +omit = */tests/* diff --git a/.travis.yml b/.travis.yml index 8cdeec8f9b9b..f42cd62448fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ install: - pip install -r requirements.txt script: - pylint src/azure_devtools - - nosetests --with-coverage --cover-branches + - nosetests --with-coverage --cover-config-file=./.coveagerc after_success: - codecov deploy: @@ -22,4 +22,4 @@ deploy: distributions: "sdist bdist_wheel" on: tags: true - python: '3.6' \ No newline at end of file + python: '3.6' From c640434845211f5955835baf1a7e092308059540 Mon Sep 17 00:00:00 2001 From: Troy Dai Date: Sun, 18 Jun 2017 22:17:17 -0700 Subject: [PATCH 084/167] Use unittest instead of nosetests to drive the automation --- .travis.yml | 2 +- src/azure_devtools/scenario_tests/tests/__init__.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 src/azure_devtools/scenario_tests/tests/__init__.py diff --git a/.travis.yml b/.travis.yml index f42cd62448fa..8aac370b00ee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,7 @@ install: - pip install -r requirements.txt script: - pylint src/azure_devtools - - nosetests --with-coverage --cover-config-file=./.coveagerc + - coverage run -m unittest discover -s src -v after_success: - codecov deploy: diff --git a/src/azure_devtools/scenario_tests/tests/__init__.py b/src/azure_devtools/scenario_tests/tests/__init__.py new file mode 100644 index 000000000000..34913fb394d7 --- /dev/null +++ b/src/azure_devtools/scenario_tests/tests/__init__.py @@ -0,0 +1,4 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- From b48db41508b7e9de270b9294aab46e815712452e Mon Sep 17 00:00:00 2001 From: Troy Dai Date: Sun, 18 Jun 2017 22:26:54 -0700 Subject: [PATCH 085/167] Fix a few code style issues --- pylintrc | 2 +- .../scenario_tests/tests/test_config.py | 6 +++++- .../tests/test_integration_test_base.py | 15 +++++++-------- .../tests/test_recording_processor.py | 5 ++--- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/pylintrc b/pylintrc index 9a75374b5076..01ae42f50e4a 100644 --- a/pylintrc +++ b/pylintrc @@ -6,7 +6,7 @@ # W0511 fixme # R0401 Cyclic import (because of https://github.com/PyCQA/pylint/issues/850) # R0913 Too many arguments - Due to the nature of the CLI many commands have large arguments set which reflect in large arguments set in corresponding methods. -disable=C0111, C0103 +disable=C0111, C0103, W0511 [FORMAT] max-line-length=120 diff --git a/src/azure_devtools/scenario_tests/tests/test_config.py b/src/azure_devtools/scenario_tests/tests/test_config.py index bc59cee93f89..88035c903b6f 100644 --- a/src/azure_devtools/scenario_tests/tests/test_config.py +++ b/src/azure_devtools/scenario_tests/tests/test_config.py @@ -1,4 +1,8 @@ -import argparse +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + import os import tempfile import unittest diff --git a/src/azure_devtools/scenario_tests/tests/test_integration_test_base.py b/src/azure_devtools/scenario_tests/tests/test_integration_test_base.py index 24d57e2e4550..bda92bbbcb9e 100644 --- a/src/azure_devtools/scenario_tests/tests/test_integration_test_base.py +++ b/src/azure_devtools/scenario_tests/tests/test_integration_test_base.py @@ -6,7 +6,6 @@ import os.path import unittest -from azure_devtools.scenario_tests.const import ENV_LIVE_TEST from azure_devtools.scenario_tests.base import IntegrationTestBase, LiveTest @@ -14,8 +13,8 @@ class TestIntegrationTestBase(unittest.TestCase): def test_integration_test_default_constructor(self): class MockTest(IntegrationTestBase): def __init__(self): - super(IntegrationTestBase, self).__init__('sample_test') - + super(MockTest, self).__init__('sample_test') + def sample_test(self): pass @@ -29,7 +28,7 @@ def sample_test(self): self.addCleanup(lambda: os.remove(random_file)) self.assertTrue(os.path.isfile(random_file)) self.assertEqual(os.path.getsize(random_file), 16 * 1024) - self.assertEqual(len(tb._cleanups), 1) + self.assertEqual(len(tb._cleanups), 1) # pylint: disable=protected-access with open(random_file, 'rb') as fq: # the file is blank self.assertFalse(any(b for b in fq.read(16 * 1024) if b != '\x00')) @@ -38,7 +37,7 @@ def sample_test(self): self.addCleanup(lambda: os.remove(random_file_2)) self.assertTrue(os.path.isfile(random_file_2)) self.assertEqual(os.path.getsize(random_file_2), 8 * 1024) - self.assertEqual(len(tb._cleanups), 2) + self.assertEqual(len(tb._cleanups), 2) # pylint: disable=protected-access with open(random_file_2, 'rb') as fq: # the file is blank self.assertTrue(any(b for b in fq.read(8 * 1024) if b != '\x00')) @@ -46,14 +45,14 @@ def sample_test(self): random_dir = tb.create_temp_dir() self.addCleanup(lambda: os.rmdir(random_dir)) self.assertTrue(os.path.isdir(random_dir)) - self.assertEqual(len(tb._cleanups), 3) + self.assertEqual(len(tb._cleanups), 3) # pylint: disable=protected-access def test_live_test_default_constructor(self): class MockTest(LiveTest): def __init__(self): - super(LiveTest, self).__init__('sample_test') + super(MockTest, self).__init__('sample_test') def sample_test(self): - self.assertFalse(True) + pass self.assertIsNone(MockTest().run(), 'The live test is not skipped as expected') diff --git a/src/azure_devtools/scenario_tests/tests/test_recording_processor.py b/src/azure_devtools/scenario_tests/tests/test_recording_processor.py index 2c819bd2ecb9..953daef87516 100644 --- a/src/azure_devtools/scenario_tests/tests/test_recording_processor.py +++ b/src/azure_devtools/scenario_tests/tests/test_recording_processor.py @@ -3,13 +3,12 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -import unittest -import uuid - try: import unittest.mock as mock except ImportError: import mock +import unittest +import uuid from azure_devtools.scenario_tests.recording_processors import RecordingProcessor, SubscriptionRecordingProcessor From 3b896d794e0770ebbb12da11a1562f2455831c80 Mon Sep 17 00:00:00 2001 From: Troy Dai Date: Sun, 18 Jun 2017 22:39:27 -0700 Subject: [PATCH 086/167] Rename coveragerc file --- .coveagerc => .coveragerc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) rename .coveagerc => .coveragerc (52%) diff --git a/.coveagerc b/.coveragerc similarity index 52% rename from .coveagerc rename to .coveragerc index 72e8e9bb6b87..c04abec58154 100644 --- a/.coveagerc +++ b/.coveragerc @@ -1,8 +1,11 @@ [run] branch = True +include = src/* +omit = + */tests/* + env/* [paths] source = src/ [report] -omit = */tests/* From c9886bf5e2448217a0f72a5d2ea28ee05ed79f98 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Thu, 22 Jun 2017 15:25:48 -0700 Subject: [PATCH 087/167] Add AccessTokenProcessor --- setup.py | 2 +- src/azure_devtools/scenario_tests/__init__.py | 5 +++-- .../scenario_tests/recording_processors.py | 16 ++++++++++++++++ .../tests/test_recording_processor.py | 18 +++++++++++++++++- 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index 3827b7a1bd41..d8d1a328390e 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ from setuptools import setup -VERSION = "0.4.1" +VERSION = "0.4.3" CLASSIFIERS = [ diff --git a/src/azure_devtools/scenario_tests/__init__.py b/src/azure_devtools/scenario_tests/__init__.py index 937fd2c06e66..b042cf6a8924 100644 --- a/src/azure_devtools/scenario_tests/__init__.py +++ b/src/azure_devtools/scenario_tests/__init__.py @@ -11,7 +11,7 @@ from .recording_processors import ( RecordingProcessor, SubscriptionRecordingProcessor, LargeRequestBodyProcessor, LargeResponseBodyProcessor, LargeResponseBodyReplacer, - OAuthRequestResponsesFilter, DeploymentNameReplacer, GeneralNameReplacer, + OAuthRequestResponsesFilter, DeploymentNameReplacer, GeneralNameReplacer, AccessTokenReplacer, ) from .utilities import create_random_name, get_sha1_hash @@ -22,6 +22,7 @@ 'RecordingProcessor', 'SubscriptionRecordingProcessor', 'LargeRequestBodyProcessor', 'LargeResponseBodyProcessor', 'LargeResponseBodyReplacer', 'OAuthRequestResponsesFilter', 'DeploymentNameReplacer', 'GeneralNameReplacer', + 'AccessTokenReplacer', 'live_only', 'record_only', 'create_random_name', 'get_sha1_hash'] -__version__ = '0.4.1' +__version__ = '0.4.3' diff --git a/src/azure_devtools/scenario_tests/recording_processors.py b/src/azure_devtools/scenario_tests/recording_processors.py index 7e33b689d7d5..d6fc22249a05 100644 --- a/src/azure_devtools/scenario_tests/recording_processors.py +++ b/src/azure_devtools/scenario_tests/recording_processors.py @@ -127,6 +127,22 @@ def process_request(self, request): return request +class AccessTokenReplacer(RecordingProcessor): + """Replace the access token for service principal authentication in a response body.""" + def __init__(self, replacement='fake_token'): + self._replacement = replacement + + def process_response(self, response): + import json + try: + body = json.loads(response['body']['string']) + body['access_token'] = self._replacement + except (KeyError, ValueError): + return response + response['body']['string'] = json.dumps(body) + return response + + class GeneralNameReplacer(RecordingProcessor): def __init__(self): self.names_name = [] diff --git a/src/azure_devtools/scenario_tests/tests/test_recording_processor.py b/src/azure_devtools/scenario_tests/tests/test_recording_processor.py index 953daef87516..54a049849abd 100644 --- a/src/azure_devtools/scenario_tests/tests/test_recording_processor.py +++ b/src/azure_devtools/scenario_tests/tests/test_recording_processor.py @@ -3,6 +3,7 @@ # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- +import json try: import unittest.mock as mock except ImportError: @@ -10,7 +11,9 @@ import unittest import uuid -from azure_devtools.scenario_tests.recording_processors import RecordingProcessor, SubscriptionRecordingProcessor +from azure_devtools.scenario_tests.recording_processors import ( + RecordingProcessor, SubscriptionRecordingProcessor, AccessTokenReplacer +) class TestRecordingProcessors(unittest.TestCase): @@ -48,6 +51,19 @@ def test_subscription_recording_processor_for_request(self): rp.process_request(mock_request) self.assertEqual(mock_request.uri, template.format(replaced_subscription_id)) + def test_access_token_processor(self): + replaced_subscription_id = 'test_fake_token' + rp = AccessTokenReplacer(replaced_subscription_id) + + TOKEN_STR = '{"token_type": "Bearer", "resource": "url", "access_token": "real_token"}' + token_response_sample = {'body': {'string': TOKEN_STR}} + + self.assertEqual(json.loads(rp.process_response(token_response_sample)['body']['string'])['access_token'], + replaced_subscription_id) + + no_token_response_sample = {'body': {'string': '{"location": "westus"}'}} + self.assertDictEqual(rp.process_response(no_token_response_sample), no_token_response_sample) + def test_subscription_recording_processor_for_response(self): replaced_subscription_id = str(uuid.uuid4()) rp = SubscriptionRecordingProcessor(replaced_subscription_id) From cf97ac73b53357f27393e238b47543c80e6e0cb9 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Thu, 15 Jun 2017 12:58:57 -0700 Subject: [PATCH 088/167] Split intro and overview into clauses --- doc/recording_vcr_tests.md | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/doc/recording_vcr_tests.md b/doc/recording_vcr_tests.md index a2425a369fb9..9d64e348e5d4 100644 --- a/doc/recording_vcr_tests.md +++ b/doc/recording_vcr_tests.md @@ -1,17 +1,35 @@ Recording Command Tests with VCR.py ======================================== -Azure CLI uses the VCR.py library to record the HTTP messages exchanged during a program run and play them back at a later time, making it useful for creating command level scenario tests. These tests can be replayed at a later time without any network activity, allowing us to detect regressions in the handling of parameters and in the compatability between AzureCLI and the PythonSDK. +Azure CLI uses the VCR.py library +to record the HTTP messages exchanged during a program run +and play them back at a later time, +making it useful for creating command level scenario tests. +These tests can be replayed at a later time without any network activity, +allowing us to detect regressions in the handling of parameters +and in the compatability between AzureCLI and the PythonSDK. ## Overview -Each command module has a `tests` folder with a file called: `test__commands.py`. This is where you will define tests. - -Tests all derive from the `VCRTestBase` class found in `azure.cli.core.test_utils.vcr_test_base`. This class exposes the VCR tests using the standard Python `unittest` framework and allows the tests to be discovered by and debugged in Visual Studio. - -The majority of tests however inherit from the `ResourceGroupVCRTestBase` class as this handles creating and tearing down the test resource group automatically, helping to ensure that tests can be recorded and cleaned up without manual creation or deletion of resources. - -After adding your test, run it. The test driver will automatically detect the test is unrecorded and record the HTTP requests and responses in a cassette .yaml file. If the test succeeds, the cassette will be preserved and future playthroughs of the test will come from the cassette. +Each command module has a `tests` folder with a file called: +`test__commands.py`. +This is where you will define tests. + +Tests all derive from the `VCRTestBase` class +found in `azure.cli.core.test_utils.vcr_test_base`. +This class exposes the VCR tests using the standard Python `unittest` framework +and allows the tests to be discovered by and debugged in Visual Studio. + +The majority of tests however inherit from the `ResourceGroupVCRTestBase` class +as this handles creating and tearing down the test resource group automatically, +helping to ensure that tests can be recorded and cleaned up +without manual creation or deletion of resources. + +After adding your test, run it. +The test driver will automatically detect the test is unrecorded +and record the HTTP requests and responses in a cassette .yaml file. +If the test succeeds, +the cassette will be preserved and future playthroughs of the test will come from the cassette. If the tests are run on TravisCI, any tests which cannot be replayed will automatically fail. From 3809c2974342e22c6d42f67010e0fdc91569d394 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Thu, 15 Jun 2017 14:40:52 -0700 Subject: [PATCH 089/167] Back to paragraphs --- doc/recording_vcr_tests.md | 34 ++++++++-------------------------- 1 file changed, 8 insertions(+), 26 deletions(-) diff --git a/doc/recording_vcr_tests.md b/doc/recording_vcr_tests.md index 9d64e348e5d4..aa1925c0b1d5 100644 --- a/doc/recording_vcr_tests.md +++ b/doc/recording_vcr_tests.md @@ -1,35 +1,17 @@ Recording Command Tests with VCR.py ======================================== -Azure CLI uses the VCR.py library -to record the HTTP messages exchanged during a program run -and play them back at a later time, -making it useful for creating command level scenario tests. -These tests can be replayed at a later time without any network activity, -allowing us to detect regressions in the handling of parameters -and in the compatability between AzureCLI and the PythonSDK. +Azure CLI uses the VCR.py library to record the HTTP messages exchanged during a program run and play them back at a later time, making it useful for creating command level scenario tests. These tests can be replayed at a later time without any network activity, allowing us to detect regressions in the handling of parameters and in the compatability between AzureCLI and the PythonSDK. ## Overview -Each command module has a `tests` folder with a file called: -`test__commands.py`. -This is where you will define tests. - -Tests all derive from the `VCRTestBase` class -found in `azure.cli.core.test_utils.vcr_test_base`. -This class exposes the VCR tests using the standard Python `unittest` framework -and allows the tests to be discovered by and debugged in Visual Studio. - -The majority of tests however inherit from the `ResourceGroupVCRTestBase` class -as this handles creating and tearing down the test resource group automatically, -helping to ensure that tests can be recorded and cleaned up -without manual creation or deletion of resources. - -After adding your test, run it. -The test driver will automatically detect the test is unrecorded -and record the HTTP requests and responses in a cassette .yaml file. -If the test succeeds, -the cassette will be preserved and future playthroughs of the test will come from the cassette. +Each command module has a `tests` folder with a file called: `test__commands.py`. This is where you will define tests. + +Tests all derive from the `VCRTestBase` class found in `azure.cli.core.test_utils.vcr_test_base`. This class exposes the VCR tests using the standard Python `unittest` framework and allows the tests to be discovered by and debugged in Visual Studio. + +The majority of tests however inherit from the `ResourceGroupVCRTestBase` class as this handles creating and tearing down the test resource group automatically, helping to ensure that tests can be recorded and cleaned up without manual creation or deletion of resources. + +After adding your test, run it. The test driver will automatically detect the test is unrecorded and record the HTTP requests and responses in a cassette .yaml file. If the test succeeds, the cassette will be preserved and future playthroughs of the test will come from the cassette. If the tests are run on TravisCI, any tests which cannot be replayed will automatically fail. From 6ab08154ae68de40dc28530fbd639b1d6e2e2b30 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Thu, 15 Jun 2017 14:42:15 -0700 Subject: [PATCH 090/167] Update intro, add vcr link --- doc/recording_vcr_tests.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/recording_vcr_tests.md b/doc/recording_vcr_tests.md index aa1925c0b1d5..d48074527d82 100644 --- a/doc/recording_vcr_tests.md +++ b/doc/recording_vcr_tests.md @@ -1,7 +1,7 @@ -Recording Command Tests with VCR.py +Recording Scenario Tests with VCR.py ======================================== -Azure CLI uses the VCR.py library to record the HTTP messages exchanged during a program run and play them back at a later time, making it useful for creating command level scenario tests. These tests can be replayed at a later time without any network activity, allowing us to detect regressions in the handling of parameters and in the compatability between AzureCLI and the PythonSDK. +The `scenario_tests` package uses the [VCR.py](https://pypi.python.org/pypi/vcrpy) library to record the HTTP messages exchanged during a program run and play them back at a later time, making it useful for creating command level scenario tests. These tests can be replayed at a later time without any network activity, allowing us to detect regressions in the handling of parameters and in the compatability between AzureCLI and the PythonSDK. ## Overview From 6e848c6686aec6bfbe4a4760bcf39710425ea2eb Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Thu, 15 Jun 2017 14:44:50 -0700 Subject: [PATCH 091/167] Update intro paragraph to be more general --- doc/recording_vcr_tests.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/recording_vcr_tests.md b/doc/recording_vcr_tests.md index d48074527d82..a352ef501b19 100644 --- a/doc/recording_vcr_tests.md +++ b/doc/recording_vcr_tests.md @@ -1,7 +1,7 @@ Recording Scenario Tests with VCR.py ======================================== -The `scenario_tests` package uses the [VCR.py](https://pypi.python.org/pypi/vcrpy) library to record the HTTP messages exchanged during a program run and play them back at a later time, making it useful for creating command level scenario tests. These tests can be replayed at a later time without any network activity, allowing us to detect regressions in the handling of parameters and in the compatability between AzureCLI and the PythonSDK. +The `scenario_tests` package uses the [VCR.py](https://pypi.python.org/pypi/vcrpy) library to record the HTTP messages exchanged during a program run and play them back at a later time, making it useful for creating "scenario tests" that interact with Azure (or other) services. These tests can be replayed at a later time without any network activity, allowing us to detect changes in the Python layers between the code being tested and the underlying REST API. ## Overview From df97434a5efe8cea481340464555b5b35f3e405f Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Thu, 15 Jun 2017 14:45:09 -0700 Subject: [PATCH 092/167] Remove reference to command modules --- doc/recording_vcr_tests.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/recording_vcr_tests.md b/doc/recording_vcr_tests.md index a352ef501b19..7233ace30032 100644 --- a/doc/recording_vcr_tests.md +++ b/doc/recording_vcr_tests.md @@ -5,8 +5,6 @@ The `scenario_tests` package uses the [VCR.py](https://pypi.python.org/pypi/vcrp ## Overview -Each command module has a `tests` folder with a file called: `test__commands.py`. This is where you will define tests. - Tests all derive from the `VCRTestBase` class found in `azure.cli.core.test_utils.vcr_test_base`. This class exposes the VCR tests using the standard Python `unittest` framework and allows the tests to be discovered by and debugged in Visual Studio. The majority of tests however inherit from the `ResourceGroupVCRTestBase` class as this handles creating and tearing down the test resource group automatically, helping to ensure that tests can be recorded and cleaned up without manual creation or deletion of resources. From 071440685a2e87c74688ff594fa9a584cef0e732 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Thu, 15 Jun 2017 14:49:01 -0700 Subject: [PATCH 093/167] Update test and module refs; clarify VCR.py role --- doc/recording_vcr_tests.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/recording_vcr_tests.md b/doc/recording_vcr_tests.md index 7233ace30032..110d1bb3795f 100644 --- a/doc/recording_vcr_tests.md +++ b/doc/recording_vcr_tests.md @@ -5,11 +5,11 @@ The `scenario_tests` package uses the [VCR.py](https://pypi.python.org/pypi/vcrp ## Overview -Tests all derive from the `VCRTestBase` class found in `azure.cli.core.test_utils.vcr_test_base`. This class exposes the VCR tests using the standard Python `unittest` framework and allows the tests to be discovered by and debugged in Visual Studio. +Tests all derive from the `ReplayableTest` class found in `azure_devtools.scenario_tests.base`. This class exposes the VCR tests using the standard Python `unittest` framework and allows the tests to be discovered by and debugged in Visual Studio. The majority of tests however inherit from the `ResourceGroupVCRTestBase` class as this handles creating and tearing down the test resource group automatically, helping to ensure that tests can be recorded and cleaned up without manual creation or deletion of resources. -After adding your test, run it. The test driver will automatically detect the test is unrecorded and record the HTTP requests and responses in a cassette .yaml file. If the test succeeds, the cassette will be preserved and future playthroughs of the test will come from the cassette. +When you run a test, the test driver will automatically detect the test is unrecorded and record the HTTP requests and responses in a .yaml file referred to by VCR.py as a "cassette." If the test succeeds, the cassette will be preserved and future playthroughs of the test will come from the cassette rather than using actual network communication. If the tests are run on TravisCI, any tests which cannot be replayed will automatically fail. From 686c7a51368a9c5289b02a1df0b78538bcf39171 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Thu, 15 Jun 2017 14:51:18 -0700 Subject: [PATCH 094/167] Remove outdated reference to class with builtin preparer --- doc/recording_vcr_tests.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/recording_vcr_tests.md b/doc/recording_vcr_tests.md index 110d1bb3795f..d1ebb35b0664 100644 --- a/doc/recording_vcr_tests.md +++ b/doc/recording_vcr_tests.md @@ -7,8 +7,6 @@ The `scenario_tests` package uses the [VCR.py](https://pypi.python.org/pypi/vcrp Tests all derive from the `ReplayableTest` class found in `azure_devtools.scenario_tests.base`. This class exposes the VCR tests using the standard Python `unittest` framework and allows the tests to be discovered by and debugged in Visual Studio. -The majority of tests however inherit from the `ResourceGroupVCRTestBase` class as this handles creating and tearing down the test resource group automatically, helping to ensure that tests can be recorded and cleaned up without manual creation or deletion of resources. - When you run a test, the test driver will automatically detect the test is unrecorded and record the HTTP requests and responses in a .yaml file referred to by VCR.py as a "cassette." If the test succeeds, the cassette will be preserved and future playthroughs of the test will come from the cassette rather than using actual network communication. If the tests are run on TravisCI, any tests which cannot be replayed will automatically fail. From 39cd932067d041c2cd2c3c6710b4d6796cc7defb Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Thu, 15 Jun 2017 15:19:52 -0700 Subject: [PATCH 095/167] Replace intro text Use intro stolen from defunct recording_vcr_tests.md --- doc/scenario_base_tests.md | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/doc/scenario_base_tests.md b/doc/scenario_base_tests.md index 1df6400d4145..268bd844715d 100644 --- a/doc/scenario_base_tests.md +++ b/doc/scenario_base_tests.md @@ -1,6 +1,33 @@ # How to write ScenarioTest based VCR test -The `ScenarioTest` class is introduced in pull request [#2393](https://github.com/Azure/azure-cli/pull/2393). It is the preferred base class for and VCR based test cases from now on. The `ScenarioTest` class is designed to be a better and easier test harness for authoring scenario based VCR test. +The `scenario_tests` package uses the [VCR.py](https://pypi.python.org/pypi/vcrpy) library +to record the HTTP messages exchanged during a program run +and play them back at a later time, +making it useful for creating "scenario tests" +that interact with Azure (or other) services. +These tests can be replayed at a later time without any network activity, +allowing us to detect changes in the Python layers +between the code being tested and the underlying REST API. + + +## Overview + +Tests all derive from the `ReplayableTest` class +found in `azure_devtools.scenario_tests.base`. +This class exposes the VCR tests using the standard Python `unittest` framework +and allows the tests to be discovered by and debugged in Visual Studio. + +When you run a test, +the test driver will automatically detect the test is unrecorded +and record the HTTP requests and responses in a .yaml file +(referred to by VCR.py as a "cassette"). +If the test succeeds, the cassette will be preserved +and future playthroughs of the test will come from the cassette +rather than using actual network communication. + +If the tests are run on TravisCI, +any tests which cannot be replayed will automatically fail. + ### Sample 1. Basic fixture ```Python From a585e0c485146af5bc86432ca7c126d69ca7db51 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Thu, 15 Jun 2017 15:46:01 -0700 Subject: [PATCH 096/167] s/ScenarioTest/ReplayableTest/ --- doc/scenario_base_tests.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/scenario_base_tests.md b/doc/scenario_base_tests.md index 268bd844715d..3c1840a8625d 100644 --- a/doc/scenario_base_tests.md +++ b/doc/scenario_base_tests.md @@ -1,4 +1,4 @@ -# How to write ScenarioTest based VCR test +# How to write ReplayableTest based VCR tests The `scenario_tests` package uses the [VCR.py](https://pypi.python.org/pypi/vcrpy) library to record the HTTP messages exchanged during a program run @@ -31,9 +31,9 @@ any tests which cannot be replayed will automatically fail. ### Sample 1. Basic fixture ```Python -from azure.cli.testsdk import ScenarioTest +from azure_devtools.scenario_tests import ReplayableTest -class StorageAccountTests(ScenarioTest): +class StorageAccountTests(ReplayableTest): def test_list_storage_account(self): self.cmd('az storage account list') ``` From 0d8b639cb17117231e1816b674d555baed717c0a Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Thu, 15 Jun 2017 15:49:54 -0700 Subject: [PATCH 097/167] Add note about semantic linefeeds --- doc/scenario_base_tests.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/scenario_base_tests.md b/doc/scenario_base_tests.md index 3c1840a8625d..15f95ba8baf6 100644 --- a/doc/scenario_base_tests.md +++ b/doc/scenario_base_tests.md @@ -204,3 +204,9 @@ Note: 1. Two storage accounts name should be assigned to different function parameters. 2. The resource group name is not required in test so the function doesn't have to declare a parameter to accept the name. However it doesn't mean that the resource group is not created. Its name is in the keyworded parameter dictionary for all the preparer to consume. It is removed before the test function is actually invoked. + +--- + +Note: This document's source uses +[semantic linefeeds](http://rhodesmill.org/brandon/2012/one-sentence-per-line/) +to make diffs and updates clearer. \ No newline at end of file From 2ab07ead4f598c84b639e2ff187e3242ca93bb76 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Fri, 16 Jun 2017 08:33:54 -0700 Subject: [PATCH 098/167] Make semantic linefeeds note an HTML comment --- doc/scenario_base_tests.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/scenario_base_tests.md b/doc/scenario_base_tests.md index 15f95ba8baf6..ec2d1d107802 100644 --- a/doc/scenario_base_tests.md +++ b/doc/scenario_base_tests.md @@ -205,8 +205,8 @@ Note: 1. Two storage accounts name should be assigned to different function parameters. 2. The resource group name is not required in test so the function doesn't have to declare a parameter to accept the name. However it doesn't mean that the resource group is not created. Its name is in the keyworded parameter dictionary for all the preparer to consume. It is removed before the test function is actually invoked. ---- - + \ No newline at end of file From 3f306f03f165b78accf25e573542daaf22fe3f60 Mon Sep 17 00:00:00 2001 From: Ian McCowan Date: Fri, 16 Jun 2017 11:45:18 -0700 Subject: [PATCH 099/167] Remove CLI examples and add links to consumers --- doc/scenario_base_tests.md | 184 +++---------------------------------- 1 file changed, 11 insertions(+), 173 deletions(-) diff --git a/doc/scenario_base_tests.md b/doc/scenario_base_tests.md index ec2d1d107802..21f222d65e26 100644 --- a/doc/scenario_base_tests.md +++ b/doc/scenario_base_tests.md @@ -28,182 +28,20 @@ rather than using actual network communication. If the tests are run on TravisCI, any tests which cannot be replayed will automatically fail. +`ReplayableTest` itself derives from `IntegrationTestBase`, +which provides some helpful methods for use in more general unit tests +but no functionality pertaining to network communication. -### Sample 1. Basic fixture -```Python -from azure_devtools.scenario_tests import ReplayableTest -class StorageAccountTests(ReplayableTest): - def test_list_storage_account(self): - self.cmd('az storage account list') -``` -Note: +## Subclassing ReplayableTest and features -1. When the test is run without recording file, the test will be run under live mode. A recording file will be created at `recording/.yaml` -2. Wrap the command in `self.cmd` method. It will assert the exit code of the command to be zero. -3. All the functions and classes your need for writing tests are included in `azure.cli.testsdk` namespace. It is recommanded __not__ to refrenced to the sub-namespace to avoid breaking changes. - -### Sample 2. Validate the return value in JSON -``` Python -class StorageAccountTests(ScenarioTest): - def test_list_storage_account(self): - accounts_list = self.cmd('az storage account list').get_output_in_json() - assert len(accounts_list) > 0 -``` -Note: - -1. The return value of `self.cmd` is an instance of class `ExecutionResult`. It has the exit code and stdout as its properties. -2. `get_output_in_json` deserialize the output to a JSON object - -Tip: - -1. Don't make any rigid assertions based on any assumptions which may not stand in a live test environment. - - -### Sample 3. Validate the return JSON value using JMESPath -``` Python -from azure.cli.testsdk import ScenarioTest, JMESPathCheck - -class StorageAccountTests(ScenarioTest): - def test_list_storage_account(self): - self.cmd('az account list-locations', - checks=[JMESPathCheck("[?name=='westus'].displayName | [0]", 'West US')]) -``` -Note: - -1. What is JMESPath? [JMESPath is a query language for JSON](http://jmespath.org/) -2. If a command is return value in JSON, multiple JMESPath based check can be added to the checks list to validate the result. -3. In addition to the `JMESPatchCheck`, there are other checks list `NoneCheck` which validate the output is `None`. The check mechanism is extensible. Any callable accept `ExecutionResult` can act as a check. - - -### Sample 4. Prepare a resource group for a test -``` Python -from azure.cli.testsdk import ScenarioTest, JMESPathCheck, ResourceGroupPreparer - -class StorageAccountTests(ScenarioTest): - @ResourceGroupPreparer() - def test_create_storage_account(self, resource_group): - self.cmd('az group show -n {}'.format(resource_group), checks=[ - JMESPathCheck('name', resource_group), - JMESPathCheck('properties.provisioningState', 'Succeeded') - ]) -``` -Note: - -1. The preparers are executed in before each test in the test class when `setUp` is executed. The resource will be cleaned up after testing. -2. The resource group name is injected to the test method as a parameter. By default 'ResourceGroupPreparer' set the value to 'resource_group' parameter. The target parameter can be customized (see following samples). -3. The resource group will be deleted in async for performance reason. - - -### Sample 5. Get more from ResourceGroupPreparer -``` Python -class StorageAccountTests(ScenarioTest): - @ResourceGroupPreparer(parameter_name='group_name', parameter_name_for_location='group_location') - def test_create_storage_account(self, group_name, group_location): - self.cmd('az group show -n {}'.format(group_name), checks=[ - JMESPathCheck('name', group_name), - JMESPathCheck('location', group_location), - JMESPathCheck('properties.provisioningState', 'Succeeded') - ]) -``` -Note: - -1. In addition to the name, the location of the resource group can be also injected into the test method. -2. Both parameters' names can be customized. -3. The test method parameter accepting the location value is optional. The test harness will inspect the method signature and decide if the value will be added to the keyworded arguments. - - -### Sample 6. Random name and name mapping -``` Python -class StorageAccountTests(ScenarioTest): - @ResourceGroupPreparer(parameter_name_for_location='location') - def test_create_storage_account(self, resource_group, location): - name = self.create_random_name(prefix='cli', length=24) - self.cmd('az storage account create -n {} -g {} --sku {} -l {}'.format( - name, resource_group, 'Standard_LRS', location)) - self.cmd('az storage account show -n {} -g {}'.format(name, resource_group), checks=[ - JMESPathCheck('name', name), - JMESPathCheck('location', location), - JMESPathCheck('sku.name', 'Standard_LRS'), - JMESPathCheck('kind', 'Storage') - ]) -``` -Note: - -One of the most important features of `ScenarioTest` is name management. For the tests to be able to run in a live environment and avoid name collision a strong name randomization is required. On the other hand, for the tests to be recorded and replay, the naming mechanism must be repeatable during playback mode. The `self.create_random_name` method helps the test achieve the goal. - -The method will create a random name during recording, and when it is called during playback, it returns a name (internally it is called moniker) based on the sequence of the name request. The order won't change once the test is written. Peek into the recording file, you find no random name. For example, note the names like 'clitest.rg000001', they aren't the names of the resources which are actually created in Azure. They're placed before the requests are persisted. -``` Yaml -- request: - body: '{"location": "westus", "tags": {"use": "az-test"}}' - headers: - Accept: [application/json] - Accept-Encoding: ['gzip, deflate'] - CommandName: [group create] - Connection: [keep-alive] - Content-Length: ['50'] - Content-Type: [application/json; charset=utf-8] - User-Agent: [python/3.5.2 (Darwin-16.4.0-x86_64-i386-64bit) requests/2.9.1 msrest/0.4.6 - msrest_azure/0.4.7 resourcemanagementclient/0.30.2 Azure-SDK-For-Python - AZURECLI/2.0.0+dev] - accept-language: [en-US] - method: PUT - uri: https://management.azure.com/subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/clitest.rg000001?api-version=2016-09-01 - response: - body: {string: '{"id":"/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/clitest.rg000001","name":"clitest.rg000001","location":"westus","tags":{"use":"az-test"},"properties":{"provisioningState":"Succeeded"}}'} - headers: - cache-control: [no-cache] - content-length: ['326'] - content-type: [application/json; charset=utf-8] - date: ['Fri, 10 Mar 2017 17:59:58 GMT'] - expires: ['-1'] - pragma: [no-cache] - strict-transport-security: [max-age=31536000; includeSubDomains] - x-ms-ratelimit-remaining-subscription-writes: ['1199'] - status: {code: 201, message: Created} -``` - -In short, for the names of any Azure resources used in the tests, always use `self.create_random_name` to generate its value. Also make sure the correct length is given to the method because different resource have different limitation of the name length. The method will always try to create the longest name possible to fully randomize the name. - - -### Sample 7. Prepare storage account for tests -``` Python -from azure.cli.testsdk import ScenarioTest, ResourceGroupPreparer, StorageAccountPreparer - -class StorageAccountTests(ScenarioTest): - @ResourceGroupPreparer() - @StorageAccountPreparer() - def test_list_storage_accounts(self, storage_account): - accounts = self.cmd('az storage account list').get_output_in_json() - search = [account for account in accounts if account['name'] == storage_account] - assert len(search) == 1 -``` -Note: - -1. Like `ResourceGroupPreparer` you can use `StorageAccountPreparer` to prepare a disposable storage account for the test. The account is deleted along with the resource group. -2. To create a storage account a resource group is required. Therefore `ResourceGroupPrepare` is needed to place above the `StorageAccountPreparer`. The preparers designed to be executed from top to bottom. (The core implementaiton of preparer is in the[AbstractPreparer](https://github.com/Azure/azure-cli/blob/master/src/azure-cli-testsdk/azure/cli/testsdk/preparers.py#L25)) -3. The preparers communicate among them by adding values to the `kwargs` of the decorated methods. Therefore the `StorageAccountPreparer` uses the resource group created in preceding `ResourceGroupPreparer`. -4. The `StorageAccountPreparer` can be further customized: -``` Python -@StorageAccountPreparer(sku='Standard_LRS', location='southcentralus', parameter_name='storage') -``` - -### Sample 8. Prepare multiple storage accounts for tests -``` Python -class StorageAccountTests(ScenarioTest): - @ResourceGroupPreparer() - @StorageAccountPreparer(parameter_name='account_1') - @StorageAccountPreparer(parameter_name='account_2') - def test_list_storage_accounts(self, account_1, account_2): - accounts_list = self.cmd('az storage account list').get_output_in_json() - assert len(accounts_list) >= 2 - assert next(acc for acc in accounts_list if acc['name'] == account_1) - assert next(acc for acc in accounts_list if acc['name'] == account_2) -``` -Note: - -1. Two storage accounts name should be assigned to different function parameters. -2. The resource group name is not required in test so the function doesn't have to declare a parameter to accept the name. However it doesn't mean that the resource group is not created. Its name is in the keyworded parameter dictionary for all the preparer to consume. It is removed before the test function is actually invoked. +The two main users of `ReplayableTest` are +[azure-cli](https://github.com/Azure/azure-cli) +and [azure-sdk-for-python](https://github.com/Azure/azure-sdk-for-python). +Each uses a subclass of `ReplayableTest` to add context-specific functionality +and preserve backward compatibility with test code +prior to the existence of `azure-devtools`. +For example, azure-cli