diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml index c08283d3b7a4..7cc858bc6991 100644 --- a/eng/pipelines/templates/steps/analyze.yml +++ b/eng/pipelines/templates/steps/analyze.yml @@ -115,6 +115,11 @@ steps: TestMarkArgument: ${{ parameters.TestMarkArgument }} AdditionalTestArgs: ${{parameters.AdditionalTestArgs}} + - template: /eng/pipeines/templates/steps/update_snippet.yml + parameters: + ScanPath: $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }} + AdditionalTestArgs: ${{parameters.AdditionalTestArgs}} + - template: ../steps/run_breaking_changes.yml parameters: ServiceDirectory: ${{ parameters.ServiceDirectory }} diff --git a/eng/pipelines/templates/steps/update_snippet.yml b/eng/pipelines/templates/steps/update_snippet.yml new file mode 100644 index 000000000000..057119c59d92 --- /dev/null +++ b/eng/pipelines/templates/steps/update_snippet.yml @@ -0,0 +1,19 @@ +parameters: + ServiceDirectory: '' + ValidateFormatting: false + EnvVars: {} + +steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.9' + inputs: + versionSpec: '3.9' + condition: succeededOrFailed() + + - task: PythonScript@0 + displayName: 'Update Snippets' + inputs: + scriptPath: 'tools/azure-sdk-tools/ci_tools/snippet_update/python_snippet_updater.py' + arguments: >- + ${{ parameters.ScanPath }} + condition: and(succeededOrFailed(), ne(variables['Skip.UpdateSnippet'],'true')) diff --git a/tools/azure-sdk-tools/ci_tools/snippet_update/__init__.py b/tools/azure-sdk-tools/ci_tools/snippet_update/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tools/azure-sdk-tools/ci_tools/snippet_update/python_snippet_updater.py b/tools/azure-sdk-tools/ci_tools/snippet_update/python_snippet_updater.py new file mode 100644 index 000000000000..ccf2d08c0ec8 --- /dev/null +++ b/tools/azure-sdk-tools/ci_tools/snippet_update/python_snippet_updater.py @@ -0,0 +1,125 @@ +import sys +import logging +from pathlib import Path +import argparse +import re +from typing import Dict + +_LOGGER = logging.getLogger(__name__) + +snippets = {} +not_up_to_date = False + +target_snippet_sources = ["samples/*.py", "samples/*/*.py"] +target_md_files = ["README.md"] + +def check_snippets() -> Dict: + return snippets + +def check_not_up_to_date() -> bool: + return not_up_to_date + +def get_snippet(file: str) -> None: + file_obj = Path(file) + with open(file_obj, 'r') as f: + content = f.read() + pattern = "# \\[START(?P[A-Z a-z0-9_]+)\\](?P[\\s\\S]+?)# \\[END[A-Z a-z0-9_]+\\]" + matches = re.findall(pattern, content) + for match in matches: + s = match + name = s[0].strip() + snippet = s[1] + # Remove extra spaces + # A sample code snippet could be like: + # \n + # # [START trio] + # from azure.core.pipeline.transport import TrioRequestsTransport + + # async with AsyncPipeline(TrioRequestsTransport(), policies=policies) as pipeline: + # return await pipeline.run(request) + # # [END trio] + # \n + # On one hand, the spaces in the beginning of the line may vary. e.g. If the snippet + # is in a class, it may have more spaces than if it is not in a class. + # On the other hand, we cannot remove all spaces because indents are part of Python syntax. + # Here is our algorithm: + # We firstly count the spaces of the # [START snippet] line. + # And for every line, we remove this amount of spaces in the beginning of the line. + # To only remove the spaces in the beginning and to make sure we only remove it once per line, + # We use replace('\n' + spaces, '\n'). + spaces = "" + for char in snippet[1:]: + if char == " ": + spaces += char + else: + break + snippet = snippet.replace("\n" + spaces, "\n") + # Remove first newline + snippet = snippet[1:].rstrip() + if snippet[-1] == "\n": + snippet = snippet[:-1] + + file_name = str(file_obj.name)[:-3] + identifier = ".".join([file_name, name]) + if identifier in snippets.keys(): + _LOGGER.warning(f'Found duplicated snippet name "{identifier}".') + _LOGGER.warning(file) + _LOGGER.debug(f"Found: {file_obj.name}.{name}") + snippets[identifier] = snippet + + +def update_snippet(file: str) -> None: + file_obj = Path(file) + with open(file_obj, 'r') as f: + content = f.read() + pattern = "(?P(?P
)\\n```python\\n[\\s\\S]*?\\n)" + matches = re.findall(pattern, content) + for match in matches: + s = match + body = s[0].strip() + header = s[1].strip() + name = s[2].strip() + _LOGGER.debug(f"Found name: {name}") + if name not in snippets.keys(): + _LOGGER.error(f'In {file}, failed to found snippet name "{name}".') + exit(1) + target_code = "".join([header, "\n```python\n", snippets[name], "\n```\n", ""]) + if body != target_code: + _LOGGER.warning(f'Snippet "{name}" is not up to date.') + global not_up_to_date + not_up_to_date = True + content = content.replace(body, target_code) + with open(file_obj, 'w') as f: + f.write(content) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "path", + nargs="?", + help=( + "The targeted path for update." + ), + ) + args = parser.parse_args() + path = sys.argv[1] + _LOGGER.info(f"Path: {path}") + for source in target_snippet_sources: + for py_file in Path(path).rglob(source): + try: + get_snippet(py_file) + except UnicodeDecodeError: + pass + for key in snippets.keys(): + _LOGGER.debug(f"Found snippet: {key}") + for target in target_md_files: + for md_file in Path(path).rglob(target): + try: + update_snippet(md_file) + except UnicodeDecodeError: + pass + if not_up_to_date: + _LOGGER.error(f'Error: code snippets are out of sync. Please run Python PythonSnippetUpdater.py "{path}" to fix it.') + exit(1) + _LOGGER.info(f"README.md under {path} is up to date.") diff --git a/tools/azure-sdk-tools/tests/README.md b/tools/azure-sdk-tools/tests/README.md new file mode 100644 index 000000000000..206abc30b8c2 --- /dev/null +++ b/tools/azure-sdk-tools/tests/README.md @@ -0,0 +1,262 @@ + +# Azure Core shared client library for Python + +Azure core provides shared exceptions and modules for Python SDK client libraries. +These libraries follow the [Azure SDK Design Guidelines for Python](https://azure.github.io/azure-sdk/python/guidelines/index.html) . + +If you are a client library developer, please reference [client library developer reference](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/core/azure-core/CLIENT_LIBRARY_DEVELOPER.md) for more information. + +[Source code](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/core/azure-core/) | [Package (Pypi)][package] | [API reference documentation](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/core/azure-core/) + +## _Disclaimer_ + +_Azure SDK Python packages support for Python 2.7 has ended 01 January 2022. For more information and questions, please refer to _ + +## Getting started + +Typically, you will not need to install azure core; +it will be installed when you install one of the client libraries using it. +In case you want to install it explicitly (to implement your own client library, for example), +you can find it [here](https://pypi.org/project/azure-core/). + +## Key concepts + +### Azure Core Library Exceptions + +#### AzureError + +AzureError is the base exception for all errors. + +```python +class AzureError(Exception): + def __init__(self, message, *args, **kwargs): + self.inner_exception = kwargs.get("error") + self.exc_type, self.exc_value, self.exc_traceback = sys.exc_info() + self.exc_type = self.exc_type.__name__ if self.exc_type else type(self.inner_exception) + self.exc_msg = "{}, {}: {}".format(message, self.exc_type, self.exc_value) # type: ignore + self.message = str(message) + self.continuation_token = kwargs.get("continuation_token") + super(AzureError, self).__init__(self.message, *args) +``` + +*message* is any message (str) to be associated with the exception. + +*args* are any additional args to be included with exception. + +*kwargs* are keyword arguments to include with the exception. Use the keyword *error* to pass in an internal exception and *continuation_token* for a token reference to continue an incomplete operation. + +**The following exceptions inherit from AzureError:** + +#### ServiceRequestError + +An error occurred while attempt to make a request to the service. No request was sent. + +#### ServiceResponseError + +The request was sent, but the client failed to understand the response. +The connection may have timed out. These errors can be retried for idempotent or safe operations. + +#### HttpResponseError + +A request was made, and a non-success status code was received from the service. + +```python +class HttpResponseError(AzureError): + def __init__(self, message=None, response=None, **kwargs): + self.reason = None + self.response = response + if response: + self.reason = response.reason + self.status_code = response.status_code + self.error = self._parse_odata_body(ODataV4Format, response) # type: Optional[ODataV4Format] + if self.error: + message = str(self.error) + else: + message = message or "Operation returned an invalid status '{}'".format( + self.reason + ) + + super(HttpResponseError, self).__init__(message=message, **kwargs) +``` + +*message* is the HTTP response error message (optional) + +*response* is the HTTP response (optional). + +*kwargs* are keyword arguments to include with the exception. + +**The following exceptions inherit from HttpResponseError:** + +#### DecodeError + +An error raised during response de-serialization. + +#### IncompleteReadError + +An error raised if peer closes the connection before we have received the complete message body. + +#### ResourceExistsError + +An error response with status code 4xx. This will not be raised directly by the Azure core pipeline. + +#### ResourceNotFoundError + +An error response, typically triggered by a 412 response (for update) or 404 (for get/post). + +#### ResourceModifiedError + +An error response with status code 4xx, typically 412 Conflict. This will not be raised directly by the Azure core pipeline. + +#### ResourceNotModifiedError + +An error response with status code 304. This will not be raised directly by the Azure core pipeline. + +#### ClientAuthenticationError + +An error response with status code 4xx. This will not be raised directly by the Azure core pipeline. + +#### TooManyRedirectsError + +An error raised when the maximum number of redirect attempts is reached. The maximum amount of redirects can be configured in the RedirectPolicy. + +```python +class TooManyRedirectsError(HttpResponseError): + def __init__(self, history, *args, **kwargs): + self.history = history + message = "Reached maximum redirect attempts." + super(TooManyRedirectsError, self).__init__(message, *args, **kwargs) +``` + +*history* is used to document the requests/responses that resulted in redirected requests. + +*args* are any additional args to be included with exception. + +*kwargs* are keyword arguments to include with the exception. + +#### StreamConsumedError + +An error thrown if you try to access the stream of `azure.core.rest.HttpResponse` or `azure.core.rest.AsyncHttpResponse` once +the response stream has been consumed. + + +```python +from azure.core.pipeline.transport import TrioRequestsTransport +async with AsyncPipeline(TrioRequestsTransport(), policies=policies) as pipeline: + return await pipeline.run(request) +``` + + +#### StreamClosedError + +An error thrown if you try to access the stream of the `azure.core.rest.HttpResponse` or `azure.core.rest.AsyncHttpResponse` once +the response stream has been closed. + +#### ResponseNotReadError + +An error thrown if you try to access the `content` of `azure.core.rest.HttpResponse` or `azure.core.rest.AsyncHttpResponse` before +reading in the response's bytes first. + +### Configurations + +When calling the methods, some properties can be configured by passing in as kwargs arguments. + +| Parameters | Description | +| --- | --- | +| headers | The HTTP Request headers. | +| request_id | The request id to be added into header. | +| user_agent | If specified, this will be added in front of the user agent string. | +| logging_enable| Use to enable per operation. Defaults to `False`. | +| logger | If specified, it will be used to log information. | +| response_encoding | The encoding to use if known for this service (will disable auto-detection). | +| proxies | Maps protocol or protocol and hostname to the URL of the proxy. | +| raw_request_hook | Callback function. Will be invoked on request. | +| raw_response_hook | Callback function. Will be invoked on response. | +| network_span_namer | A callable to customize the span name. | +| tracing_attributes | Attributes to set on all created spans. | +| permit_redirects | Whether the client allows redirects. Defaults to `True`. | +| redirect_max | The maximum allowed redirects. Defaults to `30`. | +| retry_total | Total number of retries to allow. Takes precedence over other counts. Default value is `10`. | +| retry_connect | How many connection-related errors to retry on. These are errors raised before the request is sent to the remote server, which we assume has not triggered the server to process the request. Default value is `3`. | +| retry_read | How many times to retry on read errors. These errors are raised after the request was sent to the server, so the request may have side-effects. Default value is `3`. | +| retry_status | How many times to retry on bad status codes. Default value is `3`. | +| retry_backoff_factor | A backoff factor to apply between attempts after the second try (most errors are resolved immediately by a second try without a delay). Retry policy will sleep for: `{backoff factor} * (2 ** ({number of total retries} - 1))` seconds. If the backoff_factor is 0.1, then the retry will sleep for [0.0s, 0.2s, 0.4s, ...] between retries. The default value is `0.8`. | +| retry_backoff_max | The maximum back off time. Default value is `120` seconds (2 minutes). | +| retry_mode | Fixed or exponential delay between attempts, default is `Exponential`. | +| timeout | Timeout setting for the operation in seconds, default is `604800`s (7 days). | +| connection_timeout | A single float in seconds for the connection timeout. Defaults to `300` seconds. | +| read_timeout | A single float in seconds for the read timeout. Defaults to `300` seconds. | +| connection_verify | SSL certificate verification. Enabled by default. Set to False to disable, alternatively can be set to the path to a CA_BUNDLE file or directory with certificates of trusted CAs. | +| connection_cert | Client-side certificates. You can specify a local cert to use as client side certificate, as a single file (containing the private key and the certificate) or as a tuple of both files' paths. | +| proxies | Dictionary mapping protocol or protocol and hostname to the URL of the proxy. | +| cookies | Dict or CookieJar object to send with the `Request`. | +| connection_data_block_size | The block size of data sent over the connection. Defaults to `4096` bytes. | + +### Async transport + +The async transport is designed to be opt-in. [AioHttp](https://pypi.org/project/aiohttp/) is one of the supported implementations of async transport. It is not installed by default. You need to install it separately. + +### Shared modules + +#### MatchConditions + +MatchConditions is an enum to describe match conditions. + +```python +class MatchConditions(Enum): + Unconditionally = 1 + IfNotModified = 2 + IfModified = 3 + IfPresent = 4 + IfMissing = 5 +``` + +#### CaseInsensitiveEnumMeta + +A metaclass to support case-insensitive enums. + +```python +from enum import Enum + +from azure.core import CaseInsensitiveEnumMeta + +class MyCustomEnum(str, Enum, metaclass=CaseInsensitiveEnumMeta): + FOO = 'foo' + BAR = 'bar' +``` + +#### Null Sentinel Value + +A falsy sentinel object which is supposed to be used to specify attributes +with no data. This gets serialized to `null` on the wire. + +```python +from azure.core.serialization import NULL + +assert bool(NULL) is False + +foo = Foo( + attr=NULL +) +``` + +## Contributing + +This project welcomes contributions and suggestions. Most contributions require +you to agree to a Contributor License Agreement (CLA) declaring that you have +the right to, and actually do, grant us the rights to use your contribution. +For details, visit [https://cla.microsoft.com](https://cla.microsoft.com). + +When you submit a pull request, a CLA-bot will automatically determine whether +you need to provide a CLA and decorate the PR appropriately (e.g., label, +comment). Simply follow the instructions provided by the bot. You will only +need to do this once across all repos using our CLA. + +This project has adopted the +[Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). +For more information, see the +[Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) +or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any +additional questions or comments. + + +[package]: https://pypi.org/project/azure-core/ diff --git a/tools/azure-sdk-tools/tests/README_missing_snippet.md b/tools/azure-sdk-tools/tests/README_missing_snippet.md new file mode 100644 index 000000000000..5f83426636bf --- /dev/null +++ b/tools/azure-sdk-tools/tests/README_missing_snippet.md @@ -0,0 +1,263 @@ + +# Azure Core shared client library for Python + +Azure core provides shared exceptions and modules for Python SDK client libraries. +These libraries follow the [Azure SDK Design Guidelines for Python](https://azure.github.io/azure-sdk/python/guidelines/index.html) . + +If you are a client library developer, please reference [client library developer reference](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/core/azure-core/CLIENT_LIBRARY_DEVELOPER.md) for more information. + +[Source code](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/core/azure-core/) | [Package (Pypi)][package] | [API reference documentation](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/core/azure-core/) + +## _Disclaimer_ + +_Azure SDK Python packages support for Python 2.7 has ended 01 January 2022. For more information and questions, please refer to _ + +## Getting started + +Typically, you will not need to install azure core; +it will be installed when you install one of the client libraries using it. +In case you want to install it explicitly (to implement your own client library, for example), +you can find it [here](https://pypi.org/project/azure-core/). + +## Key concepts + +### Azure Core Library Exceptions + +#### AzureError + +AzureError is the base exception for all errors. + +```python +class AzureError(Exception): + def __init__(self, message, *args, **kwargs): + self.inner_exception = kwargs.get("error") + self.exc_type, self.exc_value, self.exc_traceback = sys.exc_info() + self.exc_type = self.exc_type.__name__ if self.exc_type else type(self.inner_exception) + self.exc_msg = "{}, {}: {}".format(message, self.exc_type, self.exc_value) # type: ignore + self.message = str(message) + self.continuation_token = kwargs.get("continuation_token") + super(AzureError, self).__init__(self.message, *args) +``` + +*message* is any message (str) to be associated with the exception. + +*args* are any additional args to be included with exception. + +*kwargs* are keyword arguments to include with the exception. Use the keyword *error* to pass in an internal exception and *continuation_token* for a token reference to continue an incomplete operation. + +**The following exceptions inherit from AzureError:** + +#### ServiceRequestError + +An error occurred while attempt to make a request to the service. No request was sent. + +#### ServiceResponseError + +The request was sent, but the client failed to understand the response. +The connection may have timed out. These errors can be retried for idempotent or safe operations. + +#### HttpResponseError + +A request was made, and a non-success status code was received from the service. + +```python +class HttpResponseError(AzureError): + def __init__(self, message=None, response=None, **kwargs): + self.reason = None + self.response = response + if response: + self.reason = response.reason + self.status_code = response.status_code + self.error = self._parse_odata_body(ODataV4Format, response) # type: Optional[ODataV4Format] + if self.error: + message = str(self.error) + else: + message = message or "Operation returned an invalid status '{}'".format( + self.reason + ) + + super(HttpResponseError, self).__init__(message=message, **kwargs) +``` + +*message* is the HTTP response error message (optional) + +*response* is the HTTP response (optional). + +*kwargs* are keyword arguments to include with the exception. + +**The following exceptions inherit from HttpResponseError:** + +#### DecodeError + +An error raised during response de-serialization. + +#### IncompleteReadError + +An error raised if peer closes the connection before we have received the complete message body. + +#### ResourceExistsError + +An error response with status code 4xx. This will not be raised directly by the Azure core pipeline. + +#### ResourceNotFoundError + +An error response, typically triggered by a 412 response (for update) or 404 (for get/post). + +#### ResourceModifiedError + +An error response with status code 4xx, typically 412 Conflict. This will not be raised directly by the Azure core pipeline. + +#### ResourceNotModifiedError + +An error response with status code 304. This will not be raised directly by the Azure core pipeline. + +#### ClientAuthenticationError + +An error response with status code 4xx. This will not be raised directly by the Azure core pipeline. + +#### TooManyRedirectsError + +An error raised when the maximum number of redirect attempts is reached. The maximum amount of redirects can be configured in the RedirectPolicy. + +```python +class TooManyRedirectsError(HttpResponseError): + def __init__(self, history, *args, **kwargs): + self.history = history + message = "Reached maximum redirect attempts." + super(TooManyRedirectsError, self).__init__(message, *args, **kwargs) +``` + +*history* is used to document the requests/responses that resulted in redirected requests. + +*args* are any additional args to be included with exception. + +*kwargs* are keyword arguments to include with the exception. + +#### StreamConsumedError + +An error thrown if you try to access the stream of `azure.core.rest.HttpResponse` or `azure.core.rest.AsyncHttpResponse` once +the response stream has been consumed. + + +```python +from azure.core.pipeline.transport import TrioRequestsTransport + +async with AsyncPipeline(TrioRequestsTransport(), policies=policies) as pipeline: + return await pipeline.run(request) +``` + + +#### StreamClosedError + +An error thrown if you try to access the stream of the `azure.core.rest.HttpResponse` or `azure.core.rest.AsyncHttpResponse` once +the response stream has been closed. + +#### ResponseNotReadError + +An error thrown if you try to access the `content` of `azure.core.rest.HttpResponse` or `azure.core.rest.AsyncHttpResponse` before +reading in the response's bytes first. + +### Configurations + +When calling the methods, some properties can be configured by passing in as kwargs arguments. + +| Parameters | Description | +| --- | --- | +| headers | The HTTP Request headers. | +| request_id | The request id to be added into header. | +| user_agent | If specified, this will be added in front of the user agent string. | +| logging_enable| Use to enable per operation. Defaults to `False`. | +| logger | If specified, it will be used to log information. | +| response_encoding | The encoding to use if known for this service (will disable auto-detection). | +| proxies | Maps protocol or protocol and hostname to the URL of the proxy. | +| raw_request_hook | Callback function. Will be invoked on request. | +| raw_response_hook | Callback function. Will be invoked on response. | +| network_span_namer | A callable to customize the span name. | +| tracing_attributes | Attributes to set on all created spans. | +| permit_redirects | Whether the client allows redirects. Defaults to `True`. | +| redirect_max | The maximum allowed redirects. Defaults to `30`. | +| retry_total | Total number of retries to allow. Takes precedence over other counts. Default value is `10`. | +| retry_connect | How many connection-related errors to retry on. These are errors raised before the request is sent to the remote server, which we assume has not triggered the server to process the request. Default value is `3`. | +| retry_read | How many times to retry on read errors. These errors are raised after the request was sent to the server, so the request may have side-effects. Default value is `3`. | +| retry_status | How many times to retry on bad status codes. Default value is `3`. | +| retry_backoff_factor | A backoff factor to apply between attempts after the second try (most errors are resolved immediately by a second try without a delay). Retry policy will sleep for: `{backoff factor} * (2 ** ({number of total retries} - 1))` seconds. If the backoff_factor is 0.1, then the retry will sleep for [0.0s, 0.2s, 0.4s, ...] between retries. The default value is `0.8`. | +| retry_backoff_max | The maximum back off time. Default value is `120` seconds (2 minutes). | +| retry_mode | Fixed or exponential delay between attempts, default is `Exponential`. | +| timeout | Timeout setting for the operation in seconds, default is `604800`s (7 days). | +| connection_timeout | A single float in seconds for the connection timeout. Defaults to `300` seconds. | +| read_timeout | A single float in seconds for the read timeout. Defaults to `300` seconds. | +| connection_verify | SSL certificate verification. Enabled by default. Set to False to disable, alternatively can be set to the path to a CA_BUNDLE file or directory with certificates of trusted CAs. | +| connection_cert | Client-side certificates. You can specify a local cert to use as client side certificate, as a single file (containing the private key and the certificate) or as a tuple of both files' paths. | +| proxies | Dictionary mapping protocol or protocol and hostname to the URL of the proxy. | +| cookies | Dict or CookieJar object to send with the `Request`. | +| connection_data_block_size | The block size of data sent over the connection. Defaults to `4096` bytes. | + +### Async transport + +The async transport is designed to be opt-in. [AioHttp](https://pypi.org/project/aiohttp/) is one of the supported implementations of async transport. It is not installed by default. You need to install it separately. + +### Shared modules + +#### MatchConditions + +MatchConditions is an enum to describe match conditions. + +```python +class MatchConditions(Enum): + Unconditionally = 1 + IfNotModified = 2 + IfModified = 3 + IfPresent = 4 + IfMissing = 5 +``` + +#### CaseInsensitiveEnumMeta + +A metaclass to support case-insensitive enums. + +```python +from enum import Enum + +from azure.core import CaseInsensitiveEnumMeta + +class MyCustomEnum(str, Enum, metaclass=CaseInsensitiveEnumMeta): + FOO = 'foo' + BAR = 'bar' +``` + +#### Null Sentinel Value + +A falsy sentinel object which is supposed to be used to specify attributes +with no data. This gets serialized to `null` on the wire. + +```python +from azure.core.serialization import NULL + +assert bool(NULL) is False + +foo = Foo( + attr=NULL +) +``` + +## Contributing + +This project welcomes contributions and suggestions. Most contributions require +you to agree to a Contributor License Agreement (CLA) declaring that you have +the right to, and actually do, grant us the rights to use your contribution. +For details, visit [https://cla.microsoft.com](https://cla.microsoft.com). + +When you submit a pull request, a CLA-bot will automatically determine whether +you need to provide a CLA and decorate the PR appropriately (e.g., label, +comment). Simply follow the instructions provided by the bot. You will only +need to do this once across all repos using our CLA. + +This project has adopted the +[Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). +For more information, see the +[Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) +or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any +additional questions or comments. + + +[package]: https://pypi.org/project/azure-core/ diff --git a/tools/azure-sdk-tools/tests/README_out_of_sync.md b/tools/azure-sdk-tools/tests/README_out_of_sync.md new file mode 100644 index 000000000000..206abc30b8c2 --- /dev/null +++ b/tools/azure-sdk-tools/tests/README_out_of_sync.md @@ -0,0 +1,262 @@ + +# Azure Core shared client library for Python + +Azure core provides shared exceptions and modules for Python SDK client libraries. +These libraries follow the [Azure SDK Design Guidelines for Python](https://azure.github.io/azure-sdk/python/guidelines/index.html) . + +If you are a client library developer, please reference [client library developer reference](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/core/azure-core/CLIENT_LIBRARY_DEVELOPER.md) for more information. + +[Source code](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/core/azure-core/) | [Package (Pypi)][package] | [API reference documentation](https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/core/azure-core/) + +## _Disclaimer_ + +_Azure SDK Python packages support for Python 2.7 has ended 01 January 2022. For more information and questions, please refer to _ + +## Getting started + +Typically, you will not need to install azure core; +it will be installed when you install one of the client libraries using it. +In case you want to install it explicitly (to implement your own client library, for example), +you can find it [here](https://pypi.org/project/azure-core/). + +## Key concepts + +### Azure Core Library Exceptions + +#### AzureError + +AzureError is the base exception for all errors. + +```python +class AzureError(Exception): + def __init__(self, message, *args, **kwargs): + self.inner_exception = kwargs.get("error") + self.exc_type, self.exc_value, self.exc_traceback = sys.exc_info() + self.exc_type = self.exc_type.__name__ if self.exc_type else type(self.inner_exception) + self.exc_msg = "{}, {}: {}".format(message, self.exc_type, self.exc_value) # type: ignore + self.message = str(message) + self.continuation_token = kwargs.get("continuation_token") + super(AzureError, self).__init__(self.message, *args) +``` + +*message* is any message (str) to be associated with the exception. + +*args* are any additional args to be included with exception. + +*kwargs* are keyword arguments to include with the exception. Use the keyword *error* to pass in an internal exception and *continuation_token* for a token reference to continue an incomplete operation. + +**The following exceptions inherit from AzureError:** + +#### ServiceRequestError + +An error occurred while attempt to make a request to the service. No request was sent. + +#### ServiceResponseError + +The request was sent, but the client failed to understand the response. +The connection may have timed out. These errors can be retried for idempotent or safe operations. + +#### HttpResponseError + +A request was made, and a non-success status code was received from the service. + +```python +class HttpResponseError(AzureError): + def __init__(self, message=None, response=None, **kwargs): + self.reason = None + self.response = response + if response: + self.reason = response.reason + self.status_code = response.status_code + self.error = self._parse_odata_body(ODataV4Format, response) # type: Optional[ODataV4Format] + if self.error: + message = str(self.error) + else: + message = message or "Operation returned an invalid status '{}'".format( + self.reason + ) + + super(HttpResponseError, self).__init__(message=message, **kwargs) +``` + +*message* is the HTTP response error message (optional) + +*response* is the HTTP response (optional). + +*kwargs* are keyword arguments to include with the exception. + +**The following exceptions inherit from HttpResponseError:** + +#### DecodeError + +An error raised during response de-serialization. + +#### IncompleteReadError + +An error raised if peer closes the connection before we have received the complete message body. + +#### ResourceExistsError + +An error response with status code 4xx. This will not be raised directly by the Azure core pipeline. + +#### ResourceNotFoundError + +An error response, typically triggered by a 412 response (for update) or 404 (for get/post). + +#### ResourceModifiedError + +An error response with status code 4xx, typically 412 Conflict. This will not be raised directly by the Azure core pipeline. + +#### ResourceNotModifiedError + +An error response with status code 304. This will not be raised directly by the Azure core pipeline. + +#### ClientAuthenticationError + +An error response with status code 4xx. This will not be raised directly by the Azure core pipeline. + +#### TooManyRedirectsError + +An error raised when the maximum number of redirect attempts is reached. The maximum amount of redirects can be configured in the RedirectPolicy. + +```python +class TooManyRedirectsError(HttpResponseError): + def __init__(self, history, *args, **kwargs): + self.history = history + message = "Reached maximum redirect attempts." + super(TooManyRedirectsError, self).__init__(message, *args, **kwargs) +``` + +*history* is used to document the requests/responses that resulted in redirected requests. + +*args* are any additional args to be included with exception. + +*kwargs* are keyword arguments to include with the exception. + +#### StreamConsumedError + +An error thrown if you try to access the stream of `azure.core.rest.HttpResponse` or `azure.core.rest.AsyncHttpResponse` once +the response stream has been consumed. + + +```python +from azure.core.pipeline.transport import TrioRequestsTransport +async with AsyncPipeline(TrioRequestsTransport(), policies=policies) as pipeline: + return await pipeline.run(request) +``` + + +#### StreamClosedError + +An error thrown if you try to access the stream of the `azure.core.rest.HttpResponse` or `azure.core.rest.AsyncHttpResponse` once +the response stream has been closed. + +#### ResponseNotReadError + +An error thrown if you try to access the `content` of `azure.core.rest.HttpResponse` or `azure.core.rest.AsyncHttpResponse` before +reading in the response's bytes first. + +### Configurations + +When calling the methods, some properties can be configured by passing in as kwargs arguments. + +| Parameters | Description | +| --- | --- | +| headers | The HTTP Request headers. | +| request_id | The request id to be added into header. | +| user_agent | If specified, this will be added in front of the user agent string. | +| logging_enable| Use to enable per operation. Defaults to `False`. | +| logger | If specified, it will be used to log information. | +| response_encoding | The encoding to use if known for this service (will disable auto-detection). | +| proxies | Maps protocol or protocol and hostname to the URL of the proxy. | +| raw_request_hook | Callback function. Will be invoked on request. | +| raw_response_hook | Callback function. Will be invoked on response. | +| network_span_namer | A callable to customize the span name. | +| tracing_attributes | Attributes to set on all created spans. | +| permit_redirects | Whether the client allows redirects. Defaults to `True`. | +| redirect_max | The maximum allowed redirects. Defaults to `30`. | +| retry_total | Total number of retries to allow. Takes precedence over other counts. Default value is `10`. | +| retry_connect | How many connection-related errors to retry on. These are errors raised before the request is sent to the remote server, which we assume has not triggered the server to process the request. Default value is `3`. | +| retry_read | How many times to retry on read errors. These errors are raised after the request was sent to the server, so the request may have side-effects. Default value is `3`. | +| retry_status | How many times to retry on bad status codes. Default value is `3`. | +| retry_backoff_factor | A backoff factor to apply between attempts after the second try (most errors are resolved immediately by a second try without a delay). Retry policy will sleep for: `{backoff factor} * (2 ** ({number of total retries} - 1))` seconds. If the backoff_factor is 0.1, then the retry will sleep for [0.0s, 0.2s, 0.4s, ...] between retries. The default value is `0.8`. | +| retry_backoff_max | The maximum back off time. Default value is `120` seconds (2 minutes). | +| retry_mode | Fixed or exponential delay between attempts, default is `Exponential`. | +| timeout | Timeout setting for the operation in seconds, default is `604800`s (7 days). | +| connection_timeout | A single float in seconds for the connection timeout. Defaults to `300` seconds. | +| read_timeout | A single float in seconds for the read timeout. Defaults to `300` seconds. | +| connection_verify | SSL certificate verification. Enabled by default. Set to False to disable, alternatively can be set to the path to a CA_BUNDLE file or directory with certificates of trusted CAs. | +| connection_cert | Client-side certificates. You can specify a local cert to use as client side certificate, as a single file (containing the private key and the certificate) or as a tuple of both files' paths. | +| proxies | Dictionary mapping protocol or protocol and hostname to the URL of the proxy. | +| cookies | Dict or CookieJar object to send with the `Request`. | +| connection_data_block_size | The block size of data sent over the connection. Defaults to `4096` bytes. | + +### Async transport + +The async transport is designed to be opt-in. [AioHttp](https://pypi.org/project/aiohttp/) is one of the supported implementations of async transport. It is not installed by default. You need to install it separately. + +### Shared modules + +#### MatchConditions + +MatchConditions is an enum to describe match conditions. + +```python +class MatchConditions(Enum): + Unconditionally = 1 + IfNotModified = 2 + IfModified = 3 + IfPresent = 4 + IfMissing = 5 +``` + +#### CaseInsensitiveEnumMeta + +A metaclass to support case-insensitive enums. + +```python +from enum import Enum + +from azure.core import CaseInsensitiveEnumMeta + +class MyCustomEnum(str, Enum, metaclass=CaseInsensitiveEnumMeta): + FOO = 'foo' + BAR = 'bar' +``` + +#### Null Sentinel Value + +A falsy sentinel object which is supposed to be used to specify attributes +with no data. This gets serialized to `null` on the wire. + +```python +from azure.core.serialization import NULL + +assert bool(NULL) is False + +foo = Foo( + attr=NULL +) +``` + +## Contributing + +This project welcomes contributions and suggestions. Most contributions require +you to agree to a Contributor License Agreement (CLA) declaring that you have +the right to, and actually do, grant us the rights to use your contribution. +For details, visit [https://cla.microsoft.com](https://cla.microsoft.com). + +When you submit a pull request, a CLA-bot will automatically determine whether +you need to provide a CLA and decorate the PR appropriately (e.g., label, +comment). Simply follow the instructions provided by the bot. You will only +need to do this once across all repos using our CLA. + +This project has adopted the +[Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). +For more information, see the +[Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) +or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any +additional questions or comments. + + +[package]: https://pypi.org/project/azure-core/ diff --git a/tools/azure-sdk-tools/tests/example_async.py b/tools/azure-sdk-tools/tests/example_async.py new file mode 100644 index 000000000000..7f439e273ac9 --- /dev/null +++ b/tools/azure-sdk-tools/tests/example_async.py @@ -0,0 +1,229 @@ +# -------------------------------------------------------------------------- +# +# Copyright (c) Microsoft Corporation. All rights reserved. +# +# The MIT License (MIT) +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the ""Software""), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +# +# -------------------------------------------------------------------------- + +import pytest +from azure.core.pipeline import AsyncPipeline +from azure.core import AsyncPipelineClient +from azure.core.pipeline.policies import UserAgentPolicy, AsyncRedirectPolicy +from azure.core.pipeline.transport import HttpRequest + +import trio + + +@pytest.mark.asyncio +async def test_example_trio(): + + async def req(): + request = HttpRequest("GET", "https://bing.com/") + policies = [ + UserAgentPolicy("myuseragent"), + AsyncRedirectPolicy() + ] + # [START trio] + from azure.core.pipeline.transport import TrioRequestsTransport + + async with AsyncPipeline(TrioRequestsTransport(), policies=policies) as pipeline: + return await pipeline.run(request) + # [END trio] + response = trio.run(req) + assert isinstance(response.http_response.status_code, int) + + +@pytest.mark.asyncio +async def test_example_asyncio(): + + request = HttpRequest("GET", "https://bing.com") + policies = [ + UserAgentPolicy("myuseragent"), + AsyncRedirectPolicy() + ] + # [START asyncio] + from azure.core.pipeline.transport import AsyncioRequestsTransport + + async with AsyncPipeline(AsyncioRequestsTransport(), policies=policies) as pipeline: + response = await pipeline.run(request) + # [END asyncio] + assert pipeline._transport.session is None + assert isinstance(response.http_response.status_code, int) + + +@pytest.mark.asyncio +async def test_example_aiohttp(): + + request = HttpRequest("GET", "https://bing.com") + policies = [ + UserAgentPolicy("myuseragent"), + AsyncRedirectPolicy() + ] + # [START aiohttp] + from azure.core.pipeline.transport import AioHttpTransport + + async with AsyncPipeline(AioHttpTransport(), policies=policies) as pipeline: + response = await pipeline.run(request) + # [END aiohttp] + assert pipeline._transport.session is None + assert isinstance(response.http_response.status_code, int) + + +@pytest.mark.asyncio +async def test_example_async_pipeline(): + # [START build_async_pipeline] + from azure.core.pipeline import AsyncPipeline + from azure.core.pipeline.policies import AsyncRedirectPolicy, UserAgentPolicy + from azure.core.pipeline.transport import AioHttpTransport, HttpRequest + + # example: create request and policies + request = HttpRequest("GET", "https://bing.com") + policies = [ + UserAgentPolicy("myuseragent"), + AsyncRedirectPolicy() + ] + + # run the pipeline + async with AsyncPipeline(transport=AioHttpTransport(), policies=policies) as pipeline: + response = await pipeline.run(request) + # [END build_async_pipeline] + assert pipeline._transport.session is None + assert isinstance(response.http_response.status_code, int) + + +@pytest.mark.asyncio +async def test_example_async_pipeline_client(): + + url = "https://bing.com" + + # [START build_async_pipeline_client] + from azure.core import AsyncPipelineClient + from azure.core.pipeline.policies import AsyncRedirectPolicy, UserAgentPolicy + from azure.core.pipeline.transport import HttpRequest + + # example policies + request = HttpRequest("GET", url) + policies = [ + UserAgentPolicy("myuseragent"), + AsyncRedirectPolicy(), + ] + + async with AsyncPipelineClient(base_url=url, policies=policies) as client: + response = await client._pipeline.run(request) + # [END build_async_pipeline_client] + + assert client._pipeline._transport.session is None + assert isinstance(response.http_response.status_code, int) + + +@pytest.mark.asyncio +async def test_example_async_redirect_policy(): + url = "https://bing.com" + request = HttpRequest("GET", url) + + # [START async_redirect_policy] + from azure.core.pipeline.policies import AsyncRedirectPolicy + + redirect_policy = AsyncRedirectPolicy() + + # Client allows redirects. Defaults to True. + redirect_policy.allow = True + + # The maximum allowed redirects. The default value is 30 + redirect_policy.max_redirects = 10 + + # Alternatively you can disable redirects entirely + redirect_policy = AsyncRedirectPolicy.no_redirects() + + # It can also be overridden per operation. + async with AsyncPipelineClient(base_url=url, policies=[redirect_policy]) as client: + response = await client._pipeline.run(request, permit_redirects=True, redirect_max=5) + + # [END async_redirect_policy] + + assert client._pipeline._transport.session is None + assert isinstance(response.http_response.status_code, int) + + +@pytest.mark.asyncio +async def test_example_async_retry_policy(): + url = "https://bing.com" + request = HttpRequest("GET", "https://bing.com") + policies = [ + UserAgentPolicy("myuseragent"), + AsyncRedirectPolicy(), + ] + + # [START async_retry_policy] + from azure.core.pipeline.policies import AsyncRetryPolicy + + retry_policy = AsyncRetryPolicy() + + # Total number of retries to allow. Takes precedence over other counts. + # Default value is 10. + retry_policy.total_retries = 5 + + # How many connection-related errors to retry on. + # These are errors raised before the request is sent to the remote server, + # which we assume has not triggered the server to process the request. Default value is 3 + retry_policy.connect_retries = 2 + + # How many times to retry on read errors. + # These errors are raised after the request was sent to the server, so the + # request may have side-effects. Default value is 3. + retry_policy.read_retries = 4 + + # How many times to retry on bad status codes. Default value is 3. + retry_policy.status_retries = 3 + + # A backoff factor to apply between attempts after the second try + # (most errors are resolved immediately by a second try without a delay). + # Retry policy will sleep for: + # {backoff factor} * (2 ** ({number of total retries} - 1)) + # seconds. If the backoff_factor is 0.1, then the retry will sleep + # for [0.0s, 0.2s, 0.4s, ...] between retries. + # The default value is 0.8. + retry_policy.backoff_factor = 0.5 + + # The maximum back off time. Default value is 120 seconds (2 minutes). + retry_policy.backoff_max = 120 + + # Alternatively you can disable redirects entirely + retry_policy = AsyncRetryPolicy.no_retries() + + # All of these settings can also be configured per operation. + policies.append(retry_policy) + async with AsyncPipelineClient(base_url=url, policies=policies) as client: + response = await client._pipeline.run( + request, + retry_total=10, + retry_connect=1, + retry_read=1, + retry_status=5, + retry_backoff_factor=0.5, + retry_backoff_max=60, + retry_on_methods=['GET'] + ) + # [END async_retry_policy] + + assert client._pipeline._transport.session is None + assert isinstance(response.http_response.status_code, int) diff --git a/tools/azure-sdk-tools/tests/test_python_snippet_updater.py b/tools/azure-sdk-tools/tests/test_python_snippet_updater.py new file mode 100644 index 000000000000..c386fad0aa9b --- /dev/null +++ b/tools/azure-sdk-tools/tests/test_python_snippet_updater.py @@ -0,0 +1,38 @@ +import os +import sys +import pytest +from ci_tools.snippet_update.python_snippet_updater import get_snippet, update_snippet, check_snippets, check_not_up_to_date + + +def test_get_snippet(): + folder = os.path.dirname(os.path.abspath(__file__)) + file = os.path.join(folder, "example_async.py") + get_snippet(file) + snippets = check_snippets().keys() + assert len(snippets) == 7 + assert 'example_async.trio' in snippets + assert 'example_async.async_retry_policy' in snippets + +def test_update_snippet(): + folder = os.path.dirname(os.path.abspath(__file__)) + file = os.path.join(folder, "example_async.py") + get_snippet(file) + file_1 = os.path.join(folder, "README.md") + update_snippet(file_1) + +def test_missing_snippet(): + folder = os.path.dirname(os.path.abspath(__file__)) + file = os.path.join(folder, "example_async.py") + get_snippet(file) + file_1 = os.path.join(folder, "README_missing_snippet.md") + with pytest.raises(SystemExit): + update_snippet(file_1) + +def test_out_of_sync(): + folder = os.path.dirname(os.path.abspath(__file__)) + file = os.path.join(folder, "example_async.py") + get_snippet(file) + file_1 = os.path.join(folder, "README_out_of_sync.md") + update_snippet(file_1) + not_up_to_date = check_not_up_to_date() + assert not_up_to_date