diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/azure_devops_result_publisher.py b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/azure_devops_result_publisher.py index fe9441887d0..47fc6454009 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/azure_devops_result_publisher.py +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/azure_devops_result_publisher.py @@ -1,18 +1,21 @@ import base64 import os import logging +import time from typing import Iterable, Mapping, List, Dict, Optional, Tuple from builtins import str as text from azure.devops.connection import Connection from msrest.authentication import BasicTokenAuthentication, BasicAuthentication from azure.devops.v5_1.test import TestClient from azure.devops.v5_1.test.models import TestCaseResult, TestAttachmentRequestModel, TestSubResult +from azure.devops.exceptions import AzureDevOpsClientRequestError from helpers import get_env from defs import TestResult log = logging.getLogger(__name__) + class AzureDevOpsTestResultPublisher: def __init__(self, collection_uri, access_token, team_project, test_run_id=None): """ @@ -38,15 +41,15 @@ def is_data_driven_test(self, r: str) -> bool: return r.endswith(")") def get_ddt_base_name(self, r: str) -> str: - return r.split('(',1)[0] + return r.split('(', 1)[0] def send_attachment(self, test_client, attachment, published_result): try: # Python 3 will throw a TypeError exception because b64encode expects bytes - stream=base64.b64encode(text(attachment.text)) + stream = base64.b64encode(text(attachment.text)) except TypeError: # stream has to be a string but b64encode takes and returns bytes on Python 3 - stream=base64.b64encode(bytes(attachment.text, "utf-8")).decode("utf-8") + stream = base64.b64encode(bytes(attachment.text, "utf-8")).decode("utf-8") test_client.create_test_result_attachment( TestAttachmentRequestModel( @@ -55,7 +58,7 @@ def send_attachment(self, test_client, attachment, published_result): ), self.team_project, self.test_run_id, published_result.id) def send_sub_attachment(self, test_client, attachment, published_result, sub_result_id): - stream=base64.b64encode(bytes(attachment.text, "utf-8")).decode("utf-8") + stream = base64.b64encode(bytes(attachment.text, "utf-8")).decode("utf-8") test_client.create_test_sub_result_attachment( TestAttachmentRequestModel( @@ -63,11 +66,43 @@ def send_sub_attachment(self, test_client, attachment, published_result, sub_res stream=stream, ), self.team_project, self.test_run_id, published_result.id, sub_result_id) - def publish_results(self, test_case_results: Iterable[TestCaseResult], test_result_order: Dict[str, List[str]], results_with_attachments: Mapping[str, TestResult]) -> None: + def publish_results(self, test_case_results: Iterable[TestCaseResult], test_result_order: Dict[str, List[str]], + results_with_attachments: Mapping[str, TestResult]) -> None: connection = self.get_connection() test_client = connection.get_client("azure.devops.v5_1.test.TestClient") # type: TestClient - published_results = test_client.add_test_results_to_test_run(list(test_case_results), self.team_project, self.test_run_id) # type: List[TestCaseResult] + tries_left = 10 + succeeded = False + test_run_ended = False + + while tries_left > 0 and not test_run_ended and not succeeded: + try: + published_results = test_client. \ + add_test_results_to_test_run(list(test_case_results), + self.team_project, + self.test_run_id) # type: List[TestCaseResult] + succeeded = True + + except AzureDevOpsClientRequestError as ex: + # Odd syntax here is to deal with checking substrings of the list of args in this exception + hit_503 = len([element for element in ex.args if ('invalid status code of 503' in element)]) != 0 + test_run_ended = len([element for element in ex.args if ('It may have been deleted' in element)]) != 0 + if hit_503: + tries_left -= 1 + log.warning("Hit HTTP 503 from Azure DevOps. Will wait three seconds and try again.") + time.sleep(3) + elif test_run_ended: # Not exceptional, don't retry. + tries_left = 0 + else: + raise ex + + if test_run_ended: + log.info("Test run has ended, skipping attaching results as it would fail.") + return + + # !succeeded means a 503 and not succeeding after 10 tries (otherwise we threw already), so give a nice error + if not succeeded: + raise Exception('Failed to report test results to Azure Dev Ops after retrying. Please contact dnceng.') for published_result in published_results: @@ -89,11 +124,12 @@ def publish_results(self, test_case_results: Iterable[TestCaseResult], test_resu # is true , and uses the order of test names recorded earlier to look-up the attachments. elif published_result.sub_results is not None: sub_results_order = test_result_order[published_result.automated_test_name] - + # Sanity check if len(sub_results_order) != len(published_result.sub_results): - log.warning("Returned subresults list length does not match expected. Attachments may not pair correctly.") - + log.warning( + "Returned subresults list length does not match expected. Attachments may not pair correctly.") + for (name, sub_result) in zip(sub_results_order, published_result.sub_results): if name in results_with_attachments: result = results_with_attachments.get(name) @@ -111,25 +147,25 @@ def convert_to_sub_test(r: TestResult) -> Optional[TestSubResult]: return TestSubResult( comment=comment, display_name=text(r.name), - duration_in_ms=r.duration_seconds*1000, + duration_in_ms=r.duration_seconds * 1000, outcome="Passed" - ) + ) if r.result == "Fail": return TestSubResult( comment=comment, display_name=text(r.name), - duration_in_ms=r.duration_seconds*1000, + duration_in_ms=r.duration_seconds * 1000, outcome="Failed", stack_trace=text(r.stack_trace) if r.stack_trace is not None else None, error_message=text(r.failure_message) - ) + ) if r.result == "Skip": return TestSubResult( comment=comment, display_name=text(r.name), - duration_in_ms=r.duration_seconds*1000, + duration_in_ms=r.duration_seconds * 1000, outcome="NotExecuted" - ) + ) log.warning("Unexpected result value {} for {}".format(r.result, r.name)) return None @@ -141,7 +177,7 @@ def convert_result(r: TestResult) -> Optional[TestCaseResult]: automated_test_type=text(r.kind), automated_test_storage=self.work_item_name, priority=1, - duration_in_ms=r.duration_seconds*1000, + duration_in_ms=r.duration_seconds * 1000, outcome="Passed", state="Completed", comment=comment, @@ -153,7 +189,7 @@ def convert_result(r: TestResult) -> Optional[TestCaseResult]: automated_test_type=text(r.kind), automated_test_storage=self.work_item_name, priority=1, - duration_in_ms=r.duration_seconds*1000, + duration_in_ms=r.duration_seconds * 1000, outcome="Failed", state="Completed", error_message=text(r.failure_message), @@ -168,7 +204,7 @@ def convert_result(r: TestResult) -> Optional[TestCaseResult]: automated_test_type=text(r.kind), automated_test_storage=self.work_item_name, priority=1, - duration_in_ms=r.duration_seconds*1000, + duration_in_ms=r.duration_seconds * 1000, outcome="NotExecuted", state="Completed", error_message=text(r.skip_reason), @@ -178,13 +214,13 @@ def convert_result(r: TestResult) -> Optional[TestCaseResult]: log.warning("Unexpected result value {} for {}".format(r.result, r.name)) return None - unconverted_results = list(results) # type: List[TestResult] + unconverted_results = list(results) # type: List[TestResult] log.debug("Count of unconverted_results: {0}".format(len(unconverted_results))) # Find all DDTs, determine parent, and add to dictionary data_driven_tests = {} # type: Dict[str, TestCaseResult] - non_data_driven_tests = [] # type: List[TestCaseResult] - test_name_ordering = {} # type: Dict[str, List[str]] + non_data_driven_tests = [] # type: List[TestCaseResult] + test_name_ordering = {} # type: Dict[str, List[str]] for r in unconverted_results: if r is None: diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.py b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.py index bde3200f8f4..d596132a795 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.py +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/reporter/run.py @@ -93,7 +93,7 @@ def main(): worker.daemon = True worker.start() - log.info("Beginning reading of test results.") + log.info("Beginning to read test results...") # In case the user puts the results in HELIX_WORKITEM_UPLOAD_ROOT for upload, check there too. all_results = read_results([os.getcwd(),