Skip to content

Windows: Enable flaky test result processing #14102

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lizan merged 2 commits into from
Dec 2, 2020
Merged

Windows: Enable flaky test result processing #14102

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
430f492
Windows: Enable flaky test result processing
sunjayBhatia Nov 18, 2020
2753755
Use with to open file
sunjayBhatia Nov 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion .azure-pipelines/pipelines.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -387,9 +387,13 @@ stages:
- bash: ci/run_envoy_docker.sh ci/windows_ci_steps.sh
displayName: "Run Windows CI"
env:
CI_TARGET: "windows"
ENVOY_DOCKER_BUILD_DIR: "$(Build.StagingDirectory)"
SLACK_TOKEN: $(SLACK_TOKEN)
REPO_URI: $(Build.Repository.Uri)
BUILD_URI: $(Build.BuildUri)
ENVOY_RBE: "true"
BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=remote-msvc-cl --jobs=$(RbeJobs)"
BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=remote-msvc-cl --jobs=$(RbeJobs) --flaky_test_attempts=2"
BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
Expand Down
35 changes: 16 additions & 19 deletions ci/flaky_test/process_xml.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,17 +67,18 @@ def processFindOutput(f, problematic_tests):
# this will likely need adjustments as well.
def getGitInfo(CI_TARGET):
ret = ""
os.system("git remote -v > ${TMP_OUTPUT_PROCESS_XML}")
os.system("git describe --all >> ${TMP_OUTPUT_PROCESS_XML}")
os.system("git show >> ${TMP_OUTPUT_PROCESS_XML}")
f = open(os.environ['TMP_OUTPUT_PROCESS_XML'], 'r+', encoding='utf-8')
os.system('sh -c "git remote -v > ${TMP_OUTPUT_PROCESS_XML}"')
os.system('sh -c "git describe --all >> ${TMP_OUTPUT_PROCESS_XML}"')
os.system('sh -c "git show >> ${TMP_OUTPUT_PROCESS_XML}"')
# Fetching the URL from predefined env variable
envoy_link = os.environ["REPO_URI"]
for line in [next(f) for x in range(6)]:
if line.split('/')[0] == 'remotes':
for token in line.split('/')[1:-1]:
envoy_link += '/' + token
ret += line

with open(os.environ['TMP_OUTPUT_PROCESS_XML'], 'r+', encoding='utf-8') as f:
for line in [next(f) for x in range(6)]:
if line.split('/')[0] == 'remotes':
for token in line.split('/')[1:-1]:
envoy_link += '/' + token
ret += line

ret += "link for additional content: " + envoy_link + " \n"
ret += "azure build URI: " + os.environ["BUILD_URI"] + " \n"
Expand All @@ -100,22 +101,18 @@ def getGitInfo(CI_TARGET):
sys.exit(0)
output_msg += getGitInfo(CI_TARGET)

find_dir = '${TEST_TMPDIR}/**/**/**/**/bazel-testlogs/'
if CI_TARGET == "MacOS":
os.system('find ${TEST_TMPDIR}/ -name "attempt_*.xml" > ${TMP_OUTPUT_PROCESS_XML}')
else:
os.system(
'find ${TEST_TMPDIR}/**/**/**/**/bazel-testlogs/ -name "attempt_*.xml" > ${TMP_OUTPUT_PROCESS_XML}'
)

f = open(os.environ['TMP_OUTPUT_PROCESS_XML'], 'r+')
if f.closed:
print("cannot open {}".format(os.environ['TMP_OUTPUT_PROCESS_XML']))
find_dir = '${TEST_TMPDIR}/'
os.system(
'sh -c "/usr/bin/find {} -name attempt_*.xml > ${{TMP_OUTPUT_PROCESS_XML}}"'.format(find_dir))

# All output of find command should be either failed or flaky tests, as only then will
# a test be rerun and have an 'attempt_n.xml' file. problematic_tests holds a lookup
# table between the last_attempt xml filepath and the failed previous attempt filepath.
problematic_tests = {}
processFindOutput(f, problematic_tests)
with open(os.environ['TMP_OUTPUT_PROCESS_XML'], 'r+') as f:
processFindOutput(f, problematic_tests)

# Needed to make sure no duplicate flaky tests are going to be reported.
visited = set()
Expand Down
5 changes: 5 additions & 0 deletions ci/windows_ci_steps.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ df -h
# shellcheck source=ci/setup_cache.sh
. "$(dirname "$0")"/setup_cache.sh

[ -z "${ENVOY_SRCDIR}" ] && export ENVOY_SRCDIR=/c/source

read -ra BAZEL_STARTUP_OPTIONS <<< "${BAZEL_STARTUP_OPTIONS:-}"
# Default to msvc-cl if not overridden
read -ra BAZEL_BUILD_EXTRA_OPTIONS <<< "${BAZEL_BUILD_EXTRA_OPTIONS:---config=msvc-cl}"
Expand Down Expand Up @@ -75,6 +77,9 @@ tar czf "${ENVOY_BUILD_DIR}"/envoy_binary.tar.gz -C "${ENVOY_DELIVERY_DIR}" envo
# Test invocations of known-working tests on Windows
bazel "${BAZEL_STARTUP_OPTIONS[@]}" test "${BAZEL_BUILD_OPTIONS[@]}" //test/... --test_tag_filters=-skip_on_windows,-fails_on_windows,-flaky_on_windows --build_tests_only

echo "running flaky test reporting script"
"${ENVOY_SRCDIR}"/ci/flaky_test/run_process_xml.sh "$CI_TARGET"

# Build tests that are known-flaky or known-failing to ensure no compilation regressions
bazel "${BAZEL_STARTUP_OPTIONS[@]}" build "${BAZEL_BUILD_OPTIONS[@]}" //test/... --test_tag_filters=-skip_on_windows,fails_on_windows,flaky_on_windows --build_tests_only

Expand Down