diff --git a/eng/pipelines/common/evaluate-changed-darc-deps.yml b/eng/pipelines/common/evaluate-changed-darc-deps.yml new file mode 100644 index 0000000000000..1a816f5876b1a --- /dev/null +++ b/eng/pipelines/common/evaluate-changed-darc-deps.yml @@ -0,0 +1,15 @@ +# This step template evaluates changes in dependencies defined in `eng/Version.Details.xml`. +# For more information on how this works works look at evaluate-changed-darc-deps.sh docs +# at the beginning of that file. + +parameters: + subsetName: '' + # Array containing the arguments that are to be passed down to evaluate-changed-paths.sh + # Note that --azurevariable is always set to the dependency name, no need to pass it down. + arguments: [] + +steps: + - script: eng/pipelines/evaluate-changed-darc-deps.sh + ${{ join(' ', parameters.arguments) }} + displayName: Evaluate eng/Version.Details.xml for dependency changes + name: DarcDependenciesChanged diff --git a/eng/pipelines/common/evaluate-paths-job.yml b/eng/pipelines/common/evaluate-paths-job.yml index 42f8549cbb7f0..71f35bd4f9f4c 100644 --- a/eng/pipelines/common/evaluate-paths-job.yml +++ b/eng/pipelines/common/evaluate-paths-job.yml @@ -50,3 +50,10 @@ jobs: - --includepaths '${{ join('+', path.include) }}' - ${{ if ne(path.exclude[0], '') }}: - --excludepaths '${{ join('+', path.exclude) }}' + + - template: evaluate-changed-darc-deps.yml + parameters: + arguments: + # The commit that we're building is always a merge commit that is merging into the target branch. + # So the first parent of the commit is on the target branch and the second parent is on the source branch. + - --difftarget HEAD^1 diff --git a/eng/pipelines/common/templates/wasm-build-only.yml b/eng/pipelines/common/templates/wasm-build-only.yml index 0f9dfb22e7720..cfb6f40f09532 100644 --- a/eng/pipelines/common/templates/wasm-build-only.yml +++ b/eng/pipelines/common/templates/wasm-build-only.yml @@ -39,6 +39,7 @@ jobs: condition: >- or( eq(variables['alwaysRunVar'], true), + eq(variables['wasmDarcDependenciesChanged'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_allwasm.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_mono.containsChange'], true)) diff --git a/eng/pipelines/common/templates/wasm-build-tests.yml b/eng/pipelines/common/templates/wasm-build-tests.yml index 369624c0adcb7..c9c8689fa671e 100644 --- a/eng/pipelines/common/templates/wasm-build-tests.yml +++ b/eng/pipelines/common/templates/wasm-build-tests.yml @@ -34,6 +34,7 @@ jobs: condition: >- or( eq(variables['alwaysRunVar'], true), + eq(variables['wasmDarcDependenciesChanged'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_allwasm.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_wasmbuildtests.containsChange'], true)) # extra steps, run tests diff --git a/eng/pipelines/common/templates/wasm-debugger-tests.yml b/eng/pipelines/common/templates/wasm-debugger-tests.yml index 3bb39b74b6a17..5ac7c3ee5dccf 100644 --- a/eng/pipelines/common/templates/wasm-debugger-tests.yml +++ b/eng/pipelines/common/templates/wasm-debugger-tests.yml @@ -33,6 +33,7 @@ jobs: condition: >- or( eq(variables['alwaysRunVar'], true), + eq(variables['wasmDarcDependenciesChanged'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_allwasm.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_wasmdebuggertests.containsChange'], true)) # extra steps, run tests diff --git a/eng/pipelines/common/templates/wasm-library-tests.yml b/eng/pipelines/common/templates/wasm-library-tests.yml index 365ea7386d400..ada2484fedc37 100644 --- a/eng/pipelines/common/templates/wasm-library-tests.yml +++ b/eng/pipelines/common/templates/wasm-library-tests.yml @@ -44,6 +44,7 @@ jobs: condition: >- or( eq(variables['alwaysRunVar'], true), + eq(variables['wasmDarcDependenciesChanged'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_libraries.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_allwasm.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_mono.containsChange'], true)) diff --git a/eng/pipelines/common/templates/wasm-runtime-tests.yml b/eng/pipelines/common/templates/wasm-runtime-tests.yml index e6bb45d98887c..37154b3c6e86d 100644 --- a/eng/pipelines/common/templates/wasm-runtime-tests.yml +++ b/eng/pipelines/common/templates/wasm-runtime-tests.yml @@ -34,6 +34,7 @@ jobs: condition: >- or( eq(variables['alwaysRunVar'], true), + eq(variables['wasmDarcDependenciesChanged'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_mono.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_allwasm.containsChange'], true), eq(dependencies.evaluate_paths.outputs['SetPathVars_runtimetests.containsChange'], true)) diff --git a/eng/pipelines/common/xplat-setup.yml b/eng/pipelines/common/xplat-setup.yml index 25314635d9b44..9d45d7cddf15d 100644 --- a/eng/pipelines/common/xplat-setup.yml +++ b/eng/pipelines/common/xplat-setup.yml @@ -108,6 +108,14 @@ jobs: ${{ if eq(parameters.jobParameters.runtimeFlavor, 'coreclr') }}: value: CoreCLR + - name: wasmDarcDependenciesChanged + ${{ if eq(parameters.archType, 'wasm') }}: + value: $[ or( + eq(dependencies.evaluate_paths.outputs['DarcDependenciesChanged.Microsoft_NET_Workload_Emscripten_Manifest-7_0_100'], true), + eq(dependencies.evaluate_paths.outputs['DarcDependenciesChanged.Microsoft_DotNet_Build_Tasks_Workloads'], true), + eq(dependencies.evaluate_paths.outputs['DarcDependenciesChanged.System_Runtime_TimeZoneData'], true), + eq(dependencies.evaluate_paths.outputs['DarcDependenciesChanged.Microsoft_NET_ILLink_Tasks'], true)) ] + - ${{ each variable in parameters.variables }}: - ${{ variable }} diff --git a/eng/pipelines/evaluate-changed-darc-deps.sh b/eng/pipelines/evaluate-changed-darc-deps.sh new file mode 100755 index 0000000000000..b4181a1e50347 --- /dev/null +++ b/eng/pipelines/evaluate-changed-darc-deps.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +: ' + Compares contents of `env/Version.Details.xml` between HEAD and difftarget, and emits variables named for + dependencies that satisfy either of: + 1. version, or sha changed + 2. it is missing from one of the xmls + + The dependency names have `.` replaced with `_`. + + In order to consume these variables in a yaml pipeline, reference them via: $[ dependencies..outputs["."] ] + + Example: + -difftarget ''HEAD^1'' +' + +# Disable globbing in this bash script since we iterate over path patterns +set -f + +# Stop script if unbound variable found (use ${var:-} if intentional) +set -u + +# Stop script if command returns non-zero exit code. +# Prevents hidden errors caused by missing error code propagation. +set -e + +usage() +{ + echo "Script that emits an azure devops variable with all the dependencies that changed in 'eng/Version.Details.xml' contained in the current HEAD against the difftarget" + echo " --difftarget SHA or branch to diff against. (i.e: HEAD^1, origin/main, 0f4hd36, etc.)" + echo " --azurevariableprefix Name of azure devops variable to create if change meets filter criteria" + echo "" + + echo "Arguments can also be passed in with a single hyphen." +} + +source="${BASH_SOURCE[0]}" + +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done + +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" +eng_root=`cd -P "$scriptroot/.." && pwd` + +azure_variable_prefix='' +diff_target='' + +while [[ $# > 0 ]]; do + opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" + case "$opt" in + -help|-h) + usage + exit 0 + ;; + -difftarget) + diff_target=$2 + shift + ;; + -azurevariableprefix) + azure_variable_prefix=$2 + shift + ;; + esac + + shift +done + +if [[ -z "$diff_target" ]]; then + echo "Argument -difftarget is required" + usage + exit 1 +fi + +oldXmlPath=`mktemp` + +ci=true # Needed in order to use pipeline-logging-functions.sh +. "$eng_root/common/pipeline-logging-functions.sh" + +git show $diff_target:eng/Version.Details.xml > $oldXmlPath +# FIXME: errors? +changed_deps=$(python3 "$eng_root/pipelines/get-changed-darc-deps.py" $oldXmlPath eng/Version.Details.xml) +rm -f $oldXmlPath + +if [[ -n "$azure_variable_prefix" ]]; then + azure_variable_prefix="${azure_variable_prefix}_" +fi + +for dep in $changed_deps; do + dep=`echo $dep | tr \. _` + var_name=${azure_variable_prefix}${dep} + echo "Setting pipeline variable $var_name=true" + Write-PipelineSetVariable -name $var_name -value true +done diff --git a/eng/pipelines/get-changed-darc-deps.py b/eng/pipelines/get-changed-darc-deps.py new file mode 100644 index 0000000000000..cae52445eaa0a --- /dev/null +++ b/eng/pipelines/get-changed-darc-deps.py @@ -0,0 +1,62 @@ +# +# Emits a comma separated list of dependencies from `eng/Version.Details.xml` +# that changed as compared to another versions file +# +# - we don't really care which is old, and which is new +# - A dependency name is emitted as changed if: +# 1. version, or sha changed +# 2. it is missing from one of the xmls + +import xml.etree.ElementTree as ET +import sys +from os.path import exists + +def getDependencies(xmlfile): + tree = ET.parse(xmlfile) + root = tree.getroot() + deps = {} + for depElement in root.findall('.//Dependency'): + dep = {} + dep['Version'] = depElement.attrib['Version'] + dep['Sha'] = depElement.find('Sha').text + + deps[depElement.attrib['Name']] = dep + + return deps + +def compare(dict1, dict2): + if dict1 is None or dict2 is None: + print('Nones') + return False + + if (not isinstance(dict1, dict)) or (not isinstance(dict2, dict)): + print('Not dict') + return False + + changed_names = [] + all_keys = set(dict1.keys()) | set(dict2.keys()) + for key in all_keys: + if key not in dict1 or key not in dict2: + print(key) + # changed_names.append(key) + elif dict1[key] != dict2[key]: + print(key) + # changed_names.append(key) + + print(','.join(changed_names)) + +if len(sys.argv) != 3: + print(f'Usage: {sys.argv[0]} ') + exit(1) + +if not exists(sys.argv[1]): + print(f'Cannot find {sys.argv[1]}') + exit(1) +if not exists(sys.argv[2]): + print(f'Cannot find {sys.argv[2]}') + exit(1) + +newDeps = getDependencies(sys.argv[1]) +oldDeps = getDependencies(sys.argv[2]) + +compare(oldDeps, newDeps)