diff --git a/eng/pipelines/common/download-specific-artifact-step.yml b/eng/pipelines/common/download-specific-artifact-step.yml index f3d1bf7d0f52c4..e2037e007dfefa 100644 --- a/eng/pipelines/common/download-specific-artifact-step.yml +++ b/eng/pipelines/common/download-specific-artifact-step.yml @@ -12,10 +12,10 @@ steps: inputs: buildType: specific project: 'internal' # 'internal' or 'public' - pipeline: 'superpmi-replay' + pipeline: 'Antigen' buildVersionToDownload: specific - branchName: 'your/branch/having/artifacts' - buildId: '1274841' + branchName: 'kpathak/antigen-ci' + buildId: '1385069' downloadType: single downloadPath: '$(Build.SourcesDirectory)/__download__' artifactName: '${{ parameters.artifactName }}' diff --git a/eng/pipelines/coreclr/exploratory.yml b/eng/pipelines/coreclr/exploratory.yml new file mode 100644 index 00000000000000..4a6aa4dfe5ffb2 --- /dev/null +++ b/eng/pipelines/coreclr/exploratory.yml @@ -0,0 +1,51 @@ +# This job definition automates the exploratory tool. + +trigger: none + +schedules: +- cron: "0 14 * * 0" + displayName: Sun at 6:00 AM (UTC-8:00) + branches: + include: + - main + always: true + +jobs: + +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + buildConfig: checked + platforms: + # Linux tests are built on the OSX machines. + # - OSX_x64 + - Linux_arm + - Linux_arm64 + - Linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + - CoreClrTestBuildHost # Either OSX_x64 or Linux_x64 + jobParameters: + testGroup: outerloop + +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/jit-exploratory-job.yml + buildConfig: checked + platforms: + # Linux tests are built on the OSX machines. + # - OSX_x64 + - Linux_arm + - Linux_arm64 + - Linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + collectionType: pmi + collectionName: libraries \ No newline at end of file diff --git a/eng/pipelines/coreclr/templates/jit-exploratory-job.yml b/eng/pipelines/coreclr/templates/jit-exploratory-job.yml new file mode 100644 index 00000000000000..2eeec92b2570a4 --- /dev/null +++ b/eng/pipelines/coreclr/templates/jit-exploratory-job.yml @@ -0,0 +1,79 @@ +parameters: + buildConfig: '' + archType: '' + osGroup: '' + osSubgroup: '' + container: '' + runtimeVariant: '' + testGroup: '' + framework: net6.0 # Specify the appropriate framework when running release branches (ie netcoreapp3.0 for release/3.0) + liveLibrariesBuildConfig: '' + variables: {} + runtimeType: 'coreclr' + pool: '' + codeGenType: 'JIT' + projetFile: '' + runKind: '' + runJobTemplate: '/eng/pipelines/coreclr/templates/jit-run-exploratory-job.yml' + additionalSetupParameters: '' + collectionType: '' + collectionName: '' + +### Exploratory job + +### Each exploratory job depends on a corresponding build job with the same +### buildConfig and archType. + +jobs: +- template: ${{ parameters.runJobTemplate }} + parameters: + # Compute job name from template parameters + jobName: ${{ format('exploratory_{0}{1}_{2}_{3}_{4}_{5}_{6}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.runtimeType, parameters.codeGenType, parameters.runKind) }} + displayName: ${{ format('Exploratory {0}{1} {2} {3} {4} {5} {6}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.runtimeType, parameters.codeGenType, parameters.runKind) }} + pool: ${{ parameters.pool }} + buildConfig: ${{ parameters.buildConfig }} + archType: ${{ parameters.archType }} + osGroup: ${{ parameters.osGroup }} + osSubgroup: ${{ parameters.osSubgroup }} + runtimeVariant: ${{ parameters.runtimeVariant }} + liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} + runtimeType: ${{ parameters.runtimeType }} + codeGenType: ${{ parameters.codeGenType }} + runKind: ${{ parameters.runKind }} + testGroup: ${{ parameters.testGroup }} + collectionType: ${{ parameters.collectionType }} + collectionName: ${{ parameters.collectionName }} + additionalSetupParameters: ${{ parameters.additionalSetupParameters }} + # Test job depends on the corresponding build job + dependsOn: + - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} + + variables: ${{ parameters.variables }} + + frameworks: + - ${{ parameters.framework }} + steps: + # Extra steps that will be passed to the exploratory template and run before sending the job to helix (all of which is done in the template) + + # Optionally download live-built libraries + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(librariesDownloadDir) + cleanUnpackFolder: false + artifactFileName: '$(librariesBuildArtifactName)$(archiveExtension)' + artifactName: '$(librariesBuildArtifactName)' + displayName: 'live-built libraries' + + # Download coreclr + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(buildProductRootFolderPath) + artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' + artifactName: '$(buildProductArtifactName)' + displayName: 'Coreclr product build' + + # Create Core_Root + - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(buildConfig) $(archType) generatelayoutonly $(librariesOverrideArg) + displayName: Create Core_Root + condition: succeeded() diff --git a/eng/pipelines/coreclr/templates/jit-run-exploratory-job.yml b/eng/pipelines/coreclr/templates/jit-run-exploratory-job.yml new file mode 100644 index 00000000000000..f5754ec47aa6aa --- /dev/null +++ b/eng/pipelines/coreclr/templates/jit-run-exploratory-job.yml @@ -0,0 +1,141 @@ +parameters: + steps: [] # optional -- any additional steps that need to happen before pulling down the jitutils repo and sending the jitutils to helix (ie building your repo) + variables: [] # optional -- list of additional variables to send to the template + jobName: '' # required -- job name + displayName: '' # optional -- display name for the job. Will use jobName if not passed + pool: '' # required -- name of the Build pool + container: '' # required -- name of the container + buildConfig: '' # required -- build configuration + archType: '' # required -- targeting CPU architecture + osGroup: '' # required -- operating system for the job + osSubgroup: '' # optional -- operating system subgroup + extraSetupParameters: '' # optional -- extra arguments to pass to the setup script + frameworks: ['net6.0'] # optional -- list of frameworks to run against + continueOnError: 'false' # optional -- determines whether to continue the build if the step errors + dependsOn: '' # optional -- dependencies of the job + timeoutInMinutes: 320 # optional -- timeout for the job + enableTelemetry: false # optional -- enable for telemetry + liveLibrariesBuildConfig: '' # optional -- live-live libraries configuration to use for the run + runtimeType: 'coreclr' # optional -- Sets the runtime as coreclr or mono + codeGenType: 'JIT' # optional -- Decides on the codegen technology if running on mono + runKind: '' # required -- test category + collectionType: '' + collectionName: '' + dependOnEvaluatePaths: false + +jobs: +- template: xplat-pipeline-job.yml + parameters: + dependsOn: ${{ parameters.dependsOn }} + buildConfig: ${{ parameters.buildConfig }} + archType: ${{ parameters.archType }} + osGroup: ${{ parameters.osGroup }} + osSubgroup: ${{ parameters.osSubgroup }} + liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} + enableTelemetry: ${{ parameters.enableTelemetry }} + enablePublishBuildArtifacts: true + continueOnError: ${{ parameters.continueOnError }} + dependOnEvaluatePaths: ${{ parameters.dependOnEvaluatePaths }} + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + + ${{ if ne(parameters.displayName, '') }}: + displayName: '${{ parameters.displayName }}' + ${{ if eq(parameters.displayName, '') }}: + displayName: '${{ parameters.jobName }}' + + variables: + + - ${{ each variable in parameters.variables }}: + - ${{ if ne(variable.name, '') }}: + - name: ${{ variable.name }} + value: ${{ variable.value }} + - ${{ if ne(variable.group, '') }}: + - group: ${{ variable.group }} + + - HelixApiAccessToken: '' + - HelixPreCommand: '' + + - ${{ if eq(parameters.osGroup, 'windows') }}: + - name: PythonScript + value: 'py -3' + - name: PipScript + value: 'py -3 -m pip' + - name: Core_Root_Dir + value: '$(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}\Tests\Core_Root' + - name: IssuesLocation + value: '$(Build.SourcesDirectory)\artifacts\helixresults\' + - name: IssuesSummaryLocation + value: '$(Build.SourcesDirectory)\artifacts\issues_summary\' + + - ${{ if ne(parameters.osGroup, 'windows') }}: + - name: PythonScript + value: 'python3' + - name: PipScript + value: 'pip3' + - name: Core_Root_Dir + value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper)/Tests/Core_Root' + - name: IssuesLocation + value: '$(Build.SourcesDirectory)/artifacts/helixresults/' + - name: IssuesSummaryLocation + value: '$(Build.SourcesDirectory)/artifacts/issues_summary/' + workspace: + clean: all + pool: + ${{ parameters.pool }} + container: ${{ parameters.container }} + strategy: + matrix: + ${{ each framework in parameters.frameworks }}: + ${{ framework }}: + _Framework: ${{ framework }} + steps: + - ${{ parameters.steps }} + + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/antigen_setup.py -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) + displayName: ${{ format('Antigen setup ({0}-{1})', parameters.osGroup, parameters.archType) }} + + # Run exploratory tool in helix + - template: /eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml + parameters: + HelixSource: '$(HelixSourcePrefix)/$(Build.Repository.Name)/$(Build.SourceBranch)' # sources must start with pr/, official/, prodcon/, or agent/ + HelixType: 'test/superpmi/$(CollectionName)/$(CollectionType)/$(_Framework)/$(Architecture)' + HelixAccessToken: $(HelixApiAccessToken) + HelixTargetQueues: $(Queue) + HelixPreCommands: $(HelixPreCommand) + Creator: $(Creator) + WorkItemTimeout: 2:30 # 2.5 hours + WorkItemDirectory: '$(WorkItemDirectory)' + CorrelationPayloadDirectory: '$(CorrelationPayloadDirectory)' + ProjectFile: 'exploratory.proj' + BuildConfig: ${{ parameters.buildConfig }} + osGroup: ${{ parameters.osGroup }} + RunConfiguration: '$(RunConfiguration)' + ToolName: '$(ToolName)' + continueOnError: true # Run the future step i.e. merge-mch step even if this step fails. + + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(IssuesLocation) + includeRootFolder: false + archiveType: $(archiveType) + tarCompression: $(tarCompression) + archiveExtension: $(archiveExtension) + artifactName: 'Antigen_Issues_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + displayName: ${{ format('Upload artifacts Antigen issues for {0}-{1}', parameters.osGroup, parameters.archType ) }} + + # Always upload the available issues-summary.txt files + - task: CopyFiles@2 + displayName: Copying issues-summary.txt of all partitions + inputs: + sourceFolder: '$(IssuesLocation)' + contents: '**/issues-summary-*.txt' + targetFolder: '$(IssuesSummaryLocation)' + + - task: PublishPipelineArtifact@1 + displayName: Publish issues-summary.txt files + inputs: + targetPath: $(IssuesSummaryLocation) + artifactName: 'Issues_Summary_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' + + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/antigen_unique_issues.py -issues_directory $(IssuesSummaryLocation) + displayName: ${{ format('Print unique issues ({0})', parameters.osGroup) }} diff --git a/src/coreclr/scripts/antigen.md b/src/coreclr/scripts/antigen.md new file mode 100644 index 00000000000000..dfa74e5e88645e --- /dev/null +++ b/src/coreclr/scripts/antigen.md @@ -0,0 +1,23 @@ +# Documentation of Exploratory tool Antigen + +Antigen is a an exploratory fuzzing tool to test JIT compiler. Currently the source code is present at https://github.com/kunalspathak/antigen and will eventually become a separate repository under "dotnet" organization or part of [dotnet/jitutils](https://github.com/dotnet/jitutils) repository. + +## Overview + +Antigen generates random test cases, compile them using Roslyn APIs and then execute it against `CoreRun.exe` in baseline and test mode. Baseline mode is without tiered JIT with minimum optimizations enabled while test mode is run by setting various combination of `COMPlus_*` environment variables. The output of both modes are compared to ensure they are same. If the output is different or an assert is hit in test mode, the tool saves the produced C# file in a `UniqueIssue` folder. Similar issues are placed in `UniqueIssueN` folder. + +Antigen also comes with `Trimmer` that can be used to reduce the generated file and still reproduce the issue. Some more work needs to be done in this tool to make it efficient. + +## Pipeline + +Antigen tool is ran every night using CI pipeline. It can also be triggered on PRs. The pipeline would run Antigen tool for 3 hours on x86/x64 platforms and 2 hours for arm/arm64 platforms to generate test cases and verify if it contains issues. Once the run duration is complete, for each OS/arch, the pipeline will upload the issues that Antigen has found has an artifact that can be downloaded. The issues will be `.cs` files that will contain the program's output, environment variables that are needed to reproduce the issue. Since there can be several issues, the pipeline will just upload at most 5 issues from each `UniqueIssueN` folder. + +### Pipeline details + +1. `eng/pipeline/coreclr/jit-exploratory.yml` : This is the main pipeline which will perform Coreclr/libraries build and then further trigger `jit-exploratory-job.yml` pipeline. +1. `eng/pipeline/coreclr/templates/jit-exploratory-job.yml` : This pipeline will download all the Coreclr/libraries artifacts and create `CORE_ROOT` folder. It further triggers `jit-run-exploratory-job.yml` pipeline. +1. `eng/pipeline/coreclr/templates/jit-run-exploratory-job.yml` : This pipeline will perform the actual run in 3 steps: + * `src/coreclr/scripts/antigen_setup.py`: This script will clone the Antigen repo, build and prepare the payloads to be sent for testing. + * `src/coreclr/scripts/antigen_run.py`: This script will execute Antigen tool and upload back the issues. + * `src/coreclr/scripts/antigen_unique_issues.py`: In addition to uploading the issues, this script will also print the output of unique issues it has found so the developer can quickly take a look at them and decide which platform's artifacts to download. If there is any issue found by Antigen, this script will make the pipeline as "FAIL". +1. `src/coreclr/scripts/exploratory.proj`: This proj file is the one that creates the helix jobs. Currently, this file configures to run Antigen on 4 partitions. Thus, if Antigen can generate and test 1000 test cases on 1 machine, with current setup, the pipeline will be able to test 4000 test cases. \ No newline at end of file diff --git a/src/coreclr/scripts/antigen_run.py b/src/coreclr/scripts/antigen_run.py new file mode 100644 index 00000000000000..e97196debeedfc --- /dev/null +++ b/src/coreclr/scripts/antigen_run.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +# +## Licensed to the .NET Foundation under one or more agreements. +## The .NET Foundation licenses this file to you under the MIT license. +# +## +# Title: antigen_run.py +# +# Notes: +# +# Script to execute Antigen tool on a platform and return back the repro +# issues they found. +# +################################################################################ +################################################################################ + +import shutil +import argparse +from os import path, walk +from os.path import getsize +import os +from coreclr_arguments import * +from superpmi_setup import run_command +from superpmi import TempDir + +parser = argparse.ArgumentParser(description="description") + +parser.add_argument("-run_configuration", help="RunConfiguration") +parser.add_argument("-antigen_directory", help="Path to antigen tool") +parser.add_argument("-output_directory", help="Path to output directory") +parser.add_argument("-partition", help="Partition name") +parser.add_argument("-core_root", help="path to CORE_ROOT directory") +is_windows = platform.system() == "Windows" + + +def setup_args(args): + """ Setup the args + + Args: + args (ArgParse): args parsed by arg parser + + Returns: + args (CoreclrArguments) + + """ + coreclr_args = CoreclrArguments(args, require_built_core_root=False, require_built_product_dir=False, + require_built_test_dir=False, default_build_type="Checked") + + coreclr_args.verify(args, + "run_configuration", + lambda unused: True, + "Unable to set run_configuration") + + coreclr_args.verify(args, + "antigen_directory", + lambda antigen_directory: os.path.isdir(antigen_directory), + "antigen_directory doesn't exist") + + coreclr_args.verify(args, + "output_directory", + lambda unused: True, + "output_directory doesn't exist") + + coreclr_args.verify(args, + "partition", + lambda unused: True, + "Unable to set partition") + + coreclr_args.verify(args, + "core_root", + lambda core_root: os.path.isdir(core_root), + "core_root doesn't exist") + + return coreclr_args + + +def copy_issues(issues_directory, upload_directory, tag_name): + """Copies issue files (only top 5 smallest files from each folder) into the upload_directory + + Args: + issues_directory (string): Issues directory + upload_directory (string): Upload directory + tag_name (string): Tag name for zip file + + Returns: + [type]: [description] + """ + # Create upload_directory + if not os.path.isdir(upload_directory): + os.makedirs(upload_directory) + + # Create temp directory to copy all issues to upload. We don't want to create a sub-folder + # inside issues_directory because that will also get included twice. + with TempDir() as prep_directory: + + def sorter_by_size(pair): + """ Sorts the pair (file_name, file_size) tuple in ascending order of file_size + + Args: + pair ([(string, int)]): List of tuple of file_name, file_size + """ + pair.sort(key=lambda x: x[1], reverse=False) + return pair + + summary_of_summary = [] + for file_path, dirs, files in walk(issues_directory, topdown=True): + filename_with_size = [] + # Credit: https://stackoverflow.com/a/19859907 + dirs[:] = [d for d in dirs] + for name in files: + if not name.lower().endswith(".g.cs"): + continue + + curr_file_path = path.join(file_path, name) + size = getsize(curr_file_path) + filename_with_size.append((curr_file_path, size)) + + if len(filename_with_size) == 0: + continue + summary_file = path.join(file_path, "summary.txt") + summary_of_summary.append("**** " + file_path) + with open(summary_file, 'r') as sf: + summary_of_summary.append(sf.read()) + filename_with_size.append((summary_file, 0)) # always copy summary.txt + + # Copy atmost 5 files from each bucket + sorted_files = [f[0] for f in sorter_by_size(filename_with_size)[:6]] # sorter_by_size(filename_with_size)[:6] + print('### Copying below files from {0} to {1}:'.format(issues_directory, prep_directory)) + print('') + print(os.linesep.join(sorted_files)) + for src_file in sorted_files: + dst_file = src_file.replace(issues_directory, prep_directory) + dst_directory = path.dirname(dst_file) + if not os.path.exists(dst_directory): + os.makedirs(dst_directory) + try: + shutil.copy2(src_file, dst_file) + except PermissionError as pe_error: + print('Ignoring PermissionError: {0}'.format(pe_error)) + + issues_summary_file_name = "issues-summary-{}.txt".format(tag_name) + print("Creating {} in {}".format(issues_summary_file_name, upload_directory)) + issues_summary_file = os.path.join(upload_directory, issues_summary_file_name) + with open(issues_summary_file, 'w') as sf: + sf.write(os.linesep.join(summary_of_summary)) + + # Also copy the issues-summary inside zip folder + dst_issue_summary_file = os.path.join(prep_directory, issues_summary_file_name) + try: + shutil.copy2(issues_summary_file, dst_issue_summary_file) + except PermissionError as pe_error: + print('Ignoring PermissionError: {0}'.format(pe_error)) + + # Zip compress the files we will upload + zip_path = os.path.join(prep_directory, "AllIssues-" + tag_name) + print("Creating archive: " + zip_path) + shutil.make_archive(zip_path, 'zip', prep_directory) + + zip_path += ".zip" + dst_zip_path = os.path.join(upload_directory, "AllIssues-" + tag_name + ".zip") + print("Copying {} to {}".format(zip_path, dst_zip_path)) + try: + shutil.copy2(zip_path, dst_zip_path) + except PermissionError as pe_error: + print('Ignoring PermissionError: {0}'.format(pe_error)) + + + +def main(main_args): + """Main entrypoint + + Args: + main_args ([type]): Arguments to the script + """ + + coreclr_args = setup_args(main_args) + + antigen_directory = coreclr_args.antigen_directory + core_root = coreclr_args.core_root + tag_name = "{}-{}".format(coreclr_args.run_configuration, coreclr_args.partition) + output_directory = coreclr_args.output_directory + run_duration = 120 # Run for 2 hours + + path_to_corerun = os.path.join(core_root, "corerun") + path_to_tool = os.path.join(antigen_directory, "Antigen") + if is_windows: + path_to_corerun += ".exe" + path_to_tool += ".exe" + + # Run tool such that issues are placed in a temp folder + with TempDir() as temp_location: + run_command([path_to_tool, "-c", path_to_corerun, "-o", temp_location, "-d", str(run_duration)], _exit_on_fail=True, _long_running= True) + + # Copy issues for upload + print("Copying issues to " + output_directory) + copy_issues(temp_location, output_directory, tag_name) + +if __name__ == "__main__": + args = parser.parse_args() + sys.exit(main(args)) diff --git a/src/coreclr/scripts/antigen_setup.py b/src/coreclr/scripts/antigen_setup.py new file mode 100644 index 00000000000000..f1f8a967c71556 --- /dev/null +++ b/src/coreclr/scripts/antigen_setup.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# +## Licensed to the .NET Foundation under one or more agreements. +## The .NET Foundation licenses this file to you under the MIT license. +# +## +# Title: antigen_setup.py +# +# Notes: +# +# Script to setup cloning and building Antigen tool. It copies all the binaries +# to the correlation payload. +# +################################################################################ +################################################################################ + +import argparse +from os import path +import os +from os import listdir +from coreclr_arguments import * +from superpmi_setup import run_command, copy_directory, set_pipeline_variable +from superpmi import ChangeDir, TempDir +import tempfile + +parser = argparse.ArgumentParser(description="description") + +parser.add_argument("-arch", help="Architecture") +parser.add_argument("-platform", help="OS platform") +parser.add_argument("-source_directory", help="path to source directory") +parser.add_argument("-core_root_directory", help="path to CORE_ROOT directory") +is_windows = platform.system() == "Windows" + +def setup_args(args): + """ Setup the args. + + Args: + args (ArgParse): args parsed by arg parser + + Returns: + args (CoreclrArguments) + + """ + coreclr_args = CoreclrArguments(args, require_built_core_root=False, require_built_product_dir=False, + require_built_test_dir=False, default_build_type="Checked") + + coreclr_args.verify(args, + "source_directory", + lambda source_directory: os.path.isdir(source_directory), + "source_directory doesn't exist") + + coreclr_args.verify(args, + "core_root_directory", + lambda core_root_directory: os.path.isdir(core_root_directory), + "core_root_directory doesn't exist") + + coreclr_args.verify(args, + "arch", + lambda unused: True, + "Unable to set arch") + + coreclr_args.verify(args, + "platform", + lambda unused: True, + "Unable to set platform") + + return coreclr_args + + +def main(main_args): + """Main entrypoint + + Args: + main_args ([type]): Arguments to the script + """ + + coreclr_args = setup_args(main_args) + arch_name = coreclr_args.arch + os_name = "win" if coreclr_args.platform.lower() == "windows" else "linux" + run_configuration = "{}-{}".format(os_name, arch_name) + source_directory = coreclr_args.source_directory + + # CorrelationPayload directories + correlation_payload_directory = path.join(coreclr_args.source_directory, "payload") + scripts_src_directory = path.join(source_directory, "src", "coreclr", 'scripts') + coreroot_dst_directory = path.join(correlation_payload_directory, "CoreRoot") + antigen_dst_directory = path.join(correlation_payload_directory, "exploratory") + + helix_source_prefix = "official" + creator = "" + ci = True + if is_windows: + helix_queue = "Windows.10.Arm64" if arch_name == "arm64" else "Windows.10.Amd64.X86" + else: + if arch_name == "arm": + helix_queue = "(Ubuntu.1804.Arm32)Ubuntu.1804.Armarch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm32v7-bfcd90a-20200121150440" + elif arch_name == "arm64": + helix_queue = "(Ubuntu.1804.Arm64)Ubuntu.1804.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm64v8-20210531091519-97d8652" + else: + helix_queue = "Ubuntu.1804.Amd64" + + # create exploratory directory + print('Copying {} -> {}'.format(scripts_src_directory, coreroot_dst_directory)) + copy_directory(scripts_src_directory, coreroot_dst_directory, match_func=lambda path: any(path.endswith(extension) for extension in [".py"])) + + if is_windows: + acceptable_copy = lambda path: any(path.endswith(extension) for extension in [".py", ".dll", ".exe", ".json"]) + else: + # Need to accept files without any extension, which is how executable file's names look. + acceptable_copy = lambda path: (os.path.basename(path).find(".") == -1) or any(path.endswith(extension) for extension in [".py", ".dll", ".so", ".json", ".a"]) + + # copy CORE_ROOT + print('Copying {} -> {}'.format(coreclr_args.core_root_directory, coreroot_dst_directory)) + copy_directory(coreclr_args.core_root_directory, coreroot_dst_directory, match_func=acceptable_copy) + + try: + with TempDir() as tool_code_directory: + # clone the tool + run_command( + ["git", "clone", "--quiet", "--depth", "1", "https://github.com/kunalspathak/Antigen.git", tool_code_directory]) + + antigen_bin_directory = path.join(tool_code_directory, "bin", "Release", "net5.0") + + # build the tool + with ChangeDir(tool_code_directory): + dotnet_cmd = os.path.join(source_directory, "dotnet.cmd") + if not is_windows: + dotnet_cmd = os.path.join(source_directory, "dotnet.sh") + run_command([dotnet_cmd, "publish", "-c", "Release", "--self-contained", "-r", run_configuration, "-o", antigen_bin_directory], _exit_on_fail=True) + + if not os.path.exists(path.join(antigen_bin_directory, "Antigen.dll")): + raise FileNotFoundError("Antigen.dll not present at {}".format(antigen_bin_directory)) + + # copy antigen tool + print('Copying {} -> {}'.format(antigen_bin_directory, antigen_dst_directory)) + copy_directory(antigen_bin_directory, antigen_dst_directory, match_func=acceptable_copy) + except PermissionError as pe: + print("Skipping file. Got error: %s", pe) + + # create foo.txt in work_item directories + workitem_directory = path.join(source_directory, "workitem") + os.mkdir(workitem_directory) + foo_txt = os.path.join(workitem_directory, "foo.txt") + with open(foo_txt, "w") as foo_txt_file: + foo_txt_file.write("hello world!") + + # Set variables + print('Setting pipeline variables:') + set_pipeline_variable("CorrelationPayloadDirectory", correlation_payload_directory) + set_pipeline_variable("WorkItemDirectory", workitem_directory) + set_pipeline_variable("RunConfiguration", run_configuration) + set_pipeline_variable("Creator", creator) + set_pipeline_variable("Queue", helix_queue) + set_pipeline_variable("HelixSourcePrefix", helix_source_prefix) + +if __name__ == "__main__": + args = parser.parse_args() + sys.exit(main(args)) diff --git a/src/coreclr/scripts/antigen_unique_issues.py b/src/coreclr/scripts/antigen_unique_issues.py new file mode 100644 index 00000000000000..d96094c005965d --- /dev/null +++ b/src/coreclr/scripts/antigen_unique_issues.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +# +## Licensed to the .NET Foundation under one or more agreements. +## The .NET Foundation licenses this file to you under the MIT license. +# +## +# Title: antigen_unique_issues.py +# +# Notes: +# +# Script to identify unique issues from all partitions and print them on console. +# +################################################################################ +################################################################################ +# import sys +import argparse +import os +from os import walk +from coreclr_arguments import * +import re + +parser = argparse.ArgumentParser(description="description") + +parser.add_argument("-issues_directory", help="Path to issues directory") + +unique_issue_dir_pattern = re.compile(r"\*\*\*\* .*UniqueIssue\d+") +assertion_patterns = [re.compile(r"Assertion failed '(.*)' in '.*' during '(.*)'"), + re.compile(r"Assert failure\(PID \d+ \[0x[0-9a-f]+], Thread: \d+ \[0x[0-9a-f]+]\):(.*)")] + +def setup_args(args): + """ Setup the args. + + Args: + args (ArgParse): args parsed by arg parser + + Returns: + args (CoreclrArguments) + + """ + coreclr_args = CoreclrArguments(args, require_built_core_root=False, require_built_product_dir=False, + require_built_test_dir=False, default_build_type="Checked") + + coreclr_args.verify(args, + "run_configuration", + lambda unused: True, + "Unable to set run_configuration") + + coreclr_args.verify(args, + "issues_directory", + lambda issues_directory: os.path.isdir(issues_directory), + "issues_directory doesn't exist") + + return coreclr_args + +def print_unique_issues_summary(issues_directory): + """Merge issues-summary-*-PartitionN.txt files from each partitions + and print unique issues + + Args: + issues_directory (string): Issues directory + Returns: + Number of issues found + """ + + issues_found = 0 + unique_issues_all_partitions = {} + for file_path, dirs, files in walk(issues_directory, topdown=True): + for file_name in files: + if not file_name.startswith("issues-summary-") or "Partition" not in file_name: + continue + + issues_summary_file = os.path.join(file_path, file_name) + partition_name = file_path.split(os.sep)[-1] + add_header = True + unique_issues = [] + with open(issues_summary_file, 'r') as sf: + contents = sf.read() + unique_issues = list(filter(None, re.split(unique_issue_dir_pattern, contents))) + + # Iterate over all unique issues of this partition + for unique_issue in unique_issues: + # Find the matching assertion message + for assertion_pattern in assertion_patterns: + issue_match = re.search(assertion_pattern, unique_issue) + if issue_match is not None: + assert_string = " ".join(issue_match.groups()) + # Check if previous partitions has already seen this assert + if assert_string not in unique_issues_all_partitions: + unique_issues_all_partitions[assert_string] = unique_issue + issues_found += 1 + if add_header: + print("%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% {} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%".format(partition_name)) + add_header = False + print(unique_issue.strip()) + print("------------------------------------") + break + + print("===== Found {} unique issues.".format(issues_found)) + return issues_found + +def main(main_args): + """Main entrypoint + + Args: + main_args ([type]): Arguments to the script + """ + + coreclr_args = setup_args(main_args) + + issues_directory = coreclr_args.issues_directory + issues_found = print_unique_issues_summary(issues_directory) + return 1 if issues_found > 0 else 0 + + +if __name__ == "__main__": + args = parser.parse_args() + sys.exit(main(args)) diff --git a/src/coreclr/scripts/exploratory.proj b/src/coreclr/scripts/exploratory.proj new file mode 100644 index 00000000000000..2ac5db3b7d967d --- /dev/null +++ b/src/coreclr/scripts/exploratory.proj @@ -0,0 +1,71 @@ + + + + \ + + + / + + + + %HELIX_PYTHONPATH% + %HELIX_WORKITEM_UPLOAD_ROOT% + %HELIX_CORRELATION_PAYLOAD%\CoreRoot + %HELIX_CORRELATION_PAYLOAD%\exploratory + + $(BUILD_SOURCESDIRECTORY)\artifacts\helixresults + + + + $HELIX_PYTHONPATH + $HELIX_WORKITEM_UPLOAD_ROOT + $HELIX_CORRELATION_PAYLOAD/CoreRoot + $HELIX_CORRELATION_PAYLOAD/exploratory + + $(BUILD_SOURCESDIRECTORY)/artifacts/helixresults + + + + false + false + 2:30 + + + + + + + + + + + + @(HelixPreCommand) + @(HelixPostCommand) + $(Python) $(CoreRoot)$(FileSeparatorChar)antigen_run.py -run_configuration $(RunConfiguration) -output_directory $(OutputDirectory) -antigen_directory $(ToolPath) -core_root $(CoreRoot) + + + + + %(Identity) + + + + + + + + + + + + + Partition%(HelixWorkItem.Index) + $(WorkItemDirectory) + $(WorkItemCommand) -partition %(PartitionName) + $(WorkItemTimeout) + AllIssues-$(RunConfiguration)-%(PartitionName).zip;issues-summary-$(RunConfiguration)-%(PartitionName).txt + + + + \ No newline at end of file diff --git a/src/coreclr/scripts/superpmi_setup.py b/src/coreclr/scripts/superpmi_setup.py index 0d0bc0bb341232..60455f67ca6dec 100644 --- a/src/coreclr/scripts/superpmi_setup.py +++ b/src/coreclr/scripts/superpmi_setup.py @@ -306,7 +306,7 @@ def first_fit(sorted_by_size, max_size): return partitions -def run_command(command_to_run, _cwd=None, _exit_on_fail=False): +def run_command(command_to_run, _cwd=None, _exit_on_fail=False, _long_running=False): """ Runs the command. Args: @@ -314,20 +314,33 @@ def run_command(command_to_run, _cwd=None, _exit_on_fail=False): _cwd (string): Current working directory. _exit_on_fail (bool): If it should exit on failure. Returns: - (string, string, int): Returns a tuple of stdout, stderr, and command return code + (string, string, int): Returns a tuple of stdout, stderr, and command return code if _long_running= False + Otherwise stdout, stderr are empty. """ print("Running: " + " ".join(command_to_run)) command_stdout = "" command_stderr = "" return_code = 1 - with subprocess.Popen(command_to_run, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=_cwd) as proc: - command_stdout, command_stderr = proc.communicate() - return_code = proc.returncode - if len(command_stdout) > 0: - print(command_stdout.decode("utf-8")) - if len(command_stderr) > 0: - print(command_stderr.decode("utf-8")) + output_type = subprocess.STDOUT if _long_running else subprocess.PIPE + with subprocess.Popen(command_to_run, stdout=subprocess.PIPE, stderr=output_type, cwd=_cwd) as proc: + + # For long running command, continuosly print the output + if _long_running: + while True: + output = proc.stdout.readline() + if proc.poll() is not None: + break + if output: + print(output.strip().decode("utf-8")) + else: + command_stdout, command_stderr = proc.communicate() + if len(command_stdout) > 0: + print(command_stdout.decode("utf-8")) + if len(command_stderr) > 0: + print(command_stderr.decode("utf-8")) + + return_code = proc.returncode if _exit_on_fail and return_code != 0: print("Command failed. Exiting.") sys.exit(1)