diff --git a/eng/common/pipelines/templates/steps/prepare-pipelines.yml b/eng/common/pipelines/templates/steps/prepare-pipelines.yml index 47fc611316d..d85c48caf3a 100644 --- a/eng/common/pipelines/templates/steps/prepare-pipelines.yml +++ b/eng/common/pipelines/templates/steps/prepare-pipelines.yml @@ -71,7 +71,26 @@ steps: --debug ${{parameters.TestsConventionOptions}} displayName: Create Live Test pipelines for public repository - condition: ne('${{parameters.TestsConventionOptions}}','') + condition: and(succeeded(), ne('${{parameters.TestsConventionOptions}}','')) + env: + PATVAR: $(azuresdk-azure-sdk-devops-pipeline-generation-pat) + - script: > + $(Pipeline.Workspace)/pipeline-generator/pipeline-generator + --organization https://dev.azure.com/azure-sdk + --project internal + --prefix ${{parameters.Prefix}} + --devopspath "\${{parameters.Prefix}}" + --path $(System.DefaultWorkingDirectory)/sdk + --endpoint Azure + --repository ${{parameters.Repository}} + --convention weekly + --agentpool Hosted + --branch refs/heads/$(DefaultBranch) + --patvar PATVAR + --debug + ${{parameters.TestsConventionOptions}} + displayName: Create Weekly (Multi-Cloud) Live Test pipelines for public repository + condition: and(succeeded(), ne('${{parameters.TestsConventionOptions}}','')) env: PATVAR: $(azuresdk-azure-sdk-devops-pipeline-generation-pat) @@ -132,6 +151,6 @@ steps: --no-schedule ${{parameters.TestsConventionOptions}} displayName: Create Live Test pipelines for private repository - condition: ne('${{parameters.TestsConventionOptions}}','') + condition: and(succeeded(), ne('${{parameters.TestsConventionOptions}}','')) env: PATVAR: $(azuresdk-azure-sdk-devops-pipeline-generation-pat) diff --git a/eng/pipelines/pipeline-generation.yml b/eng/pipelines/pipeline-generation.yml index 149873d446d..a98dcb222a9 100644 --- a/eng/pipelines/pipeline-generation.yml +++ b/eng/pipelines/pipeline-generation.yml @@ -142,7 +142,13 @@ jobs: - script: | $(Pipeline.Workspace)/pipeline-generator/pipeline-generator --organization https://dev.azure.com/azure-sdk --project internal --prefix $(Prefix) --path $(Pipeline.Workspace)/$(RepositoryName)/sdk --endpoint Azure --repository Azure/$(RepositoryName) --convention tests --agentpool Hosted --branch refs/heads/$(DefaultBranch) --patvar PATVAR --debug $(TestOptions) displayName: 'Generate test pipelines for: $(RepositoryName)' - condition: ne(variables['TestOptions'],'') + condition: and(succeeded(), ne(variables['TestOptions'],'')) + env: + PATVAR: $(azuresdk-azure-sdk-devops-pipeline-generation-pat) + - script: | + $(Pipeline.Workspace)/pipeline-generator/pipeline-generator --organization https://dev.azure.com/azure-sdk --project internal --prefix $(Prefix) --path $(Pipeline.Workspace)/$(RepositoryName)/sdk --endpoint Azure --repository Azure/$(RepositoryName) --convention weekly --agentpool Hosted --branch refs/heads/$(DefaultBranch) --patvar PATVAR --debug $(TestOptions) + displayName: 'Generate weekly test pipelines (multi-cloud) for: $(RepositoryName)' + condition: and(succeeded(), ne(variables['TestOptions'],'')) env: PATVAR: $(azuresdk-azure-sdk-devops-pipeline-generation-pat) diff --git a/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Azure.Sdk.Tools.PipelineGenerator.csproj b/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Azure.Sdk.Tools.PipelineGenerator.csproj index 9a99bce12d9..548d398667b 100644 --- a/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Azure.Sdk.Tools.PipelineGenerator.csproj +++ b/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Azure.Sdk.Tools.PipelineGenerator.csproj @@ -2,7 +2,7 @@ Exe - netcoreapp2.1 + netcoreapp3.1 true pipeline-generator diff --git a/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Conventions/IntegrationTestingPipelineConvention.cs b/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Conventions/IntegrationTestingPipelineConvention.cs index 9cc208ea197..25d0bdfcc84 100644 --- a/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Conventions/IntegrationTestingPipelineConvention.cs +++ b/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Conventions/IntegrationTestingPipelineConvention.cs @@ -16,7 +16,7 @@ public IntegrationTestingPipelineConvention(ILogger logger, PipelineGenerationCo { } - protected override string GetDefinitionName(SdkComponent component) + public override string GetDefinitionName(SdkComponent component) { return component.Variant == null ? $"{Context.Prefix} - {component.Name} - tests" : $"{Context.Prefix} - {component.Name} - tests.{component.Variant}"; } @@ -25,7 +25,7 @@ protected override async Task ApplyConventionAsync(BuildDefinition definit { var hasChanges = await base.ApplyConventionAsync(definition, component); - if (EnsureDefautPullRequestTrigger(definition, overrideYaml: true, securePipeline: true)) + if (EnsureDefaultPullRequestTrigger(definition, overrideYaml: true, securePipeline: true)) { hasChanges = true; } diff --git a/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Conventions/PipelineConvention.cs b/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Conventions/PipelineConvention.cs index b83203e8a61..3830d5f69d8 100644 --- a/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Conventions/PipelineConvention.cs +++ b/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Conventions/PipelineConvention.cs @@ -26,7 +26,7 @@ public PipelineConvention(ILogger logger, PipelineGenerationContext context) public abstract string SearchPattern { get; } - protected abstract string GetDefinitionName(SdkComponent component); + public abstract string GetDefinitionName(SdkComponent component); public async Task DeleteDefinitionAsync(SdkComponent component, CancellationToken cancellationToken) { @@ -80,7 +80,7 @@ public async Task CreateOrUpdateDefinitionAsync(SdkComponent co Logger.LogDebug("Applying convention to '{0}' definition.", definitionName); var hasChanges = await ApplyConventionAsync(definition, component); - if (hasChanges) + if (hasChanges || Context.OverwriteTriggers) { if (!Context.WhatIf) { @@ -313,7 +313,7 @@ protected virtual Task ApplyConventionAsync(BuildDefinition definition, Sd return Task.FromResult(hasChanges); } - protected bool EnsureDefautPullRequestTrigger(BuildDefinition definition, bool overrideYaml = true, bool securePipeline = true) + protected bool EnsureDefaultPullRequestTrigger(BuildDefinition definition, bool overrideYaml = true, bool securePipeline = true) { bool hasChanges = false; var prTriggers = definition.Triggers.OfType(); @@ -349,23 +349,24 @@ protected bool EnsureDefautPullRequestTrigger(BuildDefinition definition, bool o { if (overrideYaml) { - if (trigger.SettingsSourceType != 1 || - trigger.BranchFilters.Contains("+*")) + // Override what is in the yaml file and use what is in the pipeline definition + if (trigger.SettingsSourceType != 1) { - // Override what is in the yaml file and use what is in the pipeline definition trigger.SettingsSourceType = 1; - trigger.BranchFilters.Add("+*"); + hasChanges = true; } - } - else - { - if (trigger.SettingsSourceType != 2) + + if (!trigger.BranchFilters.Contains("+*")) { - // Pull settings from yaml - trigger.SettingsSourceType = 2; + trigger.BranchFilters.Add("+*"); hasChanges = true; } - + } + else if (trigger.SettingsSourceType != 2) + { + // Pull settings from yaml + trigger.SettingsSourceType = 2; + hasChanges = true; } if (trigger.RequireCommentsForNonTeamMembersOnly != false || trigger.Forks.AllowSecrets != securePipeline || @@ -377,7 +378,7 @@ protected bool EnsureDefautPullRequestTrigger(BuildDefinition definition, bool o trigger.Forks.Enabled = true; trigger.RequireCommentsForNonTeamMembersOnly = false; trigger.IsCommentRequiredForPullRequest = securePipeline; - + hasChanges = true; } } @@ -390,11 +391,12 @@ protected bool EnsureDefaultScheduledTrigger(BuildDefinition definition) bool hasChanges = false; var scheduleTriggers = definition.Triggers.OfType(); - // Only add the schedule trigger if one doesn't exist. - if (scheduleTriggers == default || !scheduleTriggers.Any()) + // Only add the schedule trigger if one doesn't exist. + if (scheduleTriggers == default || !scheduleTriggers.Any() || Context.OverwriteTriggers) { var computedSchedule = CreateScheduleFromDefinition(definition); + definition.Triggers.RemoveAll(e => e is ScheduleTrigger); definition.Triggers.Add(new ScheduleTrigger { Schedules = new List { computedSchedule } @@ -409,8 +411,9 @@ protected bool EnsureDefaultCITrigger(BuildDefinition definition) { bool hasChanges = false; var ciTrigger = definition.Triggers.OfType().SingleOrDefault(); - if (ciTrigger == null) + if (ciTrigger == null || Context.OverwriteTriggers) { + definition.Triggers.RemoveAll(e => e is ContinuousIntegrationTrigger); definition.Triggers.Add(new ContinuousIntegrationTrigger() { SettingsSourceType = 2 // Get CI trigger data from yaml file diff --git a/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Conventions/PullRequestValidationPipelineConvention.cs b/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Conventions/PullRequestValidationPipelineConvention.cs index d4f66468e91..6236b58aaa5 100644 --- a/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Conventions/PullRequestValidationPipelineConvention.cs +++ b/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Conventions/PullRequestValidationPipelineConvention.cs @@ -14,7 +14,7 @@ public PullRequestValidationPipelineConvention(ILogger logger, PipelineGeneratio { } - protected override string GetDefinitionName(SdkComponent component) + public override string GetDefinitionName(SdkComponent component) { return component.Variant == null ? $"{Context.Prefix} - {component.Name} - ci" : $"{Context.Prefix} - {component.Name} - ci.{component.Variant}"; } @@ -25,7 +25,7 @@ protected override async Task ApplyConventionAsync(BuildDefinition definit { var hasChanges = await base.ApplyConventionAsync(definition, component); - if (EnsureDefautPullRequestTrigger(definition, overrideYaml: false, securePipeline: false)) + if (EnsureDefaultPullRequestTrigger(definition, overrideYaml: false, securePipeline: false)) { hasChanges = true; } diff --git a/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Conventions/UnifiedPipelineConvention.cs b/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Conventions/UnifiedPipelineConvention.cs index 2e9c51787a2..153188fc9a4 100644 --- a/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Conventions/UnifiedPipelineConvention.cs +++ b/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Conventions/UnifiedPipelineConvention.cs @@ -14,7 +14,7 @@ public UnifiedPipelineConvention(ILogger logger, PipelineGenerationContext conte { } - protected override string GetDefinitionName(SdkComponent component) + public override string GetDefinitionName(SdkComponent component) { return component.Variant == null ? $"{Context.Prefix} - {component.Name}" : $"{Context.Prefix} - {component.Name} - {component.Variant}"; } @@ -25,7 +25,7 @@ protected override async Task ApplyConventionAsync(BuildDefinition definit { var hasChanges = await base.ApplyConventionAsync(definition, component); - if (EnsureDefautPullRequestTrigger(definition, overrideYaml: true, securePipeline: true)) + if (EnsureDefaultPullRequestTrigger(definition, overrideYaml: true, securePipeline: true)) { hasChanges = true; } diff --git a/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Conventions/WeeklyIntegrationTestingPipelineConvention.cs b/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Conventions/WeeklyIntegrationTestingPipelineConvention.cs index d5156812340..d1a520feb67 100644 --- a/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Conventions/WeeklyIntegrationTestingPipelineConvention.cs +++ b/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Conventions/WeeklyIntegrationTestingPipelineConvention.cs @@ -9,7 +9,7 @@ public WeeklyIntegrationTestingPipelineConvention(ILogger logger, PipelineGenera { } - protected override string GetDefinitionName(SdkComponent component) + public override string GetDefinitionName(SdkComponent component) { var definitionName = $"{Context.Prefix} - {component.Name} - tests-weekly"; if (component.Variant != null) { @@ -23,10 +23,12 @@ protected override Schedule CreateScheduleFromDefinition(BuildDefinition definit var bucket = definition.Id % TotalBuckets; var startHours = bucket / BucketsPerHour; var startMinutes = bucket % BucketsPerHour; + var daysToBuild = new ScheduleDays[]{ ScheduleDays.Saturday, ScheduleDays.Sunday }; + var dayBucket = definition.Id % daysToBuild.Length; var schedule = new Schedule { - DaysToBuild = ScheduleDays.Saturday | ScheduleDays.Sunday, + DaysToBuild = daysToBuild[dayBucket], ScheduleOnlyWithChanges = true, StartHours = FirstSchedulingHour + startHours, StartMinutes = startMinutes * BucketSizeInMinutes, diff --git a/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/ExitCondition.cs b/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/ExitCondition.cs index 83ccf3398cf..2ad5d29e9cf 100644 --- a/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/ExitCondition.cs +++ b/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/ExitCondition.cs @@ -7,6 +7,7 @@ public enum ExitCondition Success = 0, Exception = 1, InvalidArguments = 2, - NoComponentsFound = 3 + NoComponentsFound = 3, + DuplicateComponentsFound = 4 } } \ No newline at end of file diff --git a/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/PipelineGenerationContext.cs b/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/PipelineGenerationContext.cs index 9f81fb65678..89f46486b50 100644 --- a/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/PipelineGenerationContext.cs +++ b/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/PipelineGenerationContext.cs @@ -26,18 +26,19 @@ public class PipelineGenerationContext private string devOpsPath; public PipelineGenerationContext( - string organization, - string project, - string patvar, - string endpoint, - string repository, - string branch, - string agentPool, + string organization, + string project, + string patvar, + string endpoint, + string repository, + string branch, + string agentPool, string[] variableGroups, string devOpsPath, - string prefix, + string prefix, bool whatIf, - bool noSchedule) + bool noSchedule, + bool overwriteTriggers) { this.organization = organization; this.project = project; @@ -51,16 +52,18 @@ public PipelineGenerationContext( this.Prefix = prefix; this.WhatIf = whatIf; this.NoSchedule = noSchedule; + this.OverwriteTriggers = overwriteTriggers; } public string Branch { get; } public string Prefix { get; } public bool WhatIf { get; } public bool NoSchedule { get; } + public bool OverwriteTriggers { get; } public int[] VariableGroups => this.variableGroups; public string DevOpsPath => string.IsNullOrEmpty(this.devOpsPath) ? Prefix : this.devOpsPath; - private int[] ParseIntArray(string[] strs) + private int[] ParseIntArray(string[] strs) => strs.Select(str => int.Parse(str)).ToArray(); private VssConnection cachedConnection; @@ -91,7 +94,7 @@ private async Task GetProjectClientAsync(CancellationToken ca } private TeamProjectReference cachedProjectReference; - + public async Task GetProjectReferenceAsync(CancellationToken cancellationToken) { if (cachedProjectReference == null) diff --git a/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Program.cs b/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Program.cs index 8b86c30236d..ad7d067bb1b 100644 --- a/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Program.cs +++ b/tools/pipeline-generator/Azure.Sdk.Tools.PipelineGenerator/Program.cs @@ -61,6 +61,7 @@ private static CommandLineApplication PrepareApplication(CancellationTokenSource var destroyOption = app.Option("--destroy", "Use this switch to delete the pipelines instead (DANGER!)", CommandOptionType.NoValue); var debugOption = app.Option("--debug", "Turn on debug level logging.", CommandOptionType.NoValue); var noScheduleOption = app.Option("--no-schedule", "Don't create any scheduled triggers.", CommandOptionType.NoValue); + var overwriteTriggersOption = app.Option("--overwrite-triggers", "Overwrite existing pipeline triggers (triggers may be manually modified, use with caution).", CommandOptionType.NoValue); app.OnExecute(() => { @@ -83,6 +84,7 @@ private static CommandLineApplication PrepareApplication(CancellationTokenSource openOption.HasValue(), destroyOption.HasValue(), noScheduleOption.HasValue(), + overwriteTriggersOption.HasValue(), cancellationTokenSource.Token ).Result; @@ -146,6 +148,7 @@ public async Task RunAsync( bool open, bool destroy, bool noSchedule, + bool overwriteTriggers, CancellationToken cancellationToken) { try @@ -167,7 +170,8 @@ public async Task RunAsync( devOpsPathValue, prefix, whatIf, - noSchedule + noSchedule, + overwriteTriggers ); var pipelineConvention = GetPipelineConvention(convention, context); @@ -180,6 +184,12 @@ public async Task RunAsync( } logger.LogInformation("Found {0} components", components.Count()); + + if (HasPipelineDefinitionNameDuplicates(pipelineConvention, components)) + { + return ExitCondition.DuplicateComponentsFound; + } + foreach (var component in components) { logger.LogInformation("Processing component '{0}' in '{1}'.", component.Name, component.Path); @@ -236,5 +246,36 @@ private IEnumerable ScanForComponents(string path, string searchPa var components = scanner.Scan(scanDirectory, searchPattern); return components; } + + private bool HasPipelineDefinitionNameDuplicates(PipelineConvention convention, IEnumerable components) + { + var pipelineNames = new Dictionary(); + var duplicates = new HashSet(); + + foreach (var component in components) + { + var definitionName = convention.GetDefinitionName(component); + if (pipelineNames.TryGetValue(definitionName, out var duplicate)) + { + duplicates.Add(duplicate); + duplicates.Add(component); + } + else + { + pipelineNames.Add(definitionName, component); + } + } + + if (duplicates.Count > 0) { + logger.LogError("Found multiple pipeline definitions that will result in name collisions. This can happen when nested directory names are the same."); + logger.LogError("Suggested fix: add a 'variant' to the yaml filename, e.g. 'sdk/keyvault/internal/ci.yml' => 'sdk/keyvault/internal/ci.keyvault.yml'"); + var paths = duplicates.Select(d => $"'{d.RelativeYamlPath}'"); + logger.LogError($"Pipeline definitions affected: {String.Join(", ", paths)}"); + + return true; + } + + return false; + } } }