Azure · benbp · Jan 6, 2022 · Jan 4, 2022 · Jan 4, 2022 · Jan 4, 2022
@@ -71,7 +71,26 @@ steps:
         --debug
         ${{parameters.TestsConventionOptions}}
       displayName: Create Live Test pipelines for public repository
-      condition: ne('${{parameters.TestsConventionOptions}}','')
+      condition: and(succeeded(), ne('${{parameters.TestsConventionOptions}}',''))
+      env:
+        PATVAR: $(azuresdk-azure-sdk-devops-pipeline-generation-pat)
+    - script: >
+        $(Pipeline.Workspace)/pipeline-generator/pipeline-generator
+        --organization https://dev.azure.com/azure-sdk
+        --project internal
+        --prefix ${{parameters.Prefix}}
+        --devopspath "\${{parameters.Prefix}}"
+        --path $(System.DefaultWorkingDirectory)/sdk
+        --endpoint Azure
+        --repository ${{parameters.Repository}}
+        --convention weekly
+        --agentpool Hosted
+        --branch refs/heads/$(DefaultBranch)
+        --patvar PATVAR
+        --debug
+        ${{parameters.TestsConventionOptions}}
+      displayName: Create Weekly (Multi-Cloud) Live Test pipelines for public repository
+      condition: and(succeeded(), ne('${{parameters.TestsConventionOptions}}',''))
       env:
         PATVAR: $(azuresdk-azure-sdk-devops-pipeline-generation-pat)
 
@@ -132,6 +151,6 @@ steps:
         --no-schedule
         ${{parameters.TestsConventionOptions}}
       displayName: Create Live Test pipelines for private repository
-      condition: ne('${{parameters.TestsConventionOptions}}','')
+      condition: and(succeeded(), ne('${{parameters.TestsConventionOptions}}',''))
       env:
         PATVAR: $(azuresdk-azure-sdk-devops-pipeline-generation-pat)
@@ -142,7 +142,13 @@ jobs:
   - script: |
       $(Pipeline.Workspace)/pipeline-generator/pipeline-generator --organization https://dev.azure.com/azure-sdk --project internal --prefix $(Prefix) --path $(Pipeline.Workspace)/$(RepositoryName)/sdk --endpoint Azure --repository Azure/$(RepositoryName) --convention tests --agentpool Hosted --branch refs/heads/$(DefaultBranch) --patvar PATVAR --debug $(TestOptions)
     displayName: 'Generate test pipelines for: $(RepositoryName)'
-    condition: ne(variables['TestOptions'],'')
+    condition: and(succeeded(), ne(variables['TestOptions'],''))
+    env:
+      PATVAR: $(azuresdk-azure-sdk-devops-pipeline-generation-pat)
+  - script: |
+      $(Pipeline.Workspace)/pipeline-generator/pipeline-generator --organization https://dev.azure.com/azure-sdk --project internal --prefix $(Prefix) --path $(Pipeline.Workspace)/$(RepositoryName)/sdk --endpoint Azure --repository Azure/$(RepositoryName) --convention weekly --agentpool Hosted --branch refs/heads/$(DefaultBranch) --patvar PATVAR --debug $(TestOptions)
+    displayName: 'Generate weekly test pipelines (multi-cloud) for: $(RepositoryName)'
+    condition: and(succeeded(), ne(variables['TestOptions'],''))
     env:
       PATVAR: $(azuresdk-azure-sdk-devops-pipeline-generation-pat)
 

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
 
     <PackAsTool>true</PackAsTool>
     <ToolCommandName>pipeline-generator</ToolCommandName>

@@ -16,7 +16,7 @@ public IntegrationTestingPipelineConvention(ILogger logger, PipelineGenerationCo
         {
         }
 
-        protected override string GetDefinitionName(SdkComponent component)
+        public override string GetDefinitionName(SdkComponent component)
         {
             return component.Variant == null ? $"{Context.Prefix} - {component.Name} - tests" : $"{Context.Prefix} - {component.Name} - tests.{component.Variant}";
         }
@@ -25,7 +25,7 @@ protected override async Task<bool> ApplyConventionAsync(BuildDefinition definit
         {
             var hasChanges = await base.ApplyConventionAsync(definition, component);
 
-            if (EnsureDefautPullRequestTrigger(definition, overrideYaml: true, securePipeline: true))
+            if (EnsureDefaultPullRequestTrigger(definition, overrideYaml: true, securePipeline: true))
             {
                 hasChanges = true;
             }

@@ -26,7 +26,7 @@ public PipelineConvention(ILogger logger, PipelineGenerationContext context)
 
         public abstract string SearchPattern { get; }
 
-        protected abstract string GetDefinitionName(SdkComponent component);
+        public abstract string GetDefinitionName(SdkComponent component);
 
         public async Task<BuildDefinition> DeleteDefinitionAsync(SdkComponent component, CancellationToken cancellationToken)
         {
@@ -80,7 +80,7 @@ public async Task<BuildDefinition> CreateOrUpdateDefinitionAsync(SdkComponent co
             Logger.LogDebug("Applying convention to '{0}' definition.", definitionName);
             var hasChanges = await ApplyConventionAsync(definition, component);
 
-            if (hasChanges)
+            if (hasChanges || Context.OverwriteTriggers)
             {
                 if (!Context.WhatIf)
                 {
@@ -313,7 +313,7 @@ protected virtual Task<bool> ApplyConventionAsync(BuildDefinition definition, Sd
             return Task.FromResult(hasChanges);
         }
 
-        protected bool EnsureDefautPullRequestTrigger(BuildDefinition definition, bool overrideYaml = true, bool securePipeline = true)
+        protected bool EnsureDefaultPullRequestTrigger(BuildDefinition definition, bool overrideYaml = true, bool securePipeline = true)
         {
             bool hasChanges = false;
             var prTriggers = definition.Triggers.OfType<PullRequestTrigger>();
@@ -349,23 +349,24 @@ protected bool EnsureDefautPullRequestTrigger(BuildDefinition definition, bool o
                 {
                     if (overrideYaml)
                     {
-                        if (trigger.SettingsSourceType != 1 ||
-                            trigger.BranchFilters.Contains("+*"))
+                        // Override what is in the yaml file and use what is in the pipeline definition
+                        if (trigger.SettingsSourceType != 1)
                         {
-                            // Override what is in the yaml file and use what is in the pipeline definition
                             trigger.SettingsSourceType = 1;
-                            trigger.BranchFilters.Add("+*");
+                            hasChanges = true;
                         }
-                    }
-                    else
-                    {
-                        if (trigger.SettingsSourceType != 2)
+
+                        if (!trigger.BranchFilters.Contains("+*"))
                         {
-                            // Pull settings from yaml
-                            trigger.SettingsSourceType = 2;
+                            trigger.BranchFilters.Add("+*");
                             hasChanges = true;
                         }
-
+                    }
+                    else if (trigger.SettingsSourceType != 2)
+                    {
+                        // Pull settings from yaml
+                        trigger.SettingsSourceType = 2;
+                        hasChanges = true;
                     }
                     if (trigger.RequireCommentsForNonTeamMembersOnly != false ||
                        trigger.Forks.AllowSecrets != securePipeline ||
@@ -377,7 +378,7 @@ protected bool EnsureDefautPullRequestTrigger(BuildDefinition definition, bool o
                         trigger.Forks.Enabled = true;
                         trigger.RequireCommentsForNonTeamMembersOnly = false;
                         trigger.IsCommentRequiredForPullRequest = securePipeline;
-   
+
                         hasChanges = true;
                     }
                 }
@@ -390,11 +391,12 @@ protected bool EnsureDefaultScheduledTrigger(BuildDefinition definition)
             bool hasChanges = false;
             var scheduleTriggers = definition.Triggers.OfType<ScheduleTrigger>();
 
-            // Only add the schedule trigger if one doesn't exist. 
-            if (scheduleTriggers == default || !scheduleTriggers.Any())
+            // Only add the schedule trigger if one doesn't exist.
+            if (scheduleTriggers == default || !scheduleTriggers.Any() || Context.OverwriteTriggers)
             {
                 var computedSchedule = CreateScheduleFromDefinition(definition);
 
+                definition.Triggers.RemoveAll(e => e is ScheduleTrigger);
                 definition.Triggers.Add(new ScheduleTrigger
                 {
                     Schedules = new List<Schedule> { computedSchedule }
@@ -409,8 +411,9 @@ protected bool EnsureDefaultCITrigger(BuildDefinition definition)
         {
             bool hasChanges = false;
             var ciTrigger = definition.Triggers.OfType<ContinuousIntegrationTrigger>().SingleOrDefault();
-            if (ciTrigger == null)
+            if (ciTrigger == null || Context.OverwriteTriggers)
             {
+                definition.Triggers.RemoveAll(e => e is ContinuousIntegrationTrigger);
                 definition.Triggers.Add(new ContinuousIntegrationTrigger()
                 {
                     SettingsSourceType = 2 // Get CI trigger data from yaml file

@@ -14,7 +14,7 @@ public PullRequestValidationPipelineConvention(ILogger logger, PipelineGeneratio
         {
         }
 
-        protected override string GetDefinitionName(SdkComponent component)
+        public override string GetDefinitionName(SdkComponent component)
         {
             return component.Variant == null ? $"{Context.Prefix} - {component.Name} - ci" : $"{Context.Prefix} - {component.Name} - ci.{component.Variant}";
         }
@@ -25,7 +25,7 @@ protected override async Task<bool> ApplyConventionAsync(BuildDefinition definit
         {
             var hasChanges = await base.ApplyConventionAsync(definition, component);
 
-            if (EnsureDefautPullRequestTrigger(definition, overrideYaml: false, securePipeline: false))
+            if (EnsureDefaultPullRequestTrigger(definition, overrideYaml: false, securePipeline: false))
             {
                 hasChanges = true;
             }

@@ -14,7 +14,7 @@ public UnifiedPipelineConvention(ILogger logger, PipelineGenerationContext conte
         {
         }
 
-        protected override string GetDefinitionName(SdkComponent component)
+        public override string GetDefinitionName(SdkComponent component)
         {
             return component.Variant == null ? $"{Context.Prefix} - {component.Name}" : $"{Context.Prefix} - {component.Name} - {component.Variant}";
         }
@@ -25,7 +25,7 @@ protected override async Task<bool> ApplyConventionAsync(BuildDefinition definit
         {
             var hasChanges = await base.ApplyConventionAsync(definition, component);
 
-            if (EnsureDefautPullRequestTrigger(definition, overrideYaml: true, securePipeline: true))
+            if (EnsureDefaultPullRequestTrigger(definition, overrideYaml: true, securePipeline: true))
             {
                 hasChanges = true;
             }

@@ -9,7 +9,7 @@ public WeeklyIntegrationTestingPipelineConvention(ILogger logger, PipelineGenera
         {
         }
 
-        protected override string GetDefinitionName(SdkComponent component)
+        public override string GetDefinitionName(SdkComponent component)
         {
             var definitionName = $"{Context.Prefix} - {component.Name} - tests-weekly";
             if (component.Variant != null) {
@@ -23,10 +23,12 @@ protected override Schedule CreateScheduleFromDefinition(BuildDefinition definit
             var bucket = definition.Id % TotalBuckets;
             var startHours = bucket / BucketsPerHour;
             var startMinutes = bucket % BucketsPerHour;
+            var daysToBuild = new ScheduleDays[]{ ScheduleDays.Saturday, ScheduleDays.Sunday };
+            var dayBucket = definition.Id % daysToBuild.Length;
 
             var schedule = new Schedule
             {
-                DaysToBuild = ScheduleDays.Saturday | ScheduleDays.Sunday,
+                DaysToBuild = daysToBuild[dayBucket],
                 ScheduleOnlyWithChanges = true,
                 StartHours = FirstSchedulingHour + startHours,
                 StartMinutes = startMinutes * BucketSizeInMinutes,

@@ -7,6 +7,7 @@ public enum ExitCondition
         Success = 0,
         Exception = 1,
         InvalidArguments = 2,
-        NoComponentsFound = 3
+        NoComponentsFound = 3,
+        DuplicateComponentsFound = 4
     }
 }
@@ -26,18 +26,19 @@ public class PipelineGenerationContext
         private string devOpsPath;
 
         public PipelineGenerationContext(
-            string organization, 
-            string project, 
-            string patvar, 
-            string endpoint, 
-            string repository, 
-            string branch, 
-            string agentPool, 
+            string organization,
+            string project,
+            string patvar,
+            string endpoint,
+            string repository,
+            string branch,
+            string agentPool,
             string[] variableGroups,
             string devOpsPath,
-            string prefix, 
+            string prefix,
             bool whatIf,
-            bool noSchedule)
+            bool noSchedule,
+            bool overwriteTriggers)
         {
             this.organization = organization;
             this.project = project;
@@ -51,16 +52,18 @@ public PipelineGenerationContext(
             this.Prefix = prefix;
             this.WhatIf = whatIf;
             this.NoSchedule = noSchedule;
+            this.OverwriteTriggers = overwriteTriggers;
         }
 
         public string Branch { get; }
         public string Prefix { get; }
         public bool WhatIf { get; }
         public bool NoSchedule { get; }
+        public bool OverwriteTriggers { get; }
         public int[] VariableGroups => this.variableGroups;
         public string DevOpsPath => string.IsNullOrEmpty(this.devOpsPath) ? Prefix : this.devOpsPath;
 
-        private int[] ParseIntArray(string[] strs) 
+        private int[] ParseIntArray(string[] strs)
             => strs.Select(str => int.Parse(str)).ToArray();
 
         private VssConnection cachedConnection;
@@ -91,7 +94,7 @@ private async Task<ProjectHttpClient> GetProjectClientAsync(CancellationToken ca
         }
 
         private TeamProjectReference cachedProjectReference;
-       
+
         public async Task<TeamProjectReference> GetProjectReferenceAsync(CancellationToken cancellationToken)
         {
             if (cachedProjectReference == null)

@@ -61,6 +61,7 @@ private static CommandLineApplication PrepareApplication(CancellationTokenSource
             var destroyOption = app.Option("--destroy", "Use this switch to delete the pipelines instead (DANGER!)", CommandOptionType.NoValue);
             var debugOption = app.Option("--debug", "Turn on debug level logging.", CommandOptionType.NoValue);
             var noScheduleOption = app.Option("--no-schedule", "Don't create any scheduled triggers.", CommandOptionType.NoValue);
+            var overwriteTriggersOption = app.Option("--overwrite-triggers", "Overwrite existing pipeline triggers (triggers may be manually modified, use with caution).", CommandOptionType.NoValue);
 
             app.OnExecute(() =>
             {
@@ -83,6 +84,7 @@ private static CommandLineApplication PrepareApplication(CancellationTokenSource
                     openOption.HasValue(),
                     destroyOption.HasValue(),
                     noScheduleOption.HasValue(),
+                    overwriteTriggersOption.HasValue(),
                     cancellationTokenSource.Token
                     ).Result;
 
@@ -146,6 +148,7 @@ public async Task<ExitCondition> RunAsync(
             bool open,
             bool destroy,
             bool noSchedule,
+            bool overwriteTriggers,
             CancellationToken cancellationToken)
         {
             try
@@ -167,7 +170,8 @@ public async Task<ExitCondition> RunAsync(
                     devOpsPathValue,
                     prefix,
                     whatIf,
-                    noSchedule
+                    noSchedule,
+                    overwriteTriggers
                     );
 
                 var pipelineConvention = GetPipelineConvention(convention, context);
@@ -180,6 +184,12 @@ public async Task<ExitCondition> RunAsync(
                 }
 
                 logger.LogInformation("Found {0} components", components.Count());
+
+                if (HasPipelineDefinitionNameDuplicates(pipelineConvention, components))
+                {
+                    return ExitCondition.DuplicateComponentsFound;
+                }
+
                 foreach (var component in components)
                 {
                     logger.LogInformation("Processing component '{0}' in '{1}'.", component.Name, component.Path);
@@ -236,5 +246,36 @@ private IEnumerable<SdkComponent> ScanForComponents(string path, string searchPa
             var components = scanner.Scan(scanDirectory, searchPattern);
             return components;
         }
+
+        private bool HasPipelineDefinitionNameDuplicates(PipelineConvention convention, IEnumerable<SdkComponent> components)
+        {
+            var pipelineNames = new Dictionary<string, SdkComponent>();
+            var duplicates = new HashSet<SdkComponent>();
+
+            foreach (var component in components)
+            {
+                var definitionName = convention.GetDefinitionName(component);
 return component.Variant == null ? $"{Context.Prefix} - {component.Name} - ci" : $"{Context.Prefix} - {component.Name} - ci.{component.Variant}"; 
 return component.Variant == null ? $"{Context.Prefix} - {component.Name} - ci" : $"{Context.Prefix} - {component.Name} - ci.{component.Variant}"; 
+                if (pipelineNames.TryGetValue(definitionName, out var duplicate))
+                {
+                    duplicates.Add(duplicate);
+                    duplicates.Add(component);
+                }
+                else
+                {
+                    pipelineNames.Add(definitionName, component);
+                }
+            }
+
+            if (duplicates.Count > 0) {
+                logger.LogError("Found multiple pipeline definitions that will result in name collisions. This can happen when nested directory names are the same.");
+                logger.LogError("Suggested fix: add a 'variant' to the yaml filename, e.g. 'sdk/keyvault/internal/ci.yml' => 'sdk/keyvault/internal/ci.keyvault.yml'");
+                var paths = duplicates.Select(d => $"'{d.RelativeYamlPath}'");
+                logger.LogError($"Pipeline definitions affected: {String.Join(", ", paths)}");
+
+                return true;
+            }
+
+            return false;
+        }
     }
 }