From db5b625a75c282a50226c18651c783c7a84ccc44 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Mon, 19 Jun 2023 10:33:26 -0700 Subject: [PATCH 1/4] Codeowners Team/User lookup --- .../CodeOwnersManualTester/CodeOwnerUtils.cs | 24 +++++++ .../CodeOwnersManualTester.csproj | 12 ++++ .../CodeOwnersManualTester/Program.cs | 38 +++++++++++ tools/code-owners-parser/CodeOwnersParser.sln | 6 ++ .../CodeOwnersParser/CodeownersEntry.cs | 64 ++++++++++++++++++- .../CodeOwnersParser/CodeownersFile.cs | 30 +++++++++ .../CodeOwnersParser/StorageConstants.cs | 15 +++++ 7 files changed, 186 insertions(+), 3 deletions(-) create mode 100644 tools/code-owners-parser/CodeOwnersManualTester/CodeOwnerUtils.cs create mode 100644 tools/code-owners-parser/CodeOwnersManualTester/CodeOwnersManualTester.csproj create mode 100644 tools/code-owners-parser/CodeOwnersManualTester/Program.cs create mode 100644 tools/code-owners-parser/CodeOwnersParser/StorageConstants.cs diff --git a/tools/code-owners-parser/CodeOwnersManualTester/CodeOwnerUtils.cs b/tools/code-owners-parser/CodeOwnersManualTester/CodeOwnerUtils.cs new file mode 100644 index 00000000000..1371fa4e872 --- /dev/null +++ b/tools/code-owners-parser/CodeOwnersManualTester/CodeOwnerUtils.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Azure.Sdk.Tools.CodeOwnersParser; + +namespace CodeOwnersManualTester +{ + internal class CodeOwnerUtils + { + /// + /// Wrapper function to load the CODEOWNERS file from a given path or URL and return + /// the list of codeowners entries. + /// + /// + /// List of CodeownersEntry + public static List GetCodeOwnerEntries(string codeOwnersFilePath) + { + Console.WriteLine($"Loading codeowners file, {codeOwnersFilePath}"); + return CodeownersFile.GetCodeownersEntriesFromFileOrUrl(codeOwnersFilePath); + } + } +} diff --git a/tools/code-owners-parser/CodeOwnersManualTester/CodeOwnersManualTester.csproj b/tools/code-owners-parser/CodeOwnersManualTester/CodeOwnersManualTester.csproj new file mode 100644 index 00000000000..b665948a462 --- /dev/null +++ b/tools/code-owners-parser/CodeOwnersManualTester/CodeOwnersManualTester.csproj @@ -0,0 +1,12 @@ + + + + Exe + net6.0 + enable + enable + + + + + diff --git a/tools/code-owners-parser/CodeOwnersManualTester/Program.cs b/tools/code-owners-parser/CodeOwnersManualTester/Program.cs new file mode 100644 index 00000000000..43171ce0a29 --- /dev/null +++ b/tools/code-owners-parser/CodeOwnersManualTester/Program.cs @@ -0,0 +1,38 @@ +using Azure.Sdk.Tools.CodeOwnersParser; + +namespace CodeOwnersManualTester +{ + internal class Program + { + static void Main(string[] args) + { + // Multiple repository list + List repositoryList = new List + { + "azure-sdk", + "azure-sdk-tools", + "azure-sdk-for-android", + "azure-sdk-for-c", + "azure-sdk-for-cpp", + "azure-sdk-for-go", + "azure-sdk-for-java", + "azure-sdk-for-js", + "azure-sdk-for-net", + "azure-sdk-for-python" + }; + // Single Repository List + //List repositoryList = new List + //{ + // "azure-sdk-for-java" + //}; + foreach (string repository in repositoryList) + { + string codeownersUrl = $"https://raw.githubusercontent.com/Azure/{repository}/main/.github/CODEOWNERS"; + List coEntries = CodeOwnerUtils.GetCodeOwnerEntries(codeownersUrl); + Console.WriteLine($"Total number of Codeowner entries: {coEntries.Count}"); + } + // Something to break on + Console.WriteLine("done"); + } + } +} diff --git a/tools/code-owners-parser/CodeOwnersParser.sln b/tools/code-owners-parser/CodeOwnersParser.sln index 8fa63a1ca1c..f0257053627 100644 --- a/tools/code-owners-parser/CodeOwnersParser.sln +++ b/tools/code-owners-parser/CodeOwnersParser.sln @@ -19,6 +19,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.CodeOwnersParser.Tests", "Azure.Sdk.Tools.CodeOwnersParser.Tests\Azure.Sdk.Tools.CodeOwnersParser.Tests.csproj", "{66C9FF6A-32DD-4C3C-ABE1-F1F58C1C8129}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodeOwnersManualTester", "CodeOwnersManualTester\CodeOwnersManualTester.csproj", "{4CDD76F7-4EFD-42A1-B39F-DB1728964ABF}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -41,6 +43,10 @@ Global {66C9FF6A-32DD-4C3C-ABE1-F1F58C1C8129}.Debug|Any CPU.Build.0 = Debug|Any CPU {66C9FF6A-32DD-4C3C-ABE1-F1F58C1C8129}.Release|Any CPU.ActiveCfg = Release|Any CPU {66C9FF6A-32DD-4C3C-ABE1-F1F58C1C8129}.Release|Any CPU.Build.0 = Release|Any CPU + {4CDD76F7-4EFD-42A1-B39F-DB1728964ABF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4CDD76F7-4EFD-42A1-B39F-DB1728964ABF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4CDD76F7-4EFD-42A1-B39F-DB1728964ABF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4CDD76F7-4EFD-42A1-B39F-DB1728964ABF}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/tools/code-owners-parser/CodeOwnersParser/CodeownersEntry.cs b/tools/code-owners-parser/CodeOwnersParser/CodeownersEntry.cs index 72f1d9fbf83..2066251f2b5 100644 --- a/tools/code-owners-parser/CodeOwnersParser/CodeownersEntry.cs +++ b/tools/code-owners-parser/CodeOwnersParser/CodeownersEntry.cs @@ -106,11 +106,69 @@ public void ParseOwnersAndPath(string line) line = ParsePath(line); line = RemoveCommentIfAny(line); - foreach (string author in SplitLine(line, OwnerSeparator).ToList()) + // If the line doesn't contain the OwnerSeparator AKA no owners, then the foreach loop below + // won't work. For example, the following line would end up causing "/sdk/communication" to + // be added as an owner when one is not listed + // /sdk/communication/ + if (line.Contains(OwnerSeparator)) { - if (!string.IsNullOrWhiteSpace(author)) - Owners.Add(author.Trim()); + foreach (string author in SplitLine(line, OwnerSeparator).ToList()) + { + if (!string.IsNullOrWhiteSpace(author)) + { + // If the author is a team, get the user list and add that to the Owners + if (!IsGitHubUserAlias(author)) + { + var teamUsers = GetUsersForTeam(author.Trim()); + // If the team is found in team user data, add the list of users to + // the owners and ensure the end result is a distinct list + if (teamUsers.Count > 0) + { + // The union of the two lists will ensure the result a distinct list + Owners = Owners.Union(teamUsers).ToList(); + } + // Else, the team user data did not contain an entry or there were no user + // for the team. In that case, just add the team to the list of authors + else + { + Owners.Add(author); + } + } + else + { + Owners.Add(author.Trim()); + } + } + } + } + else + { + Console.WriteLine($"Warning: CODEOWNERS line '{line}' does not have an owner entry."); + } + } + + private static List GetUsersForTeam(string teamName) + { + // The teamName in the codeowners file should be in the form /. + // The dictionary's team names do not contain the org so the org needs to + // be stripped off. Handle the case where the teamName passed in does and + // does not being with @org/ + string teamWithoutOrg = teamName.Trim(); + if (teamName.Contains('/')) + { + teamWithoutOrg = teamName.Split("/")[1].Trim(); + } + var teamUserDict = CodeownersFile.GetTeamUserData(); + if (teamUserDict != null) + { + if (teamUserDict.ContainsKey(teamWithoutOrg)) + { + Console.WriteLine($"Found team entry for {teamWithoutOrg}"); + return teamUserDict[teamWithoutOrg]; + } + Console.WriteLine($"Warning: TeamUserDictionary did not contain a team entry for {teamWithoutOrg}"); } + return new List(); } private static bool IsComment(string line) diff --git a/tools/code-owners-parser/CodeOwnersParser/CodeownersFile.cs b/tools/code-owners-parser/CodeOwnersParser/CodeownersFile.cs index 83f1dc376ee..6754fb2eac8 100644 --- a/tools/code-owners-parser/CodeOwnersParser/CodeownersFile.cs +++ b/tools/code-owners-parser/CodeOwnersParser/CodeownersFile.cs @@ -3,11 +3,15 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.Text.Json; namespace Azure.Sdk.Tools.CodeOwnersParser { public static class CodeownersFile { + + private static string teamUserBlobUri = $"https://{StorageConstants.AzureBlobAccountName}.blob.core.windows.net/{StorageConstants.AzureSdkWriteTeamsContainer}/{StorageConstants.AzureSdkWriteTeamsBlobName}"; + private static Dictionary>? teamUserDict = null; public static List GetCodeownersEntriesFromFileOrUrl( string codeownersFilePathOrUrl) { @@ -15,6 +19,32 @@ public static List GetCodeownersEntriesFromFileOrUrl( return GetCodeownersEntries(content); } + public static Dictionary>?GetTeamUserData() + { + if (null == teamUserDict) + { + Stopwatch stopWatch = new Stopwatch(); + stopWatch.Start(); + + string rawJson = FileHelpers.GetFileOrUrlContents(teamUserBlobUri); + stopWatch.Stop(); + // Get the elapsed time as a TimeSpan value. + TimeSpan ts = stopWatch.Elapsed; + + // Format and display the TimeSpan value. + string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", + ts.Hours, ts.Minutes, ts.Seconds, + ts.Milliseconds / 10); + Console.WriteLine($"Time to pull teamUserBlob: {elapsedTime}"); + var list = JsonSerializer.Deserialize>>>(rawJson); + if (null != list) + { + teamUserDict = list.ToDictionary((keyItem) => keyItem.Key, (valueItem) => valueItem.Value); + } + } + return teamUserDict; + } + public static List GetCodeownersEntries(string codeownersContent) { List entries = new List(); diff --git a/tools/code-owners-parser/CodeOwnersParser/StorageConstants.cs b/tools/code-owners-parser/CodeOwnersParser/StorageConstants.cs new file mode 100644 index 00000000000..e71428cfa60 --- /dev/null +++ b/tools/code-owners-parser/CodeOwnersParser/StorageConstants.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Azure.Sdk.Tools.CodeOwnersParser +{ + internal class StorageConstants + { + public const string AzureBlobAccountName = "azuresdkartifacts"; + public const string AzureSdkWriteTeamsContainer = "azure-sdk-write-teams"; + public const string AzureSdkWriteTeamsBlobName = "azure-sdk-write-teams-blob"; + } +} From 16d03bd87a9a56124d4d228890e1a9e88bc0615e Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Tue, 27 Jun 2023 10:51:44 -0700 Subject: [PATCH 2/4] Add a TeamUserHolder and pass that to Owners parsing. Remove the manual tester project --- .../Program.cs | 22 ++++-- .../CodeOwnersManualTester/CodeOwnerUtils.cs | 24 ------ .../CodeOwnersManualTester.csproj | 12 --- .../CodeOwnersManualTester/Program.cs | 38 --------- tools/code-owners-parser/CodeOwnersParser.sln | 6 -- .../CodeOwnersParser/CodeownersEntry.cs | 70 ++++++---------- .../CodeOwnersParser/CodeownersFile.cs | 53 ++++--------- .../DefaultStorageConstants.cs | 13 +++ .../CodeOwnersParser/StorageConstants.cs | 15 ---- .../CodeOwnersParser/TeamUserHolder.cs | 79 +++++++++++++++++++ 10 files changed, 144 insertions(+), 188 deletions(-) delete mode 100644 tools/code-owners-parser/CodeOwnersManualTester/CodeOwnerUtils.cs delete mode 100644 tools/code-owners-parser/CodeOwnersManualTester/CodeOwnersManualTester.csproj delete mode 100644 tools/code-owners-parser/CodeOwnersManualTester/Program.cs create mode 100644 tools/code-owners-parser/CodeOwnersParser/DefaultStorageConstants.cs delete mode 100644 tools/code-owners-parser/CodeOwnersParser/StorageConstants.cs create mode 100644 tools/code-owners-parser/CodeOwnersParser/TeamUserHolder.cs diff --git a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners/Program.cs b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners/Program.cs index 1661dda8d51..6a0c74aeb42 100644 --- a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners/Program.cs +++ b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners/Program.cs @@ -37,6 +37,7 @@ public static class Program /// Defaults to ".git". /// Example usage: ".git|foo|bar" /// + /// Override for the default URI where the team/storage blob data resides /// /// On STDOUT: The JSON representation of the matched CodeownersEntry. /// "new CodeownersEntry()" if no path in the CODEOWNERS data matches. @@ -48,7 +49,8 @@ public static int Main( string codeownersFilePathOrUrl, bool excludeNonUserAliases = false, string? targetDir = null, - string ignoredPathPrefixes = DefaultIgnoredPrefixes) + string ignoredPathPrefixes = DefaultIgnoredPrefixes, + string? teamStorageURI = null) { try { @@ -71,11 +73,13 @@ public static int Main( targetDir!, codeownersFilePathOrUrl, excludeNonUserAliases, - SplitIgnoredPathPrefixes()) + SplitIgnoredPathPrefixes(), + teamStorageURI) : GetCodeownersForSimplePath( targetPath, codeownersFilePathOrUrl, - excludeNonUserAliases); + excludeNonUserAliases, + teamStorageURI); string codeownersJson = JsonSerializer.Serialize( codeownersData, @@ -101,7 +105,8 @@ private static Dictionary GetCodeownersForGlobPath( string targetDir, string codeownersFilePathOrUrl, bool excludeNonUserAliases, - string[]? ignoredPathPrefixes = null) + string[]? ignoredPathPrefixes = null, + string? teamStorageURI=null) { ignoredPathPrefixes ??= Array.Empty(); @@ -110,7 +115,8 @@ private static Dictionary GetCodeownersForGlobPath( targetPath, targetDir, codeownersFilePathOrUrl, - ignoredPathPrefixes); + ignoredPathPrefixes, + teamStorageURI); if (excludeNonUserAliases) codeownersEntries.Values.ToList().ForEach(entry => entry.ExcludeNonUserAliases()); @@ -121,12 +127,14 @@ private static Dictionary GetCodeownersForGlobPath( private static CodeownersEntry GetCodeownersForSimplePath( string targetPath, string codeownersFilePathOrUrl, - bool excludeNonUserAliases) + bool excludeNonUserAliases, + string? teamStorageURI = null) { CodeownersEntry codeownersEntry = CodeownersFile.GetMatchingCodeownersEntry( targetPath, - codeownersFilePathOrUrl); + codeownersFilePathOrUrl, + teamStorageURI); if (excludeNonUserAliases) codeownersEntry.ExcludeNonUserAliases(); diff --git a/tools/code-owners-parser/CodeOwnersManualTester/CodeOwnerUtils.cs b/tools/code-owners-parser/CodeOwnersManualTester/CodeOwnerUtils.cs deleted file mode 100644 index 1371fa4e872..00000000000 --- a/tools/code-owners-parser/CodeOwnersManualTester/CodeOwnerUtils.cs +++ /dev/null @@ -1,24 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Azure.Sdk.Tools.CodeOwnersParser; - -namespace CodeOwnersManualTester -{ - internal class CodeOwnerUtils - { - /// - /// Wrapper function to load the CODEOWNERS file from a given path or URL and return - /// the list of codeowners entries. - /// - /// - /// List of CodeownersEntry - public static List GetCodeOwnerEntries(string codeOwnersFilePath) - { - Console.WriteLine($"Loading codeowners file, {codeOwnersFilePath}"); - return CodeownersFile.GetCodeownersEntriesFromFileOrUrl(codeOwnersFilePath); - } - } -} diff --git a/tools/code-owners-parser/CodeOwnersManualTester/CodeOwnersManualTester.csproj b/tools/code-owners-parser/CodeOwnersManualTester/CodeOwnersManualTester.csproj deleted file mode 100644 index b665948a462..00000000000 --- a/tools/code-owners-parser/CodeOwnersManualTester/CodeOwnersManualTester.csproj +++ /dev/null @@ -1,12 +0,0 @@ - - - - Exe - net6.0 - enable - enable - - - - - diff --git a/tools/code-owners-parser/CodeOwnersManualTester/Program.cs b/tools/code-owners-parser/CodeOwnersManualTester/Program.cs deleted file mode 100644 index 43171ce0a29..00000000000 --- a/tools/code-owners-parser/CodeOwnersManualTester/Program.cs +++ /dev/null @@ -1,38 +0,0 @@ -using Azure.Sdk.Tools.CodeOwnersParser; - -namespace CodeOwnersManualTester -{ - internal class Program - { - static void Main(string[] args) - { - // Multiple repository list - List repositoryList = new List - { - "azure-sdk", - "azure-sdk-tools", - "azure-sdk-for-android", - "azure-sdk-for-c", - "azure-sdk-for-cpp", - "azure-sdk-for-go", - "azure-sdk-for-java", - "azure-sdk-for-js", - "azure-sdk-for-net", - "azure-sdk-for-python" - }; - // Single Repository List - //List repositoryList = new List - //{ - // "azure-sdk-for-java" - //}; - foreach (string repository in repositoryList) - { - string codeownersUrl = $"https://raw.githubusercontent.com/Azure/{repository}/main/.github/CODEOWNERS"; - List coEntries = CodeOwnerUtils.GetCodeOwnerEntries(codeownersUrl); - Console.WriteLine($"Total number of Codeowner entries: {coEntries.Count}"); - } - // Something to break on - Console.WriteLine("done"); - } - } -} diff --git a/tools/code-owners-parser/CodeOwnersParser.sln b/tools/code-owners-parser/CodeOwnersParser.sln index f0257053627..8fa63a1ca1c 100644 --- a/tools/code-owners-parser/CodeOwnersParser.sln +++ b/tools/code-owners-parser/CodeOwnersParser.sln @@ -19,8 +19,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sdk.Tools.CodeOwnersParser.Tests", "Azure.Sdk.Tools.CodeOwnersParser.Tests\Azure.Sdk.Tools.CodeOwnersParser.Tests.csproj", "{66C9FF6A-32DD-4C3C-ABE1-F1F58C1C8129}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodeOwnersManualTester", "CodeOwnersManualTester\CodeOwnersManualTester.csproj", "{4CDD76F7-4EFD-42A1-B39F-DB1728964ABF}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -43,10 +41,6 @@ Global {66C9FF6A-32DD-4C3C-ABE1-F1F58C1C8129}.Debug|Any CPU.Build.0 = Debug|Any CPU {66C9FF6A-32DD-4C3C-ABE1-F1F58C1C8129}.Release|Any CPU.ActiveCfg = Release|Any CPU {66C9FF6A-32DD-4C3C-ABE1-F1F58C1C8129}.Release|Any CPU.Build.0 = Release|Any CPU - {4CDD76F7-4EFD-42A1-B39F-DB1728964ABF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {4CDD76F7-4EFD-42A1-B39F-DB1728964ABF}.Debug|Any CPU.Build.0 = Debug|Any CPU - {4CDD76F7-4EFD-42A1-B39F-DB1728964ABF}.Release|Any CPU.ActiveCfg = Release|Any CPU - {4CDD76F7-4EFD-42A1-B39F-DB1728964ABF}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/tools/code-owners-parser/CodeOwnersParser/CodeownersEntry.cs b/tools/code-owners-parser/CodeOwnersParser/CodeownersEntry.cs index 2066251f2b5..782adfd7e00 100644 --- a/tools/code-owners-parser/CodeOwnersParser/CodeownersEntry.cs +++ b/tools/code-owners-parser/CodeOwnersParser/CodeownersEntry.cs @@ -1,7 +1,10 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; namespace Azure.Sdk.Tools.CodeOwnersParser { @@ -45,7 +48,7 @@ public CodeownersEntry(string pathExpression, List owners) } private static string[] SplitLine(string line, char splitOn) - => line.Split(new char[] { splitOn }, StringSplitOptions.RemoveEmptyEntries); + => line.Split(new char[] { splitOn }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); public override string ToString() => $"HasWildcard:{ContainsWildcard} Expression:{PathExpression} " + @@ -85,14 +88,11 @@ private static IEnumerable ParseLabels(string line, string moniker) line = line[(colonPosition + 1)..].Trim(); foreach (string label in SplitLine(line, LabelSeparator).ToList()) { - if (!string.IsNullOrWhiteSpace(label)) - { - yield return label.Trim(); - } + yield return label; } } - public void ParseOwnersAndPath(string line) + public void ParseOwnersAndPath(string line, TeamUserHolder teamUserHolder) { if ( string.IsNullOrEmpty(line) @@ -114,31 +114,29 @@ public void ParseOwnersAndPath(string line) { foreach (string author in SplitLine(line, OwnerSeparator).ToList()) { - if (!string.IsNullOrWhiteSpace(author)) + // If the author is a team, get the user list and add that to the Owners + if (!IsGitHubUserAlias(author)) { - // If the author is a team, get the user list and add that to the Owners - if (!IsGitHubUserAlias(author)) + var teamUsers = teamUserHolder.GetUsersForTeam(author); + // If the team is found in team user data, add the list of users to + // the owners and ensure the end result is a distinct list + if (teamUsers.Count > 0) { - var teamUsers = GetUsersForTeam(author.Trim()); - // If the team is found in team user data, add the list of users to - // the owners and ensure the end result is a distinct list - if (teamUsers.Count > 0) - { - // The union of the two lists will ensure the result a distinct list - Owners = Owners.Union(teamUsers).ToList(); - } - // Else, the team user data did not contain an entry or there were no user - // for the team. In that case, just add the team to the list of authors - else - { - Owners.Add(author); - } + // The union of the two lists will ensure the result a distinct list + Owners = Owners.Union(teamUsers).ToList(); } + // Else, the team user data did not contain an entry or there were no user + // for the team. In that case, just add the team to the list of authors else { - Owners.Add(author.Trim()); + Owners.Add(author); } } + // If the entry isn't a team, then just add it + else + { + Owners.Add(author); + } } } else @@ -147,30 +145,6 @@ public void ParseOwnersAndPath(string line) } } - private static List GetUsersForTeam(string teamName) - { - // The teamName in the codeowners file should be in the form /. - // The dictionary's team names do not contain the org so the org needs to - // be stripped off. Handle the case where the teamName passed in does and - // does not being with @org/ - string teamWithoutOrg = teamName.Trim(); - if (teamName.Contains('/')) - { - teamWithoutOrg = teamName.Split("/")[1].Trim(); - } - var teamUserDict = CodeownersFile.GetTeamUserData(); - if (teamUserDict != null) - { - if (teamUserDict.ContainsKey(teamWithoutOrg)) - { - Console.WriteLine($"Found team entry for {teamWithoutOrg}"); - return teamUserDict[teamWithoutOrg]; - } - Console.WriteLine($"Warning: TeamUserDictionary did not contain a team entry for {teamWithoutOrg}"); - } - return new List(); - } - private static bool IsComment(string line) => line.StartsWith("#"); diff --git a/tools/code-owners-parser/CodeOwnersParser/CodeownersFile.cs b/tools/code-owners-parser/CodeOwnersParser/CodeownersFile.cs index 6754fb2eac8..110f001751a 100644 --- a/tools/code-owners-parser/CodeOwnersParser/CodeownersFile.cs +++ b/tools/code-owners-parser/CodeOwnersParser/CodeownersFile.cs @@ -10,43 +10,17 @@ namespace Azure.Sdk.Tools.CodeOwnersParser public static class CodeownersFile { - private static string teamUserBlobUri = $"https://{StorageConstants.AzureBlobAccountName}.blob.core.windows.net/{StorageConstants.AzureSdkWriteTeamsContainer}/{StorageConstants.AzureSdkWriteTeamsBlobName}"; - private static Dictionary>? teamUserDict = null; public static List GetCodeownersEntriesFromFileOrUrl( - string codeownersFilePathOrUrl) + string codeownersFilePathOrUrl, + string? teamStorageURI = null) { string content = FileHelpers.GetFileOrUrlContents(codeownersFilePathOrUrl); - return GetCodeownersEntries(content); + return GetCodeownersEntries(content, teamStorageURI); } - public static Dictionary>?GetTeamUserData() - { - if (null == teamUserDict) - { - Stopwatch stopWatch = new Stopwatch(); - stopWatch.Start(); - - string rawJson = FileHelpers.GetFileOrUrlContents(teamUserBlobUri); - stopWatch.Stop(); - // Get the elapsed time as a TimeSpan value. - TimeSpan ts = stopWatch.Elapsed; - - // Format and display the TimeSpan value. - string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", - ts.Hours, ts.Minutes, ts.Seconds, - ts.Milliseconds / 10); - Console.WriteLine($"Time to pull teamUserBlob: {elapsedTime}"); - var list = JsonSerializer.Deserialize>>>(rawJson); - if (null != list) - { - teamUserDict = list.ToDictionary((keyItem) => keyItem.Key, (valueItem) => valueItem.Value); - } - } - return teamUserDict; - } - - public static List GetCodeownersEntries(string codeownersContent) + public static List GetCodeownersEntries(string codeownersContent, string? teamStorageURI = null) { + TeamUserHolder teamUserHolder = new TeamUserHolder(teamStorageURI); List entries = new List(); // We are going to read line by line until we find a line that is not a comment @@ -58,7 +32,7 @@ public static List GetCodeownersEntries(string codeownersConten using StringReader sr = new StringReader(codeownersContent); while (sr.ReadLine() is { } line) { - entry = ProcessCodeownersLine(line, entry, entries); + entry = ProcessCodeownersLine(line, entry, entries, teamUserHolder); } return entries; @@ -66,9 +40,10 @@ public static List GetCodeownersEntries(string codeownersConten public static CodeownersEntry GetMatchingCodeownersEntry( string targetPath, - string codeownersFilePathOrUrl) + string codeownersFilePathOrUrl, + string? teamStorageURI = null) { - var codeownersEntries = GetCodeownersEntriesFromFileOrUrl(codeownersFilePathOrUrl); + var codeownersEntries = GetCodeownersEntriesFromFileOrUrl(codeownersFilePathOrUrl, teamStorageURI); return GetMatchingCodeownersEntry(targetPath, codeownersEntries); } @@ -76,11 +51,12 @@ public static Dictionary GetMatchingCodeownersEntries( GlobFilePath targetPath, string targetDir, string codeownersFilePathOrUrl, - string[]? ignoredPathPrefixes = null) + string[]? ignoredPathPrefixes = null, + string? teamStorageURI = null) { ignoredPathPrefixes ??= Array.Empty(); - var codeownersEntries = GetCodeownersEntriesFromFileOrUrl(codeownersFilePathOrUrl); + var codeownersEntries = GetCodeownersEntriesFromFileOrUrl(codeownersFilePathOrUrl, teamStorageURI); Dictionary codeownersEntriesByPath = targetPath .ResolveGlob(targetDir, ignoredPathPrefixes) @@ -104,7 +80,8 @@ public static CodeownersEntry GetMatchingCodeownersEntry( private static CodeownersEntry ProcessCodeownersLine( string line, CodeownersEntry entry, - List entries) + List entries, + TeamUserHolder teamUserHolder) { line = NormalizeLine(line); @@ -115,7 +92,7 @@ private static CodeownersEntry ProcessCodeownersLine( if (!IsCommentLine(line) || (IsCommentLine(line) && IsPlaceholderEntry(line))) { - entry.ParseOwnersAndPath(line); + entry.ParseOwnersAndPath(line, teamUserHolder); if (entry.IsValid) entries.Add(entry); diff --git a/tools/code-owners-parser/CodeOwnersParser/DefaultStorageConstants.cs b/tools/code-owners-parser/CodeOwnersParser/DefaultStorageConstants.cs new file mode 100644 index 00000000000..dfc25d9b957 --- /dev/null +++ b/tools/code-owners-parser/CodeOwnersParser/DefaultStorageConstants.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Azure.Sdk.Tools.CodeOwnersParser +{ + public class DefaultStorageConstants + { + public const string DefaultStorageURI = "https://azuresdkartifacts.blob.core.windows.net/azure-sdk-write-teams/azure-sdk-write-teams-blob"; + } +} diff --git a/tools/code-owners-parser/CodeOwnersParser/StorageConstants.cs b/tools/code-owners-parser/CodeOwnersParser/StorageConstants.cs deleted file mode 100644 index e71428cfa60..00000000000 --- a/tools/code-owners-parser/CodeOwnersParser/StorageConstants.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Azure.Sdk.Tools.CodeOwnersParser -{ - internal class StorageConstants - { - public const string AzureBlobAccountName = "azuresdkartifacts"; - public const string AzureSdkWriteTeamsContainer = "azure-sdk-write-teams"; - public const string AzureSdkWriteTeamsBlobName = "azure-sdk-write-teams-blob"; - } -} diff --git a/tools/code-owners-parser/CodeOwnersParser/TeamUserHolder.cs b/tools/code-owners-parser/CodeOwnersParser/TeamUserHolder.cs new file mode 100644 index 00000000000..f91a7d15434 --- /dev/null +++ b/tools/code-owners-parser/CodeOwnersParser/TeamUserHolder.cs @@ -0,0 +1,79 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading.Tasks; + +namespace Azure.Sdk.Tools.CodeOwnersParser +{ + public class TeamUserHolder + { + private string TeamUserStorageURI { get; set; } = DefaultStorageConstants.DefaultStorageURI; + private Dictionary>? _teamUserDict = null; + + public Dictionary> TeamUserDict + { + get + { + if (_teamUserDict == null) + { + _teamUserDict = GetTeamUserData(); + } + return _teamUserDict; + } + set + { + _teamUserDict = value; + } + } + + public TeamUserHolder(string? teamUserStorageURI) + { + if (!string.IsNullOrWhiteSpace(teamUserStorageURI)) + { + TeamUserStorageURI = teamUserStorageURI; + } + } + + private Dictionary> GetTeamUserData() + { + if (null == _teamUserDict) + { + string rawJson = FileHelpers.GetFileOrUrlContents(TeamUserStorageURI); + var list = JsonSerializer.Deserialize>>>(rawJson); + if (null != list) + { + return list.ToDictionary((keyItem) => keyItem.Key, (valueItem) => valueItem.Value); + } + Console.WriteLine($"Error! Unable to deserialize json team/user data. rawJson={rawJson}"); + return new Dictionary>(); + } + return _teamUserDict; + } + + public List GetUsersForTeam(string teamName) + { + // The teamName in the codeowners file should be in the form /. + // The dictionary's team names do not contain the org so the org needs to + // be stripped off. Handle the case where the teamName passed in does and + // does not being with @org/ + string teamWithoutOrg = teamName.Trim(); + if (teamWithoutOrg.Contains('/')) + { + teamWithoutOrg = teamWithoutOrg.Split("/")[1]; + } + if (TeamUserDict != null) + { + if (TeamUserDict.ContainsKey(teamWithoutOrg)) + { + Console.WriteLine($"Found team entry for {teamWithoutOrg}"); + return TeamUserDict[teamWithoutOrg]; + } + Console.WriteLine($"Warning: TeamUserDictionary did not contain a team entry for {teamWithoutOrg}"); + } + return new List(); + } + } +} From 5d04719c23846b7288faffb8c522287b98910468 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Tue, 27 Jun 2023 12:52:44 -0700 Subject: [PATCH 3/4] Add command line option to RetrieveCodeOwners to output json to a file --- .../Program.cs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners/Program.cs b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners/Program.cs index 6a0c74aeb42..07ca20b85b9 100644 --- a/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners/Program.cs +++ b/tools/code-owners-parser/Azure.Sdk.Tools.RetrieveCodeOwners/Program.cs @@ -37,7 +37,8 @@ public static class Program /// Defaults to ".git". /// Example usage: ".git|foo|bar" /// - /// Override for the default URI where the team/storage blob data resides + /// Override for the default URI where the team/storage blob data resides. + /// File to output the owners data to, will overwrite if the file exist. /// /// On STDOUT: The JSON representation of the matched CodeownersEntry. /// "new CodeownersEntry()" if no path in the CODEOWNERS data matches. @@ -50,7 +51,8 @@ public static int Main( bool excludeNonUserAliases = false, string? targetDir = null, string ignoredPathPrefixes = DefaultIgnoredPrefixes, - string? teamStorageURI = null) + string? teamStorageURI = null, + string? ownersDataOutputFile = null) { try { @@ -86,6 +88,16 @@ public static int Main( new JsonSerializerOptions { WriteIndented = true }); Console.WriteLine(codeownersJson); + + // If the output data file is specified, write the json to that. + if (!string.IsNullOrEmpty(ownersDataOutputFile)) + { + // False in the ctor is to overwrite, not append + using (StreamWriter outputFile = new StreamWriter(ownersDataOutputFile, false)) + { + outputFile.WriteLine(codeownersJson); + } + } return 0; string[] SplitIgnoredPathPrefixes() From def297903c782ffd7eaea7219d255ac03750e04e Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Thu, 29 Jun 2023 08:11:36 -0700 Subject: [PATCH 4/4] Add team/storage URI to the deserialize error message in the TeamUserHolder --- tools/code-owners-parser/CodeOwnersParser/TeamUserHolder.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/code-owners-parser/CodeOwnersParser/TeamUserHolder.cs b/tools/code-owners-parser/CodeOwnersParser/TeamUserHolder.cs index f91a7d15434..4a3a7c4ba60 100644 --- a/tools/code-owners-parser/CodeOwnersParser/TeamUserHolder.cs +++ b/tools/code-owners-parser/CodeOwnersParser/TeamUserHolder.cs @@ -47,7 +47,7 @@ private Dictionary> GetTeamUserData() { return list.ToDictionary((keyItem) => keyItem.Key, (valueItem) => valueItem.Value); } - Console.WriteLine($"Error! Unable to deserialize json team/user data. rawJson={rawJson}"); + Console.WriteLine($"Error! Unable to deserialize json team/user data from {TeamUserStorageURI}. rawJson={rawJson}"); return new Dictionary>(); } return _teamUserDict;