Add secrets search and bump DotUtils.MsBuild.BinlogRedactor.SensitiveDataDetector version

Directory.Packages.props

@@ -17,7 +17,7 @@
     <PackageVersion Include="Avalonia.Diagnostics" Version="$(AvaloniaVersion)" />
     <PackageVersion Include="Avalonia.Themes.Fluent" Version="$(AvaloniaVersion)" />
     <PackageVersion Include="DiffPlex" Version="1.7.2" />
-    <PackageVersion Include="DotUtils.MsBuild.BinlogRedactor.SensitiveDataDetector" Version="0.0.8" />
+    <PackageVersion Include="DotUtils.MsBuild.BinlogRedactor.SensitiveDataDetector" Version="0.0.9" />
     <PackageVersion Include="DotUtils.StreamUtils.Sources" Version="0.0.8" />
     <PackageVersion Include="FluentAssertions" Version="6.12.0" />
     <PackageVersion Include="GuiLabs.Language.Xml" Version="1.2.93" />

src/StructuredLogViewer/Controls/BuildControl.xaml.cs

@@ -15,6 +15,8 @@
 using System.Windows.Media;
 using System.Windows.Threading;
 using System.Xml;
+using DotUtils.MsBuild.SensitiveDataDetector;
+using Microsoft.Build.Experimental.ProjectCache;
 using Microsoft.Build.Framework;
 using Microsoft.Build.Logging.StructuredLogger;
 using Microsoft.Language.Xml;
@@ -79,6 +81,8 @@ public partial class BuildControl : UserControl
 
         private PropertiesAndItemsSearch propertiesAndItemsSearch;
 
+        private SecretsSearch secretsSearch;
+
         public BuildControl(Build build, string logFilePath)
         {
             InitializeComponent();
@@ -137,6 +141,7 @@ public BuildControl(Build build, string logFilePath)
             propertiesAndItemsControl.WatermarkDisplayed += UpdatePropertiesAndItemsWatermark;
             propertiesAndItemsControl.RecentItemsCategory = "PropertiesAndItems";
 
+            secretsSearch = new SecretsSearch(build);
             SetProjectContext(null);
 
             VirtualizingPanel.SetIsVirtualizing(treeView, SettingsService.EnableTreeViewVirtualization);
@@ -662,7 +667,8 @@ private void PopulateProjectGraph()
             "$csc",
             "$rar",
             "$import",
-            "$noimport"
+            "$noimport",
+            "$secret"
         };
 
         private static Inline MakeLink(string query, SearchAndResultsControl searchControl, string before = " \u2022 ", string after = "\r\n")
@@ -1024,11 +1030,30 @@ private object FindInFiles(string searchText, int maxResults, CancellationToken
                     return null;
                 }
 
-                var haystack = file.Value;
-                var resultsInFile = haystack.Find(searchText);
-                if (resultsInFile.Count > 0)
+                if (!string.IsNullOrEmpty(searchText) && searchText.StartsWith("$secret"))
                 {
-                    results.Add((file.Key, resultsInFile.Select(lineNumber => (lineNumber, haystack.GetLineText(lineNumber)))));
+                    var word = searchText.Replace("$secret", string.Empty).Trim();
+                    NodeQueryMatcher notMatcher = null;
+                    if (word.StartsWith("not(", StringComparison.OrdinalIgnoreCase) && word.EndsWith(")"))
+                    {
+                        word = word.Substring(4, word.Length - 5);
+                        notMatcher = new NodeQueryMatcher(word);
+                    }
+
+                    var searchResults = secretsSearch.SearchSecrets(file.Value.Text, notMatcher, maxResults);
+                    if (searchResults.Count > 0)
+                    {
+                        results.Add((file.Key, searchResults.Select(sr => (sr.Line - 1, sr.Secret))));
+                    }
+                }
+                else
+                {
+                    var haystack = file.Value;
+                    var resultsInFile = haystack.Find(searchText);
+                    if (resultsInFile.Count > 0)
+                    {
+                        results.Add((file.Key, resultsInFile.Select(lineNumber => (lineNumber, haystack.GetLineText(lineNumber)))));
+                    }
                 }
             }
 

src/StructuredLogViewer/MainWindow.xaml.cs

@@ -692,6 +692,7 @@ await System.Threading.Tasks.Task.Run(() =>
                 try
                 {
                     BuildAnalyzer.AnalyzeBuild(build);
+                    build.SearchExtensions.Add(new SecretsSearch(build));
                     build.SearchExtensions.Add(new NuGetSearch(build));
                 }
                 catch (Exception ex)

src/StructuredLogger.Utils/BinlogRedactor.cs

@@ -1,7 +1,6 @@
 using System;
 using System.IO;
 using System.Threading;
-using Microsoft.Build.Logging;
 using Microsoft.Build.Logging.StructuredLogger;
 using Microsoft.Build.SensitiveDataDetector;
 
@@ -61,7 +60,7 @@ public static void RedactSecrets(
                 sensitiveDataKind |= SensitiveDataKind.Username;
             }
 
-            ISensitiveDataRedactor sensitiveDataRedactor = SensitiveDataDetectorFactory.GetSecretsDetector(
+            ISensitiveDataRedactor sensitiveDataRedactor = SensitiveDataDetectorFactory.GetSecretsRedactor(
                 sensitiveDataKind,
                 redactorOptions.IdentifyReplacemenets,
                 redactorOptions.TokensToRedact);

src/StructuredLogger.Utils/SecretsSearch.cs

@@ -0,0 +1,130 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using DotUtils.MsBuild.SensitiveDataDetector;
+using Microsoft.Build.SensitiveDataDetector;
+using StructuredLogViewer;
+
+namespace Microsoft.Build.Logging.StructuredLogger
+{
+    public class SecretsSearch : ISearchExtension
+    {
+        private readonly Build _build;
+        private readonly Dictionary<SensitiveDataKind, ISensitiveDataDetector> _detectors;
+        private readonly Dictionary<string, Dictionary<SensitiveDataKind, List<SecretDescriptor>>> _secretCache = new();
+
+        public SecretsSearch(Build build)
+        {
+            _build = build ?? throw new ArgumentNullException(nameof(build));
+
+            _detectors = new()
+            {
+                { SensitiveDataKind.CommonSecrets, SensitiveDataDetectorFactory.GetSecretsDetector(SensitiveDataKind.CommonSecrets, false) },
+                { SensitiveDataKind.ExplicitSecrets, SensitiveDataDetectorFactory.GetSecretsDetector(SensitiveDataKind.ExplicitSecrets, false) },
+                { SensitiveDataKind.Username, SensitiveDataDetectorFactory.GetSecretsDetector(SensitiveDataKind.Username, false) }
+            };
+        }
+
+        public bool TryGetResults(NodeQueryMatcher matcher, IList<SearchResult> results, int maxResults)
+        {
+            if (string.Equals(matcher.TypeKeyword, "secret", StringComparison.OrdinalIgnoreCase))
+            {
+                var activeDetectors = GetActiveDetectors(matcher.NotMatchers);
+                var foundResults = ScanForSecrets(_build.StringTable.Instances, activeDetectors, maxResults);
+
+                foreach (var result in foundResults)
+                {
+                    results.Add(result);
+                }
+
+                return true;
+            }
+
+            return false;
+        }
+
+        public List<SecretDescriptor> SearchSecrets(string text, NodeQueryMatcher? matcher, int maxResults)
+        {
+            var activeDetectors = GetActiveDetectors(matcher == null ? [] : [matcher]);
+
+            var secrets = DetectSecrets(text, activeDetectors);
+
+            return secrets.Take(maxResults).ToList();
+        }
+
+        private IEnumerable<SearchResult> ScanForSecrets(IEnumerable<string> stringsPool, Dictionary<SensitiveDataKind, ISensitiveDataDetector> detectors, int maxResults)
+        {
+            var secretsSet = new HashSet<string>();
+            foreach (var text in stringsPool)
+            {
+                var secretResults = DetectSecrets(text, detectors);
+                foreach (SecretDescriptor secretDescriptor in secretResults)
+                {
+                    secretsSet.Add(secretDescriptor.Secret);
+                }
+            }
+
+            var results = new List<SearchResult>();
+            if (_build.SearchIndex is { } index)
+            {
+                foreach (var text in secretsSet)
+                {
+                    index.MaxResults = maxResults;
+                    index.MarkResultsInTree = false;
+                    IEnumerable<SearchResult> indexResults = index.FindNodes(text, CancellationToken.None);
+                    if (indexResults.Any())
+                    {
+                        results.AddRange(indexResults);
+                    }
+                }
+            }
+
+            return results;
+        }
+
+        private List<SecretDescriptor> DetectSecrets(string text, Dictionary<SensitiveDataKind, ISensitiveDataDetector> detectors)
+        {
+            if (_secretCache.TryGetValue(text, out var cachedSecrets))
+            {
+                return cachedSecrets
+                    .Where(kv => detectors.Any(d => d.Key == kv.Key))
+                    .SelectMany(kv => kv.Value)
+                    .ToList();
+            }
+
+            var results = new Dictionary<SensitiveDataKind, List<SecretDescriptor>>();
+
+            foreach (var detector in detectors)
+            {
+                Dictionary<SensitiveDataKind, List<SecretDescriptor>> detectedSecrets = detector.Value.Detect(text);
+                foreach (KeyValuePair<SensitiveDataKind, List<SecretDescriptor>> kv in detectedSecrets)
+                {
+                    if (kv.Value.Any())
+                    {
+                        results[kv.Key] = kv.Value;
+                    }
+                }
+            }
+
+            if (results.Any())
+            {
+                _secretCache[text] = results;
+            }
+
+            return results.Values.SelectMany(v => v).ToList();
+        }
+
+        private Dictionary<SensitiveDataKind, ISensitiveDataDetector> GetActiveDetectors(IList<NodeQueryMatcher> notMatchers)
+        {
+            if (!notMatchers.Any())
+            {
+                return new Dictionary<SensitiveDataKind, ISensitiveDataDetector>(_detectors);
+            }
+
+            return _detectors
+                .Where(d => !notMatchers.Any(m => Enum.TryParse<SensitiveDataKind>(m.Query, true, out var kind) && kind == d.Key))
+                .ToDictionary(k => k.Key, v => v.Value);
+        }
+    }
+}