Fix GeneratePackageVersions so that it correctly respects IncludePackages and ExcludePackages

tracer/build/_build/Build.Utilities.cs

@@ -230,30 +230,38 @@ partial class Build
            var testDir = Solution.GetProject(Projects.ClrProfilerIntegrationTests).Directory;
            var dependabotFolder = TracerDirectory / "dependabot" / "integrations";
            var definitionsFile = BuildDirectory / FileNames.DefinitionsJson;
-           var currentDependencies = DependabotFileManager.GetCurrentlyTestedVersions(dependabotFolder);
-           Logger.Information("Found {CurrentDependenciesCount} existing dependencies", currentDependencies.Count);
-           var excludedFromUpdates = ((IncludePackages, ExcludePackages) switch
-                                         {
-                                             (_, { } exclude) => currentDependencies.Where(x => ExcludePackages.Contains(x.NugetName, StringComparer.OrdinalIgnoreCase)),
-                                             ({ } include, _) => currentDependencies.Where(x => !IncludePackages.Contains(x.NugetName, StringComparer.OrdinalIgnoreCase)),
-                                             _ => Enumerable.Empty<(string NugetName, Version LatestTestedVersion)>()
-                                         }).ToDictionary(x => x.NugetName, x => x.LatestTestedVersion, StringComparer.OrdinalIgnoreCase);
-
-           foreach (var dep in excludedFromUpdates)
-           {
-               Logger.Information("Excluding package {NugetName} from update. Fixing at {Version}", dep.Key, dep.Value);
-           }
+           var supportedVersionsPath = BuildDirectory / "supported_versions.json";
 
-           var versionGenerator = new PackageVersionGenerator(TracerDirectory, testDir, excludedFromUpdates);
+           // Build the shouldQueryNuGet predicate from include/exclude filters
+           Func<string, bool> shouldUpdatePackage = (IncludePackages, ExcludePackages) switch
+           {
+               ({ } include, _) => name => include.Contains(name, StringComparer.OrdinalIgnoreCase),
+               (_, { } exclude) => name => !exclude.Contains(name, StringComparer.OrdinalIgnoreCase),
+               _ => _ => true
+           };
+
+           // Load caches for both pipelines
+           var cacheFilePath = BuildDirectory / "nuget_version_cache.json";
+           var previousVersionCache = await NuGetVersionCache.Load(cacheFilePath);
+           Logger.Information("Loaded NuGet version cache with {Count} entries", previousVersionCache.Count);
+           var previousSupportedVersions = await GenerateSupportMatrix.LoadPreviousVersions(supportedVersionsPath);
+           Logger.Information("Loaded previous supported versions with {Count} entries", previousSupportedVersions.Count);
+
+           // Pipeline A: generate .g.props/.g.cs files
+           var versionGenerator = new PackageVersionGenerator(TracerDirectory, testDir, shouldUpdatePackage, previousVersionCache);
            var testedVersions = await versionGenerator.GenerateVersions(Solution);
+           await NuGetVersionCache.Save(cacheFilePath, versionGenerator.VersionCache);
 
            var assemblies = MonitoringHomeDirectory
                            .GlobFiles("**/Datadog.Trace.dll")
                            .Select(x => x.ToString())
                            .ToList();
 
            var integrations = GenerateIntegrationDefinitions.GetAllIntegrations(assemblies, definitionsFile);
-           var distinctIntegrations = await DependabotFileManager.BuildDistinctIntegrationMaps(integrations, testedVersions);
+
+           // Pipeline B: generate dependabot files + supported_versions.json
+           var distinctIntegrations = await DependabotFileManager.BuildDistinctIntegrationMaps(
+               integrations, testedVersions, shouldUpdatePackage, previousSupportedVersions);
 
            await DependabotFileManager.UpdateIntegrations(dependabotFolder, distinctIntegrations);
 

tracer/build/_build/GeneratePackageVersions/GenerateSupportMatrix.cs

@@ -13,7 +13,7 @@ namespace GeneratePackageVersions;
 /// <summary>
 /// Used to generate the matrix of dependencies that we support
 /// </summary>
-internal static class GenerateSupportMatrix
+public static class GenerateSupportMatrix
 {
     public static async Task GenerateInstrumentationSupportMatrix(
         string outputPath,
@@ -63,6 +63,43 @@ public static async Task GenerateInstrumentationSupportMatrix(
         await JsonSerializer.SerializeAsync(file, toWrite, jsonSerializerOptions );
     }
 
+    /// <summary>
+    /// Load the previously generated supported_versions.json and return a lookup
+    /// keyed by (assemblyName, packageName), mapping to its <see cref="SupportedNuGetPackage"/> data.
+    /// The composite key ensures that integrations sharing a NuGet package but with different
+    /// support bounds each get their own entry.
+    /// Returns an empty dictionary if the file doesn't exist.
+    /// </summary>
+    public static async Task<Dictionary<(string AssemblyName, string PackageName), SupportedNuGetPackage>> LoadPreviousVersions(string path)
+    {
+        if (!File.Exists(path))
+        {
+            return new Dictionary<(string, string), SupportedNuGetPackage>();
+        }
+
+        var JsonOptions = new JsonSerializerOptions
+        {
+            PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
+        };
+        await using var openStream = File.OpenRead(path);
+        var integrations = await JsonSerializer.DeserializeAsync<List<Integration>>(openStream, JsonOptions);
+
+        var result = new Dictionary<(string AssemblyName, string PackageName), SupportedNuGetPackage>();
+        if (integrations is null)
+        {
+            return result;
+        }
+
+        foreach (var integration in integrations)
+        {
+            foreach (var package in integration.Packages)
+            {
+                result[(integration.AssemblyName, package.Name)] = package;
+            }
+        }
+
+        return result;
+    }
 
     public class Integration
     {
@@ -91,7 +128,7 @@ public class Integration
         /// If applicable, the name of the NuGet package in which we expect to find the assemblies.
         /// If empty, the package is not available in NuGet packages (e.g. installed as part of the framework)
         /// </summary>
-        public List<SupportedNuGetPackage> Packages { get; } = new();
+        public List<SupportedNuGetPackage> Packages { get; set; } = new();
     }
 
     public class SupportedNuGetPackage

tracer/build/_build/GeneratePackageVersions/NuGetPackageHelper.cs

@@ -16,35 +16,56 @@ namespace GeneratePackageVersions
 {
     public class NuGetPackageHelper
     {
-        public static async Task<IEnumerable<string>> GetNugetPackageVersions(IPackageVersionEntry entry)
+        /// <summary>
+        /// Returns all available versions for a package, unfiltered by version range.
+        /// Suitable for caching since the result is independent of any specific entry's version bounds.
+        /// </summary>
+        public static async Task<List<string>> GetAllNugetPackageVersions(string packageName)
         {
-            var searchMetadata = await GetPackageMetadatas(entry);
+            var searchMetadata = await GetPackageMetadatas(packageName);
 
-            SemanticVersion minSemanticVersion, maxSemanticVersionExclusive;
+            var packageVersions = new List<string>();
+            foreach (var md in searchMetadata)
+            {
+                if (md.Identity.HasVersion)
+                {
+                    packageVersions.Add(md.Identity.Version.ToNormalizedString());
+                }
+            }
 
-            if (!SemanticVersion.TryParse(entry.MinVersion, out minSemanticVersion))
+            return packageVersions;
+        }
+
+        /// <summary>
+        /// Filters a list of version strings to only those within the entry's [MinVersion, MaxVersionExclusive) range.
+        /// </summary>
+        public static List<string> FilterVersions(IEnumerable<string> allVersions, IPackageVersionEntry entry)
+        {
+            if (!NuGetVersion.TryParse(entry.MinVersion, out var minVersion))
             {
-                throw new ArgumentException($"MinVersion {entry.MinVersion} in integration {entry.IntegrationName} could not be parsed into a NuGet Semantic Version");
+                throw new ArgumentException($"MinVersion {entry.MinVersion} in integration {entry.IntegrationName} could not be parsed into a NuGet Version");
             }
 
-            if (!SemanticVersion.TryParse(entry.MaxVersionExclusive, out maxSemanticVersionExclusive))
+            if (!NuGetVersion.TryParse(entry.MaxVersionExclusive, out var maxVersionExclusive))
             {
-                throw new ArgumentException($"MaxVersion {entry.MaxVersionExclusive} in integration {entry.IntegrationName} could not be parsed into a NuGet Semantic Version");
+                throw new ArgumentException($"MaxVersion {entry.MaxVersionExclusive} in integration {entry.IntegrationName} could not be parsed into a NuGet Version");
             }
 
-            List<string> packageVersions = new List<string>();
-            foreach (var md in searchMetadata)
+            var result = new List<string>();
+            foreach (var versionText in allVersions)
             {
-                if (md.Identity.HasVersion && md.Identity.Version.CompareTo(minSemanticVersion) >= 0 && md.Identity.Version.CompareTo(maxSemanticVersionExclusive) < 0)
+                if (NuGetVersion.TryParse(versionText, out var version)
+                    && version.CompareTo(minVersion) >= 0
+                    && version.CompareTo(maxVersionExclusive) < 0)
                 {
-                    packageVersions.Add(md.Identity.Version.ToNormalizedString());
+                    result.Add(versionText);
                 }
             }
 
-            return packageVersions;
+            return result;
         }
 
-        public static async Task<IEnumerable<IPackageSearchMetadata>> GetPackageMetadatas(IPackageVersionEntry entry)
+        public static async Task<IEnumerable<IPackageSearchMetadata>> GetPackageMetadatas(string packageName)
         {
             var packageSource = new PackageSource("https://api.nuget.org/v3/index.json");
 
@@ -59,7 +80,7 @@ public static async Task<IEnumerable<IPackageSearchMetadata>> GetPackageMetadata
             var logger = new Logger();
 
             var searchMetadata = await packageMetadataResource.GetMetadataAsync(
-                                                                     entry.NugetPackageSearchName,
+                                                                     packageName,
                                                                      includePrerelease: false,
                                                                      includeUnlisted: true,
                                                                      sourceCacheContext,
@@ -128,4 +149,4 @@ await Task.Run(() => Log(message.Level, message.Message))
             }
         }
     }
-}
+}

tracer/build/_build/GeneratePackageVersions/NuGetVersionCache.cs

@@ -0,0 +1,60 @@
+// <copyright file="NuGetVersionCache.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.Json;
+using System.Threading.Tasks;
+
+namespace GeneratePackageVersions;
+
+/// <summary>
+/// Manages a cache of NuGet package version lists so that excluded packages
+/// can reuse previously fetched data instead of querying NuGet again.
+/// </summary>
+public static class NuGetVersionCache
+{
+    private static readonly JsonSerializerOptions JsonOptions = new()
+    {
+        WriteIndented = true,
+    };
+
+    /// <summary>
+    /// Load the version cache from disk. Returns an empty dictionary if the file doesn't exist.
+    /// </summary>
+    public static async Task<Dictionary<string, List<string>>> Load(string path)
+    {
+        if (!File.Exists(path))
+        {
+            return new Dictionary<string, List<string>>();
+        }
+
+        await using var openStream = File.OpenRead(path);
+
+        var result = await JsonSerializer.DeserializeAsync<List<KeyValuePair<string, List<string>>>>(openStream, JsonOptions)
+                     ?? new List<KeyValuePair<string, List<string>>>();
+        return new Dictionary<string, List<string>>(result);
+    }
+
+    /// <summary>
+    /// Save the version cache to disk.
+    /// </summary>
+    public static async Task Save(string path, Dictionary<string, List<string>> cache)
+    {
+        // convert to a list to make sure it has deterministic ordering
+        var ordered = cache
+            .OrderBy(x => x.Key)
+            .Select(x => new KeyValuePair<string, IEnumerable<string>>(
+                x.Key,
+                x.Value
+                    .Select(Version.Parse)
+                    .OrderBy(version => version)
+                    .Select(version => version.ToString())));
+        await using var createStream = File.Create(path);
+        await JsonSerializer.SerializeAsync(createStream, ordered, JsonOptions);
+    }
+}

tracer/build/_build/GeneratePackageVersions/PackageVersionGenerator.cs

@@ -16,19 +16,27 @@ namespace GeneratePackageVersions
 {
     public class PackageVersionGenerator
     {
-        private readonly Dictionary<string, Version> FixedMaxDependencies;
+        private readonly Func<string, bool> _shouldQueryNuGet;
         private readonly AbsolutePath _definitionsFilePath;
         private readonly PackageGroup _latestMinors;
         private readonly PackageGroup _latestMajors;
         private readonly PackageGroup _latestSpecific;
         private readonly XunitStrategyFileGenerator _strategyGenerator;
 
+        /// <summary>
+        /// The version cache, populated during generation. Entries are added for every
+        /// package that is queried from NuGet. Pre-populated with cached entries on construction.
+        /// </summary>
+        public Dictionary<string, List<string>> VersionCache { get; }
+
         public PackageVersionGenerator(
             AbsolutePath tracerDirectory,
             AbsolutePath testProjectDirectory,
-            Dictionary<string, Version> fixedMaxDependencies)
+            Func<string, bool> shouldQueryNuGet,
+            Dictionary<string, List<string>> previousVersionCache)
         {
-            FixedMaxDependencies = fixedMaxDependencies;
+            _shouldQueryNuGet = shouldQueryNuGet;
+            VersionCache = new Dictionary<string, List<string>>(previousVersionCache);
             var propsDirectory = tracerDirectory / "build";
             _definitionsFilePath = tracerDirectory / "build" / "PackageVersionsGeneratorDefinitions.json";
             _latestMinors = new PackageGroup(propsDirectory, testProjectDirectory, "LatestMinors");
@@ -67,18 +75,27 @@ private async Task<List<TestedPackage>> RunFileGeneratorWithPackageEntries(IEnum
 
                 var requiresDockerDependency = project.RequiresDockerDependency().ToString();
 
-                var packageVersions = await NuGetPackageHelper.GetNugetPackageVersions(entry);
+                // Get all versions for this package (unfiltered), using cache when possible
+                List<string> allPackageVersions;
+                if (!_shouldQueryNuGet(entry.NugetPackageSearchName)
+                    && VersionCache.TryGetValue(entry.NugetPackageSearchName, out var cached))
+                {
+                    allPackageVersions = cached;
+                }
+                else
+                {
+                    allPackageVersions = await NuGetPackageHelper.GetAllNugetPackageVersions(entry.NugetPackageSearchName);
+                    VersionCache[entry.NugetPackageSearchName] = allPackageVersions;
+                }
+
+                // Filter to this entry's version range
+                var packageVersions = NuGetPackageHelper.FilterVersions(allPackageVersions, entry);
+
                 var orderedPackageVersions =
                     packageVersions
                        .Distinct()
                        .Select(versionText => new Version(versionText));
 
-                if (FixedMaxDependencies.TryGetValue(entry.NugetPackageSearchName, out var fixedVersion))
-                {
-                    orderedPackageVersions = orderedPackageVersions
-                       .Where(x => x <= fixedVersion);
-                }
-
                 var orderedWithFramework = (
                     from version in orderedPackageVersions.OrderBy(x => x)
                     from framework in supportedTargetFrameworks

tracer/build/_build/Honeypot/DependabotFileManager.cs

@@ -9,7 +9,6 @@
 using System.Linq;
 using System.Text;
 using System.Threading.Tasks;
-using System.Xml.Linq;
 using GeneratePackageVersions;
 using Nuke.Common.IO;
 using PrepareRelease;
@@ -69,15 +68,11 @@ public static async Task UpdateIntegrations(AbsolutePath honeypotDirectory, List
             }
         }
 
-        public static List<(string NugetName, Version LatestTestedVersion)> GetCurrentlyTestedVersions(AbsolutePath honeypotFolder)
-            => Directory.EnumerateFiles(honeypotFolder, "*.csproj", SearchOption.AllDirectories)
-                .Select(XElement.Load)
-                .Descendants("PackageReference")
-                .Select(x => ((string) x.Attribute("Include"), new Version(((string) x.Attribute("Version"))!)))
-                .Distinct()
-                .ToList();
-
-        public static async Task<List<IntegrationMap>> BuildDistinctIntegrationMaps(List<InstrumentedAssembly> targets, List<PackageVersionGenerator.TestedPackage> testedVersions)
+        public static async Task<List<IntegrationMap>> BuildDistinctIntegrationMaps(
+            List<InstrumentedAssembly> targets,
+            List<PackageVersionGenerator.TestedPackage> testedVersions,
+            Func<string, bool> shouldQueryNuGet,
+            Dictionary<(string AssemblyName, string PackageName), GeneratePackageVersions.GenerateSupportMatrix.SupportedNuGetPackage> previousSupportedVersions)
         {
             var distinctIntegrations = new List<IntegrationMap>();
 
@@ -113,7 +108,9 @@ await IntegrationMap.Create(
                         assemblyName: maxVersionTarget.TargetAssembly,
                         minSupportedVersion,
                         maxSupportedVersion,
-                        testedVersions));
+                        testedVersions,
+                        shouldQueryNuGet,
+                        previousSupportedVersions));
             }
 
             return distinctIntegrations;