From bbe4547f6a81b8a88d61d593dbccecb5b0f07880 Mon Sep 17 00:00:00 2001 From: Genevieve Warren <24882762+gewarren@users.noreply.github.com> Date: Wed, 12 Aug 2026 14:21:18 -0700 Subject: [PATCH 1/3] Add new tool for downloading reference assemblies --- PackageDownloader/PackageDownloader.csproj | 14 ++ PackageDownloader/Program.cs | 252 +++++++++++++++++++++ PackageDownloader/README.md | 17 ++ 3 files changed, 283 insertions(+) create mode 100644 PackageDownloader/PackageDownloader.csproj create mode 100644 PackageDownloader/Program.cs create mode 100644 PackageDownloader/README.md diff --git a/PackageDownloader/PackageDownloader.csproj b/PackageDownloader/PackageDownloader.csproj new file mode 100644 index 00000000..632bf9c7 --- /dev/null +++ b/PackageDownloader/PackageDownloader.csproj @@ -0,0 +1,14 @@ + + + + Exe + net10.0 + enable + enable + + + + + + + diff --git a/PackageDownloader/Program.cs b/PackageDownloader/Program.cs new file mode 100644 index 00000000..a782f11e --- /dev/null +++ b/PackageDownloader/Program.cs @@ -0,0 +1,252 @@ +using System.IO.Compression; +using System.Text.Json; +using System.Xml.Linq; +using NuGet.Versioning; + +const string NuGetIndexUrl = "https://packagefeedproxy.microsoft.io/nuget/v3/index.json"; +const string NetCoreRefPackageId = "Microsoft.NETCore.App.Ref"; +const string WindowsDesktopRefPackageId = "Microsoft.WindowsDesktop.App.Ref"; +const string RefTargetFramework = "net11.0"; +const string ShimReferencesUrl = "https://raw.githubusercontent.com/dotnet/runtime/v7.0.0-preview.1.22076.8/src/libraries/shims/netfxreference.props"; +const string dotnetDir = @"C:\Users\gewarren\binaries\dotnet"; +const string versionDir = "net-11.0"; +const string windowsDesktopDir = "windowsdesktop-11.0"; +string downloadDir = Path.Combine(Path.GetTempPath(), "ref-packages"); + +using HttpClient httpClient = new(); +httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("PackageDownloader/1.0"); + +ClearDirectory(downloadDir); + +string packageBaseAddress = await GetPackageBaseAddressAsync(httpClient); +HashSet shimDllExclusions = await GetShimDllExclusionsAsync(httpClient); +HashSet xmlFilesToCopy = BuildFileNameSet( + "Microsoft.Extensions.Caching.Abstractions.xml", + "Microsoft.Extensions.Configuration.Abstractions.xml", + "Microsoft.Extensions.DependencyInjection.Abstractions.xml", + "Microsoft.Extensions.Diagnostics.Abstractions.xml", + "Microsoft.Extensions.FileProviders.Abstractions.xml", + "Microsoft.Extensions.Hosting.Abstractions.xml", + "Microsoft.Extensions.Logging.Abstractions.xml", + "Microsoft.Extensions.Options.xml", + "Microsoft.Extensions.Primitives.xml", + "System.Formats.Asn1.xml", + "System.Linq.AsyncEnumerable.xml", + "System.Net.ServerSentEvents.xml", + "System.Reflection.DispatchProxy.xml", + "System.Text.RegularExpressions.xml"); + +string netCoreExtractDir = await DownloadAndExtractLatestPackageAsync(httpClient, packageBaseAddress, NetCoreRefPackageId, downloadDir); +string netCoreRefDir = GetRefDirectory(netCoreExtractDir); +string netCoreDestination = Path.Combine(dotnetDir, versionDir); +ClearDirectory(netCoreDestination); +CopyFiles(netCoreRefDir, netCoreDestination, "*.dll", shimDllExclusions); +CopyNamedFiles(netCoreRefDir, netCoreDestination, xmlFilesToCopy); + +string windowsDesktopExtractDir = await DownloadAndExtractLatestPackageAsync(httpClient, packageBaseAddress, WindowsDesktopRefPackageId, downloadDir); +string windowsDesktopRefDir = GetRefDirectory(windowsDesktopExtractDir); +string windowsDesktopDestination = Path.Combine(dotnetDir, windowsDesktopDir); +ClearDirectory(windowsDesktopDestination); +CopyFiles( + windowsDesktopRefDir, + windowsDesktopDestination, + "*.dll", + new HashSet(StringComparer.OrdinalIgnoreCase) + { + "Microsoft.VisualBasic.dll", + "System.Drawing.dll" + }); + +// Copy System.Security.Cryptography.dll to the dependencies directory for WindowsDesktop. +string cryptoSource = Path.Combine(netCoreDestination, "System.Security.Cryptography.dll"); +if (!File.Exists(cryptoSource)) +{ + throw new FileNotFoundException("Could not find the copied System.Security.Cryptography.dll dependency.", cryptoSource); +} + +string dependenciesDestination = Path.Combine(dotnetDir, "dependencies", windowsDesktopDir); +Directory.CreateDirectory(dependenciesDestination); +File.Copy(cryptoSource, Path.Combine(dependenciesDestination, Path.GetFileName(cryptoSource)), overwrite: true); +Console.WriteLine($"Copied {Path.GetFileName(cryptoSource)} to {dependenciesDestination}."); + +static async Task GetPackageBaseAddressAsync(HttpClient httpClient) +{ + using JsonDocument serviceIndex = await GetJsonDocumentAsync(httpClient, NuGetIndexUrl); + + foreach (JsonElement resource in serviceIndex.RootElement.GetProperty("resources").EnumerateArray()) + { + if (!resource.TryGetProperty("@type", out JsonElement typeElement) || + !resource.TryGetProperty("@id", out JsonElement idElement)) + { + continue; + } + + string? resourceType = typeElement.GetString(); + string? resourceId = idElement.GetString(); + if (resourceType?.Contains("PackageBaseAddress/3.0.0", StringComparison.Ordinal) == true && + !string.IsNullOrWhiteSpace(resourceId)) + { + return resourceId.TrimEnd('/') + "/"; + } + } + + throw new InvalidOperationException($"Could not find a PackageBaseAddress resource in {NuGetIndexUrl}."); +} + +static async Task> GetShimDllExclusionsAsync(HttpClient httpClient) +{ + string propsXml = await httpClient.GetStringAsync(ShimReferencesUrl); + XDocument props = XDocument.Parse(propsXml); + + HashSet exclusions = new(StringComparer.OrdinalIgnoreCase); + foreach (XElement reference in props.Descendants("NetFxReference")) + { + string? include = reference.Attribute("Include")?.Value; + if (!string.IsNullOrWhiteSpace(include)) + { + exclusions.Add(include + ".dll"); + } + } + + if (exclusions.Count == 0) + { + throw new InvalidOperationException($"No NetFxReference entries were found in {ShimReferencesUrl}."); + } + + return exclusions; +} + +static async Task DownloadAndExtractLatestPackageAsync( + HttpClient httpClient, + string packageBaseAddress, + string packageId, + string downloadDir) +{ + string packageIdLower = packageId.ToLowerInvariant(); + string versionsUrl = $"{packageBaseAddress}{packageIdLower}/index.json"; + using JsonDocument versionsDocument = await GetJsonDocumentAsync(httpClient, versionsUrl); + + List<(string Original, NuGetVersion Parsed)> versions = []; + foreach (JsonElement versionElement in versionsDocument.RootElement.GetProperty("versions").EnumerateArray()) + { + string? version = versionElement.GetString(); + if (string.IsNullOrWhiteSpace(version)) + { + continue; + } + + if (!NuGetVersion.TryParse(version, out NuGetVersion? parsedVersion)) + { + throw new InvalidOperationException($"The feed returned an invalid NuGet version for {packageId}: {version}"); + } + + versions.Add((version, parsedVersion)); + } + + if (versions.Count == 0) + { + throw new InvalidOperationException($"No versions were found for {packageId} at {versionsUrl}."); + } + + string latestVersion = versions.MaxBy(version => version.Parsed)!.Original; + string packageDir = Path.Combine(downloadDir, packageIdLower, latestVersion); + Directory.CreateDirectory(packageDir); + + string packagePath = Path.Combine(packageDir, $"{packageIdLower}.{latestVersion}.nupkg"); + string packageUrl = $"{packageBaseAddress}{packageIdLower}/{latestVersion}/{packageIdLower}.{latestVersion}.nupkg"; + + await using (Stream packageStream = await httpClient.GetStreamAsync(packageUrl)) + await using (FileStream packageFile = File.Create(packagePath)) + { + await packageStream.CopyToAsync(packageFile); + } + + string extractDir = Path.Combine(packageDir, "extracted"); + ClearDirectory(extractDir); + ZipFile.ExtractToDirectory(packagePath, extractDir, overwriteFiles: true); + + Console.WriteLine($"Downloaded and extracted {packageId} {latestVersion}."); + return extractDir; +} + +static async Task GetJsonDocumentAsync(HttpClient httpClient, string url) +{ + await using Stream stream = await httpClient.GetStreamAsync(url); + return await JsonDocument.ParseAsync(stream); +} + +static string GetRefDirectory(string extractDir) +{ + string refDir = Path.Combine(extractDir, "ref", RefTargetFramework); + if (!Directory.Exists(refDir)) + { + throw new DirectoryNotFoundException($"The package does not contain a ref/{RefTargetFramework} directory: {extractDir}"); + } + + return refDir; +} + +static void CopyFiles(string sourceDir, string destinationDir, string searchPattern, ISet excludedFileNames) +{ + Directory.CreateDirectory(destinationDir); + + int copiedCount = 0; + foreach (string sourcePath in Directory.EnumerateFiles(sourceDir, searchPattern, SearchOption.TopDirectoryOnly)) + { + string fileName = Path.GetFileName(sourcePath); + if (excludedFileNames.Contains(fileName)) + { + continue; + } + + File.Copy(sourcePath, Path.Combine(destinationDir, fileName), overwrite: true); + copiedCount++; + } + + Console.WriteLine($"Copied {copiedCount} {searchPattern} files from {sourceDir} to {destinationDir}."); +} + +static void CopyNamedFiles(string sourceDir, string destinationDir, ISet fileNamesToCopy) +{ + Directory.CreateDirectory(destinationDir); + + int copiedCount = 0; + foreach (string fileName in fileNamesToCopy) + { + string sourcePath = Path.Combine(sourceDir, fileName); + if (!File.Exists(sourcePath)) + { + throw new FileNotFoundException("Could not find an expected XML reference file.", sourcePath); + } + + File.Copy(sourcePath, Path.Combine(destinationDir, fileName), overwrite: true); + copiedCount++; + } + + Console.WriteLine($"Copied {copiedCount} named files from {sourceDir} to {destinationDir}."); +} + +static HashSet BuildFileNameSet(params string[] fileNames) +{ + HashSet names = new(StringComparer.OrdinalIgnoreCase); + foreach (string fileName in fileNames) + { + names.Add(fileName); + if (Path.GetExtension(fileName).Length == 0) + { + names.Add(fileName + ".xml"); + } + } + + return names; +} + +static void ClearDirectory(string directory) +{ + if (Directory.Exists(directory)) + { + Directory.Delete(directory, recursive: true); + } + + Directory.CreateDirectory(directory); +} diff --git a/PackageDownloader/README.md b/PackageDownloader/README.md new file mode 100644 index 00000000..b455ab03 --- /dev/null +++ b/PackageDownloader/README.md @@ -0,0 +1,17 @@ +# PackageDownloader + +`PackageDownloader` updates local reference assemblies used to feed the .NET API +reference documentation generator. It downloads the latest .NET Core and Windows Desktop +reference assemblies from the packages available on the NuGet V3 feed +at `https://packagefeedproxy.microsoft.io/nuget/v3/index.json`. + +## Run + +From the repository root: + +```powershell +dotnet run --project PackageDownloader\PackageDownloader.csproj +``` + +The program deletes and recreates its target output directories, so don't run it +while files in those directories are being edited or used by another process. From 6b603960cc7d864ac544954453571dd99463b099 Mon Sep 17 00:00:00 2001 From: Genevieve Warren <24882762+gewarren@users.noreply.github.com> Date: Wed, 12 Aug 2026 14:32:22 -0700 Subject: [PATCH 2/3] Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- PackageDownloader/Program.cs | 22 +++++++++++++++++----- PackageDownloader/README.md | 4 +++- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/PackageDownloader/Program.cs b/PackageDownloader/Program.cs index a782f11e..d0846537 100644 --- a/PackageDownloader/Program.cs +++ b/PackageDownloader/Program.cs @@ -14,8 +14,8 @@ string downloadDir = Path.Combine(Path.GetTempPath(), "ref-packages"); using HttpClient httpClient = new(); +httpClient.Timeout = TimeSpan.FromMinutes(5); httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("PackageDownloader/1.0"); - ClearDirectory(downloadDir); string packageBaseAddress = await GetPackageBaseAddressAsync(httpClient); @@ -177,13 +177,25 @@ static async Task GetJsonDocumentAsync(HttpClient httpClient, stri static string GetRefDirectory(string extractDir) { - string refDir = Path.Combine(extractDir, "ref", RefTargetFramework); - if (!Directory.Exists(refDir)) + string refRoot = Path.Combine(extractDir, "ref"); + if (!Directory.Exists(refRoot)) + { + throw new DirectoryNotFoundException($"The package does not contain a ref directory: {extractDir}"); + } + + string preferredRefDir = Path.Combine(refRoot, RefTargetFramework); + if (Directory.Exists(preferredRefDir)) + { + return preferredRefDir; + } + + string[] candidateDirs = Directory.GetDirectories(refRoot); + if (candidateDirs.Length == 1) { - throw new DirectoryNotFoundException($"The package does not contain a ref/{RefTargetFramework} directory: {extractDir}"); + return candidateDirs[0]; } - return refDir; + throw new DirectoryNotFoundException($"The package does not contain a ref/{RefTargetFramework} directory (and could not infer one) under: {extractDir}"); } static void CopyFiles(string sourceDir, string destinationDir, string searchPattern, ISet excludedFileNames) diff --git a/PackageDownloader/README.md b/PackageDownloader/README.md index b455ab03..677a2195 100644 --- a/PackageDownloader/README.md +++ b/PackageDownloader/README.md @@ -13,5 +13,7 @@ From the repository root: dotnet run --project PackageDownloader\PackageDownloader.csproj ``` -The program deletes and recreates its target output directories, so don't run it +The program deletes and recreates its target output directories, so don't run it while files in those directories are being edited or used by another process. + +Note: the output location/version are currently configured in `Program.cs` (see `dotnetDir`, `versionDir`, and `windowsDesktopDir`). The tool also downloads shim exclusions from the `netfxreference.props` file in dotnet/runtime to decide which assemblies to skip. From deec5b3d38caa0e19265d634811890c56c30fe39 Mon Sep 17 00:00:00 2001 From: Genevieve Warren <24882762+gewarren@users.noreply.github.com> Date: Wed, 12 Aug 2026 14:50:20 -0700 Subject: [PATCH 3/3] make some values configurable --- PackageDownloader/Program.cs | 67 ++++++++++++++++++++++++++++++------ PackageDownloader/README.md | 13 ++++++- 2 files changed, 68 insertions(+), 12 deletions(-) diff --git a/PackageDownloader/Program.cs b/PackageDownloader/Program.cs index d0846537..c7b44a84 100644 --- a/PackageDownloader/Program.cs +++ b/PackageDownloader/Program.cs @@ -6,16 +6,25 @@ const string NuGetIndexUrl = "https://packagefeedproxy.microsoft.io/nuget/v3/index.json"; const string NetCoreRefPackageId = "Microsoft.NETCore.App.Ref"; const string WindowsDesktopRefPackageId = "Microsoft.WindowsDesktop.App.Ref"; -const string RefTargetFramework = "net11.0"; +const string DefaultDotnetDir = @"C:\Users\gewarren\binaries\dotnet"; +const string DefaultMajorVersion = "11.0"; const string ShimReferencesUrl = "https://raw.githubusercontent.com/dotnet/runtime/v7.0.0-preview.1.22076.8/src/libraries/shims/netfxreference.props"; -const string dotnetDir = @"C:\Users\gewarren\binaries\dotnet"; -const string versionDir = "net-11.0"; -const string windowsDesktopDir = "windowsdesktop-11.0"; -string downloadDir = Path.Combine(Path.GetTempPath(), "ref-packages"); + +(Dictionary options, string? error) = ParseArguments(args); +if (error is not null) +{ + throw new ArgumentException(error); +} + +string dotnetDir = GetOption(options, "--dotnet-dir", DefaultDotnetDir); +string majorVersion = GetOption(options, "--version", DefaultMajorVersion); +string refTargetFramework = $"net{majorVersion}"; using HttpClient httpClient = new(); httpClient.Timeout = TimeSpan.FromMinutes(5); httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("PackageDownloader/1.0"); + +string downloadDir = Path.Combine(Path.GetTempPath(), "ref-packages"); ClearDirectory(downloadDir); string packageBaseAddress = await GetPackageBaseAddressAsync(httpClient); @@ -36,16 +45,21 @@ "System.Reflection.DispatchProxy.xml", "System.Text.RegularExpressions.xml"); +// Copy the .NET Core reference assemblies to the target directory. string netCoreExtractDir = await DownloadAndExtractLatestPackageAsync(httpClient, packageBaseAddress, NetCoreRefPackageId, downloadDir); -string netCoreRefDir = GetRefDirectory(netCoreExtractDir); -string netCoreDestination = Path.Combine(dotnetDir, versionDir); +string netCoreRefDir = GetRefDirectory(netCoreExtractDir, refTargetFramework); +string netCoreDestination = Path.Combine(dotnetDir, $"net-{majorVersion}"); + ClearDirectory(netCoreDestination); CopyFiles(netCoreRefDir, netCoreDestination, "*.dll", shimDllExclusions); CopyNamedFiles(netCoreRefDir, netCoreDestination, xmlFilesToCopy); +// Copy the Windows Desktop reference assemblies to the target directory. string windowsDesktopExtractDir = await DownloadAndExtractLatestPackageAsync(httpClient, packageBaseAddress, WindowsDesktopRefPackageId, downloadDir); -string windowsDesktopRefDir = GetRefDirectory(windowsDesktopExtractDir); +string windowsDesktopRefDir = GetRefDirectory(windowsDesktopExtractDir, refTargetFramework); +string windowsDesktopDir = $"windowsdesktop-{majorVersion}"; string windowsDesktopDestination = Path.Combine(dotnetDir, windowsDesktopDir); + ClearDirectory(windowsDesktopDestination); CopyFiles( windowsDesktopRefDir, @@ -69,6 +83,7 @@ File.Copy(cryptoSource, Path.Combine(dependenciesDestination, Path.GetFileName(cryptoSource)), overwrite: true); Console.WriteLine($"Copied {Path.GetFileName(cryptoSource)} to {dependenciesDestination}."); +#region Helper methods static async Task GetPackageBaseAddressAsync(HttpClient httpClient) { using JsonDocument serviceIndex = await GetJsonDocumentAsync(httpClient, NuGetIndexUrl); @@ -175,7 +190,7 @@ static async Task GetJsonDocumentAsync(HttpClient httpClient, stri return await JsonDocument.ParseAsync(stream); } -static string GetRefDirectory(string extractDir) +static string GetRefDirectory(string extractDir, string refTargetFramework) { string refRoot = Path.Combine(extractDir, "ref"); if (!Directory.Exists(refRoot)) @@ -183,7 +198,7 @@ static string GetRefDirectory(string extractDir) throw new DirectoryNotFoundException($"The package does not contain a ref directory: {extractDir}"); } - string preferredRefDir = Path.Combine(refRoot, RefTargetFramework); + string preferredRefDir = Path.Combine(refRoot, refTargetFramework); if (Directory.Exists(preferredRefDir)) { return preferredRefDir; @@ -195,7 +210,36 @@ static string GetRefDirectory(string extractDir) return candidateDirs[0]; } - throw new DirectoryNotFoundException($"The package does not contain a ref/{RefTargetFramework} directory (and could not infer one) under: {extractDir}"); + throw new DirectoryNotFoundException($"The package does not contain a ref/{refTargetFramework} directory (and could not infer one) under: {extractDir}"); +} + +static (Dictionary Options, string? Error) ParseArguments(string[] args) +{ + Dictionary options = new(StringComparer.OrdinalIgnoreCase); + for (int index = 0; index < args.Length; index++) + { + string option = args[index]; + if (option is not ("--dotnet-dir" or "--version")) + { + return (options, $"Unknown option '{option}'. Supported options: --dotnet-dir and --version ."); + } + + if (index + 1 >= args.Length || args[index + 1].StartsWith("--", StringComparison.Ordinal)) + { + return (options, $"Option '{option}' requires a value."); + } + + options[option] = args[++index]; + } + + return (options, null); +} + +static string GetOption(IReadOnlyDictionary options, string name, string defaultValue) +{ + return options.TryGetValue(name, out string? value) && !string.IsNullOrWhiteSpace(value) + ? value + : defaultValue; } static void CopyFiles(string sourceDir, string destinationDir, string searchPattern, ISet excludedFileNames) @@ -262,3 +306,4 @@ static void ClearDirectory(string directory) Directory.CreateDirectory(directory); } +#endregion diff --git a/PackageDownloader/README.md b/PackageDownloader/README.md index 677a2195..05b6da1d 100644 --- a/PackageDownloader/README.md +++ b/PackageDownloader/README.md @@ -13,7 +13,18 @@ From the repository root: dotnet run --project PackageDownloader\PackageDownloader.csproj ``` +Override the output directory and .NET version with named arguments: + +```powershell +dotnet run --project PackageDownloader\PackageDownloader.csproj -- --dotnet-dir C:\Users\me\binaries\dotnet --version 11.0 +``` + +Options: + +- `--dotnet-dir `: root output directory. Defaults to `C:\Users\gewarren\binaries\dotnet`. +- `--version `: .NET version used to select the package `ref/net` directory and name the output directories. Defaults to `11.0`. + The program deletes and recreates its target output directories, so don't run it while files in those directories are being edited or used by another process. -Note: the output location/version are currently configured in `Program.cs` (see `dotnetDir`, `versionDir`, and `windowsDesktopDir`). The tool also downloads shim exclusions from the `netfxreference.props` file in dotnet/runtime to decide which assemblies to skip. +The tool also downloads shim exclusions from the `netfxreference.props` file in dotnet/runtime to decide which assemblies to skip.