diff --git a/PackageDownloader/PackageDownloader.csproj b/PackageDownloader/PackageDownloader.csproj new file mode 100644 index 00000000..632bf9c7 --- /dev/null +++ b/PackageDownloader/PackageDownloader.csproj @@ -0,0 +1,14 @@ + + + + Exe + net10.0 + enable + enable + + + + + + + diff --git a/PackageDownloader/Program.cs b/PackageDownloader/Program.cs new file mode 100644 index 00000000..c7b44a84 --- /dev/null +++ b/PackageDownloader/Program.cs @@ -0,0 +1,309 @@ +using System.IO.Compression; +using System.Text.Json; +using System.Xml.Linq; +using NuGet.Versioning; + +const string NuGetIndexUrl = "https://packagefeedproxy.microsoft.io/nuget/v3/index.json"; +const string NetCoreRefPackageId = "Microsoft.NETCore.App.Ref"; +const string WindowsDesktopRefPackageId = "Microsoft.WindowsDesktop.App.Ref"; +const string DefaultDotnetDir = @"C:\Users\gewarren\binaries\dotnet"; +const string DefaultMajorVersion = "11.0"; +const string ShimReferencesUrl = "https://raw.githubusercontent.com/dotnet/runtime/v7.0.0-preview.1.22076.8/src/libraries/shims/netfxreference.props"; + +(Dictionary options, string? error) = ParseArguments(args); +if (error is not null) +{ + throw new ArgumentException(error); +} + +string dotnetDir = GetOption(options, "--dotnet-dir", DefaultDotnetDir); +string majorVersion = GetOption(options, "--version", DefaultMajorVersion); +string refTargetFramework = $"net{majorVersion}"; + +using HttpClient httpClient = new(); +httpClient.Timeout = TimeSpan.FromMinutes(5); +httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("PackageDownloader/1.0"); + +string downloadDir = Path.Combine(Path.GetTempPath(), "ref-packages"); +ClearDirectory(downloadDir); + +string packageBaseAddress = await GetPackageBaseAddressAsync(httpClient); +HashSet shimDllExclusions = await GetShimDllExclusionsAsync(httpClient); +HashSet xmlFilesToCopy = BuildFileNameSet( + "Microsoft.Extensions.Caching.Abstractions.xml", + "Microsoft.Extensions.Configuration.Abstractions.xml", + "Microsoft.Extensions.DependencyInjection.Abstractions.xml", + "Microsoft.Extensions.Diagnostics.Abstractions.xml", + "Microsoft.Extensions.FileProviders.Abstractions.xml", + "Microsoft.Extensions.Hosting.Abstractions.xml", + "Microsoft.Extensions.Logging.Abstractions.xml", + "Microsoft.Extensions.Options.xml", + "Microsoft.Extensions.Primitives.xml", + "System.Formats.Asn1.xml", + "System.Linq.AsyncEnumerable.xml", + "System.Net.ServerSentEvents.xml", + "System.Reflection.DispatchProxy.xml", + "System.Text.RegularExpressions.xml"); + +// Copy the .NET Core reference assemblies to the target directory. +string netCoreExtractDir = await DownloadAndExtractLatestPackageAsync(httpClient, packageBaseAddress, NetCoreRefPackageId, downloadDir); +string netCoreRefDir = GetRefDirectory(netCoreExtractDir, refTargetFramework); +string netCoreDestination = Path.Combine(dotnetDir, $"net-{majorVersion}"); + +ClearDirectory(netCoreDestination); +CopyFiles(netCoreRefDir, netCoreDestination, "*.dll", shimDllExclusions); +CopyNamedFiles(netCoreRefDir, netCoreDestination, xmlFilesToCopy); + +// Copy the Windows Desktop reference assemblies to the target directory. +string windowsDesktopExtractDir = await DownloadAndExtractLatestPackageAsync(httpClient, packageBaseAddress, WindowsDesktopRefPackageId, downloadDir); +string windowsDesktopRefDir = GetRefDirectory(windowsDesktopExtractDir, refTargetFramework); +string windowsDesktopDir = $"windowsdesktop-{majorVersion}"; +string windowsDesktopDestination = Path.Combine(dotnetDir, windowsDesktopDir); + +ClearDirectory(windowsDesktopDestination); +CopyFiles( + windowsDesktopRefDir, + windowsDesktopDestination, + "*.dll", + new HashSet(StringComparer.OrdinalIgnoreCase) + { + "Microsoft.VisualBasic.dll", + "System.Drawing.dll" + }); + +// Copy System.Security.Cryptography.dll to the dependencies directory for WindowsDesktop. +string cryptoSource = Path.Combine(netCoreDestination, "System.Security.Cryptography.dll"); +if (!File.Exists(cryptoSource)) +{ + throw new FileNotFoundException("Could not find the copied System.Security.Cryptography.dll dependency.", cryptoSource); +} + +string dependenciesDestination = Path.Combine(dotnetDir, "dependencies", windowsDesktopDir); +Directory.CreateDirectory(dependenciesDestination); +File.Copy(cryptoSource, Path.Combine(dependenciesDestination, Path.GetFileName(cryptoSource)), overwrite: true); +Console.WriteLine($"Copied {Path.GetFileName(cryptoSource)} to {dependenciesDestination}."); + +#region Helper methods +static async Task GetPackageBaseAddressAsync(HttpClient httpClient) +{ + using JsonDocument serviceIndex = await GetJsonDocumentAsync(httpClient, NuGetIndexUrl); + + foreach (JsonElement resource in serviceIndex.RootElement.GetProperty("resources").EnumerateArray()) + { + if (!resource.TryGetProperty("@type", out JsonElement typeElement) || + !resource.TryGetProperty("@id", out JsonElement idElement)) + { + continue; + } + + string? resourceType = typeElement.GetString(); + string? resourceId = idElement.GetString(); + if (resourceType?.Contains("PackageBaseAddress/3.0.0", StringComparison.Ordinal) == true && + !string.IsNullOrWhiteSpace(resourceId)) + { + return resourceId.TrimEnd('/') + "/"; + } + } + + throw new InvalidOperationException($"Could not find a PackageBaseAddress resource in {NuGetIndexUrl}."); +} + +static async Task> GetShimDllExclusionsAsync(HttpClient httpClient) +{ + string propsXml = await httpClient.GetStringAsync(ShimReferencesUrl); + XDocument props = XDocument.Parse(propsXml); + + HashSet exclusions = new(StringComparer.OrdinalIgnoreCase); + foreach (XElement reference in props.Descendants("NetFxReference")) + { + string? include = reference.Attribute("Include")?.Value; + if (!string.IsNullOrWhiteSpace(include)) + { + exclusions.Add(include + ".dll"); + } + } + + if (exclusions.Count == 0) + { + throw new InvalidOperationException($"No NetFxReference entries were found in {ShimReferencesUrl}."); + } + + return exclusions; +} + +static async Task DownloadAndExtractLatestPackageAsync( + HttpClient httpClient, + string packageBaseAddress, + string packageId, + string downloadDir) +{ + string packageIdLower = packageId.ToLowerInvariant(); + string versionsUrl = $"{packageBaseAddress}{packageIdLower}/index.json"; + using JsonDocument versionsDocument = await GetJsonDocumentAsync(httpClient, versionsUrl); + + List<(string Original, NuGetVersion Parsed)> versions = []; + foreach (JsonElement versionElement in versionsDocument.RootElement.GetProperty("versions").EnumerateArray()) + { + string? version = versionElement.GetString(); + if (string.IsNullOrWhiteSpace(version)) + { + continue; + } + + if (!NuGetVersion.TryParse(version, out NuGetVersion? parsedVersion)) + { + throw new InvalidOperationException($"The feed returned an invalid NuGet version for {packageId}: {version}"); + } + + versions.Add((version, parsedVersion)); + } + + if (versions.Count == 0) + { + throw new InvalidOperationException($"No versions were found for {packageId} at {versionsUrl}."); + } + + string latestVersion = versions.MaxBy(version => version.Parsed)!.Original; + string packageDir = Path.Combine(downloadDir, packageIdLower, latestVersion); + Directory.CreateDirectory(packageDir); + + string packagePath = Path.Combine(packageDir, $"{packageIdLower}.{latestVersion}.nupkg"); + string packageUrl = $"{packageBaseAddress}{packageIdLower}/{latestVersion}/{packageIdLower}.{latestVersion}.nupkg"; + + await using (Stream packageStream = await httpClient.GetStreamAsync(packageUrl)) + await using (FileStream packageFile = File.Create(packagePath)) + { + await packageStream.CopyToAsync(packageFile); + } + + string extractDir = Path.Combine(packageDir, "extracted"); + ClearDirectory(extractDir); + ZipFile.ExtractToDirectory(packagePath, extractDir, overwriteFiles: true); + + Console.WriteLine($"Downloaded and extracted {packageId} {latestVersion}."); + return extractDir; +} + +static async Task GetJsonDocumentAsync(HttpClient httpClient, string url) +{ + await using Stream stream = await httpClient.GetStreamAsync(url); + return await JsonDocument.ParseAsync(stream); +} + +static string GetRefDirectory(string extractDir, string refTargetFramework) +{ + string refRoot = Path.Combine(extractDir, "ref"); + if (!Directory.Exists(refRoot)) + { + throw new DirectoryNotFoundException($"The package does not contain a ref directory: {extractDir}"); + } + + string preferredRefDir = Path.Combine(refRoot, refTargetFramework); + if (Directory.Exists(preferredRefDir)) + { + return preferredRefDir; + } + + string[] candidateDirs = Directory.GetDirectories(refRoot); + if (candidateDirs.Length == 1) + { + return candidateDirs[0]; + } + + throw new DirectoryNotFoundException($"The package does not contain a ref/{refTargetFramework} directory (and could not infer one) under: {extractDir}"); +} + +static (Dictionary Options, string? Error) ParseArguments(string[] args) +{ + Dictionary options = new(StringComparer.OrdinalIgnoreCase); + for (int index = 0; index < args.Length; index++) + { + string option = args[index]; + if (option is not ("--dotnet-dir" or "--version")) + { + return (options, $"Unknown option '{option}'. Supported options: --dotnet-dir and --version ."); + } + + if (index + 1 >= args.Length || args[index + 1].StartsWith("--", StringComparison.Ordinal)) + { + return (options, $"Option '{option}' requires a value."); + } + + options[option] = args[++index]; + } + + return (options, null); +} + +static string GetOption(IReadOnlyDictionary options, string name, string defaultValue) +{ + return options.TryGetValue(name, out string? value) && !string.IsNullOrWhiteSpace(value) + ? value + : defaultValue; +} + +static void CopyFiles(string sourceDir, string destinationDir, string searchPattern, ISet excludedFileNames) +{ + Directory.CreateDirectory(destinationDir); + + int copiedCount = 0; + foreach (string sourcePath in Directory.EnumerateFiles(sourceDir, searchPattern, SearchOption.TopDirectoryOnly)) + { + string fileName = Path.GetFileName(sourcePath); + if (excludedFileNames.Contains(fileName)) + { + continue; + } + + File.Copy(sourcePath, Path.Combine(destinationDir, fileName), overwrite: true); + copiedCount++; + } + + Console.WriteLine($"Copied {copiedCount} {searchPattern} files from {sourceDir} to {destinationDir}."); +} + +static void CopyNamedFiles(string sourceDir, string destinationDir, ISet fileNamesToCopy) +{ + Directory.CreateDirectory(destinationDir); + + int copiedCount = 0; + foreach (string fileName in fileNamesToCopy) + { + string sourcePath = Path.Combine(sourceDir, fileName); + if (!File.Exists(sourcePath)) + { + throw new FileNotFoundException("Could not find an expected XML reference file.", sourcePath); + } + + File.Copy(sourcePath, Path.Combine(destinationDir, fileName), overwrite: true); + copiedCount++; + } + + Console.WriteLine($"Copied {copiedCount} named files from {sourceDir} to {destinationDir}."); +} + +static HashSet BuildFileNameSet(params string[] fileNames) +{ + HashSet names = new(StringComparer.OrdinalIgnoreCase); + foreach (string fileName in fileNames) + { + names.Add(fileName); + if (Path.GetExtension(fileName).Length == 0) + { + names.Add(fileName + ".xml"); + } + } + + return names; +} + +static void ClearDirectory(string directory) +{ + if (Directory.Exists(directory)) + { + Directory.Delete(directory, recursive: true); + } + + Directory.CreateDirectory(directory); +} +#endregion diff --git a/PackageDownloader/README.md b/PackageDownloader/README.md new file mode 100644 index 00000000..05b6da1d --- /dev/null +++ b/PackageDownloader/README.md @@ -0,0 +1,30 @@ +# PackageDownloader + +`PackageDownloader` updates local reference assemblies used to feed the .NET API +reference documentation generator. It downloads the latest .NET Core and Windows Desktop +reference assemblies from the packages available on the NuGet V3 feed +at `https://packagefeedproxy.microsoft.io/nuget/v3/index.json`. + +## Run + +From the repository root: + +```powershell +dotnet run --project PackageDownloader\PackageDownloader.csproj +``` + +Override the output directory and .NET version with named arguments: + +```powershell +dotnet run --project PackageDownloader\PackageDownloader.csproj -- --dotnet-dir C:\Users\me\binaries\dotnet --version 11.0 +``` + +Options: + +- `--dotnet-dir `: root output directory. Defaults to `C:\Users\gewarren\binaries\dotnet`. +- `--version `: .NET version used to select the package `ref/net` directory and name the output directories. Defaults to `11.0`. + +The program deletes and recreates its target output directories, so don't run it +while files in those directories are being edited or used by another process. + +The tool also downloads shim exclusions from the `netfxreference.props` file in dotnet/runtime to decide which assemblies to skip.