diff --git a/nuget/lib/dependabot/nuget/metadata_finder.rb b/nuget/lib/dependabot/nuget/metadata_finder.rb index 56512d061cd..aae13690639 100644 --- a/nuget/lib/dependabot/nuget/metadata_finder.rb +++ b/nuget/lib/dependabot/nuget/metadata_finder.rb @@ -12,7 +12,62 @@ class MetadataFinder < Dependabot::MetadataFinders::Base def look_up_source return Source.from_url(dependency_source_url) if dependency_source_url - look_up_source_in_nuspec(dependency_nuspec_file) + src_repo = look_up_source_in_nuspec(dependency_nuspec_file) + return src_repo if src_repo + + # Fallback to getting source from the search result's projectUrl or licenseUrl. + # GitHub Packages doesn't support getting the `.nuspec`, switch to getting + # that instead once it is supported. + src_repo_from_project + end + + def src_repo_from_project + source = dependency.requirements.find { |r| r&.fetch(:source) }&.fetch(:source) + return unless source + + # Query the service index e.g. https://nuget.pkg.github.com/ORG/index.json + response = Excon.get( + source.fetch(:url), + idempotent: true, + **SharedHelpers.excon_defaults(headers: auth_header) + ) + return unless response.status == 200 + + # Extract the query url e.g. https://nuget.pkg.github.com/ORG/query + search_base = extract_search_url(response.body) + return unless search_base + + response = Excon.get( + search_base + "?q=#{dependency.name.downcase}&prerelease=true&semVerLevel=2.0.0", + idempotent: true, + **SharedHelpers.excon_defaults(headers: auth_header) + ) + return unless response.status == 200 + + # Find a projectUrl or licenseUrl that look like a source URL + extract_source_repo(response.body) + end + + def extract_search_url(body) + JSON.parse(body). + fetch("resources", []). + find { |r| r.fetch("@type") == "SearchQueryService" }&. + fetch("@id") + end + + def extract_source_repo(body) + JSON.parse(body).fetch("data", []).each do |search_result| + next unless search_result["id"].downcase == dependency.name.downcase + + if search_result.fetch("projectUrl") + source = Source.from_url(search_result.fetch("projectUrl")) + return source unless source.repo.nil? + end + if search_result.fetch("licenseUrl") + source = Source.from_url(search_result.fetch("licenseUrl")) + return source unless source.repo.nil? + end + end end def look_up_source_in_nuspec(nuspec) diff --git a/nuget/spec/dependabot/nuget/metadata_finder_spec.rb b/nuget/spec/dependabot/nuget/metadata_finder_spec.rb index d6028dc0994..89ccb275398 100644 --- a/nuget/spec/dependabot/nuget/metadata_finder_spec.rb +++ b/nuget/spec/dependabot/nuget/metadata_finder_spec.rb @@ -146,14 +146,18 @@ context "that requires authentication" do before do stub_request(:get, nuget_url).to_return(status: 404) - stub_request(:get, nuget_url). - with(basic_auth: %w(my passw0rd)). - to_return(status: 200, body: nuget_response) + stub_request(:get, "https://www.myget.org/F/exceptionless/api/v3/index.json").to_return(status: 404) end it { is_expected.to be_nil } context "with details in the credentials" do + before do + stub_request(:get, nuget_url). + with(basic_auth: %w(my passw0rd)). + to_return(status: 200, body: nuget_response) + end + let(:credentials) do [{ "type" => "git_source", @@ -171,6 +175,49 @@ it { is_expected.to eq("https://github.com/dotnet/core-setup") } end end + + context "that doesn't support .nuspec routes" do + before do + # registry doesn't support .nuspec route, so returns 404 + stub_request(:get, nuget_url).to_return(status: 404) + # fallback begins by getting the search URL from the index + stub_request(:get, "https://www.myget.org/F/exceptionless/api/v3/index.json"). + to_return(status: 200, body: fixture("nuspecs", "index.json")) + # next query for the package at the search URL returned + stub_request(:get, "https://azuresearch-usnc.nuget.org/query?prerelease=true&q=microsoft.extensions.dependencymodel&semVerLevel=2.0.0"). + to_return(status: 200, body: fixture("nuspecs", "microsoft.extensions.depdencymodel-results.json")) + end + + # data was extracted from the projectUrl in the search results + it { is_expected.to eq "https://github.com/dotnet/core-setup" } + + context "and it fails to get the index" do + before do + # registry is in a bad state + stub_request(:get, nuget_url).to_return(status: 500) + # it falls back to get search URL from the index, but it fails too + stub_request(:get, "https://www.myget.org/F/exceptionless/api/v3/index.json"). + to_return(status: 500, body: "internal server error") + end + + it { is_expected.to be_nil } + end + + context "and it fails to get the search results" do + before do + # registry doesn't support .nuspec route, so returns 404 + stub_request(:get, nuget_url).to_return(status: 404) + # fallback begins by getting the search URL from the index + stub_request(:get, "https://www.myget.org/F/exceptionless/api/v3/index.json"). + to_return(status: 200, body: fixture("nuspecs", "index.json")) + # oops, we're a little overloaded + stub_request(:get, "https://azuresearch-usnc.nuget.org/query?prerelease=true&q=microsoft.extensions.dependencymodel&semVerLevel=2.0.0"). + to_return(status: 503, body: "") + end + + it { is_expected.to be_nil } + end + end end end end diff --git a/nuget/spec/fixtures/nuspecs/index.json b/nuget/spec/fixtures/nuspecs/index.json new file mode 100644 index 00000000000..c09ca053285 --- /dev/null +++ b/nuget/spec/fixtures/nuspecs/index.json @@ -0,0 +1,192 @@ +{ + "version": "3.0.0", + "resources": [ + { + "@id": "https://azuresearch-usnc.nuget.org/query", + "@type": "SearchQueryService", + "comment": "Query endpoint of NuGet Search service (primary)" + }, + { + "@id": "https://azuresearch-ussc.nuget.org/query", + "@type": "SearchQueryService", + "comment": "Query endpoint of NuGet Search service (secondary)" + }, + { + "@id": "https://azuresearch-usnc.nuget.org/autocomplete", + "@type": "SearchAutocompleteService", + "comment": "Autocomplete endpoint of NuGet Search service (primary)" + }, + { + "@id": "https://azuresearch-ussc.nuget.org/autocomplete", + "@type": "SearchAutocompleteService", + "comment": "Autocomplete endpoint of NuGet Search service (secondary)" + }, + { + "@id": "https://azuresearch-usnc.nuget.org/", + "@type": "SearchGalleryQueryService/3.0.0-rc", + "comment": "Azure Website based Search Service used by Gallery (primary)" + }, + { + "@id": "https://azuresearch-ussc.nuget.org/", + "@type": "SearchGalleryQueryService/3.0.0-rc", + "comment": "Azure Website based Search Service used by Gallery (secondary)" + }, + { + "@id": "https://api.nuget.org/v3/registration5-semver1/", + "@type": "RegistrationsBaseUrl", + "comment": "Base URL of Azure storage where NuGet package registration info is stored" + }, + { + "@id": "https://api.nuget.org/v3-flatcontainer/", + "@type": "PackageBaseAddress/3.0.0", + "comment": "Base URL of where NuGet packages are stored, in the format https://api.nuget.org/v3-flatcontainer/{id-lower}/{version-lower}/{id-lower}.{version-lower}.nupkg" + }, + { + "@id": "https://www.nuget.org/api/v2", + "@type": "LegacyGallery" + }, + { + "@id": "https://www.nuget.org/api/v2", + "@type": "LegacyGallery/2.0.0" + }, + { + "@id": "https://www.nuget.org/api/v2/package", + "@type": "PackagePublish/2.0.0" + }, + { + "@id": "https://www.nuget.org/api/v2/symbolpackage", + "@type": "SymbolPackagePublish/4.9.0", + "comment": "The gallery symbol publish endpoint." + }, + { + "@id": "https://azuresearch-usnc.nuget.org/query", + "@type": "SearchQueryService/3.0.0-rc", + "comment": "Query endpoint of NuGet Search service (primary) used by RC clients" + }, + { + "@id": "https://azuresearch-ussc.nuget.org/query", + "@type": "SearchQueryService/3.0.0-rc", + "comment": "Query endpoint of NuGet Search service (secondary) used by RC clients" + }, + { + "@id": "https://azuresearch-usnc.nuget.org/query", + "@type": "SearchQueryService/3.5.0", + "comment": "Query endpoint of NuGet Search service (primary) that supports package type filtering" + }, + { + "@id": "https://azuresearch-ussc.nuget.org/query", + "@type": "SearchQueryService/3.5.0", + "comment": "Query endpoint of NuGet Search service (secondary) that supports package type filtering" + }, + { + "@id": "https://azuresearch-usnc.nuget.org/autocomplete", + "@type": "SearchAutocompleteService/3.0.0-rc", + "comment": "Autocomplete endpoint of NuGet Search service (primary) used by RC clients" + }, + { + "@id": "https://azuresearch-ussc.nuget.org/autocomplete", + "@type": "SearchAutocompleteService/3.0.0-rc", + "comment": "Autocomplete endpoint of NuGet Search service (secondary) used by RC clients" + }, + { + "@id": "https://azuresearch-usnc.nuget.org/autocomplete", + "@type": "SearchAutocompleteService/3.5.0", + "comment": "Autocomplete endpoint of NuGet Search service (primary) that supports package type filtering" + }, + { + "@id": "https://azuresearch-ussc.nuget.org/autocomplete", + "@type": "SearchAutocompleteService/3.5.0", + "comment": "Autocomplete endpoint of NuGet Search service (secondary) that supports package type filtering" + }, + { + "@id": "https://api.nuget.org/v3/registration5-semver1/", + "@type": "RegistrationsBaseUrl/3.0.0-rc", + "comment": "Base URL of Azure storage where NuGet package registration info is stored used by RC clients. This base URL does not include SemVer 2.0.0 packages." + }, + { + "@id": "https://www.nuget.org/packages/{id}/{version}/ReportAbuse", + "@type": "ReportAbuseUriTemplate/3.0.0-rc", + "comment": "URI template used by NuGet Client to construct Report Abuse URL for packages used by RC clients" + }, + { + "@id": "https://api.nuget.org/v3/registration5-semver1/{id-lower}/index.json", + "@type": "PackageDisplayMetadataUriTemplate/3.0.0-rc", + "comment": "URI template used by NuGet Client to construct display metadata for Packages using ID" + }, + { + "@id": "https://api.nuget.org/v3/registration5-semver1/{id-lower}/{version-lower}.json", + "@type": "PackageVersionDisplayMetadataUriTemplate/3.0.0-rc", + "comment": "URI template used by NuGet Client to construct display metadata for Packages using ID, Version" + }, + { + "@id": "https://azuresearch-usnc.nuget.org/query", + "@type": "SearchQueryService/3.0.0-beta", + "comment": "Query endpoint of NuGet Search service (primary) used by beta clients" + }, + { + "@id": "https://azuresearch-ussc.nuget.org/query", + "@type": "SearchQueryService/3.0.0-beta", + "comment": "Query endpoint of NuGet Search service (secondary) used by beta clients" + }, + { + "@id": "https://azuresearch-usnc.nuget.org/autocomplete", + "@type": "SearchAutocompleteService/3.0.0-beta", + "comment": "Autocomplete endpoint of NuGet Search service (primary) used by beta clients" + }, + { + "@id": "https://azuresearch-ussc.nuget.org/autocomplete", + "@type": "SearchAutocompleteService/3.0.0-beta", + "comment": "Autocomplete endpoint of NuGet Search service (secondary) used by beta clients" + }, + { + "@id": "https://api.nuget.org/v3/registration5-semver1/", + "@type": "RegistrationsBaseUrl/3.0.0-beta", + "comment": "Base URL of Azure storage where NuGet package registration info is stored used by Beta clients. This base URL does not include SemVer 2.0.0 packages." + }, + { + "@id": "https://www.nuget.org/packages/{id}/{version}/ReportAbuse", + "@type": "ReportAbuseUriTemplate/3.0.0-beta", + "comment": "URI template used by NuGet Client to construct Report Abuse URL for packages" + }, + { + "@id": "https://www.nuget.org/packages/{id}/{version}?_src=template", + "@type": "PackageDetailsUriTemplate/5.1.0", + "comment": "URI template used by NuGet Client to construct details URL for packages" + }, + { + "@id": "https://api.nuget.org/v3/registration5-gz-semver1/", + "@type": "RegistrationsBaseUrl/3.4.0", + "comment": "Base URL of Azure storage where NuGet package registration info is stored in GZIP format. This base URL does not include SemVer 2.0.0 packages." + }, + { + "@id": "https://api.nuget.org/v3/registration5-gz-semver2/", + "@type": "RegistrationsBaseUrl/3.6.0", + "comment": "Base URL of Azure storage where NuGet package registration info is stored in GZIP format. This base URL includes SemVer 2.0.0 packages." + }, + { + "@id": "https://api.nuget.org/v3/registration5-gz-semver2/", + "@type": "RegistrationsBaseUrl/Versioned", + "clientVersion": "4.3.0-alpha", + "comment": "Base URL of Azure storage where NuGet package registration info is stored in GZIP format. This base URL includes SemVer 2.0.0 packages." + }, + { + "@id": "https://api.nuget.org/v3-index/repository-signatures/4.7.0/index.json", + "@type": "RepositorySignatures/4.7.0", + "comment": "The endpoint for discovering information about this package source's repository signatures." + }, + { + "@id": "https://api.nuget.org/v3-index/repository-signatures/5.0.0/index.json", + "@type": "RepositorySignatures/5.0.0", + "comment": "The endpoint for discovering information about this package source's repository signatures." + }, + { + "@id": "https://api.nuget.org/v3/catalog0/index.json", + "@type": "Catalog/3.0.0", + "comment": "Index of the NuGet package catalog." + } + ], + "@context": { + "@vocab": "http://schema.nuget.org/services#", + "comment": "http://www.w3.org/2000/01/rdf-schema#comment" + } +} diff --git a/nuget/spec/fixtures/nuspecs/microsoft.extensions.depdencymodel-results.json b/nuget/spec/fixtures/nuspecs/microsoft.extensions.depdencymodel-results.json new file mode 100644 index 00000000000..489aa2ce0c9 --- /dev/null +++ b/nuget/spec/fixtures/nuspecs/microsoft.extensions.depdencymodel-results.json @@ -0,0 +1,193 @@ +{ + "@context": { + "@vocab": "http://schema.nuget.org/schema#", + "@base": "https://api.nuget.org/v3/registration5-semver1/" + }, + "totalHits": 1, + "data": [ + { + "@id": "https://api.nuget.org/v3/registration5-semver1/microsoft.extensions.dependencymodel/index.json", + "@type": "Package", + "registration": "https://api.nuget.org/v3/registration5-semver1/microsoft.extensions.dependencymodel/index.json", + "id": "Microsoft.Extensions.DependencyModel", + "version": "6.0.0", + "description": "Abstractions for reading `.deps` files.\n\nCommonly Used Types:\nMicrosoft.Extensions.DependencyModel.DependencyContext", + "summary": "", + "title": "Microsoft.Extensions.DependencyModel", + "iconUrl": "https://api.nuget.org/v3-flatcontainer/microsoft.extensions.dependencymodel/6.0.0/icon", + "licenseUrl": "https://www.nuget.org/packages/Microsoft.Extensions.DependencyModel/6.0.0/license", + "projectUrl": "https://github.com/dotnet/core-setup", + "tags": [], + "authors": [ + "Microsoft" + ], + "owners": [ + "aspnet", + "dotnetframework", + "Microsoft" + ], + "totalDownloads": 592854601, + "verified": true, + "packageTypes": [ + { + "name": "Dependency" + } + ], + "versions": [ + { + "version": "1.0.0", + "downloads": 53573949, + "@id": "https://api.nuget.org/v3/registration5-semver1/microsoft.extensions.dependencymodel/1.0.0.json" + }, + { + "version": "1.0.3", + "downloads": 70125341, + "@id": "https://api.nuget.org/v3/registration5-semver1/microsoft.extensions.dependencymodel/1.0.3.json" + }, + { + "version": "1.1.0", + "downloads": 12508665, + "@id": "https://api.nuget.org/v3/registration5-semver1/microsoft.extensions.dependencymodel/1.1.0.json" + }, + { + "version": "1.1.1", + "downloads": 4699544, + "@id": "https://api.nuget.org/v3/registration5-semver1/microsoft.extensions.dependencymodel/1.1.1.json" + }, + { + "version": "1.1.2", + "downloads": 14087944, + "@id": "https://api.nuget.org/v3/registration5-semver1/microsoft.extensions.dependencymodel/1.1.2.json" + }, + { + "version": "1.1.9", + "downloads": 746146, + "@id": "https://api.nuget.org/v3/registration5-semver1/microsoft.extensions.dependencymodel/1.1.9.json" + }, + { + "version": "2.0.0", + "downloads": 66592651, + "@id": "https://api.nuget.org/v3/registration5-semver1/microsoft.extensions.dependencymodel/2.0.0.json" + }, + { + "version": "2.0.3", + "downloads": 21618938, + "@id": "https://api.nuget.org/v3/registration5-semver1/microsoft.extensions.dependencymodel/2.0.3.json" + }, + { + "version": "2.0.4", + "downloads": 109088277, + "@id": "https://api.nuget.org/v3/registration5-semver1/microsoft.extensions.dependencymodel/2.0.4.json" + }, + { + "version": "2.1.0", + "downloads": 120196091, + "@id": "https://api.nuget.org/v3/registration5-semver1/microsoft.extensions.dependencymodel/2.1.0.json" + }, + { + "version": "3.0.0", + "downloads": 18697802, + "@id": "https://api.nuget.org/v3/registration5-semver1/microsoft.extensions.dependencymodel/3.0.0.json" + }, + { + "version": "3.0.1", + "downloads": 303086, + "@id": "https://api.nuget.org/v3/registration5-semver1/microsoft.extensions.dependencymodel/3.0.1.json" + }, + { + "version": "3.0.2", + "downloads": 65869, + "@id": "https://api.nuget.org/v3/registration5-semver1/microsoft.extensions.dependencymodel/3.0.2.json" + }, + { + "version": "3.0.3", + "downloads": 103602, + "@id": "https://api.nuget.org/v3/registration5-semver1/microsoft.extensions.dependencymodel/3.0.3.json" + }, + { + "version": "3.1.0", + "downloads": 7791483, + "@id": "https://api.nuget.org/v3/registration5-semver1/microsoft.extensions.dependencymodel/3.1.0.json" + }, + { + "version": "3.1.1", + "downloads": 4184184, + "@id": "https://api.nuget.org/v3/registration5-semver1/microsoft.extensions.dependencymodel/3.1.1.json" + }, + { + "version": "3.1.2", + "downloads": 4194473, + "@id": "https://api.nuget.org/v3/registration5-semver1/microsoft.extensions.dependencymodel/3.1.2.json" + }, + { + "version": "3.1.3", + "downloads": 10138695, + "@id": "https://api.nuget.org/v3/registration5-semver1/microsoft.extensions.dependencymodel/3.1.3.json" + }, + { + "version": "3.1.4", + "downloads": 3718896, + "@id": "https://api.nuget.org/v3/registration5-semver1/microsoft.extensions.dependencymodel/3.1.4.json" + }, + { + "version": "3.1.5", + "downloads": 5237803, + "@id": "https://api.nuget.org/v3/registration5-semver1/microsoft.extensions.dependencymodel/3.1.5.json" + }, + { + "version": "3.1.6", + "downloads": 32350043, + "@id": "https://api.nuget.org/v3/registration5-semver1/microsoft.extensions.dependencymodel/3.1.6.json" + }, + { + "version": "5.0.0", + "downloads": 24280196, + "@id": "https://api.nuget.org/v3/registration5-semver1/microsoft.extensions.dependencymodel/5.0.0.json" + }, + { + "version": "6.0.0", + "downloads": 5397556, + "@id": "https://api.nuget.org/v3/registration5-semver1/microsoft.extensions.dependencymodel/6.0.0.json" + } + ] + }, + { + "@id": "https://api.nuget.org/v3/registration5-semver1/macross.servicemodel.extensions/index.json", + "@type": "Package", + "registration": "https://api.nuget.org/v3/registration5-semver1/macross.servicemodel.extensions/index.json", + "id": "Macross.ServiceModel.Extensions", + "version": "1.0.1", + "description": "Macross Software System.ServiceModel extensions library.\n\n Provides extensions for the System.ServiceModel API.\n\n Commonly Used Types:\n System.ServiceModel.SoapClient\n\n Commonly Used Extensions:\n Microsoft.Extensions.DependencyInjection.AddSoapClient", + "summary": "", + "title": "Macross.ServiceModel.Extensions", + "licenseUrl": "https://www.nuget.org/packages/Macross.ServiceModel.Extensions/1.0.1/license", + "projectUrl": "https://github.com/Macross-Software/core/tree/develop/ClassLibraries/Macross.ServiceModel.Extensions", + "tags": [], + "authors": [ + "Macross.ServiceModel.Extensions" + ], + "owners": [ + "Macross-Software" + ], + "totalDownloads": 23530, + "verified": false, + "packageTypes": [ + { + "name": "Dependency" + } + ], + "versions": [ + { + "version": "1.0.0", + "downloads": 408, + "@id": "https://api.nuget.org/v3/registration5-semver1/macross.servicemodel.extensions/1.0.0.json" + }, + { + "version": "1.0.1", + "downloads": 22345, + "@id": "https://api.nuget.org/v3/registration5-semver1/macross.servicemodel.extensions/1.0.1.json" + } + ] + } + ] +}