nuget: fix PR missing commits in message when using private registry

nuget/lib/dependabot/nuget/metadata_finder.rb

@@ -12,7 +12,73 @@ class MetadataFinder < Dependabot::MetadataFinders::Base
       def look_up_source
         return Source.from_url(dependency_source_url) if dependency_source_url
 
-        look_up_source_in_nuspec(dependency_nuspec_file)
+        src_repo = look_up_source_in_nuspec(dependency_nuspec_file)
+        return src_repo if src_repo
+
+        # Fallback to getting source from the search result's projectUrl or licenseUrl.
+        # GitHub Packages doesn't support getting the `.nuspec`, switch to getting
+        # that instead once it is supported.
+        src_repo_from_project
+      rescue StandardError
+        # At this point in the process the PR is ready to be posted, we tried to gather commit
+        # and release notes, but have encountered an exception. So let's eat it and log it
+        # since it's better to have a PR with no info than error out.
+        # TODO how do we log this?
+        nil
+      end
+
+      def src_repo_from_project
+        source = dependency.requirements.find { |r| r&.fetch(:source) }&.fetch(:source)
+        return unless source
+
+        # Query the service index e.g. https://nuget.pkg.github.com/ORG/index.json
+        response = Excon.get(
+          source.fetch(:url),
+          idempotent: true,
+          **SharedHelpers.excon_defaults(headers: { **auth_header, "Accept" => "application/json" })
+        )
+        return unless response.status == 200
+
+        # Extract the query url e.g. https://nuget.pkg.github.com/ORG/query
+        search_base = extract_search_url(response.body)
+        return unless search_base
+
+        response = Excon.get(
+          search_base + "?q=#{dependency.name.downcase}&prerelease=true&semVerLevel=2.0.0",
+          idempotent: true,
+          **SharedHelpers.excon_defaults(headers: { **auth_header, "Accept" => "application/json" })
+        )
+        return unless response.status == 200
+
+        # Find a projectUrl or licenseUrl that look like a source URL
+        extract_source_repo(response.body)
+      end
+
+      def extract_search_url(body)
+        JSON.parse(body).
+          fetch("resources", []).
+          find { |r| r.fetch("@type") == "SearchQueryService" }&.
+          fetch("@id")
+      rescue JSON::ParserError
+        # Ignored, this is expected for some registries that don't handle this request.
+        # TODO can we log here so it's clear why the update is missing commit info?
+      end
+
+      def extract_source_repo(body)
+        JSON.parse(body).fetch("data", []).each do |search_result|
+          next unless search_result["id"].downcase == dependency.name.downcase
+
+          if search_result.key?("projectUrl")
+            source = Source.from_url(search_result.fetch("projectUrl"))
+            return source if source
+          end
+          if search_result.key?("licenseUrl")
+            source = Source.from_url(search_result.fetch("licenseUrl"))
+            return source if source
+          end
+        end
+        # failed to find a source URL
+        nil
       end
 
       def look_up_source_in_nuspec(nuspec)

nuget/spec/dependabot/nuget/metadata_finder_spec.rb

@@ -171,6 +171,79 @@
           it { is_expected.to eq("https://github.com/dotnet/core-setup") }
         end
       end
+
+      context "that doesn't support .nuspec routes" do
+        before do
+          # registry doesn't support .nuspec route, so returns 404
+          stub_request(:get, nuget_url).to_return(status: 404)
+          # fallback begins by getting the search URL from the index
+          stub_request(:get, "https://www.myget.org/F/exceptionless/api/v3/index.json").
+            to_return(status: 200, body: fixture("nuspecs", "index.json"))
+          # next query for the package at the search URL returned
+          stub_request(:get, "https://azuresearch-usnc.nuget.org/query?prerelease=true&q=microsoft.extensions.dependencymodel&semVerLevel=2.0.0").
+            to_return(status: 200, body: fixture("nuspecs", "microsoft.extensions.dependencymodel-results.json"))
+        end
+
+        # data was extracted from the projectUrl in the search results
+        it { is_expected.to eq "https://github.com/dotnet/core-setup" }
+      end
+
+      context "the index returns XML" do
+        before do
+          # registry doesn't support .nuspec route, so returns 404
+          stub_request(:get, nuget_url).to_return(status: 404)
+          # fallback tries to get the index, but gets a 200 with XML
+          # This might be due to artifactory not supporting index?
+          stub_request(:get, "https://www.myget.org/F/exceptionless/api/v3/index.json").
+            to_return(status: 200, body: '<?xml version="1.0" encoding="UTF-8"?><hello>world</hello>')
+        end
+
+        # no exceptions
+        it { is_expected.to be_nil }
+      end
+
+      context "the search results do not contain a projectUrl" do
+        before do
+          # registry doesn't support .nuspec route, so returns 404
+          stub_request(:get, nuget_url).to_return(status: 404)
+          # fallback begins by getting the search URL from the index
+          stub_request(:get, "https://www.myget.org/F/exceptionless/api/v3/index.json").
+            to_return(status: 200, body: fixture("nuspecs", "index.json"))
+          # the search results have a blank projectUrl field AND missing the licenseUrl field entirely
+          stub_request(:get, "https://azuresearch-usnc.nuget.org/query?prerelease=true&q=microsoft.extensions.dependencymodel&semVerLevel=2.0.0").
+            to_return(status: 200, body: '{"data":[{"id":"Microsoft.Extensions.DependencyModel","projectUrl":""}]}')
+        end
+
+        # no exceptions
+        it { is_expected.to be_nil }
+      end
+
+      context "and it fails to get the index" do
+        before do
+          # registry is in a bad state
+          stub_request(:get, nuget_url).to_return(status: 500)
+          # it falls back to get search URL from the index, but it fails too
+          stub_request(:get, "https://www.myget.org/F/exceptionless/api/v3/index.json").
+            to_return(status: 500, body: "internal server error")
+        end
+
+        it { is_expected.to be_nil }
+      end
+
+      context "and it fails to get the search results" do
+        before do
+          # registry doesn't support .nuspec route, so returns 404
+          stub_request(:get, nuget_url).to_return(status: 404)
+          # fallback begins by getting the search URL from the index
+          stub_request(:get, "https://www.myget.org/F/exceptionless/api/v3/index.json").
+            to_return(status: 200, body: fixture("nuspecs", "index.json"))
+          # oops, we're a little overloaded
+          stub_request(:get, "https://azuresearch-usnc.nuget.org/query?prerelease=true&q=microsoft.extensions.dependencymodel&semVerLevel=2.0.0").
+            to_return(status: 503, body: "")
+        end
+
+        it { is_expected.to be_nil }
+      end
     end
   end
 end

nuget/spec/fixtures/nuspecs/index.json

@@ -0,0 +1,192 @@
+{
+  "version": "3.0.0",
+  "resources": [
+    {
+      "@id": "https://azuresearch-usnc.nuget.org/query",
+      "@type": "SearchQueryService",
+      "comment": "Query endpoint of NuGet Search service (primary)"
+    },
+    {
+      "@id": "https://azuresearch-ussc.nuget.org/query",
+      "@type": "SearchQueryService",
+      "comment": "Query endpoint of NuGet Search service (secondary)"
+    },
+    {
+      "@id": "https://azuresearch-usnc.nuget.org/autocomplete",
+      "@type": "SearchAutocompleteService",
+      "comment": "Autocomplete endpoint of NuGet Search service (primary)"
+    },
+    {
+      "@id": "https://azuresearch-ussc.nuget.org/autocomplete",
+      "@type": "SearchAutocompleteService",
+      "comment": "Autocomplete endpoint of NuGet Search service (secondary)"
+    },
+    {
+      "@id": "https://azuresearch-usnc.nuget.org/",
+      "@type": "SearchGalleryQueryService/3.0.0-rc",
+      "comment": "Azure Website based Search Service used by Gallery (primary)"
+    },
+    {
+      "@id": "https://azuresearch-ussc.nuget.org/",
+      "@type": "SearchGalleryQueryService/3.0.0-rc",
+      "comment": "Azure Website based Search Service used by Gallery (secondary)"
+    },
+    {
+      "@id": "https://api.nuget.org/v3/registration5-semver1/",
+      "@type": "RegistrationsBaseUrl",
+      "comment": "Base URL of Azure storage where NuGet package registration info is stored"
+    },
+    {
+      "@id": "https://api.nuget.org/v3-flatcontainer/",
+      "@type": "PackageBaseAddress/3.0.0",
+      "comment": "Base URL of where NuGet packages are stored, in the format https://api.nuget.org/v3-flatcontainer/{id-lower}/{version-lower}/{id-lower}.{version-lower}.nupkg"
+    },
+    {
+      "@id": "https://www.nuget.org/api/v2",
+      "@type": "LegacyGallery"
+    },
+    {
+      "@id": "https://www.nuget.org/api/v2",
+      "@type": "LegacyGallery/2.0.0"
+    },
+    {
+      "@id": "https://www.nuget.org/api/v2/package",
+      "@type": "PackagePublish/2.0.0"
+    },
+    {
+      "@id": "https://www.nuget.org/api/v2/symbolpackage",
+      "@type": "SymbolPackagePublish/4.9.0",
+      "comment": "The gallery symbol publish endpoint."
+    },
+    {
+      "@id": "https://azuresearch-usnc.nuget.org/query",
+      "@type": "SearchQueryService/3.0.0-rc",
+      "comment": "Query endpoint of NuGet Search service (primary) used by RC clients"
+    },
+    {
+      "@id": "https://azuresearch-ussc.nuget.org/query",
+      "@type": "SearchQueryService/3.0.0-rc",
+      "comment": "Query endpoint of NuGet Search service (secondary) used by RC clients"
+    },
+    {
+      "@id": "https://azuresearch-usnc.nuget.org/query",
+      "@type": "SearchQueryService/3.5.0",
+      "comment": "Query endpoint of NuGet Search service (primary) that supports package type filtering"
+    },
+    {
+      "@id": "https://azuresearch-ussc.nuget.org/query",
+      "@type": "SearchQueryService/3.5.0",
+      "comment": "Query endpoint of NuGet Search service (secondary) that supports package type filtering"
+    },
+    {
+      "@id": "https://azuresearch-usnc.nuget.org/autocomplete",
+      "@type": "SearchAutocompleteService/3.0.0-rc",
+      "comment": "Autocomplete endpoint of NuGet Search service (primary) used by RC clients"
+    },
+    {
+      "@id": "https://azuresearch-ussc.nuget.org/autocomplete",
+      "@type": "SearchAutocompleteService/3.0.0-rc",
+      "comment": "Autocomplete endpoint of NuGet Search service (secondary) used by RC clients"
+    },
+    {
+      "@id": "https://azuresearch-usnc.nuget.org/autocomplete",
+      "@type": "SearchAutocompleteService/3.5.0",
+      "comment": "Autocomplete endpoint of NuGet Search service (primary) that supports package type filtering"
+    },
+    {
+      "@id": "https://azuresearch-ussc.nuget.org/autocomplete",
+      "@type": "SearchAutocompleteService/3.5.0",
+      "comment": "Autocomplete endpoint of NuGet Search service (secondary) that supports package type filtering"
+    },
+    {
+      "@id": "https://api.nuget.org/v3/registration5-semver1/",
+      "@type": "RegistrationsBaseUrl/3.0.0-rc",
+      "comment": "Base URL of Azure storage where NuGet package registration info is stored used by RC clients. This base URL does not include SemVer 2.0.0 packages."
+    },
+    {
+      "@id": "https://www.nuget.org/packages/{id}/{version}/ReportAbuse",
+      "@type": "ReportAbuseUriTemplate/3.0.0-rc",
+      "comment": "URI template used by NuGet Client to construct Report Abuse URL for packages used by RC clients"
+    },
+    {
+      "@id": "https://api.nuget.org/v3/registration5-semver1/{id-lower}/index.json",
+      "@type": "PackageDisplayMetadataUriTemplate/3.0.0-rc",
+      "comment": "URI template used by NuGet Client to construct display metadata for Packages using ID"
+    },
+    {
+      "@id": "https://api.nuget.org/v3/registration5-semver1/{id-lower}/{version-lower}.json",
+      "@type": "PackageVersionDisplayMetadataUriTemplate/3.0.0-rc",
+      "comment": "URI template used by NuGet Client to construct display metadata for Packages using ID, Version"
+    },
+    {
+      "@id": "https://azuresearch-usnc.nuget.org/query",
+      "@type": "SearchQueryService/3.0.0-beta",
+      "comment": "Query endpoint of NuGet Search service (primary) used by beta clients"
+    },
+    {
+      "@id": "https://azuresearch-ussc.nuget.org/query",
+      "@type": "SearchQueryService/3.0.0-beta",
+      "comment": "Query endpoint of NuGet Search service (secondary) used by beta clients"
+    },
+    {
+      "@id": "https://azuresearch-usnc.nuget.org/autocomplete",
+      "@type": "SearchAutocompleteService/3.0.0-beta",
+      "comment": "Autocomplete endpoint of NuGet Search service (primary) used by beta clients"
+    },
+    {
+      "@id": "https://azuresearch-ussc.nuget.org/autocomplete",
+      "@type": "SearchAutocompleteService/3.0.0-beta",
+      "comment": "Autocomplete endpoint of NuGet Search service (secondary) used by beta clients"
+    },
+    {
+      "@id": "https://api.nuget.org/v3/registration5-semver1/",
+      "@type": "RegistrationsBaseUrl/3.0.0-beta",
+      "comment": "Base URL of Azure storage where NuGet package registration info is stored used by Beta clients. This base URL does not include SemVer 2.0.0 packages."
+    },
+    {
+      "@id": "https://www.nuget.org/packages/{id}/{version}/ReportAbuse",
+      "@type": "ReportAbuseUriTemplate/3.0.0-beta",
+      "comment": "URI template used by NuGet Client to construct Report Abuse URL for packages"
+    },
+    {
+      "@id": "https://www.nuget.org/packages/{id}/{version}?_src=template",
+      "@type": "PackageDetailsUriTemplate/5.1.0",
+      "comment": "URI template used by NuGet Client to construct details URL for packages"
+    },
+    {
+      "@id": "https://api.nuget.org/v3/registration5-gz-semver1/",
+      "@type": "RegistrationsBaseUrl/3.4.0",
+      "comment": "Base URL of Azure storage where NuGet package registration info is stored in GZIP format. This base URL does not include SemVer 2.0.0 packages."
+    },
+    {
+      "@id": "https://api.nuget.org/v3/registration5-gz-semver2/",
+      "@type": "RegistrationsBaseUrl/3.6.0",
+      "comment": "Base URL of Azure storage where NuGet package registration info is stored in GZIP format. This base URL includes SemVer 2.0.0 packages."
+    },
+    {
+      "@id": "https://api.nuget.org/v3/registration5-gz-semver2/",
+      "@type": "RegistrationsBaseUrl/Versioned",
+      "clientVersion": "4.3.0-alpha",
+      "comment": "Base URL of Azure storage where NuGet package registration info is stored in GZIP format. This base URL includes SemVer 2.0.0 packages."
+    },
+    {
+      "@id": "https://api.nuget.org/v3-index/repository-signatures/4.7.0/index.json",
+      "@type": "RepositorySignatures/4.7.0",
+      "comment": "The endpoint for discovering information about this package source's repository signatures."
+    },
+    {
+      "@id": "https://api.nuget.org/v3-index/repository-signatures/5.0.0/index.json",
+      "@type": "RepositorySignatures/5.0.0",
+      "comment": "The endpoint for discovering information about this package source's repository signatures."
+    },
+    {
+      "@id": "https://api.nuget.org/v3/catalog0/index.json",
+      "@type": "Catalog/3.0.0",
+      "comment": "Index of the NuGet package catalog."
+    }
+  ],
+  "@context": {
+    "@vocab": "http://schema.nuget.org/services#",
+    "comment": "http://www.w3.org/2000/01/rdf-schema#comment"
+  }
+}