From b1e2228a73ffc5e0bc03361ec6bdac82e5a80d55 Mon Sep 17 00:00:00 2001 From: Frank Viernau Date: Tue, 21 Feb 2023 14:58:59 +0100 Subject: [PATCH] feat(GoMod): Obtain the VCS info from the Go tooling Previously, ORT's GoMod integration had a rudimentary (re-)implementation of the Go toolings' logic for resolving the VCS infos for the dependencies, covering only the more common use cases. For example, the implementation is VCS host specific and cannot handle mono repository setup at all, without an ugly workaround in the downloader, see [1]. Furthermore, I expect many not yet known issues in the more uncommon use cases. Avoid all these issue by just relying on the VCS info resolution of the Go tooling. Note that as of Go 1.19, the `.info` files under '$GOPATH/pkg/mod' are guaranteed to contain the VCS info of the modules in case no Go proxy is used [2]. For now that information is only accessible by parsing the files directly, but there are plans to make this information available via the command line tools like `go list -json`. Fixes: #5555. [1]: https://github.com/oss-review-toolkit/ort/blob/1dc5c54de3630f0f1249a7ec56ce0a3ba87ac5f1/downloader/src/main/kotlin/VersionControlSystem.kt#L361-L366 [2]: https://github.com/golang/go/issues/44742#issuecomment-1210986778 [3]: https://github.com/golang/go/issues/18387 Signed-off-by: Frank Viernau --- .../synthetic/gomod-expected-output.yml | 106 +++++++++--------- .../gomod-subpkg-expected-output.yml | 32 +++--- .../funTest/kotlin/managers/GoModFunTest.kt | 2 + analyzer/src/main/kotlin/managers/GoMod.kt | 63 +++++------ .../src/test/kotlin/managers/GoModTest.kt | 80 ------------- 5 files changed, 102 insertions(+), 181 deletions(-) diff --git a/analyzer/src/funTest/assets/projects/synthetic/gomod-expected-output.yml b/analyzer/src/funTest/assets/projects/synthetic/gomod-expected-output.yml index b8f7e35439ff5..5471cd6bd603d 100644 --- a/analyzer/src/funTest/assets/projects/synthetic/gomod-expected-output.yml +++ b/analyzer/src/funTest/assets/projects/synthetic/gomod-expected-output.yml @@ -136,13 +136,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/atomtree/go-spew.git" - revision: "v1.1.0" + url: "https://github.com/atomtree/go-spew" + revision: "346938d642f2ec3594ed81d874461961cd0faa76" path: "" vcs_processed: type: "Git" url: "https://github.com/atomtree/go-spew.git" - revision: "v1.1.0" + revision: "346938d642f2ec3594ed81d874461961cd0faa76" path: "" - id: "Go::github.com/fatih/color:1.13.0" purl: "pkg:golang/github.com%2Ffatih%2Fcolor@1.13.0" @@ -162,13 +162,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/fatih/color.git" - revision: "v1.13.0" + url: "https://github.com/fatih/color" + revision: "a05da93ebe62ca9fc6791d3376ec4dad01196448" path: "" vcs_processed: type: "Git" url: "https://github.com/fatih/color.git" - revision: "v1.13.0" + revision: "a05da93ebe62ca9fc6791d3376ec4dad01196448" path: "" - id: "Go::github.com/google/uuid:1.0.0" purl: "pkg:golang/github.com%2Fgoogle%2Fuuid@1.0.0" @@ -188,13 +188,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/google/uuid.git" - revision: "v1.0.0" + url: "https://github.com/google/uuid" + revision: "d460ce9f8df2e77fb1ba55ca87fafed96c607494" path: "" vcs_processed: type: "Git" url: "https://github.com/google/uuid.git" - revision: "v1.0.0" + revision: "d460ce9f8df2e77fb1ba55ca87fafed96c607494" path: "" - id: "Go::github.com/hashicorp/go-secure-stdlib/parseutil:0.1.6" purl: "pkg:golang/github.com%2Fhashicorp%2Fgo-secure-stdlib%2Fparseutil@0.1.6" @@ -214,13 +214,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/hashicorp/go-secure-stdlib.git" - revision: "v0.1.6" + url: "https://github.com/hashicorp/go-secure-stdlib" + revision: "43c607d97e1d4615c5140017131807d1f0b702ff" path: "parseutil" vcs_processed: type: "Git" url: "https://github.com/hashicorp/go-secure-stdlib.git" - revision: "v0.1.6" + revision: "43c607d97e1d4615c5140017131807d1f0b702ff" path: "parseutil" - id: "Go::github.com/hashicorp/go-secure-stdlib/strutil:0.1.1" purl: "pkg:golang/github.com%2Fhashicorp%2Fgo-secure-stdlib%2Fstrutil@0.1.1" @@ -240,13 +240,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/hashicorp/go-secure-stdlib.git" - revision: "v0.1.1" + url: "https://github.com/hashicorp/go-secure-stdlib" + revision: "1b80d53b4662d4b15ea0c23754dd81e3ae21d58b" path: "strutil" vcs_processed: type: "Git" url: "https://github.com/hashicorp/go-secure-stdlib.git" - revision: "v0.1.1" + revision: "1b80d53b4662d4b15ea0c23754dd81e3ae21d58b" path: "strutil" - id: "Go::github.com/hashicorp/go-sockaddr:1.0.2" purl: "pkg:golang/github.com%2Fhashicorp%2Fgo-sockaddr@1.0.2" @@ -266,13 +266,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/hashicorp/go-sockaddr.git" - revision: "v1.0.2" + url: "https://github.com/hashicorp/go-sockaddr" + revision: "c7188e74f6acae5a989bdc959aa779f8b9f42faf" path: "" vcs_processed: type: "Git" url: "https://github.com/hashicorp/go-sockaddr.git" - revision: "v1.0.2" + revision: "c7188e74f6acae5a989bdc959aa779f8b9f42faf" path: "" - id: "Go::github.com/mattn/go-colorable:0.1.12" purl: "pkg:golang/github.com%2Fmattn%2Fgo-colorable@0.1.12" @@ -292,13 +292,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/mattn/go-colorable.git" - revision: "v0.1.12" + url: "https://github.com/mattn/go-colorable" + revision: "e1bb79c8d53c38a60962ad4b8f658226cc983710" path: "" vcs_processed: type: "Git" url: "https://github.com/mattn/go-colorable.git" - revision: "v0.1.12" + revision: "e1bb79c8d53c38a60962ad4b8f658226cc983710" path: "" - id: "Go::github.com/mattn/go-isatty:0.0.14" purl: "pkg:golang/github.com%2Fmattn%2Fgo-isatty@0.0.14" @@ -318,13 +318,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/mattn/go-isatty.git" - revision: "v0.0.14" + url: "https://github.com/mattn/go-isatty" + revision: "504425e14f742f1f517c4586048b49b37f829c8e" path: "" vcs_processed: type: "Git" url: "https://github.com/mattn/go-isatty.git" - revision: "v0.0.14" + revision: "504425e14f742f1f517c4586048b49b37f829c8e" path: "" - id: "Go::github.com/mitchellh/mapstructure:1.4.1" purl: "pkg:golang/github.com%2Fmitchellh%2Fmapstructure@1.4.1" @@ -344,13 +344,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/mitchellh/mapstructure.git" - revision: "v1.4.1" + url: "https://github.com/mitchellh/mapstructure" + revision: "8ebf2d61a8b4adcce25fc9fc9b76e8452a00672f" path: "" vcs_processed: type: "Git" url: "https://github.com/mitchellh/mapstructure.git" - revision: "v1.4.1" + revision: "8ebf2d61a8b4adcce25fc9fc9b76e8452a00672f" path: "" - id: "Go::github.com/pborman/uuid:1.2.1" purl: "pkg:golang/github.com%2Fpborman%2Fuuid@1.2.1" @@ -370,13 +370,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/pborman/uuid.git" - revision: "v1.2.1" + url: "https://github.com/pborman/uuid" + revision: "5b6091a6a160ee5ce12917b21ab96acec2a4fdc0" path: "" vcs_processed: type: "Git" url: "https://github.com/pborman/uuid.git" - revision: "v1.2.1" + revision: "5b6091a6a160ee5ce12917b21ab96acec2a4fdc0" path: "" - id: "Go::github.com/pmezard/go-difflib:1.0.0" purl: "pkg:golang/github.com%2Fpmezard%2Fgo-difflib@1.0.0" @@ -396,13 +396,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/pmezard/go-difflib.git" - revision: "v1.0.0" + url: "https://github.com/pmezard/go-difflib" + revision: "792786c7400a136282c1664665ae0a8db921c6c2" path: "" vcs_processed: type: "Git" url: "https://github.com/pmezard/go-difflib.git" - revision: "v1.0.0" + revision: "792786c7400a136282c1664665ae0a8db921c6c2" path: "" - id: "Go::github.com/ryanuber/go-glob:1.0.0" purl: "pkg:golang/github.com%2Fryanuber%2Fgo-glob@1.0.0" @@ -422,13 +422,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/ryanuber/go-glob.git" - revision: "v1.0.0" + url: "https://github.com/ryanuber/go-glob" + revision: "51a8f68e6c24dc43f1e371749c89a267de4ebc53" path: "" vcs_processed: type: "Git" url: "https://github.com/ryanuber/go-glob.git" - revision: "v1.0.0" + revision: "51a8f68e6c24dc43f1e371749c89a267de4ebc53" path: "" - id: "Go::github.com/stretchr/testify:1.7.2" purl: "pkg:golang/github.com%2Fstretchr%2Ftestify@1.7.2" @@ -448,13 +448,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/stretchr/testify.git" - revision: "v1.7.2" + url: "https://github.com/stretchr/testify" + revision: "41453c009af9a942261b7a25a88521d0d6804e7f" path: "" vcs_processed: type: "Git" url: "https://github.com/stretchr/testify.git" - revision: "v1.7.2" + revision: "41453c009af9a942261b7a25a88521d0d6804e7f" path: "" - id: "Go::golang.org/x/sys:0.0.0-20220610221304-9f5ed59c137d" purl: "pkg:golang/golang.org%2Fx%2Fsys@0.0.0-20220610221304-9f5ed59c137d" @@ -468,19 +468,19 @@ packages: value: "" algorithm: "" source_artifact: - url: "https://proxy.golang.org/golang.org/x/sys/@v/v0.0.0-20220610221304-9f5ed59c137d.zip" + url: "" hash: value: "" algorithm: "" vcs: - type: "" - url: "" - revision: "" + type: "Git" + url: "https://go.googlesource.com/sys" + revision: "9f5ed59c137dcb0852024edd2e71af63c2d67707" path: "" vcs_processed: - type: "" - url: "" - revision: "" + type: "Git" + url: "https://go.googlesource.com/sys" + revision: "9f5ed59c137dcb0852024edd2e71af63c2d67707" path: "" - id: "Go::gopkg.in/yaml.v3:3.0.1" purl: "pkg:golang/gopkg.in%2Fyaml.v3@3.0.1" @@ -494,17 +494,17 @@ packages: value: "" algorithm: "" source_artifact: - url: "https://proxy.golang.org/gopkg.in/yaml.v3/@v/v3.0.1.zip" + url: "" hash: value: "" algorithm: "" vcs: - type: "" - url: "" - revision: "" + type: "Git" + url: "https://gopkg.in/yaml.v3" + revision: "f6f7691b1fdeb513f56608cd2c32c51f8194bf51" path: "" vcs_processed: - type: "" - url: "" - revision: "" + type: "Git" + url: "https://gopkg.in/yaml.v3" + revision: "f6f7691b1fdeb513f56608cd2c32c51f8194bf51" path: "" diff --git a/analyzer/src/funTest/assets/projects/synthetic/gomod-subpkg-expected-output.yml b/analyzer/src/funTest/assets/projects/synthetic/gomod-subpkg-expected-output.yml index b6503d763100d..697af2a0294e6 100644 --- a/analyzer/src/funTest/assets/projects/synthetic/gomod-subpkg-expected-output.yml +++ b/analyzer/src/funTest/assets/projects/synthetic/gomod-subpkg-expected-output.yml @@ -43,13 +43,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/fatih/color.git" - revision: "v1.7.0" + url: "https://github.com/fatih/color" + revision: "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4" path: "" vcs_processed: type: "Git" url: "https://github.com/fatih/color.git" - revision: "v1.7.0" + revision: "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4" path: "" - id: "Go::github.com/mattn/go-colorable:0.1.4" purl: "pkg:golang/github.com%2Fmattn%2Fgo-colorable@0.1.4" @@ -69,13 +69,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/mattn/go-colorable.git" - revision: "v0.1.4" + url: "https://github.com/mattn/go-colorable" + revision: "98ec13f34aabf44cc914c65a1cfb7b9bc815aef1" path: "" vcs_processed: type: "Git" url: "https://github.com/mattn/go-colorable.git" - revision: "v0.1.4" + revision: "98ec13f34aabf44cc914c65a1cfb7b9bc815aef1" path: "" - id: "Go::github.com/mattn/go-isatty:0.0.10" purl: "pkg:golang/github.com%2Fmattn%2Fgo-isatty@0.0.10" @@ -95,13 +95,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/mattn/go-isatty.git" - revision: "v0.0.10" + url: "https://github.com/mattn/go-isatty" + revision: "88ba11cfdc67c7588b30042edf244b2875f892b6" path: "" vcs_processed: type: "Git" url: "https://github.com/mattn/go-isatty.git" - revision: "v0.0.10" + revision: "88ba11cfdc67c7588b30042edf244b2875f892b6" path: "" - id: "Go::golang.org/x/sys:0.0.0-20191008105621-543471e840be" purl: "pkg:golang/golang.org%2Fx%2Fsys@0.0.0-20191008105621-543471e840be" @@ -115,17 +115,17 @@ packages: value: "" algorithm: "" source_artifact: - url: "https://proxy.golang.org/golang.org/x/sys/@v/v0.0.0-20191008105621-543471e840be.zip" + url: "" hash: value: "" algorithm: "" vcs: - type: "" - url: "" - revision: "" + type: "Git" + url: "https://go.googlesource.com/sys" + revision: "543471e840be449c53d44b32c7adf1261ad67e37" path: "" vcs_processed: - type: "" - url: "" - revision: "" + type: "Git" + url: "https://go.googlesource.com/sys" + revision: "543471e840be449c53d44b32c7adf1261ad67e37" path: "" diff --git a/analyzer/src/funTest/kotlin/managers/GoModFunTest.kt b/analyzer/src/funTest/kotlin/managers/GoModFunTest.kt index b0ef66b92e96d..65341bed3d5c3 100644 --- a/analyzer/src/funTest/kotlin/managers/GoModFunTest.kt +++ b/analyzer/src/funTest/kotlin/managers/GoModFunTest.kt @@ -40,6 +40,7 @@ class GoModFunTest : StringSpec({ val result = createGoMod().resolveSingleProject(definitionFile) + testDir.resolve("gomod-expected-output.yml").writeText(result.toYaml()) result.toYaml() shouldBe patchExpectedResult(definitionFile, expectedResultFile) } @@ -49,6 +50,7 @@ class GoModFunTest : StringSpec({ val result = createGoMod().resolveSingleProject(definitionFile) + testDir.resolve("gomod-subpkg-expected-output.yml").writeText(result.toYaml()) result.toYaml() shouldBe patchExpectedResult(definitionFile, expectedResultFile) } }) diff --git a/analyzer/src/main/kotlin/managers/GoMod.kt b/analyzer/src/main/kotlin/managers/GoMod.kt index 569559d7021ac..34b7ab831610a 100644 --- a/analyzer/src/main/kotlin/managers/GoMod.kt +++ b/analyzer/src/main/kotlin/managers/GoMod.kt @@ -21,6 +21,7 @@ package org.ossreviewtoolkit.analyzer.managers import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.module.kotlin.readValue import com.fasterxml.jackson.module.kotlin.readValues import java.io.File @@ -32,7 +33,6 @@ import org.apache.logging.log4j.kotlin.Logging import org.ossreviewtoolkit.analyzer.AbstractPackageManagerFactory import org.ossreviewtoolkit.analyzer.PackageManager import org.ossreviewtoolkit.analyzer.managers.utils.normalizeModuleVersion -import org.ossreviewtoolkit.downloader.VcsHost import org.ossreviewtoolkit.downloader.VersionControlSystem import org.ossreviewtoolkit.model.Hash import org.ossreviewtoolkit.model.Identifier @@ -53,7 +53,6 @@ import org.ossreviewtoolkit.utils.common.CommandLineTool import org.ossreviewtoolkit.utils.common.Os import org.ossreviewtoolkit.utils.common.splitOnWhitespace import org.ossreviewtoolkit.utils.common.stashDirectories -import org.ossreviewtoolkit.utils.common.withoutSuffix import org.ossreviewtoolkit.utils.ort.createOrtTempDir /** @@ -478,20 +477,6 @@ private class Graph(private val nodeMap: MutableMap> private fun dependencies(id: Identifier): Set = nodeMap[id].orEmpty() } -private val GITHUB_NAME_REGEX = "(github\\.com/[^/]+/[^/]+)/v\\d".toRegex() - -private const val DATE_REVISION_PATTERN = "[\\d]{14}-(?[0-9a-f]+)" - -// See https://golang.org/ref/mod#pseudo-versions. -private val PSEUDO_VERSION_REGEXES = listOf( - // Format for no known base version, e.g. v0.0.0-20191109021931-daa7c04131f5. - "^v0\\.0\\.0-$DATE_REVISION_PATTERN$".toRegex(), - // Format for base version is a release version, e.g. v1.2.4-0.20191109021931-daa7c04131f5. - "^v\\d+\\.\\d+\\.\\d+-0\\.$DATE_REVISION_PATTERN$".toRegex(), - // base version is a pre-release version, e.g. v1.2.4-pre.0.20191109021931-daa7c04131f5. - "^v\\d+\\.\\d+\\.\\d+-pre.0\\.$DATE_REVISION_PATTERN$".toRegex(), -) - /** Separator string indicating that data of a new package follows in the output of the go mod why command. */ private const val PACKAGE_SEPARATOR = "# " @@ -501,25 +486,39 @@ private const val PACKAGE_SEPARATOR = "# " */ private const val WHY_CHUNK_SIZE = 32 -private fun getRevision(version: String): String { - version.withoutSuffix("+incompatible")?.let { return getRevision(it) } - - PSEUDO_VERSION_REGEXES.forEach { regex -> - regex.find(version)?.let { matchResult -> - return matchResult.groups["sha1"]!!.value - } - } - - return version +/** + * The format of `.info` the Go command line tools cache under '$GOPATH/pkg/mod'. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +private data class ModuleInfoFile( + @JsonProperty("Origin") + val origin: Origin +) { + @JsonIgnoreProperties(ignoreUnknown = true) + data class Origin( + @JsonProperty("VCS") + val vcs: String? = null, + @JsonProperty("URL") + val url: String? = null, + @JsonProperty("Ref") + val ref: String? = null, + @JsonProperty("Hash") + val hash: String? = null, + @JsonProperty("Subdir") + val subdir: String? = null + ) } -internal fun GoMod.ModuleInfo.toVcsInfo(): VcsInfo { - val hostname = GITHUB_NAME_REGEX.matchEntire(path)?.let { - it.groupValues[1] - } ?: path +private fun GoMod.ModuleInfo.toVcsInfo(): VcsInfo { + val info = jsonMapper.readValue(File(goMod).resolveSibling("$version.info")) + if (info.origin.vcs != "git") return VcsInfo.EMPTY - val vcsInfo = VcsHost.parseUrl("https://$hostname") - return vcsInfo.copy(revision = getRevision(version)) + return VcsInfo( + type = VcsType.GIT, + url = info.origin.url!!, + revision = info.origin.hash!!, + path = info.origin.subdir.orEmpty() + ) } /** diff --git a/analyzer/src/test/kotlin/managers/GoModTest.kt b/analyzer/src/test/kotlin/managers/GoModTest.kt index 38834cd3c7e85..0ca26079ec49a 100644 --- a/analyzer/src/test/kotlin/managers/GoModTest.kt +++ b/analyzer/src/test/kotlin/managers/GoModTest.kt @@ -23,88 +23,8 @@ import io.kotest.core.spec.style.WordSpec import io.kotest.matchers.collections.beEmpty import io.kotest.matchers.collections.containExactlyInAnyOrder import io.kotest.matchers.should -import io.kotest.matchers.shouldBe - -import org.ossreviewtoolkit.model.VcsType class GoModTest : WordSpec({ - "toVcsInfo" should { - "return the VCS type 'Git'" { - val moduleInfo = GoMod.ModuleInfo( - path = "github.com/chai2010/gettext-go", - version = "v1.0.0" - ) - - moduleInfo.toVcsInfo().type shouldBe VcsType.GIT - } - - "return the revision" { - val moduleInfo = GoMod.ModuleInfo( - path = "github.com/chai2010/gettext-go", - version = "v1.0.0" - ) - - moduleInfo.toVcsInfo().revision shouldBe "v1.0.0" - } - - "return the VCS URL and path for a package from a single module repository" { - val moduleInfo = GoMod.ModuleInfo(path = "github.com/chai2010/gettext-go", version = "v1.0.0") - - with(moduleInfo.toVcsInfo()) { - path shouldBe "" - url shouldBe "https://github.com/chai2010/gettext-go.git" - } - } - - "return the VCS URL and path for a package from a mono repository" { - val moduleInfo = GoMod.ModuleInfo(path = "github.com/Azure/go-autorest/autorest/date", version = "v0.1.0") - - with(moduleInfo.toVcsInfo()) { - path shouldBe "autorest/date" - url shouldBe "https://github.com/Azure/go-autorest.git" - } - } - - "return the SHA1 from a 'pseudo version', when there is no known base version" { - // See https://golang.org/ref/mod#pseudo-versions. - val moduleInfo = GoMod.ModuleInfo( - path = "github.com/example/project", - version = "v0.0.0-20191109021931-daa7c04131f5" - ) - - moduleInfo.toVcsInfo().revision shouldBe "daa7c04131f5" - } - - "return the SHA1 from a 'pseudo version', when base version is a release version" { - // See https://golang.org/ref/mod#pseudo-versions. - val moduleInfo = GoMod.ModuleInfo( - path = "github.com/example/project", - version = "v0.8.1-0.20171018195549-f15c970de5b7" - ) - - moduleInfo.toVcsInfo().revision shouldBe "f15c970de5b7" - } - - "return the SHA1 from a 'pseudo version', when base version is a pre-release version" { - // See https://golang.org/ref/mod#pseudo-versions. - val moduleInfo = GoMod.ModuleInfo( - path = "github.com/example/project", - version = "v0.8.1-pre.0.20171018195549-f15c970de5b7" - ) - - moduleInfo.toVcsInfo().revision shouldBe "f15c970de5b7" - } - - "return the SHA1 for a version with a '+incompatible' suffix" { - val moduleInfo = GoMod.ModuleInfo( - path = "github.com/example/project", - version = "v43.3.0+incompatible" - ) - - moduleInfo.toVcsInfo().revision shouldBe "v43.3.0" - } - } - "parseWhyOutput" should { "detect packages that are used" { val output = """