diff --git a/analyzer/src/funTest/assets/projects/synthetic/gomod-expected-output.yml b/analyzer/src/funTest/assets/projects/synthetic/gomod-expected-output.yml index b8f7e35439ff5..5471cd6bd603d 100644 --- a/analyzer/src/funTest/assets/projects/synthetic/gomod-expected-output.yml +++ b/analyzer/src/funTest/assets/projects/synthetic/gomod-expected-output.yml @@ -136,13 +136,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/atomtree/go-spew.git" - revision: "v1.1.0" + url: "https://github.com/atomtree/go-spew" + revision: "346938d642f2ec3594ed81d874461961cd0faa76" path: "" vcs_processed: type: "Git" url: "https://github.com/atomtree/go-spew.git" - revision: "v1.1.0" + revision: "346938d642f2ec3594ed81d874461961cd0faa76" path: "" - id: "Go::github.com/fatih/color:1.13.0" purl: "pkg:golang/github.com%2Ffatih%2Fcolor@1.13.0" @@ -162,13 +162,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/fatih/color.git" - revision: "v1.13.0" + url: "https://github.com/fatih/color" + revision: "a05da93ebe62ca9fc6791d3376ec4dad01196448" path: "" vcs_processed: type: "Git" url: "https://github.com/fatih/color.git" - revision: "v1.13.0" + revision: "a05da93ebe62ca9fc6791d3376ec4dad01196448" path: "" - id: "Go::github.com/google/uuid:1.0.0" purl: "pkg:golang/github.com%2Fgoogle%2Fuuid@1.0.0" @@ -188,13 +188,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/google/uuid.git" - revision: "v1.0.0" + url: "https://github.com/google/uuid" + revision: "d460ce9f8df2e77fb1ba55ca87fafed96c607494" path: "" vcs_processed: type: "Git" url: "https://github.com/google/uuid.git" - revision: "v1.0.0" + revision: "d460ce9f8df2e77fb1ba55ca87fafed96c607494" path: "" - id: "Go::github.com/hashicorp/go-secure-stdlib/parseutil:0.1.6" purl: "pkg:golang/github.com%2Fhashicorp%2Fgo-secure-stdlib%2Fparseutil@0.1.6" @@ -214,13 +214,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/hashicorp/go-secure-stdlib.git" - revision: "v0.1.6" + url: "https://github.com/hashicorp/go-secure-stdlib" + revision: "43c607d97e1d4615c5140017131807d1f0b702ff" path: "parseutil" vcs_processed: type: "Git" url: "https://github.com/hashicorp/go-secure-stdlib.git" - revision: "v0.1.6" + revision: "43c607d97e1d4615c5140017131807d1f0b702ff" path: "parseutil" - id: "Go::github.com/hashicorp/go-secure-stdlib/strutil:0.1.1" purl: "pkg:golang/github.com%2Fhashicorp%2Fgo-secure-stdlib%2Fstrutil@0.1.1" @@ -240,13 +240,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/hashicorp/go-secure-stdlib.git" - revision: "v0.1.1" + url: "https://github.com/hashicorp/go-secure-stdlib" + revision: "1b80d53b4662d4b15ea0c23754dd81e3ae21d58b" path: "strutil" vcs_processed: type: "Git" url: "https://github.com/hashicorp/go-secure-stdlib.git" - revision: "v0.1.1" + revision: "1b80d53b4662d4b15ea0c23754dd81e3ae21d58b" path: "strutil" - id: "Go::github.com/hashicorp/go-sockaddr:1.0.2" purl: "pkg:golang/github.com%2Fhashicorp%2Fgo-sockaddr@1.0.2" @@ -266,13 +266,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/hashicorp/go-sockaddr.git" - revision: "v1.0.2" + url: "https://github.com/hashicorp/go-sockaddr" + revision: "c7188e74f6acae5a989bdc959aa779f8b9f42faf" path: "" vcs_processed: type: "Git" url: "https://github.com/hashicorp/go-sockaddr.git" - revision: "v1.0.2" + revision: "c7188e74f6acae5a989bdc959aa779f8b9f42faf" path: "" - id: "Go::github.com/mattn/go-colorable:0.1.12" purl: "pkg:golang/github.com%2Fmattn%2Fgo-colorable@0.1.12" @@ -292,13 +292,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/mattn/go-colorable.git" - revision: "v0.1.12" + url: "https://github.com/mattn/go-colorable" + revision: "e1bb79c8d53c38a60962ad4b8f658226cc983710" path: "" vcs_processed: type: "Git" url: "https://github.com/mattn/go-colorable.git" - revision: "v0.1.12" + revision: "e1bb79c8d53c38a60962ad4b8f658226cc983710" path: "" - id: "Go::github.com/mattn/go-isatty:0.0.14" purl: "pkg:golang/github.com%2Fmattn%2Fgo-isatty@0.0.14" @@ -318,13 +318,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/mattn/go-isatty.git" - revision: "v0.0.14" + url: "https://github.com/mattn/go-isatty" + revision: "504425e14f742f1f517c4586048b49b37f829c8e" path: "" vcs_processed: type: "Git" url: "https://github.com/mattn/go-isatty.git" - revision: "v0.0.14" + revision: "504425e14f742f1f517c4586048b49b37f829c8e" path: "" - id: "Go::github.com/mitchellh/mapstructure:1.4.1" purl: "pkg:golang/github.com%2Fmitchellh%2Fmapstructure@1.4.1" @@ -344,13 +344,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/mitchellh/mapstructure.git" - revision: "v1.4.1" + url: "https://github.com/mitchellh/mapstructure" + revision: "8ebf2d61a8b4adcce25fc9fc9b76e8452a00672f" path: "" vcs_processed: type: "Git" url: "https://github.com/mitchellh/mapstructure.git" - revision: "v1.4.1" + revision: "8ebf2d61a8b4adcce25fc9fc9b76e8452a00672f" path: "" - id: "Go::github.com/pborman/uuid:1.2.1" purl: "pkg:golang/github.com%2Fpborman%2Fuuid@1.2.1" @@ -370,13 +370,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/pborman/uuid.git" - revision: "v1.2.1" + url: "https://github.com/pborman/uuid" + revision: "5b6091a6a160ee5ce12917b21ab96acec2a4fdc0" path: "" vcs_processed: type: "Git" url: "https://github.com/pborman/uuid.git" - revision: "v1.2.1" + revision: "5b6091a6a160ee5ce12917b21ab96acec2a4fdc0" path: "" - id: "Go::github.com/pmezard/go-difflib:1.0.0" purl: "pkg:golang/github.com%2Fpmezard%2Fgo-difflib@1.0.0" @@ -396,13 +396,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/pmezard/go-difflib.git" - revision: "v1.0.0" + url: "https://github.com/pmezard/go-difflib" + revision: "792786c7400a136282c1664665ae0a8db921c6c2" path: "" vcs_processed: type: "Git" url: "https://github.com/pmezard/go-difflib.git" - revision: "v1.0.0" + revision: "792786c7400a136282c1664665ae0a8db921c6c2" path: "" - id: "Go::github.com/ryanuber/go-glob:1.0.0" purl: "pkg:golang/github.com%2Fryanuber%2Fgo-glob@1.0.0" @@ -422,13 +422,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/ryanuber/go-glob.git" - revision: "v1.0.0" + url: "https://github.com/ryanuber/go-glob" + revision: "51a8f68e6c24dc43f1e371749c89a267de4ebc53" path: "" vcs_processed: type: "Git" url: "https://github.com/ryanuber/go-glob.git" - revision: "v1.0.0" + revision: "51a8f68e6c24dc43f1e371749c89a267de4ebc53" path: "" - id: "Go::github.com/stretchr/testify:1.7.2" purl: "pkg:golang/github.com%2Fstretchr%2Ftestify@1.7.2" @@ -448,13 +448,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/stretchr/testify.git" - revision: "v1.7.2" + url: "https://github.com/stretchr/testify" + revision: "41453c009af9a942261b7a25a88521d0d6804e7f" path: "" vcs_processed: type: "Git" url: "https://github.com/stretchr/testify.git" - revision: "v1.7.2" + revision: "41453c009af9a942261b7a25a88521d0d6804e7f" path: "" - id: "Go::golang.org/x/sys:0.0.0-20220610221304-9f5ed59c137d" purl: "pkg:golang/golang.org%2Fx%2Fsys@0.0.0-20220610221304-9f5ed59c137d" @@ -468,19 +468,19 @@ packages: value: "" algorithm: "" source_artifact: - url: "https://proxy.golang.org/golang.org/x/sys/@v/v0.0.0-20220610221304-9f5ed59c137d.zip" + url: "" hash: value: "" algorithm: "" vcs: - type: "" - url: "" - revision: "" + type: "Git" + url: "https://go.googlesource.com/sys" + revision: "9f5ed59c137dcb0852024edd2e71af63c2d67707" path: "" vcs_processed: - type: "" - url: "" - revision: "" + type: "Git" + url: "https://go.googlesource.com/sys" + revision: "9f5ed59c137dcb0852024edd2e71af63c2d67707" path: "" - id: "Go::gopkg.in/yaml.v3:3.0.1" purl: "pkg:golang/gopkg.in%2Fyaml.v3@3.0.1" @@ -494,17 +494,17 @@ packages: value: "" algorithm: "" source_artifact: - url: "https://proxy.golang.org/gopkg.in/yaml.v3/@v/v3.0.1.zip" + url: "" hash: value: "" algorithm: "" vcs: - type: "" - url: "" - revision: "" + type: "Git" + url: "https://gopkg.in/yaml.v3" + revision: "f6f7691b1fdeb513f56608cd2c32c51f8194bf51" path: "" vcs_processed: - type: "" - url: "" - revision: "" + type: "Git" + url: "https://gopkg.in/yaml.v3" + revision: "f6f7691b1fdeb513f56608cd2c32c51f8194bf51" path: "" diff --git a/analyzer/src/funTest/assets/projects/synthetic/gomod-subpkg-expected-output.yml b/analyzer/src/funTest/assets/projects/synthetic/gomod-subpkg-expected-output.yml index b6503d763100d..697af2a0294e6 100644 --- a/analyzer/src/funTest/assets/projects/synthetic/gomod-subpkg-expected-output.yml +++ b/analyzer/src/funTest/assets/projects/synthetic/gomod-subpkg-expected-output.yml @@ -43,13 +43,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/fatih/color.git" - revision: "v1.7.0" + url: "https://github.com/fatih/color" + revision: "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4" path: "" vcs_processed: type: "Git" url: "https://github.com/fatih/color.git" - revision: "v1.7.0" + revision: "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4" path: "" - id: "Go::github.com/mattn/go-colorable:0.1.4" purl: "pkg:golang/github.com%2Fmattn%2Fgo-colorable@0.1.4" @@ -69,13 +69,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/mattn/go-colorable.git" - revision: "v0.1.4" + url: "https://github.com/mattn/go-colorable" + revision: "98ec13f34aabf44cc914c65a1cfb7b9bc815aef1" path: "" vcs_processed: type: "Git" url: "https://github.com/mattn/go-colorable.git" - revision: "v0.1.4" + revision: "98ec13f34aabf44cc914c65a1cfb7b9bc815aef1" path: "" - id: "Go::github.com/mattn/go-isatty:0.0.10" purl: "pkg:golang/github.com%2Fmattn%2Fgo-isatty@0.0.10" @@ -95,13 +95,13 @@ packages: algorithm: "" vcs: type: "Git" - url: "https://github.com/mattn/go-isatty.git" - revision: "v0.0.10" + url: "https://github.com/mattn/go-isatty" + revision: "88ba11cfdc67c7588b30042edf244b2875f892b6" path: "" vcs_processed: type: "Git" url: "https://github.com/mattn/go-isatty.git" - revision: "v0.0.10" + revision: "88ba11cfdc67c7588b30042edf244b2875f892b6" path: "" - id: "Go::golang.org/x/sys:0.0.0-20191008105621-543471e840be" purl: "pkg:golang/golang.org%2Fx%2Fsys@0.0.0-20191008105621-543471e840be" @@ -115,17 +115,17 @@ packages: value: "" algorithm: "" source_artifact: - url: "https://proxy.golang.org/golang.org/x/sys/@v/v0.0.0-20191008105621-543471e840be.zip" + url: "" hash: value: "" algorithm: "" vcs: - type: "" - url: "" - revision: "" + type: "Git" + url: "https://go.googlesource.com/sys" + revision: "543471e840be449c53d44b32c7adf1261ad67e37" path: "" vcs_processed: - type: "" - url: "" - revision: "" + type: "Git" + url: "https://go.googlesource.com/sys" + revision: "543471e840be449c53d44b32c7adf1261ad67e37" path: "" diff --git a/analyzer/src/main/kotlin/managers/GoMod.kt b/analyzer/src/main/kotlin/managers/GoMod.kt index 6719fc9a6898e..0dc5dcf170c6d 100644 --- a/analyzer/src/main/kotlin/managers/GoMod.kt +++ b/analyzer/src/main/kotlin/managers/GoMod.kt @@ -21,6 +21,7 @@ package org.ossreviewtoolkit.analyzer.managers import com.fasterxml.jackson.annotation.JsonIgnoreProperties import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.module.kotlin.readValue import com.fasterxml.jackson.module.kotlin.readValues import java.io.File @@ -32,7 +33,6 @@ import org.apache.logging.log4j.kotlin.Logging import org.ossreviewtoolkit.analyzer.AbstractPackageManagerFactory import org.ossreviewtoolkit.analyzer.PackageManager import org.ossreviewtoolkit.analyzer.managers.utils.normalizeModuleVersion -import org.ossreviewtoolkit.downloader.VcsHost import org.ossreviewtoolkit.downloader.VersionControlSystem import org.ossreviewtoolkit.model.Hash import org.ossreviewtoolkit.model.Identifier @@ -53,7 +53,6 @@ import org.ossreviewtoolkit.utils.common.CommandLineTool import org.ossreviewtoolkit.utils.common.Os import org.ossreviewtoolkit.utils.common.splitOnWhitespace import org.ossreviewtoolkit.utils.common.stashDirectories -import org.ossreviewtoolkit.utils.common.withoutSuffix import org.ossreviewtoolkit.utils.ort.createOrtTempDir import org.semver4j.RangesList @@ -226,7 +225,10 @@ class GoMod( val indirect: Boolean = false, @JsonProperty("Main") - val main: Boolean = false + val main: Boolean = false, + + @JsonProperty("GoMod") + val goMod: String ) private data class DepInfo( @@ -480,20 +482,6 @@ private class Graph(private val nodeMap: MutableMap> private fun dependencies(id: Identifier): Set = nodeMap[id].orEmpty() } -private val GITHUB_NAME_REGEX = "(github\\.com/[^/]+/[^/]+)/v\\d".toRegex() - -private const val DATE_REVISION_PATTERN = "[\\d]{14}-(?[0-9a-f]+)" - -// See https://golang.org/ref/mod#pseudo-versions. -private val PSEUDO_VERSION_REGEXES = listOf( - // Format for no known base version, e.g. v0.0.0-20191109021931-daa7c04131f5. - "^v0\\.0\\.0-$DATE_REVISION_PATTERN$".toRegex(), - // Format for base version is a release version, e.g. v1.2.4-0.20191109021931-daa7c04131f5. - "^v\\d+\\.\\d+\\.\\d+-0\\.$DATE_REVISION_PATTERN$".toRegex(), - // base version is a pre-release version, e.g. v1.2.4-pre.0.20191109021931-daa7c04131f5. - "^v\\d+\\.\\d+\\.\\d+-pre.0\\.$DATE_REVISION_PATTERN$".toRegex(), -) - /** Separator string indicating that data of a new package follows in the output of the go mod why command. */ private const val PACKAGE_SEPARATOR = "# " @@ -503,25 +491,39 @@ private const val PACKAGE_SEPARATOR = "# " */ private const val WHY_CHUNK_SIZE = 32 -private fun getRevision(version: String): String { - version.withoutSuffix("+incompatible")?.let { return getRevision(it) } - - PSEUDO_VERSION_REGEXES.forEach { regex -> - regex.find(version)?.let { matchResult -> - return matchResult.groups["sha1"]!!.value - } - } - - return version +/** + * The format of `.info` the Go command line tools cache under '$GOPATH/pkg/mod'. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +private data class ModuleInfoFile( + @JsonProperty("Origin") + val origin: Origin +) { + @JsonIgnoreProperties(ignoreUnknown = true) + data class Origin( + @JsonProperty("VCS") + val vcs: String? = null, + @JsonProperty("URL") + val url: String? = null, + @JsonProperty("Ref") + val ref: String? = null, + @JsonProperty("Hash") + val hash: String? = null, + @JsonProperty("Subdir") + val subdir: String? = null + ) } -internal fun GoMod.ModuleInfo.toVcsInfo(): VcsInfo { - val hostname = GITHUB_NAME_REGEX.matchEntire(path)?.let { - it.groupValues[1] - } ?: path +private fun GoMod.ModuleInfo.toVcsInfo(): VcsInfo { + val info = jsonMapper.readValue(File(goMod).resolveSibling("$version.info")) + val type = info.origin.vcs?.let { VcsType.forName(it) }.takeIf { it == VcsType.GIT } ?: return VcsInfo.EMPTY - val vcsInfo = VcsHost.parseUrl("https://$hostname") - return vcsInfo.copy(revision = getRevision(version)) + return VcsInfo( + type = type, + url = checkNotNull(info.origin.url), + revision = checkNotNull(info.origin.hash), + path = info.origin.subdir.orEmpty() + ) } /** diff --git a/analyzer/src/test/kotlin/managers/GoModTest.kt b/analyzer/src/test/kotlin/managers/GoModTest.kt index 38834cd3c7e85..0ca26079ec49a 100644 --- a/analyzer/src/test/kotlin/managers/GoModTest.kt +++ b/analyzer/src/test/kotlin/managers/GoModTest.kt @@ -23,88 +23,8 @@ import io.kotest.core.spec.style.WordSpec import io.kotest.matchers.collections.beEmpty import io.kotest.matchers.collections.containExactlyInAnyOrder import io.kotest.matchers.should -import io.kotest.matchers.shouldBe - -import org.ossreviewtoolkit.model.VcsType class GoModTest : WordSpec({ - "toVcsInfo" should { - "return the VCS type 'Git'" { - val moduleInfo = GoMod.ModuleInfo( - path = "github.com/chai2010/gettext-go", - version = "v1.0.0" - ) - - moduleInfo.toVcsInfo().type shouldBe VcsType.GIT - } - - "return the revision" { - val moduleInfo = GoMod.ModuleInfo( - path = "github.com/chai2010/gettext-go", - version = "v1.0.0" - ) - - moduleInfo.toVcsInfo().revision shouldBe "v1.0.0" - } - - "return the VCS URL and path for a package from a single module repository" { - val moduleInfo = GoMod.ModuleInfo(path = "github.com/chai2010/gettext-go", version = "v1.0.0") - - with(moduleInfo.toVcsInfo()) { - path shouldBe "" - url shouldBe "https://github.com/chai2010/gettext-go.git" - } - } - - "return the VCS URL and path for a package from a mono repository" { - val moduleInfo = GoMod.ModuleInfo(path = "github.com/Azure/go-autorest/autorest/date", version = "v0.1.0") - - with(moduleInfo.toVcsInfo()) { - path shouldBe "autorest/date" - url shouldBe "https://github.com/Azure/go-autorest.git" - } - } - - "return the SHA1 from a 'pseudo version', when there is no known base version" { - // See https://golang.org/ref/mod#pseudo-versions. - val moduleInfo = GoMod.ModuleInfo( - path = "github.com/example/project", - version = "v0.0.0-20191109021931-daa7c04131f5" - ) - - moduleInfo.toVcsInfo().revision shouldBe "daa7c04131f5" - } - - "return the SHA1 from a 'pseudo version', when base version is a release version" { - // See https://golang.org/ref/mod#pseudo-versions. - val moduleInfo = GoMod.ModuleInfo( - path = "github.com/example/project", - version = "v0.8.1-0.20171018195549-f15c970de5b7" - ) - - moduleInfo.toVcsInfo().revision shouldBe "f15c970de5b7" - } - - "return the SHA1 from a 'pseudo version', when base version is a pre-release version" { - // See https://golang.org/ref/mod#pseudo-versions. - val moduleInfo = GoMod.ModuleInfo( - path = "github.com/example/project", - version = "v0.8.1-pre.0.20171018195549-f15c970de5b7" - ) - - moduleInfo.toVcsInfo().revision shouldBe "f15c970de5b7" - } - - "return the SHA1 for a version with a '+incompatible' suffix" { - val moduleInfo = GoMod.ModuleInfo( - path = "github.com/example/project", - version = "v43.3.0+incompatible" - ) - - moduleInfo.toVcsInfo().revision shouldBe "v43.3.0" - } - } - "parseWhyOutput" should { "detect packages that are used" { val output = """