Skip to content

Commit

Permalink
feat: official support for Golang on pkg.go.dev
Browse files Browse the repository at this point in the history 
Signed-off-by: Paul Horton <[email protected]>
  • Loading branch information
@madpah
madpah committed Nov 26, 2024
1 parent 2d29bba commit e9b1af6
Show file tree
Hide file tree
Showing 7 changed files with 14,969 additions and 2,453 deletions.
106 changes: 71 additions & 35 deletions src/utils/PageParsing/Golang.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,34 @@ describe('Golang Page Parsing', () => {
const repoType = REPO_TYPES.find((e) => e.repoID == REPOS.pkgGoDev)
expect(repoType).toBeDefined()

test('Parse golang page etcd version in url', () => {
test('Parse golang.org/x/text no version in URL', () => {
const html = readFileSync(join(__dirname, 'testdata/pkg.go.dev/golang.org-x-text-0.20.0.html'))
window.document.body.innerHTML = html.toString()

const packageURL: PackageURL | undefined = getArtifactDetailsFromDOM(
ensure(repoType),
'https://pkg.go.dev/golang.org/x/text'
)

expect(packageURL).toBeDefined()
expect(packageURL?.version).toBe('v0.20.0')
expect(packageURL?.namespace).toBe('golang.org/x')
expect(packageURL?.name).toBe('text')
})

test('Parse golang.org/x/text with version in URL', () => {
const packageURL: PackageURL | undefined = getArtifactDetailsFromDOM(
ensure(repoType),
'https://pkg.go.dev/golang.org/x/[email protected]'
)

expect(packageURL).toBeDefined()
expect(packageURL?.version).toBe('v0.19.0')
expect(packageURL?.namespace).toBe('golang.org/x')
expect(packageURL?.name).toBe('text')
})

test('Parse github.com/etcd-io/etcd with version in URL', () => {
const packageURL: PackageURL | undefined = getArtifactDetailsFromDOM(
ensure(repoType),
'https://pkg.go.dev/github.com/etcd-io/[email protected]'
Expand All @@ -37,23 +64,31 @@ describe('Golang Page Parsing', () => {
expect(packageURL?.name).toBe('etcd')
})

test('Parse golang page protobuf version in url', () => {
const PackageURL = getArtifactDetailsFromDOM(
test('Parse golang.org/x/text no version in URL', () => {
const html = readFileSync(join(__dirname, 'testdata/pkg.go.dev/go.etcd.io-etcd-client-v3-3.5.17.html'))
window.document.body.innerHTML = html.toString()

const packageURL: PackageURL | undefined = getArtifactDetailsFromDOM(
ensure(repoType),
'https://pkg.go.dev/google.golang.org/[email protected]'
'https://pkg.go.dev/go.etcd.io/etcd/client/v3'
)
expect(PackageURL).toBeDefined()
expect(PackageURL?.version).toBe('v1.26.0')
expect(PackageURL?.namespace).toBe('google.golang.org')
expect(PackageURL?.name).toBe('protobuf')
expect(PackageURL?.toString()).toBe('pkg:golang/google.golang.org/[email protected]')
//write tests for this and implement it
// expect(PackageURL?.qualifiers).toBe('runtime');
// expect(PackageURL?.subpath).toBe('protoimpl');
//https://pkg.go.dev/google.golang.org/protobuf@v1.26.0/runtime/protoimpl

expect(packageURL).toBeDefined()
expect(packageURL?.version).toBe('v3.5.17')
expect(packageURL?.namespace).toBe('go.etcd.io/etcd/client')
expect(packageURL?.name).toBe('v3')
})

test('+incompatible version URL fails to parse', () => {
const packageURL: PackageURL | undefined = getArtifactDetailsFromDOM(
ensure(repoType),
'https://pkg.go.dev/github.com/etcd-io/[email protected]+incompatible'
)

expect(packageURL).not.toBeDefined()
})

test('Parse golang page protobuf version in url and subpath', () => {
test('Parse google.golang.org/protobuf with Version & Subpath', () => {
const PackageURL = getArtifactDetailsFromDOM(
ensure(repoType),
'https://pkg.go.dev/google.golang.org/[email protected]/runtime/protoimpl'
Expand All @@ -64,30 +99,31 @@ describe('Golang Page Parsing', () => {
expect(PackageURL?.name).toBe('protobuf')
expect(PackageURL?.toString()).toBe('pkg:golang/google.golang.org/[email protected]')
})
test('Parse golang page gopkg.in version in url', () => {
const PackageURL = getArtifactDetailsFromDOM(ensure(repoType), 'https://pkg.go.dev/gopkg.in/[email protected]')
expect(PackageURL).toBeDefined()
expect(PackageURL?.version).toBe('v1.61.0')
expect(PackageURL?.namespace).toBe('github.com/go-ini')
expect(PackageURL?.name).toBe('ini')
})

test('Parse golang page gopkg.in version in url', () => {
const PackageURL = getArtifactDetailsFromDOM(ensure(repoType), 'https://pkg.go.dev/gopkg.in/[email protected]')
expect(PackageURL).toBeDefined()
expect(PackageURL?.version).toBe('v2.4.0')
expect(PackageURL?.namespace).toBe('github.com/go-yaml')
expect(PackageURL?.name).toBe('yaml')
test('Parse gopkg.in/ini.v1 no version in URL', () => {
const html = readFileSync(join(__dirname, 'testdata/pkg.go.dev/gopkg.in-ini.v1-1.67.0.html'))
window.document.body.innerHTML = html.toString()

const packageURL: PackageURL | undefined = getArtifactDetailsFromDOM(
ensure(repoType),
'https://pkg.go.dev/gopkg.in/ini.v1'
)

expect(packageURL).toBeDefined()
expect(packageURL?.version).toBe('v1.67.0')
expect(packageURL?.namespace).toBe('github.com/go-ini')
expect(packageURL?.name).toBe('ini')
})
test('should parse a valid Golang page etcd', () => {
const html = readFileSync(join(__dirname, 'testdata/golang.html'))

window.document.body.innerHTML = html.toString()
const PackageURL = getArtifactDetailsFromDOM(ensure(repoType), 'https://pkg.go.dev/github.com/etcd-io/etcd')
test('Parse golang.org/x/text with version in URL', () => {
const packageURL: PackageURL | undefined = getArtifactDetailsFromDOM(
ensure(repoType),
'https://pkg.go.dev/gopkg.in/[email protected]'
)

expect(PackageURL).toBeDefined()
expect(PackageURL?.namespace).toBe('github.com/etcd-io')
expect(PackageURL?.name).toBe('etcd')
expect(PackageURL?.version).toBe('v3.3.25+incompatible')
expect(packageURL).toBeDefined()
expect(packageURL?.version).toBe('v1.65.0')
expect(packageURL?.namespace).toBe('github.com/go-ini')
expect(packageURL?.name).toBe('ini')
})
})
12 changes: 9 additions & 3 deletions src/utils/PageParsing/Golang.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,11 @@ import { generatePackageURLWithNamespace } from './PurlUtils'
https://pkg.go.dev/google.golang.org/[email protected]/runtime/protoimpl ->Todo Version: v1.26.0 ->No vulns, but different namespace and some stuff at the end
*/

const PKG_GO_DEV_VERSION_SELECTOR =
'body > div.Site-content > div > header > div.UnitHeader-content > div > div.UnitHeader-details > span:nth-child(1) > a'
// #main-content a[href="?tab=versions"]
const PKG_GO_DEV_VERSION_SELECTOR = '#main-content a[href="?tab=versions"]'
const GO_PKG_IN_V1 = /^gopkg.in\/([^.]+).*/
const GO_PKG_IN_V2 = /^gopkg.in\/([^/]+)\/([^.]+).*/
const INCOMPATIBLE_VERSION_SUFFIX = '+incompatible'

const parseGolang = (url: string): PackageURL | undefined => {
return parsePkgGoDevURLIntoPackageURL(url)
Expand All @@ -42,6 +43,9 @@ const parsePkgGoDevURLIntoPackageURL = (url: string): PackageURL | undefined =>
const nameVersion = uri.pathname.split('@')

let version = getVersionFromURI(uri)
if (version != null && version.endsWith(INCOMPATIBLE_VERSION_SUFFIX)) {
return undefined
}

if (version !== undefined) {
nameAndNamespace = getName(handleGoPkgIn(nameVersion[0].replace(/^\//, '')))
Expand All @@ -50,8 +54,10 @@ const parsePkgGoDevURLIntoPackageURL = (url: string): PackageURL | undefined =>

if (typeof found !== 'undefined') {
nameAndNamespace = getName(handleGoPkgIn(uri.pathname.replace(/^\//, '')))

version = found.text().trim().replace('Version: ', '').trim()
if (version != null && version.endsWith(INCOMPATIBLE_VERSION_SUFFIX)) {
return undefined
}
}
}

Expand Down
Loading

0 comments on commit e9b1af6

Please sign in to comment.