Skip to content

Commit 93dbe46

Browse files
keshav-spacekeshav-space
authored
Merge pull request #114 from nexB/purl2sym-metadata-4
Add metadata support for openssl, erofs-utils
2 parents 4f957a8 + 81d95fa commit 93dbe46

13 files changed

+13647
-35
lines changed

‎CHANGELOG.rst

+33
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,39 @@ Changelog
22
=========
33

44

5+
v0.5.0
6+
-------
7+
- FetchCode now supports retrieving package info for following generic packages:
8+
* pkg:generic/linux
9+
* pkg:generic/mtd-utils
10+
* pkg:generic/barebox
11+
* pkg:generic/e2fsprogs
12+
* pkg:generic/udhcp
13+
* pkg:generic/miniupnpc
14+
* pkg:generic/miniupnpd
15+
* pkg:generic/minissdpd
16+
* pkg:generic/erofs-utils
17+
* pkg:openssl/openssl
18+
19+
- FetchCode also supports retrieving package info for packages hosted on GitHub specifically.
20+
* pkg:github/avahi/avahi
21+
* pkg:github/bestouff/genext2fs
22+
* pkg:github/dosfstools/dosfstools
23+
* pkg:github/google/brotli
24+
* pkg:github/hewlettpackard/wireless-tools
25+
* pkg:github/inotify-tools/inotify-tools
26+
* pkg:github/libbpf/bpftool
27+
* pkg:github/llvm/llvm-project
28+
* pkg:github/nixos/nix
29+
* pkg:github/plougher/squashfs-tools
30+
* pkg:github/pupnp/pupnp
31+
* pkg:github/python/cpython
32+
* pkg:github/rpm-software-management/rpm
33+
* pkg:github/shadow-maint/shadow
34+
* pkg:github/sqlite/sqlite
35+
* pkg:github/u-boot/u-boot
36+
37+
538
v0.4.0
639
-------
740
- FetchCode now supports retrieving package info for following generic packages:

‎src/fetchcode/package.py

+61-3
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,11 @@
2727

2828
from fetchcode.package_util import GITHUB_SOURCE_BY_PACKAGE
2929
from fetchcode.package_util import IPKG_RELEASES
30+
from fetchcode.package_util import UDHCP_RELEASES
31+
from fetchcode.package_util import ErofsUtilsGitHubSource
3032
from fetchcode.package_util import GitHubSource
3133
from fetchcode.package_util import MiniupnpPackagesGitHubSource
34+
from fetchcode.package_util import OpenSSLGitHubSource
3235
from fetchcode.packagedcode_models import Package
3336
from fetchcode.utils import get_response
3437

@@ -242,7 +245,25 @@ def get_github_data_for_miniupnp(purl):
242245
)
243246

244247

245-
@router.route("pkg:generic/erofs-utils.*",)
248+
@router.route(
249+
"pkg:openssl/openssl.*",
250+
)
251+
def get_github_data_for_openssl(purl):
252+
"""
253+
Yield `Package` object for OpenSSL package from GitHub.
254+
"""
255+
generic_purl = PackageURL.from_string(purl)
256+
github_repo_purl = PackageURL(
257+
type="github",
258+
namespace="openssl",
259+
name="openssl",
260+
version=generic_purl.version,
261+
)
262+
263+
return OpenSSLGitHubSource.get_package_info(github_repo_purl)
264+
265+
266+
@router.route("pkg:generic/erofs-utils.*")
246267
def get_github_data_for_erofs_utils(purl):
247268
"""
248269
Yield `Package` object for erofs-utils package from GitHub.
@@ -255,7 +276,7 @@ def get_github_data_for_erofs_utils(purl):
255276
version=generic_purl.version,
256277
)
257278

258-
return GitHubSource.get_package_info(github_repo_purl)
279+
return ErofsUtilsGitHubSource.get_package_info(github_repo_purl)
259280

260281

261282
@router.route("pkg:bitbucket/.*")
@@ -384,6 +405,37 @@ def get_package_info(cls, package_url):
384405
)
385406

386407

408+
# The udhcp is no longer maintained as a standalone project.
409+
# It has been fully integrated into busybox.
410+
class UdhcpDirectoryListedSource(DirectoryListedSource):
411+
source_url = (
412+
"https://web.archive.org/web/20021209021312/http://udhcp.busybox.net/source/"
413+
)
414+
415+
@classmethod
416+
def get_package_info(cls, package_url):
417+
418+
version = package_url.version
419+
if version and version in UDHCP_RELEASES:
420+
archive = UDHCP_RELEASES[version]
421+
yield Package(
422+
homepage_url=cls.source_url,
423+
download_url=archive["url"],
424+
release_date=archive["date"],
425+
**package_url.to_dict(),
426+
)
427+
428+
else:
429+
for version, data in UDHCP_RELEASES.items():
430+
purl = PackageURL(type="generic", name="udhcp", version=version)
431+
yield Package(
432+
homepage_url=cls.source_url,
433+
download_url=data["url"],
434+
release_date=data["date"],
435+
**purl.to_dict(),
436+
)
437+
438+
387439
class IpkgDirectoryListedSource(DirectoryListedSource):
388440
source_url = "https://web.archive.org/web/20090326020239/http://handhelds.org/download/packages/ipkg/"
389441
is_nested = False
@@ -612,6 +664,7 @@ class BareboxDirectoryListedSource(DirectoryListedSource):
612664
is_nested = False
613665
ignored_files_and_dir = []
614666

667+
615668
class LinuxDirectoryListedSource(DirectoryListedSource):
616669
source_url = "https://cdn.kernel.org/pub/linux/kernel/"
617670
# Source archive ex: linux-1.2.3.tar.gz
@@ -631,8 +684,11 @@ class LinuxDirectoryListedSource(DirectoryListedSource):
631684
"uemacs/",
632685
]
633686

687+
634688
class E2fsprogsDirectoryListedSource(DirectoryListedSource):
635-
source_url = "https://mirrors.edge.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/"
689+
source_url = (
690+
"https://mirrors.edge.kernel.org/pub/linux/kernel/people/tytso/e2fsprogs/"
691+
)
636692
# Source archive ex: e2fsprogs-1.2.3.tar.gz
637693
source_archive_regex = re.compile(r"^(e2fsprogs-)(?P<version>[\w.-]*)(.tar.gz)$")
638694
is_nested = True
@@ -665,6 +721,7 @@ class E2fsprogsDirectoryListedSource(DirectoryListedSource):
665721
"pkg:generic/barebox.*",
666722
"pkg:generic/linux.*",
667723
"pkg:generic/e2fsprogs.*",
724+
"pkg:generic/udhcp.*",
668725
]
669726

670727
DIR_LISTED_SOURCE_BY_PACKAGE_NAME = {
@@ -692,6 +749,7 @@ class E2fsprogsDirectoryListedSource(DirectoryListedSource):
692749
"barebox": BareboxDirectoryListedSource,
693750
"linux": LinuxDirectoryListedSource,
694751
"e2fsprogs": E2fsprogsDirectoryListedSource,
752+
"udhcp": UdhcpDirectoryListedSource,
695753
}
696754

697755

‎src/fetchcode/package_util.py

+88-4
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,6 @@
1414
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
1515
# specific language governing permissions and limitations under the License.
1616

17-
# Since there will be no new releases of ipkg, it's better to
18-
# store them in a dictionary rather than fetching them every time.
19-
2017
import dataclasses
2118
import re
2219

@@ -211,6 +208,54 @@ class RpmGitHubSource(GitHubSource):
211208
}
212209

213210

211+
class OpenSSLGitHubSource(GitHubSource):
212+
version_regex = re.compile(r"(OpenSSL_|openssl-)(?P<version>.+)")
213+
ignored_tag_regex = None
214+
215+
@classmethod
216+
def get_package_info(cls, gh_purl):
217+
218+
packages = get_github_packages(
219+
gh_purl,
220+
cls.version_regex,
221+
cls.ignored_tag_regex,
222+
cls.get_default_package(gh_purl),
223+
)
224+
225+
for package in packages:
226+
package_dict = package.to_dict()
227+
package_dict["type"] = "openssl"
228+
package_dict["namespace"] = None
229+
package_dict["name"] = "openssl"
230+
package_dict["version"] = package_dict["version"].replace("_", ".")
231+
232+
yield package_from_dict(package_dict)
233+
234+
235+
class ErofsUtilsGitHubSource(GitHubSource):
236+
version_regex = None
237+
ignored_tag_regex = None
238+
239+
@classmethod
240+
def get_package_info(cls, gh_purl):
241+
242+
packages = get_github_packages(
243+
gh_purl,
244+
cls.version_regex,
245+
cls.ignored_tag_regex,
246+
cls.get_default_package(gh_purl),
247+
)
248+
249+
for package in packages:
250+
package_dict = package.to_dict()
251+
package_dict["type"] = "generic"
252+
package_dict["namespace"] = None
253+
package_dict["name"] = "erofs-utils"
254+
package_dict["version"] = package_dict["version"].replace("_", ".")
255+
256+
yield package_from_dict(package_dict)
257+
258+
214259
class MiniupnpPackagesGitHubSource(GitHubSource):
215260
version_regex = None
216261
ignored_tag_regex = None
@@ -231,14 +276,53 @@ def get_package_info(cls, gh_purl, package_name):
231276

232277
for package in packages:
233278
package_dict = package.to_dict()
279+
package_dict["type"] = "generic"
234280
package_dict["namespace"] = None
235281
package_dict["name"] = package_name
236-
package_dict["type"] = "generic"
237282
package_dict["version"] = package_dict["version"].replace("_", ".")
238283

239284
yield package_from_dict(package_dict)
240285

241286

287+
# Archive source https://web.archive.org/web/20021209021312/http://udhcp.busybox.net/source/
288+
UDHCP_RELEASES = {
289+
"0.9.1": {
290+
"url": "https://web.archive.org/web/20021209021312/http://udhcp.busybox.net/source//udhcp-0.9.1.tar.gz",
291+
"date": "2001-08-10T20:17:00",
292+
},
293+
"0.9.2": {
294+
"url": "https://web.archive.org/web/20021209021312/http://udhcp.busybox.net/source//udhcp-0.9.2.tar.gz",
295+
"date": "2001-08-10T20:17:00",
296+
},
297+
"0.9.3": {
298+
"url": "https://web.archive.org/web/20021209021312/http://udhcp.busybox.net/source//udhcp-0.9.3.tar.gz",
299+
"date": "2001-08-20T18:23:00",
300+
},
301+
"0.9.4": {
302+
"url": "https://web.archive.org/web/20021209021312/http://udhcp.busybox.net/source//udhcp-0.9.4.tar.gz",
303+
"date": "2001-08-27T15:41:00",
304+
},
305+
"0.9.5": {
306+
"url": "https://web.archive.org/web/20021209021312/http://udhcp.busybox.net/source//udhcp-0.9.5.tar.gz",
307+
"date": "2001-09-14T18:19:00",
308+
},
309+
"0.9.6": {
310+
"url": "https://web.archive.org/web/20021209021312/http://udhcp.busybox.net/source//udhcp-0.9.6.tar.gz",
311+
"date": "2001-10-01T13:38:00",
312+
},
313+
"0.9.7": {
314+
"url": "https://web.archive.org/web/20021209021312/http://udhcp.busybox.net/source//udhcp-0.9.7.tar.gz",
315+
"date": "2002-05-27T00:48:00",
316+
},
317+
"0.9.8": {
318+
"url": "https://web.archive.org/web/20021209021312/http://udhcp.busybox.net/source//udhcp-0.9.8.tar.gz",
319+
"date": "2002-10-31T12:10:00",
320+
},
321+
}
322+
323+
324+
# Since there will be no new releases of ipkg, it's better to
325+
# store them in a dictionary rather than fetching them every time.
242326
IPKG_RELEASES = {
243327
"0.99.88": {
244328
"url": "https://web.archive.org/web/20090326020239/http:/handhelds.org/download/packages/ipkg/ipkg-0.99.88.tar.gz",

0 commit comments

Comments
 (0)