diff --git a/cyclonedx/model/__init__.py b/cyclonedx/model/__init__.py index 0fff1bf4..c850a6c7 100644 --- a/cyclonedx/model/__init__.py +++ b/cyclonedx/model/__init__.py @@ -16,6 +16,7 @@ import re from datetime import datetime, timezone from enum import Enum +from functools import reduce from hashlib import sha1 from itertools import zip_longest from typing import Any, Iterable, Optional, Tuple, TypeVar @@ -424,16 +425,47 @@ class XsUri(serializable.helpers.BaseHelper): .. note:: See XSD definition for xsd:anyURI: http://www.datypic.com/sc/xsd/t-xsd_anyURI.html + See JSON Schema definition for iri-reference: https://tools.ietf.org/html/rfc3987 """ _INVALID_URI_REGEX = re.compile(r'%(?![0-9A-F]{2})|#.*#', re.IGNORECASE + re.MULTILINE) + __SPEC_REPLACEMENTS = ( + (' ', '%20'), + ('[', '%5B'), + (']', '%5D'), + ('<', '%3C'), + ('>', '%3E'), + ('{', '%7B'), + ('}', '%7D'), + ) + + @staticmethod + def __spec_replace(v: str, r: Tuple[str, str]) -> str: + return v.replace(*r) + + @classmethod + def _spec_migrate(cls, o: str) -> str: + """ + Make a string valid to + - XML::anyURI spec. + - JSON::iri-reference spec. + + BEST EFFORT IMPLEMENTATION + + @see http://www.w3.org/TR/xmlschema-2/#anyURI + @see http://www.datypic.com/sc/xsd/t-xsd_anyURI.html + @see https://datatracker.ietf.org/doc/html/rfc2396 + @see https://datatracker.ietf.org/doc/html/rfc3987 + """ + return reduce(cls.__spec_replace, cls.__SPEC_REPLACEMENTS, o) + def __init__(self, uri: str) -> None: if re.search(XsUri._INVALID_URI_REGEX, uri): raise InvalidUriException( f"Supplied value '{uri}' does not appear to be a valid URI." ) - self._uri = uri + self._uri = self._spec_migrate(uri) @property @serializable.json_name('.') diff --git a/tests/_data/models.py b/tests/_data/models.py index 74e492da..ea17080c 100644 --- a/tests/_data/models.py +++ b/tests/_data/models.py @@ -754,6 +754,31 @@ def get_bom_with_multiple_licenses() -> Bom: ) +def get_bom_for_issue_497_urls() -> Bom: + """regression test for issue #497 + see https://github.com/CycloneDX/cyclonedx-python-lib/issues/497 + """ + return _make_bom(components=[ + Component(name='dummy', bom_ref='dummy', external_references=[ + ExternalReference( + type=ExternalReferenceType.OTHER, + comment='nothing special', + url=XsUri('https://acme.org') + ), + ExternalReference( + type=ExternalReferenceType.OTHER, + comment='control characters', + url=XsUri('https://acme.org/?foo=sp ace&bar[23]=42<=1<2>=3>2&cb={lol}') + ), + ExternalReference( + type=ExternalReferenceType.OTHER, + comment='pre-encoded', + url=XsUri('https://acme.org/?bar%5b23%5D=42') + ), + ]) + ]) + + def bom_all_same_bomref() -> Tuple[Bom, int]: bom = Bom() bom.metadata.component = Component(name='root', bom_ref='foo', components=[ @@ -774,13 +799,18 @@ def bom_all_same_bomref() -> Tuple[Bom, int]: if n.startswith('get_bom_') and not n.endswith('_invalid') ) +all_get_bom_funct_valid_immut = tuple( + (n, f) for n, f in getmembers(sys.modules[__name__], isfunction) + if n.startswith('get_bom_') and not n.endswith('_invalid') and not n.endswith('_migrate') +) + all_get_bom_funct_invalid = tuple( (n, f) for n, f in getmembers(sys.modules[__name__], isfunction) if n.startswith('get_bom_') and n.endswith('_invalid') ) all_get_bom_funct_with_incomplete_deps = { - # List of functions that return BOM with an incomplte dependency graph. + # List of functions that return BOM with an incomplete dependency graph. # It is expected that some process auto-fixes this before actual serialization takes place. get_bom_just_complete_metadata, get_bom_with_component_setuptools_basic, @@ -797,4 +827,5 @@ def bom_all_same_bomref() -> Tuple[Bom, int]: get_bom_with_services_simple, get_bom_with_licenses, get_bom_with_multiple_licenses, + get_bom_for_issue_497_urls, } diff --git a/tests/_data/snapshots/get_bom_for_issue_497_urls-1.0.xml.bin b/tests/_data/snapshots/get_bom_for_issue_497_urls-1.0.xml.bin new file mode 100644 index 00000000..068b881e --- /dev/null +++ b/tests/_data/snapshots/get_bom_for_issue_497_urls-1.0.xml.bin @@ -0,0 +1,10 @@ + + + + + dummy + + false + + + diff --git a/tests/_data/snapshots/get_bom_for_issue_497_urls-1.1.xml.bin b/tests/_data/snapshots/get_bom_for_issue_497_urls-1.1.xml.bin new file mode 100644 index 00000000..d006b51e --- /dev/null +++ b/tests/_data/snapshots/get_bom_for_issue_497_urls-1.1.xml.bin @@ -0,0 +1,23 @@ + + + + + dummy + + + + https://acme.org + nothing special + + + https://acme.org/?bar%5b23%5D=42 + pre-encoded + + + https://acme.org/?foo=sp%20ace&bar%5B23%5D=42&lt=1%3C2&gt=3%3E2&cb=%7Blol%7D + control characters + + + + + diff --git a/tests/_data/snapshots/get_bom_for_issue_497_urls-1.2.json.bin b/tests/_data/snapshots/get_bom_for_issue_497_urls-1.2.json.bin new file mode 100644 index 00000000..db13f23c --- /dev/null +++ b/tests/_data/snapshots/get_bom_for_issue_497_urls-1.2.json.bin @@ -0,0 +1,47 @@ +{ + "components": [ + { + "bom-ref": "dummy", + "externalReferences": [ + { + "comment": "nothing special", + "type": "other", + "url": "https://acme.org" + }, + { + "comment": "pre-encoded", + "type": "other", + "url": "https://acme.org/?bar%5b23%5D=42" + }, + { + "comment": "control characters", + "type": "other", + "url": "https://acme.org/?foo=sp%20ace&bar%5B23%5D=42<=1%3C2>=3%3E2&cb=%7Blol%7D" + } + ], + "name": "dummy", + "type": "library", + "version": "" + } + ], + "dependencies": [ + { + "ref": "dummy" + } + ], + "metadata": { + "timestamp": "2023-01-07T13:44:32.312678+00:00", + "tools": [ + { + "name": "cyclonedx-python-lib", + "vendor": "CycloneDX", + "version": "TESTING" + } + ] + }, + "serialNumber": "urn:uuid:1441d33a-e0fc-45b5-af3b-61ee52a88bac", + "version": 1, + "$schema": "http://cyclonedx.org/schema/bom-1.2b.schema.json", + "bomFormat": "CycloneDX", + "specVersion": "1.2" +} \ No newline at end of file diff --git a/tests/_data/snapshots/get_bom_for_issue_497_urls-1.2.xml.bin b/tests/_data/snapshots/get_bom_for_issue_497_urls-1.2.xml.bin new file mode 100644 index 00000000..d2da5f03 --- /dev/null +++ b/tests/_data/snapshots/get_bom_for_issue_497_urls-1.2.xml.bin @@ -0,0 +1,36 @@ + + + + 2023-01-07T13:44:32.312678+00:00 + + + CycloneDX + cyclonedx-python-lib + TESTING + + + + + + dummy + + + + https://acme.org + nothing special + + + https://acme.org/?bar%5b23%5D=42 + pre-encoded + + + https://acme.org/?foo=sp%20ace&bar%5B23%5D=42&lt=1%3C2&gt=3%3E2&cb=%7Blol%7D + control characters + + + + + + + + diff --git a/tests/_data/snapshots/get_bom_for_issue_497_urls-1.3.json.bin b/tests/_data/snapshots/get_bom_for_issue_497_urls-1.3.json.bin new file mode 100644 index 00000000..23430184 --- /dev/null +++ b/tests/_data/snapshots/get_bom_for_issue_497_urls-1.3.json.bin @@ -0,0 +1,47 @@ +{ + "components": [ + { + "bom-ref": "dummy", + "externalReferences": [ + { + "comment": "nothing special", + "type": "other", + "url": "https://acme.org" + }, + { + "comment": "pre-encoded", + "type": "other", + "url": "https://acme.org/?bar%5b23%5D=42" + }, + { + "comment": "control characters", + "type": "other", + "url": "https://acme.org/?foo=sp%20ace&bar%5B23%5D=42<=1%3C2>=3%3E2&cb=%7Blol%7D" + } + ], + "name": "dummy", + "type": "library", + "version": "" + } + ], + "dependencies": [ + { + "ref": "dummy" + } + ], + "metadata": { + "timestamp": "2023-01-07T13:44:32.312678+00:00", + "tools": [ + { + "name": "cyclonedx-python-lib", + "vendor": "CycloneDX", + "version": "TESTING" + } + ] + }, + "serialNumber": "urn:uuid:1441d33a-e0fc-45b5-af3b-61ee52a88bac", + "version": 1, + "$schema": "http://cyclonedx.org/schema/bom-1.3a.schema.json", + "bomFormat": "CycloneDX", + "specVersion": "1.3" +} \ No newline at end of file diff --git a/tests/_data/snapshots/get_bom_for_issue_497_urls-1.3.xml.bin b/tests/_data/snapshots/get_bom_for_issue_497_urls-1.3.xml.bin new file mode 100644 index 00000000..e80d642e --- /dev/null +++ b/tests/_data/snapshots/get_bom_for_issue_497_urls-1.3.xml.bin @@ -0,0 +1,36 @@ + + + + 2023-01-07T13:44:32.312678+00:00 + + + CycloneDX + cyclonedx-python-lib + TESTING + + + + + + dummy + + + + https://acme.org + nothing special + + + https://acme.org/?bar%5b23%5D=42 + pre-encoded + + + https://acme.org/?foo=sp%20ace&bar%5B23%5D=42&lt=1%3C2&gt=3%3E2&cb=%7Blol%7D + control characters + + + + + + + + diff --git a/tests/_data/snapshots/get_bom_for_issue_497_urls-1.4.json.bin b/tests/_data/snapshots/get_bom_for_issue_497_urls-1.4.json.bin new file mode 100644 index 00000000..b9da7b14 --- /dev/null +++ b/tests/_data/snapshots/get_bom_for_issue_497_urls-1.4.json.bin @@ -0,0 +1,80 @@ +{ + "components": [ + { + "bom-ref": "dummy", + "externalReferences": [ + { + "comment": "nothing special", + "type": "other", + "url": "https://acme.org" + }, + { + "comment": "pre-encoded", + "type": "other", + "url": "https://acme.org/?bar%5b23%5D=42" + }, + { + "comment": "control characters", + "type": "other", + "url": "https://acme.org/?foo=sp%20ace&bar%5B23%5D=42<=1%3C2>=3%3E2&cb=%7Blol%7D" + } + ], + "name": "dummy", + "type": "library" + } + ], + "dependencies": [ + { + "ref": "dummy" + } + ], + "metadata": { + "timestamp": "2023-01-07T13:44:32.312678+00:00", + "tools": [ + { + "externalReferences": [ + { + "type": "build-system", + "url": "https://github.com/CycloneDX/cyclonedx-python-lib/actions" + }, + { + "type": "distribution", + "url": "https://pypi.org/project/cyclonedx-python-lib/" + }, + { + "type": "documentation", + "url": "https://cyclonedx-python-library.readthedocs.io/" + }, + { + "type": "issue-tracker", + "url": "https://github.com/CycloneDX/cyclonedx-python-lib/issues" + }, + { + "type": "license", + "url": "https://github.com/CycloneDX/cyclonedx-python-lib/blob/main/LICENSE" + }, + { + "type": "release-notes", + "url": "https://github.com/CycloneDX/cyclonedx-python-lib/blob/main/CHANGELOG.md" + }, + { + "type": "vcs", + "url": "https://github.com/CycloneDX/cyclonedx-python-lib" + }, + { + "type": "website", + "url": "https://github.com/CycloneDX/cyclonedx-python-lib/#readme" + } + ], + "name": "cyclonedx-python-lib", + "vendor": "CycloneDX", + "version": "TESTING" + } + ] + }, + "serialNumber": "urn:uuid:1441d33a-e0fc-45b5-af3b-61ee52a88bac", + "version": 1, + "$schema": "http://cyclonedx.org/schema/bom-1.4.schema.json", + "bomFormat": "CycloneDX", + "specVersion": "1.4" +} \ No newline at end of file diff --git a/tests/_data/snapshots/get_bom_for_issue_497_urls-1.4.xml.bin b/tests/_data/snapshots/get_bom_for_issue_497_urls-1.4.xml.bin new file mode 100644 index 00000000..76017afb --- /dev/null +++ b/tests/_data/snapshots/get_bom_for_issue_497_urls-1.4.xml.bin @@ -0,0 +1,61 @@ + + + + 2023-01-07T13:44:32.312678+00:00 + + + CycloneDX + cyclonedx-python-lib + TESTING + + + https://github.com/CycloneDX/cyclonedx-python-lib/actions + + + https://pypi.org/project/cyclonedx-python-lib/ + + + https://cyclonedx-python-library.readthedocs.io/ + + + https://github.com/CycloneDX/cyclonedx-python-lib/issues + + + https://github.com/CycloneDX/cyclonedx-python-lib/blob/main/LICENSE + + + https://github.com/CycloneDX/cyclonedx-python-lib/blob/main/CHANGELOG.md + + + https://github.com/CycloneDX/cyclonedx-python-lib + + + https://github.com/CycloneDX/cyclonedx-python-lib/#readme + + + + + + + + dummy + + + https://acme.org + nothing special + + + https://acme.org/?bar%5b23%5D=42 + pre-encoded + + + https://acme.org/?foo=sp%20ace&bar%5B23%5D=42&lt=1%3C2&gt=3%3E2&cb=%7Blol%7D + control characters + + + + + + + + diff --git a/tests/test_deserialize_json.py b/tests/test_deserialize_json.py index c3b670d0..cc85da43 100644 --- a/tests/test_deserialize_json.py +++ b/tests/test_deserialize_json.py @@ -28,13 +28,13 @@ from cyclonedx.model.license import DisjunctiveLicense, LicenseExpression, LicenseRepository from cyclonedx.schema import OutputFormat, SchemaVersion from tests import OWN_DATA_DIRECTORY, DeepCompareMixin, SnapshotMixin, mksname, uuid_generator -from tests._data.models import all_get_bom_funct_valid, all_get_bom_funct_with_incomplete_deps +from tests._data.models import all_get_bom_funct_valid_immut, all_get_bom_funct_with_incomplete_deps @ddt class TestDeserializeJson(TestCase, SnapshotMixin, DeepCompareMixin): - @named_data(*all_get_bom_funct_valid) + @named_data(*all_get_bom_funct_valid_immut) @patch('cyclonedx.model.ThisTool._version', 'TESTING') @patch('cyclonedx.model.bom_ref.uuid4', side_effect=uuid_generator(0, version=4)) def test_prepared(self, get_bom: Callable[[], Bom], *_: Any, **__: Any) -> None: diff --git a/tests/test_deserialize_xml.py b/tests/test_deserialize_xml.py index d4c5688a..c8f048d9 100644 --- a/tests/test_deserialize_xml.py +++ b/tests/test_deserialize_xml.py @@ -25,13 +25,13 @@ from cyclonedx.model.bom import Bom from cyclonedx.schema import OutputFormat, SchemaVersion from tests import DeepCompareMixin, SnapshotMixin, mksname, uuid_generator -from tests._data.models import all_get_bom_funct_valid, all_get_bom_funct_with_incomplete_deps +from tests._data.models import all_get_bom_funct_valid_immut, all_get_bom_funct_with_incomplete_deps @ddt class TestDeserializeXml(TestCase, SnapshotMixin, DeepCompareMixin): - @named_data(*all_get_bom_funct_valid) + @named_data(*all_get_bom_funct_valid_immut) @patch('cyclonedx.model.ThisTool._version', 'TESTING') @patch('cyclonedx.model.bom_ref.uuid4', side_effect=uuid_generator(0, version=4)) def test_prepared(self, get_bom: Callable[[], Bom], *_: Any, **__: Any) -> None: