Handle yarn.lock dependencies with @ in constraint #3931

Signed-off-by: Jono Yang <[email protected]>

src/packagedcode/npm.py

@@ -207,7 +207,7 @@ def assemble(cls, package_data, resource, codebase, package_adder):
 
     @classmethod
     def yield_npm_dependencies_and_resources(cls, package_resource, package_data, package_uid, codebase, package_adder):
- 
+
         # in all cases yield possible dependencies
         yield from yield_dependencies_from_package_data(package_data, package_resource.path, package_uid)
 
@@ -424,7 +424,7 @@ def get_workspace_members(cls, workspaces, codebase, workspace_root_path):
                         workspace_members.append(resource)
 
             # Case 3: This is a complex glob pattern, we are doing a full codebase walk
-            # and glob matching each resource 
+            # and glob matching each resource
             else:
                 for resource in workspace_root_path:
                     if NpmPackageJsonHandler.is_datafile(resource.location) and fnmatch.fnmatch(
@@ -469,7 +469,7 @@ def update_workspace_members(cls, workspace_members, codebase):
                 workspace_package_versions_by_base_purl[base_purl] = version
 
         # Update workspace member package information from
-        # workspace level data 
+        # workspace level data
         for base_purl, dependency in workspace_dependencies_by_base_purl.items():
             extracted_requirement = dependency.get('extracted_requirement')
             if 'workspace' in extracted_requirement:
@@ -1011,6 +1011,14 @@ def parse(cls, location, package_only=False):
                     if '"' in ns_name:
                         ns_name = ns_name.replace('"', '')
                     ns, _ , name = ns_name.rpartition('/')
+
+                    # sometimes constraints appear in the form of
+                    # wrap-ansi-cjs "npm:wrap-ansi@^7.0.0"
+                    if '@' in constraint:
+                        # "npm:wrap-ansi" should be appended to `name`, joined
+                        # with an "@"
+                        constraint_package, _, constraint = constraint.partition('@')
+                        name = f'{name}@{constraint_package}'
                     sub_dependencies.append((ns, name, constraint,))
 
                 elif line.startswith(' ' * 2):
@@ -1112,7 +1120,7 @@ def parse(cls, location, package_only=False):
                 resolved_package=resolved_package_data.to_dict(),
             )
 
-            if not dep_purl in dependencies_by_purl: 
+            if not dep_purl in dependencies_by_purl:
                 dependencies_by_purl[dep_purl] = dep.to_dict()
             else:
                 # FIXME: We have duplicate dependencies because of aliases
@@ -1176,7 +1184,7 @@ def parse(cls, location, package_only=False):
                     _, name_version = sections
                 elif len(sections) == 3:
                     _, namespace, name_version = sections
-                
+
                 name, version = name_version.split("@")
             elif major_v == "5" or is_shrinkwrap:
                 if len(sections) == 3:
@@ -1264,7 +1272,7 @@ def parse(cls, location, package_only=False):
             for key in extra_data_fields:
                 value = data.get(key, None)
                 if value is not None:
-                    extra_data_deps[key] = value 
+                    extra_data_deps[key] = value
 
             dependency_data = models.DependentPackage(
                 purl=purl,
@@ -1762,7 +1770,7 @@ def deps_mapper(deps, package, field_name, is_direct=True):
             deps_by_name[npm_name] = d
 
     for fqname, requirement in deps.items():
-        # Handle cases in ``resolutions`` with ``**`` 
+        # Handle cases in ``resolutions`` with ``**``
         # "resolutions": {
         #   "**/@typescript-eslint/eslint-plugin": "^4.1.1",
         if fqname.startswith('**'):

tests/packagedcode/data/npm/yarn-lock/v1-other-constraint/yarn.lock

@@ -0,0 +1,14 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+"@isaacs/cliui@^8.0.2":
+  version "8.0.2"
+  resolved "https://registry.yarnpkg.com/@isaacs/cliui/-/cliui-8.0.2.tgz#b37667b7bc181c168782259bab42474fbf52b550"
+  integrity sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==
+  dependencies:
+    string-width "^5.1.2"
+    string-width-cjs "npm:string-width@^4.2.0"
+    strip-ansi "^7.0.1"
+    strip-ansi-cjs "npm:strip-ansi@^6.0.1"
+    wrap-ansi "^8.1.0"

tests/packagedcode/data/npm/yarn-lock/v1-other-constraint/yarn.lock-expected

@@ -0,0 +1,157 @@
+[
+  {
+    "type": "npm",
+    "namespace": null,
+    "name": null,
+    "version": null,
+    "qualifiers": {},
+    "subpath": null,
+    "primary_language": "JavaScript",
+    "description": null,
+    "release_date": null,
+    "parties": [],
+    "keywords": [],
+    "homepage_url": null,
+    "download_url": null,
+    "size": null,
+    "sha1": null,
+    "md5": null,
+    "sha256": null,
+    "sha512": null,
+    "bug_tracking_url": null,
+    "code_view_url": null,
+    "vcs_url": null,
+    "copyright": null,
+    "holder": null,
+    "declared_license_expression": null,
+    "declared_license_expression_spdx": null,
+    "license_detections": [],
+    "other_license_expression": null,
+    "other_license_expression_spdx": null,
+    "other_license_detections": [],
+    "extracted_license_statement": null,
+    "notice_text": null,
+    "source_packages": [],
+    "file_references": [],
+    "is_private": false,
+    "is_virtual": false,
+    "extra_data": {},
+    "dependencies": [
+      {
+        "purl": "pkg:npm/%40isaacs/[email protected]",
+        "extracted_requirement": "^8.0.2",
+        "scope": "dependencies",
+        "is_runtime": true,
+        "is_optional": false,
+        "is_resolved": true,
+        "is_direct": false,
+        "resolved_package": {
+          "type": "npm",
+          "namespace": "@isaacs",
+          "name": "cliui",
+          "version": "8.0.2",
+          "qualifiers": {},
+          "subpath": null,
+          "primary_language": "JavaScript",
+          "description": null,
+          "release_date": null,
+          "parties": [],
+          "keywords": [],
+          "homepage_url": null,
+          "download_url": "https://registry.yarnpkg.com/@isaacs/cliui/-/cliui-8.0.2.tgz",
+          "size": null,
+          "sha1": "b37667b7bc181c168782259bab42474fbf52b550",
+          "md5": null,
+          "sha256": null,
+          "sha512": "3bc8dc8da6d76a578e1bd0d0d3e0115d66414df9cfe16340ab3ba224aee5978e009b118abff2763384cf8f18d8df39c109fbc15c5cee726d6dc1dc85c9b16a10",
+          "bug_tracking_url": null,
+          "code_view_url": null,
+          "vcs_url": null,
+          "copyright": null,
+          "holder": null,
+          "declared_license_expression": null,
+          "declared_license_expression_spdx": null,
+          "license_detections": [],
+          "other_license_expression": null,
+          "other_license_expression_spdx": null,
+          "other_license_detections": [],
+          "extracted_license_statement": null,
+          "notice_text": null,
+          "source_packages": [],
+          "file_references": [],
+          "is_private": false,
+          "is_virtual": true,
+          "extra_data": {},
+          "dependencies": [
+            {
+              "purl": "pkg:npm/string-width",
+              "extracted_requirement": "^5.1.2",
+              "scope": "dependencies",
+              "is_runtime": true,
+              "is_optional": false,
+              "is_resolved": false,
+              "is_direct": true,
+              "resolved_package": {},
+              "extra_data": {}
+            },
+            {
+              "purl": "pkg:npm/string-width-cjs%40%22npm:string-width",
+              "extracted_requirement": "^4.2.0",
+              "scope": "dependencies",
+              "is_runtime": true,
+              "is_optional": false,
+              "is_resolved": false,
+              "is_direct": true,
+              "resolved_package": {},
+              "extra_data": {}
+            },
+            {
+              "purl": "pkg:npm/strip-ansi",
+              "extracted_requirement": "^7.0.1",
+              "scope": "dependencies",
+              "is_runtime": true,
+              "is_optional": false,
+              "is_resolved": false,
+              "is_direct": true,
+              "resolved_package": {},
+              "extra_data": {}
+            },
+            {
+              "purl": "pkg:npm/strip-ansi-cjs%40%22npm:strip-ansi",
+              "extracted_requirement": "^6.0.1",
+              "scope": "dependencies",
+              "is_runtime": true,
+              "is_optional": false,
+              "is_resolved": false,
+              "is_direct": true,
+              "resolved_package": {},
+              "extra_data": {}
+            },
+            {
+              "purl": "pkg:npm/wrap-ansi",
+              "extracted_requirement": "^8.1.0",
+              "scope": "dependencies",
+              "is_runtime": true,
+              "is_optional": false,
+              "is_resolved": false,
+              "is_direct": true,
+              "resolved_package": {},
+              "extra_data": {}
+            }
+          ],
+          "repository_homepage_url": "https://www.npmjs.com/package/@isaacs/cliui",
+          "repository_download_url": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
+          "api_data_url": "https://registry.npmjs.org/@isaacs%2fcliui/8.0.2",
+          "datasource_id": "yarn_lock_v1",
+          "purl": "pkg:npm/%40isaacs/[email protected]"
+        },
+        "extra_data": {}
+      }
+    ],
+    "repository_homepage_url": null,
+    "repository_download_url": null,
+    "api_data_url": null,
+    "datasource_id": "yarn_lock_v1",
+    "purl": null
+  }
+]

tests/packagedcode/test_npm.py

@@ -360,6 +360,13 @@ def test_npm_yarn_with_package_json_resolve_dependencies(self):
             expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES
         )
 
+    def test_npm_yarn_lock_v1_parse_with_other_version_constraint(self):
+        test_file = self.get_test_loc('npm/yarn-lock/v1-other-constraint/yarn.lock')
+        expected_loc = self.get_test_loc(
+            'npm/yarn-lock/v1-other-constraint/yarn.lock-expected')
+        packages = npm.YarnLockV1Handler.parse(test_file)
+        self.check_packages_data(packages, expected_loc, regen=REGEN_TEST_FIXTURES)
+
     def test_is_datafile_pnpm_shrinkwrap_yaml(self):
         test_file = self.get_test_loc('npm/pnpm/shrinkwrap/v3/vuepack/shrinkwrap.yaml')
         assert npm.PnpmShrinkwrapYamlHandler.is_datafile(test_file)