Support more cargo workspace cases

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

src/packagedcode/cargo.py

@@ -21,74 +21,7 @@
 """
 
 
-class CargoTomlHandler(models.DatafileHandler):
-    datasource_id = 'cargo_toml'
-    path_patterns = ('*/Cargo.toml', '*/cargo.toml',)
-    default_package_type = 'cargo'
-    default_primary_language = 'Rust'
-    description = 'Rust Cargo.toml package manifest'
-    documentation_url = 'https://doc.rust-lang.org/cargo/reference/manifest.html'
-
-    @classmethod
-    def parse(cls, location):
-        package_data = toml.load(location, _dict=dict)
-        core_package_data = package_data.get('package', {})
-        workspace = package_data.get('workspace', {})
-
-        name = core_package_data.get('name')
-        version = core_package_data.get('version')
-        description = core_package_data.get('description') or ''
-        description = description.strip()
-
-        authors = core_package_data.get('authors') or []
-        parties = list(get_parties(person_names=authors, party_role='author'))
-
-        extracted_license_statement = core_package_data.get('license')
-        # TODO: load as a notice_text
-        license_file = core_package_data.get('license-file')
-
-        keywords = core_package_data.get('keywords') or []
-        categories = core_package_data.get('categories') or []
-        keywords.extend(categories)
-
-        # cargo dependencies are complex and can be overriden at multiple levels
-        dependencies = []
-        for key, value in core_package_data.items():
-            if key.endswith('dependencies'):
-                dependencies.extend(dependency_mapper(dependencies=value, scope=key))
-
-        # TODO: add file refs:
-        # - readme, include and exclude
-        # TODO: other URLs
-        # - documentation
-
-        vcs_url = core_package_data.get('repository')
-        homepage_url = core_package_data.get('homepage')
-        repository_homepage_url = name and f'https://crates.io/crates/{name}'
-        repository_download_url = name and version and f'https://crates.io/api/v1/crates/{name}/{version}/download'
-        api_data_url = name and f'https://crates.io/api/v1/crates/{name}'
-        extra_data = {}
-        if workspace:
-            extra_data["workspace"] = workspace
-
-        yield models.PackageData(
-            datasource_id=cls.datasource_id,
-            type=cls.default_package_type,
-            name=name,
-            version=version,
-            primary_language=cls.default_primary_language,
-            description=description,
-            parties=parties,
-            extracted_license_statement=extracted_license_statement,
-            vcs_url=vcs_url,
-            homepage_url=homepage_url,
-            repository_homepage_url=repository_homepage_url,
-            repository_download_url=repository_download_url,
-            api_data_url=api_data_url,
-            dependencies=dependencies,
-            extra_data=extra_data,
-        )
-
+class CargoBaseHandler(models.DatafileHandler):
     @classmethod
     def assemble(cls, package_data, resource, codebase, package_adder):
         """
@@ -161,6 +94,79 @@ def update_resource_package_data(cls, package_data, old_package_data, mapping=No
         return old_package_data
 
 
+
+class CargoTomlHandler(CargoBaseHandler):
+    datasource_id = 'cargo_toml'
+    path_patterns = ('*/Cargo.toml', '*/cargo.toml',)
+    default_package_type = 'cargo'
+    default_primary_language = 'Rust'
+    description = 'Rust Cargo.toml package manifest'
+    documentation_url = 'https://doc.rust-lang.org/cargo/reference/manifest.html'
+
+    @classmethod
+    def parse(cls, location):
+        package_data = toml.load(location, _dict=dict)
+        core_package_data = package_data.get('package', {})
+        workspace = package_data.get('workspace', {})
+
+        name = core_package_data.get('name')
+        version = core_package_data.get('version')
+        if isinstance(version, dict) and "workspace" in version:
+            version = "workspace"
+
+        description = core_package_data.get('description') or ''
+        description = description.strip()
+
+        authors = core_package_data.get('authors') or []
+        parties = list(get_parties(person_names=authors, party_role='author'))
+
+        extracted_license_statement = core_package_data.get('license')
+        # TODO: load as a notice_text
+        license_file = core_package_data.get('license-file')
+
+        keywords = core_package_data.get('keywords') or []
+        categories = core_package_data.get('categories') or []
+        keywords.extend(categories)
+
+        # cargo dependencies are complex and can be overriden at multiple levels
+        dependencies = []
+        for key, value in core_package_data.items():
+            if key.endswith('dependencies'):
+                dependencies.extend(dependency_mapper(dependencies=value, scope=key))
+
+        # TODO: add file refs:
+        # - readme, include and exclude
+        # TODO: other URLs
+        # - documentation
+
+        vcs_url = core_package_data.get('repository')
+        homepage_url = core_package_data.get('homepage')
+        repository_homepage_url = name and f'https://crates.io/crates/{name}'
+        repository_download_url = name and version and f'https://crates.io/api/v1/crates/{name}/{version}/download'
+        api_data_url = name and f'https://crates.io/api/v1/crates/{name}'
+        extra_data = {}
+        if workspace:
+            extra_data["workspace"] = workspace
+
+        yield models.PackageData(
+            datasource_id=cls.datasource_id,
+            type=cls.default_package_type,
+            name=name,
+            version=version,
+            primary_language=cls.default_primary_language,
+            description=description,
+            parties=parties,
+            extracted_license_statement=extracted_license_statement,
+            vcs_url=vcs_url,
+            homepage_url=homepage_url,
+            repository_homepage_url=repository_homepage_url,
+            repository_download_url=repository_download_url,
+            api_data_url=api_data_url,
+            dependencies=dependencies,
+            extra_data=extra_data,
+        )
+
+
 CARGO_ATTRIBUTE_MAPPING = {
     # Fields in PackageData model: Fields in cargo
     "homepage_url": "homepage",
@@ -173,7 +179,7 @@ def update_resource_package_data(cls, package_data, old_package_data, mapping=No
     "declared_license_expression_spdx": "declared_license_expression_spdx",
 }
 
-class CargoLockHandler(models.DatafileHandler):
+class CargoLockHandler(CargoBaseHandler):
     datasource_id = 'cargo_lock'
     path_patterns = ('*/Cargo.lock', '*/cargo.lock',)
     default_package_type = 'cargo'
@@ -220,18 +226,6 @@ def parse(cls, location):
             dependencies=dependencies,
         )
 
-    @classmethod
-    def assemble(cls, package_data, resource, codebase, package_adder):
-        """
-        Assemble Cargo.toml and possible Cargo.lock datafiles
-        """
-        yield from cls.assemble_from_many_datafiles(
-            datafile_name_patterns=('Cargo.toml', 'Cargo.lock',),
-            directory=resource.parent(codebase),
-            codebase=codebase,
-            package_adder=package_adder,
-        )
-
 
 def dependency_mapper(dependencies, scope='dependencies'):
     """

src/packagedcode/plugin_package.py

@@ -38,7 +38,8 @@
 from packagedcode.models import PackageWithResources
 
 TRACE = os.environ.get('SCANCODE_DEBUG_PACKAGE_API', False)
-
+TRACE_DEEP = os.environ.get('SCANCODE_DEBUG_PACKAGE_API_DEEP', False)
+TRACE_LICENSE = os.environ.get('SCANCODE_DEBUG_PACKAGE_LICENSE', False)
 
 def logger_debug(*args):
     pass
@@ -207,7 +208,7 @@ def process_codebase(self, codebase, strip_root=False, **kwargs):
             # If we don't detect license in package_data but there is license detected in file
             # we add the license expression from the file to a package
             modified = add_license_from_file(resource, codebase)
-            if TRACE and modified:
+            if TRACE_LICENSE and modified:
                 logger_debug(f'packagedcode: process_codebase: add_license_from_file: modified: {modified}')
 
             if codebase.has_single_resource:
@@ -216,7 +217,7 @@ def process_codebase(self, codebase, strip_root=False, **kwargs):
             # If there is referenced files in a extracted license statement, we follow
             # the references, look for license detections and add them back
             modified = list(add_referenced_license_matches_for_package(resource, codebase))
-            if TRACE and modified:
+            if TRACE_LICENSE and modified:
                 logger_debug(f'packagedcode: process_codebase: add_referenced_license_matches_for_package: modified: {modified}')
 
             # If there is a LICENSE file on the same level as the manifest, and no license
@@ -234,7 +235,7 @@ def process_codebase(self, codebase, strip_root=False, **kwargs):
                 # If there is a unknown reference to a package we add the license
                 # from the package license detection
                 modified = list(add_referenced_license_detection_from_package(resource, codebase))
-                if TRACE and modified:
+                if TRACE_LICENSE and modified:
                     logger_debug(f'packagedcode: process_codebase: add_referenced_license_matches_from_package: modified: {modified}')
 
 
@@ -244,15 +245,15 @@ def add_license_from_file(resource, codebase):
     and the file has license detections, and if so, populate the package_data license
     expression and detection fields from the file license.
     """
-    if TRACE:
+    if TRACE_LICENSE:
         logger_debug(f'packagedcode.plugin_package: add_license_from_file: resource: {resource.path}')
 
     if not resource.is_file:
         return
 
     license_detections_file = resource.license_detections
 
-    if TRACE:
+    if TRACE_LICENSE:
         logger_debug(f'add_license_from_file: license_detections_file: {license_detections_file}')
     if not license_detections_file:
         return
@@ -263,7 +264,7 @@ def add_license_from_file(resource, codebase):
 
     for pkg in package_data:
         license_detections_pkg = pkg["license_detections"]
-        if TRACE:
+        if TRACE_LICENSE:
             logger_debug(f'add_license_from_file: license_detections_pkg: {license_detections_pkg}')
 
         if not license_detections_pkg:
@@ -359,7 +360,7 @@ def get_package_and_deps(codebase, package_adder=add_to_package, strip_root=Fals
                 package_data = PackageData.from_dict(mapping=package_data)
 
                 if TRACE:
-                    logger_debug('  get_package_and_deps: package_data:', package_data)
+                    logger_debug('  get_package_and_deps: package_data.purl:', package_data.purl)
 
                 # Find a handler for this package datasource to assemble collect
                 # packages and deps
@@ -375,8 +376,6 @@ def get_package_and_deps(codebase, package_adder=add_to_package, strip_root=Fals
                 )
 
                 for item in items:
-                    if TRACE:
-                        logger_debug('    get_package_and_deps: item:', item)
 
                     if isinstance(item, Package):
                         if strip_root and not has_single_resource:
@@ -385,6 +384,8 @@ def get_package_and_deps(codebase, package_adder=add_to_package, strip_root=Fals
                                 for dfp in item.datafile_paths
                             ]
                         packages.append(item)
+                        if TRACE:
+                            logger_debug('    get_package_and_deps: Package:', item.purl)
 
                     elif isinstance(item, Dependency):
                         if strip_root and not has_single_resource:
@@ -395,6 +396,9 @@ def get_package_and_deps(codebase, package_adder=add_to_package, strip_root=Fals
                         seen_resource_paths.add(item.path)
 
                         if TRACE:
+                            logger_debug('    get_package_and_deps: Resource:', item.path)
+
+                        if TRACE_DEEP:
                             logger_debug(
                                 '    get_package_and_deps: seen_resource_path:',
                                 seen_resource_paths,