Basic Packagedcode module for handling Rust's cargo crates

setup.py

@@ -178,6 +178,7 @@ def read(*names, **kwargs):
         'packageurl-python >= 0.7.0',
         'xmltodict >= 0.11.0',
         'javaproperties >= 0.5',
+        'toml >= 0.10.0',
 
         # scancode
         'click >= 6.0.0, < 7.0.0',

src/packagedcode/__init__.py

@@ -26,6 +26,7 @@
 from __future__ import unicode_literals
 
 from packagedcode import models
+from packagedcode import cargo
 from packagedcode import freebsd
 from packagedcode import haxe
 from packagedcode import maven
@@ -54,6 +55,7 @@
     npm.NpmPackage,
     phpcomposer.PHPComposerPackage,
     haxe.HaxePackage,
+    cargo.RustCargoCrate,
     models.MeteorPackage,
     models.BowerPackage,
     freebsd.FreeBSDPackage,

src/packagedcode/cargo.py

@@ -0,0 +1,124 @@
+
+# Copyright (c) 2019 nexB Inc. and others. All rights reserved.
+# http://nexb.com and https://github.com/nexB/scancode-toolkit/
+# The ScanCode software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode require an acknowledgment.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# When you publish or redistribute any data created with ScanCode or any ScanCode
+# derivative work, you must accompany this data with the following acknowledgment:
+#
+#  Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+#  ScanCode should be considered or used as legal advice. Consult an Attorney
+#  for any legal advice.
+#  ScanCode is a free software code scanning tool from nexB Inc. and others.
+#  Visit https://github.com/nexB/scancode-toolkit/ for support and download.
+
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from collections import OrderedDict
+
+import io
+import toml
+import logging
+
+import attr
+
+from commoncode import filetype
+from commoncode import fileutils
+from packagedcode import models
+
+
+"""
+Handle Rust cargo crates
+"""
+
+TRACE = False
+
+logger = logging.getLogger(__name__)
+
+if TRACE:
+    import sys
+    logging.basicConfig(stream=sys.stdout)
+    logger.setLevel(logging.DEBUG)
+
+
+@attr.s()
+class RustCargoCrate(models.Package):
+    metafiles = ('Cargo.toml',)
+    default_type = 'cargo'
+    default_primary_language = 'Rust'
+    default_web_baseurl = "https://crates.io/"
+    default_download_baseurl = "https://crates.io/api/v1/"
+    default_api_baseurl = "https://crates.io/api/v1/"
+
+    @classmethod
+    def recognize(cls, location):
+        return parse(location)
+
+    @classmethod
+    def get_package_root(cls, manifest_resource, codebase):
+        return manifest_resource.parent(codebase)
+
+    def repository_homepage_url(self, baseurl=default_web_baseurl):
+        return '{}/crates/{}'.format(baseurl, self.name)
+
+    def repository_download_url(self, baseurl=default_download_baseurl):
+        return '{}/crates/{}/{}/download'.format(baseurl, self.name, self.version)
+
+    def api_data_url(self, baseurl=default_api_baseurl):
+        return '{}/crates/{}'.format(baseurl, self.name)
+
+    def compute_normalized_license(self):
+        return models.compute_normalized_license(self.declared_license)
+
+
+def is_cargo_toml(location):
+    return (filetype.is_file(location) and fileutils.file_name(location).lower() == 'cargo.toml')
+
+
+def parse(location):
+    """
+    Return a Package object from a Cargo.toml file or None.
+    """
+    if not is_cargo_toml(location):
+        return
+
+    with io.open(location, encoding='utf-8') as loc:
+        package_data = toml.load(location, _dict=OrderedDict)
+
+    return build_package(package_data)
+
+
+def build_package(package_data):
+    """
+    Return a Pacakge object from a package data mapping or None.
+    """
+
+    name = package_data.get('package').get('name')
+    version = package_data.get('package').get('version')
+
+    # TODO: Remove this ordered_dict_map once cargo.py is able to handle
+    # the appropriate data (source_packages, dependencies, etc..)
+    # At the moment, this is only useful for making tests pass
+    ordered_dict_map = {}
+    for key in ("source_packages", "dependencies", "keywords", "parties"):
+        ordered_dict_map[key] = OrderedDict()
+
+    package = RustCargoCrate(
+        name=name,
+        version=version,
+        **ordered_dict_map
+    )
+
+    return package

src/packagedcode/npm.py

@@ -219,7 +219,7 @@ def build_package(package_data):
         if homepage:
             homepage = homepage[0]
         else:
-            homepage =''
+            homepage = ''
     namespace, name = split_scoped_package_name(name)
     package = NpmPackage(
         namespace=namespace or None,

tests/packagedcode/data/cargo/clap/Cargo.toml

@@ -0,0 +1,92 @@
+[package]
+
+name = "clap"
+version = "2.32.0"
+authors = ["Kevin K. <[email protected]>"]
+exclude = ["examples/*", "clap-test/*", "tests/*", "benches/*", "*.png", "clap-perf/*", "*.dot"]
+repository = "https://github.com/clap-rs/clap"
+documentation = "https://docs.rs/clap/"
+homepage = "https://clap.rs/"
+readme = "README.md"
+license = "MIT"
+keywords = ["argument", "cli", "arg", "parser", "parse"]
+categories = ["command-line-interface"]
+description = """
+A simple to use, efficient, and full featured  Command Line Argument Parser
+"""
+
+[badges]
+travis-ci = { repository = "clap-rs/clap" }
+appveyor = { repository = "clap-rs/clap" }
+coveralls = { repository = "clap-rs/clap", branch = "master" }
+is-it-maintained-issue-resolution = { repository = "clap-rs/clap" }
+is-it-maintained-open-issues = { repository = "clap-rs/clap" }
+maintenance = {status = "actively-developed"}
+
+[dependencies]
+bitflags              = "1.0"
+unicode-width         = "0.1.4"
+textwrap              = "0.10.0"
+strsim    = { version = "0.8",  optional = true }
+yaml-rust = { version = "0.3.5",  optional = true }
+clippy    = { version = "~0.0.166", optional = true }
+atty      = { version = "0.2.2",  optional = true }
+vec_map   = { version = "0.8", optional = true }
+term_size = { version = "0.3.0", optional = true }
+
+[target.'cfg(not(windows))'.dependencies]
+ansi_term = { version = "0.11",  optional = true }
+
+[dev-dependencies]
+regex = "1"
+# lazy_static 1.2 requires Rust 1.24.1+
+lazy_static = "~1.1"
+version-sync = "0.5"
+
+[features]
+default     = ["suggestions", "color", "vec_map"]
+suggestions = ["strsim"]
+color       = ["ansi_term", "atty"]
+wrap_help   = ["term_size", "textwrap/term_size"]
+yaml        = ["yaml-rust"]
+unstable    = [] # for building with unstable clap features (doesn't require nightly Rust) (currently none)
+nightly     = [] # for building with unstable Rust features (currently none)
+lints       = ["clippy"] # Requires nightly Rust
+debug       = [] # Enables debug messages
+no_cargo    = [] # Enable if you're not using Cargo, disables Cargo-env-var-dependent macros
+doc         = ["yaml"] # All the features which add to documentation
+
+[profile.release]
+opt-level = 3
+debug = false
+rpath = false
+lto = true
+debug-assertions = false
+codegen-units = 1
+
+[profile.dev]
+opt-level = 0
+debug = true
+rpath = false
+lto = false
+debug-assertions = true
+codegen-units = 4
+
+[profile.test]
+opt-level = 1
+debug = true
+rpath = false
+lto = false
+debug-assertions = true
+codegen-units = 4
+
+[profile.bench]
+opt-level = 3
+debug = false
+rpath = false
+lto = true
+debug-assertions = false
+codegen-units = 1
+
+[package.metadata.docs.rs]
+features = ["doc"]

tests/packagedcode/data/cargo/clap/Cargo.toml.expected

@@ -0,0 +1,35 @@
+{
+ "type": "cargo",
+ "namespace": null,
+ "name": "clap",
+ "version": "2.32.0",
+ "qualifiers": null,
+ "subpath": null,
+ "primary_language": "Rust",
+ "description": null,
+ "release_date": null,
+ "parties": {},
+ "keywords": {},
+ "homepage_url": null,
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "license_expression": null,
+ "declared_license": null,
+ "notice_text": null,
+ "manifest_path": null,
+ "dependencies": {},
+ "contains_source_code": null,
+ "source_packages": {},
+ "purl": "pkg:cargo/[email protected]",
+ "repository_homepage_url": "https://crates.io//crates/clap",
+ "repository_download_url": "https://crates.io/api/v1//crates/clap/2.32.0/download",
+ "api_data_url": "https://crates.io/api/v1//crates/clap"
+}

tests/packagedcode/data/cargo/clippy/Cargo.toml

@@ -0,0 +1,66 @@
+[package]
+name = "clippy"
+version = "0.0.212"
+authors = [
+	"Manish Goregaokar <[email protected]>",
+	"Andre Bogus <[email protected]>",
+	"Georg Brandl <[email protected]>",
+	"Martin Carton <[email protected]>",
+	"Oliver Schneider <[email protected]>"
+]
+description = "A bunch of helpful lints to avoid common pitfalls in Rust"
+repository = "https://github.com/rust-lang/rust-clippy"
+readme = "README.md"
+license = "MIT/Apache-2.0"
+keywords = ["clippy", "lint", "plugin"]
+categories = ["development-tools", "development-tools::cargo-plugins"]
+build = "build.rs"
+edition = "2018"
+publish = false
+
+[badges]
+travis-ci = { repository = "rust-lang/rust-clippy" }
+appveyor = { repository = "rust-lang/rust-clippy" }
+
+[lib]
+name = "clippy"
+plugin = true
+test = false
+
+[[bin]]
+name = "cargo-clippy"
+test = false
+path = "src/main.rs"
+
+[[bin]]
+name = "clippy-driver"
+path = "src/driver.rs"
+
+[dependencies]
+# begin automatic update
+clippy_lints = { version = "0.0.212", path = "clippy_lints" }
+# end automatic update
+regex = "1"
+semver = "0.9"
+rustc_tools_util = { version = "0.1.1", path = "rustc_tools_util"}
+
+[dev-dependencies]
+clippy_dev = { version = "0.0.1", path = "clippy_dev" }
+cargo_metadata = "0.7.1"
+compiletest_rs = "0.3.19"
+lazy_static = "1.0"
+serde_derive = "1.0"
+clippy-mini-macro-test = { version = "0.2", path = "mini-macro" }
+serde = "1.0"
+derive-new = "0.5"
+
+# A noop dependency that changes in the Rust repository, it's a bit of a hack.
+# See the `src/tools/rustc-workspace-hack/README.md` file in `rust-lang/rust`
+# for more information.
+rustc-workspace-hack = "1.0.0"
+
+[build-dependencies]
+rustc_tools_util = { version = "0.1.1", path = "rustc_tools_util"}
+
+[features]
+debugging = []

tests/packagedcode/data/cargo/clippy/Cargo.toml.expected

@@ -0,0 +1,35 @@
+{
+ "type": "cargo",
+ "namespace": null,
+ "name": "clippy",
+ "version": "0.0.212",
+ "qualifiers": null,
+ "subpath": null,
+ "primary_language": "Rust",
+ "description": null,
+ "release_date": null,
+ "parties": {},
+ "keywords": {},
+ "homepage_url": null,
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "license_expression": null,
+ "declared_license": null,
+ "notice_text": null,
+ "manifest_path": null,
+ "dependencies": {},
+ "contains_source_code": null,
+ "source_packages": {},
+ "purl": "pkg:cargo/[email protected]",
+ "repository_homepage_url": "https://crates.io//crates/clippy",
+ "repository_download_url": "https://crates.io/api/v1//crates/clippy/0.0.212/download",
+ "api_data_url": "https://crates.io/api/v1//crates/clippy"
+}