Merge pull requests #722, #719 from amazonlinux/bottlerocket, amazonl…

…inux/license-scan-rust

extras/sdk-container/Dockerfile

@@ -336,8 +336,8 @@ FROM sdk-rust as sdk-license-scan
 
 USER root
 RUN \
-  mkdir -p /usr/libexec/tools /home/builder/license-scan && \
-  chown -R builder:builder /usr/libexec/tools /home/builder/license-scan
+  mkdir -p /usr/libexec/tools /home/builder/license-scan /usr/share/licenses/bottlerocket-license-scan && \
+  chown -R builder:builder /usr/libexec/tools /home/builder/license-scan /usr/share/licenses/bottlerocket-license-scan
 
 ARG SPDXVER="3.7"
 
@@ -354,6 +354,13 @@ COPY license-scan /home/builder/license-scan
 RUN cargo build --release --locked
 RUN install -p -m 0755 target/release/bottlerocket-license-scan /usr/libexec/tools/
 RUN cp -r license-list-data/json/details /usr/libexec/tools/spdx-data
+# quine - scan the license tool itself for licenses
+RUN \
+  /usr/libexec/tools/bottlerocket-license-scan \
+    --clarify clarify.toml \
+    --spdx-data /usr/libexec/tools/spdx-data \
+    --out-dir /usr/share/licenses/bottlerocket-license-scan/vendor \
+    cargo --locked Cargo.toml
 
 # =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=   =^..^=
 
@@ -391,6 +398,8 @@ COPY --chown=0:0 --from=sdk-go /home/builder/go/src /usr/libexec/go/src/
 
 # "sdk-license-scan" has our attribution generation tool.
 COPY --chown=0:0 --from=sdk-license-scan /usr/libexec/tools/ /usr/libexec/tools/
+# quine - include the licenses for the license scan tool itself
+COPY --chown=0:0 --from=sdk-license-scan /usr/share/licenses/bottlerocket-license-scan/ /usr/share/licenses/bottlerocket-license-scan/
 
 # Add Rust programs and libraries to the path.
 RUN \

extras/sdk-container/license-scan/Cargo.toml

@@ -8,6 +8,7 @@ publish = false
 [dependencies]
 anyhow = "1"
 askalono = "0.4"
+cargo_metadata = "0.9"
 ignore = "0.4"
 lazy_static = "1"
 serde = { version = "1", features = ["derive"] }

extras/sdk-container/license-scan/clarify.toml

@@ -0,0 +1,62 @@
+[clarify.askalono]
+expression = "Apache-2.0"
+license-files = [
+    { path = "LICENSE", hash = 0x18785531 },
+    { path = "NOTICE", hash = 0x96b3ea7d },
+]
+skip-files = [
+    "src/license.rs" # source code named "license"...
+]
+
+[clarify.backtrace-sys]
+# backtrace-sys is MIT/Apache-2.0, libbacktrace is BSD-3-Clause
+expression = "(MIT OR Apache-2.0) AND BSD-3-Clause"
+license-files = [
+    { path = "src/libbacktrace/LICENSE", hash = 0x0ce09262 },
+]
+
+[clarify.crossbeam-channel]
+expression = "(MIT OR Apache-2.0) AND BSD-2-Clause AND CC-BY-3.0"
+license-files = [
+    { path = "LICENSE-APACHE", hash = 0x24b54f4b },
+    { path = "LICENSE-MIT", hash = 0xbc436f08 },
+    { path = "LICENSE-THIRD-PARTY", hash = 0xc6242648 },
+]
+
+[clarify.crossbeam-queue]
+expression = "(MIT OR Apache-2.0) AND BSD-2-Clause-FreeBSD"
+license-files = [
+    { path = "LICENSE-APACHE", hash = 0x24b54f4b },
+    { path = "LICENSE-MIT", hash = 0xbc436f08 },
+    { path = "LICENSE-THIRD-PARTY", hash = 0x7e40bc60 },
+]
+
+[clarify.regex]
+expression = "MIT OR Apache-2.0"
+license-files = [
+    { path = "LICENSE-APACHE", hash = 0x24b54f4b },
+    { path = "LICENSE-MIT", hash = 0xb755395b },
+]
+skip-files = [
+    "src/testdata/LICENSE", # we aren't using the test data
+]
+
+[clarify.regex-syntax]
+expression = "(MIT OR Apache-2.0) AND Unicode-DFS-2016"
+license-files = [
+    { path = "LICENSE-APACHE", hash = 0x24b54f4b },
+    { path = "LICENSE-MIT", hash = 0xb755395b },
+    { path = "src/unicode_tables/LICENSE-UNICODE", hash = 0xa7f28b93 },
+]
+
+[clarify.zstd-sys]
+# zstd-sys is MIT OR Apache-2.0
+# libzstd is GPL-2.0-only OR BSD-3-Clause (selecting BSD-3-Clause)
+expression = "(MIT OR Apache-2.0) AND BSD-3-Clause"
+license-files = [
+    { path = "zstd/LICENSE", hash = 0x79cda15 },
+]
+skip-files = [
+    "zstd/COPYING", # copy of the GPLv2 we are not choosing from libzstd's dual license
+    "zstd/contrib/linux-kernel/COPYING", # kernel source and patches for adding zstd (?!), not used
+]

extras/sdk-container/license-scan/src/main.rs

@@ -2,7 +2,7 @@
 #![warn(clippy::pedantic)]
 #![allow(clippy::redundant_closure_for_method_calls)]
 
-use anyhow::{bail, ensure, Context, Result};
+use anyhow::{anyhow, bail, ensure, Context, Result};
 use askalono::{ScanStrategy, Store, TextData};
 use ignore::types::{Types, TypesBuilder};
 use ignore::WalkBuilder;
@@ -42,6 +42,18 @@ enum Cmd {
         /// Path to the vendor directory of a project.
         vendor_dir: PathBuf,
     },
+    Cargo {
+        /// Path to Cargo.toml for a project.
+        manifest_path: PathBuf,
+
+        /// Equivalent to `cargo --locked`
+        #[structopt(long)]
+        locked: bool,
+
+        /// Equivalent to `cargo --offline`
+        #[structopt(long)]
+        offline: bool,
+    },
 }
 
 fn main() -> Result<()> {
@@ -75,6 +87,53 @@ fn main() -> Result<()> {
                     &opt.out_dir.join(&repo),
                     &scanner,
                     &clarify,
+                    None,
+                )?;
+            }
+            Ok(())
+        }
+        Cmd::Cargo {
+            manifest_path,
+            locked,
+            offline,
+        } => {
+            let mut builder = cargo_metadata::MetadataCommand::new();
+            builder.manifest_path(manifest_path);
+            if locked {
+                builder.other_options(&["--locked".to_owned()]);
+            }
+            if offline {
+                builder.other_options(&["--offline".to_owned()]);
+            }
+            let metadata = builder.exec()?;
+            for package in metadata.packages {
+                if package.source.is_none() {
+                    if let Some(publish) = package.publish {
+                        if publish.is_empty() {
+                            // `package.source` is None if the project is a local project;
+                            // `package.publish` is an empty Vec if `publish = false` is set
+                            continue;
+                        }
+                    }
+                }
+                write_attribution(
+                    &package.name,
+                    package
+                        .manifest_path
+                        .parent()
+                        .expect("expected a path to Cargo.toml to have a parent"),
+                    &opt.out_dir
+                        .join(format!("{}-{}", package.name, package.version)),
+                    &scanner,
+                    &clarify,
+                    if let Some(license) = package.license {
+                        Some(Expression::parse(&unslash(&license)).map_err(|err| {
+                            // spdx errors use the lifetime of the string
+                            anyhow!(err.to_string())
+                        })?)
+                    } else {
+                        None
+                    },
                 )?;
             }
             Ok(())
@@ -203,6 +262,12 @@ lazy_static::lazy_static! {
     };
 }
 
+/// Replace '/' characters in a license string with 'OR'. (crates.io allows '/' instead of 'OR' for
+/// compatibility.)
+fn unslash(s: &str) -> String {
+    s.split('/').map(str::trim).collect::<Vec<_>>().join(" OR ")
+}
+
 /// Returns true if the file is expected to not be a license text (such as the Apache-2.0 NOTICE
 /// file).
 fn non_license(path: &Path) -> bool {
@@ -222,14 +287,19 @@ fn hash(data: &[u8]) -> u32 {
         .expect("XxHash32 returned hash larger than 32 bits")
 }
 
+#[allow(clippy::too_many_lines)] // maybe someday...
 fn write_attribution(
     name: &str,
     scan_dir: &Path,
     out_dir: &Path,
     scanner: &ScanStrategy<'_>,
     clarifications: &Clarifications,
+    stated_license: Option<Expression>,
 ) -> Result<()> {
     eprintln!("{}:", name);
+    if let Some(stated_license) = stated_license.as_ref() {
+        eprintln!("  + {} (stated in metadata)", stated_license);
+    }
     let mut files = HashMap::new();
     for entry in WalkBuilder::new(scan_dir).types(TYPES.clone()).build() {
         let entry = entry?;
@@ -266,6 +336,21 @@ fn write_attribution(
                         file_hash
                     );
                 } else {
+                    if stated_license.is_some() {
+                        // if the package states a license and we heuristically detect that this is
+                        // a top-level "either license, at your option" file, ignore it
+                        let trainwreck = data.split_whitespace().collect::<Vec<_>>().join(" ");
+                        if trainwreck.contains("under the terms of either license")
+                            || trainwreck.contains("at your option")
+                        {
+                            eprintln!(
+                                "  + {} (hash = 0x{:x}) detected as non-license file",
+                                file.display(),
+                                file_hash
+                            );
+                            continue;
+                        }
+                    }
                     bail!(
                         "failed to detect any license from {} (hash = 0x{:x}), \
                          please add a clarification",
@@ -282,15 +367,41 @@ fn write_attribution(
                     result.license.name,
                     result.score,
                 );
-                licenses.push(result.license.name);
+                if let Some(stated_license) = stated_license.as_ref() {
+                    // The license we detected should be included in the stated license string,
+                    // otherwise we know the stated license is incomplete, in which case we should
+                    // have had a clarification.
+                    ensure!(
+                        stated_license.requirements().any(|er| {
+                            // `er` is an `ExpressionReq`; `er.req` is a `LicenseReq`.
+                            // `er.req.license.id()` returns `Option<LicenseId>`.
+                            er.req.license.id().is_some()
+                                && er.req.license.id() == spdx::license_id(result.license.name)
+                        }),
+                        "detected license \"{}\" from {} is not present in the license \
+                         field \"{}\" for {}",
+                        result.license.name,
+                        file.display(),
+                        stated_license,
+                        name
+                    );
+                } else {
+                    licenses.push(result.license.name);
+                }
             }
         }
-        licenses.sort();
-        licenses.dedup();
-        let expression = licenses.join(" AND ");
-        eprintln!("  = {}", expression);
+
         copy_files(out_dir, &files, &[])?;
-        expression
+
+        if let Some(stated_license) = stated_license {
+            stated_license.to_string()
+        } else {
+            licenses.sort();
+            licenses.dedup();
+            let expression = licenses.join(" AND ");
+            eprintln!("  = {}", expression);
+            expression
+        }
     };
 
     fs::create_dir_all(out_dir)?;

packages/release/release.spec

@@ -59,6 +59,7 @@ Requires: %{_cross_os}updog
 Requires: %{_cross_os}util-linux
 Requires: %{_cross_os}preinit
 Requires: %{_cross_os}wicked
+Requires: %{_cross_os}workspaces
 
 %description
 %{summary}.

packages/workspaces/workspaces.spec

@@ -218,6 +218,12 @@ install -d %{buildroot}%{_cross_tmpfilesdir}
 install -p -m 0644 %{S:200} %{buildroot}%{_cross_tmpfilesdir}/migration.conf
 install -p -m 0644 %{S:201} %{buildroot}%{_cross_tmpfilesdir}/host-containers.conf
 
+%cross_scan_attribution --clarify %{_builddir}/workspaces/clarify.toml \
+    cargo --offline --locked %{_builddir}/workspaces/Cargo.toml
+
+%files
+%{_cross_attribution_vendor_dir}
+
 %files -n %{_cross_os}apiserver
 %{_cross_bindir}/apiserver
 %{_cross_unitdir}/apiserver.service