From 0a95d7c20a0f917b8e59dcd055694943477265e8 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 25 Aug 2023 14:44:43 +0000 Subject: [PATCH 01/15] Split out check_runtime_license_exceptions For runtime dependencies we have a much stricter license check. Combining the license check code with that for regular tools makes it harder to follow. --- src/tools/tidy/src/deps.rs | 67 +++++++++++++++++++++++++++----------- 1 file changed, 48 insertions(+), 19 deletions(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index f89faacc2d17b..835a1f3f50752 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -387,8 +387,11 @@ pub fn check(root: &Path, cargo: &Path, bad: &mut bool) { .manifest_path(root.join("Cargo.toml")) .features(cargo_metadata::CargoOpt::AllFeatures); let metadata = t!(cmd.exec()); + let runtime_ids = compute_runtime_crates(&metadata); - check_license_exceptions(&metadata, EXCEPTIONS, runtime_ids, bad); + check_runtime_license_exceptions(&metadata, runtime_ids, bad); + + check_license_exceptions(&metadata, EXCEPTIONS, bad); check_permitted_dependencies( &metadata, "rustc", @@ -403,8 +406,7 @@ pub fn check(root: &Path, cargo: &Path, bad: &mut bool) { .manifest_path(root.join("src/tools/cargo/Cargo.toml")) .features(cargo_metadata::CargoOpt::AllFeatures); let cargo_metadata = t!(cmd.exec()); - let runtime_ids = HashSet::new(); - check_license_exceptions(&cargo_metadata, EXCEPTIONS_CARGO, runtime_ids, bad); + check_license_exceptions(&cargo_metadata, EXCEPTIONS_CARGO, bad); check_rustfix(&metadata, &cargo_metadata, bad); // Check rustc_codegen_cranelift independently as it has it's own workspace. @@ -413,8 +415,7 @@ pub fn check(root: &Path, cargo: &Path, bad: &mut bool) { .manifest_path(root.join("compiler/rustc_codegen_cranelift/Cargo.toml")) .features(cargo_metadata::CargoOpt::AllFeatures); let metadata = t!(cmd.exec()); - let runtime_ids = HashSet::new(); - check_license_exceptions(&metadata, EXCEPTIONS_CRANELIFT, runtime_ids, bad); + check_license_exceptions(&metadata, EXCEPTIONS_CRANELIFT, bad); check_permitted_dependencies( &metadata, "cranelift", @@ -428,19 +429,54 @@ pub fn check(root: &Path, cargo: &Path, bad: &mut bool) { .manifest_path(root.join("src/bootstrap/Cargo.toml")) .features(cargo_metadata::CargoOpt::AllFeatures); let metadata = t!(cmd.exec()); - let runtime_ids = HashSet::new(); - check_license_exceptions(&metadata, EXCEPTIONS_BOOTSTRAP, runtime_ids, bad); + check_license_exceptions(&metadata, EXCEPTIONS_BOOTSTRAP, bad); } -/// Check that all licenses are in the valid list in `LICENSES`. +/// Check that all licenses of runtime dependencies are in the valid list in `LICENSES`. /// -/// Packages listed in `exceptions` are allowed for tools. -fn check_license_exceptions( +/// Unlike for tools we don't allow exceptions to the `LICENSES` list for the runtime with the sole +/// exception of `fortanix-sgx-abi` which is only used on x86_64-fortanix-unknown-sgx. +fn check_runtime_license_exceptions( metadata: &Metadata, - exceptions: &[(&str, &str)], runtime_ids: HashSet<&PackageId>, bad: &mut bool, ) { + for pkg in &metadata.packages { + if !runtime_ids.contains(&pkg.id) { + // Only checking dependencies of runtime libraries here. + continue; + } + if pkg.source.is_none() { + // No need to check local packages. + continue; + } + let license = match &pkg.license { + Some(license) => license, + None => { + tidy_error!(bad, "dependency `{}` does not define a license expression", pkg.id); + continue; + } + }; + if !LICENSES.contains(&license.as_str()) { + if pkg.name == "fortanix-sgx-abi" { + // This is a specific exception because SGX is considered + // "third party". See + // https://github.com/rust-lang/rust/issues/62620 for more. In + // general, these should never be added. + if pkg.license.as_deref() != Some("MPL-2.0") { + tidy_error!(bad, "invalid license `{}` in `{}`", license, pkg.id); + } + continue; + } + tidy_error!(bad, "invalid license `{}` in `{}`", license, pkg.id); + } + } +} + +/// Check that all licenses of tool dependencies are in the valid list in `LICENSES`. +/// +/// Packages listed in `exceptions` are allowed for tools. +fn check_license_exceptions(metadata: &Metadata, exceptions: &[(&str, &str)], bad: &mut bool) { // Validate the EXCEPTIONS list hasn't changed. for (name, license) in exceptions { // Check that the package actually exists. @@ -482,7 +518,7 @@ fn check_license_exceptions( // No need to check local packages. continue; } - if !runtime_ids.contains(&pkg.id) && exception_names.contains(&pkg.name.as_str()) { + if exception_names.contains(&pkg.name.as_str()) { continue; } let license = match &pkg.license { @@ -493,13 +529,6 @@ fn check_license_exceptions( } }; if !LICENSES.contains(&license.as_str()) { - if pkg.name == "fortanix-sgx-abi" { - // This is a specific exception because SGX is considered - // "third party". See - // https://github.com/rust-lang/rust/issues/62620 for more. In - // general, these should never be added. - continue; - } tidy_error!(bad, "invalid license `{}` in `{}`", license, pkg.id); } } From 3b4598e0a460496efa3300f965dac9648aafdfb3 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 25 Aug 2023 18:29:26 +0000 Subject: [PATCH 02/15] Check permitted dependencies for rustc-main Rather than just for rustc_driver. rustc-main also depends on rustc_smir which may get unique dependencies in the future. --- src/tools/tidy/src/deps.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 835a1f3f50752..88c0c3e5b692b 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -184,6 +184,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[ "is-terminal", "itertools", "itoa", + "jemalloc-sys", "jobserver", "lazy_static", "libc", @@ -396,7 +397,7 @@ pub fn check(root: &Path, cargo: &Path, bad: &mut bool) { &metadata, "rustc", PERMITTED_RUSTC_DEPENDENCIES, - &["rustc_driver", "rustc_codegen_llvm"], + &["rustc-main"], bad, ); From 4182d93487165dc22166edd7f90b2324dc0fe993 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 25 Aug 2023 19:01:36 +0000 Subject: [PATCH 03/15] Make it easier to check licenses for new workspaces --- src/tools/tidy/src/deps.rs | 112 ++++++++++++++++++++----------------- 1 file changed, 60 insertions(+), 52 deletions(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 88c0c3e5b692b..8876e0d45c211 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -31,12 +31,34 @@ const LICENSES: &[&str] = &[ // tidy-alphabetical-end ]; +type ExceptionList = &'static [(&'static str, &'static str)]; + +/// The workspaces to check for licensing and optionally permitted dependencies. +/// +/// Each entry consists of a tuple with the following elements: +/// +/// * The path to the workspace root Cargo.toml file. +/// * The list of license exceptions. +/// * Optionally a tuple of: +/// * A list of crates for which dependencies need to be explicitly allowed. +/// * The list of allowed dependencies. +const WORKSPACES: &[(&str, ExceptionList, Option<(&[&str], &[&str])>)] = &[ + ("Cargo.toml", EXCEPTIONS, Some((&["rustc-main"], PERMITTED_RUSTC_DEPENDENCIES))), + ("src/tools/cargo/Cargo.toml", EXCEPTIONS_CARGO, None), + ( + "compiler/rustc_codegen_cranelift/Cargo.toml", + EXCEPTIONS_CRANELIFT, + Some((&["rustc_codegen_cranelift"], PERMITTED_CRANELIFT_DEPENDENCIES)), + ), + ("src/bootstrap/Cargo.toml", EXCEPTIONS_BOOTSTRAP, None), +]; + /// These are exceptions to Rust's permissive licensing policy, and /// should be considered bugs. Exceptions are only allowed in Rust /// tooling. It is _crucial_ that no exception crates be dependencies /// of the Rust runtime (std/test). #[rustfmt::skip] -const EXCEPTIONS: &[(&str, &str)] = &[ +const EXCEPTIONS: ExceptionList = &[ // tidy-alphabetical-start ("ar_archive_writer", "Apache-2.0 WITH LLVM-exception"), // rustc ("colored", "MPL-2.0"), // rustfmt @@ -55,7 +77,7 @@ const EXCEPTIONS: &[(&str, &str)] = &[ // tidy-alphabetical-end ]; -const EXCEPTIONS_CARGO: &[(&str, &str)] = &[ +const EXCEPTIONS_CARGO: ExceptionList = &[ // tidy-alphabetical-start ("bitmaps", "MPL-2.0+"), ("bytesize", "Apache-2.0"), @@ -78,7 +100,7 @@ const EXCEPTIONS_CARGO: &[(&str, &str)] = &[ // tidy-alphabetical-end ]; -const EXCEPTIONS_CRANELIFT: &[(&str, &str)] = &[ +const EXCEPTIONS_CRANELIFT: ExceptionList = &[ // tidy-alphabetical-start ("cranelift-bforest", "Apache-2.0 WITH LLVM-exception"), ("cranelift-codegen", "Apache-2.0 WITH LLVM-exception"), @@ -99,7 +121,7 @@ const EXCEPTIONS_CRANELIFT: &[(&str, &str)] = &[ // tidy-alphabetical-end ]; -const EXCEPTIONS_BOOTSTRAP: &[(&str, &str)] = &[ +const EXCEPTIONS_BOOTSTRAP: ExceptionList = &[ ("ryu", "Apache-2.0 OR BSL-1.0"), // through serde ]; @@ -383,54 +405,40 @@ const PERMITTED_CRANELIFT_DEPENDENCIES: &[&str] = &[ /// `root` is path to the directory with the root `Cargo.toml` (for the workspace). `cargo` is path /// to the cargo executable. pub fn check(root: &Path, cargo: &Path, bad: &mut bool) { - let mut cmd = cargo_metadata::MetadataCommand::new(); - cmd.cargo_path(cargo) - .manifest_path(root.join("Cargo.toml")) - .features(cargo_metadata::CargoOpt::AllFeatures); - let metadata = t!(cmd.exec()); - - let runtime_ids = compute_runtime_crates(&metadata); - check_runtime_license_exceptions(&metadata, runtime_ids, bad); - - check_license_exceptions(&metadata, EXCEPTIONS, bad); - check_permitted_dependencies( - &metadata, - "rustc", - PERMITTED_RUSTC_DEPENDENCIES, - &["rustc-main"], - bad, - ); - - // Check cargo independently as it has it's own workspace. - let mut cmd = cargo_metadata::MetadataCommand::new(); - cmd.cargo_path(cargo) - .manifest_path(root.join("src/tools/cargo/Cargo.toml")) - .features(cargo_metadata::CargoOpt::AllFeatures); - let cargo_metadata = t!(cmd.exec()); - check_license_exceptions(&cargo_metadata, EXCEPTIONS_CARGO, bad); - check_rustfix(&metadata, &cargo_metadata, bad); - - // Check rustc_codegen_cranelift independently as it has it's own workspace. - let mut cmd = cargo_metadata::MetadataCommand::new(); - cmd.cargo_path(cargo) - .manifest_path(root.join("compiler/rustc_codegen_cranelift/Cargo.toml")) - .features(cargo_metadata::CargoOpt::AllFeatures); - let metadata = t!(cmd.exec()); - check_license_exceptions(&metadata, EXCEPTIONS_CRANELIFT, bad); - check_permitted_dependencies( - &metadata, - "cranelift", - PERMITTED_CRANELIFT_DEPENDENCIES, - &["rustc_codegen_cranelift"], - bad, - ); - - let mut cmd = cargo_metadata::MetadataCommand::new(); - cmd.cargo_path(cargo) - .manifest_path(root.join("src/bootstrap/Cargo.toml")) - .features(cargo_metadata::CargoOpt::AllFeatures); - let metadata = t!(cmd.exec()); - check_license_exceptions(&metadata, EXCEPTIONS_BOOTSTRAP, bad); + let mut checked_runtime_licenses = false; + let mut rust_metadata = None; + + for &(workspace, exceptions, permitted_deps) in WORKSPACES { + let mut cmd = cargo_metadata::MetadataCommand::new(); + cmd.cargo_path(cargo) + .manifest_path(root.join(workspace)) + .features(cargo_metadata::CargoOpt::AllFeatures); + let metadata = t!(cmd.exec()); + + check_license_exceptions(&metadata, exceptions, bad); + if let Some((crates, permitted_deps)) = permitted_deps { + check_permitted_dependencies(&metadata, workspace, permitted_deps, crates, bad); + } + + if workspace == "Cargo.toml" { + let runtime_ids = compute_runtime_crates(&metadata); + check_runtime_license_exceptions(&metadata, runtime_ids, bad); + checked_runtime_licenses = true; + rust_metadata = Some(metadata); + } else if workspace == "src/tools/cargo/Cargo.toml" { + check_rustfix( + rust_metadata + .as_ref() + .expect("The root workspace should be the first to be checked"), + &metadata, + bad, + ); + } + } + + // Sanity check to ensure we don't accidentally remove the workspace containing the runtime + // crates. + assert!(checked_runtime_licenses); } /// Check that all licenses of runtime dependencies are in the valid list in `LICENSES`. From 212f273dbc4c439afa955d748feb8e2a29866ac9 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 25 Aug 2023 19:23:43 +0000 Subject: [PATCH 04/15] Run licence checks for cg_gcc and rust-analyzer --- src/tools/tidy/src/deps.rs | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 8876e0d45c211..552dfc00fe391 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -43,14 +43,21 @@ type ExceptionList = &'static [(&'static str, &'static str)]; /// * A list of crates for which dependencies need to be explicitly allowed. /// * The list of allowed dependencies. const WORKSPACES: &[(&str, ExceptionList, Option<(&[&str], &[&str])>)] = &[ + // The root workspace has to be first for check_rustfix to work. ("Cargo.toml", EXCEPTIONS, Some((&["rustc-main"], PERMITTED_RUSTC_DEPENDENCIES))), - ("src/tools/cargo/Cargo.toml", EXCEPTIONS_CARGO, None), + // Outside of the alphabetical section because rustfmt formats it using multiple lines. ( "compiler/rustc_codegen_cranelift/Cargo.toml", EXCEPTIONS_CRANELIFT, Some((&["rustc_codegen_cranelift"], PERMITTED_CRANELIFT_DEPENDENCIES)), ), + // tidy-alphabetical-start + ("compiler/rustc_codegen_gcc/Cargo.toml", EXCEPTIONS_GCC, None), ("src/bootstrap/Cargo.toml", EXCEPTIONS_BOOTSTRAP, None), + ("src/tools/cargo/Cargo.toml", EXCEPTIONS_CARGO, None), + ("src/tools/rust-analyzer/Cargo.toml", EXCEPTIONS_RUST_ANALYZER, None), + ("src/tools/x/Cargo.toml", &[], None), + // tidy-alphabetical-end ]; /// These are exceptions to Rust's permissive licensing policy, and @@ -100,6 +107,19 @@ const EXCEPTIONS_CARGO: ExceptionList = &[ // tidy-alphabetical-end ]; +const EXCEPTIONS_RUST_ANALYZER: ExceptionList = &[ + // tidy-alphabetical-start + ("anymap", "BlueOak-1.0.0 OR MIT OR Apache-2.0"), + ("dissimilar", "Apache-2.0"), + ("instant", "BSD-3-Clause"), + ("notify", "CC0-1.0 OR Artistic-2.0"), + ("pulldown-cmark-to-cmark", "Apache-2.0"), + ("ryu", "Apache-2.0 OR BSL-1.0"), + ("scip", "Apache-2.0"), + ("snap", "BSD-3-Clause"), + // tidy-alphabetical-end +]; + const EXCEPTIONS_CRANELIFT: ExceptionList = &[ // tidy-alphabetical-start ("cranelift-bforest", "Apache-2.0 WITH LLVM-exception"), @@ -121,6 +141,13 @@ const EXCEPTIONS_CRANELIFT: ExceptionList = &[ // tidy-alphabetical-end ]; +const EXCEPTIONS_GCC: ExceptionList = &[ + // tidy-alphabetical-start + ("gccjit", "GPL-3.0"), + ("gccjit_sys", "GPL-3.0"), + // tidy-alphabetical-end +]; + const EXCEPTIONS_BOOTSTRAP: ExceptionList = &[ ("ryu", "Apache-2.0 OR BSL-1.0"), // through serde ]; From 6baa14787e544e1f85b341e818aaff6d18a12238 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Fri, 25 Aug 2023 19:29:28 +0000 Subject: [PATCH 05/15] Check for external package sources in all workspaces --- src/tools/tidy/src/deps.rs | 22 +++++++++--------- src/tools/tidy/src/extdeps.rs | 42 ++++++++++++++++++++++------------- 2 files changed, 38 insertions(+), 26 deletions(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 552dfc00fe391..2212b228f1b6c 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -42,21 +42,21 @@ type ExceptionList = &'static [(&'static str, &'static str)]; /// * Optionally a tuple of: /// * A list of crates for which dependencies need to be explicitly allowed. /// * The list of allowed dependencies. -const WORKSPACES: &[(&str, ExceptionList, Option<(&[&str], &[&str])>)] = &[ +pub(crate) const WORKSPACES: &[(&str, ExceptionList, Option<(&[&str], &[&str])>)] = &[ // The root workspace has to be first for check_rustfix to work. - ("Cargo.toml", EXCEPTIONS, Some((&["rustc-main"], PERMITTED_RUSTC_DEPENDENCIES))), + (".", EXCEPTIONS, Some((&["rustc-main"], PERMITTED_RUSTC_DEPENDENCIES))), // Outside of the alphabetical section because rustfmt formats it using multiple lines. ( - "compiler/rustc_codegen_cranelift/Cargo.toml", + "compiler/rustc_codegen_cranelift", EXCEPTIONS_CRANELIFT, Some((&["rustc_codegen_cranelift"], PERMITTED_CRANELIFT_DEPENDENCIES)), ), // tidy-alphabetical-start - ("compiler/rustc_codegen_gcc/Cargo.toml", EXCEPTIONS_GCC, None), - ("src/bootstrap/Cargo.toml", EXCEPTIONS_BOOTSTRAP, None), - ("src/tools/cargo/Cargo.toml", EXCEPTIONS_CARGO, None), - ("src/tools/rust-analyzer/Cargo.toml", EXCEPTIONS_RUST_ANALYZER, None), - ("src/tools/x/Cargo.toml", &[], None), + ("compiler/rustc_codegen_gcc", EXCEPTIONS_GCC, None), + ("src/bootstrap", EXCEPTIONS_BOOTSTRAP, None), + ("src/tools/cargo", EXCEPTIONS_CARGO, None), + ("src/tools/rust-analyzer", EXCEPTIONS_RUST_ANALYZER, None), + ("src/tools/x", &[], None), // tidy-alphabetical-end ]; @@ -438,7 +438,7 @@ pub fn check(root: &Path, cargo: &Path, bad: &mut bool) { for &(workspace, exceptions, permitted_deps) in WORKSPACES { let mut cmd = cargo_metadata::MetadataCommand::new(); cmd.cargo_path(cargo) - .manifest_path(root.join(workspace)) + .manifest_path(root.join(workspace).join("Cargo.toml")) .features(cargo_metadata::CargoOpt::AllFeatures); let metadata = t!(cmd.exec()); @@ -447,12 +447,12 @@ pub fn check(root: &Path, cargo: &Path, bad: &mut bool) { check_permitted_dependencies(&metadata, workspace, permitted_deps, crates, bad); } - if workspace == "Cargo.toml" { + if workspace == "." { let runtime_ids = compute_runtime_crates(&metadata); check_runtime_license_exceptions(&metadata, runtime_ids, bad); checked_runtime_licenses = true; rust_metadata = Some(metadata); - } else if workspace == "src/tools/cargo/Cargo.toml" { + } else if workspace == "src/tools/cargo" { check_rustfix( rust_metadata .as_ref() diff --git a/src/tools/tidy/src/extdeps.rs b/src/tools/tidy/src/extdeps.rs index aad57cacbb41e..2556b19a7d6b6 100644 --- a/src/tools/tidy/src/extdeps.rs +++ b/src/tools/tidy/src/extdeps.rs @@ -9,25 +9,37 @@ const ALLOWED_SOURCES: &[&str] = &["\"registry+https://github.com/rust-lang/crat /// Checks for external package sources. `root` is the path to the directory that contains the /// workspace `Cargo.toml`. pub fn check(root: &Path, bad: &mut bool) { - // `Cargo.lock` of rust. - let path = root.join("Cargo.lock"); + for &(workspace, _, _) in crate::deps::WORKSPACES { + // FIXME check other workspaces too + // `Cargo.lock` of rust. + let path = root.join(workspace).join("Cargo.lock"); - // Open and read the whole file. - let cargo_lock = t!(fs::read_to_string(&path)); + // Open and read the whole file. + let cargo_lock = t!(fs::read_to_string(&path)); - // Process each line. - for line in cargo_lock.lines() { - // Consider only source entries. - if !line.starts_with("source = ") { - continue; - } + // Process each line. + for line in cargo_lock.lines() { + // Consider only source entries. + if !line.starts_with("source = ") { + continue; + } + + // Extract source value. + let source = line.split_once('=').unwrap().1.trim(); - // Extract source value. - let source = line.split_once('=').unwrap().1.trim(); + // Ensure source is allowed. + if !ALLOWED_SOURCES.contains(&&*source) { + if workspace == "compiler/rustc_codegen_gcc" { + // FIXME stop using git dependencies for rustc_codegen_gcc? + if source + == "\"git+https://github.com/antoyo/gccjit.rs#d6e52626cfc6f487094a5d5ac66302baf3439984\"" + { + continue; + } + } - // Ensure source is allowed. - if !ALLOWED_SOURCES.contains(&&*source) { - tidy_error!(bad, "invalid source: {}", source); + tidy_error!(bad, "invalid source: {}", source); + } } } } From be1307510e65a06f7f9cb64adfd1ca0892658e29 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sat, 26 Aug 2023 09:22:18 +0000 Subject: [PATCH 06/15] Clarify some license exceptions --- src/tools/tidy/src/deps.rs | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 2212b228f1b6c..87f07f232bb20 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -78,7 +78,7 @@ const EXCEPTIONS: ExceptionList = &[ ("openssl", "Apache-2.0"), // opt-dist ("option-ext", "MPL-2.0"), // cargo-miri (via `directories`) ("rustc_apfloat", "Apache-2.0 WITH LLVM-exception"), // rustc (license is the same as LLVM uses) - ("ryu", "Apache-2.0 OR BSL-1.0"), // cargo/... (because of serde) + ("ryu", "Apache-2.0 OR BSL-1.0"), // BSL is not acceptble, but we use it under Apache-2.0 // cargo/... (because of serde) ("self_cell", "Apache-2.0"), // rustc (fluent translations) ("snap", "BSD-3-Clause"), // rustc // tidy-alphabetical-end @@ -98,7 +98,7 @@ const EXCEPTIONS_CARGO: ExceptionList = &[ ("im-rc", "MPL-2.0+"), ("normalize-line-endings", "Apache-2.0"), ("openssl", "Apache-2.0"), - ("ryu", "Apache-2.0 OR BSL-1.0"), + ("ryu", "Apache-2.0 OR BSL-1.0"), // BSL is not acceptble, but we use it under Apache-2.0 ("sha1_smol", "BSD-3-Clause"), ("similar", "Apache-2.0"), ("sized-chunks", "MPL-2.0+"), @@ -109,12 +109,12 @@ const EXCEPTIONS_CARGO: ExceptionList = &[ const EXCEPTIONS_RUST_ANALYZER: ExceptionList = &[ // tidy-alphabetical-start - ("anymap", "BlueOak-1.0.0 OR MIT OR Apache-2.0"), + ("anymap", "BlueOak-1.0.0 OR MIT OR Apache-2.0"), // BlueOak is not acceptable, but we use it under MIT OR Apache-2 .0 ("dissimilar", "Apache-2.0"), ("instant", "BSD-3-Clause"), ("notify", "CC0-1.0 OR Artistic-2.0"), ("pulldown-cmark-to-cmark", "Apache-2.0"), - ("ryu", "Apache-2.0 OR BSL-1.0"), + ("ryu", "Apache-2.0 OR BSL-1.0"), // BSL is not acceptble, but we use it under Apache-2.0 ("scip", "Apache-2.0"), ("snap", "BSD-3-Clause"), // tidy-alphabetical-end @@ -149,7 +149,7 @@ const EXCEPTIONS_GCC: ExceptionList = &[ ]; const EXCEPTIONS_BOOTSTRAP: ExceptionList = &[ - ("ryu", "Apache-2.0 OR BSL-1.0"), // through serde + ("ryu", "Apache-2.0 OR BSL-1.0"), // through serde. BSL is not acceptble, but we use it under Apache-2.0 ]; /// These are the root crates that are part of the runtime. The licenses for @@ -494,14 +494,11 @@ fn check_runtime_license_exceptions( } }; if !LICENSES.contains(&license.as_str()) { - if pkg.name == "fortanix-sgx-abi" { - // This is a specific exception because SGX is considered - // "third party". See - // https://github.com/rust-lang/rust/issues/62620 for more. In - // general, these should never be added. - if pkg.license.as_deref() != Some("MPL-2.0") { - tidy_error!(bad, "invalid license `{}` in `{}`", license, pkg.id); - } + // This is a specific exception because SGX is considered "third party". + // See https://github.com/rust-lang/rust/issues/62620 for more. + // In general, these should never be added and this exception + // should not be taken as precedent for any new target. + if pkg.name == "fortanix-sgx-abi" && pkg.license.as_deref() == Some("MPL-2.0") { continue; } tidy_error!(bad, "invalid license `{}` in `{}`", license, pkg.id); From b876f2a944dd35720650cfeb7ec2af049a132188 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sat, 26 Aug 2023 12:18:59 +0000 Subject: [PATCH 07/15] Update rand dependency for test-float-parse This removes a dependency on fuchsia-cprng which doesn't have any license info. --- src/etc/test-float-parse/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etc/test-float-parse/Cargo.toml b/src/etc/test-float-parse/Cargo.toml index 6d7b227d0ad31..a045be956acd1 100644 --- a/src/etc/test-float-parse/Cargo.toml +++ b/src/etc/test-float-parse/Cargo.toml @@ -8,7 +8,7 @@ publish = false resolver = "1" [dependencies] -rand = "0.4" +rand = "0.8" [lib] name = "test_float_parse" From 8bce7116fa8b4cef51ff52398ee659c44d79930d Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sat, 26 Aug 2023 12:19:09 +0000 Subject: [PATCH 08/15] Add two lockfiles --- .../test-various/uefi_qemu_test/Cargo.lock | 16 ++++ src/etc/test-float-parse/Cargo.lock | 75 +++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 src/ci/docker/host-x86_64/test-various/uefi_qemu_test/Cargo.lock create mode 100644 src/etc/test-float-parse/Cargo.lock diff --git a/src/ci/docker/host-x86_64/test-various/uefi_qemu_test/Cargo.lock b/src/ci/docker/host-x86_64/test-various/uefi_qemu_test/Cargo.lock new file mode 100644 index 0000000000000..e983edf205cb0 --- /dev/null +++ b/src/ci/docker/host-x86_64/test-various/uefi_qemu_test/Cargo.lock @@ -0,0 +1,16 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "r-efi" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "575fc2d9b3da54adbdfaddf6eca48fec256d977c8630a1750b8991347d1ac911" + +[[package]] +name = "uefi_qemu_test" +version = "0.0.0" +dependencies = [ + "r-efi", +] diff --git a/src/etc/test-float-parse/Cargo.lock b/src/etc/test-float-parse/Cargo.lock new file mode 100644 index 0000000000000..3f60423fed352 --- /dev/null +++ b/src/etc/test-float-parse/Cargo.lock @@ -0,0 +1,75 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "libc" +version = "0.2.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "test-float-parse" +version = "0.1.0" +dependencies = [ + "rand", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" From 9f63776121db2f5cb8833e2f7d2fac37165dde87 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sat, 26 Aug 2023 12:23:31 +0000 Subject: [PATCH 09/15] Give an error instead of a panic if the lockfile is missing --- src/tools/tidy/src/deps.rs | 5 +++++ src/tools/tidy/src/extdeps.rs | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 87f07f232bb20..2f7440cba88cc 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -436,6 +436,11 @@ pub fn check(root: &Path, cargo: &Path, bad: &mut bool) { let mut rust_metadata = None; for &(workspace, exceptions, permitted_deps) in WORKSPACES { + if !root.join(workspace).join("Cargo.lock").exists() { + tidy_error!(bad, "the `{workspace}` workspace doesn't have a Cargo.lock"); + continue; + } + let mut cmd = cargo_metadata::MetadataCommand::new(); cmd.cargo_path(cargo) .manifest_path(root.join(workspace).join("Cargo.toml")) diff --git a/src/tools/tidy/src/extdeps.rs b/src/tools/tidy/src/extdeps.rs index 2556b19a7d6b6..44d13043e461e 100644 --- a/src/tools/tidy/src/extdeps.rs +++ b/src/tools/tidy/src/extdeps.rs @@ -14,6 +14,11 @@ pub fn check(root: &Path, bad: &mut bool) { // `Cargo.lock` of rust. let path = root.join(workspace).join("Cargo.lock"); + if !path.exists() { + tidy_error!(bad, "the `{workspace}` workspace doesn't have a Cargo.lock"); + continue; + } + // Open and read the whole file. let cargo_lock = t!(fs::read_to_string(&path)); From 175e4326f73935fb4c2d976ceb2b26ea2888bee4 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 27 Aug 2023 10:29:41 +0000 Subject: [PATCH 10/15] Check more workspaces --- src/tools/tidy/src/deps.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 2f7440cba88cc..a4b1676790d4f 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -42,6 +42,7 @@ type ExceptionList = &'static [(&'static str, &'static str)]; /// * Optionally a tuple of: /// * A list of crates for which dependencies need to be explicitly allowed. /// * The list of allowed dependencies. +// FIXME auto detect all cargo workspaces pub(crate) const WORKSPACES: &[(&str, ExceptionList, Option<(&[&str], &[&str])>)] = &[ // The root workspace has to be first for check_rustfix to work. (".", EXCEPTIONS, Some((&["rustc-main"], PERMITTED_RUSTC_DEPENDENCIES))), @@ -53,8 +54,15 @@ pub(crate) const WORKSPACES: &[(&str, ExceptionList, Option<(&[&str], &[&str])>) ), // tidy-alphabetical-start ("compiler/rustc_codegen_gcc", EXCEPTIONS_GCC, None), + ("library/backtrace", &[], None), + //("library/portable-simd", &[], None), // FIXME uncomment once rust-lang/portable-simd#363 has been synced back to the rust repo + //("library/stdarch", EXCEPTIONS_STDARCH, None), // FIXME uncomment once rust-lang/stdarch#1462 lands ("src/bootstrap", EXCEPTIONS_BOOTSTRAP, None), + ("src/ci/docker/host-x86_64/test-various/uefi_qemu_test", EXCEPTIONS_UEFI_QEMU_TEST, None), + ("src/etc/test-float-parse", &[], None), ("src/tools/cargo", EXCEPTIONS_CARGO, None), + ("src/tools/miri/test-cargo-miri", &[], None), + ("src/tools/miri/test_dependencies", &[], None), ("src/tools/rust-analyzer", EXCEPTIONS_RUST_ANALYZER, None), ("src/tools/x", &[], None), // tidy-alphabetical-end @@ -84,6 +92,17 @@ const EXCEPTIONS: ExceptionList = &[ // tidy-alphabetical-end ]; +// FIXME uncomment once rust-lang/stdarch#1462 lands +/* +const EXCEPTIONS_STDARCH: ExceptionList = &[ + // tidy-alphabetical-start + ("ryu", "Apache-2.0 OR BSL-1.0"), // BSL is not acceptble, but we use it under Apache-2.0 + ("wasmparser", "Apache-2.0 WITH LLVM-exception"), + ("wasmprinter", "Apache-2.0 WITH LLVM-exception"), + // tidy-alphabetical-end +]; +*/ + const EXCEPTIONS_CARGO: ExceptionList = &[ // tidy-alphabetical-start ("bitmaps", "MPL-2.0+"), @@ -152,6 +171,10 @@ const EXCEPTIONS_BOOTSTRAP: ExceptionList = &[ ("ryu", "Apache-2.0 OR BSL-1.0"), // through serde. BSL is not acceptble, but we use it under Apache-2.0 ]; +const EXCEPTIONS_UEFI_QEMU_TEST: ExceptionList = &[ + ("r-efi", "MIT OR Apache-2.0 OR LGPL-2.1-or-later"), // LGPL is not acceptible, but we use it under MIT OR Apache-2.0 +]; + /// These are the root crates that are part of the runtime. The licenses for /// these and all their dependencies *must not* be in the exception list. const RUNTIME_CRATES: &[&str] = &["std", "core", "alloc", "test", "panic_abort", "panic_unwind"]; From 4edd74821af6e63369fe722d20fc6c689b27dbaf Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sun, 27 Aug 2023 16:41:04 +0000 Subject: [PATCH 11/15] Ensure tidy never updates Cargo.lock Otherwise the results may be outdated compared to a build at a later time. Also disable checking for the backtrace workspace until Cargo.lock is committed. --- src/tools/tidy/src/deps.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index a4b1676790d4f..07be91bbeb4c3 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -54,7 +54,7 @@ pub(crate) const WORKSPACES: &[(&str, ExceptionList, Option<(&[&str], &[&str])>) ), // tidy-alphabetical-start ("compiler/rustc_codegen_gcc", EXCEPTIONS_GCC, None), - ("library/backtrace", &[], None), + //("library/backtrace", &[], None), // FIXME uncomment once rust-lang/backtrace#562 lands //("library/portable-simd", &[], None), // FIXME uncomment once rust-lang/portable-simd#363 has been synced back to the rust repo //("library/stdarch", EXCEPTIONS_STDARCH, None), // FIXME uncomment once rust-lang/stdarch#1462 lands ("src/bootstrap", EXCEPTIONS_BOOTSTRAP, None), @@ -467,7 +467,8 @@ pub fn check(root: &Path, cargo: &Path, bad: &mut bool) { let mut cmd = cargo_metadata::MetadataCommand::new(); cmd.cargo_path(cargo) .manifest_path(root.join(workspace).join("Cargo.toml")) - .features(cargo_metadata::CargoOpt::AllFeatures); + .features(cargo_metadata::CargoOpt::AllFeatures) + .other_options(vec!["--locked".to_owned()]); let metadata = t!(cmd.exec()); check_license_exceptions(&metadata, exceptions, bad); From d2054db3ebed4e955c901c3020fa3978b6eea97e Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Mon, 28 Aug 2023 14:16:27 +0000 Subject: [PATCH 12/15] Disable tidy license checker for cg_gcc until it's deps are vendored --- src/tools/tidy/src/deps.rs | 5 ++++- src/tools/tidy/src/extdeps.rs | 9 --------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 07be91bbeb4c3..7ff5eb61fb3c0 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -53,7 +53,7 @@ pub(crate) const WORKSPACES: &[(&str, ExceptionList, Option<(&[&str], &[&str])>) Some((&["rustc_codegen_cranelift"], PERMITTED_CRANELIFT_DEPENDENCIES)), ), // tidy-alphabetical-start - ("compiler/rustc_codegen_gcc", EXCEPTIONS_GCC, None), + //("compiler/rustc_codegen_gcc", EXCEPTIONS_GCC, None), // FIXME uncomment once all deps are vendored //("library/backtrace", &[], None), // FIXME uncomment once rust-lang/backtrace#562 lands //("library/portable-simd", &[], None), // FIXME uncomment once rust-lang/portable-simd#363 has been synced back to the rust repo //("library/stdarch", EXCEPTIONS_STDARCH, None), // FIXME uncomment once rust-lang/stdarch#1462 lands @@ -160,12 +160,15 @@ const EXCEPTIONS_CRANELIFT: ExceptionList = &[ // tidy-alphabetical-end ]; +// FIXME uncomment once all deps are vendored +/* const EXCEPTIONS_GCC: ExceptionList = &[ // tidy-alphabetical-start ("gccjit", "GPL-3.0"), ("gccjit_sys", "GPL-3.0"), // tidy-alphabetical-end ]; +*/ const EXCEPTIONS_BOOTSTRAP: ExceptionList = &[ ("ryu", "Apache-2.0 OR BSL-1.0"), // through serde. BSL is not acceptble, but we use it under Apache-2.0 diff --git a/src/tools/tidy/src/extdeps.rs b/src/tools/tidy/src/extdeps.rs index 44d13043e461e..ff71ca537256f 100644 --- a/src/tools/tidy/src/extdeps.rs +++ b/src/tools/tidy/src/extdeps.rs @@ -34,15 +34,6 @@ pub fn check(root: &Path, bad: &mut bool) { // Ensure source is allowed. if !ALLOWED_SOURCES.contains(&&*source) { - if workspace == "compiler/rustc_codegen_gcc" { - // FIXME stop using git dependencies for rustc_codegen_gcc? - if source - == "\"git+https://github.com/antoyo/gccjit.rs#d6e52626cfc6f487094a5d5ac66302baf3439984\"" - { - continue; - } - } - tidy_error!(bad, "invalid source: {}", source); } } From 0d79348d47559f688cc4beeaa8a126d5eca02f25 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 10 Oct 2023 12:50:40 +0000 Subject: [PATCH 13/15] Update status for the backtrace and stdarch exceptions --- src/tools/tidy/src/deps.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 7ff5eb61fb3c0..70cc91718e2f9 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -54,9 +54,9 @@ pub(crate) const WORKSPACES: &[(&str, ExceptionList, Option<(&[&str], &[&str])>) ), // tidy-alphabetical-start //("compiler/rustc_codegen_gcc", EXCEPTIONS_GCC, None), // FIXME uncomment once all deps are vendored - //("library/backtrace", &[], None), // FIXME uncomment once rust-lang/backtrace#562 lands + //("library/backtrace", &[], None), // FIXME uncomment once rust-lang/backtrace#562 has been synced back to the rust repo //("library/portable-simd", &[], None), // FIXME uncomment once rust-lang/portable-simd#363 has been synced back to the rust repo - //("library/stdarch", EXCEPTIONS_STDARCH, None), // FIXME uncomment once rust-lang/stdarch#1462 lands + //("library/stdarch", EXCEPTIONS_STDARCH, None), // FIXME uncomment once rust-lang/stdarch#1462 has been synced back to the rust repo ("src/bootstrap", EXCEPTIONS_BOOTSTRAP, None), ("src/ci/docker/host-x86_64/test-various/uefi_qemu_test", EXCEPTIONS_UEFI_QEMU_TEST, None), ("src/etc/test-float-parse", &[], None), From f6ce2e6d4a415ed31ae65698e09372ade822c02d Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Tue, 10 Oct 2023 12:51:06 +0000 Subject: [PATCH 14/15] Update license for notify --- src/tools/tidy/src/deps.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 70cc91718e2f9..210c9e4806220 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -131,7 +131,7 @@ const EXCEPTIONS_RUST_ANALYZER: ExceptionList = &[ ("anymap", "BlueOak-1.0.0 OR MIT OR Apache-2.0"), // BlueOak is not acceptable, but we use it under MIT OR Apache-2 .0 ("dissimilar", "Apache-2.0"), ("instant", "BSD-3-Clause"), - ("notify", "CC0-1.0 OR Artistic-2.0"), + ("notify", "CC0-1.0"), ("pulldown-cmark-to-cmark", "Apache-2.0"), ("ryu", "Apache-2.0 OR BSL-1.0"), // BSL is not acceptble, but we use it under Apache-2.0 ("scip", "Apache-2.0"), From af85d3169f821901c68b4f1c8222ec2b1c887147 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sat, 28 Oct 2023 08:47:40 +0000 Subject: [PATCH 15/15] Disable tidy dep check for a couple more unvendored project --- src/tools/tidy/src/deps.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs index 210c9e4806220..c55c406b5d081 100644 --- a/src/tools/tidy/src/deps.rs +++ b/src/tools/tidy/src/deps.rs @@ -59,10 +59,10 @@ pub(crate) const WORKSPACES: &[(&str, ExceptionList, Option<(&[&str], &[&str])>) //("library/stdarch", EXCEPTIONS_STDARCH, None), // FIXME uncomment once rust-lang/stdarch#1462 has been synced back to the rust repo ("src/bootstrap", EXCEPTIONS_BOOTSTRAP, None), ("src/ci/docker/host-x86_64/test-various/uefi_qemu_test", EXCEPTIONS_UEFI_QEMU_TEST, None), - ("src/etc/test-float-parse", &[], None), + //("src/etc/test-float-parse", &[], None), // FIXME uncomment once all deps are vendored ("src/tools/cargo", EXCEPTIONS_CARGO, None), - ("src/tools/miri/test-cargo-miri", &[], None), - ("src/tools/miri/test_dependencies", &[], None), + //("src/tools/miri/test-cargo-miri", &[], None), // FIXME uncomment once all deps are vendored + //("src/tools/miri/test_dependencies", &[], None), // FIXME uncomment once all deps are vendored ("src/tools/rust-analyzer", EXCEPTIONS_RUST_ANALYZER, None), ("src/tools/x", &[], None), // tidy-alphabetical-end