diff --git a/crates/re_build_info/src/build_info.rs b/crates/re_build_info/src/build_info.rs index ea3cde437d7b..6a5d48e67996 100644 --- a/crates/re_build_info/src/build_info.rs +++ b/crates/re_build_info/src/build_info.rs @@ -42,6 +42,8 @@ pub struct BuildInfo { /// ISO 8601 / RFC 3339 build time. /// /// Example: `"2023-02-23T19:33:26Z"` + /// + /// Empty if unknown. pub datetime: &'static str, } @@ -83,7 +85,9 @@ impl std::fmt::Display for BuildInfo { write!(f, "]")?; } - write!(f, " {target_triple}")?; + if !target_triple.is_empty() { + write!(f, " {target_triple}")?; + } if !git_branch.is_empty() { write!(f, " {git_branch}")?; @@ -93,7 +97,9 @@ impl std::fmt::Display for BuildInfo { write!(f, " {git_hash}")?; } - write!(f, ", built {datetime}")?; + if !datetime.is_empty() { + write!(f, ", built {datetime}")?; + } Ok(()) } diff --git a/crates/re_build_tools/src/git.rs b/crates/re_build_tools/src/git.rs new file mode 100644 index 000000000000..03031a73ff15 --- /dev/null +++ b/crates/re_build_tools/src/git.rs @@ -0,0 +1,65 @@ +//! # Situations to consider regarding git +//! +//! ## Using the published crate +//! +//! The published crate carries its version around, which in turns gives us the git tag, which makes +//! the commit hash irrelevant. +//! We still need to compute _something_ so that we can actually build, but that value will be +//! ignored when the crate is built by the end user anyhow. +//! +//! ## Working directly within the workspace +//! +//! When working within the workspace, we can simply try and call `git` and we're done. +//! +//! ## Using an unpublished crate (e.g. `path = "…"` or `git = "…"` or `[patch.crates-io]`) +//! +//! In these cases we may or may not have access to the workspace (e.g. a `path = …` import likely +//! will, while a crate patch won't). +//! +//! This is not an issue however, as we can simply try and see what we get. +//! If we manage to compute a commit hash, great, otherwise we still have the crate version to +//! fallback on. + +use std::path::PathBuf; + +use crate::{rerun_if_changed, run_command}; + +pub fn rebuild_if_branch_or_commit_changes() { + if let Ok(head_path) = git_path("HEAD") { + rerun_if_changed(&head_path); // Track changes to branch + if let Ok(head) = std::fs::read_to_string(&head_path) { + if let Some(git_file) = head.strip_prefix("ref: ") { + if let Ok(path) = git_path(git_file) { + if path.exists() { + rerun_if_changed(path); // Track changes to commit hash + } else { + // Weird that it doesn't exist. Maybe we will miss a git hash change, + // but that is better that tracking a non-existing files (which leads to constant rebuilds). + // See https://github.com/rerun-io/rerun/issues/2380 for more + } + } + } + } + } +} + +pub fn commit_hash() -> anyhow::Result { + let git_hash = run_command("git", &["rev-parse", "HEAD"])?; + if git_hash.is_empty() { + anyhow::bail!("empty commit hash"); + } + Ok(git_hash) +} + +pub fn branch() -> anyhow::Result { + run_command("git", &["symbolic-ref", "--short", "HEAD"]) +} + +/// From : +/// +/// Resolve `$GIT_DIR/` and takes other path relocation variables such as `$GIT_OBJECT_DIRECTORY`, `$GIT_INDEX_FILE…​` into account. +/// For example, if `$GIT_OBJECT_DIRECTORY` is set to /foo/bar then `git rev-parse --git-path objects/abc` returns `/foo/bar/abc`. +fn git_path(path: &str) -> anyhow::Result { + let path = run_command("git", &["rev-parse", "--git-path", path])?; + Ok(path.into()) +} diff --git a/crates/re_build_tools/src/lib.rs b/crates/re_build_tools/src/lib.rs index 1067deb0275f..e929d18fee0a 100644 --- a/crates/re_build_tools/src/lib.rs +++ b/crates/re_build_tools/src/lib.rs @@ -1,12 +1,14 @@ #![allow(clippy::unwrap_used)] +#![warn(missing_docs)] //! This crate is to be used from `build.rs` build scripts. use anyhow::Context as _; +use std::process::Command; use std::sync::atomic::{AtomicBool, Ordering}; -use std::{path::PathBuf, process::Command}; +mod git; mod hashing; mod rebuild_detector; @@ -21,6 +23,24 @@ pub use self::rebuild_detector::{ rerun_if_changed_glob, rerun_if_changed_or_doesnt_exist, write_file_if_necessary, }; +// ------------------ + +/// Should we export the build datetime for developers in the workspace? +/// +/// It will be visible in analytics, in the viewer's about-menu, and with `rerun --version`. +/// +/// To do so accurately may incur unnecessary recompiles, so only turn this on if you really need it. +const EXPORT_BUILD_TIME_FOR_DEVELOPERS: bool = false; + +/// Should we export the current git hash/branch for developers in the workspace? +/// +/// It will be visible in analytics, in the viewer's about-menu, and with `rerun --version`. +/// +/// To do so accurately may incur unnecessary recompiles, so only turn this on if you really need it. +const EXPORT_GIT_FOR_DEVELOPERS: bool = false; + +// ------------------ + /// Atomic bool indicating whether or not to print the cargo build instructions pub(crate) static OUTPUT_CARGO_BUILD_INSTRUCTIONS: AtomicBool = AtomicBool::new(true); @@ -34,6 +54,8 @@ pub(crate) fn should_output_cargo_build_instructions() -> bool { OUTPUT_CARGO_BUILD_INSTRUCTIONS.load(Ordering::Relaxed) } +// ------------------ + /// Where is this `build.rs` build script running? pub enum Environment { /// We are running `cargo publish` (via `scripts/ci/crates.py`); _probably_ on CI. @@ -87,79 +109,84 @@ pub fn is_on_ci() -> bool { /// /// Use this crate together with the `re_build_info` crate. pub fn export_build_info_vars_for_crate(crate_name: &str) { - rebuild_if_crate_changed(crate_name); - export_build_info_env_vars(); -} + let environment = Environment::detect(); -/// # Situations to consider regarding git -/// -/// ## Using the published crate -/// -/// The published crate carries its version around, which in turns gives us the git tag, which makes -/// the commit hash irrelevant. -/// We still need to compute _something_ so that we can actually build, but that value will be -/// ignored when the crate is built by the end user anyhow. -/// -/// ## Working directly within the workspace -/// -/// When working within the workspace, we can simply try and call `git` and we're done. -/// -/// ## Using an unpublished crate (e.g. `path = "…"` or `git = "…"` or `[patch.crates-io]`) -/// -/// In these cases we may or may not have access to the workspace (e.g. a `path = …` import likely -/// will, while a crate patch won't). -/// -/// This is not an issue however, as we can simply try and see what we get. -/// If we manage to compute a commit hash, great, otherwise we still have the crate version to -/// fallback on. -fn export_build_info_env_vars() { - // target triple - set_env("RE_BUILD_TARGET_TRIPLE", &std::env::var("TARGET").unwrap()); - set_env("RE_BUILD_GIT_HASH", &git_hash().unwrap_or_default()); - set_env("RE_BUILD_GIT_BRANCH", &git_branch().unwrap_or_default()); - - // rust version - let (rustc, llvm) = rust_version().unwrap_or_default(); - set_env("RE_BUILD_RUSTC_VERSION", &rustc); - set_env("RE_BUILD_LLVM_VERSION", &llvm); - - // We need to check `IS_IN_RERUN_WORKSPACE` in the build-script (here), - // because otherwise it won't show up when compiling through maturin. - // We must also make an exception for when we build actual wheels (on CI) for release. - if is_on_ci() { - // e.g. building wheels on CI. - set_env("RE_BUILD_IS_IN_RERUN_WORKSPACE", "no"); + let export_datetime = match environment { + Environment::PublishingCrates | Environment::CI => true, + + Environment::DeveloperInWorkspace => EXPORT_BUILD_TIME_FOR_DEVELOPERS, + + // Datetime won't always be accurate unless we rebuild as soon as a dependency changes, + // and we don't want to add that burden to our users. + Environment::UsedAsDependency => false, + }; + + let export_git_info = match environment { + Environment::PublishingCrates | Environment::CI => true, + + Environment::DeveloperInWorkspace => EXPORT_GIT_FOR_DEVELOPERS, + + // We shouldn't show the users git hash/branch in the rerun viewer. + Environment::UsedAsDependency => false, + }; + + if export_datetime { + set_env("RE_BUILD_DATETIME", &date_time()); + + // The only way to make sure the build datetime is up-to-date is to run + // `build.rs` on every build, and there is really no good way of doing + // so except to manually check if any files have changed: + rebuild_if_crate_changed(crate_name); + } else { + set_env("RE_BUILD_DATETIME", ""); + } + + if export_git_info { + set_env("RE_BUILD_GIT_HASH", &git::commit_hash().unwrap_or_default()); + set_env("RE_BUILD_GIT_BRANCH", &git::branch().unwrap_or_default()); + + // Make sure the above are up-to-date + git::rebuild_if_branch_or_commit_changes(); } else { - set_env( - "RE_BUILD_IS_IN_RERUN_WORKSPACE", - &std::env::var("IS_IN_RERUN_WORKSPACE").unwrap_or_default(), - ); + set_env("RE_BUILD_GIT_HASH", ""); + set_env("RE_BUILD_GIT_BRANCH", ""); } + // Stuff that doesn't change, so doesn't need rebuilding: + { + // target triple + set_env("RE_BUILD_TARGET_TRIPLE", &std::env::var("TARGET").unwrap()); + + // rust version + let (rustc, llvm) = rust_llvm_versions().unwrap_or_default(); + set_env("RE_BUILD_RUSTC_VERSION", &rustc); + set_env("RE_BUILD_LLVM_VERSION", &llvm); + + // We need to check `IS_IN_RERUN_WORKSPACE` in the build-script (here), + // because otherwise it won't show up when compiling through maturin. + // We must also make an exception for when we build actual wheels (on CI) for release. + if is_on_ci() { + // e.g. building wheels on CI. + set_env("RE_BUILD_IS_IN_RERUN_WORKSPACE", "no"); + } else { + set_env( + "RE_BUILD_IS_IN_RERUN_WORKSPACE", + &std::env::var("IS_IN_RERUN_WORKSPACE").unwrap_or_default(), + ); + } + } +} + +/// ISO 8601 / RFC 3339 build time. +/// +/// Example: `"2023-02-23T19:33:26Z"` +fn date_time() -> String { let time_format = time::format_description::parse("[year]-[month]-[day]T[hour]:[minute]:[second]Z").unwrap(); - let date_time = time::OffsetDateTime::now_utc() + + time::OffsetDateTime::now_utc() .format(&time_format) - .unwrap(); - set_env("RE_BUILD_DATETIME", &date_time); - - // Make sure we re-run the build script if the branch or commit changes: - if let Ok(head_path) = git_path("HEAD") { - rerun_if_changed(&head_path); // Track changes to branch - if let Ok(head) = std::fs::read_to_string(&head_path) { - if let Some(git_file) = head.strip_prefix("ref: ") { - if let Ok(path) = git_path(git_file) { - if path.exists() { - rerun_if_changed(path); // Track changes to commit hash - } else { - // Weird that it doesn't exist. Maybe we will miss a git hash change, - // but that is better that tracking a non-existing files (which leads to constant rebuilds). - // See https://github.com/rerun-io/rerun/issues/2380 for more - } - } - } - } - } + .unwrap() } fn set_env(name: &str, value: &str) { @@ -184,32 +211,11 @@ fn run_command(cmd: &str, args: &[&str]) -> anyhow::Result { Ok(String::from_utf8(output.stdout)?.trim().to_owned()) } -fn git_hash() -> anyhow::Result { - let git_hash = run_command("git", &["rev-parse", "HEAD"])?; - if git_hash.is_empty() { - anyhow::bail!("empty commit hash"); - } - Ok(git_hash) -} - -fn git_branch() -> anyhow::Result { - run_command("git", &["symbolic-ref", "--short", "HEAD"]) -} - -/// From : -/// -/// Resolve `$GIT_DIR/` and takes other path relocation variables such as `$GIT_OBJECT_DIRECTORY`, `$GIT_INDEX_FILE…​` into account. -/// For example, if `$GIT_OBJECT_DIRECTORY` is set to /foo/bar then `git rev-parse --git-path objects/abc` returns `/foo/bar/abc`. -fn git_path(path: &str) -> anyhow::Result { - let path = run_command("git", &["rev-parse", "--git-path", path])?; - Ok(path.into()) -} - /// Returns `(rustc, LLVM)` versions. /// /// Defaults to `"unknown"` if, for whatever reason, the output from `rustc -vV` did not contain /// version information and/or the output format underwent breaking changes. -fn rust_version() -> anyhow::Result<(String, String)> { +fn rust_llvm_versions() -> anyhow::Result<(String, String)> { let cmd = std::env::var("RUSTC").unwrap_or("rustc".into()); let args = &["-vV"]; diff --git a/crates/re_build_tools/src/rebuild_detector.rs b/crates/re_build_tools/src/rebuild_detector.rs index 365a2c0a24ac..a4b3124f3c03 100644 --- a/crates/re_build_tools/src/rebuild_detector.rs +++ b/crates/re_build_tools/src/rebuild_detector.rs @@ -22,6 +22,7 @@ fn should_run() -> bool { // Dependencies shouldn't change on CI, but who knows 🤷‍♂️ Environment::CI => true, + // Yes - this is what we want tracking for. Environment::DeveloperInWorkspace => true, // Definitely not @@ -34,6 +35,9 @@ fn should_run() -> bool { /// /// This will work even if the package depends on crates that are outside of the workspace, /// included with `path = …` +/// +/// However, this is a complex beast, and may have bugs in it. +/// Maybe it is even causing spurious re-compiles (). pub fn rebuild_if_crate_changed(pkg_name: &str) { if !should_run() { return; diff --git a/crates/re_types/Cargo.toml b/crates/re_types/Cargo.toml index 31d2e0a507de..d55a60a38b28 100644 --- a/crates/re_types/Cargo.toml +++ b/crates/re_types/Cargo.toml @@ -45,6 +45,14 @@ serde = ["dep:serde", "re_string_interner/serde"] ## Only useful for testing purposes. testing = [] +## Special hack! +## +## If you configure rust-analyzer to set `--all-features, +## then this feature will be set and that will ensure that +## `rust-analyzer` won't run the (expensive) `build.rs`! +__opt_out_of_auto_rebuild = [] + + [dependencies] # Rerun re_error.workspace = true diff --git a/crates/re_types/build.rs b/crates/re_types/build.rs index d4fbb80a5fdc..f50c021e1f5b 100644 --- a/crates/re_types/build.rs +++ b/crates/re_types/build.rs @@ -38,14 +38,32 @@ fn should_run() -> bool { return false; } + if re_build_tools::get_and_track_env_var("CARGO_FEATURE___OPT_OUT_OF_AUTO_REBUILD").is_ok() { + eprintln!("__opt_out_of_auto_rebuild feature detected: Skipping re_types/build.rs"); + return false; + } + match Environment::detect() { // we should have been run before publishing Environment::PublishingCrates => false, - // YES! We run it to verify that the generated code is up-to-date. - Environment::CI => true, - - Environment::DeveloperInWorkspace => true, + // No - we run a manual `cargo codegen` on CI in `.github/workflows/contrib_checks.yml` + // (`no-codegen-changes`) to check out that the generated files are in-sync with the input. + Environment::CI => false, + + Environment::DeveloperInWorkspace => { + // This `build.rs` depends on having a bunch of tools installed (`clang-format`, …) + // and when some random contributor clones our repository, + // they likely won't have it, and we shouldn't need it. + // We really only need this `build.rs` for the convenience of + // developers who changes the input files (*.fbs) who then don't want to manually + // run `cargo codegen`. + // So: we only run this `build.rs` automatically after a developer + // has once run the codegen MANUALLY first using `cargo codegen`. + // That will produce the `source_hash.txt` file. + + Path::new(SOURCE_HASH_PATH).exists() + } // We ship pre-built source files for users Environment::UsedAsDependency => false, @@ -57,11 +75,6 @@ fn main() { return; } - // Only re-build if source-hash exists - if !Path::new(SOURCE_HASH_PATH).exists() { - return; - } - rerun_if_changed(SOURCE_HASH_PATH); for path in iter_dir(DEFINITIONS_DIR_PATH, Some(&["fbs"])) { rerun_if_changed(&path); @@ -87,13 +100,6 @@ fn main() { } } - // Detect desyncs between definitions and generated when running on CI, and - // crash the build accordingly. - #[allow(clippy::manual_assert)] - if re_build_tools::is_on_ci() { - panic!("re_types' fbs definitions and generated code are out-of-sync!"); - } - let (report, reporter) = re_types_builder::report::init(); // passes 1 through 3: bfbs, semantic, arrow registry diff --git a/crates/re_types_builder/build.rs b/crates/re_types_builder/build.rs index 64c7304b4e48..44de031db821 100644 --- a/crates/re_types_builder/build.rs +++ b/crates/re_types_builder/build.rs @@ -29,7 +29,19 @@ fn should_run() -> bool { // The code we're generating here is actual source code that gets committed into the repository. Environment::CI => false, - Environment::DeveloperInWorkspace => true, + Environment::DeveloperInWorkspace => { + // This `build.rs` depends on having `flatc` installed, + // and when some random contributor clones our repository, + // they likely won't have it, and we shouldn't need it. + // We really only need this `build.rs` for the convenience of + // developers who changes the input file (reflection.fbs), + // which again is rare. + // So: we only run this `build.rs` automatically after a developer + // has once run the codegen MANUALLY first using `cargo codegen`. + // That will produce the `source_hash.txt` file. + + Path::new(SOURCE_HASH_PATH).exists() + } // We ship pre-built source files for users Environment::UsedAsDependency => false, @@ -41,11 +53,6 @@ fn main() { return; } - // Only re-build if source-hash exists - if !Path::new(SOURCE_HASH_PATH).exists() { - return; - } - // We're building an actual build graph here, and Cargo has no idea about it. // // Worse: some nodes in our build graph actually output artifacts into the src/ directory, diff --git a/crates/re_types_builder/src/lib.rs b/crates/re_types_builder/src/lib.rs index 3385e6fa0a7d..9de3112c604f 100644 --- a/crates/re_types_builder/src/lib.rs +++ b/crates/re_types_builder/src/lib.rs @@ -266,7 +266,7 @@ pub fn generate_lang_agnostic( } /// Generates a .gitattributes file that marks up all generated files as generated -pub fn generate_gitattributes_for_generated_files( +fn generate_gitattributes_for_generated_files( output_path: &impl AsRef, files: impl Iterator, ) { @@ -292,6 +292,7 @@ pub fn generate_gitattributes_for_generated_files( codegen::write_file(&path, &content); } +/// This will automatically emit a `rerun-if-changed` clause for all the files that were hashed. pub fn compute_re_types_builder_hash() -> String { compute_crate_hash("re_types_builder") } @@ -303,6 +304,7 @@ pub struct SourceLocations<'a> { pub cpp_output_dir: &'a str, } +/// Also triggers a re-build if anything that affects the hash changes. pub fn compute_re_types_hash(locations: &SourceLocations<'_>) -> String { // NOTE: We need to hash both the flatbuffers definitions as well as the source code of the // code generator itself! diff --git a/crates/re_viewer/src/ui/rerun_menu.rs b/crates/re_viewer/src/ui/rerun_menu.rs index c474e497a6ca..21d6953bd50a 100644 --- a/crates/re_viewer/src/ui/rerun_menu.rs +++ b/crates/re_viewer/src/ui/rerun_menu.rs @@ -126,27 +126,30 @@ impl App { ui.style_mut().wrap = Some(false); - let rustc_version = if rustc_version.is_empty() { - "unknown" + let git_hash_suffix = if git_hash.is_empty() { + String::new() } else { - rustc_version + let short_git_hash = &git_hash[..std::cmp::min(git_hash.len(), 7)]; + format!("({short_git_hash})") }; - let llvm_version = if llvm_version.is_empty() { - "unknown" - } else { - llvm_version - }; + let mut label = format!( + "{crate_name} {version} {git_hash_suffix}\n\ + {target_triple}" + ); + + if !rustc_version.is_empty() { + label += &format!("\nrustc {rustc_version}"); + if !llvm_version.is_empty() { + label += &format!(", LLVM {llvm_version}"); + } + } - let short_git_hash = &git_hash[..std::cmp::min(git_hash.len(), 7)]; + if !datetime.is_empty() { + label += &format!("\nbuilt {datetime}"); + } - ui.label(format!( - "{crate_name} {version} ({short_git_hash})\n\ - {target_triple}\n\ - rustc {rustc_version}\n\ - LLVM {llvm_version}\n\ - Built {datetime}", - )); + ui.label(label); } fn options_menu_ui(&mut self, ui: &mut egui::Ui, _frame: &mut eframe::Frame) { diff --git a/crates/re_web_viewer_server/Cargo.toml b/crates/re_web_viewer_server/Cargo.toml index 7f73b5612ccb..3bff1c722b7f 100644 --- a/crates/re_web_viewer_server/Cargo.toml +++ b/crates/re_web_viewer_server/Cargo.toml @@ -34,7 +34,10 @@ all-features = true ## ## When set: will skip building the Web Viewer Wasm. ## This makes the CI much faster, but the resulting web server -## will crash if you rey to run it! +## will crash if you try to run it! +## +## BONUS: If you configure rust-analyzer to use `--all-features, +## you will also have rust-analyzer opting-out of building wasm! __ci = [] ## Enable telemetry using our analytics SDK. diff --git a/crates/re_web_viewer_server/build.rs b/crates/re_web_viewer_server/build.rs index 4d38932a257a..1bb70743d19b 100644 --- a/crates/re_web_viewer_server/build.rs +++ b/crates/re_web_viewer_server/build.rs @@ -8,6 +8,13 @@ fn should_run() -> bool { #![allow(clippy::match_same_arms)] use re_build_tools::Environment; + if get_and_track_env_var("CARGO_FEATURE___CI").is_ok() { + // If the `__ci` feature is set we skip building the web viewer wasm, saving a lot of time. + // This feature is set on CI (hence the name), but also with `--all-features`, which is set by rust analyzer, bacon, etc. + eprintln!("__ci feature detected: Skipping building of web viewer wasm."); + return false; + } + match Environment::detect() { // We should build the web viewer before starting crate publishing Environment::PublishingCrates => false, @@ -39,16 +46,10 @@ fn main() { // or patched!). rebuild_if_crate_changed("re_viewer"); - if get_and_track_env_var("CARGO_FEATURE___CI").is_ok() { - // If the `__ci` feature is set we skip building the web viewer wasm, saving a lot of time. - // This feature is set on CI (hence the name), but also with `--all-features`, which is set by rust analyzer, bacon, etc. - eprintln!("__ci feature detected: Skipping building of web viewer wasm."); - } else { - let release = re_build_tools::get_and_track_env_var("PROFILE").unwrap() == "release"; - if let Err(err) = - re_build_web_viewer::build(release, is_tracked_env_var_set("RERUN_BUILD_WEBGPU")) - { - panic!("Failed to build web viewer: {}", re_error::format(err)); - } + let release = re_build_tools::get_and_track_env_var("PROFILE").unwrap() == "release"; + if let Err(err) = + re_build_web_viewer::build(release, is_tracked_env_var_set("RERUN_BUILD_WEBGPU")) + { + panic!("Failed to build web viewer: {}", re_error::format(err)); } }