diff --git a/ci/check-profiling.sh b/ci/check-profiling.sh index c69d109e1a..c6ff394e83 100755 --- a/ci/check-profiling.sh +++ b/ci/check-profiling.sh @@ -32,9 +32,10 @@ cargo build -p collector --bin rustc-fake # --profiles Check \ # --cargo $bindir/cargo \ # --include helloworld \ -# --scenarios Full -#test -f results/perf-Test-helloworld-Check-Full -#grep -q "PERFILE" results/perf-Test-helloworld-Check-Full +# --scenarios Full \ +# --frontend-threads 1 +#test -f results/perf-Test-helloworld-Check-Full-frontend_threads_1 +#grep -q "PERFILE" results/perf-Test-helloworld-Check-Full-frontend_threads_1 # oprofile: untested... it's not used much, and might have the same problems # that `perf` has due to virtualized hardware. @@ -50,13 +51,14 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --profiles Check \ --cargo $bindir/cargo \ --include helloworld \ - --scenarios Full -test -f results/cgout-Test-helloworld-Check-Full -grep -q "events: Ir" results/cgout-Test-helloworld-Check-Full -test -f results/cgann-Test-helloworld-Check-Full -grep -q "PROGRAM TOTALS" results/cgann-Test-helloworld-Check-Full + --scenarios Full \ + --frontend-threads 1 +test -f results/cgout-Test-helloworld-Check-Full-frontend_threads_1 +grep -q "events: Ir" results/cgout-Test-helloworld-Check-Full-frontend_threads_1 +test -f results/cgann-Test-helloworld-Check-Full-frontend_threads_1 +grep -q "PROGRAM TOTALS" results/cgann-Test-helloworld-Check-Full-frontend_threads_1 # Ensure that we also profile the memory allocator -grep -q "malloc" results/cgann-Test-helloworld-Check-Full +grep -q "malloc" results/cgann-Test-helloworld-Check-Full-frontend_threads_1 # Callgrind. RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot=debug \ @@ -66,11 +68,12 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --profiles Check \ --cargo $bindir/cargo \ --include helloworld \ - --scenarios Full -test -f results/clgout-Test-helloworld-Check-Full -grep -q "creator: callgrind" results/clgout-Test-helloworld-Check-Full -test -f results/clgann-Test-helloworld-Check-Full -grep -q "Profile data file" results/clgann-Test-helloworld-Check-Full + --scenarios Full \ + --frontend-threads 1 +test -f results/clgout-Test-helloworld-Check-Full-frontend_threads_1 +grep -q "creator: callgrind" results/clgout-Test-helloworld-Check-Full-frontend_threads_1 +test -f results/clgann-Test-helloworld-Check-Full-frontend_threads_1 +grep -q "Profile data file" results/clgann-Test-helloworld-Check-Full-frontend_threads_1 # DHAT. RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot=debug \ @@ -80,9 +83,10 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --profiles Check \ --cargo $bindir/cargo \ --include helloworld \ - --scenarios Full -test -f results/dhout-Test-helloworld-Check-Full -grep -q "dhatFileVersion" results/dhout-Test-helloworld-Check-Full + --scenarios Full \ + --frontend-threads 1 +test -f results/dhout-Test-helloworld-Check-Full-frontend_threads_1 +grep -q "dhatFileVersion" results/dhout-Test-helloworld-Check-Full-frontend_threads_1 # DHAT (copy mode). @@ -95,9 +99,10 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --profiles Check \ --cargo $bindir/cargo \ --include helloworld \ - --scenarios Full -test -f results/dhcopy-Test-helloworld-Check-Full -grep -q "dhatFileVersion" results/dhcopy-Test-helloworld-Check-Full + --scenarios Full \ + --frontend-threads 1 +test -f results/dhcopy-Test-helloworld-Check-Full-frontend_threads_1 +grep -q "dhatFileVersion" results/dhcopy-Test-helloworld-Check-Full-frontend_threads_1 # Massif. RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot=debug \ @@ -107,9 +112,10 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --profiles Check \ --cargo $bindir/cargo \ --include helloworld \ - --scenarios Full -test -f results/msout-Test-helloworld-Check-Full -grep -q "snapshot=0" results/msout-Test-helloworld-Check-Full + --scenarios Full \ + --frontend-threads 1 +test -f results/msout-Test-helloworld-Check-Full-frontend_threads_1 +grep -q "snapshot=0" results/msout-Test-helloworld-Check-Full-frontend_threads_1 # Bytehound. # This is currently broken in CI, commenting out to fix CI for this. @@ -120,8 +126,9 @@ grep -q "snapshot=0" results/msout-Test-helloworld-Check-Full # --profiles Check \ # --cargo $bindir/cargo \ # --include helloworld \ -# --scenarios Full -# test -f results/bhout-Test-helloworld-Check-Full +# --scenarios Full \ +# --frontend-threads 1 +# test -f results/bhout-Test-helloworld-Check-Full-frontend_threads_1 # eprintln. The output file is empty because a vanilla rustc doesn't print # anything to stderr. @@ -132,9 +139,10 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --profiles Check \ --cargo $bindir/cargo \ --include helloworld \ - --scenarios Full -test -f results/eprintln-Test-helloworld-Check-Full -test ! -s results/eprintln-Test-helloworld-Check-Full + --scenarios Full \ + --frontend-threads 1 +test -f results/eprintln-Test-helloworld-Check-Full-frontend_threads_1 +test ! -s results/eprintln-Test-helloworld-Check-Full-frontend_threads_1 # llvm-lines. `Debug` not `Check` because it doesn't support `Check` profiles. # Including both `helloworld` and `regex-automata-0.4.8` benchmarks, as they exercise the @@ -147,11 +155,12 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --profiles Debug \ --cargo $bindir/cargo \ --include helloworld,regex-automata-0.4.8 \ - --scenarios Full -test -f results/ll-Test-helloworld-Debug-Full -grep -q "Lines.*Copies.*Function name" results/ll-Test-helloworld-Debug-Full -test -f results/ll-Test-regex-automata-0.4.8-Debug-Full -grep -q "Lines.*Copies.*Function name" results/ll-Test-regex-automata-0.4.8-Debug-Full + --scenarios Full \ + --frontend-threads 1 +test -f results/ll-Test-helloworld-Debug-Full-frontend_threads_1 +grep -q "Lines.*Copies.*Function name" results/ll-Test-helloworld-Debug-Full-frontend_threads_1 +test -f results/ll-Test-regex-automata-0.4.8-Debug-Full-frontend_threads_1 +grep -q "Lines.*Copies.*Function name" results/ll-Test-regex-automata-0.4.8-Debug-Full-frontend_threads_1 # llvm-ir. `Debug` not `Check` because it works better that way. RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot=debug \ @@ -161,9 +170,10 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --profiles Debug \ --cargo $bindir/cargo \ --include helloworld \ - --scenarios Full -test -f results/llir-Test-helloworld-Debug-Full -grep -q "; ModuleID" results/llir-Test-helloworld-Debug-Full + --scenarios Full \ + --frontend-threads 1 +test -f results/llir-Test-helloworld-Debug-Full-frontend_threads_1 +grep -q "; ModuleID" results/llir-Test-helloworld-Debug-Full-frontend_threads_1 # mono-items. `Debug` not `Check` because it works better that way. RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot=debug \ @@ -173,9 +183,10 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --profiles Debug \ --cargo $bindir/cargo \ --include helloworld \ - --scenarios Full -test -f results/mono-items-Test-helloworld-Debug-Full/raw -grep -q "MONO_ITEM" results/mono-items-Test-helloworld-Debug-Full/raw + --scenarios Full \ + --frontend-threads 1 +test -f results/mono-items-Test-helloworld-Debug-Full-frontend_threads_1/raw +grep -q "MONO_ITEM" results/mono-items-Test-helloworld-Debug-Full-frontend_threads_1/raw # dep-graph. `IncrFull` not `Full` because it doesn't work with `Full`. RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot=debug \ @@ -185,9 +196,10 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --profiles Check \ --cargo $bindir/cargo \ --include helloworld \ - --scenarios IncrFull -test -f results/dep-graph-Test-helloworld-Check-IncrFull.txt -grep -q "hir_owner" results/dep-graph-Test-helloworld-Check-IncrFull.txt + --scenarios IncrFull \ + --frontend-threads 1 +test -f results/dep-graph-Test-helloworld-Check-IncrFull-frontend_threads_1.txt +grep -q "hir_owner" results/dep-graph-Test-helloworld-Check-IncrFull-frontend_threads_1.txt #---------------------------------------------------------------------------- # Test option handling @@ -200,23 +212,24 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= profile_local eprintln $bindir/rustc \ --id Builds1 \ --cargo $bindir/cargo \ - --include helloworld -test -f results/eprintln-Builds1-helloworld-Check-Full -test -f results/eprintln-Builds1-helloworld-Check-IncrFull -test -f results/eprintln-Builds1-helloworld-Check-IncrPatched0 -test -f results/eprintln-Builds1-helloworld-Check-IncrUnchanged -test -f results/eprintln-Builds1-helloworld-Debug-Full -test -f results/eprintln-Builds1-helloworld-Debug-IncrFull -test -f results/eprintln-Builds1-helloworld-Debug-IncrPatched0 -test -f results/eprintln-Builds1-helloworld-Debug-IncrUnchanged -test -f results/eprintln-Builds1-helloworld-Opt-Full -test -f results/eprintln-Builds1-helloworld-Opt-IncrFull -test -f results/eprintln-Builds1-helloworld-Opt-IncrPatched0 -test -f results/eprintln-Builds1-helloworld-Opt-IncrUnchanged -test ! -e results/eprintln-Builds1-helloworld-Doc-Full -test ! -e results/eprintln-Builds1-helloworld-Doc-IncrFull -test ! -e results/eprintln-Builds1-helloworld-Doc-IncrPatched0 -test ! -e results/eprintln-Builds1-helloworld-Doc-IncrUnchanged + --include helloworld \ + --frontend-threads 1 +test -f results/eprintln-Builds1-helloworld-Check-Full-frontend_threads_1 +test -f results/eprintln-Builds1-helloworld-Check-IncrFull-frontend_threads_1 +test -f results/eprintln-Builds1-helloworld-Check-IncrPatched0-frontend_threads_1 +test -f results/eprintln-Builds1-helloworld-Check-IncrUnchanged-frontend_threads_1 +test -f results/eprintln-Builds1-helloworld-Debug-Full-frontend_threads_1 +test -f results/eprintln-Builds1-helloworld-Debug-IncrFull-frontend_threads_1 +test -f results/eprintln-Builds1-helloworld-Debug-IncrPatched0-frontend_threads_1 +test -f results/eprintln-Builds1-helloworld-Debug-IncrUnchanged-frontend_threads_1 +test -f results/eprintln-Builds1-helloworld-Opt-Full-frontend_threads_1 +test -f results/eprintln-Builds1-helloworld-Opt-IncrFull-frontend_threads_1 +test -f results/eprintln-Builds1-helloworld-Opt-IncrPatched0-frontend_threads_1 +test -f results/eprintln-Builds1-helloworld-Opt-IncrUnchanged-frontend_threads_1 +test ! -e results/eprintln-Builds1-helloworld-Doc-Full-frontend_threads_1 +test ! -e results/eprintln-Builds1-helloworld-Doc-IncrFull-frontend_threads_1 +test ! -e results/eprintln-Builds1-helloworld-Doc-IncrPatched0-frontend_threads_1 +test ! -e results/eprintln-Builds1-helloworld-Doc-IncrUnchanged-frontend_threads_1 # With `--profiles Doc` specified, `Check`/`Debug`/`Opt` files must not be # present, and `Doc` files must be present (but not for incremental runs). @@ -226,23 +239,24 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --id Builds2 \ --profiles Doc \ --cargo $bindir/cargo \ - --include helloworld -test ! -e results/eprintln-Builds2-helloworld-Check-Full -test ! -e results/eprintln-Builds2-helloworld-Check-IncrFull -test ! -e results/eprintln-Builds2-helloworld-Check-IncrUnchanged -test ! -e results/eprintln-Builds2-helloworld-Check-IncrPatched0 -test ! -e results/eprintln-Builds2-helloworld-Debug-Full -test ! -e results/eprintln-Builds2-helloworld-Debug-IncrFull -test ! -e results/eprintln-Builds2-helloworld-Debug-IncrUnchanged -test ! -e results/eprintln-Builds2-helloworld-Debug-IncrPatched0 -test ! -e results/eprintln-Builds2-helloworld-Opt-Full -test ! -e results/eprintln-Builds2-helloworld-Opt-IncrFull -test ! -e results/eprintln-Builds2-helloworld-Opt-IncrUnchanged -test ! -e results/eprintln-Builds2-helloworld-Opt-IncrPatched0 -test -f results/eprintln-Builds2-helloworld-Doc-Full -test ! -f results/eprintln-Builds2-helloworld-Doc-IncrFull -test ! -f results/eprintln-Builds2-helloworld-Doc-IncrPatched0 -test ! -f results/eprintln-Builds2-helloworld-Doc-IncrUnchanged + --include helloworld \ + --frontend-threads 1 +test ! -e results/eprintln-Builds2-helloworld-Check-Full-frontend_threads_1 +test ! -e results/eprintln-Builds2-helloworld-Check-IncrFull-frontend_threads_1 +test ! -e results/eprintln-Builds2-helloworld-Check-IncrUnchanged-frontend_threads_1 +test ! -e results/eprintln-Builds2-helloworld-Check-IncrPatched0-frontend_threads_1 +test ! -e results/eprintln-Builds2-helloworld-Debug-Full-frontend_threads_1 +test ! -e results/eprintln-Builds2-helloworld-Debug-IncrFull-frontend_threads_1 +test ! -e results/eprintln-Builds2-helloworld-Debug-IncrUnchanged-frontend_threads_1 +test ! -e results/eprintln-Builds2-helloworld-Debug-IncrPatched0-frontend_threads_1 +test ! -e results/eprintln-Builds2-helloworld-Opt-Full-frontend_threads_1 +test ! -e results/eprintln-Builds2-helloworld-Opt-IncrFull-frontend_threads_1 +test ! -e results/eprintln-Builds2-helloworld-Opt-IncrUnchanged-frontend_threads_1 +test ! -e results/eprintln-Builds2-helloworld-Opt-IncrPatched0-frontend_threads_1 +test -f results/eprintln-Builds2-helloworld-Doc-Full-frontend_threads_1 +test ! -f results/eprintln-Builds2-helloworld-Doc-IncrFull-frontend_threads_1 +test ! -f results/eprintln-Builds2-helloworld-Doc-IncrPatched0-frontend_threads_1 +test ! -f results/eprintln-Builds2-helloworld-Doc-IncrUnchanged-frontend_threads_1 # With `--scenarios IncrUnchanged` specified, `IncrFull` and `IncrUnchanged` # files must be present. @@ -253,11 +267,12 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --profiles Check \ --cargo $bindir/cargo \ --include helloworld \ - --scenarios IncrUnchanged -test ! -e results/eprintln-Runs1-helloworld-Check-Full -test -f results/eprintln-Runs1-helloworld-Check-IncrFull -test -f results/eprintln-Runs1-helloworld-Check-IncrUnchanged -test ! -e results/eprintln-Runs1-helloworld-Check-IncrPatched0 + --scenarios IncrUnchanged \ + --frontend-threads 1 +test ! -e results/eprintln-Runs1-helloworld-Check-Full-frontend_threads_1 +test -f results/eprintln-Runs1-helloworld-Check-IncrFull-frontend_threads_1 +test -f results/eprintln-Runs1-helloworld-Check-IncrUnchanged-frontend_threads_1 +test ! -e results/eprintln-Runs1-helloworld-Check-IncrPatched0-frontend_threads_1 # With `--scenarios IncrPatched` specified, `IncrFull` and `IncrPatched0` files # must be present. @@ -268,11 +283,12 @@ RUST_BACKTRACE=1 RUST_LOG=raw_cargo_messages=trace,collector=debug,rust_sysroot= --profiles Check \ --cargo $bindir/cargo \ --include helloworld \ - --scenarios IncrPatched -test ! -e results/eprintln-Runs2-helloworld-Check-Full -test -f results/eprintln-Runs2-helloworld-Check-IncrFull -test ! -e results/eprintln-Runs2-helloworld-Check-IncrUnchanged -test -f results/eprintln-Runs2-helloworld-Check-IncrPatched0 + --scenarios IncrPatched \ + --frontend-threads 1 +test ! -e results/eprintln-Runs2-helloworld-Check-Full-frontend_threads_1 +test -f results/eprintln-Runs2-helloworld-Check-IncrFull-frontend_threads_1 +test ! -e results/eprintln-Runs2-helloworld-Check-IncrUnchanged-frontend_threads_1 +test -f results/eprintln-Runs2-helloworld-Check-IncrPatched0-frontend_threads_1 kill $PING_LOOP_PID exit 0 diff --git a/collector/README.md b/collector/README.md index 7cc766d900..ee390a1d50 100644 --- a/collector/README.md +++ b/collector/README.md @@ -160,6 +160,9 @@ The following options alter the behaviour of the `bench_local` subcommand. `IncrUnchanged`, `IncrPatched`, and `All`. The default is `All`. Note that `IncrFull` is always run if either of `IncrUnchanged` or `IncrPatched` are run (even if not requested). +- `--frontend-threads `: comma-separated list of thread counts for + the parallel frontend. Maps directly to the `-Zthreads` rustc option. + The default is `1`. - `--backends `: the codegen backends to be benchmarked. The possible choices are one or more (comma-separated) of `Llvm`, `Cranelift`. The default is `Llvm`. @@ -204,7 +207,7 @@ and discover all benchmarks within them. If you only want to run benchmark(s) fr you can use this to speed up the runtime benchmarking or profiling commands. The `bench_runtime_local` command also shares some options with the `bench_local` command, notably -`--id`, `--db`, `--cargo`, `--cargo-config`, `--include`, `--exclude` and `--iterations`. +`--id`, `--db`, `--cargo`, `--cargo-config`, `--include`, `--exclude` and `--iterations`. ### How to view the measurements on your own machine @@ -489,6 +492,7 @@ The following options alter the behaviour of the `profile_local` subcommand. diff files will also be produced. - `--rustdoc ` as for `bench_local`. - `--scenarios `: as for `bench_local`. +- `--frontend-threads `: as for `bench_local`. - `--backends `: as for `bench_local`. - `--jobs `: execute `` benchmarks in parallel. This is only allowed for certain profilers whose results are not affected by system noise (e.g. `callgrind` or `eprintln`). @@ -511,7 +515,7 @@ build directory (at least Valgrind 3.22 is required), like this: ``` DEP_VALGRIND=/include cargo run --release --bin collector \ - --features precise-cachegrind profile_runtime cachegrind + --features precise-cachegrind profile_runtime cachegrind ``` ## Codegen diff @@ -534,7 +538,7 @@ binary artifacts (executables, libraries). You can compare the binary statistics [--profile ] \ [--backend ] ``` - + You can also compare (diff) the size statistics between two compilers: ```bash ./target/release/collector binary_stats compile `` --include --rustc2 diff --git a/collector/src/bin/collector.rs b/collector/src/bin/collector.rs index b5d3dc2429..9e8dc31880 100644 --- a/collector/src/bin/collector.rs +++ b/collector/src/bin/collector.rs @@ -36,6 +36,7 @@ use collector::benchmark_set::{get_benchmark_set, BenchmarkSetId, BenchmarkSetMe use collector::codegen::{codegen_diff, CodegenType}; use collector::compile::benchmark::category::Category; use collector::compile::benchmark::codegen_backend::CodegenBackend; +use collector::compile::benchmark::parallel_frontend::FrontendThreads; use collector::compile::benchmark::profile::Profile; use collector::compile::benchmark::scenario::Scenario; use collector::compile::benchmark::target::Target; @@ -116,6 +117,7 @@ struct CompileBenchmarkConfig { self_profile_storage: Option>, bench_rustc: bool, targets: Vec, + frontend_threads_counts: Vec, } struct RuntimeBenchmarkConfig { @@ -166,50 +168,54 @@ fn generate_diffs( benchmarks: &[Benchmark], profiles: &[Profile], scenarios: &[Scenario], + frontend_threads_counts: &[FrontendThreads], errors: &mut BenchmarkErrors, profiler: &Profiler, ) -> Vec { let mut annotated_diffs = Vec::new(); for benchmark in benchmarks { for &profile in profiles { - for scenario in scenarios.iter().flat_map(|scenario| { - if profile.is_doc() && scenario.is_incr() { - return vec![]; - } - match scenario { - Scenario::Full | Scenario::IncrFull | Scenario::IncrUnchanged => { - vec![format!("{:?}", scenario)] + for &frontend_threads in frontend_threads_counts { + for scenario in scenarios.iter().flat_map(|scenario| { + if profile.is_doc() && scenario.is_incr() { + return vec![]; + } + match scenario { + Scenario::Full | Scenario::IncrFull | Scenario::IncrUnchanged => { + vec![format!("{:?}", scenario)] + } + Scenario::IncrPatched => (0..benchmark.patches.len()) + .map(|i| format!("{scenario:?}{i}")) + .collect::>(), + } + }) { + let filename = |prefix, id| { + format!( + "{}-{}-{}-{:?}-{}-frontend_threads_{}{}", + prefix, + id, + benchmark.name, + profile, + scenario, + frontend_threads.get(), + profiler.postfix() + ) + }; + let id_diff = format!("{id1}-{id2}"); + let prefix = profiler.prefix(); + let prefix2 = profiler.prefix2(); + let left = out_dir.join(filename(prefix, id1)); + let right = out_dir.join(filename(prefix, id2)); + let output = out_dir.join(filename(&format!("{prefix2}-diff"), &id_diff)); + + if let Err(e) = profiler.diff(&left, &right, &output) { + errors.incr(); + eprintln!("collector error: {e:?}"); + continue; } - Scenario::IncrPatched => (0..benchmark.patches.len()) - .map(|i| format!("{scenario:?}{i}")) - .collect::>(), - } - }) { - let filename = |prefix, id| { - format!( - "{}-{}-{}-{:?}-{}{}", - prefix, - id, - benchmark.name, - profile, - scenario, - profiler.postfix() - ) - }; - let id_diff = format!("{id1}-{id2}"); - let prefix = profiler.prefix(); - let prefix2 = profiler.prefix2(); - let left = out_dir.join(filename(prefix, id1)); - let right = out_dir.join(filename(prefix, id2)); - let output = out_dir.join(filename(&format!("{prefix2}-diff"), &id_diff)); - - if let Err(e) = profiler.diff(&left, &right, &output) { - errors.incr(); - eprintln!("collector error: {e:?}"); - continue; - } - annotated_diffs.push(output); + annotated_diffs.push(output); + } } } } @@ -227,6 +233,7 @@ fn profile_compile( backends: &[CodegenBackend], errors: &mut BenchmarkErrors, targets: &[Target], + frontend_threads_counts: &[FrontendThreads], ) { eprintln!("Profiling {} with {:?}", toolchain.id, profiler); if let Profiler::SelfProfile = profiler { @@ -248,6 +255,7 @@ fn profile_compile( toolchain, Some(1), targets, + frontend_threads_counts, // We always want to profile everything &hashbrown::HashSet::new(), )); @@ -411,6 +419,27 @@ struct CompileTimeOptions { /// It should be a path to the `clippy-driver` binary. #[arg(long)] clippy: Option, + + /// Parallel frontend thread count in comma-separated list + #[arg(long = "frontend-threads", value_delimiter = ',')] + frontend_threads_counts: Vec, +} + +impl CompileTimeOptions { + fn validate_and_normalize_frontend_threads(&self) -> Vec { + if self.frontend_threads_counts.is_empty() { + return FrontendThreads::default_threads_counts(); + } + + let mut threads = self + .frontend_threads_counts + .iter() + .map(|&x| FrontendThreads::new(x)) + .collect::>(); + threads.sort(); + threads.dedup(); + threads + } } #[derive(Debug, clap::Args)] @@ -992,6 +1021,7 @@ fn main_result() -> anyhow::Result { purge, } => { log_db(&db); + let frontend_threads_counts = opts.validate_and_normalize_frontend_threads(); let profiles = opts.profiles.0; let scenarios = opts.scenarios.0; let backends = opts.codegen_backends.0; @@ -1044,6 +1074,7 @@ fn main_result() -> anyhow::Result { }, bench_rustc: bench_rustc.bench_rustc, targets: vec![Target::host()], + frontend_threads_counts, }; rt.block_on(run_benchmarks(conn.as_mut(), shared, Some(config), None))?; @@ -1084,6 +1115,7 @@ fn main_result() -> anyhow::Result { let profiles = &opts.profiles.0; let scenarios = &opts.scenarios.0; + let frontend_threads_counts = opts.validate_and_normalize_frontend_threads(); let backends = &opts.codegen_backends.0; let mut benchmarks = get_compile_benchmarks(&compile_benchmark_dir, (&local).into())?; @@ -1123,6 +1155,7 @@ fn main_result() -> anyhow::Result { backends, &mut errors, &[Target::host()], + &frontend_threads_counts, ); Ok(id) }; @@ -1141,6 +1174,7 @@ fn main_result() -> anyhow::Result { &benchmarks, profiles, scenarios, + &frontend_threads_counts, &mut errors, &profiler, ); @@ -1680,6 +1714,7 @@ async fn create_benchmark_configs( }, bench_rustc, targets: vec![job.target().into()], + frontend_threads_counts: FrontendThreads::default_threads_counts(), }) } else { None @@ -2135,6 +2170,11 @@ async fn bench_published_artifact( } else { Scenario::all_non_incr() }; + let frontend_threads_counts = if collector::version_supports_parallel_frontend(&toolchain.id) { + FrontendThreads::default_threads_counts() + } else { + vec![FrontendThreads::new(1)] + }; // Exclude benchmarks that don't work with a stable compiler. let mut compile_benchmarks = get_compile_benchmarks(dirs.compile, CompileBenchmarkFilter::All)?; @@ -2168,6 +2208,7 @@ async fn bench_published_artifact( self_profile_storage: None, bench_rustc: false, targets: vec![Target::host()], + frontend_threads_counts, }), Some(RuntimeBenchmarkConfig::new( runtime_suite, @@ -2274,6 +2315,7 @@ async fn bench_compile( &shared.toolchain, config.iterations, &config.targets, + &config.frontend_threads_counts, &collector.measured_compile_test_cases, )) .await diff --git a/collector/src/compile/benchmark/mod.rs b/collector/src/compile/benchmark/mod.rs index 049e66ee80..922878f738 100644 --- a/collector/src/compile/benchmark/mod.rs +++ b/collector/src/compile/benchmark/mod.rs @@ -1,5 +1,6 @@ use crate::compile::benchmark::category::Category; use crate::compile::benchmark::codegen_backend::CodegenBackend; +use crate::compile::benchmark::parallel_frontend::FrontendThreads; use crate::compile::benchmark::patch::Patch; use crate::compile::benchmark::profile::Profile; use crate::compile::benchmark::scenario::Scenario; @@ -19,6 +20,7 @@ use tempfile::TempDir; pub mod category; pub mod codegen_backend; +pub mod parallel_frontend; pub(crate) mod patch; pub mod profile; pub mod scenario; @@ -195,6 +197,7 @@ impl Benchmark { profile: Profile, backend: CodegenBackend, target: Target, + frontend_threads: FrontendThreads, ) -> CargoProcess<'a> { let mut cargo_args = self .config @@ -236,6 +239,7 @@ impl Benchmark { touch_file: self.config.touch_file.clone(), jobserver: None, target, + frontend_threads, workspace_package: self.config.package.clone(), } } @@ -251,6 +255,7 @@ impl Benchmark { toolchain: &Toolchain, iterations: Option, targets: &[Target], + frontend_threads_counts: &[FrontendThreads], already_computed: &hashbrown::HashSet, ) -> anyhow::Result<()> { if self.config.disabled { @@ -288,6 +293,7 @@ impl Benchmark { profile: Profile, backend: CodegenBackend, target: Target, + frontend_threads: FrontendThreads, } // Materialize the test cases that we want to benchmark @@ -297,32 +303,36 @@ impl Benchmark { for backend in backends { for profile in &profiles { for target in targets { - // Do we have any scenarios left to compute? - let remaining_scenarios = scenarios - .iter() - .filter(|scenario| { - self.should_run_scenario( - scenario, - profile, - backend, - target, - already_computed, - ) - }) - .copied() - .collect::>(); - if remaining_scenarios.is_empty() { - continue; - } + for frontend_threads in frontend_threads_counts { + // Do we have any scenarios left to compute? + let remaining_scenarios = scenarios + .iter() + .filter(|scenario| { + self.should_run_scenario( + scenario, + frontend_threads, + profile, + backend, + target, + already_computed, + ) + }) + .copied() + .collect::>(); + if remaining_scenarios.is_empty() { + continue; + } - let temp_dir = self.make_temp_dir(&self.path)?; - benchmark_dirs.push(BenchmarkDir { - dir: temp_dir, - scenarios: remaining_scenarios, - profile: *profile, - backend: *backend, - target: *target, - }); + let temp_dir = self.make_temp_dir(&self.path)?; + benchmark_dirs.push(BenchmarkDir { + dir: temp_dir, + scenarios: remaining_scenarios, + profile: *profile, + backend: *backend, + target: *target, + frontend_threads: *frontend_threads, + }); + } } } } @@ -382,6 +392,7 @@ impl Benchmark { benchmark_dir.profile, benchmark_dir.backend, benchmark_dir.target, + benchmark_dir.frontend_threads, ) .jobserver(server) .run_rustc(false) @@ -421,9 +432,10 @@ impl Benchmark { let profile = benchmark_dir.profile; let target = benchmark_dir.target; let scenarios = &benchmark_dir.scenarios; + let frontend_threads = benchmark_dir.frontend_threads; eprintln!( - "Running {}: {:?} + {:?} + {:?} + {:?}", - self.name, profile, scenarios, backend, target, + "Running {}: {:?} + {:?} + {:?} + {:?} + {:?}", + self.name, profile, scenarios, backend, target, frontend_threads, ); // We want at least two runs for all benchmarks (since we run @@ -445,10 +457,17 @@ impl Benchmark { // A full non-incremental build. if scenarios.contains(&Scenario::Full) { - self.mk_cargo_process(toolchain, cwd, profile, backend, target) - .processor(processor, Scenario::Full, "Full", None) - .run_rustc(true) - .await?; + self.mk_cargo_process( + toolchain, + cwd, + profile, + backend, + target, + frontend_threads, + ) + .processor(processor, Scenario::Full, "Full", None) + .run_rustc(true) + .await?; } // Rustdoc does not support incremental compilation @@ -456,20 +475,34 @@ impl Benchmark { // An incremental build from scratch (slowest incremental case). // This is required for any subsequent incremental builds. if scenarios.iter().any(|s| s.is_incr()) { - self.mk_cargo_process(toolchain, cwd, profile, backend, target) - .incremental(true) - .processor(processor, Scenario::IncrFull, "IncrFull", None) - .run_rustc(true) - .await?; + self.mk_cargo_process( + toolchain, + cwd, + profile, + backend, + target, + frontend_threads, + ) + .incremental(true) + .processor(processor, Scenario::IncrFull, "IncrFull", None) + .run_rustc(true) + .await?; } // An incremental build with no changes (fastest incremental case). if scenarios.contains(&Scenario::IncrUnchanged) { - self.mk_cargo_process(toolchain, cwd, profile, backend, target) - .incremental(true) - .processor(processor, Scenario::IncrUnchanged, "IncrUnchanged", None) - .run_rustc(true) - .await?; + self.mk_cargo_process( + toolchain, + cwd, + profile, + backend, + target, + frontend_threads, + ) + .incremental(true) + .processor(processor, Scenario::IncrUnchanged, "IncrUnchanged", None) + .run_rustc(true) + .await?; } if scenarios.contains(&Scenario::IncrPatched) { @@ -480,16 +513,18 @@ impl Benchmark { // An incremental build with some changes (realistic // incremental case). let scenario_str = format!("IncrPatched{i}"); - self.mk_cargo_process(toolchain, cwd, profile, backend, target) - .incremental(true) - .processor( - processor, - Scenario::IncrPatched, - &scenario_str, - Some(patch), - ) - .run_rustc(true) - .await?; + self.mk_cargo_process( + toolchain, + cwd, + profile, + backend, + target, + frontend_threads, + ) + .incremental(true) + .processor(processor, Scenario::IncrPatched, &scenario_str, Some(patch)) + .run_rustc(true) + .await?; } } } @@ -515,6 +550,7 @@ impl Benchmark { fn should_run_scenario( &self, scenario: &Scenario, + frontend_threads: &FrontendThreads, profile: &Profile, backend: &CodegenBackend, target: &Target, @@ -529,6 +565,7 @@ impl Benchmark { let profile: database::Profile = (*profile).into(); let backend: database::CodegenBackend = (*backend).into(); let target: database::Target = (*target).into(); + let frontend_threads: database::FrontendThreads = (*frontend_threads).into(); match scenario { // For these scenarios, we can simply check if they were benchmarked or not @@ -544,6 +581,7 @@ impl Benchmark { Scenario::IncrUnchanged => database::Scenario::IncrementalFresh, Scenario::IncrPatched => unreachable!(), }, + frontend_threads, }; !already_computed.contains(&test_case) } @@ -557,6 +595,7 @@ impl Benchmark { benchmark, profile, scenario: database::Scenario::IncrementalPatch(patch.name), + frontend_threads, backend, target, }; diff --git a/collector/src/compile/benchmark/parallel_frontend.rs b/collector/src/compile/benchmark/parallel_frontend.rs new file mode 100644 index 0000000000..39e7bb99e5 --- /dev/null +++ b/collector/src/compile/benchmark/parallel_frontend.rs @@ -0,0 +1,42 @@ +use std::num::NonZeroU32; + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] +pub struct FrontendThreads(NonZeroU32); + +impl FrontendThreads { + pub fn new(val: u32) -> Self { + Self( + val.try_into() + .map_err(|_| log::error!("Number of frontend threads cannot be zero")) + .unwrap(), + ) + } + pub fn get(&self) -> u32 { + self.0.get() + } + // Default thread counts for the parallel frontend + pub fn default_threads_counts() -> Vec { + database::FrontendThreads::default_threads_counts() + .iter() + .map(|v| (*v).into()) + .collect() + } +} + +impl From for FrontendThreads { + fn from(item: u32) -> Self { + Self::new(item) + } +} + +impl From for FrontendThreads { + fn from(value: database::FrontendThreads) -> Self { + FrontendThreads::new(value.0) + } +} + +impl From for database::FrontendThreads { + fn from(value: FrontendThreads) -> Self { + database::FrontendThreads(value.get()) + } +} diff --git a/collector/src/compile/execute/bencher.rs b/collector/src/compile/execute/bencher.rs index f4b8d4f7bb..433a8c2188 100644 --- a/collector/src/compile/execute/bencher.rs +++ b/collector/src/compile/execute/bencher.rs @@ -27,6 +27,7 @@ pub struct RecordedSelfProfile { profile: database::Profile, codegen_backend: database::CodegenBackend, target: database::Target, + frontend_threads: database::FrontendThreads, files: SelfProfileFiles, } @@ -87,6 +88,7 @@ impl<'a> BenchProcessor<'a> { } } + #[allow(clippy::too_many_arguments)] async fn insert_stats( &mut self, collection: CollectionId, @@ -94,6 +96,7 @@ impl<'a> BenchProcessor<'a> { profile: database::Profile, backend: CodegenBackend, target: Target, + frontend_threads: database::FrontendThreads, stats: Stats, ) { let mut buf = FuturesUnordered::new(); @@ -106,6 +109,7 @@ impl<'a> BenchProcessor<'a> { scenario, backend.into(), target.into(), + frontend_threads, stat, value, )); @@ -168,6 +172,7 @@ impl Processor for BenchProcessor<'_> { execute::store_documentation_size_into_stats(&mut res.0, &doc_dir); } } + let frontend_threads = database::FrontendThreads::from(data.frontend_threads); let scenario = match data.scenario { Scenario::Full => database::Scenario::Empty, @@ -189,6 +194,7 @@ impl Processor for BenchProcessor<'_> { profile, codegen_backend: data.backend.into(), target: data.target.into(), + frontend_threads, files, }); @@ -205,6 +211,7 @@ impl Processor for BenchProcessor<'_> { profile, data.backend, data.target, + frontend_threads, res.0, ) .await; @@ -255,6 +262,7 @@ impl Processor for BenchProcessor<'_> { benchmark: self.benchmark.clone(), profile: profile.profile, scenario: profile.scenario, + frontend_threads: profile.frontend_threads, backend: profile.codegen_backend, target: profile.target, }; diff --git a/collector/src/compile/execute/mod.rs b/collector/src/compile/execute/mod.rs index 94ec42ef12..b5c524f875 100644 --- a/collector/src/compile/execute/mod.rs +++ b/collector/src/compile/execute/mod.rs @@ -1,6 +1,7 @@ //! Execute benchmarks. use crate::compile::benchmark::codegen_backend::CodegenBackend; +use crate::compile::benchmark::parallel_frontend::FrontendThreads; use crate::compile::benchmark::patch::Patch; use crate::compile::benchmark::profile::Profile; use crate::compile::benchmark::scenario::Scenario; @@ -132,6 +133,7 @@ pub struct CargoProcess<'a> { pub touch_file: Option, pub jobserver: Option, pub target: Target, + pub frontend_threads: FrontendThreads, pub workspace_package: Option, } @@ -358,13 +360,14 @@ impl<'a> CargoProcess<'a> { // really. pub async fn run_rustc(&mut self, needs_final: bool) -> anyhow::Result<()> { log::info!( - "run_rustc with incremental={}, profile={:?}, scenario={:?}, patch={:?}, backend={:?}, target={:?}, phase={}", + "run_rustc with incremental={}, profile={:?}, scenario={:?}, patch={:?}, backend={:?}, target={:?}, frontend_threads={}, phase={}", self.incremental, self.profile, self.processor_etc.as_ref().map(|v| v.1), self.processor_etc.as_ref().and_then(|v| v.3), self.backend, self.target, + self.frontend_threads.get(), if needs_final { "benchmark" } else { "dependencies" } ); @@ -406,6 +409,12 @@ impl<'a> CargoProcess<'a> { let mut cmd = self.base_command(self.cwd, cargo_subcommand); cmd.arg("-p").arg(self.get_pkgid(self.cwd)?); + // Whatever arguments were passed from the external environment, we only look + // at the config value + cmd.env( + "RUSTC_THREAD_COUNT", + self.frontend_threads.get().to_string(), + ); match self.profile { Profile::Check => { cmd.arg("--profile").arg("check"); @@ -537,6 +546,7 @@ impl<'a> CargoProcess<'a> { patch, backend: self.backend, target: self.target, + frontend_threads: self.frontend_threads, }; match processor.process_output(&data, output).await { Ok(Retry::No) => return Ok(()), @@ -603,6 +613,7 @@ pub struct ProcessOutputData<'a> { patch: Option<&'a Patch>, backend: CodegenBackend, target: Target, + frontend_threads: FrontendThreads, } /// Trait used by `Benchmark::measure()` to provide different kinds of diff --git a/collector/src/compile/execute/profiler.rs b/collector/src/compile/execute/profiler.rs index c276f46861..c00a5f6584 100644 --- a/collector/src/compile/execute/profiler.rs +++ b/collector/src/compile/execute/profiler.rs @@ -131,11 +131,16 @@ impl Processor for ProfileProcessor<'_> { Box::pin(async move { fs::create_dir_all(self.output_dir)?; - // Produce a name of the form $PREFIX-$ID-$BENCHMARK-$PROFILE-$SCENARIO. + // Produce a name of the form $PREFIX-$ID-$BENCHMARK-$PROFILE-$SCENARIO-$FRONTEND_THREADS_. let out_file = |prefix: &str| -> String { format!( - "{}-{}-{}-{:?}-{}", - prefix, self.id, data.name, data.profile, data.scenario_str + "{}-{}-{}-{:?}-{}-frontend_threads_{}", + prefix, + self.id, + data.name, + data.profile, + data.scenario_str, + data.frontend_threads.get(), ) }; diff --git a/collector/src/lib.rs b/collector/src/lib.rs index b0dd797db7..428a98fe77 100644 --- a/collector/src/lib.rs +++ b/collector/src/lib.rs @@ -139,6 +139,15 @@ pub fn version_supports_incremental(version_str: &str) -> bool { } } +pub fn version_supports_parallel_frontend(version_str: &str) -> bool { + if let Ok(version) = version_str.parse::() { + version >= semver::Version::new(1, 33, 0) + } else { + assert!(version_str.starts_with("beta") || version_str.starts_with("master")); + true + } +} + /// Rounds serialized and deserialized floats to 2 decimal places. pub mod round_float { use serde::{Deserialize, Deserializer, Serializer}; diff --git a/collector/src/self_profile.rs b/collector/src/self_profile.rs index ccd4a402c9..499815ade7 100644 --- a/collector/src/self_profile.rs +++ b/collector/src/self_profile.rs @@ -3,7 +3,8 @@ use crate::compile::execute::SelfProfileFiles; use analyzeme::ProfilingData; use anyhow::Context; use database::{ - ArtifactId, ArtifactIdNumber, CodegenBackend, CollectionId, Profile, Scenario, Target, + ArtifactId, ArtifactIdNumber, CodegenBackend, CollectionId, FrontendThreads, Profile, Scenario, + Target, }; use reqwest::StatusCode; use std::future::Future; @@ -12,7 +13,12 @@ use std::path::{Path, PathBuf}; use std::pin::Pin; use std::time::Instant; -/// Uniquely identifies a self-profile archive. +/// Identifies a self-profile archive. Provides a single location to write the archive to. +/// +/// However, this struct provides several paths for possible archive locations to support +/// already recorded legacy profiles. These paths are orthogonal to SelfProfileId enum variants +/// and are not dependent on the database layout. +/// Currently works for `//` part only. #[derive(Debug, Hash, PartialEq, Eq, Clone)] pub enum SelfProfileId { /// Legacy ID with artifact ID number and collection ID @@ -29,15 +35,49 @@ pub enum SelfProfileId { Simple { artifact_id: ArtifactId, benchmark: BenchmarkName, + target: Target, + backend: CodegenBackend, profile: Profile, scenario: Scenario, - backend: CodegenBackend, - target: Target, + frontend_threads: FrontendThreads, }, } impl SelfProfileId { - fn relative_file_path(&self) -> PathBuf { + /// FIXME: DRY when removing SelfProfileId::Legacy support + fn relative_file_path_to_read_variants(&self) -> Vec { + match self { + Self::Legacy { .. } => vec![self.relative_file_path_to_write()], + // Here we don't include the `frontend_threads` part + Self::Simple { + artifact_id, + benchmark, + target, + backend: codegen_backend, + profile, + scenario, + .. + } => { + let primary = self.relative_file_path_to_write(); + let secondary = { + let artifact_name = match artifact_id { + ArtifactId::Commit(c) => &c.sha, + ArtifactId::Tag(name) => name, + }; + PathBuf::from("self-profile") + .join(artifact_name) + .join(benchmark.0.as_str()) + .join(target.to_string()) + .join(codegen_backend.to_string()) + .join(profile.to_string()) + .join(scenario.to_id()) + .join("self-profile.mm_profdata.sz") + }; + vec![primary, secondary] + } + } + } + fn relative_file_path_to_write(&self) -> PathBuf { match self { // self-profile//// // /self-profile-.mm_profdata.sz @@ -53,13 +93,22 @@ impl SelfProfileId { .join(profile.to_string()) .join(scenario.to_id()) .join(format!("self-profile-{collection}.mm_profdata.sz")), - // self-profile////// - // /self-profile.mm_profdata.sz + + // self-profile/ + // / + // / + // / + // / + // / + // / + // / + // self-profile.mm_profdata.sz SelfProfileId::Simple { artifact_id, benchmark, profile, scenario, + frontend_threads, backend: codegen_backend, target, } => { @@ -74,6 +123,7 @@ impl SelfProfileId { .join(codegen_backend.to_string()) .join(profile.to_string()) .join(scenario.to_id()) + .join(format!("frontend_threads_{}", frontend_threads.0)) .join("self-profile.mm_profdata.sz") } } @@ -108,6 +158,8 @@ pub trait SelfProfileStorage { /// Load the raw byte data of the self-profile with the given ID. /// Returns `None` if data for the ID was not found. + /// + /// It's this function's responsibility to run through possible read path variants #[allow(clippy::type_complexity)] fn load_raw( &self, @@ -131,8 +183,15 @@ impl LocalSelfProfileStorage { Self::new(Path::new("self-profile-storage")) } - fn path(&self, id: &SelfProfileId) -> PathBuf { - self.directory.join(id.relative_file_path()) + fn path_to_write(&self, id: &SelfProfileId) -> PathBuf { + self.directory.join(id.relative_file_path_to_write()) + } + + fn path_to_read_variants(&self, id: &SelfProfileId) -> Vec { + id.relative_file_path_to_read_variants() + .into_iter() + .map(|p| self.directory.join(p)) + .collect() } } @@ -142,7 +201,7 @@ impl SelfProfileStorage for LocalSelfProfileStorage { id: SelfProfileId, files: SelfProfileFiles, ) -> Pin> + Send>> { - let path = self.path(&id); + let path = self.path_to_write(&id); Box::pin(async move { tokio::fs::create_dir_all(path.parent().unwrap()).await?; match files { @@ -159,15 +218,18 @@ impl SelfProfileStorage for LocalSelfProfileStorage { &self, id: SelfProfileId, ) -> Pin>>> + Send>> { - let path = self.path(&id); + let path_variants = self.path_to_read_variants(&id); Box::pin(async move { - if !path.is_file() { - return Ok(None); + for path in path_variants { + if !path.is_file() { + continue; + } + let data = tokio::fs::read(&path).await.with_context(|| { + anyhow::anyhow!("Cannot read self-profile data from {}", path.display()) + })?; + return Ok(Some(data)); } - let data = tokio::fs::read(&path).await.with_context(|| { - anyhow::anyhow!("Cannot read self-profile data from {}", path.display()) - })?; - Ok(Some(data)) + Ok(None) }) } } @@ -238,7 +300,7 @@ impl SelfProfileStorage for S3SelfProfileStorage { let ctx = self.write_ctx.clone(); Box::pin(async move { - let file_path = _id.relative_file_path(); + let file_path = _id.relative_file_path_to_write(); let compressed = match _files { SelfProfileFiles::Eight { file } => { let start = Instant::now(); @@ -309,55 +371,64 @@ impl SelfProfileStorage for S3SelfProfileStorage { &self, id: SelfProfileId, ) -> Pin>>> + Send>> { - let path = id.relative_file_path(); - let url = format!( - "https://perf-data.rust-lang.org/{}", - path.to_str().expect("Non UTF-8 path used for self-profile") - ); + let path_variants = id.relative_file_path_to_read_variants(); + let url_variants: Vec<_> = path_variants + .iter() + .map(|p| { + format!( + "https://perf-data.rust-lang.org/{}", + p.to_str().expect("Non UTF-8 path used for self-profile") + ) + }) + .collect(); Box::pin(async move { - log::trace!("Downloading {url}"); - let start = Instant::now(); - let resp = match reqwest::get(&url).await { - Ok(r) => r, - Err(e) => return Err(anyhow::anyhow!("{e:?}")), - }; + for url in url_variants { + log::trace!("Downloading {url}"); + let start = Instant::now(); + let resp = match reqwest::get(&url).await { + Ok(r) => r, + Err(e) => return Err(anyhow::anyhow!("{e:?}")), + }; - if !resp.status().is_success() { - // Hitting an unknown path is returned as forbidden - if resp.status() == StatusCode::FORBIDDEN { - return Ok(None); + if !resp.status().is_success() { + // Hitting an unknown path is returned as forbidden + if resp.status() == StatusCode::FORBIDDEN { + continue; + } + return Err(anyhow::anyhow!( + "Upstream status {:?} is not successful.\nurl={url}", + resp.status(), + )); } - return Err(anyhow::anyhow!( - "Upstream status {:?} is not successful.\nurl={url}", - resp.status(), - )); - } - let compressed = match resp.bytes().await { - Ok(b) => b, - Err(e) => { - return Err(anyhow::anyhow!("Could not download from upstream: {e:?}")); - } - }; + let compressed = match resp.bytes().await { + Ok(b) => b, + Err(e) => { + return Err(anyhow::anyhow!("Could not download from upstream: {e:?}")); + } + }; - log::trace!( - "downloaded {} bytes in {:?}", - compressed.len(), - start.elapsed() - ); - - // The decompression is blocking, so we should not do it in the async task directly - let data = tokio::task::spawn_blocking(move || { - let mut data = Vec::new(); - match snap::read::FrameDecoder::new(Cursor::new(compressed)).read_to_end(&mut data) - { - Ok(_) => Ok(data), - Err(e) => Err(anyhow::anyhow!("Could not decode self-profile data: {e:?}")), - } - }) - .await??; + log::trace!( + "downloaded {} bytes in {:?}", + compressed.len(), + start.elapsed() + ); + + // The decompression is blocking, so we should not do it in the async task directly + let data = tokio::task::spawn_blocking(move || { + let mut data = Vec::new(); + match snap::read::FrameDecoder::new(Cursor::new(compressed)) + .read_to_end(&mut data) + { + Ok(_) => Ok(data), + Err(e) => Err(anyhow::anyhow!("Could not decode self-profile data: {e:?}")), + } + }) + .await??; - Ok(Some(data)) + return Ok(Some(data)); + } + Ok(None) }) } } diff --git a/database/schema.md b/database/schema.md index 19c334271a..66ddb5dc26 100644 --- a/database/schema.md +++ b/database/schema.md @@ -16,27 +16,28 @@ Below are some diagrams showing the basic layout of the database schema for thes Here is the diagram for compile-time benchmarks: ``` - ┌────────────┐ ┌───────────────┐ ┌────────────┐ - │ benchmark │ │ collection │ │ artifact │ - ├────────────┤ ├───────────────┤ ├────────────┤ -┌►│ name * │ │ id * │◄┐│ id * │◄┐ -│ │ stabilized │ │ perf_commit │ ││ name │ │ -│ │ │ │ │ ││ date │ │ -│ │ │ │ │ ││ type │ │ -│ └────────────┘ └───────────────┘ │└────────────┘ │ -│ │ │ -│ │ │ -│ ┌───────────────┐ ┌──────────┐ │ │ -│ │ pstat_series │ │ pstat │ │ │ -│ ├───────────────┤ ├──────────┤ │ │ -│ │ id * │◄┐│ id * │ │ │ -└─┤ crate │ └┤ series │ │ │ - │ profile │ │ aid ├───┼───────────────┘ - │ scenario │ │ cid │ │ - │ backend │ │ value ├───┘ - │ metric │ └──────────┘ - │ target │ - └───────────────┘ + ┌────────────┐ ┌───────────────┐ ┌────────────┐ + │ benchmark │ │ collection │ │ artifact │ + ├────────────┤ ├───────────────┤ ├────────────┤ +┌►│ name * │ │ id * │◄────┐ │ id * │◄┐ +│ │ stabilized │ │ perf_commit │ │ │ name │ │ +│ │ │ │ │ │ │ date │ │ +│ │ │ │ │ │ │ type │ │ +│ └────────────┘ └───────────────┘ │ └────────────┘ │ +│ │ │ +│ │ │ +│ ┌───────────────────┐ ┌──────────┐ │ │ +│ │ pstat_series │ │ pstat │ │ │ +│ ├───────────────────┤ ├──────────┤ │ │ +│ │ id * │◄┐│ id * │ │ │ +└─┤ crate │ └┤ series │ │ │ + │ profile │ │ aid ├───┼──────────────────┘ + │ scenario │ │ cid │ │ + │ backend │ │ value ├───┘ + │ target │ └──────────┘ + │ frontend_threads | + │ metric | + └───────────────────┘ ``` For runtime benchmarks the schema very similar, but there are different table names: @@ -84,7 +85,7 @@ Columns: ### benchmark -The different types of compile-time benchmarks that are run. +The different types of compile-time benchmarks that are run. The table stores the name of the benchmark, whether it is capable of being run using the stable compiler, and its category. The benchmark name is used as a foreign key in many of the other tables. @@ -101,7 +102,7 @@ Columns: ### pstat_series Describes the parametrization of a compile-time benchmark. Contains a unique combination -of a crate, profile, scenario and the metric being collected. +of a crate, profile, scenario, metric and parallel frontend thread count being collected. Columns: @@ -109,6 +110,8 @@ Columns: * **profile** (`text`): What type of compilation is happening - check build, optimized build (a.k.a. release build), debug build, or doc build. * **scenario** (`text`): Describes how much of the incremental cache is full. An empty incremental cache means that the compiler must do a full build. * **backend** (`text`): Codegen backend used for compilation, for example 'llvm' +* **target** (`text`): target triple of the artifact, defaults to `x86_64-unknown-linux-gnu` +* **frontend_threads** (`text`): Parallel frontend thread count ('-Zthreads=N'). * **metric** (`text`): the type of metric being collected. This corresponds to a [`statistic description`](../docs/glossary.md). @@ -193,7 +196,7 @@ Columns: * **id** (`BIGINT` / `SERIAL`): Primary key identifier for the error row; auto increments with each new error. * **aid** (`INTERGER`): References the artifact id column. -* **context** (`TEXT NOT NULL`): A little message to be able to understand a +* **context** (`TEXT NOT NULL`): A little message to be able to understand a bit more about why or where the error occured. * **message** (`TEXT NOT NULL`): The error message. * **job_id** (`INTEGER`): A nullable job_id which, if it exists it will inform diff --git a/database/src/bin/import-sqlite.rs b/database/src/bin/import-sqlite.rs index c30512315f..9706449cf1 100644 --- a/database/src/bin/import-sqlite.rs +++ b/database/src/bin/import-sqlite.rs @@ -45,7 +45,7 @@ async fn main() { let sqlite_aid = sqlite_conn.artifact_id(&aid).await; let postgres_aid = postgres_conn.artifact_id(&aid).await; - for (&(benchmark, profile, scenario, backend, target, metric), id) in + for (&(benchmark, profile, scenario, backend, target, frontend_threads, metric), id) in sqlite_idx.compile_statistic_descriptions() { if benchmarks.insert(benchmark) { @@ -75,6 +75,7 @@ async fn main() { scenario, backend, target, + frontend_threads, metric.as_str(), stat, ) diff --git a/database/src/bin/postgres-to-sqlite.rs b/database/src/bin/postgres-to-sqlite.rs index a779e1cc94..56e01d63ad 100644 --- a/database/src/bin/postgres-to-sqlite.rs +++ b/database/src/bin/postgres-to-sqlite.rs @@ -189,12 +189,13 @@ impl Table for PstatSeries { } fn postgres_select_statement(&self, _since_weeks_ago: Option) -> String { - "select id, crate, profile, scenario, backend, target, metric from ".to_string() + "SELECT id, crate, profile, scenario, backend, target, frontend_threads, metric FROM " + .to_string() + self.name() } fn sqlite_insert_statement(&self) -> &'static str { - "insert into pstat_series (id, crate, profile, scenario, backend, target, metric) VALUES (?, ?, ?, ?, ?, ?, ?)" + "INSERT INTO pstat_series (id, crate, profile, scenario, backend, target, frontend_threads, metric) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" } fn sqlite_execute_insert(&self, statement: &mut rusqlite::Statement, row: tokio_postgres::Row) { diff --git a/database/src/bin/sqlite-to-postgres.rs b/database/src/bin/sqlite-to-postgres.rs index 1eb2c6888a..345494b9b3 100644 --- a/database/src/bin/sqlite-to-postgres.rs +++ b/database/src/bin/sqlite-to-postgres.rs @@ -242,6 +242,7 @@ struct PstatSeriesRow<'a> { scenario: &'a str, backend: &'a str, target: &'a str, + frontend_threads: u32, metric: &'a str, } @@ -251,11 +252,11 @@ impl Table for PstatSeries { } fn sqlite_attributes() -> &'static str { - "id, crate, profile, scenario, backend, target, metric" + "id, crate, profile, scenario, backend, target, frontend_threads, metric" } fn postgres_attributes() -> &'static str { - "id, crate, profile, scenario, backend, target, metric" + "id, crate, profile, scenario, backend, target, frontend_threads, metric" } fn postgres_generated_id_attribute() -> Option<&'static str> { @@ -271,7 +272,8 @@ impl Table for PstatSeries { scenario: row.get_ref(3).unwrap().as_str().unwrap(), backend: row.get_ref(4).unwrap().as_str().unwrap(), target: row.get_ref(5).unwrap().as_str().unwrap(), - metric: row.get_ref(6).unwrap().as_str().unwrap(), + frontend_threads: row.get(6).unwrap(), + metric: row.get_ref(7).unwrap().as_str().unwrap(), }) .unwrap(); } diff --git a/database/src/lib.rs b/database/src/lib.rs index 7820e306c3..836623c609 100644 --- a/database/src/lib.rs +++ b/database/src/lib.rs @@ -21,6 +21,29 @@ intern!(pub struct Metric); intern!(pub struct Benchmark); intern!(pub struct TargetName); +#[derive(Serialize, Deserialize, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub struct FrontendThreads(pub u32); + +impl FrontendThreads { + // Default thread counts for the parallel frontend + pub fn default_threads_counts() -> Vec { + vec![FrontendThreads(1)] + } + pub fn single(self) -> bool { + self.0 == 1 + } +} + +impl std::str::FromStr for FrontendThreads { + type Err = String; + fn from_str(s: &str) -> Result { + let v: u32 = s + .parse() + .map_err(|e: std::num::ParseIntError| e.to_string())?; + Ok(Self(v)) + } +} + pub fn intern_target_name(target: &str) -> TargetName { intern(target) } @@ -562,7 +585,15 @@ pub struct Index { artifacts: Indexed>, /// Id lookup of compile stat description ids /// For legacy reasons called `pstat_series` in the database, and so the name is kept here. - pstat_series: Indexed<(Benchmark, Profile, Scenario, CodegenBackend, Target, Metric)>, + pstat_series: Indexed<( + Benchmark, + Profile, + Scenario, + CodegenBackend, + Target, + FrontendThreads, + Metric, + )>, /// Id lookup of runtime stat description ids runtime_pstat_series: Indexed<(Benchmark, Target, Metric)>, } @@ -684,6 +715,7 @@ pub enum DbLabel { scenario: Scenario, backend: CodegenBackend, target: Target, + frontend_threads: FrontendThreads, metric: Metric, }, } @@ -702,11 +734,18 @@ impl Lookup for DbLabel { profile, scenario, backend, - metric, target, - } => index - .pstat_series - .get(&(*benchmark, *profile, *scenario, *backend, *target, *metric)), + frontend_threads, + metric, + } => index.pstat_series.get(&( + *benchmark, + *profile, + *scenario, + *backend, + *target, + *frontend_threads, + *metric, + )), } } } @@ -722,6 +761,15 @@ impl Lookup for ArtifactId { } pub type StatisticalDescriptionId = u32; +pub type CompileStatisticDescription = ( + Benchmark, + Profile, + Scenario, + CodegenBackend, + Target, + FrontendThreads, + Metric, +); impl Index { pub async fn load(conn: &mut dyn pool::Connection) -> Index { @@ -754,7 +802,7 @@ impl Index { self.pstat_series .map .keys() - .map(|(_, _, _, _, _, metric)| metric) + .map(|(_, _, _, _, _, _, metric)| metric) .collect::>() .into_iter() .map(|s| s.to_string()) @@ -777,7 +825,7 @@ impl Index { self.pstat_series .map .keys() - .map(|(_, _, _, _, target, _)| target) + .map(|(_, _, _, _, target, _, _)| target) .collect::>() .into_iter() .cloned() @@ -790,12 +838,7 @@ impl Index { // for it as keeping indices around would be annoying. pub fn compile_statistic_descriptions( &self, - ) -> impl Iterator< - Item = ( - &(Benchmark, Profile, Scenario, CodegenBackend, Target, Metric), - StatisticalDescriptionId, - ), - > + '_ { + ) -> impl Iterator + '_ { self.pstat_series .map .iter() diff --git a/database/src/pool.rs b/database/src/pool.rs index 5de872082e..ebe3873242 100644 --- a/database/src/pool.rs +++ b/database/src/pool.rs @@ -5,7 +5,7 @@ use crate::{ BenchmarkRequestWithErrors, BenchmarkSet, CodegenBackend, CollectorConfig, CompileBenchmark, PendingBenchmarkRequests, Target, }; -use crate::{CollectionId, Index, Profile, Scenario}; +use crate::{CollectionId, FrontendThreads, Index, Profile, Scenario}; use chrono::{DateTime, Utc}; use hashbrown::{HashMap, HashSet}; use std::sync::{Arc, Mutex}; @@ -58,6 +58,7 @@ pub trait Connection: Send + Sync { scenario: Scenario, backend: CodegenBackend, target: Target, + frontend_threads: FrontendThreads, metric: &str, value: f64, ); @@ -720,6 +721,7 @@ mod tests { #[tokio::test] async fn get_compile_test_cases_with_data() { + const MOCK_FRONTEND_THREADS: FrontendThreads = FrontendThreads(1); run_db_test(|ctx| async { let db = ctx.db(); @@ -742,6 +744,7 @@ mod tests { Scenario::IncrementalFresh, CodegenBackend::Llvm, Target::X86_64UnknownLinuxGnu, + MOCK_FRONTEND_THREADS, Metric::CacheMisses.as_str(), 1.0, ) @@ -755,6 +758,7 @@ mod tests { benchmark: "benchmark".into(), profile: Profile::Check, scenario: Scenario::IncrementalFresh, + frontend_threads: MOCK_FRONTEND_THREADS, backend: CodegenBackend::Llvm, target: Target::X86_64UnknownLinuxGnu, }]) diff --git a/database/src/pool/postgres.rs b/database/src/pool/postgres.rs index ecc7f093a8..0f108a3b6c 100644 --- a/database/src/pool/postgres.rs +++ b/database/src/pool/postgres.rs @@ -7,10 +7,10 @@ use crate::{ BenchmarkJobKind, BenchmarkJobStatus, BenchmarkRequest, BenchmarkRequestIndex, BenchmarkRequestInsertResult, BenchmarkRequestStatus, BenchmarkRequestType, BenchmarkRequestWithErrors, BenchmarkSet, CodegenBackend, CollectionId, CollectorConfig, - Commit, CommitType, CompileBenchmark, Date, Index, PendingBenchmarkRequests, Profile, Scenario, - Target, BENCHMARK_JOB_STATUS_FAILURE_STR, BENCHMARK_JOB_STATUS_IN_PROGRESS_STR, - BENCHMARK_JOB_STATUS_QUEUED_STR, BENCHMARK_JOB_STATUS_SUCCESS_STR, - BENCHMARK_REQUEST_MASTER_STR, BENCHMARK_REQUEST_RELEASE_STR, + Commit, CommitType, CompileBenchmark, Date, FrontendThreads, Index, PendingBenchmarkRequests, + Profile, Scenario, Target, BENCHMARK_JOB_STATUS_FAILURE_STR, + BENCHMARK_JOB_STATUS_IN_PROGRESS_STR, BENCHMARK_JOB_STATUS_QUEUED_STR, + BENCHMARK_JOB_STATUS_SUCCESS_STR, BENCHMARK_REQUEST_MASTER_STR, BENCHMARK_REQUEST_RELEASE_STR, BENCHMARK_REQUEST_STATUS_ARTIFACTS_READY_STR, BENCHMARK_REQUEST_STATUS_COMPLETED_STR, BENCHMARK_REQUEST_STATUS_IN_PROGRESS_STR, BENCHMARK_REQUEST_STATUS_WAITING_FOR_ARTIFACTS_STR, BENCHMARK_REQUEST_TRY_STR, @@ -443,6 +443,12 @@ static MIGRATIONS: &[&str] = &[ "#, r#"ALTER TABLE job_queue ADD COLUMN is_optional BOOLEAN NOT NULL DEFAULT FALSE"#, r#"ALTER TABLE benchmark_request ADD COLUMN targets TEXT NOT NULL DEFAULT ''"#, + // Add frontend_threads to pstat_series + r#" + ALTER TABLE pstat_series ADD frontend_threads INT NOT NULL DEFAULT 1; + ALTER TABLE pstat_series DROP CONSTRAINT test_case; + ALTER TABLE pstat_series ADD CONSTRAINT test_case UNIQUE(crate, profile, scenario, backend, target, frontend_threads, metric); + "#, ]; #[async_trait::async_trait] @@ -635,8 +641,8 @@ impl PostgresConnection { .await .unwrap(), get_error: conn.prepare("select context, message from error where aid = $1").await.unwrap(), - insert_pstat_series: conn.prepare("insert into pstat_series (crate, profile, scenario, backend, target, metric) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT DO NOTHING RETURNING id").await.unwrap(), - select_pstat_series: conn.prepare("select id from pstat_series where crate = $1 and profile = $2 and scenario = $3 and backend = $4 and target = $5 and metric = $6").await.unwrap(), + insert_pstat_series: conn.prepare("insert into pstat_series (crate, profile, scenario, backend, target, frontend_threads, metric) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT DO NOTHING RETURNING id").await.unwrap(), + select_pstat_series: conn.prepare("select id from pstat_series where crate = $1 and profile = $2 and scenario = $3 and backend = $4 and target = $5 and frontend_threads = $6 and metric = $7").await.unwrap(), collection_id: conn.prepare("insert into collection (perf_commit) VALUES ($1) returning id").await.unwrap(), get_benchmarks: conn.prepare(" select name, category @@ -688,7 +694,7 @@ impl PostgresConnection { WHERE tag IS NOT NULL ").await.unwrap(), get_compile_test_cases_with_measurements: conn.prepare(" - SELECT DISTINCT crate, profile, scenario, backend, target + SELECT DISTINCT crate, profile, scenario, backend, target, frontend_threads FROM pstat_series WHERE id IN ( SELECT DISTINCT series @@ -887,7 +893,7 @@ where pstat_series: self .conn() .query( - "select id, crate, profile, scenario, backend, target, metric from pstat_series;", + "select id, crate, profile, scenario, backend, target, frontend_threads, metric from pstat_series;", &[], ) .await @@ -902,7 +908,8 @@ where row.get::<_, String>(3).as_str().parse().unwrap(), CodegenBackend::from_str(row.get::<_, String>(4).as_str()).unwrap(), Target::from_str(row.get::<_, String>(5).as_str()).unwrap(), - row.get::<_, String>(6).as_str().into(), + FrontendThreads(row.get::<_, i32>(6) as u32), + row.get::<_, String>(7).as_str().into(), ), ) }) @@ -1022,6 +1029,7 @@ where scenario: Scenario, backend: CodegenBackend, target: Target, + frontend_threads: FrontendThreads, metric: &str, stat: f64, ) { @@ -1029,11 +1037,20 @@ where let scenario = scenario.to_string(); let backend = backend.to_string(); let target = target.to_string(); + let frontend_threads = frontend_threads.0 as i32; let sid = self .conn() .query_opt( &self.statements().select_pstat_series, - &[&benchmark, &profile, &scenario, &backend, &target, &metric], + &[ + &benchmark, + &profile, + &scenario, + &backend, + &target, + &frontend_threads, + &metric, + ], ) .await .unwrap(); @@ -1043,14 +1060,30 @@ where self.conn() .query_opt( &self.statements().insert_pstat_series, - &[&benchmark, &profile, &scenario, &backend, &target, &metric], + &[ + &benchmark, + &profile, + &scenario, + &backend, + &target, + &frontend_threads, + &metric, + ], ) .await .unwrap(); self.conn() .query_one( &self.statements().select_pstat_series, - &[&benchmark, &profile, &scenario, &backend, &target, &metric], + &[ + &benchmark, + &profile, + &scenario, + &backend, + &target, + &frontend_threads, + &metric, + ], ) .await .unwrap() @@ -1627,6 +1660,7 @@ where scenario: row.get::<_, &str>(2).parse().unwrap(), backend: CodegenBackend::from_str(row.get::<_, &str>(3)).unwrap(), target: Target::from_str(row.get::<_, &str>(4)).unwrap(), + frontend_threads: FrontendThreads(row.get::<_, i32>(5) as u32), }) .collect()) } diff --git a/database/src/pool/sqlite.rs b/database/src/pool/sqlite.rs index 959653c735..7851b65736 100644 --- a/database/src/pool/sqlite.rs +++ b/database/src/pool/sqlite.rs @@ -6,7 +6,8 @@ use crate::{ ArtifactId, Benchmark, BenchmarkJob, BenchmarkJobConclusion, BenchmarkJobKind, BenchmarkRequest, BenchmarkRequestIndex, BenchmarkRequestInsertResult, BenchmarkRequestStatus, BenchmarkRequestWithErrors, BenchmarkSet, CodegenBackend, CollectionId, CollectorConfig, - Commit, CommitType, CompileBenchmark, Date, PendingBenchmarkRequests, Profile, Target, + Commit, CommitType, CompileBenchmark, Date, FrontendThreads, PendingBenchmarkRequests, Profile, + Target, }; use crate::{ArtifactIdNumber, Index}; use chrono::{DateTime, TimeZone, Utc}; @@ -448,6 +449,25 @@ static MIGRATIONS: &[Migration] = &[ ALTER TABLE runtime_pstat_series_with_target RENAME TO runtime_pstat_series; "#, ), + // Add frontend_threads as an unique constraint, defaulting to '1' + Migration::without_foreign_key_constraints( + r#" + CREATE TABLE pstat_series_with_frontend_threads( + id INTEGER PRIMARY KEY NOT NULL, + crate TEXT NOT NULL REFERENCES benchmark(name) ON DELETE CASCADE ON UPDATE CASCADE, + profile TEXT NOT NULL, + scenario TEXT NOT NULL, + backend TEXT NOT NULL, + target TEXT NOT NULL DEFAULT 'x86_64-unknown-linux-gnu', + frontend_threads INTEGER NOT NULL DEFAULT 1, + metric TEXT NOT NULL, + UNIQUE(crate, profile, scenario, backend, target, frontend_threads, metric) + ); + INSERT INTO pstat_series_with_frontend_threads select id, crate, profile, scenario, backend, 'x86_64-unknown-linux-gnu', 1, metric FROM pstat_series; + DROP TABLE pstat_series; + ALTER TABLE pstat_series_with_frontend_threads RENAME TO pstat_series; + "#, + ), ]; #[async_trait::async_trait] @@ -575,7 +595,7 @@ impl Connection for SqliteConnection { let pstat_series = self .raw() .prepare( - "select id, crate, profile, scenario, backend, target, metric from pstat_series;", + "select id, crate, profile, scenario, backend, target, frontend_threads, metric from pstat_series;", ) .unwrap() .query_map(params![], |row| { @@ -587,7 +607,8 @@ impl Connection for SqliteConnection { row.get::<_, String>(3)?.as_str().parse().unwrap(), CodegenBackend::from_str(row.get::<_, String>(4)?.as_str()).unwrap(), Target::from_str(row.get::<_, String>(5)?.as_str()).unwrap(), - row.get::<_, String>(6)?.as_str().into(), + FrontendThreads(row.get::<_, i32>(6)? as u32), + row.get::<_, String>(7)?.as_str().into(), ), )) }) @@ -723,6 +744,7 @@ impl Connection for SqliteConnection { scenario: crate::Scenario, backend: CodegenBackend, target: Target, + frontend_threads: FrontendThreads, metric: &str, value: f64, ) { @@ -730,20 +752,22 @@ impl Connection for SqliteConnection { let scenario = scenario.to_string(); let backend = backend.to_string(); let target = target.to_string(); - self.raw_ref().execute("insert or ignore into pstat_series (crate, profile, scenario, backend, target, metric) VALUES (?, ?, ?, ?, ?, ?)", params![ + self.raw_ref().execute("insert or ignore into pstat_series (crate, profile, scenario, backend, target, frontend_threads, metric) VALUES (?, ?, ?, ?, ?, ?, ?)", params![ &benchmark, &profile, &scenario, &backend, &target, + &frontend_threads.0, &metric, ]).unwrap(); - let sid: i32 = self.raw_ref().query_row("select id from pstat_series where crate = ? and profile = ? and scenario = ? and backend = ? and target = ? and metric = ?", params![ + let sid: i32 = self.raw_ref().query_row("select id from pstat_series where crate = ? and profile = ? and scenario = ? and backend = ? and target = ? and frontend_threads = ? and metric = ?", params![ &benchmark, &profile, &scenario, &backend, &target, + &frontend_threads.0, &metric, ], |r| r.get(0)).unwrap(); self.raw_ref() @@ -1082,7 +1106,7 @@ impl Connection for SqliteConnection { Ok(self .raw_ref() .prepare_cached( - "SELECT DISTINCT crate, profile, scenario, backend, target + "SELECT DISTINCT crate, profile, scenario, backend, target, frontend_threads FROM pstat_series WHERE id IN ( SELECT DISTINCT series @@ -1097,6 +1121,7 @@ impl Connection for SqliteConnection { scenario: row.get::<_, String>(2)?.parse().unwrap(), backend: row.get::<_, String>(3)?.parse().unwrap(), target: row.get::<_, String>(4)?.parse().unwrap(), + frontend_threads: FrontendThreads(row.get::<_, i32>(5)? as u32), }) })? .collect::>()?) diff --git a/database/src/selector.rs b/database/src/selector.rs index 042dbf932a..5d4535c587 100644 --- a/database/src/selector.rs +++ b/database/src/selector.rs @@ -29,7 +29,7 @@ use std::{ use crate::{ interpolate::Interpolate, metric::Metric, ArtifactId, ArtifactIdIter, Benchmark, - CodegenBackend, Connection, Index, Lookup, Profile, Scenario, Target, + CodegenBackend, Connection, FrontendThreads, Index, Lookup, Profile, Scenario, Target, }; #[derive(Debug)] @@ -192,8 +192,9 @@ pub struct CompileBenchmarkQuery { scenario: Selector, profile: Selector, backend: Selector, - metric: Selector, target: Selector, + frontend_threads: Selector, + metric: Selector, } impl CompileBenchmarkQuery { @@ -217,6 +218,11 @@ impl CompileBenchmarkQuery { self } + pub fn frontend_threads(mut self, selector: Selector) -> Self { + self.frontend_threads = selector; + self + } + pub fn target(mut self, selector: Selector) -> Self { self.target = selector; self @@ -233,8 +239,9 @@ impl CompileBenchmarkQuery { profile: Selector::All, scenario: Selector::All, backend: Selector::All, - metric: Selector::One(metric.as_str().into()), target: Selector::All, + frontend_threads: Selector::All, + metric: Selector::One(metric.as_str().into()), } } } @@ -246,8 +253,9 @@ impl Default for CompileBenchmarkQuery { scenario: Selector::All, profile: Selector::All, backend: Selector::All, - metric: Selector::All, target: Selector::All, + frontend_threads: Selector::All, + metric: Selector::All, } } } @@ -263,21 +271,28 @@ impl BenchmarkQuery for CompileBenchmarkQuery { ) -> Result>, String> { let mut statistic_descriptions: Vec<_> = index .compile_statistic_descriptions() - .filter(|(&(b, p, s, backend, target, metric), _)| { - self.benchmark.matches(b) - && self.profile.matches(p) - && self.scenario.matches(s) - && self.backend.matches(backend) - && self.target.matches(target) - && self.metric.matches(metric) - }) + .filter( + |(&(b, p, s, backend, target, frontend_threads, metric), _)| { + self.benchmark.matches(b) + && self.profile.matches(p) + && self.scenario.matches(s) + && self.backend.matches(backend) + && self.target.matches(target) + && self.frontend_threads.matches(frontend_threads) + && self.metric.matches(metric) + }, + ) .map( - |(&(benchmark, profile, scenario, backend, target, metric), sid)| { + |( + &(benchmark, profile, scenario, backend, target, frontend_threads, metric), + sid, + )| { ( CompileTestCase { benchmark, profile, scenario, + frontend_threads, backend, target, }, @@ -336,6 +351,7 @@ pub struct CompileTestCase { pub scenario: Scenario, pub backend: CodegenBackend, pub target: Target, + pub frontend_threads: FrontendThreads, } impl TestCase for CompileTestCase {} diff --git a/docs/glossary.md b/docs/glossary.md index 1793589370..5253f61357 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -1,6 +1,6 @@ # Glossary -The following is a glossary of domain specific terminology. Although benchmarks are a seemingly simple domain, they have a surprising amount of complexity. It is therefore useful to ensure that the vocabulary used to describe the domain is consistent and precise to avoid confusion. +The following is a glossary of domain specific terminology. Although benchmarks are a seemingly simple domain, they have a surprising amount of complexity. It is therefore useful to ensure that the vocabulary used to describe the domain is consistent and precise to avoid confusion. ## Common terms @@ -15,8 +15,8 @@ The following is a glossary of domain specific terminology. Although benchmarks - `check` corresponds to running `cargo check`. - `debug` corresponds to running `cargo build`. - `opt` corresponds to running `cargo build --release`. - - `doc` corresponds to running rustdoc with the JSON output format. - - `doc-json` corresponds to running rustdoc. + - `doc` corresponds to running rustdoc. + - `doc-json` corresponds to running rustdoc with the JSON output format. - `clippy` corresponds to running `cargo clippy`. * **scenario**: describes the incremental cache state and an optional change in the source since last compilation. - `full`: incremental compilation is not used. @@ -28,6 +28,8 @@ The following is a glossary of domain specific terminology. Although benchmarks - `cranelift`: experimental backend designed for quicker non-optimized builds * **target**: compilation target for which the benchmark is compiled. - `x86_64-unknown-linux-gnu`: the default x64 Linux target +* **frontend threads**: an integer corresponding to number of threads used by the frontend part + of the compiler. Maps directly to the `-Zthreads` rustc option. * **category**: a high-level group of benchmarks. Currently, there are three categories, primary (mostly real-world crates), secondary (mostly stress tests), and stable (old real-world crates, only used for the dashboard). * **artifact type**: describes what kind of artifact does the benchmark build. Either `library` or `binary`. @@ -47,7 +49,7 @@ The following is a glossary of domain specific terminology. Although benchmarks - For compile-time benchmarks, it is a combination of a benchmark, a profile, a scenario, a codegen backend and a target. - For runtime benchmarks, it a combination of a benchmark and a target. * **test**: the act of running an artifact under a test case. Each test is composed of many iterations. -* **test iteration**: a single iteration that makes up a test. Note: we currently normally run 3 test iterations for each test. +* **test iteration**: a single iteration that makes up a test. Note: we currently normally run 3 test iterations for each test. * **test result**: the set of all gathered statistics from running a test. Currently, the minimum value of a statistic from all the test iterations is used for analysis calculations and the website. * **statistic**: a single measured value of a metric in a test iteration * **statistic description**: the combination of a metric and a test case which describes a statistic. @@ -57,7 +59,7 @@ The following is a glossary of domain specific terminology. Although benchmarks ## Analysis * **artifact comparisons**: the comparison of two artifacts. This is composed of many test result comparisons. The [comparison page](https://perf.rust-lang.org/compare.html) shows a single artifact comparison between two artifacts. -* **test result comparison**: the delta between two test results for the same test case but different artifacts. The [comparison page](https://perf.rust-lang.org/compare.html) lists all the test result comparisons as percentages between two runs. +* **test result comparison**: the delta between two test results for the same test case but different artifacts. The [comparison page](https://perf.rust-lang.org/compare.html) lists all the test result comparisons as percentages between two runs. * **significance threshold**: the threshold at which a test result comparison is considered "significant" (i.e., a real change in performance and not just noise). You can see how this is calculated [here](https://github.com/rust-lang/rustc-perf/blob/master/docs/comparison-analysis.md#what-makes-a-test-result-significant). * **significant test result comparison**: a test result comparison above the significance threshold. Significant test result comparisons can be thought of as being "statistically significant". * **relevant test result comparison**: a test result comparison can be significant but still not be relevant (i.e., worth paying attention to). Relevance is a factor of the test result comparison's significance and magnitude. Comparisons are considered relevant if they are significant and have at least a small magnitude . @@ -67,9 +69,9 @@ The following is a glossary of domain specific terminology. Although benchmarks These terms are related to the [job queue system](./job-queue.md) that distributes benchmarking jobs across available collectors. -- **benchmark request**: a request for a benchmarking a *run* on a given *artifact*. Can be either created from a try build on a PR, or it is automatically created from merged master/release *artifacts*. +- **benchmark request**: a request for a benchmarking a *run* on a given *artifact*. Can be either created from a try build on a PR, or it is automatically created from merged master/release *artifacts*. - **collector**: a machine that performs benchmarks. -- **benchmark set**: a subset of a compile/runtime/bootstrap benchmark suite that is executed by a collector in a single job. +- **benchmark set**: a subset of a compile/runtime/bootstrap benchmark suite that is executed by a collector in a single job. - **job**: a high-level "work item" that defines a set of *test cases* that should be benchmarked on a specific collector. - **job queue**: a queue of *jobs*. diff --git a/site/src/request_handlers/graph.rs b/site/src/request_handlers/graph.rs index ccb0c28e0a..c06ad416a8 100644 --- a/site/src/request_handlers/graph.rs +++ b/site/src/request_handlers/graph.rs @@ -10,11 +10,11 @@ use crate::api::{detail_graphs, detail_sections, graphs, runtime_detail_graphs, use crate::load::SiteCtxt; use crate::self_profile::fetch_self_profile; -use database::interpolate::IsInterpolated; use database::selector::{ CompileBenchmarkQuery, CompileTestCase, RuntimeBenchmarkQuery, Selector, SeriesResponse, }; use database::{self, ArtifactId, CodegenBackend, Profile, Scenario, Target}; +use database::{interpolate::IsInterpolated, FrontendThreads}; /// Returns data for before/after graphs when comparing a single test result comparison /// for a compile-time benchmark. @@ -99,6 +99,7 @@ pub async fn handle_compile_detail_sections( backend: CodegenBackend, target: Target, ) -> Option { + const MOCK_FRONTEND_THREADS: FrontendThreads = FrontendThreads(1); let id = SelfProfileId::Simple { artifact_id: aid, benchmark: benchmark.into(), @@ -106,6 +107,7 @@ pub async fn handle_compile_detail_sections( scenario, backend, target, + frontend_threads: MOCK_FRONTEND_THREADS, }; fetch_self_profile(ctxt, id, None) .await diff --git a/site/src/request_handlers/self_profile.rs b/site/src/request_handlers/self_profile.rs index 6e33492bda..ca9b48b1f7 100644 --- a/site/src/request_handlers/self_profile.rs +++ b/site/src/request_handlers/self_profile.rs @@ -6,7 +6,7 @@ use crate::server::{maybe_compressed_response, Response, ResponseHeaders}; use brotli::enc::BrotliEncoderParams; use collector::compile::benchmark::BenchmarkName; use collector::SelfProfileId; -use database::{metric::Metric, CommitType}; +use database::{metric::Metric, CommitType, FrontendThreads}; use database::{selector, CodegenBackend, Target}; use database::{ArtifactId, Profile}; use headers::{ContentType, Header}; @@ -225,6 +225,7 @@ async fn get_self_profile_id( &scenario.to_id(), ) .await; + const MOCK_FRONTEND_THREADS: FrontendThreads = FrontendThreads(1); let id = match aids_and_cids.first().copied() { // We have a record in the DB, assume that it is the legacy ID @@ -243,6 +244,7 @@ async fn get_self_profile_id( scenario, backend, target, + frontend_threads: MOCK_FRONTEND_THREADS, }, };