Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
e65243b
Benchmark native lockfile reads and writes
zanieb Sep 11, 2026
d11644a
Locate immutable inputs from the lockfile benchmark
zanieb Sep 11, 2026
4d0ffb6
Benchmark extraction of published Python wheels
zanieb Sep 11, 2026
233c415
Share wheel inputs with their first benchmark consumer
zanieb Sep 11, 2026
325beb0
Benchmark fresh and overlapping wheel installs
zanieb Sep 11, 2026
4e18434
Benchmark extraction of a published source distribution
zanieb Sep 11, 2026
4db25af
Benchmark native package builds over Django sources
zanieb Sep 11, 2026
f9f26f5
Benchmark parsing published core metadata
zanieb Sep 11, 2026
8efbf5e
Benchmark frozen project inspection commands
zanieb Sep 11, 2026
ec9c976
Prime real installed-environment benchmark fixtures
zanieb Sep 11, 2026
1ef77fa
Format the benchmark environment instructions
zanieb Sep 11, 2026
98a6021
Benchmark real installed-environment operations
zanieb Sep 11, 2026
105865b
Benchmark no-op project commands in a real environment
zanieb Sep 11, 2026
e502273
Benchmark bytecode compilation in a populated environment
zanieb Sep 11, 2026
118e374
Benchmark real wheel filename compatibility
zanieb Sep 11, 2026
ea0131d
Benchmark preparation of real publish artifacts
zanieb Sep 11, 2026
83dfb09
Benchmark real compiled requirements files
zanieb Sep 11, 2026
933d4d7
Benchmark discovery of real Python installations
zanieb Sep 11, 2026
198fe4a
Benchmark PufferLib conflict-heavy lockfiles
zanieb Sep 11, 2026
7c9cde4
Benchmark real PyPI Simple API HTML parsing
zanieb Sep 11, 2026
43d332e
Benchmark cache maintenance over real wheels
zanieb Sep 11, 2026
9907f30
Benchmark virtual environment creation
zanieb Sep 11, 2026
0ddbf27
Replay immutable package fixtures over loopback
zanieb Sep 11, 2026
8bc60d8
Benchmark concurrent remote wheel installs
zanieb Sep 11, 2026
1d0ef43
Benchmark concurrent package-index lookups
zanieb Sep 11, 2026
c486950
Benchmark package-index credential lookup
zanieb Sep 11, 2026
e2d8c62
Benchmark outdated checks for a real lockfile
zanieb Sep 11, 2026
a8f0fe8
Benchmark authenticated S3 index requests
zanieb Sep 11, 2026
3b6afee
Benchmark OSV audits of a historical project
zanieb Sep 11, 2026
93a5189
Prepare multiple sizes of frozen project environments
zanieb Sep 11, 2026
cb6377a
Benchmark installed environments across workload sizes
zanieb Sep 11, 2026
0a528c4
Benchmark no-op projects across workload sizes
zanieb Sep 11, 2026
3e2fd9a
Benchmark cache maintenance across project sizes
zanieb Sep 12, 2026
174f7a7
Benchmark TLS metadata handshakes
zanieb Sep 12, 2026
ed0672c
Ablate the preferred post-quantum TLS key share
zanieb Sep 12, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions .github/workflows/bench.yml
Original file line number Diff line number Diff line change
Expand Up @@ -54,11 +54,17 @@ jobs:
- name: "Prepare immutable benchmark fixtures"
run: python3 scripts/benchmark/prepare-fixtures.py

- name: "Build optimized benchmark command"
run: cargo build --locked --profile profiling --bin uv

- name: "Prime installed-environment fixtures"
run: python3 scripts/benchmark/prepare-environments.py --discovery --project-caches

- name: "Build benchmarks"
run: cargo codspeed build -m walltime --profile profiling -p uv-bench --bench uv --bench workspace_discovery
run: cargo codspeed build -m walltime --profile profiling -p uv-bench --bench uv --bench workspace_discovery --bench lockfile --bench wheel_extract --bench wheel_install --bench sdist_extract --bench package_build --bench project_inspection --bench installed_environment --bench project_noop --bench installed_bytecode --bench publish_preparation --bench python_discovery --bench cache_management --bench virtualenv_creation --bench concurrent_downloads --bench multi_index --bench outdated_tree --bench s3_signing --bench osv_audit --bench tls_handshake

- name: "Create artifact archive"
run: tar -cvf benchmarks-walltime.tar target/codspeed target/debug/uv .cache
run: tar -cvf benchmarks-walltime.tar target/codspeed target/debug/uv target/profiling/uv .cache

- name: "Upload benchmark artifacts"
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
Expand Down
14 changes: 13 additions & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -333,7 +333,7 @@ rcgen = { version = "0.14.5", features = [
"pem",
"ring",
], default-features = false }
rustls = { version = "0.23.36" }
rustls = { version = "0.23.36", default-features = false }
similar = { version = "3.0.0" }
webpki = { package = "rustls-webpki", version = "0.103.10" }
x509-parser = { version = "0.18.0" }
Expand Down
147 changes: 144 additions & 3 deletions crates/uv-bench/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,147 @@ name = "uv_pep440"
path = "benches/uv_pep440.rs"
harness = false

[[bench]]
name = "lockfile"
path = "benches/lockfile.rs"
harness = false

[[bench]]
name = "wheel_extract"
path = "benches/wheel_extract.rs"
harness = false

[[bench]]
name = "wheel_install"
path = "benches/wheel_install.rs"
harness = false

[[bench]]
name = "sdist_extract"
path = "benches/sdist_extract.rs"
harness = false

[[bench]]
name = "package_build"
path = "benches/package_build.rs"
harness = false

[[bench]]
name = "core_metadata"
path = "benches/core_metadata.rs"
harness = false

[[bench]]
name = "project_inspection"
path = "benches/project_inspection.rs"
harness = false

[[bench]]
name = "installed_environment"
path = "benches/installed_environment.rs"
harness = false

[[bench]]
name = "project_noop"
path = "benches/project_noop.rs"
harness = false

[[bench]]
name = "installed_bytecode"
path = "benches/installed_bytecode.rs"
harness = false

[[bench]]
name = "wheel_tags"
path = "benches/wheel_tags.rs"
harness = false

[[bench]]
name = "publish_preparation"
path = "benches/publish_preparation.rs"
harness = false

[[bench]]
name = "requirements_files"
path = "benches/requirements_files.rs"
harness = false

[[bench]]
name = "python_discovery"
path = "benches/python_discovery.rs"
harness = false

[[bench]]
name = "conflict_markers"
path = "benches/conflict_markers.rs"
harness = false

[[bench]]
name = "simple_api_html"
path = "benches/simple_api_html.rs"
harness = false

[[bench]]
name = "cache_management"
path = "benches/cache_management.rs"
harness = false

[[bench]]
name = "virtualenv_creation"
path = "benches/virtualenv_creation.rs"
harness = false

[[bench]]
name = "concurrent_downloads"
path = "benches/concurrent_downloads.rs"
harness = false

[[bench]]
name = "multi_index"
path = "benches/multi_index.rs"
harness = false

[[bench]]
name = "credential_lookup"
path = "benches/credential_lookup.rs"
harness = false

[[bench]]
name = "outdated_tree"
path = "benches/outdated_tree.rs"
harness = false

[[bench]]
name = "s3_signing"
path = "benches/s3_signing.rs"
harness = false

[[bench]]
name = "osv_audit"
path = "benches/osv_audit.rs"
harness = false

[[bench]]
name = "tls_handshake"
path = "benches/tls_handshake.rs"
harness = false

[[bench]]
name = "uv_pypi_types"
path = "benches/uv_pypi_types.rs"
harness = false

[dependencies]
fs-err = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
tempfile = { workspace = true }
toml = { workspace = true }

[dev-dependencies]
uv = { workspace = true }
uv-auth = { workspace = true }
uv-build-backend = { workspace = true }
uv-cache = { workspace = true }
uv-cli = { workspace = true }
uv-client = { workspace = true }
Expand All @@ -57,22 +191,29 @@ uv-platform-tags = { workspace = true }
uv-preview = { workspace = true }
uv-pypi-types = { workspace = true }
uv-python = { workspace = true }
uv-redacted = { workspace = true }
uv-requirements-txt = { workspace = true }
uv-resolver = { workspace = true }
uv-types = { workspace = true }
uv-version = { workspace = true }
uv-workspace = { workspace = true }

anyhow = { workspace = true }
astral-tl = { workspace = true }
async-trait = { workspace = true }
async_zip = { workspace = true }
clap = { workspace = true }
criterion = { version = "5.0.1", default-features = false, package = "codspeed-criterion-compat", features = ["async_tokio"] }
flate2 = { workspace = true }
fs-err = { workspace = true }
futures = { workspace = true }
http = { workspace = true }
jiff = { workspace = true }
serde_json = { workspace = true }
rcgen = { workspace = true }
reqwest = { workspace = true }
reqwest-middleware = { workspace = true }
rustls = { workspace = true }
sha2 = { workspace = true }
tar-codec = { workspace = true }
tempfile = { workspace = true }
tokio = { workspace = true }
tokio-util = { workspace = true }
toml = { workspace = true }
114 changes: 114 additions & 0 deletions crates/uv-bench/benches/cache_management.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
//! Inspect and clean caches populated by real project environments.

mod common;

use std::path::Path;
use std::process::Command;

use criterion::{
BatchSize, BenchmarkId, Criterion, SamplingMode, criterion_group, criterion_main,
measurement::WallTime,
};
use uv_bench::{
EnvironmentFixture, PreparedEnvironment, environment_fixtures, is_codspeed_simulation,
run_command, uv_command_with_cache,
};

fn copy_cache(source: &Path, destination: &Path) -> std::io::Result<()> {
fs_err::create_dir_all(destination)?;
for entry in fs_err::read_dir(source)? {
let entry = entry?;
let target = destination.join(entry.file_name());
let file_type = entry.file_type()?;
if file_type.is_symlink() {
let link = fs_err::read_link(entry.path())?;
assert!(link.is_relative(), "Cache links must be relocatable");
#[cfg(unix)]
fs_err::os::unix::fs::symlink(link, target)?;
#[cfg(windows)]
if fs_err::metadata(entry.path())?.is_dir() {
fs_err::os::windows::fs::symlink_dir(link, target)?;
} else {
fs_err::os::windows::fs::symlink_file(link, target)?;
}
} else if file_type.is_dir() {
copy_cache(&entry.path(), &target)?;
} else {
fs_err::copy(entry.path(), target)?;
}
}
Ok(())
}

struct PackageCache {
// Keep an installed environment alive so cached wheel files have real installation links.
_environment: PreparedEnvironment,
directory: tempfile::TempDir,
}

impl PackageCache {
fn prepare(fixture: &EnvironmentFixture) -> Self {
let directory = tempfile::tempdir().expect("Failed to create package cache");
let source = Path::new("../../.cache/bench-caches").join(&fixture.name);
assert!(
source.is_dir(),
"Missing project cache. Run `python3 scripts/benchmark/prepare-environments.py --project-caches`."
);
copy_cache(&source, directory.path()).expect("Failed to copy project cache");
let environment = PreparedEnvironment::from_fixture_with_cache(fixture, directory.path());
Self {
_environment: environment,
directory,
}
}

fn command(&self) -> Command {
let mut command = uv_command_with_cache(self.directory.path());
command.arg("--offline");
command
}
}

fn cache_management(c: &mut Criterion<WallTime>) {
if is_codspeed_simulation() {
return;
}
let mut group = c.benchmark_group("cache_management");
// Reconstructing a populated cache is expensive even though setup is not timed.
group.sampling_mode(SamplingMode::Flat);
for fixture in environment_fixtures() {
for (name, arguments) in [
("size", &["size", "--output-format", "machine"][..]),
("prune", &["prune"][..]),
("clean_one", &["clean", "packaging"][..]),
(
"clean_packages",
&["clean", "packaging", "pyyaml", "click"][..],
),
] {
group.bench_function(BenchmarkId::new(name, &fixture.name), |b| {
b.iter_batched(
|| {
let cache = PackageCache::prepare(&fixture);
let mut command = cache.command();
command.arg("cache").args(arguments);
(cache, command)
},
|(cache, mut command)| {
run_command(&mut command);
cache
},
BatchSize::PerIteration,
);
});
}
}
group.finish();
}

criterion_group! {
name = caches;
config = common::walltime_criterion();
targets = cache_management
}
criterion_main!(caches);
Loading
Loading