Skip to content

Commit

Permalink
Don't run 3rd party bench suites on CI (#1787)
Browse files Browse the repository at this point in the history
* dont run 3rd party bench suites on CI

* typo

* and other annoyances
  • Loading branch information
teh-cmc authored Apr 6, 2023
1 parent 83cd229 commit a4f59b2
Show file tree
Hide file tree
Showing 4 changed files with 31 additions and 6 deletions.
5 changes: 5 additions & 0 deletions crates/re_arrow_store/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,11 @@ deadlock_detection = ["parking_lot/deadlock_detection"]
## Integration with `polars`, to efficiently use the datastore with dataframes.
polars = ["dep:polars-core", "dep:polars-ops"]

## When set, disables costly benchmark suites that measure the performance of third-party
## libraries.
## Commonly set implicitly by --all-features, e.g. on CI.
dont_bench_third_party = []


[dependencies]
# Rerun dependencies:
Expand Down
10 changes: 8 additions & 2 deletions crates/re_arrow_store/benches/arrow2.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use arrow2::{
array::{Array, PrimitiveArray, StructArray, UnionArray},
compute::aggregate::estimated_bytes_size,
};
use criterion::{criterion_group, criterion_main, Criterion};
use criterion::{criterion_group, Criterion};
use itertools::Itertools;
use re_log_types::{
component_types::{InstanceKey, Point2D, Rect2D},
Expand All @@ -21,7 +21,13 @@ use re_log_types::{
// ---

criterion_group!(benches, erased_clone, estimated_size_bytes);
criterion_main!(benches);

#[cfg(not(feature = "dont_bench_third_party"))]
criterion::criterion_main!(benches);

// Don't run these benchmarks on CI: they measure the performance of third-party libraries.
#[cfg(feature = "dont_bench_third_party")]
fn main() {}

// ---

Expand Down
10 changes: 8 additions & 2 deletions crates/re_arrow_store/benches/arrow2_convert.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;

use arrow2::{array::PrimitiveArray, datatypes::PhysicalType, types::PrimitiveType};
use criterion::{criterion_group, criterion_main, Criterion};
use criterion::{criterion_group, Criterion};
use re_log_types::{
component_types::InstanceKey, external::arrow2_convert::deserialize::TryIntoCollection,
Component as _, DataCell,
Expand All @@ -13,7 +13,13 @@ use re_log_types::{
// ---

criterion_group!(benches, serialize, deserialize);
criterion_main!(benches);

#[cfg(not(feature = "dont_bench_third_party"))]
criterion::criterion_main!(benches);

// Don't run these benchmarks on CI: they measure the performance of third-party libraries.
#[cfg(feature = "dont_bench_third_party")]
fn main() {}

// ---

Expand Down
12 changes: 10 additions & 2 deletions crates/re_arrow_store/benches/vectors.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,21 @@
#[global_allocator]
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;

use criterion::{criterion_group, criterion_main, Criterion};
use criterion::{criterion_group, Criterion};

use smallvec::SmallVec;
use tinyvec::TinyVec;

// ---

criterion_group!(benches, sort, split, swap, swap_opt);
criterion_main!(benches);

#[cfg(not(feature = "dont_bench_third_party"))]
criterion::criterion_main!(benches);

// Don't run these benchmarks on CI: they measure the performance of third-party libraries.
#[cfg(feature = "dont_bench_third_party")]
fn main() {}

// ---

Expand Down

1 comment on commit a4f59b2

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Rust Benchmark

Benchmark suite Current: a4f59b2 Previous: 83cd229 Ratio
datastore/num_rows=1000/num_instances=1000/packed=false/insert/default 11349971 ns/iter (± 765409) 14609436 ns/iter (± 504287) 0.78
datastore/num_rows=1000/num_instances=1000/packed=false/insert/bucketsz=0 14186745 ns/iter (± 971991) 17282541 ns/iter (± 261439) 0.82
datastore/num_rows=1000/num_instances=1000/packed=false/insert/bucketsz=2 13810443 ns/iter (± 1046810) 16317228 ns/iter (± 172901) 0.85
datastore/num_rows=1000/num_instances=1000/packed=false/insert/bucketsz=32 12565873 ns/iter (± 1018807) 14626634 ns/iter (± 227080) 0.86
datastore/num_rows=1000/num_instances=1000/packed=false/insert/bucketsz=2048 12830888 ns/iter (± 1057665) 14536660 ns/iter (± 182791) 0.88
datastore/num_rows=1000/num_instances=1000/packed=true/insert/default 11625253 ns/iter (± 815420) 14307661 ns/iter (± 354591) 0.81
datastore/num_rows=1000/num_instances=1000/packed=true/insert/bucketsz=0 13591259 ns/iter (± 1002800) 16803928 ns/iter (± 250663) 0.81
datastore/num_rows=1000/num_instances=1000/packed=true/insert/bucketsz=2 13446012 ns/iter (± 816476) 16152235 ns/iter (± 251798) 0.83
datastore/num_rows=1000/num_instances=1000/packed=true/insert/bucketsz=32 11638917 ns/iter (± 1000817) 14432392 ns/iter (± 175193) 0.81
datastore/num_rows=1000/num_instances=1000/packed=true/insert/bucketsz=2048 10936563 ns/iter (± 833334) 14214420 ns/iter (± 182551) 0.77
datastore/num_rows=1000/num_instances=1000/packed=false/latest_at/default 1852 ns/iter (± 17) 1818 ns/iter (± 19) 1.02
datastore/num_rows=1000/num_instances=1000/packed=false/latest_at/bucketsz=0 1858 ns/iter (± 6) 1852 ns/iter (± 10) 1.00
datastore/num_rows=1000/num_instances=1000/packed=false/latest_at/bucketsz=2 1871 ns/iter (± 5) 1844 ns/iter (± 10) 1.01
datastore/num_rows=1000/num_instances=1000/packed=false/latest_at/bucketsz=32 1843 ns/iter (± 2) 1828 ns/iter (± 11) 1.01
datastore/num_rows=1000/num_instances=1000/packed=false/latest_at/bucketsz=2048 1821 ns/iter (± 7) 1810 ns/iter (± 11) 1.01
datastore/num_rows=1000/num_instances=1000/packed=true/latest_at/default 1826 ns/iter (± 19) 1813 ns/iter (± 37) 1.01
datastore/num_rows=1000/num_instances=1000/packed=true/latest_at/bucketsz=0 1827 ns/iter (± 4) 1815 ns/iter (± 2) 1.01
datastore/num_rows=1000/num_instances=1000/packed=true/latest_at/bucketsz=2 1830 ns/iter (± 4) 1828 ns/iter (± 21) 1.00
datastore/num_rows=1000/num_instances=1000/packed=true/latest_at/bucketsz=32 1818 ns/iter (± 5) 1816 ns/iter (± 4) 1.00
datastore/num_rows=1000/num_instances=1000/packed=true/latest_at/bucketsz=2048 1786 ns/iter (± 8) 1806 ns/iter (± 6) 0.99
datastore/num_rows=1000/num_instances=1000/packed=false/latest_at_missing/primary/default 280 ns/iter (± 0) 281 ns/iter (± 0) 1.00
datastore/num_rows=1000/num_instances=1000/packed=false/latest_at_missing/secondaries/default 435 ns/iter (± 0) 433 ns/iter (± 0) 1.00
datastore/num_rows=1000/num_instances=1000/packed=false/latest_at_missing/primary/bucketsz=0 280 ns/iter (± 1) 282 ns/iter (± 1) 0.99
datastore/num_rows=1000/num_instances=1000/packed=false/latest_at_missing/secondaries/bucketsz=0 442 ns/iter (± 0) 443 ns/iter (± 0) 1.00
datastore/num_rows=1000/num_instances=1000/packed=false/latest_at_missing/primary/bucketsz=2 281 ns/iter (± 0) 281 ns/iter (± 0) 1
datastore/num_rows=1000/num_instances=1000/packed=false/latest_at_missing/secondaries/bucketsz=2 443 ns/iter (± 0) 441 ns/iter (± 0) 1.00
datastore/num_rows=1000/num_instances=1000/packed=false/latest_at_missing/primary/bucketsz=32 280 ns/iter (± 0) 281 ns/iter (± 0) 1.00
datastore/num_rows=1000/num_instances=1000/packed=false/latest_at_missing/secondaries/bucketsz=32 440 ns/iter (± 16) 438 ns/iter (± 0) 1.00
datastore/num_rows=1000/num_instances=1000/packed=false/latest_at_missing/primary/bucketsz=2048 280 ns/iter (± 0) 281 ns/iter (± 0) 1.00
datastore/num_rows=1000/num_instances=1000/packed=false/latest_at_missing/secondaries/bucketsz=2048 435 ns/iter (± 1) 434 ns/iter (± 0) 1.00
datastore/num_rows=1000/num_instances=1000/packed=true/latest_at_missing/primary/default 280 ns/iter (± 0) 280 ns/iter (± 0) 1
datastore/num_rows=1000/num_instances=1000/packed=true/latest_at_missing/secondaries/default 437 ns/iter (± 0) 433 ns/iter (± 0) 1.01
datastore/num_rows=1000/num_instances=1000/packed=true/latest_at_missing/primary/bucketsz=0 280 ns/iter (± 0) 280 ns/iter (± 0) 1
datastore/num_rows=1000/num_instances=1000/packed=true/latest_at_missing/secondaries/bucketsz=0 444 ns/iter (± 0) 441 ns/iter (± 0) 1.01
datastore/num_rows=1000/num_instances=1000/packed=true/latest_at_missing/primary/bucketsz=2 280 ns/iter (± 0) 280 ns/iter (± 0) 1
datastore/num_rows=1000/num_instances=1000/packed=true/latest_at_missing/secondaries/bucketsz=2 442 ns/iter (± 0) 441 ns/iter (± 0) 1.00
datastore/num_rows=1000/num_instances=1000/packed=true/latest_at_missing/primary/bucketsz=32 280 ns/iter (± 0) 281 ns/iter (± 0) 1.00
datastore/num_rows=1000/num_instances=1000/packed=true/latest_at_missing/secondaries/bucketsz=32 441 ns/iter (± 1) 438 ns/iter (± 0) 1.01
datastore/num_rows=1000/num_instances=1000/packed=true/latest_at_missing/primary/bucketsz=2048 280 ns/iter (± 0) 281 ns/iter (± 0) 1.00
datastore/num_rows=1000/num_instances=1000/packed=true/latest_at_missing/secondaries/bucketsz=2048 435 ns/iter (± 0) 434 ns/iter (± 0) 1.00
datastore/num_rows=1000/num_instances=1000/packed=false/range/default 14970366 ns/iter (± 1282811) 15896286 ns/iter (± 186023) 0.94
datastore/num_rows=1000/num_instances=1000/packed=false/range/bucketsz=0 2308704 ns/iter (± 86952) 4254261 ns/iter (± 223654) 0.54
datastore/num_rows=1000/num_instances=1000/packed=false/range/bucketsz=2 2171256 ns/iter (± 127706) 4215094 ns/iter (± 285494) 0.52
datastore/num_rows=1000/num_instances=1000/packed=false/range/bucketsz=32 1917166 ns/iter (± 23125) 2950765 ns/iter (± 423513) 0.65
datastore/num_rows=1000/num_instances=1000/packed=false/range/bucketsz=2048 1916937 ns/iter (± 36236) 3257660 ns/iter (± 411620) 0.59
datastore/num_rows=1000/num_instances=1000/packed=true/range/default 12573729 ns/iter (± 981756) 15159934 ns/iter (± 214189) 0.83
datastore/num_rows=1000/num_instances=1000/packed=true/range/bucketsz=0 2242329 ns/iter (± 29233) 3480879 ns/iter (± 411042) 0.64
datastore/num_rows=1000/num_instances=1000/packed=true/range/bucketsz=2 2207098 ns/iter (± 20962) 3204907 ns/iter (± 521873) 0.69
datastore/num_rows=1000/num_instances=1000/packed=true/range/bucketsz=32 1909705 ns/iter (± 12786) 2326665 ns/iter (± 288980) 0.82
datastore/num_rows=1000/num_instances=1000/packed=true/range/bucketsz=2048 1870439 ns/iter (± 19149) 2087887 ns/iter (± 153601) 0.90
mono_points_arrow/generate_message_bundles 48953436 ns/iter (± 784727) 45911970 ns/iter (± 357076) 1.07
mono_points_arrow/generate_messages 151442586 ns/iter (± 1357997) 136614555 ns/iter (± 1184878) 1.11
mono_points_arrow/encode_log_msg 190039808 ns/iter (± 2209008) 186563920 ns/iter (± 1472145) 1.02
mono_points_arrow/encode_total 389108956 ns/iter (± 2732339) 365290280 ns/iter (± 2156673) 1.07
mono_points_arrow/decode_log_msg 225962077 ns/iter (± 1091102) 214285171 ns/iter (± 867824) 1.05
mono_points_arrow/decode_message_bundles 85275679 ns/iter (± 805355) 73334124 ns/iter (± 582758) 1.16
mono_points_arrow/decode_total 310486635 ns/iter (± 1838279) 286893529 ns/iter (± 1586194) 1.08
mono_points_arrow_batched/generate_message_bundles 43142753 ns/iter (± 1485484) 41910675 ns/iter (± 406621) 1.03
mono_points_arrow_batched/generate_messages 10591935 ns/iter (± 1124589) 10675491 ns/iter (± 397830) 0.99
mono_points_arrow_batched/encode_log_msg 1514524 ns/iter (± 5056) 1558696 ns/iter (± 6816) 0.97
mono_points_arrow_batched/encode_total 55749591 ns/iter (± 2072791) 54065444 ns/iter (± 1018378) 1.03
mono_points_arrow_batched/decode_log_msg 866510 ns/iter (± 4291) 876074 ns/iter (± 4624) 0.99
mono_points_arrow_batched/decode_message_bundles 12617842 ns/iter (± 1122973) 15167256 ns/iter (± 352627) 0.83
mono_points_arrow_batched/decode_total 14108013 ns/iter (± 1201951) 16475803 ns/iter (± 291291) 0.86
batch_points_arrow/generate_message_bundles 309405 ns/iter (± 1015) 308390 ns/iter (± 951) 1.00
batch_points_arrow/generate_messages 5799 ns/iter (± 14) 5774 ns/iter (± 12) 1.00
batch_points_arrow/encode_log_msg 371711 ns/iter (± 2053) 385969 ns/iter (± 1924) 0.96
batch_points_arrow/encode_total 709898 ns/iter (± 3721) 716772 ns/iter (± 3786) 0.99
batch_points_arrow/decode_log_msg 360931 ns/iter (± 1527) 346465 ns/iter (± 1932) 1.04
batch_points_arrow/decode_message_bundles 2228 ns/iter (± 7) 2198 ns/iter (± 9) 1.01
batch_points_arrow/decode_total 366955 ns/iter (± 1881) 354356 ns/iter (± 3234) 1.04
arrow_mono_points/insert 7094643017 ns/iter (± 18111834) 6254175453 ns/iter (± 27831290) 1.13
arrow_mono_points/query 1839765 ns/iter (± 26832) 1864039 ns/iter (± 21263) 0.99
arrow_batch_points/insert 3026248 ns/iter (± 14318) 3087774 ns/iter (± 59713) 0.98
arrow_batch_points/query 17227 ns/iter (± 76) 16368 ns/iter (± 52) 1.05
arrow_batch_vecs/insert 43821 ns/iter (± 1313) 42764 ns/iter (± 123) 1.02
arrow_batch_vecs/query 506882 ns/iter (± 946) 506158 ns/iter (± 638) 1.00
tuid/Tuid::random 34 ns/iter (± 0) 34 ns/iter (± 0) 1

This comment was automatically generated by workflow using github-action-benchmark.

Please sign in to comment.