Skip to content

Commit

Permalink
New data APIs 0: ClampedZip iterator machinery (#5573)
Browse files Browse the repository at this point in the history
This introduces a new temporary `re_query2` crate, which won't ever be
published.
It will replace the existing `re_query` crate once all the necessary
features have been backported.

As of this PR, this crate only contains the `ClampedZip` iterator
machinery, which is code generated for all the different arities.

Since I'm very, _very tired_ of the awful DX of macros, I implemented a
very low-tech code generator in the crate itself
(`src/bin/clamped_zip.rs`) that just spews the generated code on stdout.
That seems like the right complexity-to-maintenance tradeoff,
considering that iterator combinators don't really ever change.

`ClampedZip` naturally works with more than one required component,
finally!

- Fixes #4742
- Fixes #2750

Here's an example of one of these combinators:
```rust
/// Returns a new [`ClampedZip1x2`] iterator.
///
/// The number of elements in a clamped zip iterator corresponds to the number of elements in the
/// shortest of its required iterators (`r0`).
///
/// Optional iterators (`o0`, `o1`) will repeat their latest values if they happen to be to short
/// to be zipped with the shortest of the required iterators.
///
/// If an optional iterator is not only too short but actually empty, its associated default function
/// (`o0_default_fn`, `o1_default_fn`) will be executed and the resulting value repeated as necessary.
pub fn clamped_zip_1x2<R0, O0, O1, D0, D1>(
    r0: R0,
    o0: O0,
    o0_default_fn: D0,
    o1: O1,
    o1_default_fn: D1,
) -> ClampedZip1x2<R0::IntoIter, O0::IntoIter, O1::IntoIter, D0, D1>
where
    R0: IntoIterator,
    O0: IntoIterator,
    O0::Item: Clone,
    O1: IntoIterator,
    O1::Item: Clone,
    D0: Fn() -> O0::Item,
    D1: Fn() -> O1::Item,
{
    ClampedZip1x2 {
        r0: r0.into_iter(),
        o0: o0.into_iter(),
        o1: o1.into_iter(),
        o0_default_fn,
        o1_default_fn,
        o0_latest_value: None,
        o1_latest_value: None,
    }
}

/// Implements a clamped zip iterator combinator with 2 required iterators and 2 optional
/// iterators.
///
/// See [`clamped_zip_1x2`] for more information.
pub struct ClampedZip1x2<R0, O0, O1, D0, D1>
where
    R0: Iterator,
    O0: Iterator,
    O0::Item: Clone,
    O1: Iterator,
    O1::Item: Clone,
    D0: Fn() -> O0::Item,
    D1: Fn() -> O1::Item,
{
    r0: R0,
    o0: O0,
    o1: O1,
    o0_default_fn: D0,
    o1_default_fn: D1,

    o0_latest_value: Option<O0::Item>,
    o1_latest_value: Option<O1::Item>,
}

impl<R0, O0, O1, D0, D1> Iterator for ClampedZip1x2<R0, O0, O1, D0, D1>
where
    R0: Iterator,
    O0: Iterator,
    O0::Item: Clone,
    O1: Iterator,
    O1::Item: Clone,
    D0: Fn() -> O0::Item,
    D1: Fn() -> O1::Item,
{
    type Item = (R0::Item, O0::Item, O1::Item);

    #[inline]
    fn next(&mut self) -> Option<Self::Item> {
        let r0_next = self.r0.next()?;
        let o0_next = self.o0.next().or(self.o0_latest_value.take());
        let o1_next = self.o1.next().or(self.o1_latest_value.take());

        self.o0_latest_value = o0_next.clone();
        self.o1_latest_value = o1_next.clone();

        Some((
            r0_next,
            o0_next.unwrap_or_else(|| (self.o0_default_fn)()),
            o1_next.unwrap_or_else(|| (self.o1_default_fn)()),
        ))
    }
}
```

---

Part of a PR series to completely revamp the data APIs in preparation
for the removal of instance keys and the introduction of promises:
- #5573
- #5574
- #5581
- #5605
- #5606
- #5633
- #5673
- #5679
- #5687
- #5755
- TODO
- TODO

Builds on top of the static data PR series:
- #5534
  • Loading branch information
teh-cmc authored Apr 8, 2024
1 parent b6b5e28 commit e158114
Show file tree
Hide file tree
Showing 11 changed files with 3,839 additions and 2 deletions.
29 changes: 29 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@ version = "0.15.0-alpha.5"
# In particular: if we compile rerun 0.3.0-alpha.0 we only want it to use
# re_log_types 0.3.0-alpha.0, NOT 0.3.0-alpha.4 even though it is newer and semver-compatible.
re_analytics = { path = "crates/re_analytics", version = "=0.15.0-alpha.5", default-features = false }
re_build_search_index = { path = "crates/re_build_search_index", version = "=0.15.0-alpha.5", default-features = false }
re_build_examples = { path = "crates/re_build_examples", version = "=0.15.0-alpha.5", default-features = false }
re_build_info = { path = "crates/re_build_info", version = "=0.15.0-alpha.5", default-features = false }
re_build_search_index = { path = "crates/re_build_search_index", version = "=0.15.0-alpha.5", default-features = false }
re_build_tools = { path = "crates/re_build_tools", version = "=0.15.0-alpha.5", default-features = false }
re_build_web_viewer = { path = "crates/re_build_web_viewer", version = "=0.15.0-alpha.5", default-features = false }
re_crash_handler = { path = "crates/re_crash_handler", version = "=0.15.0-alpha.5", default-features = false }
Expand All @@ -46,6 +46,7 @@ re_log_encoding = { path = "crates/re_log_encoding", version = "=0.15.0-alpha.5"
re_log_types = { path = "crates/re_log_types", version = "=0.15.0-alpha.5", default-features = false }
re_memory = { path = "crates/re_memory", version = "=0.15.0-alpha.5", default-features = false }
re_query = { path = "crates/re_query", version = "=0.15.0-alpha.5", default-features = false }
re_query2 = { path = "crates/re_query2", version = "=0.15.0-alpha.5", default-features = false }
re_query_cache = { path = "crates/re_query_cache", version = "=0.15.0-alpha.5", default-features = false }
re_renderer = { path = "crates/re_renderer", version = "=0.15.0-alpha.5", default-features = false }
re_sdk = { path = "crates/re_sdk", version = "=0.15.0-alpha.5", default-features = false }
Expand Down
72 changes: 72 additions & 0 deletions crates/re_query2/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
[package]
name = "re_query2"
authors.workspace = true
description = "Temporary crate meant to replace re_query"
edition.workspace = true
homepage.workspace = true
include.workspace = true
license.workspace = true
# TODO(cmc): Replace re_query with this crate. Never publish this one.
publish = false
readme = "README.md"
repository.workspace = true
rust-version.workspace = true
version.workspace = true

[package.metadata.docs.rs]
all-features = true


[features]
default = []

## Enable (de)serialization using serde.
serde = ["dep:serde"]

## Enable codegen helper binaries (generates ClampedZip & RangeZip implementations).
codegen = ["unindent"]


[dependencies]
# Rerun dependencies:
re_data_store.workspace = true
re_format = { workspace = true, features = ["arrow"] }
re_log_types.workspace = true
re_types_core.workspace = true
re_log.workspace = true
re_tracing.workspace = true
re_tuid.workspace = true

# External dependencies:
ahash.workspace = true
anyhow.workspace = true
arrow2.workspace = true
backtrace.workspace = true
document-features.workspace = true
itertools = { workspace = true }
nohash-hasher.workspace = true
serde = { workspace = true, features = ["derive", "rc"], optional = true }
smallvec.workspace = true
thiserror.workspace = true

# Optional:
unindent = { workspace = true, optional = true }


[dev-dependencies]
re_types = { workspace = true, features = ["datagen"] }

criterion.workspace = true
itertools = { workspace = true }
mimalloc.workspace = true
rand = { workspace = true, features = ["std", "std_rng"] }
similar-asserts.workspace = true


[lib]
bench = false


[[bin]]
name = "clamped_zip"
required-features = ["codegen"]
5 changes: 5 additions & 0 deletions crates/re_query2/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
# re_query2

Temporary crate for implementing the new data APIs. Not published.

Will replace `re_query` when ready.
Loading

0 comments on commit e158114

Please sign in to comment.