introduce re_types

Cargo.toml

@@ -56,6 +56,7 @@ re_tensor_ops = { path = "crates/re_tensor_ops", version = "0.7.0-alpha.0", defa
 re_time_panel = { path = "crates/re_time_panel", version = "=0.7.0-alpha.0", default-features = false }
 re_tracing = { path = "crates/re_tracing", version = "0.7.0-alpha.0", default-features = false }
 re_tuid = { path = "crates/re_tuid", version = "0.7.0-alpha.0", default-features = false }
+re_types = { path = "crates/re_types", version = "=0.7.0-alpha.0", default-features = false }
 re_types_builder = { path = "crates/re_types_builder", version = "=0.7.0-alpha.0", default-features = false }
 re_ui = { path = "crates/re_ui", version = "0.7.0-alpha.0", default-features = false }
 re_viewer = { path = "crates/re_viewer", version = "0.7.0-alpha.0", default-features = false }

crates/re_types/Cargo.toml

@@ -0,0 +1,61 @@
+[package]
+name = "re_types"
+authors.workspace = true
+description = "The standard Rerun data types, component types, and archetypes."
+edition.workspace = true
+homepage.workspace = true
+include.workspace = true
+license.workspace = true
+publish = true
+readme = "README.md"
+repository.workspace = true
+rust-version.workspace = true
+version.workspace = true
+
+
+[package.metadata.docs.rs]
+all-features = true
+
+
+[features]
+default = []
+
+## Enable color conversions.
+ecolor = ["dep:ecolor"]
+
+## Add support for some math operations using [`glam`](https://crates.io/crates/glam/).
+glam = ["dep:glam", "dep:macaw"]
+
+
+[dependencies]
+
+# External
+arrow2 = { workspace = true, features = [
+  "io_ipc",
+  "io_print",
+  "compute_concatenate",
+] }
+bytemuck = { version = "1.11", features = ["derive", "extern_crate_alloc"] }
+document-features = "0.2"
+
+# External (optional)
+ecolor = { workspace = true, optional = true }
+glam = { workspace = true, optional = true }
+macaw = { workspace = true, optional = true }
+
+
+[dev-dependencies]
+
+# External
+glam.workspace = true
+itertools.workspace = true
+
+
+[build-dependencies]
+
+# Rerun
+re_build_tools.workspace = true
+re_types_builder.workspace = true
+
+# External
+xshell = "0.2"

crates/re_types/README.md

@@ -0,0 +1,12 @@
+# re_types
+
+Part of the [`rerun`](https://github.com/rerun-io/rerun) family of crates.
+
+[![Latest version](https://img.shields.io/crates/v/re_types.svg)](https://crates.io/crates/re_types)
+[![Documentation](https://docs.rs/re_types/badge.svg)](https://docs.rs/re_types)
+![MIT](https://img.shields.io/badge/license-MIT-blue.svg)
+![Apache](https://img.shields.io/badge/license-Apache-blue.svg)
+
+The standard Rerun data types, component types, and archetypes.
+
+This crate includes both the language-agnostic definitions (flatbuffers IDL) as well as the generated code.

crates/re_types/build.rs

@@ -0,0 +1,104 @@
+//! Generates Rust & Python code from flatbuffers definitions.
+
+use xshell::{cmd, Shell};
+
+use re_build_tools::{
+    compute_crate_hash, compute_dir_hash, compute_strings_hash, is_tracked_env_var_set, iter_dir,
+    read_versioning_hash, rerun_if_changed, rerun_if_changed_or_doesnt_exist,
+    write_versioning_hash,
+};
+
+// NOTE: Don't need to add extra context to xshell invocations, it does so on its own.
+
+// ---
+
+const SOURCE_HASH_PATH: &str = "./source_hash.txt";
+const DEFINITIONS_DIR_PATH: &str = "./definitions";
+const RUST_OUTPUT_DIR_PATH: &str = ".";
+const PYTHON_OUTPUT_DIR_PATH: &str = "../../rerun_py/rerun2";
+
+fn main() {
+    if std::env::var("CI").is_ok() {
+        // Don't run on CI!
+        //
+        // The code we're generating here is actual source code that gets committed into the
+        // repository.
+        return;
+    }
+
+    if !is_tracked_env_var_set("IS_IN_RERUN_WORKSPACE") {
+        // Only run if we are in the rerun workspace, not on users machines.
+        return;
+    }
+    if is_tracked_env_var_set("RERUN_IS_PUBLISHING") {
+        // We don't need to rebuild - we should have done so beforehand!
+        // See `RELEASES.md`
+        return;
+    }
+
+    rerun_if_changed_or_doesnt_exist(SOURCE_HASH_PATH);
+    for path in iter_dir(DEFINITIONS_DIR_PATH, Some(&[".fbs"])) {
+        rerun_if_changed(&path);
+    }
+
+    // NOTE: We need to hash both the flatbuffers definitions as well as the source code of the
+    // code generator itself!
+    let re_types_builder_hash = compute_crate_hash("re_types_builder");
+    let cur_hash = read_versioning_hash(SOURCE_HASH_PATH);
+    let new_hash = compute_dir_hash(DEFINITIONS_DIR_PATH, Some(&[".fbs"]));
+    let new_hash_final = compute_strings_hash(&[&re_types_builder_hash, &new_hash]);
+
+    // Leave these be please, very useful when debugging.
+    eprintln!("re_types_builder_hash: {re_types_builder_hash:?}");
+    eprintln!("cur_hash: {cur_hash:?}");
+    eprintln!("new_hash: {new_hash:?}");
+    eprintln!("new_hash_final: {new_hash_final:?}");
+
+    if let Some(cur_hash) = cur_hash {
+        if cur_hash == new_hash_final {
+            // Neither the source of the code generator nor the IDL definitions have changed, no need
+            // to do anything at this point.
+            return;
+        }
+    }
+
+    let sh = Shell::new().unwrap();
+
+    re_types_builder::generate_rust_code(
+        DEFINITIONS_DIR_PATH,
+        RUST_OUTPUT_DIR_PATH,
+        "./definitions/rerun/archetypes.fbs",
+    );
+
+    // NOTE: We're purposefully ignoring the error here.
+    //
+    // In the very unlikely chance that the user doesn't have `cargo` in their $PATH, there's
+    // still no good reason to fail the build.
+    //
+    // The CI will catch the unformatted files at PR time and complain appropriately anyhow.
+    cmd!(sh, "cargo fmt").run().ok();
+
+    re_types_builder::generate_python_code(
+        DEFINITIONS_DIR_PATH,
+        PYTHON_OUTPUT_DIR_PATH,
+        "./definitions/rerun/archetypes.fbs",
+    );
+
+    // NOTE: We're purposefully ignoring the error here.
+    //
+    // If the user doesn't have `black` in their $PATH, there's still no good reason to fail
+    // the build.
+    //
+    // The CI will catch the unformatted files at PR time and complain appropriately anyhow.
+    cmd!(sh, "black {PYTHON_OUTPUT_DIR_PATH}").run().ok();
+
+    // NOTE: We're purposefully ignoring the error here.
+    //
+    // If the user doesn't have `ruff` in their $PATH, there's still no good reason to fail
+    // the build.
+    //
+    // The CI will catch the unformatted files at PR time and complain appropriately anyhow.
+    cmd!(sh, "ruff --fix {PYTHON_OUTPUT_DIR_PATH}").run().ok();
+
+    write_versioning_hash(SOURCE_HASH_PATH, new_hash_final);
+}

crates/re_types/definitions/arrow/attributes.fbs

@@ -0,0 +1,12 @@
+namespace arrow;
+
+/// Marks a union as sparse from Arrow's standpoint, affecting its Arrow datatype.
+///
+/// Only applies to unions.
+attribute "arrow.attr.sparse_union";
+
+/// Marks a single-field object as transparent from Arrow's standpoint, affecting its Arrow
+/// datatype.
+///
+/// This is generally most useful for getting rid of extraneous `struct` layers.
+attribute "arrow.attr.transparent";

crates/re_types/definitions/fbs/attributes.fbs

@@ -0,0 +1,16 @@
+namespace fbs.attributes;
+
+/// Mandatory attribute that applies to all kinds of objects: structs, enums, unions and even the
+/// fields within.
+///
+/// This defines a stable order between objects of the same kind, e.g. the order in which fields of a
+/// struct should be laid out when generating code.
+/// This is always required since flatbuffers works entirely with unordered maps internally, which
+/// would results in flaky code generation.
+///
+/// In unions, this effectively defines the arrow type of each variant, since that depends on the
+/// fields's order!
+///
+/// NOTE: We do not use flatbuffers' builtin `id` attribute as it only works on `table`s, whereas we
+/// need a stable order for all kinds of things.
+attribute "order";

crates/re_types/definitions/fbs/reflection.fbs

@@ -0,0 +1,157 @@
+// This schema defines objects that represent a parsed schema, like
+// the binary version of a .fbs file.
+// This could be used to operate on unknown FlatBuffers at runtime.
+// It can even ... represent itself (!)
+
+namespace reflection;
+
+// These must correspond to the enum in idl.h.
+enum BaseType : byte {
+    None,
+    UType,
+    Bool,
+    Byte,
+    UByte,
+    Short,
+    UShort,
+    Int,
+    UInt,
+    Long,
+    ULong,
+    Float,
+    Double,
+    String,
+    Vector,
+    Obj,     // Used for tables & structs.
+    Union,
+    Array,
+    Vector64,
+
+    // Add any new type above this value.
+    MaxBaseType
+}
+
+table Type {
+    base_type:BaseType;
+    element:BaseType = None;  // Only if base_type == Vector
+                              // or base_type == Array.
+    index:int = -1;  // If base_type == Object, index into "objects" below.
+                     // If base_type == Union, UnionType, or integral derived
+                     // from an enum, index into "enums" below.
+                     // If base_type == Vector && element == Union or UnionType.
+    fixed_length:uint16 = 0;  // Only if base_type == Array.
+    /// The size (octets) of the `base_type` field.
+    base_size:uint = 4; // 4 Is a common size due to offsets being that size.
+    /// The size (octets) of the `element` field, if present.
+    element_size:uint = 0;
+}
+
+table KeyValue {
+    key:string (required, key);
+    value:string;
+}
+
+table EnumVal {
+    name:string (required);
+    value:long (key);
+    object:Object (deprecated);
+    union_type:Type;
+    documentation:[string];
+    attributes:[KeyValue];
+}
+
+table Enum {
+    name:string (required, key);
+    values:[EnumVal] (required);  // In order of their values.
+    is_union:bool = false;
+    underlying_type:Type (required);
+    attributes:[KeyValue];
+    documentation:[string];
+    /// File that this Enum is declared in.
+    declaration_file: string;
+}
+
+table Field {
+    name:string (required, key);
+    type:Type (required);
+    id:ushort;
+    offset:ushort;  // Offset into the vtable for tables, or into the struct.
+    default_integer:long = 0;
+    default_real:double = 0.0;
+    deprecated:bool = false;
+    required:bool = false;
+    key:bool = false;
+    attributes:[KeyValue];
+    documentation:[string];
+    optional:bool = false;
+    /// Number of padding octets to always add after this field. Structs only.
+    padding:uint16 = 0;
+    /// If the field uses 64-bit offsets.
+    offset64:bool = false;
+}
+
+table Object {  // Used for both tables and structs.
+    name:string (required, key);
+    fields:[Field] (required);  // Sorted.
+    is_struct:bool = false;
+    minalign:int;
+    bytesize:int;  // For structs.
+    attributes:[KeyValue];
+    documentation:[string];
+    /// File that this Object is declared in.
+    declaration_file: string;
+}
+
+table RPCCall {
+    name:string (required, key);
+    request:Object (required);      // must be a table (not a struct)
+    response:Object (required);     // must be a table (not a struct)
+    attributes:[KeyValue];
+    documentation:[string];
+}
+
+table Service {
+    name:string (required, key);
+    calls:[RPCCall];
+    attributes:[KeyValue];
+    documentation:[string];
+    /// File that this Service is declared in.
+    declaration_file: string;
+}
+
+/// New schema language features that are not supported by old code generators.
+enum AdvancedFeatures : ulong (bit_flags) {
+    AdvancedArrayFeatures,
+    AdvancedUnionFeatures,
+    OptionalScalars,
+    DefaultVectorsAndStrings,
+}
+
+/// File specific information.
+/// Symbols declared within a file may be recovered by iterating over all
+/// symbols and examining the `declaration_file` field.
+table SchemaFile {
+  /// Filename, relative to project root.
+  filename:string (required, key);
+  /// Names of included files, relative to project root.
+  included_filenames:[string];
+}
+
+table Schema {
+    objects:[Object] (required);    // Sorted.
+    enums:[Enum] (required);        // Sorted.
+    file_ident:string;
+    file_ext:string;
+    root_table:Object;
+    services:[Service];             // Sorted.
+    advanced_features:AdvancedFeatures;
+    /// All the files used in this compilation. Files are relative to where
+    /// flatc was invoked.
+    fbs_files:[SchemaFile];         // Sorted.
+}
+
+root_type Schema;
+
+file_identifier "BFBS";
+file_extension "bfbs";
+