Add "Crazy Robotaxi" game demo as a extension to interactive-drive

integrations/omnidreams/ludus-renderer/ludus_renderer/dynamic_scene.py

@@ -29,6 +29,7 @@
     CubePool,
     TimestampedScene,
 )
+from .clipgt import _get_obstacle_color
 
 
 class ObjectTrajectory(Protocol):
@@ -196,12 +197,7 @@ def _host_array(value: Any, dtype: np.dtype[Any]) -> np.ndarray:
     )
     colors_host = np.asarray(
         [
-            [1.0, 0.55, 0.15, 0.55, 0.18, 0.04]
-            if any(
-                token in actor.object_type.lower()
-                for token in ("truck", "bus", "trailer")
-            )
-            else [0.25, 0.75, 1.0, 0.08, 0.30, 0.55]
+            np.asarray(_get_obstacle_color(actor.object_type)).reshape(-1)
             for actor in actors
         ],
         dtype=np.float32,

integrations/omnidreams/omnidreams/interactive_drive/README.md

@@ -282,6 +282,40 @@ The HUD also subscribes to the backend's `/bev_stream` and shows a top-down
 BEV minimap below the steering and pedal controls; pass `--no-bev` to skip
 the extra rasterizer dispatch when you don't need it.
 
+Enable the overlay-only taxi game with `--taxi-game`. The HUD selects
+road-valid pickups from the scene's car-lane centerlines (falling back to the
+recorded route when lane data is unavailable). Every valid pickup remains
+available, but the forward camera shows only the three nearest pickups that are
+currently inside its view; the BEV draws every pickup that is inside its local
+map coverage and omits out-of-view targets. While seeking, the compass follows
+the nearest available pickup as the player moves. The initial layout includes a
+pickup constrained to project inside the starting camera view and preferably be
+no farther than 200 meters. Collecting any pickup randomly selects a reachable
+dropoff at least 200 routed meters away through the scene's directed car-lane
+graph, falling back to a shorter fare only when no such destination is
+reachable. Pickup and dropoff candidates are kept at least 100 meters inside the
+playable map bounds. Taxi mode starts a trip timer scaled by the shortest legal
+road distance, with a 2x deadline multiplier after pickup, and shows the active
+target, direction arrow, score, trip time, and global game time.
+Each game starts with 60 seconds. A successful
+dropoff awards 500 points plus 100 points per whole trip second remaining and
+adds 30 seconds; an expired trip timer cancels that fare. When the leaderboard
+is non-empty, its top score is shown beside the player's live score.
+When the global clock expires, the game freezes and shows the global top-ten
+leaderboard. Qualifying players enter a 1-12 character name before the board
+appears; zero-point runs are not recorded. Scores persist at
+`$FLASHDREAMS_CACHE_DIR/interactive-drive/highscores.csv`; use
+`--taxi-highscores PATH` to override it. New games use fresh random fare layouts
+by default; pass `--taxi-seed N` when debugging to reproduce the same layout for
+the same scene. Manual reset, OOB respawn, and scene changes start a fresh run
+without saving the unfinished score.
+
+Taxi mode also owns its arcade driving policy: rollouts start stationary,
+`S` brakes and then reverses after stopping, `Space` applies the handbrake,
+steering and acceleration respond more quickly, and `--traffic-density`
+selects the fraction of recorded motor traffic retained. These changes are not
+applied to normal Interactive Drive or `--game-mode`.
+
 **Steering wheel support.** Drop a profile YAML (devices, axis map, FFB
 settings) into `configs/wheels/` and the HUD will pick it up at startup. With
 `--wheel-profile auto` (the default), the HUD scans `/dev/input/by-id` first,
@@ -377,6 +411,10 @@ This is the lighter-weight path that matches the older standalone
 `interactive-drive` script: a single Vulkan window for the omnidreams
 output, no HUD chrome, no scene selector.
 
+Because Taxi mode depends on its HUD and BEV overlays, ``--taxi-game`` cannot
+be combined with bare ``--no-hud`` mode. Omit ``--no-hud`` to use the native
+Taxi HUD, or add ``--stream-mjpeg PORT`` to use the browser HUD.
+
 ```bash
 uv run --package flashdreams-omnidreams interactive-drive --no-hud
 ```
@@ -421,6 +459,9 @@ matches the desktop modes' affordances:
   server's `/state` endpoint at 10 Hz. Reads `--` until the simulation
   has produced its first chunk; numeric the moment chunks start
   arriving.
+- With `--taxi-game`, a **taxi HUD and BEV target pin** show the same active
+  pickup or dropoff, timer, direction arrow, and
+  score as the local HUD.
 - **WASD chiclets** light up while the corresponding direction key is
   held. The page tracks the `keydown`/`keyup` set locally so the
   highlight is zero-latency (no server round-trip); arrow keys light
@@ -444,6 +485,36 @@ ssh -L 8080:localhost:8080 <user>@<host>
 
 Then open `http://localhost:8080/`.
 
+#### Crazy Robotaxi alignment diagnostics
+
+Use `--taxi-alignment-diagnostics` when investigating a generated-view/map
+disconnect. The option captures Taxi-aligned frame data before HUD overlays and
+forces a PhysX collider snapshot for every simulated frame:
+
+```bash
+UV_CACHE_DIR=./uv-cache LUDUS_PHYSX_CACHE=./.cache/ludus-physx \
+uv run --package flashdreams-omnidreams interactive-drive \
+  --backend omnidreams \
+  --manifest example_world_model.yaml \
+  --taxi-game \
+  --stream-mjpeg 8080 \
+  --taxi-alignment-diagnostics ./alignment-diagnostics
+```
+
+Each launch creates a timestamped run directory containing:
+
+- `frames/frame_*.png`: synchronized HD-map conditioning, generated RGB, BEV,
+  and pre-policy PhysX contact-pose panels.
+- `telemetry.csv`: authoritative vehicle, conditioning-rig, and pre-policy
+  PhysX contact poses with their numerical position and yaw differences.
+- `metadata.json`: scene, variant, camera calibration, and captured-frame count.
+
+Drive through the collision being investigated, exit cleanly, and inspect the
+first contact sheets where the BEV and generated view begin to disagree. The
+pose-error columns distinguish app/physics synchronization faults from cases
+where the generated RGB fails to follow an otherwise consistent conditioning
+trajectory.
+
 For a richer browser frontend with lower latency, prefer the separate
 `omnidreams.webrtc.server` entry point.
 
@@ -560,6 +631,25 @@ oriented HD-map boxes for both RGB and BEV/model inputs. The first topology
 change replaces one scene slot; subsequent chunks update the actor cube pool in
 place without clearing static map or camera buffers.
 
+With `--taxi-game`, the app's Taxi trajectory remains the sole authority for
+ego heading. PhysX contacts still resolve translation and velocity, while the
+Taxi adapter prevents contact rotation from diverging from the HD-map and
+world-model conditioning trajectory. The adapter also insets vehicle chassis
+boxes by 16 cm on each horizontal axis to reduce rectangular-corner snagging;
+the reusable Ludus renderer and normal Interactive Drive collision models are
+unchanged.
+
+Each simulated frame also carries its authoritative ego state alongside the rig
+pose derived from that state. Camera conditioning, the current BEV, native HUD,
+MJPEG telemetry, taxi arrow, and waypoint projection all consume that same
+frame record. The runtime may simulate the next chunk ahead of presentation,
+but presentation never reads that future boundary state for current visuals.
+The Omnidreams output responds causally to the preceding HD-map frame, so Crazy
+Robotaxi delays that synchronized frame record by one presented frame and pairs
+it with the generated RGB that it produced. Frame zero remains tied to the
+rollout's unsimulated initial pose. This Taxi-only presenter policy is shared by
+native and MJPEG modes; normal Interactive Drive presentation is unchanged.
+
 ``GameEntity.to_game_engine_dict()`` and
 ``DynamicActorTrajectory.to_game_engine_dict()`` expose JSON-compatible
 component data and timestamped transform keyframes for an external game engine.

integrations/omnidreams/omnidreams/interactive_drive/_pipeline_fakes.py

@@ -55,19 +55,24 @@ def minimal_scene() -> SceneBundle:
 
 
 def make_trajectory(chunk_size: int) -> TrajectoryChunk:
-    return TrajectoryChunk(
-        timestamps_us=np.arange(chunk_size, dtype=np.int64),
-        rig_poses_world=np.repeat(
-            np.eye(4, dtype=np.float32)[None], chunk_size, axis=0
-        ),
-        boundary_state_after_chunk=VehicleState(
+    states = tuple(
+        VehicleState(
             x_m=0.0,
             y_m=0.0,
             z_m=0.0,
             yaw_rad=0.0,
             speed_mps=0.0,
             steer_rad=0.0,
+        )
+        for _ in range(chunk_size)
+    )
+    return TrajectoryChunk(
+        timestamps_us=np.arange(chunk_size, dtype=np.int64),
+        rig_poses_world=np.repeat(
+            np.eye(4, dtype=np.float32)[None], chunk_size, axis=0
         ),
+        vehicle_states=states,
+        boundary_state_after_chunk=states[-1],
     )
 
 

integrations/omnidreams/omnidreams/interactive_drive/app.py

@@ -9,6 +9,7 @@
 
 import numpy as np
 from loguru import logger
+from omnidreams.interactive_drive.application import InteractiveDriveApplication
 from omnidreams.interactive_drive.backends.base import RenderBackend
 from omnidreams.interactive_drive.config import AppConfig
 from omnidreams.interactive_drive.input.keyboard import (
@@ -29,8 +30,11 @@
     EgoVehicleKinematics,
     build_ground_snapper,
     build_map_bounds,
+    integrate_vehicle,
     state_from_initial_pose,
+    step_physics_world,
 )
+from omnidreams.interactive_drive.simulation.game_physics import GamePhysicsWorld
 from omnidreams.interactive_drive.simulation.ground_snap import GroundSnapper
 from omnidreams.interactive_drive.simulation.map_bounds import MapBounds
 from omnidreams.interactive_drive.streaming_presenter import (
@@ -69,6 +73,8 @@ def __init__(
         *,
         trace_sink: TraceSink | None = None,
         close_presenter_on_exit: bool = True,
+        keyboard: KeyboardState | None = None,
+        application: InteractiveDriveApplication | None = None,
     ) -> None:
         """Construct the engine and begin model warmup.
 
@@ -85,7 +91,8 @@ def __init__(
         """
         self._config = config
         self._backend = backend
-        self._keyboard = KeyboardState()
+        self._keyboard = keyboard or KeyboardState()
+        self._application = application
         if config.backend == "omnidreams":
             self._keyboard.set_view_mode("model_rgb")
         if presenter is None:
@@ -98,6 +105,8 @@ def __init__(
             bind_keyboard = getattr(self._presenter, "bind_keyboard", None)
             if callable(bind_keyboard):
                 bind_keyboard(self._keyboard)
+        if self._application is not None:
+            self._application.configure_presenter(self._presenter)
         # When ``False`` the caller (the demo's outer scene-change loop)
         # owns the presenter's lifecycle: it constructs one presenter at
         # startup, reuses it across many scenes, and only closes it when
@@ -196,6 +205,8 @@ def load_scene(
         cached = self._cached_scene(scene_path, variant, prompt_override)
         if cached is not None:
             self._scene, self._map_bounds, self._ground_snapper = cached
+            if self._application is not None:
+                self._application.load_scene(self._scene, self._map_bounds)
             self._pipeline.request_scene(self._scene)
             return True
 
@@ -248,6 +259,8 @@ def _parse() -> None:
             # while we were committing the loaded bundle, leave the presenter in
             # close/requested state for the outer loop to consume.
             return False
+        if self._application is not None:
+            self._application.load_scene(self._scene, self._map_bounds)
         self._pipeline.request_scene(self._scene)
         return True
 
@@ -434,6 +447,8 @@ def run_scene(self) -> None:
         """
         if self._scene is None or self._map_bounds is None:
             raise RuntimeError("load_scene() must be called before run_scene()")
+        if self._application is not None:
+            self._application.configure_scene_presenter(self._presenter, self._scene)
         # Seed the loop's initial ``last_presented_frame`` with the scene's
         # first frame. The loop overlays a live loading status over it (see
         # ``_loading_status_message``) until the first generated chunk
@@ -449,23 +464,57 @@ def run_scene(self) -> None:
         # OOB respawn, so switch the indicator to "Resetting..." for those.
         loading_status = self._loading_status_message
         while not self._presenter.should_close:
+            if self._application is None:
+                vehicle_config = self._config.vehicle
+                initial_speed_mps = 10.0
+                integrate_fn = integrate_vehicle
+                physics_world_factory = GamePhysicsWorld
+                physics_step_fn = step_physics_world
+                visual_flare_enabled = self._config.visual_flare_enabled
+                ground_snapper = self._ground_snapper
+                capture_physics_debug = False
+                include_initial_state_in_first_chunk = False
+            else:
+                rollout_spec = self._application.rollout_spec(
+                    self._scene,
+                    default_vehicle=self._config.vehicle,
+                    default_visual_flare_enabled=self._config.visual_flare_enabled,
+                )
+                vehicle_config = rollout_spec.vehicle_config
+                initial_speed_mps = rollout_spec.initial_speed_mps
+                integrate_fn = rollout_spec.integrate_fn
+                physics_world_factory = rollout_spec.physics_world_factory
+                physics_step_fn = rollout_spec.physics_step_fn
+                visual_flare_enabled = rollout_spec.visual_flare_enabled
+                ground_snapper = rollout_spec.ground_snapper
+                capture_physics_debug = rollout_spec.capture_physics_debug
+                include_initial_state_in_first_chunk = (
+                    rollout_spec.include_initial_state_in_first_chunk
+                )
             simulation = EgoVehicleKinematics(
                 initial_state=state_from_initial_pose(
                     initial_rig_to_world=self._scene.initial_rig_to_world,
                     initial_yaw_rad=self._scene.initial_yaw_rad,
-                    # Start each rollout at a fixed 10 m/s so the ego is
-                    # already rolling on initial load (and after a manual
-                    # reset / OOB respawn), instead of launching at the
-                    # clip's full recorded speed.
-                    initial_speed_mps=10.0,
+                    initial_speed_mps=initial_speed_mps,
                 ),
-                vehicle_config=self._config.vehicle,
-                ground_snapper=self._ground_snapper,
+                vehicle_config=vehicle_config,
+                ground_snapper=ground_snapper,
                 initial_timestamp_us=self._scene.initial_timestamp_us,
                 map_bounds=self._map_bounds,
                 oob_margin_m=self._config.oob_margin_m,
                 oob_warning_zone_m=self._config.oob_warning_zone_m,
                 scene=self._scene,
+                integrate_fn=integrate_fn,
+                physics_world_factory=physics_world_factory,
+                physics_step_fn=physics_step_fn,
+                include_initial_state_in_first_chunk=(
+                    include_initial_state_in_first_chunk
+                ),
+            )
+            runtime_application = (
+                None
+                if self._application is None
+                else self._application.create_runtime(self._scene, simulation)
             )
             # Publish the freshly-built initial state up front so read-side
             # speed readouts (the HUD speed digit, the browser ``/state``
@@ -474,7 +523,10 @@ def run_scene(self) -> None:
             # screen through the "Resetting..." window until the new rollout
             # requested its first chunk -- the "reset doesn't reset the
             # displayed speed" symptom.
-            self._keyboard.update_telemetry(simulation.current_state)
+            if runtime_application is not None:
+                runtime_application.publish_boundary(simulation.current_state)
+            else:
+                self._keyboard.update_telemetry(simulation.current_state)
             input_backend = KeyboardInputBackend(self._keyboard)
             try:
                 reset_requested = run_main_loop(
@@ -496,10 +548,12 @@ def run_scene(self) -> None:
                         stop_after_consumed_chunks=(
                             self._config.stop_after_consumed_chunks
                         ),
-                        visual_flare_enabled=self._config.visual_flare_enabled,
+                        visual_flare_enabled=visual_flare_enabled,
+                        capture_physics_debug=capture_physics_debug,
                     ),
                     loading_status=loading_status,
                     trace_context=self._trace_context,
+                    runtime_application=runtime_application,
                 )
             finally:
                 simulation.close()