diff --git a/crates/network/src/client_game_state.rs b/crates/network/src/client_game_state.rs index 610c910907..c475e98e30 100644 --- a/crates/network/src/client_game_state.rs +++ b/crates/network/src/client_game_state.rs @@ -14,7 +14,7 @@ use ambient_gizmos::render::GizmoRenderer; use ambient_gpu::gpu::Gpu; use ambient_native_std::{asset_cache::AssetCache, color::Color, math::interpolate, shapes::Ray}; use ambient_renderer::{RenderTarget, Renderer, RendererConfig, RendererTarget}; -use ambient_world_audio::systems::{setup_audio, spatial_audio_systems}; +use ambient_world_audio::systems::{audio_systems, setup_audio}; use glam::{vec2, Mat4, Vec2, Vec3, Vec3Swizzles}; use ambient_core::player::{player, user_id}; @@ -65,8 +65,7 @@ impl ClientGameState { vec![ Box::new(client_systems), Box::new(world_instance_systems(true)), - Box::new(spatial_audio_systems()), - Box::new(ambient_world_audio::systems::audio_systems()), + Box::new(audio_systems()), ], ); let mut renderer = Renderer::new( diff --git a/crates/world_audio/examples/client_spatial.rs b/crates/world_audio/examples/client_spatial.rs index 024c54e4a0..110faea878 100644 --- a/crates/world_audio/examples/client_spatial.rs +++ b/crates/world_audio/examples/client_spatial.rs @@ -62,9 +62,8 @@ fn spawn_emitters(world: &mut World) { } fn init(app: &mut App) { - app.systems.add(Box::new( - ambient_world_audio::systems::spatial_audio_systems(), - )); + app.systems + .add(Box::new(ambient_world_audio::systems::audio_systems())); let world = &mut app.world; let _assets = world.resource(asset_cache()).clone(); diff --git a/crates/world_audio/src/systems.rs b/crates/world_audio/src/systems.rs index 4fc1f39729..f14fd80e5f 100644 --- a/crates/world_audio/src/systems.rs +++ b/crates/world_audio/src/systems.rs @@ -18,24 +18,128 @@ use glam::{vec4, Mat4}; use parking_lot::Mutex; use std::str::FromStr; +/// Initializes the HRTF sphere and adds the appropriate resources +/// +/// TODO: customizer IR sphere selection +pub fn setup_audio(world: &mut World) -> anyhow::Result<()> { + let hrtf = Arc::new(HrtfLib::load(Cursor::new(include_bytes!( + "../IRC_1002_C.bin" + )))?); + world.add_resource(hrtf_lib(), hrtf); + Ok(()) +} + +/// This translates elements RHS Z-up coordinate system to the HRIR sphere LHS Y-up +/// +pub const Y_UP_LHS: Mat4 = Mat4::from_cols( + vec4(1.0, 0.0, 0.0, 0.0), + vec4(0.0, 0.0, 1.0, 0.0), + vec4(0.0, 1.0, 0.0, 0.0), + vec4(0.0, 0.0, 0.0, 1.0), +); + pub fn audio_systems() -> SystemGroup { SystemGroup::new( "audio", vec![ + query((spatial_audio_player(), play_now())).to_system(|q, world, qs, _| { + for (audio_entity, _) in q.collect_cloned(world, qs) { + let amp = world.get(audio_entity, amplitude()).unwrap_or(1.0); + // TODO: find a way to get looping to work + // let looping = world.get(audio_entity, looping()).unwrap_or(false); + world.remove_component(audio_entity, play_now()).unwrap(); + + let assets = world.resource(asset_cache()).clone(); + let runtime = world.resource(runtime()).clone(); + let async_run = world.resource(async_run()).clone(); + let url = world.get_ref(audio_entity, audio_url()).unwrap(); + let url = AbsAssetUrl::from_str(url) + .unwrap() + .to_download_url(&assets) + .unwrap(); + + runtime.spawn(async move { + let track = AudioFromUrl { url: url.clone() }.get(&assets).await; + async_run.run(move |world| { + let listener_id = + world.get(audio_entity, spatial_audio_listener()).unwrap(); + let emitter_id = + world.get(audio_entity, spatial_audio_emitter()).unwrap(); + let pos_listener = world.get(listener_id, translation()).unwrap(); + let rot = world.get(listener_id, rotation()).unwrap(); + let pos_emitter = world.get(emitter_id, translation()).unwrap(); + + let listener = Arc::new(parking_lot::Mutex::new(AudioListener::new( + Mat4::from_rotation_translation(rot, pos_listener), + glam::Vec3::X * 0.3, + ))); + let emitter = Arc::new(parking_lot::Mutex::new(AudioEmitter { + amplitude: amp, + attenuation: Attenuation::InversePoly { + quad: 0.1, + lin: 0.0, + constant: 1.0, + }, + pos: pos_emitter, + })); + world + .add_component(emitter_id, audio_emitter(), emitter.clone()) + .unwrap(); + world + .add_component(listener_id, audio_listener(), listener.clone()) + .unwrap(); + + let sender = world.resource(crate::audio_sender()); + let hrtf_lib = world.resource(hrtf_lib()); + let source = + track.unwrap().decode().spatial(hrtf_lib, listener, emitter); + sender.send(crate::AudioMessage::Spatial(source)).unwrap(); + }); + }); + } + }), + // Updates the volume of audio emitters in the world + query((audio_emitter(), local_to_world())).to_system(|q, world, qs, _| { + for (_, (emitter, ltw)) in q.iter(world, qs) { + let (_, _, pos) = ltw.to_scale_rotation_translation(); + let mut emitter = emitter.lock(); + emitter.pos = pos; + } + }), + query((audio_listener(), local_to_world())).to_system_with_name( + "update_audio_listener", + |q, world, qs, _| { + for (_, (listener, <w)) in q.iter(world, qs) { + let mut listener = listener.lock(); + listener.transform = Y_UP_LHS * ltw; + } + }, + ), query((playing_sound(), stop_now())).to_system(|q, world, qs, _| { for (playing_entity, _) in q.collect_cloned(world, qs) { let sender = world.resource(crate::audio_sender()); sender .send(crate::AudioMessage::StopById(playing_entity.to_base64())) .unwrap(); - let p = world.get(playing_entity, parent()).unwrap(); - let c = world.get_ref(p, children()).unwrap(); - let new_c = c + let p = world.get(playing_entity, parent()); + if p.is_err() { + eprintln!("No parent component on playing entity; cannot stop audio."); + continue; + } + let parent_entity = p.unwrap(); + + let c = world.get_ref(parent_entity, children()); + if c.is_err() { + eprintln!("No children component on parent entity; cannot stop audio."); + continue; + } + let new_children = c + .unwrap() .iter() .filter(|&&e| e != playing_entity) .cloned() .collect::>(); - world.set(p, children(), new_c).unwrap(); + world.set(parent_entity, children(), new_children).unwrap(); world.despawn(playing_entity); } }), @@ -72,9 +176,8 @@ pub fn audio_systems() -> SystemGroup { .unwrap(); } }), - query((audio_player(), play_now())).to_system(|q, world, qs, _| { - for (audio_entity, _) in q.collect_cloned(world, qs) { - // TODO: should check if these components exist + query((audio_player(), play_now(), audio_url())).to_system(|q, world, qs, _| { + for (audio_entity, (_, _, url)) in q.collect_cloned(world, qs) { let amp = world.get(audio_entity, amplitude()).unwrap_or(1.0); let pan = world.get(audio_entity, panning()).unwrap_or(0.0); let freq = world.get(audio_entity, onepole_lpf()).unwrap_or(20000.0); @@ -85,8 +188,7 @@ pub fn audio_systems() -> SystemGroup { let assets = world.resource(asset_cache()).clone(); let runtime = world.resource(runtime()).clone(); let async_run = world.resource(async_run()).clone(); - let url = world.get_ref(audio_entity, audio_url()).unwrap(); - let url = AbsAssetUrl::from_str(url) + let url = AbsAssetUrl::from_str(&url) .unwrap() .to_download_url(&assets) .unwrap(); @@ -99,7 +201,16 @@ pub fn audio_systems() -> SystemGroup { let id_share_clone = id_share.clone(); async_run.run(move |world| { let sender = world.resource(crate::audio_sender()); - let id_vec = world.get_ref(audio_entity, children()).unwrap(); + let id_vec = world.get_ref(audio_entity, children()); + if id_vec.is_err() { + eprintln!("No children component on parent entity; cannot play audio."); + return; + } + let id_vec = id_vec.unwrap(); + if id_vec.is_empty() { + eprintln!("No children component on parent entity; cannot play audio."); + return; + } let id = id_vec.last().unwrap(); id_share.lock().replace(*id); @@ -133,8 +244,13 @@ pub fn audio_systems() -> SystemGroup { if !world.exists(audio_entity) { return; } - let child = world.get_ref(audio_entity, children()).unwrap(); + let child = world.get_ref(audio_entity, children()); + if child.is_err() { + eprintln!("No children component on parent entity; cannot auto stop audio."); + return; + } let new_child = child + .unwrap() .iter() .filter(|c| *c != &id_share_clone.lock().unwrap()) .cloned() @@ -149,107 +265,6 @@ pub fn audio_systems() -> SystemGroup { ) } -/// Initializes the HRTF sphere and adds the appropriate resources -/// -/// TODO: customizer IR sphere selection -pub fn setup_audio(world: &mut World) -> anyhow::Result<()> { - let hrtf = Arc::new(HrtfLib::load(Cursor::new(include_bytes!( - "../IRC_1002_C.bin" - )))?); - world.add_resource(hrtf_lib(), hrtf); - Ok(()) -} - -/// This translates elements RHS Z-up coordinate system to the HRIR sphere LHS Y-up -/// -pub const Y_UP_LHS: Mat4 = Mat4::from_cols( - vec4(1.0, 0.0, 0.0, 0.0), - vec4(0.0, 0.0, 1.0, 0.0), - vec4(0.0, 1.0, 0.0, 0.0), - vec4(0.0, 0.0, 0.0, 1.0), -); - -pub fn spatial_audio_systems() -> SystemGroup { - SystemGroup::new( - "spatial_audio", - vec![ - query((spatial_audio_player(), play_now())).to_system(|q, world, qs, _| { - for (audio_entity, _) in q.collect_cloned(world, qs) { - let amp = world.get(audio_entity, amplitude()).unwrap_or(1.0); - // TODO: find a way to get looping to work - // let looping = world.get(audio_entity, looping()).unwrap_or(false); - world.remove_component(audio_entity, play_now()).unwrap(); - - let assets = world.resource(asset_cache()).clone(); - let runtime = world.resource(runtime()).clone(); - let async_run = world.resource(async_run()).clone(); - let url = world.get_ref(audio_entity, audio_url()).unwrap(); - let url = AbsAssetUrl::from_str(url) - .unwrap() - .to_download_url(&assets) - .unwrap(); - - runtime.spawn(async move { - let track = AudioFromUrl { url: url.clone() }.get(&assets).await; - async_run.run(move |world| { - let listener_id = - world.get(audio_entity, spatial_audio_listener()).unwrap(); - let emitter_id = - world.get(audio_entity, spatial_audio_emitter()).unwrap(); - let pos_listener = world.get(listener_id, translation()).unwrap(); - let rot = world.get(listener_id, rotation()).unwrap(); - let pos_emitter = world.get(emitter_id, translation()).unwrap(); - - let listener = Arc::new(parking_lot::Mutex::new(AudioListener::new( - Mat4::from_rotation_translation(rot, pos_listener), - glam::Vec3::X * 0.3, - ))); - let emitter = Arc::new(parking_lot::Mutex::new(AudioEmitter { - amplitude: amp, - attenuation: Attenuation::InversePoly { - quad: 0.1, - lin: 0.0, - constant: 1.0, - }, - pos: pos_emitter, - })); - world - .add_component(emitter_id, audio_emitter(), emitter.clone()) - .unwrap(); - world - .add_component(listener_id, audio_listener(), listener.clone()) - .unwrap(); - - let sender = world.resource(crate::audio_sender()); - let hrtf_lib = world.resource(hrtf_lib()); - let source = - track.unwrap().decode().spatial(hrtf_lib, listener, emitter); - sender.send(crate::AudioMessage::Spatial(source)).unwrap(); - }); - }); - } - }), - // Updates the volume of audio emitters in the world - query((audio_emitter(), local_to_world())).to_system(|q, world, qs, _| { - for (_, (emitter, ltw)) in q.iter(world, qs) { - let (_, _, pos) = ltw.to_scale_rotation_translation(); - let mut emitter = emitter.lock(); - emitter.pos = pos; - } - }), - query((audio_listener(), local_to_world())).to_system_with_name( - "update_audio_listener", - |q, world, qs, _| { - for (_, (listener, <w)) in q.iter(world, qs) { - let mut listener = listener.lock(); - listener.transform = Y_UP_LHS * ltw; - } - }, - ), - ], - ) -} - pub fn client_systems() -> SystemGroup { - SystemGroup::new("Spatial audio", vec![Box::new(spatial_audio_systems())]) + SystemGroup::new("audio", vec![Box::new(audio_systems())]) }