Skip to content

Commit

Permalink
improve and simplify host audio code
Browse files Browse the repository at this point in the history
  • Loading branch information
@chaosprint
chaosprint committed Aug 8, 2023
1 parent 4156fcd commit 8f1b175
Show file tree
Hide file tree
Showing 3 changed files with 132 additions and 119 deletions.
5 changes: 2 additions & 3 deletions crates/network/src/client_game_state.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ use ambient_gizmos::render::GizmoRenderer;
use ambient_gpu::gpu::Gpu;
use ambient_native_std::{asset_cache::AssetCache, color::Color, math::interpolate, shapes::Ray};
use ambient_renderer::{RenderTarget, Renderer, RendererConfig, RendererTarget};
use ambient_world_audio::systems::{setup_audio, spatial_audio_systems};
use ambient_world_audio::systems::{audio_systems, setup_audio};
use glam::{vec2, Mat4, Vec2, Vec3, Vec3Swizzles};

use ambient_core::player::{player, user_id};
Expand Down Expand Up @@ -65,8 +65,7 @@ impl ClientGameState {
vec![
Box::new(client_systems),
Box::new(world_instance_systems(true)),
Box::new(spatial_audio_systems()),
Box::new(ambient_world_audio::systems::audio_systems()),
Box::new(audio_systems()),
],
);
let mut renderer = Renderer::new(
Expand Down
5 changes: 2 additions & 3 deletions crates/world_audio/examples/client_spatial.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,9 +62,8 @@ fn spawn_emitters(world: &mut World) {
}

fn init(app: &mut App) {
app.systems.add(Box::new(
ambient_world_audio::systems::spatial_audio_systems(),
));
app.systems
.add(Box::new(ambient_world_audio::systems::audio_systems()));

let world = &mut app.world;
let _assets = world.resource(asset_cache()).clone();
Expand Down
241 changes: 128 additions & 113 deletions crates/world_audio/src/systems.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,24 +18,128 @@ use glam::{vec4, Mat4};
use parking_lot::Mutex;
use std::str::FromStr;

/// Initializes the HRTF sphere and adds the appropriate resources
///
/// TODO: customizer IR sphere selection
pub fn setup_audio(world: &mut World) -> anyhow::Result<()> {
let hrtf = Arc::new(HrtfLib::load(Cursor::new(include_bytes!(
"../IRC_1002_C.bin"
)))?);
world.add_resource(hrtf_lib(), hrtf);
Ok(())
}

/// This translates elements RHS Z-up coordinate system to the HRIR sphere LHS Y-up
/// <https://github.com/mrDIMAS/hrir_sphere_builder/blob/e52a10ece678a2b80a0978f7cf23f3ad9cce41c3/src/hrtf_builder.cpp#L155-L162>
pub const Y_UP_LHS: Mat4 = Mat4::from_cols(
vec4(1.0, 0.0, 0.0, 0.0),
vec4(0.0, 0.0, 1.0, 0.0),
vec4(0.0, 1.0, 0.0, 0.0),
vec4(0.0, 0.0, 0.0, 1.0),
);

pub fn audio_systems() -> SystemGroup {
SystemGroup::new(
"audio",
vec![
query((spatial_audio_player(), play_now())).to_system(|q, world, qs, _| {
for (audio_entity, _) in q.collect_cloned(world, qs) {
let amp = world.get(audio_entity, amplitude()).unwrap_or(1.0);
// TODO: find a way to get looping to work
// let looping = world.get(audio_entity, looping()).unwrap_or(false);
world.remove_component(audio_entity, play_now()).unwrap();

let assets = world.resource(asset_cache()).clone();
let runtime = world.resource(runtime()).clone();
let async_run = world.resource(async_run()).clone();
let url = world.get_ref(audio_entity, audio_url()).unwrap();
let url = AbsAssetUrl::from_str(url)
.unwrap()
.to_download_url(&assets)
.unwrap();

runtime.spawn(async move {
let track = AudioFromUrl { url: url.clone() }.get(&assets).await;
async_run.run(move |world| {
let listener_id =
world.get(audio_entity, spatial_audio_listener()).unwrap();
let emitter_id =
world.get(audio_entity, spatial_audio_emitter()).unwrap();
let pos_listener = world.get(listener_id, translation()).unwrap();
let rot = world.get(listener_id, rotation()).unwrap();
let pos_emitter = world.get(emitter_id, translation()).unwrap();

let listener = Arc::new(parking_lot::Mutex::new(AudioListener::new(
Mat4::from_rotation_translation(rot, pos_listener),
glam::Vec3::X * 0.3,
)));
let emitter = Arc::new(parking_lot::Mutex::new(AudioEmitter {
amplitude: amp,
attenuation: Attenuation::InversePoly {
quad: 0.1,
lin: 0.0,
constant: 1.0,
},
pos: pos_emitter,
}));
world
.add_component(emitter_id, audio_emitter(), emitter.clone())
.unwrap();
world
.add_component(listener_id, audio_listener(), listener.clone())
.unwrap();

let sender = world.resource(crate::audio_sender());
let hrtf_lib = world.resource(hrtf_lib());
let source =
track.unwrap().decode().spatial(hrtf_lib, listener, emitter);
sender.send(crate::AudioMessage::Spatial(source)).unwrap();
});
});
}
}),
// Updates the volume of audio emitters in the world
query((audio_emitter(), local_to_world())).to_system(|q, world, qs, _| {
for (_, (emitter, ltw)) in q.iter(world, qs) {
let (_, _, pos) = ltw.to_scale_rotation_translation();
let mut emitter = emitter.lock();
emitter.pos = pos;
}
}),
query((audio_listener(), local_to_world())).to_system_with_name(
"update_audio_listener",
|q, world, qs, _| {
for (_, (listener, &ltw)) in q.iter(world, qs) {
let mut listener = listener.lock();
listener.transform = Y_UP_LHS * ltw;
}
},
),
query((playing_sound(), stop_now())).to_system(|q, world, qs, _| {
for (playing_entity, _) in q.collect_cloned(world, qs) {
let sender = world.resource(crate::audio_sender());
sender
.send(crate::AudioMessage::StopById(playing_entity.to_base64()))
.unwrap();
let p = world.get(playing_entity, parent()).unwrap();
let c = world.get_ref(p, children()).unwrap();
let new_c = c
let p = world.get(playing_entity, parent());
if p.is_err() {
eprintln!("No parent component on playing entity; cannot stop audio.");
continue;
}
let parent_entity = p.unwrap();

let c = world.get_ref(parent_entity, children());
if c.is_err() {
eprintln!("No children component on parent entity; cannot stop audio.");
continue;
}
let new_children = c
.unwrap()
.iter()
.filter(|&&e| e != playing_entity)
.cloned()
.collect::<Vec<_>>();
world.set(p, children(), new_c).unwrap();
world.set(parent_entity, children(), new_children).unwrap();
world.despawn(playing_entity);
}
}),
Expand Down Expand Up @@ -72,9 +176,8 @@ pub fn audio_systems() -> SystemGroup {
.unwrap();
}
}),
query((audio_player(), play_now())).to_system(|q, world, qs, _| {
for (audio_entity, _) in q.collect_cloned(world, qs) {
// TODO: should check if these components exist
query((audio_player(), play_now(), audio_url())).to_system(|q, world, qs, _| {
for (audio_entity, (_, _, url)) in q.collect_cloned(world, qs) {
let amp = world.get(audio_entity, amplitude()).unwrap_or(1.0);
let pan = world.get(audio_entity, panning()).unwrap_or(0.0);
let freq = world.get(audio_entity, onepole_lpf()).unwrap_or(20000.0);
Expand All @@ -85,8 +188,7 @@ pub fn audio_systems() -> SystemGroup {
let assets = world.resource(asset_cache()).clone();
let runtime = world.resource(runtime()).clone();
let async_run = world.resource(async_run()).clone();
let url = world.get_ref(audio_entity, audio_url()).unwrap();
let url = AbsAssetUrl::from_str(url)
let url = AbsAssetUrl::from_str(&url)
.unwrap()
.to_download_url(&assets)
.unwrap();
Expand All @@ -99,7 +201,16 @@ pub fn audio_systems() -> SystemGroup {
let id_share_clone = id_share.clone();
async_run.run(move |world| {
let sender = world.resource(crate::audio_sender());
let id_vec = world.get_ref(audio_entity, children()).unwrap();
let id_vec = world.get_ref(audio_entity, children());
if id_vec.is_err() {
eprintln!("No children component on parent entity; cannot play audio.");
return;
}
let id_vec = id_vec.unwrap();
if id_vec.is_empty() {
eprintln!("No children component on parent entity; cannot play audio.");
return;
}
let id = id_vec.last().unwrap();
id_share.lock().replace(*id);

Expand Down Expand Up @@ -133,8 +244,13 @@ pub fn audio_systems() -> SystemGroup {
if !world.exists(audio_entity) {
return;
}
let child = world.get_ref(audio_entity, children()).unwrap();
let child = world.get_ref(audio_entity, children());
if child.is_err() {
eprintln!("No children component on parent entity; cannot auto stop audio.");
return;
}
let new_child = child
.unwrap()
.iter()
.filter(|c| *c != &id_share_clone.lock().unwrap())
.cloned()
Expand All @@ -149,107 +265,6 @@ pub fn audio_systems() -> SystemGroup {
)
}

/// Initializes the HRTF sphere and adds the appropriate resources
///
/// TODO: customizer IR sphere selection
pub fn setup_audio(world: &mut World) -> anyhow::Result<()> {
let hrtf = Arc::new(HrtfLib::load(Cursor::new(include_bytes!(
"../IRC_1002_C.bin"
)))?);
world.add_resource(hrtf_lib(), hrtf);
Ok(())
}

/// This translates elements RHS Z-up coordinate system to the HRIR sphere LHS Y-up
/// <https://github.com/mrDIMAS/hrir_sphere_builder/blob/e52a10ece678a2b80a0978f7cf23f3ad9cce41c3/src/hrtf_builder.cpp#L155-L162>
pub const Y_UP_LHS: Mat4 = Mat4::from_cols(
vec4(1.0, 0.0, 0.0, 0.0),
vec4(0.0, 0.0, 1.0, 0.0),
vec4(0.0, 1.0, 0.0, 0.0),
vec4(0.0, 0.0, 0.0, 1.0),
);

pub fn spatial_audio_systems() -> SystemGroup {
SystemGroup::new(
"spatial_audio",
vec![
query((spatial_audio_player(), play_now())).to_system(|q, world, qs, _| {
for (audio_entity, _) in q.collect_cloned(world, qs) {
let amp = world.get(audio_entity, amplitude()).unwrap_or(1.0);
// TODO: find a way to get looping to work
// let looping = world.get(audio_entity, looping()).unwrap_or(false);
world.remove_component(audio_entity, play_now()).unwrap();

let assets = world.resource(asset_cache()).clone();
let runtime = world.resource(runtime()).clone();
let async_run = world.resource(async_run()).clone();
let url = world.get_ref(audio_entity, audio_url()).unwrap();
let url = AbsAssetUrl::from_str(url)
.unwrap()
.to_download_url(&assets)
.unwrap();

runtime.spawn(async move {
let track = AudioFromUrl { url: url.clone() }.get(&assets).await;
async_run.run(move |world| {
let listener_id =
world.get(audio_entity, spatial_audio_listener()).unwrap();
let emitter_id =
world.get(audio_entity, spatial_audio_emitter()).unwrap();
let pos_listener = world.get(listener_id, translation()).unwrap();
let rot = world.get(listener_id, rotation()).unwrap();
let pos_emitter = world.get(emitter_id, translation()).unwrap();

let listener = Arc::new(parking_lot::Mutex::new(AudioListener::new(
Mat4::from_rotation_translation(rot, pos_listener),
glam::Vec3::X * 0.3,
)));
let emitter = Arc::new(parking_lot::Mutex::new(AudioEmitter {
amplitude: amp,
attenuation: Attenuation::InversePoly {
quad: 0.1,
lin: 0.0,
constant: 1.0,
},
pos: pos_emitter,
}));
world
.add_component(emitter_id, audio_emitter(), emitter.clone())
.unwrap();
world
.add_component(listener_id, audio_listener(), listener.clone())
.unwrap();

let sender = world.resource(crate::audio_sender());
let hrtf_lib = world.resource(hrtf_lib());
let source =
track.unwrap().decode().spatial(hrtf_lib, listener, emitter);
sender.send(crate::AudioMessage::Spatial(source)).unwrap();
});
});
}
}),
// Updates the volume of audio emitters in the world
query((audio_emitter(), local_to_world())).to_system(|q, world, qs, _| {
for (_, (emitter, ltw)) in q.iter(world, qs) {
let (_, _, pos) = ltw.to_scale_rotation_translation();
let mut emitter = emitter.lock();
emitter.pos = pos;
}
}),
query((audio_listener(), local_to_world())).to_system_with_name(
"update_audio_listener",
|q, world, qs, _| {
for (_, (listener, &ltw)) in q.iter(world, qs) {
let mut listener = listener.lock();
listener.transform = Y_UP_LHS * ltw;
}
},
),
],
)
}

pub fn client_systems() -> SystemGroup {
SystemGroup::new("Spatial audio", vec![Box::new(spatial_audio_systems())])
SystemGroup::new("audio", vec![Box::new(audio_systems())])
}

0 comments on commit 8f1b175

Please sign in to comment.