From bc65445af44a335a0586a393c792614330258249 Mon Sep 17 00:00:00 2001 From: Eskil Queseth Date: Sat, 12 Jun 2021 02:30:01 +0200 Subject: simplify audio output infrastructure --- mumd/src/audio/input.rs | 79 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 67 insertions(+), 12 deletions(-) (limited to 'mumd/src/audio/input.rs') diff --git a/mumd/src/audio/input.rs b/mumd/src/audio/input.rs index e45ff27..162dd2c 100644 --- a/mumd/src/audio/input.rs +++ b/mumd/src/audio/input.rs @@ -8,20 +8,49 @@ use crate::error::{AudioError, AudioStream}; use crate::state::StatePhase; pub fn callback( - mut input_sender: futures_channel::mpsc::Sender, + mut input_sender: futures_channel::mpsc::Sender>, + transformers: Vec Option<&mut [f32]> + Send + 'static>>, + frame_size: u32, + sample_rate: u32, + channels: u16, input_volume_receiver: watch::Receiver, phase_watcher: watch::Receiver, ) -> impl FnMut(&[T], &InputCallbackInfo) + Send + 'static { + let buffer_size = (sample_rate * frame_size / 400) as usize; + let mut opus_encoder = opus::Encoder::new( + sample_rate, + match channels { + 1 => opus::Channels::Mono, + 2 => opus::Channels::Stereo, + _ => unimplemented!("Only 1 or 2 channels supported, got {}", channels), + }, + opus::Application::Voip, + ) + .unwrap(); + let mut buffer = Vec::with_capacity(buffer_size); + move |data: &[T], _info: &InputCallbackInfo| { if !matches!(&*phase_watcher.borrow(), StatePhase::Connected(_)) { return; } let input_volume = *input_volume_receiver.borrow(); - for sample in data.iter().map(|e| e.to_f32()).map(|e| e * input_volume) { - if let Err(_e) = input_sender.try_send(sample) { - warn!("Error sending audio: {}", _e); + let mut data = data.iter().map(|e| e.to_f32()).map(|e| e * input_volume); + + while buffer.len() + data.len() > buffer_size { + buffer.extend(data.by_ref().take(buffer_size - buffer.len())); + let encoded = transformers + .iter() + .try_fold(&mut buffer[..], |acc, e| e(acc)) + .map(|buf| opus_encoder.encode_vec_float(&*buf, buffer_size).unwrap()); + + if let Some(encoded) = encoded { + if let Err(e) = input_sender.try_send(encoded) { + warn!("Error sending audio: {}", e); + } } + buffer.clear(); } + buffer.extend(data); } } @@ -29,13 +58,13 @@ pub trait AudioInputDevice { fn play(&self) -> Result<(), AudioError>; fn pause(&self) -> Result<(), AudioError>; fn set_volume(&self, volume: f32); - fn sample_receiver(&mut self) -> futures_channel::mpsc::Receiver; + fn sample_receiver(&mut self) -> Option>>; fn num_channels(&self) -> usize; } pub struct DefaultAudioInputDevice { stream: cpal::Stream, - sample_receiver: Option>, + sample_receiver: Option>>, volume_sender: watch::Sender, channels: u16, } @@ -44,6 +73,7 @@ impl DefaultAudioInputDevice { pub fn new( input_volume: f32, phase_watcher: watch::Receiver, + frame_size: u32, ) -> Result { let sample_rate = SampleRate(SAMPLE_RATE); @@ -73,20 +103,46 @@ impl DefaultAudioInputDevice { let (volume_sender, input_volume_receiver) = watch::channel::(input_volume); + let transformers = vec![]; + let input_stream = match input_supported_sample_format { SampleFormat::F32 => input_device.build_input_stream( &input_config, - callback::(sample_sender, input_volume_receiver, phase_watcher), + callback::( + sample_sender, + transformers, + frame_size, + SAMPLE_RATE, + input_config.channels, + input_volume_receiver, + phase_watcher + ), err_fn, ), SampleFormat::I16 => input_device.build_input_stream( &input_config, - callback::(sample_sender, input_volume_receiver, phase_watcher), + callback::( + sample_sender, + transformers, + frame_size, + SAMPLE_RATE, + input_config.channels, + input_volume_receiver, + phase_watcher + ), err_fn, ), SampleFormat::U16 => input_device.build_input_stream( &input_config, - callback::(sample_sender, input_volume_receiver, phase_watcher), + callback::( + sample_sender, + transformers, + frame_size, + SAMPLE_RATE, + input_config.channels, + input_volume_receiver, + phase_watcher + ), err_fn, ), } @@ -116,9 +172,8 @@ impl AudioInputDevice for DefaultAudioInputDevice { fn set_volume(&self, volume: f32) { self.volume_sender.send(volume).unwrap(); } - fn sample_receiver(&mut self) -> futures_channel::mpsc::Receiver { - let ret = self.sample_receiver.take(); - ret.unwrap() + fn sample_receiver(&mut self) -> Option>> { + self.sample_receiver.take() } fn num_channels(&self) -> usize { self.channels as usize -- cgit v1.2.1 From 3cb2c612fb030278aaf5e4d49d42cf689bdc9cc0 Mon Sep 17 00:00:00 2001 From: Eskil Queseth Date: Sat, 12 Jun 2021 15:58:44 +0200 Subject: re-add noise gate --- mumd/src/audio/input.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'mumd/src/audio/input.rs') diff --git a/mumd/src/audio/input.rs b/mumd/src/audio/input.rs index 162dd2c..b0e621b 100644 --- a/mumd/src/audio/input.rs +++ b/mumd/src/audio/input.rs @@ -4,12 +4,13 @@ use log::*; use tokio::sync::watch; use crate::audio::SAMPLE_RATE; +use crate::audio::transformers::{create_noise_gate}; use crate::error::{AudioError, AudioStream}; use crate::state::StatePhase; pub fn callback( mut input_sender: futures_channel::mpsc::Sender>, - transformers: Vec Option<&mut [f32]> + Send + 'static>>, + mut transformers: Vec Option<&mut [f32]> + Send + 'static>>, frame_size: u32, sample_rate: u32, channels: u16, @@ -39,7 +40,7 @@ pub fn callback( while buffer.len() + data.len() > buffer_size { buffer.extend(data.by_ref().take(buffer_size - buffer.len())); let encoded = transformers - .iter() + .iter_mut() .try_fold(&mut buffer[..], |acc, e| e(acc)) .map(|buf| opus_encoder.encode_vec_float(&*buf, buffer_size).unwrap()); @@ -103,7 +104,7 @@ impl DefaultAudioInputDevice { let (volume_sender, input_volume_receiver) = watch::channel::(input_volume); - let transformers = vec![]; + let transformers = vec![Box::new(create_noise_gate(10, 0.6)) as Box Option<&mut [f32]> + Send + 'static>]; let input_stream = match input_supported_sample_format { SampleFormat::F32 => input_device.build_input_stream( -- cgit v1.2.1 From 5b2e2c4e266c61f35f7bcfbc603e02070b54e7d0 Mon Sep 17 00:00:00 2001 From: Eskil Queseth Date: Sat, 12 Jun 2021 16:47:36 +0200 Subject: FnMut -> Trait --- mumd/src/audio/input.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'mumd/src/audio/input.rs') diff --git a/mumd/src/audio/input.rs b/mumd/src/audio/input.rs index b0e621b..9c2d93c 100644 --- a/mumd/src/audio/input.rs +++ b/mumd/src/audio/input.rs @@ -4,13 +4,13 @@ use log::*; use tokio::sync::watch; use crate::audio::SAMPLE_RATE; -use crate::audio::transformers::{create_noise_gate}; +use crate::audio::transformers::{NoiseGate, Transformer}; use crate::error::{AudioError, AudioStream}; use crate::state::StatePhase; pub fn callback( mut input_sender: futures_channel::mpsc::Sender>, - mut transformers: Vec Option<&mut [f32]> + Send + 'static>>, + mut transformers: Vec>, frame_size: u32, sample_rate: u32, channels: u16, @@ -41,7 +41,7 @@ pub fn callback( buffer.extend(data.by_ref().take(buffer_size - buffer.len())); let encoded = transformers .iter_mut() - .try_fold(&mut buffer[..], |acc, e| e(acc)) + .try_fold(&mut buffer[..], |acc, e| e.transform(acc)) .map(|buf| opus_encoder.encode_vec_float(&*buf, buffer_size).unwrap()); if let Some(encoded) = encoded { @@ -74,7 +74,7 @@ impl DefaultAudioInputDevice { pub fn new( input_volume: f32, phase_watcher: watch::Receiver, - frame_size: u32, + frame_size: u32, // blocks of 2.5 ms ) -> Result { let sample_rate = SampleRate(SAMPLE_RATE); @@ -104,7 +104,7 @@ impl DefaultAudioInputDevice { let (volume_sender, input_volume_receiver) = watch::channel::(input_volume); - let transformers = vec![Box::new(create_noise_gate(10, 0.6)) as Box Option<&mut [f32]> + Send + 'static>]; + let transformers = vec![Box::new(NoiseGate::new(200)) as Box]; let input_stream = match input_supported_sample_format { SampleFormat::F32 => input_device.build_input_stream( -- cgit v1.2.1 From 09cf98857e39d75ed6034259a6fefe48e5998a71 Mon Sep 17 00:00:00 2001 From: Eskil Queseth Date: Sat, 12 Jun 2021 17:09:15 +0200 Subject: lift inits one level --- mumd/src/audio/input.rs | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) (limited to 'mumd/src/audio/input.rs') diff --git a/mumd/src/audio/input.rs b/mumd/src/audio/input.rs index 9c2d93c..78a7202 100644 --- a/mumd/src/audio/input.rs +++ b/mumd/src/audio/input.rs @@ -11,23 +11,11 @@ use crate::state::StatePhase; pub fn callback( mut input_sender: futures_channel::mpsc::Sender>, mut transformers: Vec>, - frame_size: u32, - sample_rate: u32, - channels: u16, + mut opus_encoder: opus::Encoder, + buffer_size: usize, input_volume_receiver: watch::Receiver, phase_watcher: watch::Receiver, ) -> impl FnMut(&[T], &InputCallbackInfo) + Send + 'static { - let buffer_size = (sample_rate * frame_size / 400) as usize; - let mut opus_encoder = opus::Encoder::new( - sample_rate, - match channels { - 1 => opus::Channels::Mono, - 2 => opus::Channels::Stereo, - _ => unimplemented!("Only 1 or 2 channels supported, got {}", channels), - }, - opus::Application::Voip, - ) - .unwrap(); let mut buffer = Vec::with_capacity(buffer_size); move |data: &[T], _info: &InputCallbackInfo| { @@ -104,6 +92,19 @@ impl DefaultAudioInputDevice { let (volume_sender, input_volume_receiver) = watch::channel::(input_volume); + let mut opus_encoder = opus::Encoder::new( + sample_rate.0, + match input_config.channels { + 1 => opus::Channels::Mono, + 2 => opus::Channels::Stereo, + _ => unimplemented!("Only 1 or 2 channels supported, got {}", input_config.channels), + }, + opus::Application::Voip, + ) + .unwrap(); + // 4 blocks @ 2.5 ms meaning 10 ms total + let buffer_size = (sample_rate.0 * frame_size / 400) as usize; + let transformers = vec![Box::new(NoiseGate::new(200)) as Box]; let input_stream = match input_supported_sample_format { @@ -112,9 +113,8 @@ impl DefaultAudioInputDevice { callback::( sample_sender, transformers, - frame_size, - SAMPLE_RATE, - input_config.channels, + opus_encoder, + buffer_size, input_volume_receiver, phase_watcher ), @@ -125,9 +125,8 @@ impl DefaultAudioInputDevice { callback::( sample_sender, transformers, - frame_size, - SAMPLE_RATE, - input_config.channels, + opus_encoder, + buffer_size, input_volume_receiver, phase_watcher ), @@ -138,9 +137,8 @@ impl DefaultAudioInputDevice { callback::( sample_sender, transformers, - frame_size, - SAMPLE_RATE, - input_config.channels, + opus_encoder, + buffer_size, input_volume_receiver, phase_watcher ), -- cgit v1.2.1 From 200dbfe0d2707a718debf4c26d66da718d16449b Mon Sep 17 00:00:00 2001 From: Eskil Queseth Date: Sat, 12 Jun 2021 17:10:53 +0200 Subject: resolve warnings --- mumd/src/audio/input.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mumd/src/audio/input.rs') diff --git a/mumd/src/audio/input.rs b/mumd/src/audio/input.rs index 78a7202..490cf04 100644 --- a/mumd/src/audio/input.rs +++ b/mumd/src/audio/input.rs @@ -92,7 +92,7 @@ impl DefaultAudioInputDevice { let (volume_sender, input_volume_receiver) = watch::channel::(input_volume); - let mut opus_encoder = opus::Encoder::new( + let opus_encoder = opus::Encoder::new( sample_rate.0, match input_config.channels { 1 => opus::Channels::Mono, -- cgit v1.2.1 From d67ab0c4b37a4e32d874d7dc9aa4c4b43376f80d Mon Sep 17 00:00:00 2001 From: Eskil Queseth Date: Sat, 12 Jun 2021 17:26:22 +0200 Subject: shorten default noise gate open time to .5s --- mumd/src/audio/input.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mumd/src/audio/input.rs') diff --git a/mumd/src/audio/input.rs b/mumd/src/audio/input.rs index 490cf04..b84cefb 100644 --- a/mumd/src/audio/input.rs +++ b/mumd/src/audio/input.rs @@ -105,7 +105,7 @@ impl DefaultAudioInputDevice { // 4 blocks @ 2.5 ms meaning 10 ms total let buffer_size = (sample_rate.0 * frame_size / 400) as usize; - let transformers = vec![Box::new(NoiseGate::new(200)) as Box]; + let transformers = vec![Box::new(NoiseGate::new(50)) as Box]; let input_stream = match input_supported_sample_format { SampleFormat::F32 => input_device.build_input_stream( -- cgit v1.2.1 From c143931f9948a0249213bdaab2658a31d4dddaf2 Mon Sep 17 00:00:00 2001 From: Eskil Queseth Date: Sat, 12 Jun 2021 17:40:41 +0200 Subject: add documentation --- mumd/src/audio/input.rs | 1 - 1 file changed, 1 deletion(-) (limited to 'mumd/src/audio/input.rs') diff --git a/mumd/src/audio/input.rs b/mumd/src/audio/input.rs index b84cefb..7a7ff0f 100644 --- a/mumd/src/audio/input.rs +++ b/mumd/src/audio/input.rs @@ -102,7 +102,6 @@ impl DefaultAudioInputDevice { opus::Application::Voip, ) .unwrap(); - // 4 blocks @ 2.5 ms meaning 10 ms total let buffer_size = (sample_rate.0 * frame_size / 400) as usize; let transformers = vec![Box::new(NoiseGate::new(50)) as Box]; -- cgit v1.2.1 From 9481f3edb37d44957273a8b856ac823d2b5b5f28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kapten=20Z=E2=88=85=E2=88=85m?= <55669224+default-username-852@users.noreply.github.com> Date: Sun, 13 Jun 2021 12:06:34 +0200 Subject: Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Gustav Sörnäs --- mumd/src/audio/input.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'mumd/src/audio/input.rs') diff --git a/mumd/src/audio/input.rs b/mumd/src/audio/input.rs index 7a7ff0f..a1227e3 100644 --- a/mumd/src/audio/input.rs +++ b/mumd/src/audio/input.rs @@ -93,15 +93,15 @@ impl DefaultAudioInputDevice { let (volume_sender, input_volume_receiver) = watch::channel::(input_volume); let opus_encoder = opus::Encoder::new( - sample_rate.0, - match input_config.channels { - 1 => opus::Channels::Mono, - 2 => opus::Channels::Stereo, - _ => unimplemented!("Only 1 or 2 channels supported, got {}", input_config.channels), - }, - opus::Application::Voip, - ) - .unwrap(); + sample_rate.0, + match input_config.channels { + 1 => opus::Channels::Mono, + 2 => opus::Channels::Stereo, + _ => unimplemented!("Only 1 or 2 channels supported, got {}", input_config.channels), + }, + opus::Application::Voip, + ) + .unwrap(); let buffer_size = (sample_rate.0 * frame_size / 400) as usize; let transformers = vec![Box::new(NoiseGate::new(50)) as Box]; -- cgit v1.2.1