diff --git a/crates/robotuna/src/lib.rs b/crates/robotuna/src/lib.rs index 883d084..4feb099 100644 --- a/crates/robotuna/src/lib.rs +++ b/crates/robotuna/src/lib.rs @@ -1,19 +1,19 @@ #![allow(incomplete_features)] #![feature(generic_associated_types)] -use std::time::Duration; - use baseplug::{MidiReceiver, Plugin, ProcessContext}; use ringbuf::{Consumer, Producer, RingBuffer}; use serde::{Deserialize, Serialize}; use utils::buffers::*; +use utils::delay::*; use utils::logs::*; +use utils::pitch::*; mod tuna; -const BUFFER_LEN: usize = 2 << 10; -const OVERLAP: usize = BUFFER_LEN / 3; +const BUFFER_LEN: usize = 2 << 9; +const DELAY_LEN: usize = 4000; baseplug::model! { #[derive(Debug, Serialize, Deserialize)] @@ -21,7 +21,7 @@ baseplug::model! { #[model(min = 0.0, max = 1.0)] #[parameter(name = "manual/snap")] manual: f32, - #[model(min = 0.1, max = 2.0)] + #[model(min = 0.1, max = 2.1)] #[parameter(name = "frequency gain")] freq_gain: f32, } @@ -40,31 +40,33 @@ struct RoboTuna { /// Current midi note note: Option, + /// Current pitches + pitch_l: Option, + pitch_r: Option, + /// Current recording buffer /// Input goes here recording_buffer: Buffers, - /// The next recording buffer we'll use. It gets a bit of the end of the `recording_buffer` - /// so we can do overlap - next_recording_buffer: Buffers, - /// Current playing buffer - /// Output comes from here - playing_buffer: Buffers, - /// Next playing buffer we'll use - /// We start using it at the end of the previous buffer so we can overlap - next_playing_buffer: Buffers, /// Ringbuf producer so we can send audio chunks to the processing thread - recordings: Producer, + recordings: Producer, /// Ringbuf consumer so we can receive processed buffers from the processing threads - processed: Consumer>, + processed: Consumer, /// Contains some empty buffers so we can reuse them instead of doing allocations /// Buffers here are not actually empty, since we don't spend any time clearing them /// But since they will be overwritten, this isn't an issue empty_buffers: Vec>, -} -// LMAO let's go, i think this works + /// Keeps delay lines for playing + delays: DelayLines, + + /// Floating indexes so we can do interpolation + delay_idx_l: f32, + delay_idx_r: f32, + /// true indexes so we can know how much we're drifting away + true_idx: usize, +} impl Plugin for RoboTuna { const NAME: &'static str = "robotuna"; @@ -80,8 +82,8 @@ impl Plugin for RoboTuna { fn new(_sample_rate: f32, _model: &RoboTunaModel) -> Self { setup_logging("robotuna.log"); - let (recordings, recording_rx) = RingBuffer::::new(10).split(); - let (processed_tx, processed) = RingBuffer::>::new(10).split(); + let (recordings, recording_rx) = RingBuffer::::new(30).split(); + let (processed_tx, processed) = RingBuffer::::new(30).split(); // Spawn analysis thread std::thread::spawn(move || { @@ -89,7 +91,7 @@ impl Plugin for RoboTuna { }); // keep some empty buffer around so we can swap them - let mut empty_buffers = Vec::with_capacity(50); + let mut empty_buffers = Vec::with_capacity(80); const BUF: Buffers = Buffers::new(); empty_buffers.append(&mut vec![BUF; 30]); @@ -97,13 +99,19 @@ impl Plugin for RoboTuna { Self { note: None, + pitch_l: None, + pitch_r: None, recording_buffer: Buffers::new(), - next_recording_buffer: Buffers::new(), - playing_buffer: Buffers::new(), - next_playing_buffer: Buffers::new(), recordings, processed, empty_buffers, + delays: DelayLines::::new(), + + delay_idx_l: 0.0, + delay_idx_r: 0.0, + // We start this at a high number cause idk + // We'll catch up when we start playing + true_idx: 500, } } @@ -113,92 +121,174 @@ impl Plugin for RoboTuna { let output = &mut ctx.outputs[0].buffers; for i in 0..ctx.nframes { - // Record - - // Add to main buffer + // append input to main buffer let full = self .recording_buffer .write_advance(input[0][i], input[1][i]); - // If we're in overlap section, also add to next buffer - if self.recording_buffer.idx > BUFFER_LEN - OVERLAP { - self.next_recording_buffer - .write_advance(input[0][i], input[1][i]); - } - // If we finish the buffer, switch them + // If we fill the buffer, switch it with an empty one if full { - // get the empty buffer from unused buffer list - let mut buf = self - .empty_buffers - .pop() - .expect("should have an empty buffer"); - buf.reset(); - std::mem::swap(&mut buf, &mut self.recording_buffer); - buf.reset(); - - std::mem::swap(&mut self.next_recording_buffer, &mut self.recording_buffer); - - let _ = self.recordings.push(tuna::ProcessChunk { - buffers: buf, - sample_rate: ctx.sample_rate as u32, - note: self.note, - manual: model.manual[i] <= 0.5, - freq_gain: model.freq_gain[i], - }); - } - - // Play - - // Get values from main buffer - let (mut l, mut r, full) = self.playing_buffer.read_advance(); - // If we're in overlap section, also play from next buffer - if self.playing_buffer.idx > BUFFER_LEN - OVERLAP { - let (l1, r1, _) = self.next_playing_buffer.read_advance(); - - // How much into overlap we are, from 0 to 1 - let overlap = - (OVERLAP - (BUFFER_LEN - self.playing_buffer.idx)) as f32 / OVERLAP as f32; - - // Linearly crossfade - // lineal crossfade works well for two waves that are highly correlated - l *= 1. - overlap; - r *= 1. - overlap; - l += overlap * l1; - r += overlap * r1; - } - // If we finish the buffer, switch them - if full { - // We try to switch like with the recording buffer, but since there might not be a processed - // buffer yet, we do it in a loop and retry every 1 millisecond - // After 10 iterations we give up. Since we didn't swap the buffer, we're gonna play the last one - // again. This isn't ideal, but it's better than silence ig (it might not be, idk) - - // The 10 iterations is arbitrary, as is the 1 millisecond wait time - - for _ in 0..10 { - if let Some(mut buf) = self.processed.pop() { + // we have to loop here, cause when the daw renders audio it tries to do it faster than + // real time. if we don't loop and wait, the processing thread gets stuck with all of the buffers, + // and we run out of empty ones to switch to + // the loop-wait ensures that we don't panic when there isn't an empty buffer + loop { + // get the empty buffer from unused buffer list + if let Some(mut buf) = self.empty_buffers.pop() { buf.reset(); - std::mem::swap(&mut buf, &mut self.playing_buffer); + // swap it with recording buffer + std::mem::swap(&mut buf, &mut self.recording_buffer); buf.reset(); - std::mem::swap(&mut self.next_playing_buffer, &mut self.playing_buffer); - - // Stick buf in unused buffer list - self.empty_buffers.push(buf); - - // Exit loop + // pass it to the processor thread + let _ = self.recordings.push(tuna::ProcessorInput { + buffers: buf, + sample_rate: ctx.sample_rate as u32, + }); break; - } else { - log::info!("didn't have a processed buffer to swap to, retrying"); } - std::thread::sleep(Duration::from_millis(1)); + std::thread::sleep(std::time::Duration::from_micros(10)); } } + // Try to get a processed buffer from the processor thread + if let Some(tuna::ProcessorOutput { + buffers, + pitch_l, + pitch_r, + }) = self.processed.pop() + { + self.empty_buffers.push(buffers); + + // Update current pitch + // We use `or`, so we keep the old value if the current one is None + self.pitch_l = pitch_l.or(self.pitch_l); + self.pitch_r = pitch_r.or(self.pitch_r); + } + + // Play from delay line according to pitch + let (l, r) = self.shift( + input[0][i], + input[1][i], + ctx.sample_rate, + model.freq_gain[i], + model.manual[i] < 0.5, + ); + output[0][i] = l; output[1][i] = r; } } } +impl RoboTuna { + fn advancement_rate(&self, freq_gain: f32, manual: bool) -> (f32, f32) { + // TODO Deal with pitch detection failing + let current_pitch_l = self.pitch_l.unwrap_or(220.0); + let current_pitch_r = self.pitch_r.unwrap_or(220.0); + + if manual { + // If we're on manual, get the expected frequency from the midi note + if let Some(expected) = self.note.map(midi_note_to_pitch) { + let l = expected / current_pitch_l; + let r = expected / current_pitch_r; + (freq_gain * l, freq_gain * r) + } else { + // If there's no note, we just do frequency gain + (freq_gain, freq_gain) + } + } else { + // If we're on snap, get the closest note + let expected_l = closest_note_freq(current_pitch_l); + let expected_r = closest_note_freq(current_pitch_r); + + let l = expected_l / current_pitch_l; + let r = expected_r / current_pitch_r; + (freq_gain * l, freq_gain * r) + } + } + + fn shift( + &mut self, + l: f32, + r: f32, + sample_rate: f32, + freq_gain: f32, + manual: bool, + ) -> (f32, f32) { + // so um this code will probably not make any sense if i don't write an explanation of the + // general thing it's trying to achieve + // if i've forgoten to write it up and you want to understand the code, ping me and uh yeah + + // add input to delay line + self.delays.write_and_advance(l, r); + + // get period of left & right + let period_l = sample_rate / self.pitch_l.unwrap_or(220.0); + let period_r = sample_rate / self.pitch_r.unwrap_or(220.0); + + // advance indexes + let (adv_l, adv_r) = self.advancement_rate(freq_gain, manual); + self.delay_idx_l += adv_l; + self.delay_idx_r += adv_r; + self.true_idx += 1; + + // get how close we are to the input idx, so we know if we have to interpolate/jump + let l_diff = self.true_idx as f32 - self.delay_idx_l; + let r_diff = self.true_idx as f32 - self.delay_idx_r; + + // get the current value + let mut l = self.delays.l.floating_index(self.delay_idx_l); + let mut r = self.delays.r.floating_index(self.delay_idx_r); + + // Interpolation + // if we are close to having to jump, we start interpolating with the jump destination + // interpolate when we're one third of the period away from jumping + + // TODO change to a non-linear interpolation + + const DIV: f32 = 2.0 / 3.0; + if l_diff - period_l < (period_l / DIV) { + let a = (l_diff - period_l) / (period_l / DIV); + l *= a; + l += (1.0 - a) * self.delays.l.floating_index(self.delay_idx_l - period_l); + } + if 3.0 * period_l - l_diff < (period_l / DIV) { + let a = (3.0 * period_l - l_diff) / (period_l / DIV); + l *= a; + l += (1.0 - a) * self.delays.l.floating_index(self.delay_idx_l - period_l); + } + if r_diff - period_r < (period_r / DIV) { + let a = (r_diff - period_r) / (period_r / DIV); + r *= a; + r += (1.0 - a) * self.delays.r.floating_index(self.delay_idx_r - period_r); + } + if 3.0 * period_r - r_diff < (period_r / DIV) { + let a = (3.0 * period_r - r_diff) / (period_r / DIV); + r *= a; + r += (1.0 - a) * self.delays.r.floating_index(self.delay_idx_r - period_r); + } + + // Check if we need to advance/go back `period` samples + // we want to be between the second and third period + // so ideally we want {l,r}_diff == 2.0 * period_{l,r} + + // We are about to get to the first period + if l_diff < period_l { + self.delay_idx_l -= period_l; + } + // We are about to get to the fourth period + if l_diff > 3.0 * period_l { + self.delay_idx_l += period_l; + } + if r_diff < period_r { + self.delay_idx_r -= period_r; + } + if r_diff > 3.0 * period_r { + self.delay_idx_r += period_r; + } + + (l, r) + } +} impl MidiReceiver for RoboTuna { fn midi_input(&mut self, _model: &RoboTunaModelProcess, data: [u8; 3]) { match data[0] { diff --git a/crates/robotuna/src/tuna.rs b/crates/robotuna/src/tuna.rs index 93e8bb0..a4fcc42 100644 --- a/crates/robotuna/src/tuna.rs +++ b/crates/robotuna/src/tuna.rs @@ -5,127 +5,35 @@ use utils::pitch::*; use crate::BUFFER_LEN; -type SampleRate = u32; - -pub struct ProcessChunk { +pub struct ProcessorInput { pub(crate) buffers: Buffers, - pub(crate) sample_rate: SampleRate, - /// Midi note number to shift frequency to - pub(crate) note: Option, - /// If true, will listen to note - /// If false, will snap to closest note - pub(crate) manual: bool, - /// Extra frequency shifting to do - pub(crate) freq_gain: f32, + pub(crate) sample_rate: u32, } -pub fn tuna(mut inputs: Consumer, mut outputs: Producer>) { - // Keep track of last detected note, and use it in case of not detecting a new one - let mut prev_l_freq: Option = None; - let mut prev_r_freq: Option = None; +pub struct ProcessorOutput { + pub(crate) buffers: Buffers, + pub(crate) pitch_l: Option, + pub(crate) pitch_r: Option, +} +pub fn tuna(mut inputs: Consumer, mut outputs: Producer) { let mut detector_l = generate_pitch_detector(BUFFER_LEN); let mut detector_r = generate_pitch_detector(BUFFER_LEN); - // Sample rates get overriden on first iteration, so we just set 48k - let mut shifter_l = generate_vocoder(48000); - let mut shifter_r = generate_vocoder(48000); - loop { - if let Some(ProcessChunk { - buffers: recording, + if let Some(ProcessorInput { + buffers, sample_rate, - note, - manual, - freq_gain, }) = inputs.pop() { - log::info!("got a buffer to process"); + let pitch_l = pitch_detect(&mut detector_l, &buffers.l, sample_rate).map(|a| a.0); + let pitch_r = pitch_detect(&mut detector_r, &buffers.r, sample_rate).map(|a| a.0); - // If we're on manual mode, and we don't have a note, just pass through - if manual && note.is_none() { - let _ = outputs.push(recording); - continue; - } - - // TODO It does weird stereo things - - // Update sample rate - shifter_l.set_sample_rate(sample_rate as f64); - shifter_r.set_sample_rate(sample_rate as f64); - - // Left - let l = recording.l; - // Try detecting note - let l = if let Some((actual, _clarity)) = pitch_detect(&mut detector_l, &l, sample_rate) - { - log::info!("L: detected actual pitch: {}", actual); - - // If note is found, set it as previous, and pitch shift - prev_l_freq = Some(actual); - - // If it's on manual mode, convert midi note to pitch - // If not, snap to closest frequency - let expected = if manual { - midi_note_to_pitch(note.expect("We wouldn't be here if note is None")) - } else { - closest_note_freq(actual) - }; - - // Perform pitch shift - // `expected / actual` is how much to shift the pitch - // If the actual pitch is 400, and expected is 800, we want to shift by 2 - pitch_shift(&mut shifter_l, &l, freq_gain * expected / actual) - } else if let Some(actual) = prev_l_freq { - log::info!("L: reusing actual pitch: {}", actual); - - let expected = if manual { - midi_note_to_pitch(note.expect("We wouldn't be here if note is None")) - } else { - closest_note_freq(actual) - }; - - pitch_shift(&mut shifter_l, &l, freq_gain * expected / actual) - } else { - log::info!("L: no actual pitch"); - - // If there's nothing, leave it as is - l - }; - - // Same thing for the right side - let r = recording.r; - let r = if let Some((actual, _clarity)) = pitch_detect(&mut detector_r, &r, sample_rate) - { - log::info!("R: detected actual pitch: {}", actual); - - prev_r_freq = Some(actual); - - let expected = if manual { - midi_note_to_pitch(note.expect("We wouldn't be here if note is None")) - } else { - closest_note_freq(actual) - }; - - pitch_shift(&mut shifter_r, &l, freq_gain * expected / actual) - } else if let Some(actual) = prev_r_freq { - log::info!("R: reusing actual pitch: {}", actual); - - let expected = if manual { - midi_note_to_pitch(note.expect("We wouldn't be here if note is None")) - } else { - closest_note_freq(actual) - }; - - pitch_shift(&mut shifter_r, &l, freq_gain * expected / actual) - } else { - log::info!("R: no actual pitch"); - - r - }; - - let _ = outputs.push(Buffers::from(l, r)); - log::info!("finished processing a buffer"); + let _ = outputs.push(ProcessorOutput { + buffers, + pitch_l, + pitch_r, + }); } } } diff --git a/crates/utils/src/delay.rs b/crates/utils/src/delay.rs index df31bf7..026681e 100644 --- a/crates/utils/src/delay.rs +++ b/crates/utils/src/delay.rs @@ -35,12 +35,22 @@ impl DelayLine { /// Indexes the buffer but interpolates between the current and the next sample pub fn floating_index(&self, val: f32) -> f32 { let idx = val.trunc() as usize; - let a = val.fract(); + let frac = val.fract(); - let one = self.wrapped_index(idx); - let two = self.wrapped_index(idx + 1); + // TODO uhm idk what this should be, but we don't want an underflow so yeah, + let xm1 = if idx == 0 { + 0.0 + } else { + self.wrapped_index(idx - 1) + }; + let x0 = self.wrapped_index(idx); + let x1 = self.wrapped_index(idx + 1); + let x2 = self.wrapped_index(idx + 2); - (1.0 - a) * one + a * two + // linear interpolation + // return (1.0 - frac) * x0 + frac * x1; + + crate::hermite(frac, xm1, x0, x1, x2) } /// Get a reference to the delay line's index. diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 37364e8..c6f51cf 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -3,3 +3,13 @@ pub mod delay; pub mod logs; pub mod pitch; pub mod threeband; + +pub fn hermite(frac: f32, xm1: f32, x0: f32, x1: f32, x2: f32) -> f32 { + let c = (x1 - xm1) * 0.5; + let v = x0 - x1; + let w = c + v; + let a = w + v + (x2 - x0) * 0.5; + let b_neg = w + a; + + (((a * frac) - b_neg) * frac + c) * frac + x0 +}