[robotuna] switch to non-pvoc implementation

crates/robotuna/src/lib.rs

@@ -1,19 +1,19 @@
 #![allow(incomplete_features)]
 #![feature(generic_associated_types)]
 
-use std::time::Duration;
-
 use baseplug::{MidiReceiver, Plugin, ProcessContext};
 use ringbuf::{Consumer, Producer, RingBuffer};
 use serde::{Deserialize, Serialize};
 
 use utils::buffers::*;
+use utils::delay::*;
 use utils::logs::*;
+use utils::pitch::*;
 
 mod tuna;
 
-const BUFFER_LEN: usize = 2 << 10;
+const BUFFER_LEN: usize = 2 << 9;
-const OVERLAP: usize = BUFFER_LEN / 3;
+const DELAY_LEN: usize = 4000;
 
 baseplug::model! {
     #[derive(Debug, Serialize, Deserialize)]
@@ -21,7 +21,7 @@ baseplug::model! {
         #[model(min = 0.0, max = 1.0)]
         #[parameter(name = "manual/snap")]
         manual: f32,
-        #[model(min = 0.1, max = 2.0)]
+        #[model(min = 0.1, max = 2.1)]
         #[parameter(name = "frequency gain")]
         freq_gain: f32,
     }
@@ -40,31 +40,33 @@ struct RoboTuna {
     /// Current midi note
     note: Option<u8>,
 
+    /// Current pitches
+    pitch_l: Option<f32>,
+    pitch_r: Option<f32>,
+
     /// Current recording buffer
     /// Input goes here
     recording_buffer: Buffers<BUFFER_LEN>,
-    /// The next recording buffer we'll use. It gets a bit of the end of the `recording_buffer`
-    /// so we can do overlap
-    next_recording_buffer: Buffers<BUFFER_LEN>,
-    /// Current playing buffer
-    /// Output comes from here
-    playing_buffer: Buffers<BUFFER_LEN>,
-    /// Next playing buffer we'll use
-    /// We start using it at the end of the previous buffer so we can overlap
-    next_playing_buffer: Buffers<BUFFER_LEN>,
 
     /// Ringbuf producer so we can send audio chunks to the processing thread
-    recordings: Producer<tuna::ProcessChunk>,
+    recordings: Producer<tuna::ProcessorInput>,
     /// Ringbuf consumer so we can receive processed buffers from the processing threads
-    processed: Consumer<Buffers<BUFFER_LEN>>,
+    processed: Consumer<tuna::ProcessorOutput>,
 
     /// Contains some empty buffers so we can reuse them instead of doing allocations
     /// Buffers here are not actually empty, since we don't spend any time clearing them
     /// But since they will be overwritten, this isn't an issue
     empty_buffers: Vec<Buffers<BUFFER_LEN>>,
-}
 
-// LMAO let's go, i think this works
+    /// Keeps delay lines for playing
+    delays: DelayLines<DELAY_LEN>,
+
+    /// Floating indexes so we can do interpolation
+    delay_idx_l: f32,
+    delay_idx_r: f32,
+    /// true indexes so we can know how much we're drifting away
+    true_idx: usize,
+}
 
 impl Plugin for RoboTuna {
     const NAME: &'static str = "robotuna";
@@ -80,8 +82,8 @@ impl Plugin for RoboTuna {
     fn new(_sample_rate: f32, _model: &RoboTunaModel) -> Self {
         setup_logging("robotuna.log");
 
-        let (recordings, recording_rx) = RingBuffer::<tuna::ProcessChunk>::new(10).split();
+        let (recordings, recording_rx) = RingBuffer::<tuna::ProcessorInput>::new(30).split();
-        let (processed_tx, processed) = RingBuffer::<Buffers<BUFFER_LEN>>::new(10).split();
+        let (processed_tx, processed) = RingBuffer::<tuna::ProcessorOutput>::new(30).split();
 
         // Spawn analysis thread
         std::thread::spawn(move || {
@@ -89,7 +91,7 @@ impl Plugin for RoboTuna {
         });
 
         // keep some empty buffer around so we can swap them
-        let mut empty_buffers = Vec::with_capacity(50);
+        let mut empty_buffers = Vec::with_capacity(80);
         const BUF: Buffers<BUFFER_LEN> = Buffers::new();
         empty_buffers.append(&mut vec![BUF; 30]);
 
@@ -97,13 +99,19 @@ impl Plugin for RoboTuna {
 
         Self {
             note: None,
+            pitch_l: None,
+            pitch_r: None,
             recording_buffer: Buffers::new(),
-            next_recording_buffer: Buffers::new(),
-            playing_buffer: Buffers::new(),
-            next_playing_buffer: Buffers::new(),
             recordings,
             processed,
             empty_buffers,
+            delays: DelayLines::<DELAY_LEN>::new(),
+
+            delay_idx_l: 0.0,
+            delay_idx_r: 0.0,
+            // We start this at a high number cause idk
+            // We'll catch up when we start playing
+            true_idx: 500,
         }
     }
 
@@ -113,92 +121,174 @@ impl Plugin for RoboTuna {
         let output = &mut ctx.outputs[0].buffers;
 
         for i in 0..ctx.nframes {
-            // Record
+            // append input to main buffer
-
-            // Add to main buffer
             let full = self
                 .recording_buffer
                 .write_advance(input[0][i], input[1][i]);
-            // If we're in overlap section, also add to next buffer
+            // If we fill the buffer, switch it with an empty one
-            if self.recording_buffer.idx > BUFFER_LEN - OVERLAP {
-                self.next_recording_buffer
-                    .write_advance(input[0][i], input[1][i]);
-            }
-            // If we finish the buffer, switch them
             if full {
-                // get the empty buffer from unused buffer list
+                // we have to loop here, cause when the daw renders audio it tries to do it faster than
-                let mut buf = self
+                // real time. if we don't loop and wait, the processing thread gets stuck with all of the buffers,
-                    .empty_buffers
+                // and we run out of empty ones to switch to
-                    .pop()
+                // the loop-wait ensures that we don't panic when there isn't an empty buffer
-                    .expect("should have an empty buffer");
+                loop {
-                buf.reset();
+                    // get the empty buffer from unused buffer list
-                std::mem::swap(&mut buf, &mut self.recording_buffer);
+                    if let Some(mut buf) = self.empty_buffers.pop() {
-                buf.reset();
-
-                std::mem::swap(&mut self.next_recording_buffer, &mut self.recording_buffer);
-
-                let _ = self.recordings.push(tuna::ProcessChunk {
-                    buffers: buf,
-                    sample_rate: ctx.sample_rate as u32,
-                    note: self.note,
-                    manual: model.manual[i] <= 0.5,
-                    freq_gain: model.freq_gain[i],
-                });
-            }
-
-            // Play
-
-            // Get values from main buffer
-            let (mut l, mut r, full) = self.playing_buffer.read_advance();
-            // If we're in overlap section, also play from next buffer
-            if self.playing_buffer.idx > BUFFER_LEN - OVERLAP {
-                let (l1, r1, _) = self.next_playing_buffer.read_advance();
-
-                // How much into overlap we are, from 0 to 1
-                let overlap =
-                    (OVERLAP - (BUFFER_LEN - self.playing_buffer.idx)) as f32 / OVERLAP as f32;
-
-                // Linearly crossfade
-                // lineal crossfade works well for two waves that are highly correlated
-                l *= 1. - overlap;
-                r *= 1. - overlap;
-                l += overlap * l1;
-                r += overlap * r1;
-            }
-            // If we finish the buffer, switch them
-            if full {
-                // We try to switch like with the recording buffer, but since there might not be a processed
-                // buffer yet, we do it in a loop and retry every 1 millisecond
-                // After 10 iterations we give up. Since we didn't swap the buffer, we're gonna play the last one
-                // again. This isn't ideal, but it's better than silence ig (it might not be, idk)
-
-                // The 10 iterations is arbitrary, as is the 1 millisecond wait time
-
-                for _ in 0..10 {
-                    if let Some(mut buf) = self.processed.pop() {
                         buf.reset();
-                        std::mem::swap(&mut buf, &mut self.playing_buffer);
+                        // swap it with recording buffer
+                        std::mem::swap(&mut buf, &mut self.recording_buffer);
                         buf.reset();
 
-                        std::mem::swap(&mut self.next_playing_buffer, &mut self.playing_buffer);
+                        // pass it to the processor thread
-
+                        let _ = self.recordings.push(tuna::ProcessorInput {
-                        // Stick buf in unused buffer list
+                            buffers: buf,
-                        self.empty_buffers.push(buf);
+                            sample_rate: ctx.sample_rate as u32,
-
+                        });
-                        // Exit loop
                         break;
-                    } else {
-                        log::info!("didn't have a processed buffer to swap to, retrying");
                     }
-                    std::thread::sleep(Duration::from_millis(1));
+                    std::thread::sleep(std::time::Duration::from_micros(10));
                 }
             }
 
+            // Try to get a processed buffer from the processor thread
+            if let Some(tuna::ProcessorOutput {
+                buffers,
+                pitch_l,
+                pitch_r,
+            }) = self.processed.pop()
+            {
+                self.empty_buffers.push(buffers);
+
+                // Update current pitch
+                // We use `or`, so we keep the old value if the current one is None
+                self.pitch_l = pitch_l.or(self.pitch_l);
+                self.pitch_r = pitch_r.or(self.pitch_r);
+            }
+
+            // Play from delay line according to pitch
+            let (l, r) = self.shift(
+                input[0][i],
+                input[1][i],
+                ctx.sample_rate,
+                model.freq_gain[i],
+                model.manual[i] < 0.5,
+            );
+
             output[0][i] = l;
             output[1][i] = r;
         }
     }
 }
+impl RoboTuna {
+    fn advancement_rate(&self, freq_gain: f32, manual: bool) -> (f32, f32) {
+        // TODO Deal with pitch detection failing
+        let current_pitch_l = self.pitch_l.unwrap_or(220.0);
+        let current_pitch_r = self.pitch_r.unwrap_or(220.0);
+
+        if manual {
+            // If we're on manual, get the expected frequency from the midi note
+            if let Some(expected) = self.note.map(midi_note_to_pitch) {
+                let l = expected / current_pitch_l;
+                let r = expected / current_pitch_r;
+                (freq_gain * l, freq_gain * r)
+            } else {
+                // If there's no note, we just do frequency gain
+                (freq_gain, freq_gain)
+            }
+        } else {
+            // If we're on snap, get the closest note
+            let expected_l = closest_note_freq(current_pitch_l);
+            let expected_r = closest_note_freq(current_pitch_r);
+
+            let l = expected_l / current_pitch_l;
+            let r = expected_r / current_pitch_r;
+            (freq_gain * l, freq_gain * r)
+        }
+    }
+
+    fn shift(
+        &mut self,
+        l: f32,
+        r: f32,
+        sample_rate: f32,
+        freq_gain: f32,
+        manual: bool,
+    ) -> (f32, f32) {
+        // so um this code will probably not make any sense if i don't write an explanation of the
+        // general thing it's trying to achieve
+        // if i've forgoten to write it up and you want to understand the code, ping me and uh yeah
+
+        // add input to delay line
+        self.delays.write_and_advance(l, r);
+
+        // get period of left & right
+        let period_l = sample_rate / self.pitch_l.unwrap_or(220.0);
+        let period_r = sample_rate / self.pitch_r.unwrap_or(220.0);
+
+        // advance indexes
+        let (adv_l, adv_r) = self.advancement_rate(freq_gain, manual);
+        self.delay_idx_l += adv_l;
+        self.delay_idx_r += adv_r;
+        self.true_idx += 1;
+
+        // get how close we are to the input idx, so we know if we have to interpolate/jump
+        let l_diff = self.true_idx as f32 - self.delay_idx_l;
+        let r_diff = self.true_idx as f32 - self.delay_idx_r;
+
+        // get the current value
+        let mut l = self.delays.l.floating_index(self.delay_idx_l);
+        let mut r = self.delays.r.floating_index(self.delay_idx_r);
+
+        // Interpolation
+        // if we are close to having to jump, we start interpolating with the jump destination
+        // interpolate when we're one third of the period away from jumping
+
+        // TODO change to a non-linear interpolation
+
+        const DIV: f32 = 2.0 / 3.0;
+        if l_diff - period_l < (period_l / DIV) {
+            let a = (l_diff - period_l) / (period_l / DIV);
+            l *= a;
+            l += (1.0 - a) * self.delays.l.floating_index(self.delay_idx_l - period_l);
+        }
+        if 3.0 * period_l - l_diff < (period_l / DIV) {
+            let a = (3.0 * period_l - l_diff) / (period_l / DIV);
+            l *= a;
+            l += (1.0 - a) * self.delays.l.floating_index(self.delay_idx_l - period_l);
+        }
+        if r_diff - period_r < (period_r / DIV) {
+            let a = (r_diff - period_r) / (period_r / DIV);
+            r *= a;
+            r += (1.0 - a) * self.delays.r.floating_index(self.delay_idx_r - period_r);
+        }
+        if 3.0 * period_r - r_diff < (period_r / DIV) {
+            let a = (3.0 * period_r - r_diff) / (period_r / DIV);
+            r *= a;
+            r += (1.0 - a) * self.delays.r.floating_index(self.delay_idx_r - period_r);
+        }
+
+        // Check if we need to advance/go back `period` samples
+        // we want to be between the second and third period
+        // so ideally we want {l,r}_diff == 2.0 * period_{l,r}
+
+        // We are about to get to the first period
+        if l_diff < period_l {
+            self.delay_idx_l -= period_l;
+        }
+        // We are about to get to the fourth period
+        if l_diff > 3.0 * period_l {
+            self.delay_idx_l += period_l;
+        }
+        if r_diff < period_r {
+            self.delay_idx_r -= period_r;
+        }
+        if r_diff > 3.0 * period_r {
+            self.delay_idx_r += period_r;
+        }
+
+        (l, r)
+    }
+}
 impl MidiReceiver for RoboTuna {
     fn midi_input(&mut self, _model: &RoboTunaModelProcess, data: [u8; 3]) {
         match data[0] {

crates/robotuna/src/tuna.rs

@@ -5,127 +5,35 @@ use utils::pitch::*;
 
 use crate::BUFFER_LEN;
 
-type SampleRate = u32;
+pub struct ProcessorInput {
-
-pub struct ProcessChunk {
     pub(crate) buffers: Buffers<BUFFER_LEN>,
-    pub(crate) sample_rate: SampleRate,
+    pub(crate) sample_rate: u32,
-    /// Midi note number to shift frequency to
-    pub(crate) note: Option<u8>,
-    /// If true, will listen to note
-    /// If false, will snap to closest note
-    pub(crate) manual: bool,
-    /// Extra frequency shifting to do
-    pub(crate) freq_gain: f32,
 }
 
-pub fn tuna(mut inputs: Consumer<ProcessChunk>, mut outputs: Producer<Buffers<BUFFER_LEN>>) {
+pub struct ProcessorOutput {
-    // Keep track of last detected note, and use it in case of not detecting a new one
+    pub(crate) buffers: Buffers<BUFFER_LEN>,
-    let mut prev_l_freq: Option<f32> = None;
+    pub(crate) pitch_l: Option<f32>,
-    let mut prev_r_freq: Option<f32> = None;
+    pub(crate) pitch_r: Option<f32>,
+}
 
+pub fn tuna(mut inputs: Consumer<ProcessorInput>, mut outputs: Producer<ProcessorOutput>) {
     let mut detector_l = generate_pitch_detector(BUFFER_LEN);
     let mut detector_r = generate_pitch_detector(BUFFER_LEN);
 
-    // Sample rates get overriden on first iteration, so we just set 48k
-    let mut shifter_l = generate_vocoder(48000);
-    let mut shifter_r = generate_vocoder(48000);
-
     loop {
-        if let Some(ProcessChunk {
+        if let Some(ProcessorInput {
-            buffers: recording,
+            buffers,
             sample_rate,
-            note,
-            manual,
-            freq_gain,
         }) = inputs.pop()
         {
-            log::info!("got a buffer to process");
+            let pitch_l = pitch_detect(&mut detector_l, &buffers.l, sample_rate).map(|a| a.0);
+            let pitch_r = pitch_detect(&mut detector_r, &buffers.r, sample_rate).map(|a| a.0);
 
-            // If we're on manual mode, and we don't have a note, just pass through
+            let _ = outputs.push(ProcessorOutput {
-            if manual && note.is_none() {
+                buffers,
-                let _ = outputs.push(recording);
+                pitch_l,
-                continue;
+                pitch_r,
-            }
+            });
-
-            // TODO It does weird stereo things
-
-            // Update sample rate
-            shifter_l.set_sample_rate(sample_rate as f64);
-            shifter_r.set_sample_rate(sample_rate as f64);
-
-            // Left
-            let l = recording.l;
-            // Try detecting note
-            let l = if let Some((actual, _clarity)) = pitch_detect(&mut detector_l, &l, sample_rate)
-            {
-                log::info!("L: detected actual pitch: {}", actual);
-
-                // If note is found, set it as previous, and pitch shift
-                prev_l_freq = Some(actual);
-
-                // If it's on manual mode, convert midi note to pitch
-                // If not, snap to closest frequency
-                let expected = if manual {
-                    midi_note_to_pitch(note.expect("We wouldn't be here if note is None"))
-                } else {
-                    closest_note_freq(actual)
-                };
-
-                // Perform pitch shift
-                // `expected / actual` is how much to shift the pitch
-                // If the actual pitch is 400, and expected is 800, we want to shift by 2
-                pitch_shift(&mut shifter_l, &l, freq_gain * expected / actual)
-            } else if let Some(actual) = prev_l_freq {
-                log::info!("L: reusing actual pitch: {}", actual);
-
-                let expected = if manual {
-                    midi_note_to_pitch(note.expect("We wouldn't be here if note is None"))
-                } else {
-                    closest_note_freq(actual)
-                };
-
-                pitch_shift(&mut shifter_l, &l, freq_gain * expected / actual)
-            } else {
-                log::info!("L: no actual pitch");
-
-                // If there's nothing, leave it as is
-                l
-            };
-
-            // Same thing for the right side
-            let r = recording.r;
-            let r = if let Some((actual, _clarity)) = pitch_detect(&mut detector_r, &r, sample_rate)
-            {
-                log::info!("R: detected actual pitch: {}", actual);
-
-                prev_r_freq = Some(actual);
-
-                let expected = if manual {
-                    midi_note_to_pitch(note.expect("We wouldn't be here if note is None"))
-                } else {
-                    closest_note_freq(actual)
-                };
-
-                pitch_shift(&mut shifter_r, &l, freq_gain * expected / actual)
-            } else if let Some(actual) = prev_r_freq {
-                log::info!("R: reusing actual pitch: {}", actual);
-
-                let expected = if manual {
-                    midi_note_to_pitch(note.expect("We wouldn't be here if note is None"))
-                } else {
-                    closest_note_freq(actual)
-                };
-
-                pitch_shift(&mut shifter_r, &l, freq_gain * expected / actual)
-            } else {
-                log::info!("R: no actual pitch");
-
-                r
-            };
-
-            let _ = outputs.push(Buffers::from(l, r));
-            log::info!("finished processing a buffer");
         }
     }
 }

crates/utils/src/delay.rs

@@ -35,12 +35,22 @@ impl<const LEN: usize> DelayLine<LEN> {
     /// Indexes the buffer but interpolates between the current and the next sample
     pub fn floating_index(&self, val: f32) -> f32 {
         let idx = val.trunc() as usize;
-        let a = val.fract();
+        let frac = val.fract();
 
-        let one = self.wrapped_index(idx);
+        // TODO uhm idk what this should be, but we don't want an underflow so yeah,
-        let two = self.wrapped_index(idx + 1);
+        let xm1 = if idx == 0 {
+            0.0
+        } else {
+            self.wrapped_index(idx - 1)
+        };
+        let x0 = self.wrapped_index(idx);
+        let x1 = self.wrapped_index(idx + 1);
+        let x2 = self.wrapped_index(idx + 2);
 
-        (1.0 - a) * one + a * two
+        // linear interpolation
+        // return (1.0 - frac) * x0 + frac * x1;
+
+        crate::hermite(frac, xm1, x0, x1, x2)
     }
 
     /// Get a reference to the delay line's index.

crates/utils/src/lib.rs

@@ -3,3 +3,13 @@ pub mod delay;
 pub mod logs;
 pub mod pitch;
 pub mod threeband;
+
+pub fn hermite(frac: f32, xm1: f32, x0: f32, x1: f32, x2: f32) -> f32 {
+    let c = (x1 - xm1) * 0.5;
+    let v = x0 - x1;
+    let w = c + v;
+    let a = w + v + (x2 - x0) * 0.5;
+    let b_neg = w + a;
+
+    (((a * frac) - b_neg) * frac + c) * frac + x0
+}