From c721192f86e24d0ac8778e920e32c954e8b74b2d Mon Sep 17 00:00:00 2001 From: annieversary Date: Thu, 16 Sep 2021 18:31:19 +0200 Subject: [PATCH] [robotuna] fix pitch shifting by using pvoc --- crates/robotuna/Cargo.toml | 2 +- crates/robotuna/src/lib.rs | 171 +++++++++---------------------------- 2 files changed, 40 insertions(+), 133 deletions(-) diff --git a/crates/robotuna/Cargo.toml b/crates/robotuna/Cargo.toml index 78345a6..484bd3e 100644 --- a/crates/robotuna/Cargo.toml +++ b/crates/robotuna/Cargo.toml @@ -11,5 +11,5 @@ baseplug = { git = "https://github.com/wrl/baseplug.git", rev = "9cec68f31cca9c0 ringbuf = "0.2.5" serde = "1.0.126" log = "0.4.14" - +pvoc = { path = "../pvoc-rs" } utils = { path = "../utils" } diff --git a/crates/robotuna/src/lib.rs b/crates/robotuna/src/lib.rs index d2dab4b..5acae38 100644 --- a/crates/robotuna/src/lib.rs +++ b/crates/robotuna/src/lib.rs @@ -2,14 +2,13 @@ #![feature(generic_associated_types)] use baseplug::{MidiReceiver, Plugin, ProcessContext}; +use pvoc::{FreqBin, PhaseVocoder}; use serde::{Deserialize, Serialize}; -use utils::delay::*; use utils::logs::*; use utils::pitch::*; -const BUFFER_LEN: usize = 2 << 9; -const DELAY_LEN: usize = 4000; +const DET_LEN: usize = 2 << 8; baseplug::model! { #[derive(Debug, Serialize, Deserialize)] @@ -40,16 +39,9 @@ struct RoboTuna { pitch_l: Option, pitch_r: Option, - detector_thread: pitch_detection::PitchDetectorThread, + detector_thread: pitch_detection::PitchDetectorThread, - /// Keeps delay lines for playing - delays: DelayLines, - - /// Floating indexes so we can do interpolation - delay_idx_l: f32, - delay_idx_r: f32, - /// true indexes so we can know how much we're drifting away - true_idx: usize, + pvoc: PhaseVocoder, } impl Plugin for RoboTuna { @@ -63,10 +55,10 @@ impl Plugin for RoboTuna { type Model = RoboTunaModel; #[inline] - fn new(_sample_rate: f32, _model: &RoboTunaModel) -> Self { + fn new(sample_rate: f32, _model: &RoboTunaModel) -> Self { setup_logging("robotuna.log"); - let detector_thread = pitch_detection::PitchDetectorThread::::new(); + let detector_thread = pitch_detection::PitchDetectorThread::::new(); log::info!("finished init"); @@ -74,15 +66,10 @@ impl Plugin for RoboTuna { note: None, pitch_l: None, pitch_r: None, + detector_thread, - delays: DelayLines::::new(), - - delay_idx_l: 0.0, - delay_idx_r: 0.0, - // We start this at a high number cause idk - // We'll catch up when we start playing - true_idx: 500, + pvoc: PhaseVocoder::new(2, sample_rate as f64, 128, 4), } } @@ -92,47 +79,57 @@ impl Plugin for RoboTuna { let output = &mut ctx.outputs[0].buffers; for i in 0..ctx.nframes { - // pass input to pitch detector + // pass input to pitch detectors self.detector_thread .write(input[0][i], input[1][i], ctx.sample_rate as u32); - // Try to get a processed buffer from the processor thread + // Try to get a pitch from short detector thread if let Some((pitch_l, pitch_r)) = self.detector_thread.try_get_pitch() { // Update current pitch // We use `or`, so we keep the old value if the current one is None self.pitch_l = pitch_l.or(self.pitch_l); self.pitch_r = pitch_r.or(self.pitch_r); } - - // Play from delay line according to pitch - let (l, r) = self.shift( - input[0][i], - input[1][i], - ctx.sample_rate, - model.freq_gain[i], - model.manual[i] < 0.5, - ); - - output[0][i] = l; - output[1][i] = r; } + + let shift = self.shift(model.freq_gain[0], model.manual[0] < 0.5); + self.pvoc.process( + input, + output, + |channels: usize, bins: usize, input: &[Vec], output: &mut [Vec]| { + for i in 0..channels { + for j in 0..bins / 2 { + let index = ((j as f64) * shift[i]) as usize; + if index < bins / 2 { + output[i][index].freq = input[i][j].freq * shift[i]; + output[i][index].amp += input[i][j].amp; + } + } + } + }, + ); } } impl RoboTuna { - fn advancement_rate(&self, freq_gain: f32, manual: bool) -> (f32, f32) { - // TODO Deal with pitch detection failing - let current_pitch_l = self.pitch_l.unwrap_or(220.0); - let current_pitch_r = self.pitch_r.unwrap_or(220.0); + fn pitch(&self) -> (f32, f32) { + let l = self.pitch_l.unwrap_or(220.0); + let r = self.pitch_r.unwrap_or(220.0); + + (l, r) + } + + fn shift(&self, freq_gain: f32, manual: bool) -> [f64; 2] { + let (current_pitch_l, current_pitch_r) = self.pitch(); if manual { // If we're on manual, get the expected frequency from the midi note if let Some(expected) = self.note.map(midi_note_to_pitch) { let l = expected / current_pitch_l; let r = expected / current_pitch_r; - (freq_gain * l, freq_gain * r) + [(freq_gain * l) as f64, (freq_gain * r) as f64] } else { // If there's no note, we just do frequency gain - (freq_gain, freq_gain) + [freq_gain as f64, freq_gain as f64] } } else { // If we're on snap, get the closest note @@ -141,99 +138,9 @@ impl RoboTuna { let l = expected_l / current_pitch_l; let r = expected_r / current_pitch_r; - (freq_gain * l, freq_gain * r) + [(freq_gain * l) as f64, (freq_gain * r) as f64] } } - - fn shift( - &mut self, - l: f32, - r: f32, - sample_rate: f32, - freq_gain: f32, - manual: bool, - ) -> (f32, f32) { - // so um this code will probably not make any sense if i don't write an explanation of the - // general thing it's trying to achieve - // if i've forgoten to write it up and you want to understand the code, ping me and uh yeah - - // add input to delay line - self.delays.write_and_advance(l, r); - - // get period of left & right - let period_l = sample_rate / self.pitch_l.unwrap_or(220.0); - let period_r = sample_rate / self.pitch_r.unwrap_or(220.0); - - // advance indexes - let (adv_l, adv_r) = self.advancement_rate(freq_gain, manual); - self.delay_idx_l += adv_l; - self.delay_idx_r += adv_r; - self.true_idx += 1; - - // get the current value - let mut l = self.delays.l.floating_index(self.delay_idx_l); - let mut r = self.delays.r.floating_index(self.delay_idx_r); - - // get how close we are to the input idx, so we know if we have to interpolate/jump - let l_diff = self.true_idx as f32 - self.delay_idx_l; - let r_diff = self.true_idx as f32 - self.delay_idx_r; - - // TODO change to a non-linear interpolation - - const DIV: f32 = 2.0 / 3.0; - if l_diff - period_l < (period_l / DIV) { - let a = (l_diff - period_l) / (period_l / DIV); - l *= a; - l += (1.0 - a) * self.delays.l.floating_index(self.delay_idx_l - period_l); - // crossfade - // if we are close to having to jump, we start crossfading with the jump destination - // crossfade when we're one third of the period away from jumping - // when we get close to jumping back - if l_diff - period_l < cf_len_l { - // cross goes from 1 (when l_diff is at the max) to 0 (when l_diff == period_l) - let cross = (l_diff - period_l) / cf_len_l; - let (fade_in, fade_out) = ep_crossfade(1.0 - cross); - l = fade_out * l + fade_in * self.delays.l.floating_index(self.delay_idx_l - period_l); - } - // when we get close to jumping foward - if MAX_PERIOD * period_l - l_diff < cf_len_l { - // cross goes from 1 (when l_diff is at the min) to 0 (when l_diff == 3.0 * period_l) - let cross = (MAX_PERIOD * period_l - l_diff) / cf_len_l; - let (fade_in, fade_out) = ep_crossfade(1.0 - cross); - l = fade_out * l + fade_in * self.delays.l.floating_index(self.delay_idx_l + period_l); - } - if r_diff - period_r < (period_r / DIV) { - let a = (r_diff - period_r) / (period_r / DIV); - r *= a; - r += (1.0 - a) * self.delays.r.floating_index(self.delay_idx_r - period_r); - } - if 3.0 * period_r - r_diff < (period_r / DIV) { - let a = (3.0 * period_r - r_diff) / (period_r / DIV); - r *= a; - r += (1.0 - a) * self.delays.r.floating_index(self.delay_idx_r - period_r); - } - - // Check if we need to advance/go back `period` samples - // we want to be between the second and third period - // so ideally we want {l,r}_diff == 2.0 * period_{l,r} - - // We are about to get to the first period - if l_diff < period_l { - self.delay_idx_l -= period_l; - } - // We are about to get to the fourth period - if l_diff > 3.0 * period_l { - self.delay_idx_l += period_l; - } - if r_diff < period_r { - self.delay_idx_r -= period_r; - } - if r_diff > 3.0 * period_r { - self.delay_idx_r += period_r; - } - - (l, r) - } } impl MidiReceiver for RoboTuna { fn midi_input(&mut self, _model: &RoboTunaModelProcess, data: [u8; 3]) {