annie/unnieversal
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

[robotuna] switch to non-pvoc implementation

Browse source
This commit is contained in:
annieversary 2021-08-02 16:56:42 +02:00
parent 7f6ee9a828
commit 2418efc63b
4 changed files with 224 additions and 206 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

276
crates/robotuna/src/lib.rs
View file

@ -1,19 +1,19 @@
#![allow(incomplete_features)]
#![feature(generic_associated_types)]
use std::time::Duration;
use baseplug::{MidiReceiver, Plugin, ProcessContext};
use ringbuf::{Consumer, Producer, RingBuffer};
use serde::{Deserialize, Serialize};
use utils::buffers::*;
use utils::delay::*;
use utils::logs::*;
use utils::pitch::*;
mod tuna;
const BUFFER_LEN: usize = 2 << 10;
const OVERLAP: usize = BUFFER_LEN / 3;
const BUFFER_LEN: usize = 2 << 9;
const DELAY_LEN: usize = 4000;
baseplug::model! {
#[derive(Debug, Serialize, Deserialize)]
@ -21,7 +21,7 @@ baseplug::model! {
#[model(min = 0.0, max = 1.0)]
#[parameter(name = "manual/snap")]
manual: f32,
#[model(min = 0.1, max = 2.0)]
#[model(min = 0.1, max = 2.1)]
#[parameter(name = "frequency gain")]
freq_gain: f32,
}
@ -40,31 +40,33 @@ struct RoboTuna {
/// Current midi note
note: Option<u8>,
/// Current pitches
pitch_l: Option<f32>,
pitch_r: Option<f32>,
/// Current recording buffer
/// Input goes here
recording_buffer: Buffers<BUFFER_LEN>,
/// The next recording buffer we'll use. It gets a bit of the end of the `recording_buffer`
/// so we can do overlap
next_recording_buffer: Buffers<BUFFER_LEN>,
/// Current playing buffer
/// Output comes from here
playing_buffer: Buffers<BUFFER_LEN>,
/// Next playing buffer we'll use
/// We start using it at the end of the previous buffer so we can overlap
next_playing_buffer: Buffers<BUFFER_LEN>,
/// Ringbuf producer so we can send audio chunks to the processing thread
recordings: Producer<tuna::ProcessChunk>,
recordings: Producer<tuna::ProcessorInput>,
/// Ringbuf consumer so we can receive processed buffers from the processing threads
processed: Consumer<Buffers<BUFFER_LEN>>,
processed: Consumer<tuna::ProcessorOutput>,
/// Contains some empty buffers so we can reuse them instead of doing allocations
/// Buffers here are not actually empty, since we don't spend any time clearing them
/// But since they will be overwritten, this isn't an issue
empty_buffers: Vec<Buffers<BUFFER_LEN>>,
}
// LMAO let's go, i think this works
/// Keeps delay lines for playing
delays: DelayLines<DELAY_LEN>,
/// Floating indexes so we can do interpolation
delay_idx_l: f32,
delay_idx_r: f32,
/// true indexes so we can know how much we're drifting away
true_idx: usize,
}
impl Plugin for RoboTuna {
const NAME: &'static str = "robotuna";
@ -80,8 +82,8 @@ impl Plugin for RoboTuna {
fn new(_sample_rate: f32, _model: &RoboTunaModel) -> Self {
setup_logging("robotuna.log");
let (recordings, recording_rx) = RingBuffer::<tuna::ProcessChunk>::new(10).split();
let (processed_tx, processed) = RingBuffer::<Buffers<BUFFER_LEN>>::new(10).split();
let (recordings, recording_rx) = RingBuffer::<tuna::ProcessorInput>::new(30).split();
let (processed_tx, processed) = RingBuffer::<tuna::ProcessorOutput>::new(30).split();
// Spawn analysis thread
std::thread::spawn(move || {
@ -89,7 +91,7 @@ impl Plugin for RoboTuna {
});
// keep some empty buffer around so we can swap them
let mut empty_buffers = Vec::with_capacity(50);
let mut empty_buffers = Vec::with_capacity(80);
const BUF: Buffers<BUFFER_LEN> = Buffers::new();
empty_buffers.append(&mut vec![BUF; 30]);
@ -97,13 +99,19 @@ impl Plugin for RoboTuna {
Self {
note: None,
pitch_l: None,
pitch_r: None,
recording_buffer: Buffers::new(),
next_recording_buffer: Buffers::new(),
playing_buffer: Buffers::new(),
next_playing_buffer: Buffers::new(),
recordings,
processed,
empty_buffers,
delays: DelayLines::<DELAY_LEN>::new(),
delay_idx_l: 0.0,
delay_idx_r: 0.0,
// We start this at a high number cause idk
// We'll catch up when we start playing
true_idx: 500,
}
}
@ -113,92 +121,174 @@ impl Plugin for RoboTuna {
let output = &mut ctx.outputs[0].buffers;
for i in 0..ctx.nframes {
// Record
// Add to main buffer
// append input to main buffer
let full = self
.recording_buffer
.write_advance(input[0][i], input[1][i]);
// If we're in overlap section, also add to next buffer
if self.recording_buffer.idx > BUFFER_LEN - OVERLAP {
self.next_recording_buffer
.write_advance(input[0][i], input[1][i]);
}
// If we finish the buffer, switch them
// If we fill the buffer, switch it with an empty one
if full {
// get the empty buffer from unused buffer list
let mut buf = self
.empty_buffers
.pop()
.expect("should have an empty buffer");
buf.reset();
std::mem::swap(&mut buf, &mut self.recording_buffer);
buf.reset();
std::mem::swap(&mut self.next_recording_buffer, &mut self.recording_buffer);
let _ = self.recordings.push(tuna::ProcessChunk {
buffers: buf,
sample_rate: ctx.sample_rate as u32,
note: self.note,
manual: model.manual[i] <= 0.5,
freq_gain: model.freq_gain[i],
});
}
// Play
// Get values from main buffer
let (mut l, mut r, full) = self.playing_buffer.read_advance();
// If we're in overlap section, also play from next buffer
if self.playing_buffer.idx > BUFFER_LEN - OVERLAP {
let (l1, r1, _) = self.next_playing_buffer.read_advance();
// How much into overlap we are, from 0 to 1
let overlap =
(OVERLAP - (BUFFER_LEN - self.playing_buffer.idx)) as f32 / OVERLAP as f32;
// Linearly crossfade
// lineal crossfade works well for two waves that are highly correlated
l *= 1. - overlap;
r *= 1. - overlap;
l += overlap * l1;
r += overlap * r1;
}
// If we finish the buffer, switch them
if full {
// We try to switch like with the recording buffer, but since there might not be a processed
// buffer yet, we do it in a loop and retry every 1 millisecond
// After 10 iterations we give up. Since we didn't swap the buffer, we're gonna play the last one
// again. This isn't ideal, but it's better than silence ig (it might not be, idk)
// The 10 iterations is arbitrary, as is the 1 millisecond wait time
for _ in 0..10 {
if let Some(mut buf) = self.processed.pop() {
// we have to loop here, cause when the daw renders audio it tries to do it faster than
// real time. if we don't loop and wait, the processing thread gets stuck with all of the buffers,
// and we run out of empty ones to switch to
// the loop-wait ensures that we don't panic when there isn't an empty buffer
loop {
// get the empty buffer from unused buffer list
if let Some(mut buf) = self.empty_buffers.pop() {
buf.reset();
std::mem::swap(&mut buf, &mut self.playing_buffer);
// swap it with recording buffer
std::mem::swap(&mut buf, &mut self.recording_buffer);
buf.reset();
std::mem::swap(&mut self.next_playing_buffer, &mut self.playing_buffer);
// Stick buf in unused buffer list
self.empty_buffers.push(buf);
// Exit loop
// pass it to the processor thread
let _ = self.recordings.push(tuna::ProcessorInput {
buffers: buf,
sample_rate: ctx.sample_rate as u32,
});
break;
} else {
log::info!("didn't have a processed buffer to swap to, retrying");
}
std::thread::sleep(Duration::from_millis(1));
std::thread::sleep(std::time::Duration::from_micros(10));
}
}
// Try to get a processed buffer from the processor thread
if let Some(tuna::ProcessorOutput {
buffers,
pitch_l,
pitch_r,
}) = self.processed.pop()
{
self.empty_buffers.push(buffers);
// Update current pitch
// We use `or`, so we keep the old value if the current one is None
self.pitch_l = pitch_l.or(self.pitch_l);
self.pitch_r = pitch_r.or(self.pitch_r);
}
// Play from delay line according to pitch
let (l, r) = self.shift(
input[0][i],
input[1][i],
ctx.sample_rate,
model.freq_gain[i],
model.manual[i] < 0.5,
);
output[0][i] = l;
output[1][i] = r;
}
}
}
impl RoboTuna {
fn advancement_rate(&self, freq_gain: f32, manual: bool) -> (f32, f32) {
// TODO Deal with pitch detection failing
let current_pitch_l = self.pitch_l.unwrap_or(220.0);
let current_pitch_r = self.pitch_r.unwrap_or(220.0);
if manual {
// If we're on manual, get the expected frequency from the midi note
if let Some(expected) = self.note.map(midi_note_to_pitch) {
let l = expected / current_pitch_l;
let r = expected / current_pitch_r;
(freq_gain * l, freq_gain * r)
} else {
// If there's no note, we just do frequency gain
(freq_gain, freq_gain)
}
} else {
// If we're on snap, get the closest note
let expected_l = closest_note_freq(current_pitch_l);
let expected_r = closest_note_freq(current_pitch_r);
let l = expected_l / current_pitch_l;
let r = expected_r / current_pitch_r;
(freq_gain * l, freq_gain * r)
}
}
fn shift(
&mut self,
l: f32,
r: f32,
sample_rate: f32,
freq_gain: f32,
manual: bool,
) -> (f32, f32) {
// so um this code will probably not make any sense if i don't write an explanation of the
// general thing it's trying to achieve
// if i've forgoten to write it up and you want to understand the code, ping me and uh yeah
// add input to delay line
self.delays.write_and_advance(l, r);
// get period of left & right
let period_l = sample_rate / self.pitch_l.unwrap_or(220.0);
let period_r = sample_rate / self.pitch_r.unwrap_or(220.0);
// advance indexes
let (adv_l, adv_r) = self.advancement_rate(freq_gain, manual);
self.delay_idx_l += adv_l;
self.delay_idx_r += adv_r;
self.true_idx += 1;
// get how close we are to the input idx, so we know if we have to interpolate/jump
let l_diff = self.true_idx as f32 - self.delay_idx_l;
let r_diff = self.true_idx as f32 - self.delay_idx_r;
// get the current value
let mut l = self.delays.l.floating_index(self.delay_idx_l);
let mut r = self.delays.r.floating_index(self.delay_idx_r);
// Interpolation
// if we are close to having to jump, we start interpolating with the jump destination
// interpolate when we're one third of the period away from jumping
// TODO change to a non-linear interpolation
const DIV: f32 = 2.0 / 3.0;
if l_diff - period_l < (period_l / DIV) {
let a = (l_diff - period_l) / (period_l / DIV);
l *= a;
l += (1.0 - a) * self.delays.l.floating_index(self.delay_idx_l - period_l);
}
if 3.0 * period_l - l_diff < (period_l / DIV) {
let a = (3.0 * period_l - l_diff) / (period_l / DIV);
l *= a;
l += (1.0 - a) * self.delays.l.floating_index(self.delay_idx_l - period_l);
}
if r_diff - period_r < (period_r / DIV) {
let a = (r_diff - period_r) / (period_r / DIV);
r *= a;
r += (1.0 - a) * self.delays.r.floating_index(self.delay_idx_r - period_r);
}
if 3.0 * period_r - r_diff < (period_r / DIV) {
let a = (3.0 * period_r - r_diff) / (period_r / DIV);
r *= a;
r += (1.0 - a) * self.delays.r.floating_index(self.delay_idx_r - period_r);
}
// Check if we need to advance/go back `period` samples
// we want to be between the second and third period
// so ideally we want {l,r}_diff == 2.0 * period_{l,r}
// We are about to get to the first period
if l_diff < period_l {
self.delay_idx_l -= period_l;
}
// We are about to get to the fourth period
if l_diff > 3.0 * period_l {
self.delay_idx_l += period_l;
}
if r_diff < period_r {
self.delay_idx_r -= period_r;
}
if r_diff > 3.0 * period_r {
self.delay_idx_r += period_r;
}
(l, r)
}
}
impl MidiReceiver for RoboTuna {
fn midi_input(&mut self, _model: &RoboTunaModelProcess, data: [u8; 3]) {
match data[0] {

126
crates/robotuna/src/tuna.rs
View file

@ -5,127 +5,35 @@ use utils::pitch::*;
use crate::BUFFER_LEN;
type SampleRate = u32;
pub struct ProcessChunk {
pub struct ProcessorInput {
pub(crate) buffers: Buffers<BUFFER_LEN>,
pub(crate) sample_rate: SampleRate,
/// Midi note number to shift frequency to
pub(crate) note: Option<u8>,
/// If true, will listen to note
/// If false, will snap to closest note
pub(crate) manual: bool,
/// Extra frequency shifting to do
pub(crate) freq_gain: f32,
pub(crate) sample_rate: u32,
}
pub fn tuna(mut inputs: Consumer<ProcessChunk>, mut outputs: Producer<Buffers<BUFFER_LEN>>) {
// Keep track of last detected note, and use it in case of not detecting a new one
let mut prev_l_freq: Option<f32> = None;
let mut prev_r_freq: Option<f32> = None;
pub struct ProcessorOutput {
pub(crate) buffers: Buffers<BUFFER_LEN>,
pub(crate) pitch_l: Option<f32>,
pub(crate) pitch_r: Option<f32>,
}
pub fn tuna(mut inputs: Consumer<ProcessorInput>, mut outputs: Producer<ProcessorOutput>) {
let mut detector_l = generate_pitch_detector(BUFFER_LEN);
let mut detector_r = generate_pitch_detector(BUFFER_LEN);
// Sample rates get overriden on first iteration, so we just set 48k
let mut shifter_l = generate_vocoder(48000);
let mut shifter_r = generate_vocoder(48000);
loop {
if let Some(ProcessChunk {
buffers: recording,
if let Some(ProcessorInput {
buffers,
sample_rate,
note,
manual,
freq_gain,
}) = inputs.pop()
{
log::info!("got a buffer to process");
let pitch_l = pitch_detect(&mut detector_l, &buffers.l, sample_rate).map(|a| a.0);
let pitch_r = pitch_detect(&mut detector_r, &buffers.r, sample_rate).map(|a| a.0);
// If we're on manual mode, and we don't have a note, just pass through
if manual && note.is_none() {
let _ = outputs.push(recording);
continue;
}
// TODO It does weird stereo things
// Update sample rate
shifter_l.set_sample_rate(sample_rate as f64);
shifter_r.set_sample_rate(sample_rate as f64);
// Left
let l = recording.l;
// Try detecting note
let l = if let Some((actual, _clarity)) = pitch_detect(&mut detector_l, &l, sample_rate)
{
log::info!("L: detected actual pitch: {}", actual);
// If note is found, set it as previous, and pitch shift
prev_l_freq = Some(actual);
// If it's on manual mode, convert midi note to pitch
// If not, snap to closest frequency
let expected = if manual {
midi_note_to_pitch(note.expect("We wouldn't be here if note is None"))
} else {
closest_note_freq(actual)
};
// Perform pitch shift
// `expected / actual` is how much to shift the pitch
// If the actual pitch is 400, and expected is 800, we want to shift by 2
pitch_shift(&mut shifter_l, &l, freq_gain * expected / actual)
} else if let Some(actual) = prev_l_freq {
log::info!("L: reusing actual pitch: {}", actual);
let expected = if manual {
midi_note_to_pitch(note.expect("We wouldn't be here if note is None"))
} else {
closest_note_freq(actual)
};
pitch_shift(&mut shifter_l, &l, freq_gain * expected / actual)
} else {
log::info!("L: no actual pitch");
// If there's nothing, leave it as is
l
};
// Same thing for the right side
let r = recording.r;
let r = if let Some((actual, _clarity)) = pitch_detect(&mut detector_r, &r, sample_rate)
{
log::info!("R: detected actual pitch: {}", actual);
prev_r_freq = Some(actual);
let expected = if manual {
midi_note_to_pitch(note.expect("We wouldn't be here if note is None"))
} else {
closest_note_freq(actual)
};
pitch_shift(&mut shifter_r, &l, freq_gain * expected / actual)
} else if let Some(actual) = prev_r_freq {
log::info!("R: reusing actual pitch: {}", actual);
let expected = if manual {
midi_note_to_pitch(note.expect("We wouldn't be here if note is None"))
} else {
closest_note_freq(actual)
};
pitch_shift(&mut shifter_r, &l, freq_gain * expected / actual)
} else {
log::info!("R: no actual pitch");
r
};
let _ = outputs.push(Buffers::from(l, r));
log::info!("finished processing a buffer");
let _ = outputs.push(ProcessorOutput {
buffers,
pitch_l,
pitch_r,
});
}
}
}

18
crates/utils/src/delay.rs
View file

@ -35,12 +35,22 @@ impl<const LEN: usize> DelayLine<LEN> {
/// Indexes the buffer but interpolates between the current and the next sample
pub fn floating_index(&self, val: f32) -> f32 {
let idx = val.trunc() as usize;
let a = val.fract();
let frac = val.fract();
let one = self.wrapped_index(idx);
let two = self.wrapped_index(idx + 1);
// TODO uhm idk what this should be, but we don't want an underflow so yeah,
let xm1 = if idx == 0 {
0.0
} else {
self.wrapped_index(idx - 1)
};
let x0 = self.wrapped_index(idx);
let x1 = self.wrapped_index(idx + 1);
let x2 = self.wrapped_index(idx + 2);
(1.0 - a) * one + a * two
// linear interpolation
// return (1.0 - frac) * x0 + frac * x1;
crate::hermite(frac, xm1, x0, x1, x2)
}
/// Get a reference to the delay line's index.

10
crates/utils/src/lib.rs
View file

@ -3,3 +3,13 @@ pub mod delay;
pub mod logs;
pub mod pitch;
pub mod threeband;
pub fn hermite(frac: f32, xm1: f32, x0: f32, x1: f32, x2: f32) -> f32 {
let c = (x1 - xm1) * 0.5;
let v = x0 - x1;
let w = c + v;
let a = w + v + (x2 - x0) * 0.5;
let b_neg = w + a;
(((a * frac) - b_neg) * frac + c) * frac + x0
}