From dfbe21fe6e773b2659ebc5b1bf1b02d89e3c599c Mon Sep 17 00:00:00 2001 From: Siavash Sameni Date: Fri, 10 Apr 2026 21:31:09 +0400 Subject: [PATCH] =?UTF-8?q?feat(tauri-engine):=20Phase=203b/3c=20re-port?= =?UTF-8?q?=20=E2=80=94=20DRED=20reconstruction=20on=20the=20live=20Tauri?= =?UTF-8?q?=20mobile=20engine?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The original Phase 3b landed on wzp-client/CallDecoder and Phase 3c landed on wzp-android/src/engine.rs. Both of those are DEAD CODE on feat/desktop-audio-rewrite: the legacy Kotlin app in android/app/ is not built by the Tauri mobile pipeline, and the Tauri engine bypasses CallDecoder by calling wzp_codec::create_decoder directly. The live Android call engine lives at desktop/src-tauri/src/engine.rs with two `pub async fn start` functions — one cfg-gated on Android (Oboe via wzp-native) and one for desktop (CPAL). Both recv tasks were using `let mut decoder = wzp_codec::create_decoder(...)` which returns `Box` and doesn't expose the inherent `reconstruct_from_dred` method. Changes: New helper struct `DredRecvState` at the top of engine.rs, wrapping: - DredDecoderHandle (libopus DRED side-channel parser) - DredState scratch (for parse_into) - DredState last_good (cached valid state, swapped on success) - last_good_seq: Option (DRED anchor sequence) - expected_seq: Option (for gap detection) - dred_reconstructions / classical_plc_invocations counters With three methods: - ingest_opus(seq, payload): parse DRED, swap on success - fill_gap_to(decoder, current_seq, frame_samples, scratch, emit): detect gap back from expected_seq, reconstruct each missing frame via DRED if state covers it, fall through to classical decoder.decode_lost() when it doesn't. Calls emit() once per frame with a slice the caller uses for AGC + playout write. - reset_on_profile_switch(): invalidate tracking when codec changes Both recv tasks (Android @ ~line 297 and desktop @ ~line 907): - Decoder type changed from `Box` via `wzp_codec::create_decoder` to concrete `AdaptiveDecoder::new(profile)` so we can call the inherent reconstruct_from_dred method. - Added `use wzp_proto::traits::AudioDecoder;` at the top of engine.rs to bring decode/decode_lost/set_profile trait methods into scope on the concrete type. - New `current_profile` local alongside `current_codec` (used for frame_duration lookups that drive the DRED sample offset math). - On codec/profile switch, call dred_recv.reset_on_profile_switch() because the cached DRED state is tied to the old profile's frame rate. - For each arriving Opus source packet: 1. dred_recv.ingest_opus(seq, payload) — parse DRED 2. dred_recv.fill_gap_to(...) — detect gap and reconstruct missing frames, each emitted through a closure that does AGC + playout write (wzp_native on Android, playout_ring on desktop) 3. Normal decoder.decode() fallthrough for the current packet (unchanged) - Codec2 packets skip the DRED path entirely (is_opus() gate) — libopus can't reconstruct Codec2 audio. Ordering invariant: gap reconstruction writes to playout BEFORE the current packet's decoded audio, preserving temporal order since the playout ring is FIFO. The closure captures the `spk_muted` flag once before the gap loop to avoid mid-gap-fill state changes. Kept `crates/wzp-android/src/engine.rs` and `crates/wzp-android/src/ stats.rs` from the earlier Phase 3c commit as-is — they're dead code on feat/desktop-audio-rewrite but harmless, and deleting them would diverge this branch from an independently-useful intermediate state. The old Phase 3c commit (505a834) stays as historical reference. Verification: - cargo check -p wzp-codec -p wzp-client -p wzp-relay: 0 errors - cargo check -p wzp-desktop: only pre-existing `tauri::generate_context!()` panic on missing ../dist (Vite output not built on host) — no Rust compile errors from our changes - cargo test -p wzp-codec --lib: 69 passing (unchanged) - cargo test -p wzp-client --lib: 35 passing + 1 ignored (unchanged) Next: scripts/build-tauri-android.sh to get the actual Tauri APK — NOT build-and-notify.sh which builds the dead legacy android/app. Co-Authored-By: Claude Opus 4.6 (1M context) --- desktop/src-tauri/src/engine.rs | 209 +++++++++++++++++++++++++++++++- 1 file changed, 207 insertions(+), 2 deletions(-) diff --git a/desktop/src-tauri/src/engine.rs b/desktop/src-tauri/src/engine.rs index 29998d4..eefba1e 100644 --- a/desktop/src-tauri/src/engine.rs +++ b/desktop/src-tauri/src/engine.rs @@ -25,6 +25,7 @@ use wzp_client::audio_io::{AudioCapture, AudioPlayback}; // Android (where wzp-client is pulled in with default-features=false). use wzp_client::call::{CallConfig, CallEncoder}; +use wzp_proto::traits::AudioDecoder; use wzp_proto::{CodecId, MediaTransport, QualityProfile}; const FRAME_SAMPLES_40MS: usize = 1920; @@ -93,6 +94,134 @@ pub struct CallEngine { _audio_handle: SyncWrapper, } +/// Phase 3b/3c DRED reconstruction state for a recv task. +/// +/// Wraps the libopus 1.5 DRED decoder + two `DredState` buffers (scratch + +/// cached last-good) + sequence tracking needed to fill packet-loss gaps +/// with neural redundancy reconstruction. Lives inside the recv task of +/// `CallEngine::start` and is reset on codec/profile switches. +/// +/// The original Phase 3c port landed on `crates/wzp-android/src/engine.rs`, +/// which turned out to be dead code on the Tauri mobile pipeline — the +/// live Android audio recv path is in *this* file. This helper rehomes +/// the same logic to the correct engine. +struct DredRecvState { + dred_decoder: wzp_codec::dred_ffi::DredDecoderHandle, + scratch: wzp_codec::dred_ffi::DredState, + last_good: wzp_codec::dred_ffi::DredState, + last_good_seq: Option, + expected_seq: Option, + pub dred_reconstructions: u64, + pub classical_plc_invocations: u64, +} + +impl DredRecvState { + fn new() -> Self { + Self { + dred_decoder: wzp_codec::dred_ffi::DredDecoderHandle::new() + .expect("opus_dred_decoder_create failed at call setup"), + scratch: wzp_codec::dred_ffi::DredState::new() + .expect("opus_dred_alloc failed at call setup (scratch)"), + last_good: wzp_codec::dred_ffi::DredState::new() + .expect("opus_dred_alloc failed at call setup (good state)"), + last_good_seq: None, + expected_seq: None, + dred_reconstructions: 0, + classical_plc_invocations: 0, + } + } + + /// Parse DRED side-channel data from an arriving Opus source packet + /// into the scratch state; on success, swap it into the cached good + /// state and record the sequence number as the new anchor. + /// + /// Call this BEFORE `fill_gap_to` so the anchor reflects the freshest + /// DRED source available for gap reconstruction. + fn ingest_opus(&mut self, seq: u16, payload: &[u8]) { + match self.dred_decoder.parse_into(&mut self.scratch, payload) { + Ok(available) if available > 0 => { + std::mem::swap(&mut self.scratch, &mut self.last_good); + self.last_good_seq = Some(seq); + } + _ => { + // Packet carried no DRED data, or parse failed — keep + // the cached good state (it may still cover upcoming + // gaps from a warm-up period). + } + } + } + + /// On an arriving packet with sequence `current_seq`, detect any gap + /// from `expected_seq` to `current_seq - 1` and fill the missing + /// frames via DRED reconstruction (if state covers them) or classical + /// Opus PLC fallback. The `emit` callback is invoked once per + /// reconstructed/concealed frame with a `&mut [i16]` slice of length + /// `frame_samples`; the caller is responsible for AGC + playout. + /// + /// Updates `expected_seq` to `current_seq + 1` on return. + fn fill_gap_to( + &mut self, + decoder: &mut wzp_codec::AdaptiveDecoder, + current_seq: u16, + frame_samples: usize, + pcm_scratch: &mut [i16], + mut emit: F, + ) where + F: FnMut(&mut [i16]), + { + const MAX_GAP_FRAMES: u16 = 16; + if let Some(expected) = self.expected_seq { + let gap = current_seq.wrapping_sub(expected); + if gap > 0 && gap <= MAX_GAP_FRAMES { + let available = self.last_good.samples_available(); + for gap_idx in 0..gap { + let missing_seq = expected.wrapping_add(gap_idx); + let offset_samples = match self.last_good_seq { + Some(anchor) => { + let delta = anchor.wrapping_sub(missing_seq); + if delta == 0 || delta > MAX_GAP_FRAMES { + -1 // skip DRED, fall through to PLC + } else { + delta as i32 * frame_samples as i32 + } + } + None => -1, + }; + let out = &mut pcm_scratch[..frame_samples]; + let reconstructed = if offset_samples > 0 && offset_samples <= available { + decoder + .reconstruct_from_dred(&self.last_good, offset_samples, out) + .ok() + } else { + None + }; + match reconstructed { + Some(_n) => { + self.dred_reconstructions += 1; + emit(out); + } + None => { + if decoder.decode_lost(out).is_ok() { + self.classical_plc_invocations += 1; + emit(out); + } + } + } + } + } + } + self.expected_seq = Some(current_seq.wrapping_add(1)); + } + + /// Invalidate sequence tracking on profile switch. The cached DRED + /// state is tied to the old profile's frame rate so offsets would + /// produce wrong reconstructions until the next good-state parse. + fn reset_on_profile_switch(&mut self) { + self.last_good_seq = None; + self.expected_seq = None; + } +} + impl CallEngine { /// Android engine path — uses the standalone `wzp-native` cdylib /// (loaded at startup via `crate::wzp_native::init()`) for Oboe-backed @@ -294,10 +423,18 @@ impl CallEngine { let recv_rx_codec = rx_codec.clone(); tokio::spawn(async move { let initial_profile = resolve_quality(&quality).unwrap_or(QualityProfile::GOOD); - let mut decoder = wzp_codec::create_decoder(initial_profile); + // Phase 3b/3c: use concrete AdaptiveDecoder (not Box) so we can call the inherent + // reconstruct_from_dred method on packet-loss gaps. + let mut decoder = wzp_codec::AdaptiveDecoder::new(initial_profile) + .expect("failed to create adaptive decoder"); + let mut current_profile = initial_profile; let mut current_codec = initial_profile.codec; let mut agc = wzp_codec::AutoGainControl::new(); let mut pcm = vec![0i16; FRAME_SAMPLES_40MS]; + // Phase 3b/3c DRED reconstruction state — see DredRecvState + // above for the full flow. + let mut dred_recv = DredRecvState::new(); info!(codec = ?current_codec, "recv task starting (android/oboe)"); // ─── Decoded-PCM recorder (debug) ──────────────────────────── @@ -372,8 +509,44 @@ impl CallEngine { }; info!(from = ?current_codec, to = ?pkt.header.codec_id, "recv: switching decoder"); let _ = decoder.set_profile(new_profile); + current_profile = new_profile; current_codec = pkt.header.codec_id; + // Phase 3c: new profile → offsets in the + // cached DRED state are invalid; reset. + dred_recv.reset_on_profile_switch(); } + + // Phase 3b/3c DRED flow for Opus packets: + // 1. parse DRED from this packet → last_good + // 2. detect gap back to expected_seq and + // reconstruct missing frames via DRED + // (or classical PLC if no state covers) + // 3. then decode the current packet normally + // (unchanged fall-through below) + // + // Codec2 packets skip DRED entirely — libopus + // can't reconstruct them and the parse is a + // no-op. + if pkt.header.codec_id.is_opus() { + dred_recv.ingest_opus(pkt.header.seq, &pkt.payload); + let frame_samples_now = (48_000 + * current_profile.frame_duration_ms as usize) + / 1000; + let spk_muted_flag = recv_spk.load(Ordering::Relaxed); + dred_recv.fill_gap_to( + &mut decoder, + pkt.header.seq, + frame_samples_now, + &mut pcm, + |samples| { + agc.process_frame(samples); + if !spk_muted_flag { + let _ = crate::wzp_native::audio_write_playout(samples); + } + }, + ); + } + match decoder.decode(&pkt.payload, &mut pcm) { Ok(n) => { last_decode_n = n; @@ -732,10 +905,16 @@ impl CallEngine { let recv_rx_codec = rx_codec.clone(); tokio::spawn(async move { let initial_profile = resolve_quality(&quality).unwrap_or(QualityProfile::GOOD); - let mut decoder = wzp_codec::create_decoder(initial_profile); + // Phase 3b/3c: concrete AdaptiveDecoder (not Box) so we + // can call reconstruct_from_dred. Same reasoning as the + // Android recv path above. + let mut decoder = wzp_codec::AdaptiveDecoder::new(initial_profile) + .expect("failed to create adaptive decoder"); + let mut current_profile = initial_profile; let mut current_codec = initial_profile.codec; let mut agc = wzp_codec::AutoGainControl::new(); let mut pcm = vec![0i16; FRAME_SAMPLES_40MS]; // big enough for any codec + let mut dred_recv = DredRecvState::new(); loop { if !recv_r.load(Ordering::Relaxed) { @@ -772,8 +951,34 @@ impl CallEngine { }; info!(from = ?current_codec, to = ?pkt.header.codec_id, "recv: switching decoder"); let _ = decoder.set_profile(new_profile); + current_profile = new_profile; current_codec = pkt.header.codec_id; + dred_recv.reset_on_profile_switch(); } + + // Phase 3b/3c: parse DRED + fill gaps before + // decoding the current packet. See the Android + // start() recv task for full commentary. + if pkt.header.codec_id.is_opus() { + dred_recv.ingest_opus(pkt.header.seq, &pkt.payload); + let frame_samples_now = (48_000 + * current_profile.frame_duration_ms as usize) + / 1000; + let spk_muted_flag = recv_spk.load(Ordering::Relaxed); + dred_recv.fill_gap_to( + &mut decoder, + pkt.header.seq, + frame_samples_now, + &mut pcm, + |samples| { + agc.process_frame(samples); + if !spk_muted_flag { + playout_ring.write(samples); + } + }, + ); + } + if let Ok(n) = decoder.decode(&pkt.payload, &mut pcm) { agc.process_frame(&mut pcm[..n]); if !recv_spk.load(Ordering::Relaxed) {