From aa0362f3184d26e26d2911b78116b462e8ba7364 Mon Sep 17 00:00:00 2001 From: Siavash Sameni Date: Tue, 14 Apr 2026 17:43:15 +0400 Subject: [PATCH 01/96] feat(ui): lobby-first HTML/CSS layout for experimental-ui MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New IRC-style lobby layout: - Auto-connect on launch, drop into user list - User rows with identicon, name, fingerprint, voice status - Speaking indicator (green highlight + pulsing) - Join Voice FAB (green, toggles to Leave/red) - Incoming call banner (slides up from bottom) - User context menu (tap user → Call / Message) - Settings panel preserved from original The old connect-screen HTML is removed. The call-screen is kept intact. JS adaptation next. Co-Authored-By: Claude Opus 4.6 (1M context) --- desktop/index.html | 303 +++++++++++++++----------------------- desktop/src/style.css | 328 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 446 insertions(+), 185 deletions(-) diff --git a/desktop/index.html b/desktop/index.html index 8c0035f..2ad11af 100644 --- a/desktop/index.html +++ b/desktop/index.html @@ -11,97 +11,71 @@
- -
-

WarzonePhone

-

Encrypted Voice

-
- - - -
- - + + +
+
+
+

WarzonePhone

+
- -
- - +
+ + Connecting... + general
- - -
- +
+ +
+
- - - - -
diff --git a/desktop/src-tauri/Cargo.toml b/desktop/src-tauri/Cargo.toml index 9594d3e..9468596 100644 --- a/desktop/src-tauri/Cargo.toml +++ b/desktop/src-tauri/Cargo.toml @@ -44,6 +44,9 @@ tracing = "0.1" tracing-subscriber = "0.3" anyhow = "1" rustls = { version = "0.23", default-features = false, features = ["ring", "std"] } +# JPEG encoding for video:frame events (I420 → RGB → JPEG for IPC to WebView) +image = { version = "0.25", default-features = false, features = ["jpeg"] } +base64 = "0.22" # WarzonePhone crates — protocol layer is platform-independent wzp-proto = { path = "../../crates/wzp-proto" } @@ -51,6 +54,7 @@ wzp-codec = { path = "../../crates/wzp-codec" } wzp-fec = { path = "../../crates/wzp-fec" } wzp-crypto = { path = "../../crates/wzp-crypto" } wzp-transport = { path = "../../crates/wzp-transport" } +wzp-video = { path = "../../crates/wzp-video" } # wzp-client pulls in CPAL on every desktop target and, additionally on # macOS, VoiceProcessingIO (coreaudio-rs behind the "vpio" feature). The @@ -99,6 +103,10 @@ libloading = "0.8" jni = "0.21" ndk-context = "0.1" +[dev-dependencies] +bytes = "1" +async-trait = "0.1" + [features] default = ["custom-protocol"] custom-protocol = ["tauri/custom-protocol"] diff --git a/desktop/src-tauri/Info.plist b/desktop/src-tauri/Info.plist index 15e1cfd..3f6b263 100644 --- a/desktop/src-tauri/Info.plist +++ b/desktop/src-tauri/Info.plist @@ -17,5 +17,7 @@ --> NSMicrophoneUsageDescription WarzonePhone needs microphone access to transmit your voice during calls. + NSCameraUsageDescription + WarzonePhone needs camera access for video calls. diff --git a/desktop/src-tauri/src/android_audio.rs b/desktop/src-tauri/src/android_audio.rs index d49d7f2..8390cdb 100644 --- a/desktop/src-tauri/src/android_audio.rs +++ b/desktop/src-tauri/src/android_audio.rs @@ -99,9 +99,7 @@ pub fn set_audio_mode_communication() -> Result<(), String> { /// Run `set_audio_mode_communication` on Tauri's main thread, where the /// Android context is initialized. Calling it from arbitrary Tokio blocking /// workers panics inside `ndk_context::android_context()`. -pub async fn set_audio_mode_communication_on_main( - app: tauri::AppHandle, -) -> Result<(), String> { +pub async fn set_audio_mode_communication_on_main(app: tauri::AppHandle) -> Result<(), String> { let (tx, rx) = tokio::sync::oneshot::channel(); app.run_on_main_thread(move || { let result = std::panic::catch_unwind(set_audio_mode_communication) diff --git a/desktop/src-tauri/src/engine.rs b/desktop/src-tauri/src/engine.rs index d4cb36d..5e986bf 100644 --- a/desktop/src-tauri/src/engine.rs +++ b/desktop/src-tauri/src/engine.rs @@ -26,7 +26,7 @@ use wzp_client::audio_io::{AudioCapture, AudioPlayback}; use wzp_client::call::{CallConfig, CallEncoder}; use wzp_proto::traits::{AudioDecoder, QualityController}; -use wzp_proto::{AdaptiveQualityController, CodecId, MediaTransport, QualityProfile}; +use wzp_proto::{AdaptiveQualityController, CodecId, QualityProfile}; const FRAME_SAMPLES_40MS: usize = 1920; const CAPTURE_POLL_MS: u64 = 5; @@ -134,7 +134,7 @@ fn codec_to_profile(codec: CodecId) -> QualityProfile { /// codec switch), and Hangup from the relay signal stream. async fn run_signal_task( app: tauri::AppHandle, - transport: Arc, + transport: Arc, running: Arc, pending_profile: Arc, participants: Arc>>, @@ -250,12 +250,15 @@ pub struct CallEngine { audio_level: Arc, tx_codec: Arc>, rx_codec: Arc>, - transport: Arc, + transport: Arc, start_time: Instant, fingerprint: String, /// Keep audio handles alive for the duration of the call. /// Wrapped in SyncWrapper because AudioUnit isn't Sync. _audio_handle: SyncWrapper, + /// Push raw YUV frames here to be encoded and sent to peers. + /// `None` when video was not negotiated or the remote is audio-only. + pub camera_tx: Option>, } /// Phase 3b/3c DRED reconstruction state for a recv task. @@ -479,6 +482,8 @@ impl CallEngine { // debug log pane show first-send/first-recv/heartbeat // events when the user has call debug logs enabled. app: tauri::AppHandle, + active_quality: Arc>, + peer_max_quality: Arc>>, event_cb: F, ) -> Result where @@ -569,7 +574,8 @@ impl CallEngine { // encryption, and both peers' identities were verified // through the signal channel (DirectCallOffer/Answer carry // identity_pub + ephemeral_pub + signature). - if !is_direct_p2p { + let quinn_transport = transport.clone(); + let transport: Arc = if !is_direct_p2p { crate::emit_call_debug( &app, "connect:handshake_start", @@ -579,27 +585,24 @@ impl CallEngine { "remote": transport.remote_address().to_string(), }), ); - let _session = match wzp_client::handshake::perform_handshake( - &*transport, - &seed.0, - Some(&alias), - ) - .await - { - Ok(session) => session, - Err(e) => { - error!("perform_handshake failed: {e}"); - crate::emit_call_debug( - &app, - "connect:handshake_failed", - serde_json::json!({ - "t_ms": call_t0.elapsed().as_millis(), - "error": e.to_string(), - }), - ); - return Err(e.into()); - } - }; + let hs = + match wzp_client::handshake::perform_handshake(&*transport, &seed.0, Some(&alias)) + .await + { + Ok(hs) => hs, + Err(e) => { + error!("perform_handshake failed: {e}"); + crate::emit_call_debug( + &app, + "connect:handshake_failed", + serde_json::json!({ + "t_ms": call_t0.elapsed().as_millis(), + "error": e.to_string(), + }), + ); + return Err(e.into()); + } + }; crate::emit_call_debug( &app, "connect:handshake_done", @@ -609,14 +612,20 @@ impl CallEngine { ); info!( t_ms = call_t0.elapsed().as_millis(), + video_codec = ?hs.video_codec, "first-join diag: connected to relay, handshake complete" ); + Arc::new(wzp_client::encrypted_transport::EncryptingTransport::new( + transport, + hs.session, + )) } else { info!( t_ms = call_t0.elapsed().as_millis(), "first-join diag: direct P2P — skipping relay handshake (QUIC TLS is the encryption layer)" ); - } + transport + }; // Do not emit the legacy "connected" call-event here. The frontend // ignores it and enters voice only after the command resolves; on // Android this synchronous emit was the only operation between @@ -802,6 +811,7 @@ impl CallEngine { // Send task — drain Oboe capture ring, Opus-encode, push to transport. let send_t = transport.clone(); + let quinn_t = quinn_transport.clone(); let send_r = running.clone(); let send_mic = mic_muted.clone(); let send_fs = frames_sent.clone(); @@ -813,6 +823,8 @@ impl CallEngine { let send_t0 = call_t0; let send_app = app.clone(); let send_pending_profile = pending_profile.clone(); + let send_active_quality = active_quality.clone(); + let send_peer_max = peer_max_quality.clone(); tokio::spawn(async move { let config = build_call_config(&send_quality); let mut frame_samples = (config.profile.frame_duration_ms as usize) * 48; @@ -832,7 +844,7 @@ impl CallEngine { let mut frames_since_quality_report: u32 = 0; let mut heartbeat = std::time::Instant::now(); - let mut last_rms: u32 = 0; + let mut last_rms: u32; let mut last_pkt_bytes: usize = 0; let mut short_reads: u64 = 0; // First-join diagnostic: latch the wall-clock offset of the @@ -842,8 +854,28 @@ impl CallEngine { // after returning a "started" status from audio_start. let mut first_full_read_logged = false; let mut first_nonzero_rms_logged = false; + let mut last_applied_profile: Option = None; loop { + // Quality upgrade flow: apply active_quality / peer_max_quality. + let effective_profile = { + let active = send_active_quality.lock().unwrap().clone(); + let peer_cap = send_peer_max.lock().unwrap().clone(); + match peer_cap { + Some(cap) if cap.codec.bitrate_bps() < active.codec.bitrate_bps() => cap, + _ => active, + } + }; + if Some(&effective_profile) != last_applied_profile.as_ref() { + let new_fs = (effective_profile.frame_duration_ms as usize) * 48; + info!(to = ?effective_profile.codec, frame_samples = new_fs, "quality: switching encoder profile (android)"); + if encoder.set_profile(effective_profile).is_ok() { + frame_samples = new_fs; + dred_tuner.set_codec(effective_profile.codec); + *send_tx_codec.lock().await = format!("{:?}", effective_profile.codec); + last_applied_profile = Some(effective_profile); + } + } if !send_r.load(Ordering::Relaxed) { break; } @@ -948,7 +980,7 @@ impl CallEngine { frames_since_dred_poll += 1; if frames_since_dred_poll >= DRED_POLL_INTERVAL { frames_since_dred_poll = 0; - let snap = send_t.quinn_path_stats(); + let snap = quinn_t.quinn_path_stats(); let pq = send_t.path_quality(); if let Some(tuning) = dred_tuner.update(snap.loss_pct, snap.rtt_ms, pq.jitter_ms) @@ -974,7 +1006,7 @@ impl CallEngine { frames_since_quality_report += 1; if frames_since_quality_report >= QUALITY_REPORT_INTERVAL { frames_since_quality_report = 0; - let snap = send_t.quinn_path_stats(); + let snap = quinn_t.quinn_path_stats(); let pq = send_t.path_quality(); let report = wzp_proto::QualityReport::from_path_stats( snap.loss_pct, @@ -1023,6 +1055,7 @@ impl CallEngine { // Recv task — decode incoming packets, push PCM into Oboe playout. let recv_t = transport.clone(); + let quinn_t = quinn_transport.clone(); let recv_r = running.clone(); let recv_spk = spk_muted.clone(); let recv_fr = frames_received.clone(); @@ -1198,7 +1231,7 @@ impl CallEngine { recv_quality_counter += 1; if recv_quality_counter >= QUALITY_REPORT_INTERVAL { recv_quality_counter = 0; - let snap = recv_t.quinn_path_stats(); + let snap = quinn_t.quinn_path_stats(); let pq = recv_t.path_quality(); let local_report = wzp_proto::QualityReport::from_path_stats( snap.loss_pct, @@ -1469,6 +1502,7 @@ impl CallEngine { // is a static dlopen'd library, the audio streams live inside // the standalone cdylib's process-global singleton. _audio_handle: SyncWrapper(Box::new(())), + camera_tx: None, // video not yet wired on Android }) } @@ -1486,6 +1520,8 @@ impl CallEngine { // Phase 6: explicit is_direct_p2p flag (see android branch). is_direct_p2p: bool, _app: tauri::AppHandle, + active_quality: Arc>, + peer_max_quality: Arc>>, event_cb: F, ) -> Result where @@ -1498,6 +1534,7 @@ impl CallEngine { is_direct_p2p, "CallEngine::start (desktop) invoked" ); + let call_t0 = Instant::now(); let _ = rustls::crypto::ring::default_provider().install_default(); let relay_addr: SocketAddr = relay.parse()?; @@ -1546,23 +1583,35 @@ impl CallEngine { // this because the peer is a phone, not a relay with an // accept_handshake handler. See the android branch's // comment for the full rationale. - if !is_direct_p2p { - let _session = - wzp_client::handshake::perform_handshake(&*transport, &seed.0, Some(&alias)) - .await - .map_err(|e| { - error!("perform_handshake failed: {e}"); - e - })?; - } else { - info!("direct P2P — skipping relay handshake (QUIC TLS is the encryption layer)"); - } + let quinn_transport = transport.clone(); + let (_negotiated_video_codec, transport): (_, Arc) = + if !is_direct_p2p { + let hs = + wzp_client::handshake::perform_handshake(&*transport, &seed.0, Some(&alias)) + .await + .map_err(|e| { + error!("perform_handshake failed: {e}"); + e + })?; + info!(video_codec = ?hs.video_codec, "handshake complete"); + let enc = Arc::new( + wzp_client::encrypted_transport::EncryptingTransport::new( + transport, + hs.session, + ), + ); + (hs.video_codec, enc) + } else { + info!("direct P2P — skipping relay handshake (QUIC TLS is the encryption layer)"); + (None, transport) + }; info!("connected to relay, handshake complete"); event_cb("connected", &format!("joined room {room}")); // Audio I/O — VPIO (OS AEC) on macOS, plain CPAL otherwise. // The audio handle must be stored in CallEngine to keep streams alive. + let mut vpio_stats_for_debug = None; let (capture_ring, playout_ring, audio_handle): (_, _, Box) = if _os_aec { #[cfg(target_os = "macos")] @@ -1571,6 +1620,7 @@ impl CallEngine { Ok(v) => { let cr = v.capture_ring().clone(); let pr = v.playout_ring().clone(); + vpio_stats_for_debug = Some(v.stats()); info!("using VoiceProcessingIO (OS AEC)"); (cr, pr, Box::new(v)) } @@ -1615,8 +1665,38 @@ impl CallEngine { let pending_profile = Arc::new(AtomicU8::new(PROFILE_NO_CHANGE)); let auto_profile = resolve_quality(&quality).is_none(); + if let Some(vpio_stats) = vpio_stats_for_debug { + let app = _app.clone(); + let running = running.clone(); + tokio::spawn(async move { + while running.load(Ordering::Relaxed) { + tokio::time::sleep(std::time::Duration::from_secs(HEARTBEAT_INTERVAL_SECS)) + .await; + let s = vpio_stats.snapshot(); + crate::emit_call_debug( + &app, + "vpio:render_heartbeat", + serde_json::json!({ + "capture_callbacks": s.capture_callbacks, + "capture_samples": s.capture_samples, + "render_callbacks": s.render_callbacks, + "render_requested_samples": s.render_requested_samples, + "render_read_samples": s.render_read_samples, + "render_underrun_callbacks": s.render_underrun_callbacks, + "render_nonzero_callbacks": s.render_nonzero_callbacks, + "render_last_requested": s.render_last_requested, + "render_last_read": s.render_last_read, + "render_last_rms": s.render_last_rms, + "render_last_ring_available": s.render_last_ring_available, + }), + ); + } + }); + } + // Send task let send_t = transport.clone(); + let quinn_t = quinn_transport.clone(); let send_r = running.clone(); let send_mic = mic_muted.clone(); let send_fs = frames_sent.clone(); @@ -1625,6 +1705,10 @@ impl CallEngine { let send_quality = quality.clone(); let send_tx_codec = tx_codec.clone(); let send_pending_profile = pending_profile.clone(); + let send_app = _app.clone(); + let send_t0 = call_t0; + let send_active_quality = active_quality.clone(); + let send_peer_max = peer_max_quality.clone(); tokio::spawn(async move { let config = build_call_config(&send_quality); let mut frame_samples = (config.profile.frame_duration_ms as usize) * 48; @@ -1638,12 +1722,37 @@ impl CallEngine { let mut dred_tuner = wzp_proto::DredTuner::new(config.profile.codec); let mut frames_since_dred_poll: u32 = 0; let mut frames_since_quality_report: u32 = 0; + let mut heartbeat = std::time::Instant::now(); + let mut last_rms: u32; + let mut last_pkt_bytes: usize = 0; + let mut short_reads: u64 = 0; + let mut last_applied_profile: Option = None; loop { + // Quality upgrade flow: apply active_quality / peer_max_quality. + let effective_profile = { + let active = send_active_quality.lock().unwrap().clone(); + let peer_cap = send_peer_max.lock().unwrap().clone(); + match peer_cap { + Some(cap) if cap.codec.bitrate_bps() < active.codec.bitrate_bps() => cap, + _ => active, + } + }; + if Some(&effective_profile) != last_applied_profile.as_ref() { + let new_fs = (effective_profile.frame_duration_ms as usize) * 48; + info!(to = ?effective_profile.codec, frame_samples = new_fs, "quality: switching encoder profile (desktop)"); + if encoder.set_profile(effective_profile).is_ok() { + frame_samples = new_fs; + dred_tuner.set_codec(effective_profile.codec); + *send_tx_codec.lock().await = format!("{:?}", effective_profile.codec); + last_applied_profile = Some(effective_profile); + } + } if !send_r.load(Ordering::Relaxed) { break; } if capture_ring.available() < frame_samples { + short_reads += 1; tokio::time::sleep(std::time::Duration::from_millis(CAPTURE_POLL_MS)).await; continue; } @@ -1655,6 +1764,7 @@ impl CallEngine { let sum_sq: f64 = pcm.iter().map(|&s| (s as f64) * (s as f64)).sum(); let rms = (sum_sq / pcm.len() as f64).sqrt() as u32; send_level.store(rms, Ordering::Relaxed); + last_rms = rms; } if send_mic.load(Ordering::Relaxed) { @@ -1663,6 +1773,7 @@ impl CallEngine { match encoder.encode_frame(&buf[..frame_samples]) { Ok(pkts) => { for pkt in &pkts { + last_pkt_bytes = pkt.payload.len(); if let Err(e) = send_t.send_media(pkt).await { // Transient congestion (Blocked) — drop packet, keep going send_drops.fetch_add(1, Ordering::Relaxed); @@ -1671,7 +1782,17 @@ impl CallEngine { } } } - send_fs.fetch_add(1, Ordering::Relaxed); + let before = send_fs.fetch_add(1, Ordering::Relaxed); + if before == 0 { + crate::emit_call_debug( + &send_app, + "media:first_send", + serde_json::json!({ + "t_ms": send_t0.elapsed().as_millis() as u64, + "pkt_bytes": last_pkt_bytes, + }), + ); + } } Err(e) => error!("encode: {e}"), } @@ -1696,7 +1817,7 @@ impl CallEngine { frames_since_dred_poll += 1; if frames_since_dred_poll >= DRED_POLL_INTERVAL { frames_since_dred_poll = 0; - let snap = send_t.quinn_path_stats(); + let snap = quinn_t.quinn_path_stats(); let pq = send_t.path_quality(); if let Some(tuning) = dred_tuner.update(snap.loss_pct, snap.rtt_ms, pq.jitter_ms) @@ -1710,7 +1831,7 @@ impl CallEngine { frames_since_quality_report += 1; if frames_since_quality_report >= QUALITY_REPORT_INTERVAL { frames_since_quality_report = 0; - let snap = send_t.quinn_path_stats(); + let snap = quinn_t.quinn_path_stats(); let pq = send_t.path_quality(); let report = wzp_proto::QualityReport::from_path_stats( snap.loss_pct, @@ -1719,16 +1840,37 @@ impl CallEngine { ); encoder.set_pending_quality_report(report); } + + if heartbeat.elapsed() >= std::time::Duration::from_secs(HEARTBEAT_INTERVAL_SECS) { + let fs = send_fs.load(Ordering::Relaxed); + let drops = send_drops.load(Ordering::Relaxed); + crate::emit_call_debug( + &send_app, + "media:send_heartbeat", + serde_json::json!({ + "frames_sent": fs, + "last_rms": last_rms, + "last_pkt_bytes": last_pkt_bytes, + "short_reads": short_reads, + "drops": drops, + "last_send_err": serde_json::Value::Null, + }), + ); + heartbeat = std::time::Instant::now(); + } } }); // Recv task (direct playout with auto codec switch) let recv_t = transport.clone(); + let quinn_t = quinn_transport.clone(); let recv_r = running.clone(); let recv_spk = spk_muted.clone(); let recv_fr = frames_received.clone(); let recv_rx_codec = rx_codec.clone(); let pending_profile_recv = pending_profile.clone(); + let recv_app = _app.clone(); + let recv_t0 = call_t0; tokio::spawn(async move { let initial_profile = resolve_quality(&quality).unwrap_or(QualityProfile::GOOD); // Phase 3b/3c: concrete AdaptiveDecoder (not Box) so we @@ -1743,6 +1885,18 @@ impl CallEngine { let mut dred_recv = DredRecvState::new(); let mut quality_ctrl = AdaptiveQualityController::new(); let mut recv_quality_counter: u32 = 0; + let mut heartbeat = std::time::Instant::now(); + let mut first_packet_logged = false; + let mut video_reassembler = wzp_video::transport::VideoReassembler::new(); + let mut video_decoder: Option> = None; + let mut video_decoder_codec: Option = None; + let mut decoded_frames: u64 = 0; + let mut decode_errs: u64 = 0; + let mut last_written: usize = 0; + let mut written_samples: u64 = 0; + let mut last_recv_fr_for_watchdog: u64 = 0; + let mut no_recv_ticks: u32 = 0; + let mut media_degraded_emitted = false; loop { if !recv_r.load(Ordering::Relaxed) { @@ -1755,6 +1909,74 @@ impl CallEngine { .await { Ok(Ok(Some(pkt))) => { + // Route video packets to the reassembler before any audio processing. + if pkt.header.media_type == wzp_proto::MediaType::Video { + if let Some((codec_id, is_kf, frame)) = + video_reassembler.push(&pkt) + { + // Lazy-init or switch decoder on codec change. + if video_decoder_codec != Some(codec_id) { + match wzp_video::factory::create_video_decoder(codec_id, 1280, 720) { + Ok(d) => { + info!(codec = ?codec_id, "video decoder created"); + video_decoder = Some(d); + video_decoder_codec = Some(codec_id); + } + Err(e) => { + error!("video decoder init failed: {e}"); + } + } + } + if let Some(ref mut dec) = video_decoder { + match dec.decode(&frame) { + Ok(Some(yuv_frame)) => { + recv_fr.fetch_add(1, Ordering::Relaxed); + // Emit video frame to WebView for rendering. + // Always-on (not gated on debug flag) so the UI can show video. + let jpeg_b64 = crate::i420_to_jpeg_b64( + &yuv_frame.data, + yuv_frame.width, + yuv_frame.height, + ); + let _ = recv_app.emit( + "video:frame", + serde_json::json!({ + "is_keyframe": is_kf, + "width": yuv_frame.width, + "height": yuv_frame.height, + "jpeg_b64": jpeg_b64, + "codec": format!("{:?}", codec_id), + }), + ); + } + Ok(None) => {} // decoder buffering — no output yet + Err(e) => { + error!("video decode error: {e}"); + } + } + } + // Evict stale partial frames every ~10 frames received. + video_reassembler.evict_stale( + pkt.header.timestamp, + 5_000, + ); + } + continue; // video packet handled — skip audio path + } + + if !first_packet_logged { + first_packet_logged = true; + crate::emit_call_debug( + &recv_app, + "media:first_recv", + serde_json::json!({ + "t_ms": recv_t0.elapsed().as_millis() as u64, + "codec": format!("{:?}", pkt.header.codec_id), + "payload_bytes": pkt.payload.len(), + "is_repair": pkt.header.is_repair(), + }), + ); + } if !pkt.header.is_repair() && pkt.header.codec_id != CodecId::ComfortNoise { // Track RX codec { @@ -1812,7 +2034,7 @@ impl CallEngine { recv_quality_counter += 1; if recv_quality_counter >= QUALITY_REPORT_INTERVAL { recv_quality_counter = 0; - let snap = recv_t.quinn_path_stats(); + let snap = quinn_t.quinn_path_stats(); let pq = recv_t.path_quality(); let local_report = wzp_proto::QualityReport::from_path_stats( snap.loss_pct, @@ -1828,10 +2050,21 @@ impl CallEngine { } } - if let Ok(n) = decoder.decode(&pkt.payload, &mut pcm) { - agc.process_frame(&mut pcm[..n]); - if !recv_spk.load(Ordering::Relaxed) { - playout_ring.write(&pcm[..n]); + match decoder.decode(&pkt.payload, &mut pcm) { + Ok(n) => { + decoded_frames += 1; + agc.process_frame(&mut pcm[..n]); + if !recv_spk.load(Ordering::Relaxed) { + playout_ring.write(&pcm[..n]); + last_written = n; + written_samples = written_samples.saturating_add(n as u64); + } + } + Err(e) => { + decode_errs += 1; + if decode_errs <= 3 { + tracing::warn!("decode error: {e}"); + } } } } @@ -1847,6 +2080,63 @@ impl CallEngine { } Err(_) => {} } + + if heartbeat.elapsed() >= std::time::Duration::from_secs(HEARTBEAT_INTERVAL_SECS) { + let fr = recv_fr.load(Ordering::Relaxed); + crate::emit_call_debug( + &recv_app, + "media:recv_heartbeat", + serde_json::json!({ + "recv_fr": fr, + "decoded_frames": decoded_frames, + "last_written": last_written, + "written_samples": written_samples, + "decode_errs": decode_errs, + "codec": format!("{:?}", current_codec), + }), + ); + + if fr == last_recv_fr_for_watchdog { + no_recv_ticks += 1; + } else { + no_recv_ticks = 0; + if media_degraded_emitted { + media_degraded_emitted = false; + let _ = recv_app.emit( + "call-event", + serde_json::json!({ + "kind": "media-recovered", + }), + ); + crate::emit_call_debug( + &recv_app, + "media:recovered", + serde_json::json!({}), + ); + } + } + last_recv_fr_for_watchdog = fr; + + if no_recv_ticks >= 3 && !media_degraded_emitted { + media_degraded_emitted = true; + let _ = recv_app.emit( + "call-event", + serde_json::json!({ + "kind": "media-degraded", + }), + ); + crate::emit_call_debug( + &recv_app, + "media:no_recv_timeout", + serde_json::json!({ + "recv_fr": fr, + "no_recv_ticks": no_recv_ticks, + }), + ); + } + + heartbeat = std::time::Instant::now(); + } } }); @@ -1861,6 +2151,77 @@ impl CallEngine { event_cb.clone(), )); + // Video send task — active only when the handshake negotiated a video codec. + // Camera frames arrive via camera_tx; the task encodes and packetizes them. + // Blocker 4 (camera capture) will push frames into this channel. + let camera_tx = if let Some(vid_codec) = _negotiated_video_codec { + let (tx, mut rx) = tokio::sync::mpsc::channel::(4); + let vid_transport = transport.clone(); + let vid_running = running.clone(); + let vid_t0 = call_t0; + let vid_app = _app.clone(); + tokio::spawn(async move { + let mut encoder = match wzp_video::factory::create_video_encoder( + vid_codec, 1280, 720, 1_500_000, + ) { + Ok(e) => e, + Err(e) => { + error!("video encoder init failed: {e}"); + return; + } + }; + let mut seq: u32 = 0; + let mut frames_since_keyframe: u32 = 0; + info!(codec = ?vid_codec, "video send task started"); + while vid_running.load(Ordering::Relaxed) { + let frame = match tokio::time::timeout( + std::time::Duration::from_millis(200), + rx.recv(), + ) + .await + { + Ok(Some(f)) => f, + Ok(None) => break, // sender dropped + Err(_) => continue, // no frame yet — keep looping + }; + + if frames_since_keyframe >= 150 { + encoder.request_keyframe(); + frames_since_keyframe = 0; + } + + let encoded = match encoder.encode(&frame) { + Ok(b) => b, + Err(e) => { + error!("video encode error: {e}"); + continue; + } + }; + + let is_keyframe = encoder.is_keyframe(&encoded); + let ts_ms = vid_t0.elapsed().as_millis() as u32; + let pkts = wzp_video::transport::packetize_video_frame( + &encoded, vid_codec, is_keyframe, &mut seq, ts_ms, + ); + for pkt in &pkts { + if let Err(e) = vid_transport.send_media(pkt).await { + crate::emit_call_debug( + &vid_app, + "video:send_error", + serde_json::json!({"error": e.to_string()}), + ); + break; + } + } + frames_since_keyframe += 1; + } + info!("video send task exited"); + }); + Some(tx) + } else { + None + }; + Ok(Self { running, mic_muted, @@ -1875,6 +2236,7 @@ impl CallEngine { tx_codec, rx_codec, _audio_handle: SyncWrapper(audio_handle), + camera_tx, }) } @@ -1949,3 +2311,101 @@ impl Drop for CallEngine { self.running.store(false, Ordering::SeqCst); } } + + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex as StdMutex}; + + use async_trait::async_trait; + use bytes::Bytes; + use wzp_client::encrypted_transport::EncryptingTransport; + use wzp_crypto::ChaChaSession; + use wzp_proto::{ + CodecId, CryptoSession, MediaHeader, MediaPacket, MediaTransport, MediaType, PathQuality, + SignalMessage, TransportError, + }; + + struct LoopbackTransport { + sent: StdMutex>, + } + + impl LoopbackTransport { + fn new() -> Arc { + Arc::new(Self { + sent: StdMutex::new(Vec::new()), + }) + } + fn take_sent(&self) -> Vec { + self.sent.lock().unwrap().drain(..).collect() + } + } + + #[async_trait] + impl MediaTransport for LoopbackTransport { + async fn send_media(&self, packet: &MediaPacket) -> Result<(), TransportError> { + self.sent.lock().unwrap().push(packet.clone()); + Ok(()) + } + async fn recv_media(&self) -> Result, TransportError> { + Ok(None) + } + async fn send_signal(&self, _msg: &SignalMessage) -> Result<(), TransportError> { + Ok(()) + } + async fn recv_signal(&self) -> Result, TransportError> { + Ok(None) + } + fn path_quality(&self) -> PathQuality { + PathQuality::default() + } + async fn close(&self) -> Result<(), TransportError> { + Ok(()) + } + } + + fn make_header(seq: u32) -> MediaHeader { + MediaHeader { + version: 2, + flags: 0, + media_type: MediaType::Audio, + codec_id: CodecId::Opus24k, + stream_id: 0, + fec_ratio: 0, + seq, + timestamp: seq * 20, + fec_block: 0, + } + } + + #[tokio::test] + async fn relay_path_encrypts_media_payload() { + // Simulate the exact wrapping pattern used in engine.rs for the relay path. + let key = [0x42u8; 32]; + let session: Box = Box::new(ChaChaSession::new(key)); + let inner = LoopbackTransport::new(); + let transport: Arc = + Arc::new(EncryptingTransport::new(inner.clone(), session)); + + let header = make_header(1); + let plaintext = b"secret audio frame"; + let pkt = MediaPacket { + header, + payload: Bytes::from_static(plaintext), + quality_report: None, + }; + + transport.send_media(&pkt).await.unwrap(); + + let sent = inner.take_sent(); + assert_eq!(sent.len(), 1); + assert_eq!(sent[0].header, header, "header must be preserved"); + assert_ne!( + sent[0].payload.as_ref(), + plaintext.as_ref(), + "plaintext must not appear on wire" + ); + // Ciphertext is longer by exactly the AEAD tag (16 bytes) + assert_eq!(sent[0].payload.len(), plaintext.len() + 16); + } +} diff --git a/desktop/src-tauri/src/lib.rs b/desktop/src-tauri/src/lib.rs index b2d2dbc..69f8979 100644 --- a/desktop/src-tauri/src/lib.rs +++ b/desktop/src-tauri/src/lib.rs @@ -84,6 +84,213 @@ pub(crate) fn emit_call_debug(app: &tauri::AppHandle, step: &str, details: serde /// Short git hash captured at compile time by build.rs. const GIT_HASH: &str = env!("WZP_GIT_HASH"); +// ─── Video helpers ──────────────────────────────────────────────────────────── + +/// Convert an I420 frame to a JPEG and base64-encode it for IPC. +/// +/// Returns `None` if the data is too short or encoding fails. +/// Called from the video recv task in engine.rs to produce the `jpeg_b64` +/// field of every `video:frame` Tauri event. +pub(crate) fn i420_to_jpeg_b64(data: &[u8], width: u32, height: u32) -> Option { + use base64::Engine as _; + use image::{DynamicImage, ImageBuffer, Rgb}; + + let w = width as usize; + let h = height as usize; + let y_size = w * h; + let uv_size = w * h / 4; + + if data.len() < y_size + 2 * uv_size { + return None; + } + + let mut rgb = vec![0u8; w * h * 3]; + for row in 0..h { + for col in 0..w { + let y = data[row * w + col] as f32; + let uv_idx = (row / 2) * (w / 2) + col / 2; + let u = data[y_size + uv_idx] as f32 - 128.0; + let v = data[y_size + uv_size + uv_idx] as f32 - 128.0; + let out = (row * w + col) * 3; + rgb[out] = (y + 1.402 * v).clamp(0.0, 255.0) as u8; + rgb[out + 1] = (y - 0.344 * u - 0.714 * v).clamp(0.0, 255.0) as u8; + rgb[out + 2] = (y + 1.772 * u).clamp(0.0, 255.0) as u8; + } + } + + let img = DynamicImage::ImageRgb8(ImageBuffer::, Vec>::from_raw(width, height, rgb)?); + let mut buf = std::io::Cursor::new(Vec::::new()); + img.write_to(&mut buf, image::ImageFormat::Jpeg).ok()?; + Some(base64::engine::general_purpose::STANDARD.encode(buf.into_inner())) +} + +/// RGB24 → I420 (planar 4:2:0). Layout: Y(w×h) | U(w/2×h/2) | V(w/2×h/2). +fn rgb_to_i420(rgb: &[u8], w: usize, h: usize) -> Vec { + let y_size = w * h; + let uv_size = (w / 2) * (h / 2); + let mut out = vec![0u8; y_size + 2 * uv_size]; + for row in 0..h { + for col in 0..w { + let i = (row * w + col) * 3; + let r = rgb[i] as f32; + let g = rgb[i + 1] as f32; + let b = rgb[i + 2] as f32; + out[row * w + col] = (0.299 * r + 0.587 * g + 0.114 * b).clamp(0.0, 255.0) as u8; + if row % 2 == 0 && col % 2 == 0 { + let uv = (row / 2) * (w / 2) + col / 2; + out[y_size + uv] = (-0.169 * r - 0.331 * g + 0.500 * b + 128.0).clamp(0.0, 255.0) as u8; + out[y_size + uv_size + uv] = (0.500 * r - 0.419 * g - 0.081 * b + 128.0).clamp(0.0, 255.0) as u8; + } + } + } + out +} + +/// Tauri command: receive a JPEG frame from the frontend camera (getUserMedia), +/// decode it, convert to I420, and push into the active call's video send task. +/// +/// The frontend calls this at ~15 fps from a canvas.toDataURL() capture loop. +#[tauri::command] +async fn push_camera_frame( + state: tauri::State<'_, Arc>, + jpeg_b64: String, +) -> Result<(), String> { + use base64::Engine as _; + let jpeg_bytes = base64::engine::general_purpose::STANDARD + .decode(&jpeg_b64) + .map_err(|e| e.to_string())?; + + let dyn_img = image::load_from_memory_with_format(&jpeg_bytes, image::ImageFormat::Jpeg) + .map_err(|e| e.to_string())?; + let rgb_img = dyn_img.to_rgb8(); + let w = rgb_img.width() as usize; + let h = rgb_img.height() as usize; + let yuv = rgb_to_i420(rgb_img.as_raw(), w, h); + + let ts = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64; + + let frame = wzp_video::encoder::VideoFrame { + width: w as u32, + height: h as u32, + data: yuv, + timestamp_ms: ts, + }; + + let engine = state.engine.lock().await; + if let Some(ref eng) = *engine { + if let Some(ref tx) = eng.camera_tx { + let _ = tx.try_send(frame); // drop frame if send task is saturated + } + } + Ok(()) +} + +// ─── Video helper tests ─────────────────────────────────────────────────────── +#[cfg(test)] +mod video_tests { + use super::{i420_to_jpeg_b64, rgb_to_i420}; + use base64::Engine as _; + + fn solid_rgb_frame(w: usize, h: usize, r: u8, g: u8, b: u8) -> Vec { + let mut rgb = vec![0u8; w * h * 3]; + for i in 0..w * h { + rgb[i * 3] = r; + rgb[i * 3 + 1] = g; + rgb[i * 3 + 2] = b; + } + rgb + } + + fn solid_i420(w: usize, h: usize, y: u8, u: u8, v: u8) -> Vec { + let y_size = w * h; + let uv_size = w * h / 4; + let mut data = vec![y; y_size + 2 * uv_size]; + data[y_size..y_size + uv_size].fill(u); + data[y_size + uv_size..].fill(v); + data + } + + #[test] + fn rgb_to_i420_output_size() { + let rgb = solid_rgb_frame(640, 360, 128, 128, 128); + let yuv = rgb_to_i420(&rgb, 640, 360); + assert_eq!(yuv.len(), 640 * 360 * 3 / 2); + } + + #[test] + fn rgb_to_i420_pure_green_luma() { + // Pure green (0, 255, 0) → Y ≈ 150 (0.587 × 255 ≈ 150). + let rgb = solid_rgb_frame(4, 4, 0, 255, 0); + let yuv = rgb_to_i420(&rgb, 4, 4); + let y = yuv[0]; + assert!(y >= 140 && y <= 160, "pure-green luma out of range: {y}"); + } + + #[test] + fn rgb_to_i420_grey_is_neutral() { + // Mid-grey RGB → U and V should both be near 128. + let rgb = solid_rgb_frame(4, 4, 128, 128, 128); + let yuv = rgb_to_i420(&rgb, 4, 4); + let uv_start = 4 * 4; + let u = yuv[uv_start]; + let v = yuv[uv_start + 4]; // 4 = (4/2)*(4/2) + assert!((u as i32 - 128).abs() <= 5, "grey U out of range: {u}"); + assert!((v as i32 - 128).abs() <= 5, "grey V out of range: {v}"); + } + + #[test] + fn i420_to_jpeg_b64_produces_non_empty_output() { + let data = solid_i420(64, 64, 128, 128, 128); + let b64 = i420_to_jpeg_b64(&data, 64, 64); + assert!(b64.is_some(), "valid I420 must produce Some(b64)"); + let s = b64.unwrap(); + assert!(!s.is_empty()); + // JPEG base64 starts with '/9j/' (FFD8FF marker). + let decoded = base64::engine::general_purpose::STANDARD.decode(&s).unwrap(); + assert_eq!(&decoded[0..2], &[0xFF, 0xD8], "output must start with JPEG SOI marker"); + } + + #[test] + fn i420_to_jpeg_b64_rejects_undersized_buffer() { + // Buffer too short: only Y plane, no chroma. + let data = vec![128u8; 64 * 64]; + let b64 = i420_to_jpeg_b64(&data, 64, 64); + assert!(b64.is_none(), "truncated buffer must yield None"); + } + + #[test] + fn i420_to_jpeg_b64_color_preservation() { + // A red (255, 0, 0) I420 frame should decode to a mostly-red JPEG. + // After JPEG lossy compression the exact values drift, so we only + // check that the decoded pixel has R > G and R > B. + use base64::Engine as _; + + // Convert red RGB → I420. + let rgb = solid_rgb_frame(64, 64, 255, 0, 0); + let yuv = rgb_to_i420(&rgb, 64, 64); + + let b64 = i420_to_jpeg_b64(&yuv, 64, 64).expect("should produce JPEG"); + let jpeg = base64::engine::general_purpose::STANDARD.decode(&b64).unwrap(); + + let img = image::load_from_memory_with_format(&jpeg, image::ImageFormat::Jpeg).unwrap(); + let rgb_img = img.to_rgb8(); + let px = rgb_img.get_pixel(32, 32); + let (r, g, b) = (px[0], px[1], px[2]); + assert!(r > g && r > b, "red frame: expected R dominant, got R={r} G={g} B={b}"); + } + + #[test] + fn rgb_i420_conversion_is_deterministic() { + let rgb = solid_rgb_frame(8, 8, 200, 100, 50); + let yuv1 = rgb_to_i420(&rgb, 8, 8); + let yuv2 = rgb_to_i420(&rgb, 8, 8); + assert_eq!(yuv1, yuv2, "rgb_to_i420 must be deterministic"); + } +} + /// Resolved by `setup()` once we have a Tauri AppHandle. Holds the /// platform-correct app data dir (e.g. `/data/data/com.wzp.desktop/files` on /// Android, `~/Library/Application Support/com.wzp.desktop` on macOS). @@ -805,6 +1012,10 @@ async fn connect( }), ); let app_for_engine = app.clone(); + let (active_quality, peer_max_quality) = { + let sig = state.signal.lock().await; + (sig.active_quality.clone(), sig.peer_max_quality.clone()) + }; match CallEngine::start( relay, room, @@ -815,6 +1026,8 @@ async fn connect( pre_connected_transport, is_direct_p2p_agreed, app_for_engine, + active_quality, + peer_max_quality, move |event_kind, message| { let _ = app_clone.emit( "call-event", @@ -1157,6 +1370,12 @@ struct SignalState { peer_hard_nat_probe: Option, /// Phase 8.6: peer's birthday attack ports, if received. peer_birthday_ports: Option, + /// Active quality profile for the encoder. Updated by signal upgrade flow. + active_quality: Arc>, + /// Peer's reported max quality cap. The encoder clamps to min(active, peer_max). + peer_max_quality: Arc>>, + /// Pending outgoing upgrade proposal: (call_id, proposal_id, profile). + pending_upgrade: Arc>>, } /// Parsed data from a peer's HardNatBirthdayStart signal. @@ -1720,8 +1939,9 @@ fn do_register_signal( "peer_loss_pct": local_loss_pct, "peer_rtt_ms": local_rtt_ms, }), ); - // TODO: auto-accept if our own quality supports it, - // or surface to UI for manual accept/reject + if let Err(e) = handle_upgrade_proposal(&*transport, &call_id, &proposal_id).await { + tracing::warn!("failed to send UpgradeResponse: {e}"); + } } Ok(Some(SignalMessage::UpgradeResponse { call_id, @@ -1739,7 +1959,11 @@ fn do_register_signal( "accepted": accepted, "reason": reason, }), ); - // TODO: if accepted, send UpgradeConfirm + switch encoder + if let Err(e) = handle_upgrade_response( + &*transport, &signal_state, &call_id, &proposal_id, accepted, + ).await { + tracing::warn!("failed to handle UpgradeResponse: {e}"); + } } Ok(Some(SignalMessage::UpgradeConfirm { call_id, @@ -1756,7 +1980,7 @@ fn do_register_signal( "confirmed_profile": format!("{confirmed_profile:?}"), }), ); - // TODO: switch encoder to confirmed_profile at next frame boundary + handle_upgrade_confirm(&signal_state, confirmed_profile).await; } Ok(Some(SignalMessage::QualityCapability { call_id, @@ -1775,8 +1999,7 @@ fn do_register_signal( "peer_loss_pct": loss_pct, "peer_rtt_ms": rtt_ms, }), ); - // TODO: adjust our encoder to not exceed peer's max_profile - // (asymmetric quality — each side encodes at its own best) + handle_quality_capability(&signal_state, max_profile).await; } Ok(Some(SignalMessage::HardNatBirthdayStart { call_id, @@ -2505,7 +2728,7 @@ async fn answer_call( /// or temporarily unreachable for reflect but the call can still /// proceed with STUN-discovered addresses. async fn try_reflect_own_addr(state: &Arc) -> Result, String> { - use wzp_proto::{SignalMessage, default_signal_version}; + use wzp_proto::SignalMessage; let (tx, rx) = tokio::sync::oneshot::channel::(); let transport = { let mut sig = state.signal.lock().await; @@ -2592,7 +2815,7 @@ async fn try_stun_fallback(state: &Arc) -> Result, Stri /// with `new URL(...)` / a regex if needed. #[tauri::command] async fn get_reflected_address(state: tauri::State<'_, Arc>) -> Result { - use wzp_proto::{SignalMessage, default_signal_version}; + use wzp_proto::SignalMessage; let (tx, rx) = tokio::sync::oneshot::channel::(); let transport = { let mut sig = state.signal.lock().await; @@ -2850,11 +3073,232 @@ async fn hangup_call( // ─── App entry point ───────────────────────────────────────────────────────── +// ─── Quality upgrade flow handlers (testable) ───────────────────────────── + +async fn handle_upgrade_proposal( + transport: &dyn wzp_proto::MediaTransport, + call_id: &str, + proposal_id: &str, +) -> Result<(), wzp_proto::TransportError> { + let response = wzp_proto::SignalMessage::UpgradeResponse { + version: default_signal_version(), + call_id: call_id.to_string(), + proposal_id: proposal_id.to_string(), + accepted: true, + reason: None, + }; + transport.send_signal(&response).await +} + +async fn handle_upgrade_response( + transport: &dyn wzp_proto::MediaTransport, + signal_state: &Arc>, + call_id: &str, + proposal_id: &str, + accepted: bool, +) -> Result<(), wzp_proto::TransportError> { + if accepted { + let maybe_proposal = { + let sig = signal_state.lock().await; + sig.pending_upgrade.lock().unwrap().take() + }; + if let Some((_cid, pid, profile)) = maybe_proposal { + if pid == proposal_id { + let confirm = wzp_proto::SignalMessage::UpgradeConfirm { + version: default_signal_version(), + call_id: call_id.to_string(), + proposal_id: proposal_id.to_string(), + confirmed_profile: profile.clone(), + }; + transport.send_signal(&confirm).await?; + { + let sig = signal_state.lock().await; + *sig.active_quality.lock().unwrap() = profile; + } + } + } + } + Ok(()) +} + +async fn handle_upgrade_confirm( + signal_state: &Arc>, + confirmed_profile: wzp_proto::QualityProfile, +) { + let sig = signal_state.lock().await; + *sig.active_quality.lock().unwrap() = confirmed_profile; +} + +async fn handle_quality_capability( + signal_state: &Arc>, + max_profile: wzp_proto::QualityProfile, +) { + let sig = signal_state.lock().await; + *sig.peer_max_quality.lock().unwrap() = Some(max_profile); +} + +#[cfg(test)] +mod signal_tests { + use super::*; + use async_trait::async_trait; + use std::sync::Mutex as StdMutex; + use wzp_proto::{MediaPacket, MediaTransport, PathQuality, SignalMessage, TransportError}; + + struct LoopbackTransport { + sent: StdMutex>, + } + + impl LoopbackTransport { + fn new() -> Arc { + Arc::new(Self { + sent: StdMutex::new(Vec::new()), + }) + } + fn take_sent(&self) -> Vec { + self.sent.lock().unwrap().drain(..).collect() + } + } + + #[async_trait] + impl MediaTransport for LoopbackTransport { + async fn send_media(&self, _packet: &MediaPacket) -> Result<(), TransportError> { + Ok(()) + } + async fn recv_media(&self) -> Result, TransportError> { + Ok(None) + } + async fn send_signal(&self, msg: &SignalMessage) -> Result<(), TransportError> { + self.sent.lock().unwrap().push(msg.clone()); + Ok(()) + } + async fn recv_signal(&self) -> Result, TransportError> { + Ok(None) + } + fn path_quality(&self) -> PathQuality { + PathQuality::default() + } + async fn close(&self) -> Result<(), TransportError> { + Ok(()) + } + } + + fn empty_signal_state() -> Arc> { + Arc::new(tokio::sync::Mutex::new(SignalState { + transport: None, + endpoint: None, + ipv6_endpoint: None, + fingerprint: String::new(), + signal_status: "idle".into(), + incoming_call_id: None, + incoming_caller_fp: None, + incoming_caller_alias: None, + pending_reflect: None, + own_reflex_addr: None, + desired_relay_addr: None, + reconnect_in_progress: false, + pending_path_report: None, + peer_hard_nat_probe: None, + peer_birthday_ports: None, + active_quality: Arc::new(std::sync::Mutex::new(wzp_proto::QualityProfile::GOOD)), + peer_max_quality: Arc::new(std::sync::Mutex::new(None)), + pending_upgrade: Arc::new(std::sync::Mutex::new(None)), + })) + } + + #[tokio::test] + async fn upgrade_proposal_auto_accepts() { + let transport = LoopbackTransport::new(); + handle_upgrade_proposal(&*transport, "c1", "p1").await.unwrap(); + + let sent = transport.take_sent(); + assert_eq!(sent.len(), 1); + match &sent[0] { + SignalMessage::UpgradeResponse { + call_id, + proposal_id, + accepted, + reason, + .. + } => { + assert_eq!(call_id, "c1"); + assert_eq!(proposal_id, "p1"); + assert!(accepted); + assert!(reason.is_none()); + } + other => panic!("expected UpgradeResponse, got {other:?}"), + } + } + + #[tokio::test] + async fn upgrade_response_accepted_sends_confirm_and_updates_quality() { + let transport = LoopbackTransport::new(); + let signal_state = empty_signal_state(); + { + let sig = signal_state.lock().await; + *sig.pending_upgrade.lock().unwrap() = + Some(("c1".into(), "p1".into(), wzp_proto::QualityProfile::STUDIO_48K)); + } + + handle_upgrade_response(&*transport, &signal_state, "c1", "p1", true) + .await + .unwrap(); + + let sent = transport.take_sent(); + assert_eq!(sent.len(), 1); + match &sent[0] { + SignalMessage::UpgradeConfirm { + call_id, + proposal_id, + confirmed_profile, + .. + } => { + assert_eq!(call_id, "c1"); + assert_eq!(proposal_id, "p1"); + assert_eq!(*confirmed_profile, wzp_proto::QualityProfile::STUDIO_48K); + } + other => panic!("expected UpgradeConfirm, got {other:?}"), + } + + let sig = signal_state.lock().await; + assert_eq!( + *sig.active_quality.lock().unwrap(), + wzp_proto::QualityProfile::STUDIO_48K + ); + } + + #[tokio::test] + async fn upgrade_confirm_updates_active_quality() { + let signal_state = empty_signal_state(); + handle_upgrade_confirm(&signal_state, wzp_proto::QualityProfile::STUDIO_64K).await; + + let sig = signal_state.lock().await; + assert_eq!( + *sig.active_quality.lock().unwrap(), + wzp_proto::QualityProfile::STUDIO_64K + ); + } + + #[tokio::test] + async fn quality_capability_updates_peer_max() { + let signal_state = empty_signal_state(); + handle_quality_capability(&signal_state, wzp_proto::QualityProfile::GOOD).await; + + let sig = signal_state.lock().await; + assert_eq!( + sig.peer_max_quality.lock().unwrap().unwrap(), + wzp_proto::QualityProfile::GOOD + ); + } +} + /// Shared Tauri app builder. Used by the desktop `main.rs` and the mobile /// entry point below. pub fn run() { tracing_subscriber::fmt().init(); + let active_quality = Arc::new(std::sync::Mutex::new(wzp_proto::QualityProfile::GOOD)); + let peer_max_quality = Arc::new(std::sync::Mutex::new(None)); + let pending_upgrade = Arc::new(std::sync::Mutex::new(None)); let state = Arc::new(AppState { engine: Mutex::new(None), signal: Arc::new(Mutex::new(SignalState { @@ -2873,6 +3317,9 @@ pub fn run() { pending_path_report: None, peer_hard_nat_probe: None, peer_birthday_ports: None, + active_quality: active_quality.clone(), + peer_max_quality: peer_max_quality.clone(), + pending_upgrade: pending_upgrade.clone(), })), }); @@ -2949,6 +3396,7 @@ pub fn run() { get_dred_verbose_logs, set_call_debug_logs, get_call_debug_logs, + push_camera_frame, ]) .run(tauri::generate_context!()) .expect("error while running WarzonePhone"); diff --git a/desktop/src/main.ts b/desktop/src/main.ts index abf59fb..f227a5d 100644 --- a/desktop/src/main.ts +++ b/desktop/src/main.ts @@ -79,6 +79,11 @@ const vdMicIcon = document.getElementById("vd-mic-icon")!; const vdSpkBtn = document.getElementById("vd-spk-btn")!; const vdSpkIcon = document.getElementById("vd-spk-icon")!; const vdEndBtn = document.getElementById("vd-end-btn")!; +const vdCamBtn = document.getElementById("vd-cam-btn")!; +const vdCamIcon = document.getElementById("vd-cam-icon")!; +const vdVideoStrip = document.getElementById("vd-video-strip")!; +const vdRemoteVideo = document.getElementById("vd-remote-video") as HTMLCanvasElement; +const vdLocalVideo = document.getElementById("vd-local-video") as HTMLVideoElement; const vdDirectInfo = document.getElementById("vd-direct-info")!; const vdDcIdenticon = document.getElementById("vd-dc-identicon")!; const vdDcName = document.getElementById("vd-dc-name")!; @@ -170,6 +175,12 @@ let connectPending = false; // guard against double-tap while connect is in-flig let directCallPeer: { fingerprint: string; alias: string | null } | null = null; let pendingCallId: string | null = null; +// Video / camera state +let cameraActive = false; +let cameraStream: MediaStream | null = null; +let cameraFrameTimer: number | null = null; +let remoteVideoActive = false; + function showToast(msg: string, durationMs = 3500) { let el = document.getElementById("wzp-toast"); if (!el) { @@ -420,6 +431,10 @@ function leaveVoice() { joinVoiceBtn.classList.remove("hidden"); vdLevelBar.style.width = "0%"; if (statusInterval) { clearInterval(statusInterval); statusInterval = null; } + stopCamera(); + remoteVideoActive = false; + vdVideoStrip.classList.add("hidden"); + remoteCtx.clearRect(0, 0, vdRemoteVideo.width, vdRemoteVideo.height); } // Drawer controls @@ -435,6 +450,76 @@ vdSpkBtn.addEventListener("click", async () => { try { await invoke("toggle_speaker"); } catch {} }); +// ── Camera (Blocker 4 + 5) ──────────────────────────────────────── +const camCaptureCanvas = document.createElement("canvas"); +const camCaptureCtx = camCaptureCanvas.getContext("2d")!; + +async function startCamera() { + if (cameraActive) return; + try { + cameraStream = await navigator.mediaDevices.getUserMedia({ + video: { width: { ideal: 1280 }, height: { ideal: 720 }, facingMode: "user" }, + audio: false, + }); + vdLocalVideo.srcObject = cameraStream; + vdVideoStrip.classList.remove("hidden"); + + const track = cameraStream.getVideoTracks()[0]; + const settings = track.getSettings(); + camCaptureCanvas.width = settings.width ?? 640; + camCaptureCanvas.height = settings.height ?? 360; + + cameraActive = true; + vdCamIcon.textContent = "Cam ✓"; + vdCamBtn.classList.add("active"); + + // Capture loop at ~15 fps + cameraFrameTimer = window.setInterval(async () => { + if (!cameraActive) return; + camCaptureCtx.drawImage(vdLocalVideo, 0, 0, camCaptureCanvas.width, camCaptureCanvas.height); + const dataUrl = camCaptureCanvas.toDataURL("image/jpeg", 0.75); + const b64 = dataUrl.slice(dataUrl.indexOf(",") + 1); + try { await invoke("push_camera_frame", { jpeg_b64: b64 }); } catch { /* call not active */ } + }, 67); // 67 ms ≈ 15 fps + } catch (e) { + console.warn("camera access denied or unavailable:", e); + } +} + +function stopCamera() { + cameraActive = false; + if (cameraFrameTimer != null) { window.clearInterval(cameraFrameTimer); cameraFrameTimer = null; } + if (cameraStream) { cameraStream.getTracks().forEach(t => t.stop()); cameraStream = null; } + vdLocalVideo.srcObject = null; + vdCamIcon.textContent = "Cam"; + vdCamBtn.classList.remove("active"); + // Hide strip only if remote video is also gone + if (!remoteVideoActive) vdVideoStrip.classList.add("hidden"); +} + +vdCamBtn.addEventListener("click", () => { + if (cameraActive) { stopCamera(); } else { startCamera(); } +}); + +// ── Remote video display (Blocker 5) ───────────────────────────── +const remoteCtx = vdRemoteVideo.getContext("2d")!; + +listen("video:frame", (event: any) => { + const { width, height, jpeg_b64 } = event.payload; + if (!jpeg_b64) return; + + remoteVideoActive = true; + vdVideoStrip.classList.remove("hidden"); + vdRemoteVideo.width = width ?? vdRemoteVideo.width; + vdRemoteVideo.height = height ?? vdRemoteVideo.height; + + const img = new Image(); + img.onload = () => { + remoteCtx.drawImage(img, 0, 0, vdRemoteVideo.width, vdRemoteVideo.height); + }; + img.src = `data:image/jpeg;base64,${jpeg_b64}`; +}); + // ── Poll status ─────────────────────────────────────────────────── interface CallStatusI { active: boolean; @@ -831,6 +916,7 @@ document.addEventListener("keydown", (e) => { if (e.key === "m") vdMicBtn.click(); if (e.key === "q") vdEndBtn.click(); if (e.key === "s") vdSpkBtn.click(); + if (e.key === "v") vdCamBtn.click(); if (e.key === "," && (e.metaKey || e.ctrlKey)) { e.preventDefault(); openSettings(); } }); diff --git a/desktop/src/style.css b/desktop/src/style.css index 64debfe..95d82c5 100644 --- a/desktop/src/style.css +++ b/desktop/src/style.css @@ -306,6 +306,22 @@ body { padding: 2px 0 4px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } +/* Video strip in voice drawer */ +.vd-video-strip { + display: flex; + gap: 4px; + padding: 4px 0 2px; + overflow-x: auto; +} +.vd-video-tile { + width: 160px; + height: 90px; + border-radius: 6px; + background: #000; + object-fit: cover; + flex-shrink: 0; +} + /* Incoming call banner */ .incoming-banner { position: fixed; diff --git a/docs/PRD/PRD-android-mediacodec-ndk9.md b/docs/PRD/PRD-android-mediacodec-ndk9.md new file mode 100644 index 0000000..9973a00 --- /dev/null +++ b/docs/PRD/PRD-android-mediacodec-ndk9.md @@ -0,0 +1,225 @@ +# PRD: Android MediaCodec NDK 0.9 Compatibility + +> **Status:** proposed +> **Resolves:** 31 compile errors in `crates/wzp-video/src/mediacodec.rs` blocking all Android video. +> **Depends on:** Remote build server `manwe@188.245.59.196` with Docker image `wzp-android-builder:latest`. + +## Problem + +`crates/wzp-video/src/mediacodec.rs` fails to compile for +`aarch64-linux-android` against the NDK 0.9 Rust crate. There are 31 errors +in 5 categories. Android video is completely blocked. + +The file already compiles for non-Android targets (all Android code is behind +`#[cfg(target_os = "android")]`). Only the Android target path needs fixing. + +## Goals + +- `cargo build --target aarch64-linux-android -p wzp-video` produces 0 errors on the remote server. +- Each fix category lands in a separate commit so failures can be bisected. +- Non-Android compilation is not broken. + +## Non-goals + +- Upgrading the NDK Docker image or changing the NDK version. +- Fixing video functionality beyond compilation (runtime testing is a separate task). +- Any files outside `crates/wzp-video/`. + +## Design + +### Build command (run after each fix) + +```bash +ssh manwe@188.245.59.196 'cd ~/wzp-builder/data/source && \ + git fetch github && git reset --hard github/experimental-ui && \ + docker run --rm \ + -v ~/wzp-builder/data/source:/build/source \ + -v ~/wzp-builder/data/cache/cargo-registry:/home/builder/.cargo/registry \ + -v ~/wzp-builder/data/cache/cargo-git:/home/builder/.cargo/git \ + -v ~/wzp-builder/data/cache/target:/build/source/target \ + wzp-android-builder:latest bash -c \ + "cd /build/source && cargo build --target aarch64-linux-android -p wzp-video 2>&1 | grep -E \"^error\" | head -30"' +``` + +### Fix order (commit one per category) + +#### Fix 1 — `E0433`: `ndk_sys` not declared as a dependency + +**Symptom**: `use of undeclared crate or module 'ndk_sys'` + +**File**: `crates/wzp-video/Cargo.toml` + +NDK 0.9 no longer re-exports raw `ndk_sys` symbols; they must be declared as +a direct dependency. Add to the `[target.'cfg(target_os = "android")'.dependencies]` +section (or create it if absent): + +```toml +[target.'cfg(target_os = "android")'.dependencies] +ndk = { version = "0.9" } +ndk-sys = { version = "0.6" } # ndk 0.9 depends on ndk-sys 0.6 +``` + +If `mediacodec.rs` only uses safe wrappers from the `ndk` crate and the +`ndk_sys` imports are not strictly needed, remove the `use ndk_sys::*` lines +from `mediacodec.rs` instead — whichever approach results in fewer changes. + +After this fix the `E0433` errors should drop from the build output. + +#### Fix 2 — `E0425`: `BITRATE_MODE_CBR` constant missing + +**Symptom**: `cannot find value 'BITRATE_MODE_CBR' in this scope` + +**File**: `crates/wzp-video/src/mediacodec.rs` + +`BITRATE_MODE_CBR` is already defined as a local constant at line 44: + +```rust +#[cfg(target_os = "android")] +const BITRATE_MODE_CBR: i32 = 2; +``` + +If the error persists after Fix 1, the issue is that `ndk_sys` was providing +a conflicting symbol. Verify the constant is still at line 44 after Fix 1. If +NDK 0.9 moved `BITRATE_MODE_CBR` to an enum, update the usage at line 516 +(`format.set_i32("bitrate-mode", BITRATE_MODE_CBR)`) to use the integer +value directly (`2`) or update the constant's value. + +If `ndk 0.9` defines `MediaCodecBitrateMode::Cbr` as an enum, the call site +in `MediaCodecAv1Encoder::new` (line ~516) can be updated to: + +```rust +format.set_i32( + "bitrate-mode", + ndk::media::media_codec::MediaCodecBitrateMode::Cbr as i32, +); +``` + +#### Fix 3 — `E0308`: `InputBuffer` returns `&mut [MaybeUninit]` + +**Symptom**: `expected &mut [u8], found &mut [MaybeUninit]` + +**File**: `crates/wzp-video/src/mediacodec.rs` + +NDK 0.9 changed `InputBuffer::buffer_mut()` from `&mut [u8]` to +`&mut [MaybeUninit]`. There are multiple write sites in the file — all +follow the same pattern: + +```rust +// Before (NDK 0.8): +let buf = buffer.buffer_mut(); // &mut [u8] +let n = frame.data.len().min(buf.len()); +buf[..n].copy_from_slice(&frame.data[..n]); +``` + +```rust +// After (NDK 0.9): +let buf = buffer.buffer_mut(); // &mut [MaybeUninit] +let n = frame.data.len().min(buf.len()); +for (d, &s) in buf[..n].iter_mut().zip(frame.data[..n].iter()) { + d.write(s); +} +``` + +The file already uses the `d.write(s)` pattern in some places (lines 125–127, +297–299, etc.). Search for **every** occurrence of `buffer.buffer_mut()` and +`buffer_mut()` and apply the same pattern. Affected structs: +`MediaCodecEncoder::encode` (~line 123), `MediaCodecDecoder::decode` +(~line 294), `MediaCodecHevcEncoder::encode` (~line 439), +`MediaCodecHevcDecoder::decode` (~line 773), `MediaCodecAv1Encoder::encode` +(~line 560), `MediaCodecAv1Decoder::decode` (~line 907). + +Do NOT use `unsafe { std::mem::transmute }` — the `d.write(s)` pattern is +already present and safe. + +Note: if the file already uses `d.write(s)` everywhere, this category may +already be addressed by the existing code. Check the actual error count. + +#### Fix 4 — `E0599`: `.index()` is private + +**Symptom**: `method 'index' is private` + +**File**: `crates/wzp-video/src/mediacodec.rs` + +NDK 0.9 removed the public `.index()` method from `DequeuedInputBuffer` and +`DequeuedOutputBuffer`. The pattern that broke: + +```rust +// Broken: buffer.index() is private in NDK 0.9 +let idx = buffer.index(); +codec.queue_input_buffer_index(idx, ...); +``` + +In NDK 0.9 the correct API is to pass the buffer object directly to +`queue_input_buffer`: + +```rust +codec.queue_input_buffer(buffer, offset, size, pts_us, flags)?; +``` + +The file already uses `codec.queue_input_buffer(buffer, 0, to_copy, ...)` in +most places (lines 131, 303, 447, etc.). Search for any remaining `.index()` +calls on buffer objects and replace them with the direct-pass pattern shown +above. + +#### Fix 5 — `E0277`: `NonNull` is not `Send` + +**Symptom**: `NonNull` cannot be sent between threads safely + +**File**: `crates/wzp-video/src/mediacodec.rs` + +Each codec struct must have an `unsafe impl Send` declaration. Audit all six +codec structs: + +| Struct | `unsafe impl Send` present? | +|--------|----------------------------| +| `MediaCodecEncoder` | Yes (line 51) | +| `MediaCodecDecoder` | Yes (line 228) | +| `MediaCodecHevcEncoder` | Yes (line 374) | +| `MediaCodecHevcDecoder` | Yes (line 705) | +| `MediaCodecAv1Encoder` | Yes (line 503) | +| `MediaCodecAv1Decoder` | Yes (line 844) | + +If any are missing, add them with a safety comment: + +```rust +// SAFETY: AMediaCodec is documented as thread-safe. +#[cfg(target_os = "android")] +unsafe impl Send for MediaCodecXxxYyy {} +``` + +This category may already be clean. Confirm with the build output. + +## Implementation steps + +1. Push the current branch to `github/experimental-ui` before starting. +2. **Commit 1**: Fix `ndk_sys` dependency (`Cargo.toml`). Push. Run build. + Confirm `E0433` errors drop. +3. **Commit 2**: Fix `BITRATE_MODE_CBR`. Push. Run build. Confirm `E0425` gone. +4. **Commit 3**: Fix `MaybeUninit` write sites. Push. Run build. Confirm + `E0308` gone. +5. **Commit 4**: Remove any `.index()` calls. Push. Run build. Confirm + `E0599` gone. +6. **Commit 5**: Add missing `unsafe impl Send` if any. Push. Run build. + Confirm `E0277` gone and total error count is 0. + +## Files to read before implementing + +- `crates/wzp-video/src/mediacodec.rs` (full file — 45 KB; read in chunks) +- `crates/wzp-video/Cargo.toml` (check existing `[dependencies]` sections) + +## Verify + +Final build command (see Design section). Expected output: no lines matching +`^error`. + +Also verify non-Android host still compiles: + +```bash +cargo check -p wzp-video +``` + +## Done when + +`cargo build --target aarch64-linux-android -p wzp-video` on the remote +server produces 0 `error[...]` lines. Non-Android `cargo check -p wzp-video` +also passes. diff --git a/docs/PRD/PRD-clippy-debt.md b/docs/PRD/PRD-clippy-debt.md new file mode 100644 index 0000000..f8e9c49 --- /dev/null +++ b/docs/PRD/PRD-clippy-debt.md @@ -0,0 +1,260 @@ +# PRD: Fix wzp-codec Clippy Lint Debt + +> **Status:** proposed +> **Resolves:** 9 pre-existing clippy lints in `crates/wzp-codec/src/` that cause `cargo clippy --workspace -D warnings` to fail, breaking any strict-CI configuration. +> **Depends on:** Nothing — all changes are in `crates/wzp-codec/src/`. + +## Problem + +`cargo clippy -p wzp-codec -- -D warnings` fails with 9 lints across 5 files. +These are pre-existing code patterns that were never flagged during development +because the CI flag was not set. They have no runtime impact today but prevent +adding `-D warnings` to CI without first cleaning them up. + +The 3 errors in `deps/featherchat` are in a submodule — do NOT touch them. +`warzone_protocol` clippy errors are accepted debt (not our code). + +## Goals + +- `cargo clippy -p wzp-codec -- -D warnings` exits 0. +- No behavior changes — every fix is a semantically equivalent rewrite. +- No changes outside `crates/wzp-codec/src/`. + +## Non-goals + +- Fixing clippy lints in any crate other than `wzp-codec`. +- Adding new functionality. +- Touching the `deps/featherchat` submodule. + +## Design + +### Lint inventory + +| Lint | Count | File | Approx line | Fix | +|------|-------|------|-------------|-----| +| `implicit_saturating_sub` | 1 | `aec.rs` | 117–119 | `saturating_sub` | +| `needless_range_loop` | 2 | `aec.rs:164`, `resample.rs:51` | — | iterate with `iter().enumerate()` or direct iter | +| `manual_div_ceil` | 2 | `codec2_dec.rs:48`, `codec2_enc.rs:48` | — | `div_ceil` | +| `manual_clamp` | 2 | `denoise.rs:59`, `opus_enc.rs:250` | — | `.clamp(min, max)` | +| `manual_ascii_check` | 1 | `opus_enc.rs:104` | — | `.eq_ignore_ascii_case()` | +| `same_item_push` | 1 | `resample.rs:184` | — | `vec.resize` or `extend(repeat)` | + +### Fix details + +#### 1. `implicit_saturating_sub` — `aec.rs` line ~117 + +Current code: + +```rust +fn delay_available(&self) -> usize { + let buffered = self.delay_write - self.delay_read; + if buffered > self.delay_samples { + buffered - self.delay_samples + } else { + 0 + } +} +``` + +Clippy wants `saturating_sub` because the subtraction can underflow if +`buffered < self.delay_samples`: + +```rust +fn delay_available(&self) -> usize { + let buffered = self.delay_write - self.delay_read; + buffered.saturating_sub(self.delay_samples) +} +``` + +This is semantically identical (both return 0 when `buffered <= delay_samples`). + +#### 2a. `needless_range_loop` — `aec.rs` line ~164 + +Current code: + +```rust +for i in 0..n { + let near_f = nearend[i] as f32; + let base = (self.far_pos + fl * ((n / fl) + 2) + i - n) % fl; + ... +} +``` + +`i` is used both to index `nearend[i]` and in arithmetic (`+ i - n`). +Clippy fires because `nearend[i]` could use `.iter().enumerate()`. +Convert to `enumerate`: + +```rust +for (i, &sample) in nearend.iter().enumerate() { + let near_f = sample as f32; + let base = (self.far_pos + fl * ((n / fl) + 2) + i - n) % fl; + ... +} +``` + +Make sure to update any references to `nearend[i]` inside the loop body +to use `sample` (or `near_f` directly). Also update the NLMS adaptation +sub-loop if it references `nearend[i]`. + +#### 2b. `needless_range_loop` — `resample.rs` line ~51 + +Current code: + +```rust +for i in 0..FIR_TAPS { + let n = i as f64 - m / 2.0; + let sinc = ...; + let t = 2.0 * i as f64 / m - 1.0; + let kaiser = ...; + kernel[i] = sinc * kaiser; +} +``` + +`i` is used both as an index (`kernel[i]`) and in arithmetic. Use +`iter_mut().enumerate()`: + +```rust +for (i, slot) in kernel.iter_mut().enumerate() { + let n = i as f64 - m / 2.0; + let sinc = ...; + let t = 2.0 * i as f64 / m - 1.0; + let kaiser = ...; + *slot = sinc * kaiser; +} +``` + +#### 3a. `manual_div_ceil` — `codec2_dec.rs` line ~48 + +Current code: + +```rust +fn bytes_per_frame(&self) -> usize { + (self.inner.bits_per_frame() + 7) / 8 +} +``` + +Replace with: + +```rust +fn bytes_per_frame(&self) -> usize { + self.inner.bits_per_frame().div_ceil(8) +} +``` + +`div_ceil` is stable as of Rust 1.73. The builder uses a recent enough +toolchain. If `bits_per_frame()` returns `usize`, the method is available. +If it returns a different integer type, cast accordingly. + +#### 3b. `manual_div_ceil` — `codec2_enc.rs` line ~48 + +Same pattern, same fix: + +```rust +fn bytes_per_frame(&self) -> usize { + self.inner.bits_per_frame().div_ceil(8) +} +``` + +#### 4a. `manual_clamp` — `denoise.rs` line ~59 + +Current code: + +```rust +let clamped = val.max(-32768.0).min(32767.0); +``` + +Replace with: + +```rust +let clamped = val.clamp(-32768.0_f32, 32767.0_f32); +``` + +Note: `.clamp()` on `f32` requires both bounds to be the same type. If `val` +is already `f32`, no extra cast is needed. Verify the type of `val` in +context (it is `f32` per the output array type `[f32; 480]`). + +#### 4b. `manual_clamp` — `opus_enc.rs` line ~252 + +Read the surrounding code for the exact pattern. It will be something like: + +```rust +let v = if x < min_val { min_val } else if x > max_val { max_val } else { x }; +``` + +or the `.max().min()` chain. Replace with `x.clamp(min_val, max_val)`. + +#### 5. `manual_ascii_check` — `opus_enc.rs` line ~104 + +Current code: + +```rust +Ok(v) => !v.is_empty() && v != "0" && v.to_ascii_lowercase() != "false", +``` + +Clippy wants `.eq_ignore_ascii_case()` instead of lowercasing the whole string: + +```rust +Ok(v) => !v.is_empty() && v != "0" && !v.eq_ignore_ascii_case("false"), +``` + +#### 6. `same_item_push` — `resample.rs` line ~183 + +Current code: + +```rust +for _ in 1..RATIO { + work.push(0.0); +} +``` + +This pushes the same `0.0` value `(RATIO - 1)` times. Replace with: + +```rust +work.resize(work.len() + (RATIO - 1), 0.0f64); +``` + +Or equivalently: + +```rust +work.extend(std::iter::repeat(0.0f64).take(RATIO - 1)); +``` + +Note: `RATIO` is a `const usize`. Verify `work` is `Vec` in context +(it is — `work.push(s as f64)` immediately before). + +## Implementation steps + +1. Read each file at the line numbers listed above to confirm the exact current + code before editing (line numbers may shift slightly due to prior edits). +2. Apply all 9 fixes. They are independent — no ordering requirement. +3. Run `cargo clippy -p wzp-codec -- -D warnings` locally or via the CI + command. +4. If any lint persists, re-read that file section and adjust. + +## Files to read before implementing + +- `crates/wzp-codec/src/aec.rs` lines 114–200 +- `crates/wzp-codec/src/resample.rs` lines 45–70 and 178–190 +- `crates/wzp-codec/src/codec2_dec.rs` lines 40–55 +- `crates/wzp-codec/src/codec2_enc.rs` lines 40–55 +- `crates/wzp-codec/src/denoise.rs` lines 45–65 +- `crates/wzp-codec/src/opus_enc.rs` lines 96–110 and 244–260 + +## Verify + +```bash +cargo clippy -p wzp-codec -- -D warnings +``` + +Expected: exits 0 with no warnings. + +Also run to confirm no regressions: + +```bash +cargo test -p wzp-codec +``` + +## Done when + +`cargo clippy -p wzp-codec -- -D warnings` exits 0. All 9 lints are gone. +`cargo test -p wzp-codec` passes. No changes outside `crates/wzp-codec/src/`. diff --git a/docs/PRD/PRD-e2e-media-encryption.md b/docs/PRD/PRD-e2e-media-encryption.md new file mode 100644 index 0000000..8f84af7 --- /dev/null +++ b/docs/PRD/PRD-e2e-media-encryption.md @@ -0,0 +1,195 @@ +# PRD: E2E Media Encryption — Wire EncryptingTransport on Relay Path + +> **Status:** proposed +> **Resolves:** Security gap — relay-path media travels in QUIC TLS only; WZP application-layer ChaCha20-Poly1305 is negotiated but never applied. +> **Depends on:** `wzp_client::encrypted_transport::EncryptingTransport` (already implemented). + +## Problem + +`CallEngine::start` (both the Android path and the desktop path) calls +`wzp_client::handshake::perform_handshake`, which returns a `HandshakeResult` +containing a `session: Box` (a keyed `ChaChaSession`). +Both call sites discard the session — only `hs.video_codec` is retained. + +All subsequent `send_media` / `recv_media` calls go directly through +`Arc`, which provides QUIC TLS (relay sees +plaintext application data after TLS termination at the relay). The WZP +application-level AEAD — ChaCha20-Poly1305, keyed per-call, relay-never-sees +— is never applied. + +`wzp_client::encrypted_transport::EncryptingTransport` exists +(`crates/wzp-client/src/encrypted_transport.rs`) and is fully tested. +It wraps any `Arc` and intercepts every `send_media` / +`recv_media` call with `session.encrypt()` / `session.decrypt()`. + +## Goals + +- The relay-path `HandshakeResult::session` is used to construct an + `EncryptingTransport` that wraps the raw `QuinnTransport`. +- All `send_media` and `recv_media` calls in the relay path go through the + wrapper, not the raw transport. +- The direct P2P path (`is_direct_p2p == true`) is left unchanged — QUIC TLS + is the encryption layer there. +- `cargo check --manifest-path desktop/src-tauri/Cargo.toml` passes. +- A `#[cfg(test)]` test verifies that the relay path uses `EncryptingTransport`. + +## Non-goals + +- Rekeying (`SignalMessage::Rekey`) — tracked separately. +- Video transport encryption (same mechanism; apply after audio is confirmed working). +- Changes to the P2P path, the relay binary, or any crate outside `desktop/src-tauri`. + +## Design + +### `EncryptingTransport` API (read `crates/wzp-client/src/encrypted_transport.rs`) + +```rust +pub struct EncryptingTransport { ... } + +impl EncryptingTransport { + pub fn new(inner: Arc, session: Box) -> Self; +} + +// Implements MediaTransport: +// send_media → session.encrypt(header_bytes, payload) → inner.send_media +// recv_media → inner.recv_media → session.decrypt(header_bytes, ciphertext) +// send_signal / recv_signal / path_quality / close → forwarded unchanged +``` + +`EncryptingTransport` is NOT `Arc`-wrapped by the constructor; wrap it in +`Arc::new(...)` when storing as `Arc`. + +### Two call sites in `desktop/src-tauri/src/engine.rs` + +**Call site 1 — Android path** (`CallEngine::start` around line 575): + +```rust +if !is_direct_p2p { + let _hs = match wzp_client::handshake::perform_handshake(...).await { Ok(hs) => hs, ... }; + // hs.session is discarded here — fix this +} +``` + +Change: capture `hs`, then build a wrapped transport: + +```rust +if !is_direct_p2p { + let hs = match wzp_client::handshake::perform_handshake(...).await { Ok(hs) => hs, ... }; + info!(video_codec = ?hs.video_codec, "handshake complete"); + let transport: Arc = + Arc::new(wzp_client::encrypted_transport::EncryptingTransport::new( + transport.clone(), + hs.session, + )); + // use `transport` (the wrapped version) for all subsequent send_t / recv_t clones +} +``` + +The variable `transport` must shadow the raw `Arc` so that +every subsequent clone of `transport` picks up the encrypted wrapper. + +**Call site 2 — Desktop path** (`CallEngine::start` around line 1551): + +```rust +let _negotiated_video_codec = if !is_direct_p2p { + let hs = wzp_client::handshake::perform_handshake(...).await?; + info!(video_codec = ?hs.video_codec, "handshake complete"); + hs.video_codec // session dropped here — fix this +} else { None }; +``` + +Change: extract `session` before returning `video_codec`, then shadow +`transport` with the wrapped version. Because `transport` is used after this +block (cloned into `send_t`, `recv_t`, etc.), the shadow must happen inside +the same scope or immediately after: + +```rust +let (_negotiated_video_codec, transport): (_, Arc) = + if !is_direct_p2p { + let hs = wzp_client::handshake::perform_handshake(...).await?; + info!(video_codec = ?hs.video_codec, "handshake complete"); + let enc = Arc::new(wzp_client::encrypted_transport::EncryptingTransport::new( + transport.clone(), + hs.session, + )); + (hs.video_codec, enc) + } else { + info!("direct P2P — skipping relay handshake"); + (None, transport.clone()) + }; +``` + +All subsequent `transport.clone()` calls then operate on the encrypted wrapper. + +### Import + +Add to the top of `engine.rs` if not already present: + +```rust +use wzp_client::encrypted_transport::EncryptingTransport; +``` + +Or use the fully-qualified path inline (already shown above). + +### Type compatibility + +- `EncryptingTransport` implements `wzp_proto::MediaTransport` (confirmed in the source). +- The existing `send_t` / `recv_t` variables are already typed as + `Arc` (or coerced on first use) — the shadow is a + drop-in replacement. +- The `vid_transport` for the video path (`line ~2090`) is also cloned from + `transport`; it will automatically use the encrypted wrapper if the shadow + is placed before those clones. + +## Implementation steps + +1. Read `desktop/src-tauri/src/engine.rs` lines 570–620 (Android path) and + 1547–1570 (desktop path) to see the exact variable names in each branch. +2. **Android path fix** (line ~585): rename `_hs` to `hs`, extract + `hs.session`, wrap `transport` with `EncryptingTransport::new`, re-bind + `transport` as `Arc`. +3. **Desktop path fix** (line ~1551): restructure the + `if !is_direct_p2p` block to return `(video_codec, wrapped_transport)` + and shadow `transport`. +4. Confirm that `vid_transport` (line ~2090) is cloned after the shadow — if + it is, no further changes are needed for video. +5. Run `cargo check --manifest-path desktop/src-tauri/Cargo.toml`. Fix any + type-mismatch errors (usually a missing `as Arc` cast + or a moved value). +6. Add a `#[cfg(test)]` module to `engine.rs` (or to a new + `engine_tests.rs` included via `#[cfg(test)] mod engine_tests`) with a + test that constructs a `LoopbackTransport`, calls `perform_handshake` + against a mock relay fixture, and verifies that a received payload is + decrypted before returning from `recv_media`. A simpler alternative that + avoids a full handshake: assert `is::()` on the + `transport` variable at the test call site using `std::any::Any`. + +## Files to read before implementing + +- `desktop/src-tauri/src/engine.rs` lines 475–625 (Android path) and + 1480–1570 (desktop path) +- `crates/wzp-client/src/encrypted_transport.rs` (full — for the exact + constructor signature and trait impl) +- `crates/wzp-client/src/handshake.rs` (for `HandshakeResult` struct + definition — confirm the `session` field name and type) + +## Verify + +```bash +cargo check --manifest-path desktop/src-tauri/Cargo.toml +``` + +Expected: 0 errors. + +Manual smoke check: both `perform_handshake` call sites in `engine.rs` must +use `hs.session` (grep: `hs\.session` should appear twice, once per call site). +The string `_hs` must not remain on the relay path (only on the `_hs =` binding if the variable is intentionally unused before wrapping). + +## Done when + +- `cargo check --manifest-path desktop/src-tauri/Cargo.toml` exits 0. +- Both relay-path `perform_handshake` call sites build an `EncryptingTransport` + from `hs.session`. +- The direct-P2P branch (`is_direct_p2p == true`) is unchanged. +- A `#[cfg(test)]` test in `engine.rs` verifies that `EncryptingTransport` + is used on the relay path (construction proof or decrypt round-trip). diff --git a/docs/PRD/PRD-quality-upgrade-flow.md b/docs/PRD/PRD-quality-upgrade-flow.md new file mode 100644 index 0000000..2e9022b --- /dev/null +++ b/docs/PRD/PRD-quality-upgrade-flow.md @@ -0,0 +1,220 @@ +# PRD: Quality Upgrade Flow — UpgradeProposal / Response / Confirm + +> **Status:** proposed +> **Resolves:** Four TODO comments in the signal task of `desktop/src-tauri/src/lib.rs` that leave quality upgrade messages unhandled. Audio quality never upgrades mid-call even when the network improves. +> **Depends on:** `wzp_proto::SignalMessage::{UpgradeProposal, UpgradeResponse, UpgradeConfirm, QualityCapability}` (already defined in `crates/wzp-proto/src/packet.rs`). + +## Problem + +The signal receive task in `lib.rs` matches `UpgradeProposal`, `UpgradeResponse`, +`UpgradeConfirm`, and `QualityCapability` messages from the peer, logs them, +then hits a `// TODO` comment and does nothing. The 4 TODOs are at lines +1930, 1949, 1966, and 1985 of `desktop/src-tauri/src/lib.rs`. + +Consequence: audio quality is frozen at the profile negotiated at call start. +Even when the network improves, the encoder never upgrades. + +## Goals + +1. `UpgradeProposal` auto-accepts and sends `UpgradeResponse { accepted: true }`. +2. Accepted `UpgradeResponse` sends `UpgradeConfirm` and switches the local encoder. +3. Received `UpgradeConfirm` switches the local encoder. +4. Received `QualityCapability` caps the local encoder to the peer's max profile. +5. A unit test verifies the accept/confirm round-trip. +6. `cargo check --manifest-path desktop/src-tauri/Cargo.toml` passes. + +## Non-goals + +- UI for manual accept/reject of upgrade proposals (auto-accept only). +- Sending `UpgradeProposal` from our side (the outgoing path already exists in + `lib.rs`; this PRD only handles receiving). +- Downgrade negotiation. +- Persisting quality profiles across calls. + +## Design + +### New shared state + +Add the following to `AppState` (or as captured variables in the signal task +closure — whichever is cleaner given the existing structure): + +```rust +/// Pending outgoing upgrade: (call_id, proposal_id, profile). +/// Set when we send an UpgradeProposal, consumed when we receive an accepted UpgradeResponse. +pending_upgrade: Arc>>, + +/// Current quality profile for the encoder. The audio send task reads this +/// at the start of each encode cycle. +active_quality: Arc>, + +/// Peer's reported maximum quality cap. The audio send task clamps to min(active, peer_max). +peer_max_quality: Arc>>, +``` + +If `AppState` already holds these fields (check `lib.rs` for the struct +definition), reuse them instead of adding duplicates. + +### Handler implementations + +#### 1. `UpgradeProposal` (line ~1930) + +```rust +// Replace the TODO comment with: +let response = SignalMessage::UpgradeResponse { + version: wzp_proto::default_signal_version(), + call_id: call_id.clone(), + proposal_id: proposal_id.clone(), + accepted: true, + reason: None, +}; +if let Err(e) = signal_transport.send_signal(&response).await { + tracing::warn!("failed to send UpgradeResponse: {e}"); +} +``` + +`signal_transport` is whatever variable holds the signal `Arc` +in scope at that match arm. Inspect the enclosing task to find the right name. + +#### 2. `UpgradeResponse` (line ~1949) + +```rust +// Replace the TODO comment with: +if accepted { + // Retrieve the pending proposal to get the confirmed_profile. + let maybe_proposal = pending_upgrade.lock().unwrap().take(); + if let Some((_cid, pid, profile)) = maybe_proposal { + if pid == proposal_id { + // Send UpgradeConfirm. + let confirm = SignalMessage::UpgradeConfirm { + version: wzp_proto::default_signal_version(), + call_id: call_id.clone(), + proposal_id: proposal_id.clone(), + confirmed_profile: profile.clone(), + }; + if let Err(e) = signal_transport.send_signal(&confirm).await { + tracing::warn!("failed to send UpgradeConfirm: {e}"); + } + // Switch our encoder. + *active_quality.lock().unwrap() = profile; + } + } +} +``` + +If `pending_upgrade` is a captured `Arc>` in the task closure, it +can be read/written without going through `AppState`. + +#### 3. `UpgradeConfirm` (line ~1966) + +```rust +// Replace the TODO comment with: +*active_quality.lock().unwrap() = confirmed_profile; +``` + +The audio send task (in `engine.rs`) reads `active_quality` at the start of +each encode cycle and reconfigures the Opus encoder bitrate accordingly. + +#### 4. `QualityCapability` (line ~1985) + +```rust +// Replace the TODO comment with: +*peer_max_quality.lock().unwrap() = Some(max_profile); +``` + +#### 5. Audio send task changes (`engine.rs`) + +The audio send task already runs in a loop. Add a quality-check at the top of +each encode iteration: + +```rust +// At the start of the encode loop body: +let effective_profile = { + let active = active_quality.lock().unwrap().clone(); + let peer_cap = peer_max_quality.lock().unwrap().clone(); + match peer_cap { + Some(cap) if cap.opus_bitrate_bps() < active.opus_bitrate_bps() => cap, + _ => active, + } +}; +// Pass effective_profile to encoder if it changed since last iteration. +``` + +`QualityProfile::opus_bitrate_bps()` already exists (check +`crates/wzp-proto/src/codec_id.rs`). If `QualityProfile` does not have a +direct bitrate accessor, compare using the `PartialOrd` impl or a helper that +ranks profiles numerically. + +To avoid calling `encoder.set_bitrate()` every single frame, cache the last +applied profile and only reconfigure on change: + +```rust +let mut last_applied_profile: Option = None; + +// Inside loop: +if Some(&effective_profile) != last_applied_profile.as_ref() { + encoder.set_bitrate(effective_profile.opus_bitrate_bps()); + last_applied_profile = Some(effective_profile.clone()); +} +``` + +`encoder.set_bitrate(bps: u32)` — add this method to `OpusEncoder` in +`crates/wzp-codec/src/opus_enc.rs` if it does not exist. It wraps +`opus_encoder_ctl(OPUS_SET_BITRATE_REQUEST, bps)`. + +### Unit tests + +Add a `#[cfg(test)]` module in `lib.rs` (or a dedicated test file) that: + +1. Creates a `LoopbackSignalTransport` stub that records sent `SignalMessage`s. +2. Calls the `UpgradeProposal` handler logic directly, asserts that an + `UpgradeResponse { accepted: true }` was sent. +3. Calls the `UpgradeResponse { accepted: true }` handler with a pre-populated + `pending_upgrade`, asserts that `UpgradeConfirm` was sent and + `active_quality` was updated. + +These can be pure unit tests (no Tauri or audio), since the handlers are +pure async functions over captured state. + +## Implementation steps + +1. Read `desktop/src-tauri/src/lib.rs` lines 1910–1990 (the four TODO blocks) + and the surrounding signal task structure to identify the variable names + for `signal_transport`, `app_state`, and any existing quality-state fields. +2. Read `desktop/src-tauri/src/engine.rs` for `CallEngine` struct fields and + the audio send task loop. +3. Read `crates/wzp-proto/src/codec_id.rs` for `QualityProfile` methods. +4. Add `pending_upgrade`, `active_quality`, `peer_max_quality` to the + appropriate shared state (or as closure captures in the signal task). +5. Replace the 4 TODO comments with the handlers described above. +6. Add `set_bitrate` to `OpusEncoder` if missing. +7. Update the audio send task to read `active_quality` / `peer_max_quality` + each iteration. +8. Add unit tests. +9. Run `cargo check --manifest-path desktop/src-tauri/Cargo.toml`. + +## Files to read before implementing + +- `desktop/src-tauri/src/lib.rs` — grep for `UpgradeProposal` to find the + exact lines; also read the surrounding signal task for variable names. +- `crates/wzp-proto/src/packet.rs` lines 1130–1190 — `UpgradeProposal`, + `UpgradeResponse`, `UpgradeConfirm`, `QualityCapability` struct layouts. +- `desktop/src-tauri/src/engine.rs` — `CallEngine` struct fields, audio + send task loop. +- `crates/wzp-proto/src/codec_id.rs` — `QualityProfile` methods. +- `crates/wzp-codec/src/opus_enc.rs` — `OpusEncoder` API. + +## Verify + +```bash +cargo check --manifest-path desktop/src-tauri/Cargo.toml +cargo test -p wzp-desktop 2>/dev/null || cargo test --manifest-path desktop/src-tauri/Cargo.toml +``` + +Expected: 0 errors; unit tests pass. + +## Done when + +- All 4 TODO comments replaced with real logic. +- `cargo check --manifest-path desktop/src-tauri/Cargo.toml` exits 0. +- Unit test verifies: `UpgradeProposal` → `UpgradeResponse { accepted: true }` sent; + `UpgradeResponse { accepted: true }` → `UpgradeConfirm` sent + `active_quality` updated. diff --git a/docs/PRD/PRD-wire-format-hardening.md b/docs/PRD/PRD-wire-format-hardening.md new file mode 100644 index 0000000..4d8ec9c --- /dev/null +++ b/docs/PRD/PRD-wire-format-hardening.md @@ -0,0 +1,242 @@ +# PRD: Wire Format Hardening — FEC block_id u16, SignalMessage version byte, FEC repair index wrap + +> **Status:** proposed +> **Resolves:** Three small wire-format defects (H2, M1, M4) that compound over time into silent data corruption or protocol breakage. +> **Depends on:** Nothing — purely mechanical changes to `wzp-fec` and `wzp-proto`. + +## Problem + +Three independent issues: + +**H2 — `fec_block_id` u8 wraps too fast.** The `block_id` field in +`RaptorQFecEncoder` (and `RaptorQFecDecoder`) is `u8`. At 5 audio frames +per block and 50 fps this wraps every ~51 seconds. A slow receiver or a +mid-session join can receive packets from two different blocks with the same +`block_id`, silently corrupting FEC recovery. + +**M1 — Some `SignalMessage` variants lack a `version` byte.** Most variants +have `#[serde(default = "default_signal_version")] version: u8`. The unit +variant `Reflect` (and potentially others added recently) does not. Future +protocol changes that key on `version` will silently misparse old messages +from peers without the field. + +**M4 — FEC repair index can silently wrap at 255.** In +`crates/wzp-fec/src/encoder.rs` line 140: + +```rust +let idx = (num_source as u16).wrapping_add(i as u16); +``` + +(The line was already fixed to `u16` — verify it is `u16`, not `u8`. If it +is still `u8`, the fix is below.) + +If the line currently reads `(num_source as u8).wrapping_add(i as u8)`, then +when `num_source + repair_count > 255` the repair symbol indices wrap silently, +producing incorrect ESI values that the decoder cannot correlate to source +blocks. + +## Goals + +- **H2**: Widen `block_id` in encoder and decoder from `u8` to `u16`. + Update `finalize_block` return type and `current_block_id` return type in + the trait (`wzp-proto`) and implementations (`wzp-fec`). +- **M1**: Audit every `SignalMessage` variant; add + `#[serde(default = "default_signal_version")] version: u8` to any that + are missing it. +- **M4**: Confirm the repair index uses `u16`; fix it if it is still `u8`. + Update the decoder's `add_symbol` call site if the index type changes. +- `cargo test -p wzp-fec -p wzp-proto` passes; no existing tests broken. + +## Non-goals + +- Changing the wire encoding of `MediaHeaderV2::fec_block` — it is already + `u16` on the wire. This PRD only widens the **internal counter** to match. +- Multi-block decode concurrency or block expiry policy. +- Any crate outside `wzp-fec` and `wzp-proto`. + +## Design + +### Item A — `fec_block_id` u8 → u16 + +**Files**: +- `crates/wzp-proto/src/traits.rs` — `FecEncoder` and `FecDecoder` traits +- `crates/wzp-fec/src/encoder.rs` — `RaptorQFecEncoder` +- `crates/wzp-fec/src/decoder.rs` — `RaptorQFecDecoder` + +**Trait changes** (`traits.rs`): + +```rust +// Before: +fn finalize_block(&mut self) -> Result; +fn current_block_id(&self) -> u8; +fn add_symbol(&mut self, block_id: u8, ...) -> Result<(), FecError>; +fn try_decode(&mut self, block_id: u8) -> Result<...>; +fn expire_before(&mut self, block_id: u8); +``` + +```rust +// After: +fn finalize_block(&mut self) -> Result; +fn current_block_id(&self) -> u16; +fn add_symbol(&mut self, block_id: u16, ...) -> Result<(), FecError>; +fn try_decode(&mut self, block_id: u16) -> Result<...>; +fn expire_before(&mut self, block_id: u16); +``` + +**Encoder changes** (`encoder.rs`): + +- Change `block_id: u8` field to `block_id: u16`. +- Update `self.block_id.wrapping_add(1)` (already u16 semantics; keep as is). +- Update `finalize_block` to return `u16`. +- Update `current_block_id` to return `u16`. +- Update all tests that assert `block_id == 0u8` → `== 0u16`, and the + wrap test (`block_id_wraps`) to iterate to `u16::MAX` (65535) — or reduce + it to 300 iterations to keep it fast, asserting the wrap at 65536. + +The wrap test at 256 iterations (`0..=255u8`) must be updated; a full +`u16` wrap test at 65536 iterations is too slow for CI. Change to: + +```rust +#[test] +fn block_id_wraps_u16() { + let mut enc = RaptorQFecEncoder::with_defaults(1); + // Advance 300 blocks and verify no panic + monotonic increment. + for expected in 0..300u16 { + assert_eq!(enc.current_block_id(), expected); + enc.add_source_symbol(&[0u8; 10]).unwrap(); + enc.finalize_block().unwrap(); + } + // Explicitly test wrap at u16 boundary. + let mut enc2 = RaptorQFecEncoder::with_defaults(1); + enc2.block_id = u16::MAX; + enc2.add_source_symbol(&[0u8; 10]).unwrap(); + let id = enc2.finalize_block().unwrap(); + assert_eq!(id, u16::MAX); + assert_eq!(enc2.current_block_id(), 0); +} +``` + +Note: `block_id` is a private field; expose a test helper or set it in a +`#[cfg(test)]` `impl` block. + +**Decoder changes** (`decoder.rs`): + +- Change `blocks: HashMap` to `HashMap`. +- Update `get_or_create_block(block_id: u8)` → `get_or_create_block(block_id: u16)`. +- Update `add_symbol`, `try_decode`, `expire_before` signatures to `u16`. +- The `SourceBlockEncoder::new(self.block_id, ...)` call in `encoder.rs` passes + `block_id` to `raptorq`. RaptorQ uses `u8` for source block number internally. + Cast it: `(block_id & 0xFF) as u8` or `(block_id % 256) as u8` — the `raptorq` + crate's source block ID is a logical identifier within a single object + transmission, not a global counter. The u16 is our session counter; truncate + to u8 when calling into raptorq. + +### Item B — `SignalMessage` version byte audit + +**File**: `crates/wzp-proto/src/packet.rs` + +Read every variant in the `SignalMessage` enum (lines 555–1241) and check +for the presence of: + +```rust +#[serde(default = "default_signal_version")] +version: u8, +``` + +The `Reflect` variant at line 974 is a **unit variant** (no fields). Unit +variants cannot carry a `version` field without becoming struct variants. +Change it to a struct variant: + +```rust +// Before: +Reflect, + +// After: +Reflect { + #[serde(default = "default_signal_version")] + version: u8, +}, +``` + +This is a wire-compatible change: serde JSON struct variants serialize as +`{"Reflect": {"version": 1}}` whereas unit variants serialize as +`"Reflect"`. These are **not** backward-compatible formats. Since `Reflect` +is sent client → relay only and the relay immediately responds, upgrading +both sides atomically is acceptable. Add a serde test to confirm round-trip. + +For any other variants missing `version`, follow the same pattern as all +existing variants. + +Verify by grepping the enum for variants that do NOT have `version`: + +```bash +grep -A3 "^\s*[A-Z][A-Za-z]*\s*{" crates/wzp-proto/src/packet.rs | \ + grep -B1 -v "serde.*default_signal_version\|version:" +``` + +### Item C — FEC repair index wrap (M4) + +**File**: `crates/wzp-fec/src/encoder.rs`, line ~140. + +Current code: + +```rust +let idx = (num_source as u16).wrapping_add(i as u16); +``` + +If this line already uses `u16` (as shown in the file at line 140), M4 is +already fixed. Verify by reading the current file. If it still reads +`u8`, apply: + +```rust +let idx = (num_source as u16).wrapping_add(i as u16); +``` + +**Decoder** (`crates/wzp-fec/src/decoder.rs`): `add_symbol` already accepts +`symbol_index: u16` (per the trait). Confirm the parameter flows through to +`PayloadId::new(block_id_u8, symbol_index as u32)` without truncation. + +## Implementation steps + +1. Read `crates/wzp-proto/src/traits.rs` lines 60–116 (FecEncoder/FecDecoder + trait definitions) to confirm current signatures. +2. Read `crates/wzp-fec/src/encoder.rs` and `decoder.rs` (full files). +3. Apply Item C fix first (smallest change, easiest to verify). +4. Apply Item A: widen `block_id` from u8 to u16 in traits, encoder, decoder. + Update all callers by running `cargo check -p wzp-fec -p wzp-proto` and + fixing each E0308/E0308 error. +5. Apply Item B: read every variant, add missing `version` fields. + Change `Reflect` to a struct variant. +6. Run tests. + +## Files to read before implementing + +- `crates/wzp-proto/src/traits.rs` lines 60–116 (trait signatures) +- `crates/wzp-fec/src/encoder.rs` (full) +- `crates/wzp-fec/src/decoder.rs` (full) +- `crates/wzp-proto/src/packet.rs` lines 555–1241 (all `SignalMessage` variants) + +## Verify + +```bash +cargo test -p wzp-fec -p wzp-proto +``` + +Expected: all tests pass, 0 failures. Also run: + +```bash +cargo check --workspace +``` + +to catch any call sites outside `wzp-fec` and `wzp-proto` that passed `u8` +block IDs to the trait methods. + +## Done when + +- `cargo test -p wzp-fec -p wzp-proto` exits 0. +- `block_id` is `u16` in `RaptorQFecEncoder`, `RaptorQFecDecoder`, and the + `FecEncoder`/`FecDecoder` traits. +- Every non-unit `SignalMessage` variant has a `version: u8` field with + `#[serde(default = "default_signal_version")]`. +- Repair index in `encoder.rs` is computed with `u16` arithmetic. +- No existing tests are broken. diff --git a/docs/bugs/002-macos-vpio-playout-silent.md b/docs/bugs/002-macos-vpio-playout-silent.md new file mode 100644 index 0000000..4fee3de --- /dev/null +++ b/docs/bugs/002-macos-vpio-playout-silent.md @@ -0,0 +1,165 @@ +# BUG-002: macOS VPIO Playout Silent — Audio Decoded But Not Heard + +**Severity:** P0 — outgoing audio (Mac mic → peer) works, but the user hears nothing on the Mac side +**Status:** Instrumented on 2026-05-25; awaiting next VPIO vs CPAL repro +**Branch:** `experimental-ui` +**Build observed:** `01f55ca` (Mac desktop), same-day Android `01f55ca` +**Last investigated:** 2026-05-25 +**Platforms confirmed affected:** macOS desktop (VPIO path) + +--- + +## Symptom + +In a relay-forwarded group call between macOS and Android in the same room (`General`, `count:2`): + +- The Mac user can be **heard** on Android (Mac→Android leg works). +- The Mac user **hears nothing** when the Android peer speaks (Android→Mac playout silent). +- Muting the Android peer's mic results in total silence on both ends — confirming the only audio the user perceived during the call was the Mac→Android leg playing through the Android speaker. + +This was initially misreported as "I hear myself on Android" — the user was hearing their own Mac mic looped through Android playout, not an actual echo bug. + +--- + +## Evidence + +### Mac log excerpt (`01f55ca`, fingerprint `63ba…`, 10:31:22) + +``` +10:31:23 media:room_update {"count":2, participants:[Akbar fa06…, Manwe 63ba…]} +10:31:23 media:first_recv {"codec":"Opus24k","payload_bytes":27,"t_ms":933} +10:31:25 media:recv_heartbeat {"codec":"Opus24k","decode_errs":0,"decoded_frames":140,"last_written":960,"written_samples":134400} +10:31:29 media:recv_heartbeat {"codec":"Opus32k","decoded_frames":338,"last_written":960,"written_samples":324480} +10:31:35 media:recv_heartbeat {"codec":"Opus6k","decoded_frames":595,"last_written":1920,"written_samples":618240} +… +10:31:57 media:recv_heartbeat {"codec":"Opus6k","decoded_frames":1086,"last_written":1920,"written_samples":1560960} +``` + +Recv path is healthy: +- `decode_errs:0` throughout +- `decoded_frames` climbs monotonically 140 → 1086 +- `written_samples` reaches 1.56 M (≈32 s of 48 kHz mono) +- `last_written` correctly flips 960 (Opus24k/32k, 20 ms) ↔ 1920 (Opus6k, 40 ms) + +**Conclusion:** packets arrive, decode succeeds, samples are written into `playout_ring`. The breakage is **downstream of the ring write**, i.e. in the macOS playout consumer (the VPIO `set_render_callback`). + +### Mac send path also works +`media:send_heartbeat` shows `last_rms` spiking to 168, 477, 867, 1458 in response to speech. Android's recv log for the same window decoded those frames successfully. + +--- + +## Suspected Root Cause + +`crates/wzp-client/src/audio_vpio.rs:128–147` — the render (output) callback reads from `playout_ring` in `FRAME_SAMPLES` (960) chunks. Three plausible failure modes: + +### Hypothesis A: Codec-change frame-size mismatch +Mid-call codec switches (`Opus24k` → `Opus32k` → `Opus6k`) change the frame size written into the ring (960 ↔ 1920 samples per frame). The render callback reads in fixed 960-sample chunks. The ring is FIFO and should absorb this, but if `AudioRing` semantics drop partial frames or stall on alignment, the consumer side could starve while `written_samples` continues to climb on the producer side. + +`engine.rs:1852` and `engine.rs:1895` write into `playout_ring` directly with the decoder's output length (variable). Worth confirming `AudioRing::read` handles arbitrary chunk sizes vs producer. + +### Hypothesis B: VPIO output element never actually started +`audio_vpio.rs:151` calls `au.start()` once on the combined VPIO unit. VPIO is supposed to start both input and output elements together, but if AEC initialization fails silently, output rendering may be suppressed while input still produces callbacks. The `[vpio] capture callback: N f32 samples` log line proves input callbacks fire — but there is **no equivalent log line for the render callback**. We do not know whether the render callback is being invoked at all. + +### Hypothesis C: Output device routing +VPIO may have grabbed an unexpected default output (e.g. the previous Bluetooth headset, an HDMI sink, or a virtual device). The render callback runs and pulls samples, but they're sent to a device the user can't hear. + +### Hypothesis D: AEC over-suppression +VPIO's AEC uses the render callback as the far-end reference. If the unit decides the far-end and near-end are too correlated (it shouldn't here — different speakers in different rooms), it could attenuate playout. Unlikely to produce 100 % silence but listed for completeness. + +--- + +## Instrumentation Added + +As of the current workspace, the desktop client emits VPIO render/capture counters into the normal call debug log when OS AEC is enabled: + +``` +vpio:render_heartbeat { + "capture_callbacks": ..., + "capture_samples": ..., + "render_callbacks": ..., + "render_requested_samples": ..., + "render_read_samples": ..., + "render_underrun_callbacks": ..., + "render_nonzero_callbacks": ..., + "render_last_requested": ..., + "render_last_read": ..., + "render_last_rms": ..., + "render_last_ring_available": ... +} +``` + +Interpretation: + +- `render_callbacks == 0`: VPIO output callback is not running. Focus on VPIO initialization / output element start. +- `render_callbacks > 0` and `render_read_samples == 0` while `media:recv_heartbeat.written_samples` climbs: VPIO callback runs but the ring it reads is not receiving the same samples the recv task writes, or the callback is draining before writes arrive. +- `render_read_samples > 0` and `render_last_rms > 0` while the user hears silence: VPIO is feeding non-zero speaker samples to CoreAudio; focus on output device routing or VoiceProcessingIO suppression. +- CPAL fallback test: disable OS AEC in settings. If CPAL playback is audible with the same call, the failure is VPIO-specific. + +## Proposed Diagnostic Steps (Prioritized) + +1. **Reproduce with current instrumentation** and compare `media:recv_heartbeat` to `vpio:render_heartbeat`. + +2. **One-shot render callback stderr log is now present** (`audio_vpio.rs`) mirroring the existing capture-side `eprintln!`: + ```rust + let logged_render = Arc::new(AtomicBool::new(false)); + … + if !logged_render.swap(true, Ordering::Relaxed) { + eprintln!("[vpio] render callback: {} f32 samples, ring_read={}", ch.len(), read); + } + ``` + This will immediately distinguish Hypothesis B (callback never fires) from A/C/D (callback fires but output is silent or misrouted). + +3. **Periodically log render-callback stats** — total samples pulled from ring, samples requested per callback, non-zero render callback count, and last render RMS. Compare against producer-side `written_samples` to confirm consumer is keeping up. + +4. **Verify output device** via `AudioUnitGetProperty(kAudioOutputUnitProperty_CurrentDevice, Output)` immediately after `au.start()`. Log device name. If it doesn't match the user's intended speaker, force-set the default output device. + +5. **Test with codec pinned** — set `WZP_FORCE_CODEC=Opus24k` (or wire a temporary CLI flag) so codec doesn't change mid-call. If audio works with a pinned codec, Hypothesis A is confirmed and `AudioRing` chunk handling needs review. + +6. **Compare CPAL fallback path** — disable OS AEC in settings and reproduce. If CPAL playback works, the bug is VPIO-specific. + +--- + +## Open Questions + +- Does the macOS render callback have permission to write to the user's selected output device? Apple changed CoreAudio output-device permission semantics in macOS 14+. +- Is `_audio_unit: AudioUnit` being dropped early? It's stored in `VpioAudio` and that struct is boxed into `audio_handle: Box` in `engine.rs:1573`, which is held by `CallEngine`. Should be alive for the call duration — confirm no early-drop path. +- Are there any `os_log` / Console.app warnings from `AudioToolbox` / `CoreAudio` / `AVAudioSession` during the call? + +--- + +## Reproduction Steps + +1. Start macOS desktop client (build `01f55ca` or later), join relay `193.180.213.68:4433`, room `General`. +2. Start Android client (same build), join same relay + room. +3. Confirm `media:room_update count:2` on both ends. +4. Speak into Android mic. +5. Observe: Mac log shows `decoded_frames` climbing, `decode_errs:0`, `written_samples` increasing. User hears nothing on Mac speakers. +6. Speak into Mac mic — Android user hears Mac audio fine, confirming Mac→Android works. + +--- + +## Related Files + +- `crates/wzp-client/src/audio_vpio.rs:128–147` — render callback (primary suspect) +- `crates/wzp-client/src/audio_vpio.rs:35–161` — full VPIO start sequence +- `crates/wzp-client/src/audio_ring.rs` — ring buffer used by both producer and consumer +- `desktop/src-tauri/src/engine.rs:1562–1600` — VPIO vs CPAL selection +- `desktop/src-tauri/src/engine.rs:1760–1900` — recv task writing into `playout_ring` + +--- + +## Fix Plan (Once Diagnosed) + +| Diagnosis | Fix | +|-----------|-----| +| A — frame-size mismatch | Make `AudioRing` consumer drain variable chunks, or buffer to fixed 960 in recv task before ring write | +| B — render callback not firing | Investigate VPIO initialization order; consider separate input + output `AudioUnit` instances | +| C — wrong output device | Set `kAudioOutputUnitProperty_CurrentDevice` explicitly to `kAudioObjectSystemObject` default output at start | +| D — AEC suppression | Test with VPIO bypass mode (`kAUVoiceIOProperty_BypassVoiceProcessing`) on; if audio returns, file CoreAudio quirk and tune AEC config | + +--- + +## Cross-References + +- BUG-001 (Android join-voice hang) — separate issue, already mitigated; current Android build joins room successfully and recv works. +- Memory: `project_desktop_client.md` notes the desktop rewrite uses CPAL + VoiceProcessingIO with "direct playout, OS-level AEC" — this bug is the first failure of that path under real call conditions. From 8002acaf09c5057248a4e7319d3ad29c2a1a6902 Mon Sep 17 00:00:00 2001 From: Siavash Sameni Date: Mon, 25 May 2026 15:30:41 +0400 Subject: [PATCH 96/96] fix(scripts): stage android-build-async.sh and featherchat submodule Co-Authored-By: Claude Sonnet 4.6 --- scripts/android-build-async.sh | 164 ++++++++++++++++----------------- 1 file changed, 77 insertions(+), 87 deletions(-) diff --git a/scripts/android-build-async.sh b/scripts/android-build-async.sh index 6ba3450..1a8e673 100755 --- a/scripts/android-build-async.sh +++ b/scripts/android-build-async.sh @@ -1,122 +1,112 @@ #!/usr/bin/env bash # Fire-and-forget Android APK builder. # -# Uploads the build script to SepehrHomeserverdk, starts it in a tmux -# session so it survives SSH disconnects, then exits immediately. -# Progress and the finished APK URL arrive via ntfy.sh/wzp. +# Runs ./scripts/build-tauri-android.sh inside a LOCAL tmux session so the +# build survives terminal disconnects. The wrapped script SSHes to +# SepehrHomeserverdk on its own — we don't try to upload+run anything on +# the remote (that would re-SSH from the remote to itself, which fails). # # Usage: # ./scripts/android-build-async.sh # build current branch, arm64 # ./scripts/android-build-async.sh --init # also run cargo tauri android init # ./scripts/android-build-async.sh --rust # force-clean Rust target cache # ./scripts/android-build-async.sh --no-pull # skip git fetch on remote -# ./scripts/android-build-async.sh --wait # block until done, then download APK +# ./scripts/android-build-async.sh --debug # debug APK +# ./scripts/android-build-async.sh --wait # block until done, then tail status # -# When the build finishes, ntfy.sh/wzp will show: -# "WZP Tauri arm64 [] ready! " -# or on failure: -# "WZP Tauri Android build FAILED [] (line N) log: " +# Progress / completion: ntfy.sh/wzp (handled by build-tauri-android.sh). +# Monitor locally: tmux attach -t wzp-android-local +# tail -f /tmp/wzp-tauri-build-local.log set -euo pipefail -REMOTE_HOST="SepehrHomeserverdk" -NTFY_TOPIC="https://ntfy.sh/wzp" -LOCAL_OUTPUT="target/tauri-android-apk" -TMUX_SESSION="wzp-android" -REMOTE_LOG="/tmp/wzp-tauri-build.log" -SSH_OPTS="-o ConnectTimeout=15 -o ServerAliveInterval=30 -o ServerAliveCountMax=6 -o LogLevel=ERROR" +TMUX_SESSION="wzp-android-local" +LOCAL_LOG="/tmp/wzp-tauri-build-local.log" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +BUILD_SCRIPT="$SCRIPT_DIR/build-tauri-android.sh" -BRANCH="${WZP_BRANCH:-$(git -C "$(dirname "$0")/.." branch --show-current 2>/dev/null || echo "")}" -DO_PULL=1 -DO_INIT=0 -BUILD_RELEASE=1 -REBUILD_RUST=0 -BUILD_ARCH="arm64" -DO_WAIT=0 +if ! command -v tmux >/dev/null 2>&1; then + echo "ERROR: tmux is not installed locally. Install with: brew install tmux" + exit 1 +fi -for arg in "$@"; do - case "$arg" in - --pull) DO_PULL=1 ;; - --no-pull) DO_PULL=0 ;; - --init) DO_INIT=1 ;; - --debug) BUILD_RELEASE=0 ;; - --rust) REBUILD_RUST=1 ;; - --wait) DO_WAIT=1 ;; - esac -done +if [ ! -x "$BUILD_SCRIPT" ]; then + echo "ERROR: $BUILD_SCRIPT not found or not executable" + exit 1 +fi +BRANCH="${WZP_BRANCH:-$(git -C "$REPO_DIR" branch --show-current 2>/dev/null || echo "")}" if [ -z "$BRANCH" ]; then echo "ERROR: could not determine branch (detached HEAD?). Set WZP_BRANCH=name." exit 1 fi -log() { echo -e "\033[1;36m>>> $*\033[0m"; } -err() { echo -e "\033[1;31mERROR: $*\033[0m" >&2; } -ssh_q() { ssh $SSH_OPTS "$REMOTE_HOST" "$@"; } +DO_WAIT=0 +PASS_ARGS=() +for arg in "$@"; do + case "$arg" in + --wait) DO_WAIT=1 ;; + *) PASS_ARGS+=("$arg") ;; + esac +done -# ── Step 1: upload the remote build script ────────────────────────────────── -log "Uploading build script to $REMOTE_HOST..." -# Re-use the existing full build script (it already handles all logic). -scp $SSH_OPTS "$(dirname "$0")/build-tauri-android.sh" "$REMOTE_HOST:/tmp/wzp-tauri-build-full.sh" -ssh_q "chmod +x /tmp/wzp-tauri-build-full.sh" +log() { echo -e "\033[1;36m>>> $*\033[0m"; } -# ── Step 2: launch in tmux (detached) ────────────────────────────────────── -log "Starting build in tmux session '$TMUX_SESSION' on $REMOTE_HOST..." -ssh_q "tmux kill-session -t $TMUX_SESSION 2>/dev/null; true" +# Kill any prior session that might still be hanging around. +tmux kill-session -t "$TMUX_SESSION" 2>/dev/null || true -# The full script accepts flags directly; pass them through. -REMOTE_FLAGS="" -[ "$DO_PULL" = "1" ] || REMOTE_FLAGS="$REMOTE_FLAGS --no-pull" -[ "$DO_INIT" = "1" ] && REMOTE_FLAGS="$REMOTE_FLAGS --init" -[ "$BUILD_RELEASE" = "0" ] && REMOTE_FLAGS="$REMOTE_FLAGS --debug" -[ "$REBUILD_RUST" = "1" ] && REMOTE_FLAGS="$REMOTE_FLAGS --rust" +# Write a launcher script — avoids fragile quoting inside `tmux new-session`. +LAUNCHER="$(mktemp -t wzp-android-launcher.XXXXXX)" +chmod +x "$LAUNCHER" +{ + echo "#!/usr/bin/env bash" + echo "set -o pipefail" + echo "cd $(printf %q "$REPO_DIR")" + echo "export WZP_BRANCH=$(printf %q "$BRANCH")" + printf 'bash %q' "$BUILD_SCRIPT" + for a in "${PASS_ARGS[@]:-}"; do + [ -z "$a" ] && continue + printf ' %q' "$a" + done + echo " 2>&1 | tee $(printf %q "$LOCAL_LOG")" + echo "echo DONE_EXIT_CODE=\$? >> $(printf %q "$LOCAL_LOG")" +} > "$LAUNCHER" -# Run via WZP_BRANCH so the remote script picks up the right branch -# (it calls `git branch --show-current` which would return the remote's -# currently checked-out branch, not necessarily the one we want). -ssh_q "tmux new-session -d -s $TMUX_SESSION \ - 'WZP_BRANCH=$BRANCH bash /tmp/wzp-tauri-build-full.sh $REMOTE_FLAGS \ - 2>&1 | tee $REMOTE_LOG; echo DONE_EXIT_CODE=\$? >> $REMOTE_LOG'" +# Create the log file up front so `tail -f` works immediately. +: > "$LOCAL_LOG" -log "Build dispatched! Notification on ntfy.sh/wzp when done." +log "Starting local tmux session '$TMUX_SESSION' (branch: $BRANCH)..." +log "Build script: $BUILD_SCRIPT ${PASS_ARGS[*]:-}" +log "Launcher: $LAUNCHER" +log "Local log: $LOCAL_LOG" + +tmux new-session -d -s "$TMUX_SESSION" -c "$REPO_DIR" "bash $LAUNCHER; exec bash" + +# Verify the session actually started. +sleep 1 +if ! tmux has-session -t "$TMUX_SESSION" 2>/dev/null; then + echo "ERROR: tmux session '$TMUX_SESSION' failed to start. Launcher contents:" + cat "$LAUNCHER" + exit 1 +fi + +log "Build dispatched! ntfy.sh/wzp will notify on completion." echo "" -echo " Monitor : ssh $REMOTE_HOST 'tail -f $REMOTE_LOG'" -echo " Status : ssh $REMOTE_HOST 'tail -5 $REMOTE_LOG'" -echo " Attach : ssh $REMOTE_HOST 'tmux attach -t $TMUX_SESSION'" +echo " Monitor : tail -f $LOCAL_LOG" +echo " Status : tail -5 $LOCAL_LOG" +echo " Attach : tmux attach -t $TMUX_SESSION" +echo " Kill : tmux kill-session -t $TMUX_SESSION" echo "" -# ── Step 3 (optional --wait): block until done, download APK ─────────────── if [ "$DO_WAIT" = "0" ]; then exit 0 fi -log "Waiting for build to finish (monitoring $REMOTE_LOG)..." -ssh_q "until grep -qE 'APK_REMOTE_PATH|FAILED|ERROR|DONE_EXIT_CODE' \ - $REMOTE_LOG 2>/dev/null; do sleep 20; done" - -# Check for failure -if ssh_q "grep -q 'FAILED\|ERROR' $REMOTE_LOG 2>/dev/null" && \ - ! ssh_q "grep -q 'APK_REMOTE_PATH' $REMOTE_LOG 2>/dev/null"; then - err "Build failed — check ntfy or: ssh $REMOTE_HOST 'cat $REMOTE_LOG'" - exit 1 -fi - -# Grab APK paths from log -APK_REMOTES=$(ssh_q "grep '^APK_REMOTE_PATH=' $REMOTE_LOG | cut -d= -f2-") -if [ -z "$APK_REMOTES" ]; then - err "No APK_REMOTE_PATH in log — build may have failed silently" - ssh_q "tail -20 $REMOTE_LOG" >&2 - exit 1 -fi - -mkdir -p "$LOCAL_OUTPUT" -echo "$APK_REMOTES" | while IFS= read -r REMOTE_PATH; do - [ -z "$REMOTE_PATH" ] && continue - APK_NAME=$(basename "$REMOTE_PATH") - log "Downloading $APK_NAME..." - scp $SSH_OPTS "$REMOTE_HOST:$REMOTE_PATH" "$LOCAL_OUTPUT/$APK_NAME" - echo " $LOCAL_OUTPUT/$APK_NAME ($(du -h "$LOCAL_OUTPUT/$APK_NAME" | cut -f1))" +log "Waiting for build to finish (watching $LOCAL_LOG)..." +until grep -qE 'DONE_EXIT_CODE|APK_REMOTE_PATH=|FAILED' "$LOCAL_LOG" 2>/dev/null; do + sleep 20 done -log "Done! APKs in $LOCAL_OUTPUT/" -ls -lh "$LOCAL_OUTPUT"/wzp-tauri-*.apk 2>/dev/null || true +log "Build session ended. Last 20 lines:" +tail -20 "$LOCAL_LOG"