manawenuz/wz-phone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(video): skip startup black frames
Some checks failed
Mirror to GitHub / mirror (push) Failing after 29s
Details
Build Release Binaries / build-amd64 (push) Failing after 3m2s
Details

Browse Source
This commit is contained in:
Siavash Sameni
2026-05-25 21:35:00 +04:00
parent d2046060b5
commit ee654cd1ef
2 changed files with 158 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
desktop/src-tauri/src/engine.rs
View File

@@ -181,6 +181,22 @@ fn should_log_video_sample(frame_no: u64, is_keyframe: bool) -> bool {
frame_no <= 5 || is_keyframe || frame_no % 30 == 0 frame_no <= 5 || is_keyframe || frame_no % 30 == 0
} }
fn is_startup_black_i420(data: &[u8], width: u32, height: u32) -> bool {
let y_size = width as usize * height as usize;
let uv_size = y_size / 4;
if data.len() < y_size + uv_size * 2 {
return false;
}
let y = &data[..y_size];
let u = &data[y_size..y_size + uv_size];
let v = &data[y_size + uv_size..y_size + uv_size * 2];
let y_nonblack = y.iter().any(|&b| b > 3);
let u_chroma = u.iter().any(|&b| !(124..=132).contains(&b));
let v_chroma = v.iter().any(|&b| !(124..=132).contains(&b));
!y_nonblack && !u_chroma && !v_chroma
}
/// Resolve a quality string from the UI to a QualityProfile. /// Resolve a quality string from the UI to a QualityProfile.
/// Returns None for "auto" (use default adaptive behavior). /// Returns None for "auto" (use default adaptive behavior).
fn resolve_quality(quality: &str) -> Option<QualityProfile> { fn resolve_quality(quality: &str) -> Option<QualityProfile> {
@@ -689,7 +705,10 @@ impl CallEngine {
// through the signal channel (DirectCallOffer/Answer carry // through the signal channel (DirectCallOffer/Answer carry
// identity_pub + ephemeral_pub + signature). // identity_pub + ephemeral_pub + signature).
let quinn_transport = transport.clone(); let quinn_transport = transport.clone();
let (_negotiated_video_codec, transport): (Option<wzp_proto::CodecId>, Arc<dyn wzp_proto::MediaTransport>) = if !is_direct_p2p { let (_negotiated_video_codec, transport): (
Option<wzp_proto::CodecId>,
Arc<dyn wzp_proto::MediaTransport>,
) = if !is_direct_p2p {
crate::emit_call_debug( crate::emit_call_debug(
&app, &app,
"connect:handshake_start", "connect:handshake_start",
@@ -1114,9 +1133,7 @@ impl CallEngine {
snap.lost_packets, snap.lost_packets,
snap.loss_pct, snap.loss_pct,
); );
if let Some(tuning) = if let Some(tuning) = dred_tuner.update(win_loss, snap.rtt_ms, pq.jitter_ms) {
dred_tuner.update(win_loss, snap.rtt_ms, pq.jitter_ms)
{
encoder.apply_dred_tuning(tuning); encoder.apply_dred_tuning(tuning);
if wzp_codec::dred_verbose_logs() { if wzp_codec::dred_verbose_logs() {
info!( info!(
@@ -1306,9 +1323,7 @@ impl CallEngine {
}), }),
); );
} }
if let Some((codec_id, is_kf, frame)) = if let Some((codec_id, is_kf, frame)) = video_reassembler.push(&pkt) {
video_reassembler.push(&pkt)
{
video_reassembled_samples += 1; video_reassembled_samples += 1;
if !video_first_reassembled_logged { if !video_first_reassembled_logged {
video_first_reassembled_logged = true; video_first_reassembled_logged = true;
@@ -1352,7 +1367,9 @@ impl CallEngine {
"platform": "android", "platform": "android",
}), }),
); );
match wzp_video::factory::create_video_decoder(codec_id, 1280, 720) { match wzp_video::factory::create_video_decoder(
codec_id, 1280, 720,
) {
Ok(d) => { Ok(d) => {
info!(codec = ?codec_id, "video decoder created (android)"); info!(codec = ?codec_id, "video decoder created (android)");
crate::emit_call_debug( crate::emit_call_debug(
@@ -1415,7 +1432,8 @@ impl CallEngine {
}), }),
); );
} }
if should_log_video_sample(video_decoded_samples, is_kf) { if should_log_video_sample(video_decoded_samples, is_kf)
{
crate::emit_call_debug( crate::emit_call_debug(
&recv_app, &recv_app,
"video:decoded_frame_sample", "video:decoded_frame_sample",
@@ -1882,9 +1900,9 @@ impl CallEngine {
"platform": "android", "platform": "android",
}), }),
); );
let mut encoder = match wzp_video::factory::create_video_encoder( let mut encoder =
vid_codec, 1280, 720, 1_500_000, match wzp_video::factory::create_video_encoder(vid_codec, 1280, 720, 1_500_000)
) { {
Ok(e) => { Ok(e) => {
crate::emit_call_debug( crate::emit_call_debug(
&vid_app, &vid_app,
@@ -1919,6 +1937,7 @@ impl CallEngine {
let mut camera_frames: u64 = 0; let mut camera_frames: u64 = 0;
let mut empty_encodes: u64 = 0; let mut empty_encodes: u64 = 0;
let mut encoded_frame_samples: u64 = 0; let mut encoded_frame_samples: u64 = 0;
let mut skipped_startup_black_frames: u64 = 0;
let mut wait_ticks: u64 = 0; let mut wait_ticks: u64 = 0;
encoder.request_keyframe(); encoder.request_keyframe();
crate::emit_call_debug( crate::emit_call_debug(
@@ -1995,6 +2014,30 @@ impl CallEngine {
} }
}; };
if !first_send_logged
&& skipped_startup_black_frames < 30
&& is_startup_black_i420(&frame.data, frame.width, frame.height)
{
skipped_startup_black_frames += 1;
encoder.request_keyframe();
if skipped_startup_black_frames == 1
|| skipped_startup_black_frames % 10 == 0
{
crate::emit_call_debug(
&vid_app,
"video:startup_black_frame_skipped",
serde_json::json!({
"t_ms": vid_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", vid_codec),
"camera_frames": camera_frames,
"skipped": skipped_startup_black_frames,
"platform": "android",
}),
);
}
continue;
}
if frames_since_keyframe >= 150 { if frames_since_keyframe >= 150 {
encoder.request_keyframe(); encoder.request_keyframe();
crate::emit_call_debug( crate::emit_call_debug(
@@ -2050,7 +2093,11 @@ impl CallEngine {
let is_keyframe = encoder.is_keyframe(&encoded); let is_keyframe = encoder.is_keyframe(&encoded);
let ts_ms = vid_t0.elapsed().as_millis() as u32; let ts_ms = vid_t0.elapsed().as_millis() as u32;
let pkts = wzp_video::transport::packetize_video_frame( let pkts = wzp_video::transport::packetize_video_frame(
&encoded, vid_codec, is_keyframe, &mut seq, ts_ms, &encoded,
vid_codec,
is_keyframe,
&mut seq,
ts_ms,
); );
if encoded_frame_samples < 5 { if encoded_frame_samples < 5 {
encoded_frame_samples += 1; encoded_frame_samples += 1;
@@ -2495,9 +2542,7 @@ impl CallEngine {
snap.lost_packets, snap.lost_packets,
snap.loss_pct, snap.loss_pct,
); );
if let Some(tuning) = if let Some(tuning) = dred_tuner.update(win_loss, snap.rtt_ms, pq.jitter_ms) {
dred_tuner.update(win_loss, snap.rtt_ms, pq.jitter_ms)
{
encoder.apply_dred_tuning(tuning); encoder.apply_dred_tuning(tuning);
} }
} }
@@ -2613,9 +2658,7 @@ impl CallEngine {
}), }),
); );
} }
if let Some((codec_id, is_kf, frame)) = if let Some((codec_id, is_kf, frame)) = video_reassembler.push(&pkt) {
video_reassembler.push(&pkt)
{
video_reassembled_samples += 1; video_reassembled_samples += 1;
if !video_first_reassembled_logged { if !video_first_reassembled_logged {
video_first_reassembled_logged = true; video_first_reassembled_logged = true;
@@ -2660,7 +2703,9 @@ impl CallEngine {
"platform": "desktop", "platform": "desktop",
}), }),
); );
match wzp_video::factory::create_video_decoder(codec_id, 1280, 720) { match wzp_video::factory::create_video_decoder(
codec_id, 1280, 720,
) {
Ok(d) => { Ok(d) => {
info!(codec = ?codec_id, "video decoder created"); info!(codec = ?codec_id, "video decoder created");
crate::emit_call_debug( crate::emit_call_debug(
@@ -2726,7 +2771,8 @@ impl CallEngine {
}), }),
); );
} }
if should_log_video_sample(video_decoded_samples, is_kf) { if should_log_video_sample(video_decoded_samples, is_kf)
{
crate::emit_call_debug( crate::emit_call_debug(
&recv_app, &recv_app,
"video:decoded_frame_sample", "video:decoded_frame_sample",
@@ -2807,10 +2853,7 @@ impl CallEngine {
} }
} }
// Evict stale partial frames every ~10 frames received. // Evict stale partial frames every ~10 frames received.
video_reassembler.evict_stale( video_reassembler.evict_stale(pkt.header.timestamp, 5_000);
pkt.header.timestamp,
5_000,
);
} }
continue; // video packet handled — skip audio path continue; // video packet handled — skip audio path
} }
@@ -3039,9 +3082,9 @@ impl CallEngine {
"platform": "desktop", "platform": "desktop",
}), }),
); );
let mut encoder = match wzp_video::factory::create_video_encoder( let mut encoder =
vid_codec, 1280, 720, 1_500_000, match wzp_video::factory::create_video_encoder(vid_codec, 1280, 720, 1_500_000)
) { {
Ok(e) => { Ok(e) => {
crate::emit_call_debug( crate::emit_call_debug(
&vid_app, &vid_app,
@@ -3076,6 +3119,7 @@ impl CallEngine {
let mut camera_frames: u64 = 0; let mut camera_frames: u64 = 0;
let mut empty_encodes: u64 = 0; let mut empty_encodes: u64 = 0;
let mut encoded_frame_samples: u64 = 0; let mut encoded_frame_samples: u64 = 0;
let mut skipped_startup_black_frames: u64 = 0;
let mut wait_ticks: u64 = 0; let mut wait_ticks: u64 = 0;
encoder.request_keyframe(); encoder.request_keyframe();
crate::emit_call_debug( crate::emit_call_debug(
@@ -3152,6 +3196,30 @@ impl CallEngine {
} }
}; };
if !first_send_logged
&& skipped_startup_black_frames < 30
&& is_startup_black_i420(&frame.data, frame.width, frame.height)
{
skipped_startup_black_frames += 1;
encoder.request_keyframe();
if skipped_startup_black_frames == 1
|| skipped_startup_black_frames % 10 == 0
{
crate::emit_call_debug(
&vid_app,
"video:startup_black_frame_skipped",
serde_json::json!({
"t_ms": vid_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", vid_codec),
"camera_frames": camera_frames,
"skipped": skipped_startup_black_frames,
"platform": "desktop",
}),
);
}
continue;
}
if frames_since_keyframe >= 150 { if frames_since_keyframe >= 150 {
encoder.request_keyframe(); encoder.request_keyframe();
crate::emit_call_debug( crate::emit_call_debug(
@@ -3207,7 +3275,11 @@ impl CallEngine {
let is_keyframe = encoder.is_keyframe(&encoded); let is_keyframe = encoder.is_keyframe(&encoded);
let ts_ms = vid_t0.elapsed().as_millis() as u32; let ts_ms = vid_t0.elapsed().as_millis() as u32;
let pkts = wzp_video::transport::packetize_video_frame( let pkts = wzp_video::transport::packetize_video_frame(
&encoded, vid_codec, is_keyframe, &mut seq, ts_ms, &encoded,
vid_codec,
is_keyframe,
&mut seq,
ts_ms,
); );
if encoded_frame_samples < 5 { if encoded_frame_samples < 5 {
encoded_frame_samples += 1; encoded_frame_samples += 1;
@@ -3383,7 +3455,6 @@ impl Drop for CallEngine {
} }
} }
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use std::sync::{Arc, Mutex as StdMutex}; use std::sync::{Arc, Mutex as StdMutex};

3
desktop/src/main.ts
View File

@@ -605,10 +605,12 @@ const remoteCtx = vdRemoteVideo.getContext("2d")!;
const vdRemotePlaceholder = document.getElementById("vd-remote-placeholder")!; const vdRemotePlaceholder = document.getElementById("vd-remote-placeholder")!;
const vdRemoteCounter = document.getElementById("vd-remote-counter")!; const vdRemoteCounter = document.getElementById("vd-remote-counter")!;
let remoteFrameCount = 0; let remoteFrameCount = 0;
let remoteFrameSerial = 0;
listen("video:frame", (event: any) => { listen("video:frame", (event: any) => {
const { width, height, jpeg_b64 } = event.payload; const { width, height, jpeg_b64 } = event.payload;
if (!jpeg_b64) return; if (!jpeg_b64) return;
const frameSerial = ++remoteFrameSerial;
remoteVideoActive = true; remoteVideoActive = true;
vdVideoStrip.classList.remove("hidden"); vdVideoStrip.classList.remove("hidden");
@@ -620,6 +622,7 @@ listen("video:frame", (event: any) => {
const img = new Image(); const img = new Image();
img.onload = () => { img.onload = () => {
if (frameSerial !== remoteFrameSerial) return;
remoteCtx.drawImage(img, 0, 0, vdRemoteVideo.width, vdRemoteVideo.height); remoteCtx.drawImage(img, 0, 0, vdRemoteVideo.width, vdRemoteVideo.height);
}; };
img.src = `data:image/jpeg;base64,${jpeg_b64}`; img.src = `data:image/jpeg;base64,${jpeg_b64}`;