manawenuz/wz-phone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(video): add frame metadata and Android encode diagnostics
Some checks failed
Mirror to GitHub / mirror (push) Failing after 41s
Details
Build Release Binaries / build-amd64 (push) Failing after 4m7s
Details

Browse Source
This commit is contained in:
Siavash Sameni
2026-05-26 11:28:17 +04:00
parent 9a7745978b
commit 112472609e
8 changed files with 574 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
desktop/src-tauri/src/engine.rs
View File

@@ -1385,7 +1385,7 @@ impl CallEngine {
// Video pipeline state — mirror of the desktop recv task.
let mut video_reassembler = wzp_video::transport::VideoReassembler::new();
let mut video_decoder: Option<Box<dyn wzp_video::decoder::VideoDecoder>> = None;
let mut video_decoder_codec: Option<wzp_proto::CodecId> = None;
let mut video_decoder_key: Option<(wzp_proto::CodecId, u32, u32)> = None;
let mut video_first_recv_logged = false;
let mut video_first_reassembled_logged = false;
let mut video_reassembled_samples: u64 = 0;
@@ -1466,7 +1466,14 @@ impl CallEngine {
}),
);
}
if let Some((codec_id, is_kf, frame)) = video_reassembler.push(&pkt) {
if let Some(reassembled) = video_reassembler.push(&pkt) {
let codec_id = reassembled.codec_id;
let is_kf = reassembled.is_keyframe;
let frame_width =
reassembled.width.unwrap_or(video_width as u16) as u32;
let frame_height =
reassembled.height.unwrap_or(video_height as u16) as u32;
let frame = reassembled.data;
video_reassembled_samples += 1;
if !video_first_reassembled_logged {
video_first_reassembled_logged = true;
@@ -1484,6 +1491,14 @@ impl CallEngine {
);
}
if should_log_video_sample(video_reassembled_samples, is_kf) {
crate::maybe_dump_video_bytes(
&recv_app,
"remote_encoded_reassembled",
"android",
video_reassembled_samples,
&frame,
codec_id,
);
crate::emit_call_debug(
&recv_app,
"video:reassembled_frame",
@@ -1517,22 +1532,23 @@ impl CallEngine {
video_reassembler.evict_stale(pkt.header.timestamp, 5_000);
continue;
}
if video_decoder_codec != Some(codec_id) {
let decoder_key = (codec_id, frame_width, frame_height);
if video_decoder_key != Some(decoder_key) {
crate::emit_call_debug(
&recv_app,
"video:decoder_init_start",
serde_json::json!({
"t_ms": recv_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", codec_id),
"width": video_width,
"height": video_height,
"width": frame_width,
"height": frame_height,
"platform": "android",
}),
);
match wzp_video::factory::create_video_decoder(
codec_id,
video_width,
video_height,
frame_width,
frame_height,
) {
Ok(d) => {
info!(codec = ?codec_id, "video decoder created (android)");
@@ -1546,7 +1562,7 @@ impl CallEngine {
}),
);
video_decoder = Some(d);
video_decoder_codec = Some(codec_id);
video_decoder_key = Some(decoder_key);
}
Err(e) => {
error!("video decoder init failed: {e}");
@@ -2304,11 +2320,21 @@ impl CallEngine {
is_keyframe,
&mut seq,
ts_ms,
frame.width,
frame.height,
);
video_packets_total += pkts.len() as u64;
video_bytes_total += encoded.len() as u64;
if encoded_frame_samples < 5 {
encoded_frame_samples += 1;
crate::maybe_dump_video_bytes(
&vid_app,
"local_encoded",
"android",
encoded_frame_samples,
&encoded,
vid_codec,
);
let packet_payload_bytes: usize =
pkts.iter().map(|pkt| pkt.payload.len()).sum();
crate::emit_call_debug(
@@ -2870,7 +2896,7 @@ impl CallEngine {
let mut first_packet_logged = false;
let mut video_reassembler = wzp_video::transport::VideoReassembler::new();
let mut video_decoder: Option<Box<dyn wzp_video::decoder::VideoDecoder>> = None;
let mut video_decoder_codec: Option<wzp_proto::CodecId> = None;
let mut video_decoder_key: Option<(wzp_proto::CodecId, u32, u32)> = None;
let mut video_first_recv_logged_desktop = false;
let mut video_first_reassembled_logged = false;
let mut video_reassembled_samples: u64 = 0;
@@ -2956,7 +2982,14 @@ impl CallEngine {
}),
);
}
if let Some((codec_id, is_kf, frame)) = video_reassembler.push(&pkt) {
if let Some(reassembled) = video_reassembler.push(&pkt) {
let codec_id = reassembled.codec_id;
let is_kf = reassembled.is_keyframe;
let frame_width =
reassembled.width.unwrap_or(video_width as u16) as u32;
let frame_height =
reassembled.height.unwrap_or(video_height as u16) as u32;
let frame = reassembled.data;
video_reassembled_samples += 1;
if !video_first_reassembled_logged {
video_first_reassembled_logged = true;
@@ -2974,6 +3007,14 @@ impl CallEngine {
);
}
if should_log_video_sample(video_reassembled_samples, is_kf) {
crate::maybe_dump_video_bytes(
&recv_app,
"remote_encoded_reassembled",
"desktop",
video_reassembled_samples,
&frame,
codec_id,
);
crate::emit_call_debug(
&recv_app,
"video:reassembled_frame",
@@ -3008,22 +3049,23 @@ impl CallEngine {
continue;
}
// Lazy-init or switch decoder on codec change.
if video_decoder_codec != Some(codec_id) {
let decoder_key = (codec_id, frame_width, frame_height);
if video_decoder_key != Some(decoder_key) {
crate::emit_call_debug(
&recv_app,
"video:decoder_init_start",
serde_json::json!({
"t_ms": recv_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", codec_id),
"width": video_width,
"height": video_height,
"width": frame_width,
"height": frame_height,
"platform": "desktop",
}),
);
match wzp_video::factory::create_video_decoder(
codec_id,
video_width,
video_height,
frame_width,
frame_height,
) {
Ok(d) => {
info!(codec = ?codec_id, "video decoder created");
@@ -3037,7 +3079,7 @@ impl CallEngine {
}),
);
video_decoder = Some(d);
video_decoder_codec = Some(codec_id);
video_decoder_key = Some(decoder_key);
}
Err(e) => {
error!("video decoder init failed: {e}");
@@ -3640,11 +3682,21 @@ impl CallEngine {
is_keyframe,
&mut seq,
ts_ms,
frame.width,
frame.height,
);
video_packets_total += pkts.len() as u64;
video_bytes_total += encoded.len() as u64;
if encoded_frame_samples < 5 {
encoded_frame_samples += 1;
crate::maybe_dump_video_bytes(
&vid_app,
"local_encoded",
"desktop",
encoded_frame_samples,
&encoded,
vid_codec,
);
let packet_payload_bytes: usize =
pkts.iter().map(|pkt| pkt.payload.len()).sum();
crate::emit_call_debug(

55
desktop/src-tauri/src/lib.rs
View File

@@ -205,6 +205,61 @@ pub(crate) fn maybe_dump_video_jpeg(
}
}
pub(crate) fn maybe_dump_video_bytes(
app: &tauri::AppHandle,
stage: &str,
platform: &str,
frame_no: u64,
bytes: &[u8],
codec: wzp_proto::CodecId,
) {
if !should_dump_frame(frame_no) || bytes.is_empty() {
return;
}
let ext = match codec {
wzp_proto::CodecId::H265Main => "h265",
wzp_proto::CodecId::Av1Main => "obu",
_ => "h264",
};
let seq = FRAME_DUMP_WRITES.fetch_add(1, Ordering::Relaxed) + 1;
let dir = identity_dir().join("frame-dumps");
let file_name = format!("{seq:06}_{platform}_{stage}_f{frame_no:06}.{ext}");
let path = dir.join(file_name);
let result = std::fs::create_dir_all(&dir).and_then(|_| std::fs::write(&path, bytes));
match result {
Ok(()) => emit_call_debug(
app,
"video:byte_dump",
serde_json::json!({
"stage": stage,
"platform": platform,
"frame_no": frame_no,
"codec": format!("{:?}", codec),
"bytes": bytes.len(),
"path": path,
}),
),
Err(e) => {
if seq <= 5 || seq % 30 == 0 {
emit_call_debug(
app,
"video:byte_dump_failed",
serde_json::json!({
"stage": stage,
"platform": platform,
"frame_no": frame_no,
"codec": format!("{:?}", codec),
"error": e.to_string(),
"path": path,
}),
);
}
}
}
}
/// RGB24 → I420 (planar 4:2:0). Layout: Y(w×h) | U(w/2×h/2) | V(w/2×h/2).
fn rgb_to_i420(rgb: &[u8], w: usize, h: usize) -> Vec<u8> {
let y_size = w * h;

4
desktop/src/main.ts
View File

@@ -607,7 +607,7 @@ function drawCameraFrameForSend() {
const vh = vdLocalVideo.videoHeight || camCaptureCanvas.height;
if (!vw || !vh) return;
const scale = Math.max(cameraSendWidth / vw, cameraSendHeight / vh);
const scale = Math.min(cameraSendWidth / vw, cameraSendHeight / vh);
const dw = vw * scale;
const dh = vh * scale;
const dx = (cameraSendWidth - dw) / 2;
@@ -631,6 +631,8 @@ async function captureAndPushCameraFrame() {
frame_no: cameraCaptureFrameNo,
width: camCaptureCanvas.width,
height: camCaptureCanvas.height,
source_width: vdLocalVideo.videoWidth || null,
source_height: vdLocalVideo.videoHeight || null,
jpeg_b64_len: b64.length,
capture_clock: getVideoFrameCallbackApi() ? "video_frame_callback" : "interval",
});