manawenuz/wz-phone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(video): preserve annex-b mediacodec output
Some checks failed
Mirror to GitHub / mirror (push) Failing after 31s
Details
Build Release Binaries / build-amd64 (push) Failing after 3m35s
Details

Browse Source
This commit is contained in:
Siavash Sameni
2026-05-25 20:20:22 +04:00
parent d57ebe3d2c
commit 06d28a9280
3 changed files with 101 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
crates/wzp-video/src/mediacodec.rs
View File

@@ -992,8 +992,13 @@ fn extract_vps_sps_pps(annex_b: &[u8]) -> HevcParameterSets {
/// (4-byte start codes `0x00 0x00 0x00 0x01`). /// (4-byte start codes `0x00 0x00 0x00 0x01`).
#[allow(dead_code)] #[allow(dead_code)]
fn avcc_to_annexb(data: &[u8]) -> Vec<u8> { fn avcc_to_annexb(data: &[u8]) -> Vec<u8> {
if starts_with_annex_b_start_code(data) {
return data.to_vec();
}
let mut out = Vec::with_capacity(data.len() + data.len() / 4); let mut out = Vec::with_capacity(data.len() + data.len() / 4);
let mut offset = 0; let mut offset = 0;
let mut saw_nal = false;
while offset + 4 <= data.len() { while offset + 4 <= data.len() {
let nal_len = u32::from_be_bytes([ let nal_len = u32::from_be_bytes([
data[offset], data[offset],
@@ -1003,15 +1008,20 @@ fn avcc_to_annexb(data: &[u8]) -> Vec<u8> {
]) as usize; ]) as usize;
offset += 4; offset += 4;
if offset + nal_len > data.len() { if offset + nal_len > data.len() {
break; return if saw_nal { out } else { data.to_vec() };
} }
out.extend_from_slice(&[0x00, 0x00, 0x00, 0x01]); out.extend_from_slice(&[0x00, 0x00, 0x00, 0x01]);
out.extend_from_slice(&data[offset..offset + nal_len]); out.extend_from_slice(&data[offset..offset + nal_len]);
offset += nal_len; offset += nal_len;
saw_nal = true;
} }
out out
} }
fn starts_with_annex_b_start_code(data: &[u8]) -> bool {
data.starts_with(&[0x00, 0x00, 0x01]) || data.starts_with(&[0x00, 0x00, 0x00, 0x01])
}
/// Parse an Annex-B access unit and return the first SPS and PPS found. /// Parse an Annex-B access unit and return the first SPS and PPS found.
#[allow(dead_code)] #[allow(dead_code)]
fn extract_sps_pps(annex_b: &[u8]) -> (Option<Vec<u8>>, Option<Vec<u8>>) { fn extract_sps_pps(annex_b: &[u8]) -> (Option<Vec<u8>>, Option<Vec<u8>>) {
@@ -1163,6 +1173,16 @@ mod tests {
assert_eq!(annex_b, expected); assert_eq!(annex_b, expected);
} }
#[test]
fn avcc_to_annexb_passes_through_annexb() {
let annex_b = vec![
0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xC0, 0x1E,
0x00, 0x00, 0x00, 0x01, 0x65, 0x88, 0x84, 0x21,
];
assert_eq!(avcc_to_annexb(&annex_b), annex_b);
}
#[test] #[test]
fn hevc_mediacodec_encoder_returns_not_initialized_on_non_android() { fn hevc_mediacodec_encoder_returns_not_initialized_on_non_android() {
let enc = MediaCodecHevcEncoder::new(1280, 720, 2_000_000); let enc = MediaCodecHevcEncoder::new(1280, 720, 2_000_000);

78
desktop/src-tauri/src/engine.rs
View File

@@ -1184,6 +1184,7 @@ impl CallEngine {
let mut video_decoder_codec: Option<wzp_proto::CodecId> = None; let mut video_decoder_codec: Option<wzp_proto::CodecId> = None;
let mut video_first_recv_logged = false; let mut video_first_recv_logged = false;
let mut video_first_reassembled_logged = false; let mut video_first_reassembled_logged = false;
let mut video_reassembled_samples: u64 = 0;
let mut video_first_decoded_logged = false; let mut video_first_decoded_logged = false;
let mut video_decoder_buffering_count: u64 = 0; let mut video_decoder_buffering_count: u64 = 0;
@@ -1217,6 +1218,7 @@ impl CallEngine {
if let Some((codec_id, is_kf, frame)) = if let Some((codec_id, is_kf, frame)) =
video_reassembler.push(&pkt) video_reassembler.push(&pkt)
{ {
video_reassembled_samples += 1;
if !video_first_reassembled_logged { if !video_first_reassembled_logged {
video_first_reassembled_logged = true; video_first_reassembled_logged = true;
crate::emit_call_debug( crate::emit_call_debug(
@@ -1231,6 +1233,20 @@ impl CallEngine {
}), }),
); );
} }
if video_reassembled_samples <= 5 {
crate::emit_call_debug(
&recv_app,
"video:reassembled_frame",
serde_json::json!({
"t_ms": recv_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", codec_id),
"is_keyframe": is_kf,
"frame_bytes": frame.len(),
"frame_no": video_reassembled_samples,
"platform": "android",
}),
);
}
if video_decoder_codec != Some(codec_id) { if video_decoder_codec != Some(codec_id) {
crate::emit_call_debug( crate::emit_call_debug(
&recv_app, &recv_app,
@@ -1778,6 +1794,7 @@ impl CallEngine {
let mut first_camera_frame_logged = false; let mut first_camera_frame_logged = false;
let mut camera_frames: u64 = 0; let mut camera_frames: u64 = 0;
let mut empty_encodes: u64 = 0; let mut empty_encodes: u64 = 0;
let mut encoded_frame_samples: u64 = 0;
let mut wait_ticks: u64 = 0; let mut wait_ticks: u64 = 0;
encoder.request_keyframe(); encoder.request_keyframe();
crate::emit_call_debug( crate::emit_call_debug(
@@ -1894,6 +1911,26 @@ impl CallEngine {
let pkts = wzp_video::transport::packetize_video_frame( let pkts = wzp_video::transport::packetize_video_frame(
&encoded, vid_codec, is_keyframe, &mut seq, ts_ms, &encoded, vid_codec, is_keyframe, &mut seq, ts_ms,
); );
if encoded_frame_samples < 5 {
encoded_frame_samples += 1;
let packet_payload_bytes: usize =
pkts.iter().map(|pkt| pkt.payload.len()).sum();
crate::emit_call_debug(
&vid_app,
"video:encoded_frame",
serde_json::json!({
"t_ms": vid_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", vid_codec),
"camera_frames": camera_frames,
"encoded_bytes": encoded.len(),
"packet_payload_bytes": packet_payload_bytes,
"packets": pkts.len(),
"is_keyframe": is_keyframe,
"sample_no": encoded_frame_samples,
"platform": "android",
}),
);
}
if !first_send_logged && !pkts.is_empty() { if !first_send_logged && !pkts.is_empty() {
first_send_logged = true; first_send_logged = true;
crate::emit_call_debug( crate::emit_call_debug(
@@ -1904,6 +1941,8 @@ impl CallEngine {
"codec": format!("{:?}", vid_codec), "codec": format!("{:?}", vid_codec),
"packets": pkts.len(), "packets": pkts.len(),
"first_pkt_bytes": pkts[0].payload.len(), "first_pkt_bytes": pkts[0].payload.len(),
"last_pkt_bytes": pkts.last().map(|pkt| pkt.payload.len()).unwrap_or(0),
"encoded_bytes": encoded.len(),
"is_keyframe": is_keyframe, "is_keyframe": is_keyframe,
}), }),
); );
@@ -2389,6 +2428,7 @@ impl CallEngine {
let mut video_decoder_codec: Option<wzp_proto::CodecId> = None; let mut video_decoder_codec: Option<wzp_proto::CodecId> = None;
let mut video_first_recv_logged_desktop = false; let mut video_first_recv_logged_desktop = false;
let mut video_first_reassembled_logged = false; let mut video_first_reassembled_logged = false;
let mut video_reassembled_samples: u64 = 0;
let mut video_first_decoded_logged = false; let mut video_first_decoded_logged = false;
let mut video_decoder_buffering_count: u64 = 0; let mut video_decoder_buffering_count: u64 = 0;
let mut decoded_frames: u64 = 0; let mut decoded_frames: u64 = 0;
@@ -2427,6 +2467,7 @@ impl CallEngine {
if let Some((codec_id, is_kf, frame)) = if let Some((codec_id, is_kf, frame)) =
video_reassembler.push(&pkt) video_reassembler.push(&pkt)
{ {
video_reassembled_samples += 1;
if !video_first_reassembled_logged { if !video_first_reassembled_logged {
video_first_reassembled_logged = true; video_first_reassembled_logged = true;
crate::emit_call_debug( crate::emit_call_debug(
@@ -2441,6 +2482,20 @@ impl CallEngine {
}), }),
); );
} }
if video_reassembled_samples <= 5 {
crate::emit_call_debug(
&recv_app,
"video:reassembled_frame",
serde_json::json!({
"t_ms": recv_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", codec_id),
"is_keyframe": is_kf,
"frame_bytes": frame.len(),
"frame_no": video_reassembled_samples,
"platform": "desktop",
}),
);
}
// Lazy-init or switch decoder on codec change. // Lazy-init or switch decoder on codec change.
if video_decoder_codec != Some(codec_id) { if video_decoder_codec != Some(codec_id) {
crate::emit_call_debug( crate::emit_call_debug(
@@ -2838,6 +2893,7 @@ impl CallEngine {
let mut first_camera_frame_logged = false; let mut first_camera_frame_logged = false;
let mut camera_frames: u64 = 0; let mut camera_frames: u64 = 0;
let mut empty_encodes: u64 = 0; let mut empty_encodes: u64 = 0;
let mut encoded_frame_samples: u64 = 0;
let mut wait_ticks: u64 = 0; let mut wait_ticks: u64 = 0;
encoder.request_keyframe(); encoder.request_keyframe();
crate::emit_call_debug( crate::emit_call_debug(
@@ -2954,6 +3010,26 @@ impl CallEngine {
let pkts = wzp_video::transport::packetize_video_frame( let pkts = wzp_video::transport::packetize_video_frame(
&encoded, vid_codec, is_keyframe, &mut seq, ts_ms, &encoded, vid_codec, is_keyframe, &mut seq, ts_ms,
); );
if encoded_frame_samples < 5 {
encoded_frame_samples += 1;
let packet_payload_bytes: usize =
pkts.iter().map(|pkt| pkt.payload.len()).sum();
crate::emit_call_debug(
&vid_app,
"video:encoded_frame",
serde_json::json!({
"t_ms": vid_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", vid_codec),
"camera_frames": camera_frames,
"encoded_bytes": encoded.len(),
"packet_payload_bytes": packet_payload_bytes,
"packets": pkts.len(),
"is_keyframe": is_keyframe,
"sample_no": encoded_frame_samples,
"platform": "desktop",
}),
);
}
if !first_send_logged && !pkts.is_empty() { if !first_send_logged && !pkts.is_empty() {
first_send_logged = true; first_send_logged = true;
crate::emit_call_debug( crate::emit_call_debug(
@@ -2964,6 +3040,8 @@ impl CallEngine {
"codec": format!("{:?}", vid_codec), "codec": format!("{:?}", vid_codec),
"packets": pkts.len(), "packets": pkts.len(),
"first_pkt_bytes": pkts[0].payload.len(), "first_pkt_bytes": pkts[0].payload.len(),
"last_pkt_bytes": pkts.last().map(|pkt| pkt.payload.len()).unwrap_or(0),
"encoded_bytes": encoded.len(),
"is_keyframe": is_keyframe, "is_keyframe": is_keyframe,
}), }),
); );

4
desktop/src/style.css
View File

@@ -258,7 +258,7 @@ body {
border-top: 1px solid var(--surface2); border-top: 1px solid var(--surface2);
padding: 0 16px; padding: 0 16px;
padding-bottom: env(safe-area-inset-bottom, 8px); padding-bottom: env(safe-area-inset-bottom, 8px);
z-index: 50; z-index: 70;
animation: drawerUp 0.25s ease-out; animation: drawerUp 0.25s ease-out;
box-shadow: 0 -4px 20px rgba(0,0,0,0.4); box-shadow: 0 -4px 20px rgba(0,0,0,0.4);
} }
@@ -324,7 +324,7 @@ body {
right: 0; right: 0;
bottom: 96px; /* leave room for voice drawer */ bottom: 96px; /* leave room for voice drawer */
background: #000; background: #000;
z-index: 50; z-index: 40;
overflow: hidden; overflow: hidden;
} }
.vd-remote-stage { .vd-remote-stage {