manawenuz/wz-phone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(video): improve android stream smoothness
Some checks failed
Mirror to GitHub / mirror (push) Failing after 27s
Details
Build Release Binaries / build-amd64 (push) Failing after 3m35s
Details

Browse Source
This commit is contained in:
Siavash Sameni
2026-05-26 09:57:10 +04:00
parent 31b2caa54d
commit f85efb9576
3 changed files with 223 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
crates/wzp-video/src/mediacodec.rs
View File

@@ -67,7 +67,7 @@ impl MediaCodecEncoder {
format.set_i32("height", height as i32);
format.set_i32("bitrate", bitrate_bps as i32);
format.set_i32("frame-rate", 30);
format.set_i32("i-frame-interval", 1);
format.set_i32("i-frame-interval", 4);
format.set_i32("color-format", COLOR_FORMAT_YUV420_SEMIPLANAR);
let codec = MediaCodec::from_encoder_type("video/avc").ok_or_else(|| {
@@ -170,7 +170,8 @@ impl VideoEncoder for MediaCodecEncoder {
if nals.is_empty() {
return (packet[0] & 0x1F) == 5;
}
nals.iter().any(|nal| !nal.is_empty() && (nal[0] & 0x1F) == 5)
nals.iter()
.any(|nal| !nal.is_empty() && (nal[0] & 0x1F) == 5)
}
}
@@ -419,7 +420,7 @@ impl MediaCodecHevcEncoder {
format.set_i32("height", height as i32);
format.set_i32("bitrate", bitrate_bps as i32);
format.set_i32("frame-rate", 30);
format.set_i32("i-frame-interval", 1);
format.set_i32("i-frame-interval", 4);
format.set_i32("color-format", COLOR_FORMAT_YUV420_PLANAR);
let codec = MediaCodec::from_encoder_type("video/hevc").ok_or_else(|| {
@@ -470,7 +471,11 @@ impl VideoEncoder for MediaCodecHevcEncoder {
.dequeue_input_buffer(std::time::Duration::from_millis(10))
{
Ok(ndk::media::media_codec::DequeuedInputBufferResult::Buffer(mut buffer)) => {
let flags = if self.force_keyframe { AMEDIACODEC_BUFFER_FLAG_KEY_FRAME } else { 0 };
let flags = if self.force_keyframe {
AMEDIACODEC_BUFFER_FLAG_KEY_FRAME
} else {
0
};
let to_copy = {
let buf = buffer.buffer_mut();
let n = frame.data.len().min(buf.len());
@@ -480,7 +485,13 @@ impl VideoEncoder for MediaCodecHevcEncoder {
n
};
self.codec
.queue_input_buffer(buffer, 0, to_copy, frame.timestamp_ms as u64 * 1000, flags)
.queue_input_buffer(
buffer,
0,
to_copy,
frame.timestamp_ms as u64 * 1000,
flags,
)
.map_err(|e| {
VideoError::PlatformError(format!("queue_input_buffer failed: {e}"))
})?;
@@ -592,7 +603,11 @@ impl VideoEncoder for MediaCodecAv1Encoder {
.dequeue_input_buffer(std::time::Duration::from_millis(0))
{
Ok(ndk::media::media_codec::DequeuedInputBufferResult::Buffer(mut buffer)) => {
let flags = if self.force_keyframe { AMEDIACODEC_BUFFER_FLAG_KEY_FRAME } else { 0 };
let flags = if self.force_keyframe {
AMEDIACODEC_BUFFER_FLAG_KEY_FRAME
} else {
0
};
let to_copy = {
let buf = buffer.buffer_mut();
let n = frame.data.len().min(buf.len());
@@ -602,7 +617,13 @@ impl VideoEncoder for MediaCodecAv1Encoder {
n
};
self.codec
.queue_input_buffer(buffer, 0, to_copy, frame.timestamp_ms as u64 * 1000, flags)
.queue_input_buffer(
buffer,
0,
to_copy,
frame.timestamp_ms as u64 * 1000,
flags,
)
.map_err(|e| {
VideoError::PlatformError(format!(
"AV1 encoder queue_input_buffer failed: {e}"
@@ -1162,9 +1183,9 @@ fn i420_len(width: usize, height: usize) -> Result<usize, VideoError> {
#[cfg(target_os = "android")]
fn i420_to_nv12(src: &[u8], width: usize, height: usize) -> Result<Vec<u8>, VideoError> {
let y_size = width
.checked_mul(height)
.ok_or_else(|| VideoError::InvalidInput(format!("invalid frame dimensions {width}x{height}")))?;
let y_size = width.checked_mul(height).ok_or_else(|| {
VideoError::InvalidInput(format!("invalid frame dimensions {width}x{height}"))
})?;
let uv_size = y_size / 4;
let expected = y_size + uv_size * 2;
if src.len() < expected {

274
desktop/src-tauri/src/engine.rs
View File

@@ -8,11 +8,11 @@
//! `start()` that returns an error, so the frontend's `connect` command
//! still fails cleanly but the rest of the engine code links in.
use base64::Engine as _;
use std::net::SocketAddr;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering};
use std::time::Instant;
use base64::Engine as _;
use tauri::Emitter;
use tokio::sync::Mutex;
use tracing::{error, info};
@@ -183,6 +183,7 @@ fn should_log_video_sample(frame_no: u64, is_keyframe: bool) -> bool {
}
const VIDEO_KEYFRAME_INTERVAL_FRAMES: u32 = 120;
const VIDEO_BITRATE_BPS: u32 = 900_000;
const VIDEO_PLI_MIN_INTERVAL_MS: u128 = 250;
#[derive(Default)]
@@ -397,10 +398,7 @@ async fn run_signal_task(
);
pending_profile.store(idx, Ordering::Release);
}
Ok(Ok(Some(wzp_proto::SignalMessage::PictureLossIndication {
stream_id,
..
}))) => {
Ok(Ok(Some(wzp_proto::SignalMessage::PictureLossIndication { stream_id, .. }))) => {
force_video_keyframe.store(true, Ordering::Release);
crate::emit_call_debug(
&app,
@@ -1556,7 +1554,8 @@ impl CallEngine {
);
}
let jpeg_b64 = jpeg_bytes.as_ref().map(|bytes| {
base64::engine::general_purpose::STANDARD.encode(bytes)
base64::engine::general_purpose::STANDARD
.encode(bytes)
});
let jpeg_ok = jpeg_b64.is_some();
if !video_first_decoded_logged {
@@ -2049,40 +2048,43 @@ impl CallEngine {
"codec": format!("{:?}", vid_codec),
"width": 1280,
"height": 720,
"bitrate_bps": 1_500_000,
"bitrate_bps": VIDEO_BITRATE_BPS,
"platform": "android",
}),
);
let mut encoder =
match wzp_video::factory::create_video_encoder(vid_codec, 1280, 720, 1_500_000)
{
Ok(e) => {
crate::emit_call_debug(
&vid_app,
"video:encoder_started",
serde_json::json!({
"t_ms": vid_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", vid_codec),
"platform": "android",
}),
);
e
}
Err(e) => {
error!("video encoder init failed (android): {e}");
crate::emit_call_debug(
&vid_app,
"video:encoder_init_failed",
serde_json::json!({
"t_ms": vid_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", vid_codec),
"platform": "android",
"error": e.to_string(),
}),
);
return;
}
};
let mut encoder = match wzp_video::factory::create_video_encoder(
vid_codec,
1280,
720,
VIDEO_BITRATE_BPS,
) {
Ok(e) => {
crate::emit_call_debug(
&vid_app,
"video:encoder_started",
serde_json::json!({
"t_ms": vid_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", vid_codec),
"platform": "android",
}),
);
e
}
Err(e) => {
error!("video encoder init failed (android): {e}");
crate::emit_call_debug(
&vid_app,
"video:encoder_init_failed",
serde_json::json!({
"t_ms": vid_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", vid_codec),
"platform": "android",
"error": e.to_string(),
}),
);
return;
}
};
let mut seq: u32 = 0;
let mut frames_since_keyframe: u32 = 0;
let mut first_send_logged = false;
@@ -2090,6 +2092,16 @@ impl CallEngine {
let mut camera_frames: u64 = 0;
let mut empty_encodes: u64 = 0;
let mut encoded_frame_samples: u64 = 0;
let mut send_heartbeat = std::time::Instant::now();
let mut encoded_frames_total: u64 = 0;
let mut encoded_keyframes_total: u64 = 0;
let mut video_packets_total: u64 = 0;
let mut video_bytes_total: u64 = 0;
let mut last_heartbeat_camera_frames: u64 = 0;
let mut last_heartbeat_encoded_frames: u64 = 0;
let mut last_heartbeat_packets: u64 = 0;
let mut last_heartbeat_bytes: u64 = 0;
let mut last_heartbeat_empty_encodes: u64 = 0;
let mut skipped_startup_black_frames: u64 = 0;
let mut wait_ticks: u64 = 0;
encoder.request_keyframe();
@@ -2191,14 +2203,13 @@ impl CallEngine {
continue;
}
let keyframe_reason =
if vid_force_keyframe.swap(false, Ordering::AcqRel) {
Some("pli")
} else if frames_since_keyframe >= VIDEO_KEYFRAME_INTERVAL_FRAMES {
Some("periodic")
} else {
None
};
let keyframe_reason = if vid_force_keyframe.swap(false, Ordering::AcqRel) {
Some("pli")
} else if frames_since_keyframe >= VIDEO_KEYFRAME_INTERVAL_FRAMES {
Some("periodic")
} else {
None
};
if let Some(reason) = keyframe_reason {
encoder.request_keyframe();
crate::emit_call_debug(
@@ -2252,6 +2263,10 @@ impl CallEngine {
}
let is_keyframe = encoder.is_keyframe(&encoded);
encoded_frames_total += 1;
if is_keyframe {
encoded_keyframes_total += 1;
}
let ts_ms = vid_t0.elapsed().as_millis() as u32;
let pkts = wzp_video::transport::packetize_video_frame(
&encoded,
@@ -2260,6 +2275,8 @@ impl CallEngine {
&mut seq,
ts_ms,
);
video_packets_total += pkts.len() as u64;
video_bytes_total += encoded.len() as u64;
if encoded_frame_samples < 5 {
encoded_frame_samples += 1;
let packet_payload_bytes: usize =
@@ -2311,6 +2328,40 @@ impl CallEngine {
break;
}
}
if send_heartbeat.elapsed() >= std::time::Duration::from_secs(2) {
let dt_ms = send_heartbeat.elapsed().as_millis().max(1) as f64;
let encoded_delta = encoded_frames_total - last_heartbeat_encoded_frames;
let camera_delta = camera_frames - last_heartbeat_camera_frames;
let packets_delta = video_packets_total - last_heartbeat_packets;
let bytes_delta = video_bytes_total - last_heartbeat_bytes;
let empty_delta = empty_encodes - last_heartbeat_empty_encodes;
crate::emit_call_debug(
&vid_app,
"video:send_heartbeat",
serde_json::json!({
"t_ms": vid_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", vid_codec),
"platform": "android",
"camera_frames": camera_frames,
"encoded_frames": encoded_frames_total,
"encoded_keyframes": encoded_keyframes_total,
"empty_encodes": empty_encodes,
"pkts_sent": video_packets_total,
"bytes_sent": video_bytes_total,
"camera_fps": (camera_delta as f64) * 1000.0 / dt_ms,
"encoded_fps": (encoded_delta as f64) * 1000.0 / dt_ms,
"packets_per_sec": (packets_delta as f64) * 1000.0 / dt_ms,
"kbps": (bytes_delta as f64) * 8.0 / dt_ms,
"empty_encodes_delta": empty_delta,
}),
);
last_heartbeat_camera_frames = camera_frames;
last_heartbeat_encoded_frames = encoded_frames_total;
last_heartbeat_packets = video_packets_total;
last_heartbeat_bytes = video_bytes_total;
last_heartbeat_empty_encodes = empty_encodes;
send_heartbeat = std::time::Instant::now();
}
frames_since_keyframe += 1;
}
crate::emit_call_debug(
@@ -2980,7 +3031,8 @@ impl CallEngine {
);
}
let jpeg_b64 = jpeg_bytes.as_ref().map(|bytes| {
base64::engine::general_purpose::STANDARD.encode(bytes)
base64::engine::general_purpose::STANDARD
.encode(bytes)
});
let jpeg_ok = jpeg_b64.is_some();
if !video_first_decoded_logged {
@@ -3315,40 +3367,43 @@ impl CallEngine {
"codec": format!("{:?}", vid_codec),
"width": 1280,
"height": 720,
"bitrate_bps": 1_500_000,
"bitrate_bps": VIDEO_BITRATE_BPS,
"platform": "desktop",
}),
);
let mut encoder =
match wzp_video::factory::create_video_encoder(vid_codec, 1280, 720, 1_500_000)
{
Ok(e) => {
crate::emit_call_debug(
&vid_app,
"video:encoder_started",
serde_json::json!({
"t_ms": vid_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", vid_codec),
"platform": "desktop",
}),
);
e
}
Err(e) => {
error!("video encoder init failed: {e}");
crate::emit_call_debug(
&vid_app,
"video:encoder_init_failed",
serde_json::json!({
"t_ms": vid_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", vid_codec),
"platform": "desktop",
"error": e.to_string(),
}),
);
return;
}
};
let mut encoder = match wzp_video::factory::create_video_encoder(
vid_codec,
1280,
720,
VIDEO_BITRATE_BPS,
) {
Ok(e) => {
crate::emit_call_debug(
&vid_app,
"video:encoder_started",
serde_json::json!({
"t_ms": vid_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", vid_codec),
"platform": "desktop",
}),
);
e
}
Err(e) => {
error!("video encoder init failed: {e}");
crate::emit_call_debug(
&vid_app,
"video:encoder_init_failed",
serde_json::json!({
"t_ms": vid_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", vid_codec),
"platform": "desktop",
"error": e.to_string(),
}),
);
return;
}
};
let mut seq: u32 = 0;
let mut frames_since_keyframe: u32 = 0;
let mut first_send_logged = false;
@@ -3356,6 +3411,16 @@ impl CallEngine {
let mut camera_frames: u64 = 0;
let mut empty_encodes: u64 = 0;
let mut encoded_frame_samples: u64 = 0;
let mut send_heartbeat = std::time::Instant::now();
let mut encoded_frames_total: u64 = 0;
let mut encoded_keyframes_total: u64 = 0;
let mut video_packets_total: u64 = 0;
let mut video_bytes_total: u64 = 0;
let mut last_heartbeat_camera_frames: u64 = 0;
let mut last_heartbeat_encoded_frames: u64 = 0;
let mut last_heartbeat_packets: u64 = 0;
let mut last_heartbeat_bytes: u64 = 0;
let mut last_heartbeat_empty_encodes: u64 = 0;
let mut skipped_startup_black_frames: u64 = 0;
let mut wait_ticks: u64 = 0;
encoder.request_keyframe();
@@ -3457,14 +3522,13 @@ impl CallEngine {
continue;
}
let keyframe_reason =
if vid_force_keyframe.swap(false, Ordering::AcqRel) {
Some("pli")
} else if frames_since_keyframe >= VIDEO_KEYFRAME_INTERVAL_FRAMES {
Some("periodic")
} else {
None
};
let keyframe_reason = if vid_force_keyframe.swap(false, Ordering::AcqRel) {
Some("pli")
} else if frames_since_keyframe >= VIDEO_KEYFRAME_INTERVAL_FRAMES {
Some("periodic")
} else {
None
};
if let Some(reason) = keyframe_reason {
encoder.request_keyframe();
crate::emit_call_debug(
@@ -3518,6 +3582,10 @@ impl CallEngine {
}
let is_keyframe = encoder.is_keyframe(&encoded);
encoded_frames_total += 1;
if is_keyframe {
encoded_keyframes_total += 1;
}
let ts_ms = vid_t0.elapsed().as_millis() as u32;
let pkts = wzp_video::transport::packetize_video_frame(
&encoded,
@@ -3526,6 +3594,8 @@ impl CallEngine {
&mut seq,
ts_ms,
);
video_packets_total += pkts.len() as u64;
video_bytes_total += encoded.len() as u64;
if encoded_frame_samples < 5 {
encoded_frame_samples += 1;
let packet_payload_bytes: usize =
@@ -3577,6 +3647,40 @@ impl CallEngine {
break;
}
}
if send_heartbeat.elapsed() >= std::time::Duration::from_secs(2) {
let dt_ms = send_heartbeat.elapsed().as_millis().max(1) as f64;
let encoded_delta = encoded_frames_total - last_heartbeat_encoded_frames;
let camera_delta = camera_frames - last_heartbeat_camera_frames;
let packets_delta = video_packets_total - last_heartbeat_packets;
let bytes_delta = video_bytes_total - last_heartbeat_bytes;
let empty_delta = empty_encodes - last_heartbeat_empty_encodes;
crate::emit_call_debug(
&vid_app,
"video:send_heartbeat",
serde_json::json!({
"t_ms": vid_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", vid_codec),
"platform": "desktop",
"camera_frames": camera_frames,
"encoded_frames": encoded_frames_total,
"encoded_keyframes": encoded_keyframes_total,
"empty_encodes": empty_encodes,
"pkts_sent": video_packets_total,
"bytes_sent": video_bytes_total,
"camera_fps": (camera_delta as f64) * 1000.0 / dt_ms,
"encoded_fps": (encoded_delta as f64) * 1000.0 / dt_ms,
"packets_per_sec": (packets_delta as f64) * 1000.0 / dt_ms,
"kbps": (bytes_delta as f64) * 8.0 / dt_ms,
"empty_encodes_delta": empty_delta,
}),
);
last_heartbeat_camera_frames = camera_frames;
last_heartbeat_encoded_frames = encoded_frames_total;
last_heartbeat_packets = video_packets_total;
last_heartbeat_bytes = video_bytes_total;
last_heartbeat_empty_encodes = empty_encodes;
send_heartbeat = std::time::Instant::now();
}
frames_since_keyframe += 1;
}
crate::emit_call_debug(

5
desktop/src/main.ts
View File

@@ -574,7 +574,8 @@ const CAMERA_SEND_WIDTH = 1280;
const CAMERA_SEND_HEIGHT = 720;
let cameraCaptureFrameNo = 0;
let cameraPushFailures = 0;
const CAMERA_CAPTURE_INTERVAL_MS = 67; // ≈ 15 fps
const CAMERA_CAPTURE_INTERVAL_MS = 33; // ≈ 30 fps
const CAMERA_JPEG_QUALITY = 0.7;
function drawCameraFrameForSend() {
const vw = vdLocalVideo.videoWidth || camCaptureCanvas.width;
@@ -598,7 +599,7 @@ async function captureAndPushCameraFrame() {
cameraCaptureFrameNo++;
try {
drawCameraFrameForSend();
const dataUrl = camCaptureCanvas.toDataURL("image/jpeg", 0.75);
const dataUrl = camCaptureCanvas.toDataURL("image/jpeg", CAMERA_JPEG_QUALITY);
const b64 = dataUrl.slice(dataUrl.indexOf(",") + 1);
if (cameraCaptureFrameNo === 1 || cameraCaptureFrameNo % 150 === 0) {
debugLog("camera:capture_frame", {