fix(video): improve android stream smoothness
Some checks failed
Mirror to GitHub / mirror (push) Failing after 27s
Build Release Binaries / build-amd64 (push) Failing after 3m35s

This commit is contained in:
Siavash Sameni
2026-05-26 09:57:10 +04:00
parent 31b2caa54d
commit f85efb9576
3 changed files with 223 additions and 97 deletions

View File

@@ -67,7 +67,7 @@ impl MediaCodecEncoder {
format.set_i32("height", height as i32); format.set_i32("height", height as i32);
format.set_i32("bitrate", bitrate_bps as i32); format.set_i32("bitrate", bitrate_bps as i32);
format.set_i32("frame-rate", 30); format.set_i32("frame-rate", 30);
format.set_i32("i-frame-interval", 1); format.set_i32("i-frame-interval", 4);
format.set_i32("color-format", COLOR_FORMAT_YUV420_SEMIPLANAR); format.set_i32("color-format", COLOR_FORMAT_YUV420_SEMIPLANAR);
let codec = MediaCodec::from_encoder_type("video/avc").ok_or_else(|| { let codec = MediaCodec::from_encoder_type("video/avc").ok_or_else(|| {
@@ -170,7 +170,8 @@ impl VideoEncoder for MediaCodecEncoder {
if nals.is_empty() { if nals.is_empty() {
return (packet[0] & 0x1F) == 5; return (packet[0] & 0x1F) == 5;
} }
nals.iter().any(|nal| !nal.is_empty() && (nal[0] & 0x1F) == 5) nals.iter()
.any(|nal| !nal.is_empty() && (nal[0] & 0x1F) == 5)
} }
} }
@@ -419,7 +420,7 @@ impl MediaCodecHevcEncoder {
format.set_i32("height", height as i32); format.set_i32("height", height as i32);
format.set_i32("bitrate", bitrate_bps as i32); format.set_i32("bitrate", bitrate_bps as i32);
format.set_i32("frame-rate", 30); format.set_i32("frame-rate", 30);
format.set_i32("i-frame-interval", 1); format.set_i32("i-frame-interval", 4);
format.set_i32("color-format", COLOR_FORMAT_YUV420_PLANAR); format.set_i32("color-format", COLOR_FORMAT_YUV420_PLANAR);
let codec = MediaCodec::from_encoder_type("video/hevc").ok_or_else(|| { let codec = MediaCodec::from_encoder_type("video/hevc").ok_or_else(|| {
@@ -470,7 +471,11 @@ impl VideoEncoder for MediaCodecHevcEncoder {
.dequeue_input_buffer(std::time::Duration::from_millis(10)) .dequeue_input_buffer(std::time::Duration::from_millis(10))
{ {
Ok(ndk::media::media_codec::DequeuedInputBufferResult::Buffer(mut buffer)) => { Ok(ndk::media::media_codec::DequeuedInputBufferResult::Buffer(mut buffer)) => {
let flags = if self.force_keyframe { AMEDIACODEC_BUFFER_FLAG_KEY_FRAME } else { 0 }; let flags = if self.force_keyframe {
AMEDIACODEC_BUFFER_FLAG_KEY_FRAME
} else {
0
};
let to_copy = { let to_copy = {
let buf = buffer.buffer_mut(); let buf = buffer.buffer_mut();
let n = frame.data.len().min(buf.len()); let n = frame.data.len().min(buf.len());
@@ -480,7 +485,13 @@ impl VideoEncoder for MediaCodecHevcEncoder {
n n
}; };
self.codec self.codec
.queue_input_buffer(buffer, 0, to_copy, frame.timestamp_ms as u64 * 1000, flags) .queue_input_buffer(
buffer,
0,
to_copy,
frame.timestamp_ms as u64 * 1000,
flags,
)
.map_err(|e| { .map_err(|e| {
VideoError::PlatformError(format!("queue_input_buffer failed: {e}")) VideoError::PlatformError(format!("queue_input_buffer failed: {e}"))
})?; })?;
@@ -592,7 +603,11 @@ impl VideoEncoder for MediaCodecAv1Encoder {
.dequeue_input_buffer(std::time::Duration::from_millis(0)) .dequeue_input_buffer(std::time::Duration::from_millis(0))
{ {
Ok(ndk::media::media_codec::DequeuedInputBufferResult::Buffer(mut buffer)) => { Ok(ndk::media::media_codec::DequeuedInputBufferResult::Buffer(mut buffer)) => {
let flags = if self.force_keyframe { AMEDIACODEC_BUFFER_FLAG_KEY_FRAME } else { 0 }; let flags = if self.force_keyframe {
AMEDIACODEC_BUFFER_FLAG_KEY_FRAME
} else {
0
};
let to_copy = { let to_copy = {
let buf = buffer.buffer_mut(); let buf = buffer.buffer_mut();
let n = frame.data.len().min(buf.len()); let n = frame.data.len().min(buf.len());
@@ -602,7 +617,13 @@ impl VideoEncoder for MediaCodecAv1Encoder {
n n
}; };
self.codec self.codec
.queue_input_buffer(buffer, 0, to_copy, frame.timestamp_ms as u64 * 1000, flags) .queue_input_buffer(
buffer,
0,
to_copy,
frame.timestamp_ms as u64 * 1000,
flags,
)
.map_err(|e| { .map_err(|e| {
VideoError::PlatformError(format!( VideoError::PlatformError(format!(
"AV1 encoder queue_input_buffer failed: {e}" "AV1 encoder queue_input_buffer failed: {e}"
@@ -1162,9 +1183,9 @@ fn i420_len(width: usize, height: usize) -> Result<usize, VideoError> {
#[cfg(target_os = "android")] #[cfg(target_os = "android")]
fn i420_to_nv12(src: &[u8], width: usize, height: usize) -> Result<Vec<u8>, VideoError> { fn i420_to_nv12(src: &[u8], width: usize, height: usize) -> Result<Vec<u8>, VideoError> {
let y_size = width let y_size = width.checked_mul(height).ok_or_else(|| {
.checked_mul(height) VideoError::InvalidInput(format!("invalid frame dimensions {width}x{height}"))
.ok_or_else(|| VideoError::InvalidInput(format!("invalid frame dimensions {width}x{height}")))?; })?;
let uv_size = y_size / 4; let uv_size = y_size / 4;
let expected = y_size + uv_size * 2; let expected = y_size + uv_size * 2;
if src.len() < expected { if src.len() < expected {

View File

@@ -8,11 +8,11 @@
//! `start()` that returns an error, so the frontend's `connect` command //! `start()` that returns an error, so the frontend's `connect` command
//! still fails cleanly but the rest of the engine code links in. //! still fails cleanly but the rest of the engine code links in.
use base64::Engine as _;
use std::net::SocketAddr; use std::net::SocketAddr;
use std::sync::Arc; use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering}; use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering};
use std::time::Instant; use std::time::Instant;
use base64::Engine as _;
use tauri::Emitter; use tauri::Emitter;
use tokio::sync::Mutex; use tokio::sync::Mutex;
use tracing::{error, info}; use tracing::{error, info};
@@ -183,6 +183,7 @@ fn should_log_video_sample(frame_no: u64, is_keyframe: bool) -> bool {
} }
const VIDEO_KEYFRAME_INTERVAL_FRAMES: u32 = 120; const VIDEO_KEYFRAME_INTERVAL_FRAMES: u32 = 120;
const VIDEO_BITRATE_BPS: u32 = 900_000;
const VIDEO_PLI_MIN_INTERVAL_MS: u128 = 250; const VIDEO_PLI_MIN_INTERVAL_MS: u128 = 250;
#[derive(Default)] #[derive(Default)]
@@ -397,10 +398,7 @@ async fn run_signal_task(
); );
pending_profile.store(idx, Ordering::Release); pending_profile.store(idx, Ordering::Release);
} }
Ok(Ok(Some(wzp_proto::SignalMessage::PictureLossIndication { Ok(Ok(Some(wzp_proto::SignalMessage::PictureLossIndication { stream_id, .. }))) => {
stream_id,
..
}))) => {
force_video_keyframe.store(true, Ordering::Release); force_video_keyframe.store(true, Ordering::Release);
crate::emit_call_debug( crate::emit_call_debug(
&app, &app,
@@ -1556,7 +1554,8 @@ impl CallEngine {
); );
} }
let jpeg_b64 = jpeg_bytes.as_ref().map(|bytes| { let jpeg_b64 = jpeg_bytes.as_ref().map(|bytes| {
base64::engine::general_purpose::STANDARD.encode(bytes) base64::engine::general_purpose::STANDARD
.encode(bytes)
}); });
let jpeg_ok = jpeg_b64.is_some(); let jpeg_ok = jpeg_b64.is_some();
if !video_first_decoded_logged { if !video_first_decoded_logged {
@@ -2049,40 +2048,43 @@ impl CallEngine {
"codec": format!("{:?}", vid_codec), "codec": format!("{:?}", vid_codec),
"width": 1280, "width": 1280,
"height": 720, "height": 720,
"bitrate_bps": 1_500_000, "bitrate_bps": VIDEO_BITRATE_BPS,
"platform": "android", "platform": "android",
}), }),
); );
let mut encoder = let mut encoder = match wzp_video::factory::create_video_encoder(
match wzp_video::factory::create_video_encoder(vid_codec, 1280, 720, 1_500_000) vid_codec,
{ 1280,
Ok(e) => { 720,
crate::emit_call_debug( VIDEO_BITRATE_BPS,
&vid_app, ) {
"video:encoder_started", Ok(e) => {
serde_json::json!({ crate::emit_call_debug(
"t_ms": vid_t0.elapsed().as_millis() as u64, &vid_app,
"codec": format!("{:?}", vid_codec), "video:encoder_started",
"platform": "android", serde_json::json!({
}), "t_ms": vid_t0.elapsed().as_millis() as u64,
); "codec": format!("{:?}", vid_codec),
e "platform": "android",
} }),
Err(e) => { );
error!("video encoder init failed (android): {e}"); e
crate::emit_call_debug( }
&vid_app, Err(e) => {
"video:encoder_init_failed", error!("video encoder init failed (android): {e}");
serde_json::json!({ crate::emit_call_debug(
"t_ms": vid_t0.elapsed().as_millis() as u64, &vid_app,
"codec": format!("{:?}", vid_codec), "video:encoder_init_failed",
"platform": "android", serde_json::json!({
"error": e.to_string(), "t_ms": vid_t0.elapsed().as_millis() as u64,
}), "codec": format!("{:?}", vid_codec),
); "platform": "android",
return; "error": e.to_string(),
} }),
}; );
return;
}
};
let mut seq: u32 = 0; let mut seq: u32 = 0;
let mut frames_since_keyframe: u32 = 0; let mut frames_since_keyframe: u32 = 0;
let mut first_send_logged = false; let mut first_send_logged = false;
@@ -2090,6 +2092,16 @@ impl CallEngine {
let mut camera_frames: u64 = 0; let mut camera_frames: u64 = 0;
let mut empty_encodes: u64 = 0; let mut empty_encodes: u64 = 0;
let mut encoded_frame_samples: u64 = 0; let mut encoded_frame_samples: u64 = 0;
let mut send_heartbeat = std::time::Instant::now();
let mut encoded_frames_total: u64 = 0;
let mut encoded_keyframes_total: u64 = 0;
let mut video_packets_total: u64 = 0;
let mut video_bytes_total: u64 = 0;
let mut last_heartbeat_camera_frames: u64 = 0;
let mut last_heartbeat_encoded_frames: u64 = 0;
let mut last_heartbeat_packets: u64 = 0;
let mut last_heartbeat_bytes: u64 = 0;
let mut last_heartbeat_empty_encodes: u64 = 0;
let mut skipped_startup_black_frames: u64 = 0; let mut skipped_startup_black_frames: u64 = 0;
let mut wait_ticks: u64 = 0; let mut wait_ticks: u64 = 0;
encoder.request_keyframe(); encoder.request_keyframe();
@@ -2191,14 +2203,13 @@ impl CallEngine {
continue; continue;
} }
let keyframe_reason = let keyframe_reason = if vid_force_keyframe.swap(false, Ordering::AcqRel) {
if vid_force_keyframe.swap(false, Ordering::AcqRel) { Some("pli")
Some("pli") } else if frames_since_keyframe >= VIDEO_KEYFRAME_INTERVAL_FRAMES {
} else if frames_since_keyframe >= VIDEO_KEYFRAME_INTERVAL_FRAMES { Some("periodic")
Some("periodic") } else {
} else { None
None };
};
if let Some(reason) = keyframe_reason { if let Some(reason) = keyframe_reason {
encoder.request_keyframe(); encoder.request_keyframe();
crate::emit_call_debug( crate::emit_call_debug(
@@ -2252,6 +2263,10 @@ impl CallEngine {
} }
let is_keyframe = encoder.is_keyframe(&encoded); let is_keyframe = encoder.is_keyframe(&encoded);
encoded_frames_total += 1;
if is_keyframe {
encoded_keyframes_total += 1;
}
let ts_ms = vid_t0.elapsed().as_millis() as u32; let ts_ms = vid_t0.elapsed().as_millis() as u32;
let pkts = wzp_video::transport::packetize_video_frame( let pkts = wzp_video::transport::packetize_video_frame(
&encoded, &encoded,
@@ -2260,6 +2275,8 @@ impl CallEngine {
&mut seq, &mut seq,
ts_ms, ts_ms,
); );
video_packets_total += pkts.len() as u64;
video_bytes_total += encoded.len() as u64;
if encoded_frame_samples < 5 { if encoded_frame_samples < 5 {
encoded_frame_samples += 1; encoded_frame_samples += 1;
let packet_payload_bytes: usize = let packet_payload_bytes: usize =
@@ -2311,6 +2328,40 @@ impl CallEngine {
break; break;
} }
} }
if send_heartbeat.elapsed() >= std::time::Duration::from_secs(2) {
let dt_ms = send_heartbeat.elapsed().as_millis().max(1) as f64;
let encoded_delta = encoded_frames_total - last_heartbeat_encoded_frames;
let camera_delta = camera_frames - last_heartbeat_camera_frames;
let packets_delta = video_packets_total - last_heartbeat_packets;
let bytes_delta = video_bytes_total - last_heartbeat_bytes;
let empty_delta = empty_encodes - last_heartbeat_empty_encodes;
crate::emit_call_debug(
&vid_app,
"video:send_heartbeat",
serde_json::json!({
"t_ms": vid_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", vid_codec),
"platform": "android",
"camera_frames": camera_frames,
"encoded_frames": encoded_frames_total,
"encoded_keyframes": encoded_keyframes_total,
"empty_encodes": empty_encodes,
"pkts_sent": video_packets_total,
"bytes_sent": video_bytes_total,
"camera_fps": (camera_delta as f64) * 1000.0 / dt_ms,
"encoded_fps": (encoded_delta as f64) * 1000.0 / dt_ms,
"packets_per_sec": (packets_delta as f64) * 1000.0 / dt_ms,
"kbps": (bytes_delta as f64) * 8.0 / dt_ms,
"empty_encodes_delta": empty_delta,
}),
);
last_heartbeat_camera_frames = camera_frames;
last_heartbeat_encoded_frames = encoded_frames_total;
last_heartbeat_packets = video_packets_total;
last_heartbeat_bytes = video_bytes_total;
last_heartbeat_empty_encodes = empty_encodes;
send_heartbeat = std::time::Instant::now();
}
frames_since_keyframe += 1; frames_since_keyframe += 1;
} }
crate::emit_call_debug( crate::emit_call_debug(
@@ -2980,7 +3031,8 @@ impl CallEngine {
); );
} }
let jpeg_b64 = jpeg_bytes.as_ref().map(|bytes| { let jpeg_b64 = jpeg_bytes.as_ref().map(|bytes| {
base64::engine::general_purpose::STANDARD.encode(bytes) base64::engine::general_purpose::STANDARD
.encode(bytes)
}); });
let jpeg_ok = jpeg_b64.is_some(); let jpeg_ok = jpeg_b64.is_some();
if !video_first_decoded_logged { if !video_first_decoded_logged {
@@ -3315,40 +3367,43 @@ impl CallEngine {
"codec": format!("{:?}", vid_codec), "codec": format!("{:?}", vid_codec),
"width": 1280, "width": 1280,
"height": 720, "height": 720,
"bitrate_bps": 1_500_000, "bitrate_bps": VIDEO_BITRATE_BPS,
"platform": "desktop", "platform": "desktop",
}), }),
); );
let mut encoder = let mut encoder = match wzp_video::factory::create_video_encoder(
match wzp_video::factory::create_video_encoder(vid_codec, 1280, 720, 1_500_000) vid_codec,
{ 1280,
Ok(e) => { 720,
crate::emit_call_debug( VIDEO_BITRATE_BPS,
&vid_app, ) {
"video:encoder_started", Ok(e) => {
serde_json::json!({ crate::emit_call_debug(
"t_ms": vid_t0.elapsed().as_millis() as u64, &vid_app,
"codec": format!("{:?}", vid_codec), "video:encoder_started",
"platform": "desktop", serde_json::json!({
}), "t_ms": vid_t0.elapsed().as_millis() as u64,
); "codec": format!("{:?}", vid_codec),
e "platform": "desktop",
} }),
Err(e) => { );
error!("video encoder init failed: {e}"); e
crate::emit_call_debug( }
&vid_app, Err(e) => {
"video:encoder_init_failed", error!("video encoder init failed: {e}");
serde_json::json!({ crate::emit_call_debug(
"t_ms": vid_t0.elapsed().as_millis() as u64, &vid_app,
"codec": format!("{:?}", vid_codec), "video:encoder_init_failed",
"platform": "desktop", serde_json::json!({
"error": e.to_string(), "t_ms": vid_t0.elapsed().as_millis() as u64,
}), "codec": format!("{:?}", vid_codec),
); "platform": "desktop",
return; "error": e.to_string(),
} }),
}; );
return;
}
};
let mut seq: u32 = 0; let mut seq: u32 = 0;
let mut frames_since_keyframe: u32 = 0; let mut frames_since_keyframe: u32 = 0;
let mut first_send_logged = false; let mut first_send_logged = false;
@@ -3356,6 +3411,16 @@ impl CallEngine {
let mut camera_frames: u64 = 0; let mut camera_frames: u64 = 0;
let mut empty_encodes: u64 = 0; let mut empty_encodes: u64 = 0;
let mut encoded_frame_samples: u64 = 0; let mut encoded_frame_samples: u64 = 0;
let mut send_heartbeat = std::time::Instant::now();
let mut encoded_frames_total: u64 = 0;
let mut encoded_keyframes_total: u64 = 0;
let mut video_packets_total: u64 = 0;
let mut video_bytes_total: u64 = 0;
let mut last_heartbeat_camera_frames: u64 = 0;
let mut last_heartbeat_encoded_frames: u64 = 0;
let mut last_heartbeat_packets: u64 = 0;
let mut last_heartbeat_bytes: u64 = 0;
let mut last_heartbeat_empty_encodes: u64 = 0;
let mut skipped_startup_black_frames: u64 = 0; let mut skipped_startup_black_frames: u64 = 0;
let mut wait_ticks: u64 = 0; let mut wait_ticks: u64 = 0;
encoder.request_keyframe(); encoder.request_keyframe();
@@ -3457,14 +3522,13 @@ impl CallEngine {
continue; continue;
} }
let keyframe_reason = let keyframe_reason = if vid_force_keyframe.swap(false, Ordering::AcqRel) {
if vid_force_keyframe.swap(false, Ordering::AcqRel) { Some("pli")
Some("pli") } else if frames_since_keyframe >= VIDEO_KEYFRAME_INTERVAL_FRAMES {
} else if frames_since_keyframe >= VIDEO_KEYFRAME_INTERVAL_FRAMES { Some("periodic")
Some("periodic") } else {
} else { None
None };
};
if let Some(reason) = keyframe_reason { if let Some(reason) = keyframe_reason {
encoder.request_keyframe(); encoder.request_keyframe();
crate::emit_call_debug( crate::emit_call_debug(
@@ -3518,6 +3582,10 @@ impl CallEngine {
} }
let is_keyframe = encoder.is_keyframe(&encoded); let is_keyframe = encoder.is_keyframe(&encoded);
encoded_frames_total += 1;
if is_keyframe {
encoded_keyframes_total += 1;
}
let ts_ms = vid_t0.elapsed().as_millis() as u32; let ts_ms = vid_t0.elapsed().as_millis() as u32;
let pkts = wzp_video::transport::packetize_video_frame( let pkts = wzp_video::transport::packetize_video_frame(
&encoded, &encoded,
@@ -3526,6 +3594,8 @@ impl CallEngine {
&mut seq, &mut seq,
ts_ms, ts_ms,
); );
video_packets_total += pkts.len() as u64;
video_bytes_total += encoded.len() as u64;
if encoded_frame_samples < 5 { if encoded_frame_samples < 5 {
encoded_frame_samples += 1; encoded_frame_samples += 1;
let packet_payload_bytes: usize = let packet_payload_bytes: usize =
@@ -3577,6 +3647,40 @@ impl CallEngine {
break; break;
} }
} }
if send_heartbeat.elapsed() >= std::time::Duration::from_secs(2) {
let dt_ms = send_heartbeat.elapsed().as_millis().max(1) as f64;
let encoded_delta = encoded_frames_total - last_heartbeat_encoded_frames;
let camera_delta = camera_frames - last_heartbeat_camera_frames;
let packets_delta = video_packets_total - last_heartbeat_packets;
let bytes_delta = video_bytes_total - last_heartbeat_bytes;
let empty_delta = empty_encodes - last_heartbeat_empty_encodes;
crate::emit_call_debug(
&vid_app,
"video:send_heartbeat",
serde_json::json!({
"t_ms": vid_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", vid_codec),
"platform": "desktop",
"camera_frames": camera_frames,
"encoded_frames": encoded_frames_total,
"encoded_keyframes": encoded_keyframes_total,
"empty_encodes": empty_encodes,
"pkts_sent": video_packets_total,
"bytes_sent": video_bytes_total,
"camera_fps": (camera_delta as f64) * 1000.0 / dt_ms,
"encoded_fps": (encoded_delta as f64) * 1000.0 / dt_ms,
"packets_per_sec": (packets_delta as f64) * 1000.0 / dt_ms,
"kbps": (bytes_delta as f64) * 8.0 / dt_ms,
"empty_encodes_delta": empty_delta,
}),
);
last_heartbeat_camera_frames = camera_frames;
last_heartbeat_encoded_frames = encoded_frames_total;
last_heartbeat_packets = video_packets_total;
last_heartbeat_bytes = video_bytes_total;
last_heartbeat_empty_encodes = empty_encodes;
send_heartbeat = std::time::Instant::now();
}
frames_since_keyframe += 1; frames_since_keyframe += 1;
} }
crate::emit_call_debug( crate::emit_call_debug(

View File

@@ -574,7 +574,8 @@ const CAMERA_SEND_WIDTH = 1280;
const CAMERA_SEND_HEIGHT = 720; const CAMERA_SEND_HEIGHT = 720;
let cameraCaptureFrameNo = 0; let cameraCaptureFrameNo = 0;
let cameraPushFailures = 0; let cameraPushFailures = 0;
const CAMERA_CAPTURE_INTERVAL_MS = 67; // ≈ 15 fps const CAMERA_CAPTURE_INTERVAL_MS = 33; // ≈ 30 fps
const CAMERA_JPEG_QUALITY = 0.7;
function drawCameraFrameForSend() { function drawCameraFrameForSend() {
const vw = vdLocalVideo.videoWidth || camCaptureCanvas.width; const vw = vdLocalVideo.videoWidth || camCaptureCanvas.width;
@@ -598,7 +599,7 @@ async function captureAndPushCameraFrame() {
cameraCaptureFrameNo++; cameraCaptureFrameNo++;
try { try {
drawCameraFrameForSend(); drawCameraFrameForSend();
const dataUrl = camCaptureCanvas.toDataURL("image/jpeg", 0.75); const dataUrl = camCaptureCanvas.toDataURL("image/jpeg", CAMERA_JPEG_QUALITY);
const b64 = dataUrl.slice(dataUrl.indexOf(",") + 1); const b64 = dataUrl.slice(dataUrl.indexOf(",") + 1);
if (cameraCaptureFrameNo === 1 || cameraCaptureFrameNo % 150 === 0) { if (cameraCaptureFrameNo === 1 || cameraCaptureFrameNo % 150 === 0) {
debugLog("camera:capture_frame", { debugLog("camera:capture_frame", {