diff --git a/crates/wzp-video/src/mediacodec.rs b/crates/wzp-video/src/mediacodec.rs index 9347d29..eafe765 100644 --- a/crates/wzp-video/src/mediacodec.rs +++ b/crates/wzp-video/src/mediacodec.rs @@ -67,7 +67,7 @@ impl MediaCodecEncoder { format.set_i32("height", height as i32); format.set_i32("bitrate", bitrate_bps as i32); format.set_i32("frame-rate", 30); - format.set_i32("i-frame-interval", 1); + format.set_i32("i-frame-interval", 4); format.set_i32("color-format", COLOR_FORMAT_YUV420_SEMIPLANAR); let codec = MediaCodec::from_encoder_type("video/avc").ok_or_else(|| { @@ -170,7 +170,8 @@ impl VideoEncoder for MediaCodecEncoder { if nals.is_empty() { return (packet[0] & 0x1F) == 5; } - nals.iter().any(|nal| !nal.is_empty() && (nal[0] & 0x1F) == 5) + nals.iter() + .any(|nal| !nal.is_empty() && (nal[0] & 0x1F) == 5) } } @@ -419,7 +420,7 @@ impl MediaCodecHevcEncoder { format.set_i32("height", height as i32); format.set_i32("bitrate", bitrate_bps as i32); format.set_i32("frame-rate", 30); - format.set_i32("i-frame-interval", 1); + format.set_i32("i-frame-interval", 4); format.set_i32("color-format", COLOR_FORMAT_YUV420_PLANAR); let codec = MediaCodec::from_encoder_type("video/hevc").ok_or_else(|| { @@ -470,7 +471,11 @@ impl VideoEncoder for MediaCodecHevcEncoder { .dequeue_input_buffer(std::time::Duration::from_millis(10)) { Ok(ndk::media::media_codec::DequeuedInputBufferResult::Buffer(mut buffer)) => { - let flags = if self.force_keyframe { AMEDIACODEC_BUFFER_FLAG_KEY_FRAME } else { 0 }; + let flags = if self.force_keyframe { + AMEDIACODEC_BUFFER_FLAG_KEY_FRAME + } else { + 0 + }; let to_copy = { let buf = buffer.buffer_mut(); let n = frame.data.len().min(buf.len()); @@ -480,7 +485,13 @@ impl VideoEncoder for MediaCodecHevcEncoder { n }; self.codec - .queue_input_buffer(buffer, 0, to_copy, frame.timestamp_ms as u64 * 1000, flags) + .queue_input_buffer( + buffer, + 0, + to_copy, + frame.timestamp_ms as u64 * 1000, + flags, + ) .map_err(|e| { VideoError::PlatformError(format!("queue_input_buffer failed: {e}")) })?; @@ -592,7 +603,11 @@ impl VideoEncoder for MediaCodecAv1Encoder { .dequeue_input_buffer(std::time::Duration::from_millis(0)) { Ok(ndk::media::media_codec::DequeuedInputBufferResult::Buffer(mut buffer)) => { - let flags = if self.force_keyframe { AMEDIACODEC_BUFFER_FLAG_KEY_FRAME } else { 0 }; + let flags = if self.force_keyframe { + AMEDIACODEC_BUFFER_FLAG_KEY_FRAME + } else { + 0 + }; let to_copy = { let buf = buffer.buffer_mut(); let n = frame.data.len().min(buf.len()); @@ -602,7 +617,13 @@ impl VideoEncoder for MediaCodecAv1Encoder { n }; self.codec - .queue_input_buffer(buffer, 0, to_copy, frame.timestamp_ms as u64 * 1000, flags) + .queue_input_buffer( + buffer, + 0, + to_copy, + frame.timestamp_ms as u64 * 1000, + flags, + ) .map_err(|e| { VideoError::PlatformError(format!( "AV1 encoder queue_input_buffer failed: {e}" @@ -1162,9 +1183,9 @@ fn i420_len(width: usize, height: usize) -> Result { #[cfg(target_os = "android")] fn i420_to_nv12(src: &[u8], width: usize, height: usize) -> Result, VideoError> { - let y_size = width - .checked_mul(height) - .ok_or_else(|| VideoError::InvalidInput(format!("invalid frame dimensions {width}x{height}")))?; + let y_size = width.checked_mul(height).ok_or_else(|| { + VideoError::InvalidInput(format!("invalid frame dimensions {width}x{height}")) + })?; let uv_size = y_size / 4; let expected = y_size + uv_size * 2; if src.len() < expected { diff --git a/desktop/src-tauri/src/engine.rs b/desktop/src-tauri/src/engine.rs index df75108..3f4c688 100644 --- a/desktop/src-tauri/src/engine.rs +++ b/desktop/src-tauri/src/engine.rs @@ -8,11 +8,11 @@ //! `start()` that returns an error, so the frontend's `connect` command //! still fails cleanly but the rest of the engine code links in. +use base64::Engine as _; use std::net::SocketAddr; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering}; use std::time::Instant; -use base64::Engine as _; use tauri::Emitter; use tokio::sync::Mutex; use tracing::{error, info}; @@ -183,6 +183,7 @@ fn should_log_video_sample(frame_no: u64, is_keyframe: bool) -> bool { } const VIDEO_KEYFRAME_INTERVAL_FRAMES: u32 = 120; +const VIDEO_BITRATE_BPS: u32 = 900_000; const VIDEO_PLI_MIN_INTERVAL_MS: u128 = 250; #[derive(Default)] @@ -397,10 +398,7 @@ async fn run_signal_task( ); pending_profile.store(idx, Ordering::Release); } - Ok(Ok(Some(wzp_proto::SignalMessage::PictureLossIndication { - stream_id, - .. - }))) => { + Ok(Ok(Some(wzp_proto::SignalMessage::PictureLossIndication { stream_id, .. }))) => { force_video_keyframe.store(true, Ordering::Release); crate::emit_call_debug( &app, @@ -1556,7 +1554,8 @@ impl CallEngine { ); } let jpeg_b64 = jpeg_bytes.as_ref().map(|bytes| { - base64::engine::general_purpose::STANDARD.encode(bytes) + base64::engine::general_purpose::STANDARD + .encode(bytes) }); let jpeg_ok = jpeg_b64.is_some(); if !video_first_decoded_logged { @@ -2049,40 +2048,43 @@ impl CallEngine { "codec": format!("{:?}", vid_codec), "width": 1280, "height": 720, - "bitrate_bps": 1_500_000, + "bitrate_bps": VIDEO_BITRATE_BPS, "platform": "android", }), ); - let mut encoder = - match wzp_video::factory::create_video_encoder(vid_codec, 1280, 720, 1_500_000) - { - Ok(e) => { - crate::emit_call_debug( - &vid_app, - "video:encoder_started", - serde_json::json!({ - "t_ms": vid_t0.elapsed().as_millis() as u64, - "codec": format!("{:?}", vid_codec), - "platform": "android", - }), - ); - e - } - Err(e) => { - error!("video encoder init failed (android): {e}"); - crate::emit_call_debug( - &vid_app, - "video:encoder_init_failed", - serde_json::json!({ - "t_ms": vid_t0.elapsed().as_millis() as u64, - "codec": format!("{:?}", vid_codec), - "platform": "android", - "error": e.to_string(), - }), - ); - return; - } - }; + let mut encoder = match wzp_video::factory::create_video_encoder( + vid_codec, + 1280, + 720, + VIDEO_BITRATE_BPS, + ) { + Ok(e) => { + crate::emit_call_debug( + &vid_app, + "video:encoder_started", + serde_json::json!({ + "t_ms": vid_t0.elapsed().as_millis() as u64, + "codec": format!("{:?}", vid_codec), + "platform": "android", + }), + ); + e + } + Err(e) => { + error!("video encoder init failed (android): {e}"); + crate::emit_call_debug( + &vid_app, + "video:encoder_init_failed", + serde_json::json!({ + "t_ms": vid_t0.elapsed().as_millis() as u64, + "codec": format!("{:?}", vid_codec), + "platform": "android", + "error": e.to_string(), + }), + ); + return; + } + }; let mut seq: u32 = 0; let mut frames_since_keyframe: u32 = 0; let mut first_send_logged = false; @@ -2090,6 +2092,16 @@ impl CallEngine { let mut camera_frames: u64 = 0; let mut empty_encodes: u64 = 0; let mut encoded_frame_samples: u64 = 0; + let mut send_heartbeat = std::time::Instant::now(); + let mut encoded_frames_total: u64 = 0; + let mut encoded_keyframes_total: u64 = 0; + let mut video_packets_total: u64 = 0; + let mut video_bytes_total: u64 = 0; + let mut last_heartbeat_camera_frames: u64 = 0; + let mut last_heartbeat_encoded_frames: u64 = 0; + let mut last_heartbeat_packets: u64 = 0; + let mut last_heartbeat_bytes: u64 = 0; + let mut last_heartbeat_empty_encodes: u64 = 0; let mut skipped_startup_black_frames: u64 = 0; let mut wait_ticks: u64 = 0; encoder.request_keyframe(); @@ -2191,14 +2203,13 @@ impl CallEngine { continue; } - let keyframe_reason = - if vid_force_keyframe.swap(false, Ordering::AcqRel) { - Some("pli") - } else if frames_since_keyframe >= VIDEO_KEYFRAME_INTERVAL_FRAMES { - Some("periodic") - } else { - None - }; + let keyframe_reason = if vid_force_keyframe.swap(false, Ordering::AcqRel) { + Some("pli") + } else if frames_since_keyframe >= VIDEO_KEYFRAME_INTERVAL_FRAMES { + Some("periodic") + } else { + None + }; if let Some(reason) = keyframe_reason { encoder.request_keyframe(); crate::emit_call_debug( @@ -2252,6 +2263,10 @@ impl CallEngine { } let is_keyframe = encoder.is_keyframe(&encoded); + encoded_frames_total += 1; + if is_keyframe { + encoded_keyframes_total += 1; + } let ts_ms = vid_t0.elapsed().as_millis() as u32; let pkts = wzp_video::transport::packetize_video_frame( &encoded, @@ -2260,6 +2275,8 @@ impl CallEngine { &mut seq, ts_ms, ); + video_packets_total += pkts.len() as u64; + video_bytes_total += encoded.len() as u64; if encoded_frame_samples < 5 { encoded_frame_samples += 1; let packet_payload_bytes: usize = @@ -2311,6 +2328,40 @@ impl CallEngine { break; } } + if send_heartbeat.elapsed() >= std::time::Duration::from_secs(2) { + let dt_ms = send_heartbeat.elapsed().as_millis().max(1) as f64; + let encoded_delta = encoded_frames_total - last_heartbeat_encoded_frames; + let camera_delta = camera_frames - last_heartbeat_camera_frames; + let packets_delta = video_packets_total - last_heartbeat_packets; + let bytes_delta = video_bytes_total - last_heartbeat_bytes; + let empty_delta = empty_encodes - last_heartbeat_empty_encodes; + crate::emit_call_debug( + &vid_app, + "video:send_heartbeat", + serde_json::json!({ + "t_ms": vid_t0.elapsed().as_millis() as u64, + "codec": format!("{:?}", vid_codec), + "platform": "android", + "camera_frames": camera_frames, + "encoded_frames": encoded_frames_total, + "encoded_keyframes": encoded_keyframes_total, + "empty_encodes": empty_encodes, + "pkts_sent": video_packets_total, + "bytes_sent": video_bytes_total, + "camera_fps": (camera_delta as f64) * 1000.0 / dt_ms, + "encoded_fps": (encoded_delta as f64) * 1000.0 / dt_ms, + "packets_per_sec": (packets_delta as f64) * 1000.0 / dt_ms, + "kbps": (bytes_delta as f64) * 8.0 / dt_ms, + "empty_encodes_delta": empty_delta, + }), + ); + last_heartbeat_camera_frames = camera_frames; + last_heartbeat_encoded_frames = encoded_frames_total; + last_heartbeat_packets = video_packets_total; + last_heartbeat_bytes = video_bytes_total; + last_heartbeat_empty_encodes = empty_encodes; + send_heartbeat = std::time::Instant::now(); + } frames_since_keyframe += 1; } crate::emit_call_debug( @@ -2980,7 +3031,8 @@ impl CallEngine { ); } let jpeg_b64 = jpeg_bytes.as_ref().map(|bytes| { - base64::engine::general_purpose::STANDARD.encode(bytes) + base64::engine::general_purpose::STANDARD + .encode(bytes) }); let jpeg_ok = jpeg_b64.is_some(); if !video_first_decoded_logged { @@ -3315,40 +3367,43 @@ impl CallEngine { "codec": format!("{:?}", vid_codec), "width": 1280, "height": 720, - "bitrate_bps": 1_500_000, + "bitrate_bps": VIDEO_BITRATE_BPS, "platform": "desktop", }), ); - let mut encoder = - match wzp_video::factory::create_video_encoder(vid_codec, 1280, 720, 1_500_000) - { - Ok(e) => { - crate::emit_call_debug( - &vid_app, - "video:encoder_started", - serde_json::json!({ - "t_ms": vid_t0.elapsed().as_millis() as u64, - "codec": format!("{:?}", vid_codec), - "platform": "desktop", - }), - ); - e - } - Err(e) => { - error!("video encoder init failed: {e}"); - crate::emit_call_debug( - &vid_app, - "video:encoder_init_failed", - serde_json::json!({ - "t_ms": vid_t0.elapsed().as_millis() as u64, - "codec": format!("{:?}", vid_codec), - "platform": "desktop", - "error": e.to_string(), - }), - ); - return; - } - }; + let mut encoder = match wzp_video::factory::create_video_encoder( + vid_codec, + 1280, + 720, + VIDEO_BITRATE_BPS, + ) { + Ok(e) => { + crate::emit_call_debug( + &vid_app, + "video:encoder_started", + serde_json::json!({ + "t_ms": vid_t0.elapsed().as_millis() as u64, + "codec": format!("{:?}", vid_codec), + "platform": "desktop", + }), + ); + e + } + Err(e) => { + error!("video encoder init failed: {e}"); + crate::emit_call_debug( + &vid_app, + "video:encoder_init_failed", + serde_json::json!({ + "t_ms": vid_t0.elapsed().as_millis() as u64, + "codec": format!("{:?}", vid_codec), + "platform": "desktop", + "error": e.to_string(), + }), + ); + return; + } + }; let mut seq: u32 = 0; let mut frames_since_keyframe: u32 = 0; let mut first_send_logged = false; @@ -3356,6 +3411,16 @@ impl CallEngine { let mut camera_frames: u64 = 0; let mut empty_encodes: u64 = 0; let mut encoded_frame_samples: u64 = 0; + let mut send_heartbeat = std::time::Instant::now(); + let mut encoded_frames_total: u64 = 0; + let mut encoded_keyframes_total: u64 = 0; + let mut video_packets_total: u64 = 0; + let mut video_bytes_total: u64 = 0; + let mut last_heartbeat_camera_frames: u64 = 0; + let mut last_heartbeat_encoded_frames: u64 = 0; + let mut last_heartbeat_packets: u64 = 0; + let mut last_heartbeat_bytes: u64 = 0; + let mut last_heartbeat_empty_encodes: u64 = 0; let mut skipped_startup_black_frames: u64 = 0; let mut wait_ticks: u64 = 0; encoder.request_keyframe(); @@ -3457,14 +3522,13 @@ impl CallEngine { continue; } - let keyframe_reason = - if vid_force_keyframe.swap(false, Ordering::AcqRel) { - Some("pli") - } else if frames_since_keyframe >= VIDEO_KEYFRAME_INTERVAL_FRAMES { - Some("periodic") - } else { - None - }; + let keyframe_reason = if vid_force_keyframe.swap(false, Ordering::AcqRel) { + Some("pli") + } else if frames_since_keyframe >= VIDEO_KEYFRAME_INTERVAL_FRAMES { + Some("periodic") + } else { + None + }; if let Some(reason) = keyframe_reason { encoder.request_keyframe(); crate::emit_call_debug( @@ -3518,6 +3582,10 @@ impl CallEngine { } let is_keyframe = encoder.is_keyframe(&encoded); + encoded_frames_total += 1; + if is_keyframe { + encoded_keyframes_total += 1; + } let ts_ms = vid_t0.elapsed().as_millis() as u32; let pkts = wzp_video::transport::packetize_video_frame( &encoded, @@ -3526,6 +3594,8 @@ impl CallEngine { &mut seq, ts_ms, ); + video_packets_total += pkts.len() as u64; + video_bytes_total += encoded.len() as u64; if encoded_frame_samples < 5 { encoded_frame_samples += 1; let packet_payload_bytes: usize = @@ -3577,6 +3647,40 @@ impl CallEngine { break; } } + if send_heartbeat.elapsed() >= std::time::Duration::from_secs(2) { + let dt_ms = send_heartbeat.elapsed().as_millis().max(1) as f64; + let encoded_delta = encoded_frames_total - last_heartbeat_encoded_frames; + let camera_delta = camera_frames - last_heartbeat_camera_frames; + let packets_delta = video_packets_total - last_heartbeat_packets; + let bytes_delta = video_bytes_total - last_heartbeat_bytes; + let empty_delta = empty_encodes - last_heartbeat_empty_encodes; + crate::emit_call_debug( + &vid_app, + "video:send_heartbeat", + serde_json::json!({ + "t_ms": vid_t0.elapsed().as_millis() as u64, + "codec": format!("{:?}", vid_codec), + "platform": "desktop", + "camera_frames": camera_frames, + "encoded_frames": encoded_frames_total, + "encoded_keyframes": encoded_keyframes_total, + "empty_encodes": empty_encodes, + "pkts_sent": video_packets_total, + "bytes_sent": video_bytes_total, + "camera_fps": (camera_delta as f64) * 1000.0 / dt_ms, + "encoded_fps": (encoded_delta as f64) * 1000.0 / dt_ms, + "packets_per_sec": (packets_delta as f64) * 1000.0 / dt_ms, + "kbps": (bytes_delta as f64) * 8.0 / dt_ms, + "empty_encodes_delta": empty_delta, + }), + ); + last_heartbeat_camera_frames = camera_frames; + last_heartbeat_encoded_frames = encoded_frames_total; + last_heartbeat_packets = video_packets_total; + last_heartbeat_bytes = video_bytes_total; + last_heartbeat_empty_encodes = empty_encodes; + send_heartbeat = std::time::Instant::now(); + } frames_since_keyframe += 1; } crate::emit_call_debug( diff --git a/desktop/src/main.ts b/desktop/src/main.ts index 8889c4e..38e6e81 100644 --- a/desktop/src/main.ts +++ b/desktop/src/main.ts @@ -574,7 +574,8 @@ const CAMERA_SEND_WIDTH = 1280; const CAMERA_SEND_HEIGHT = 720; let cameraCaptureFrameNo = 0; let cameraPushFailures = 0; -const CAMERA_CAPTURE_INTERVAL_MS = 67; // ≈ 15 fps +const CAMERA_CAPTURE_INTERVAL_MS = 33; // ≈ 30 fps +const CAMERA_JPEG_QUALITY = 0.7; function drawCameraFrameForSend() { const vw = vdLocalVideo.videoWidth || camCaptureCanvas.width; @@ -598,7 +599,7 @@ async function captureAndPushCameraFrame() { cameraCaptureFrameNo++; try { drawCameraFrameForSend(); - const dataUrl = camCaptureCanvas.toDataURL("image/jpeg", 0.75); + const dataUrl = camCaptureCanvas.toDataURL("image/jpeg", CAMERA_JPEG_QUALITY); const b64 = dataUrl.slice(dataUrl.indexOf(",") + 1); if (cameraCaptureFrameNo === 1 || cameraCaptureFrameNo % 150 === 0) { debugLog("camera:capture_frame", {