feat(video): add codec and resolution controls
Some checks failed
Build Release Binaries / build-amd64 (push) Failing after 3m38s
Mirror to GitHub / mirror (push) Failing after 38s

This commit is contained in:
Siavash Sameni
2026-05-26 10:05:20 +04:00
parent f85efb9576
commit 9a7745978b
7 changed files with 250 additions and 92 deletions

View File

@@ -174,6 +174,22 @@
OS Echo Cancellation
</label>
</div>
<div class="settings-section">
<h3>Video</h3>
<label>Codec
<select id="s-video-codec">
<option value="h264">H.264</option>
<option value="h265">H.265 / HEVC</option>
</select>
</label>
<label>Room Resolution
<select id="s-video-resolution">
<option value="640x360">640 x 360</option>
<option value="960x540">960 x 540</option>
<option value="1280x720">1280 x 720</option>
</select>
</label>
</div>
<div class="settings-section">
<h3>Relays</h3>
<div id="s-relay-list"></div>

View File

@@ -186,6 +186,19 @@ const VIDEO_KEYFRAME_INTERVAL_FRAMES: u32 = 120;
const VIDEO_BITRATE_BPS: u32 = 900_000;
const VIDEO_PLI_MIN_INTERVAL_MS: u128 = 250;
fn parse_video_codec(codec: &str) -> wzp_proto::CodecId {
match codec.to_ascii_lowercase().as_str() {
"h265" | "h265main" | "hevc" => wzp_proto::CodecId::H265Main,
"av1" | "av1main" => wzp_proto::CodecId::Av1Main,
_ => wzp_proto::CodecId::H264Baseline,
}
}
fn clamp_video_dimension(value: u32, fallback: u32) -> u32 {
let value = if value == 0 { fallback } else { value };
value.clamp(160, 1920) & !1
}
#[derive(Default)]
struct VideoContinuity {
expected_seq: Option<u32>,
@@ -684,17 +697,26 @@ impl CallEngine {
app: tauri::AppHandle,
active_quality: Arc<std::sync::Mutex<wzp_proto::QualityProfile>>,
peer_max_quality: Arc<std::sync::Mutex<Option<wzp_proto::QualityProfile>>>,
video_codec_preference: String,
video_width: u32,
video_height: u32,
event_cb: F,
) -> Result<Self, anyhow::Error>
where
F: Fn(&str, &str) + Send + Sync + 'static,
{
let call_t0 = std::time::Instant::now();
let preferred_video_codec = parse_video_codec(&video_codec_preference);
let video_width = clamp_video_dimension(video_width, 1280);
let video_height = clamp_video_dimension(video_height, 720);
info!(
%relay, %room, %alias, %quality,
has_reuse = reuse_endpoint.is_some(),
has_pre_connected = pre_connected_transport.is_some(),
is_direct_p2p,
video_codec = ?preferred_video_codec,
video_width,
video_height,
t_ms = 0u128,
"CallEngine::start (android) invoked"
);
@@ -788,24 +810,28 @@ impl CallEngine {
"remote": transport.remote_address().to_string(),
}),
);
let hs =
match wzp_client::handshake::perform_handshake(&*transport, &seed.0, Some(&alias))
.await
{
Ok(hs) => hs,
Err(e) => {
error!("perform_handshake failed: {e}");
crate::emit_call_debug(
&app,
"connect:handshake_failed",
serde_json::json!({
"t_ms": call_t0.elapsed().as_millis(),
"error": e.to_string(),
}),
);
return Err(e.into());
}
};
let hs = match wzp_client::handshake::perform_handshake_with_video_codecs(
&*transport,
&seed.0,
Some(&alias),
vec![preferred_video_codec],
)
.await
{
Ok(hs) => hs,
Err(e) => {
error!("perform_handshake failed: {e}");
crate::emit_call_debug(
&app,
"connect:handshake_failed",
serde_json::json!({
"t_ms": call_t0.elapsed().as_millis(),
"error": e.to_string(),
}),
);
return Err(e.into());
}
};
crate::emit_call_debug(
&app,
"connect:handshake_done",
@@ -829,7 +855,7 @@ impl CallEngine {
t_ms = call_t0.elapsed().as_millis(),
"first-join diag: direct P2P — skipping relay handshake (QUIC TLS is the encryption layer)"
);
(Some(wzp_proto::CodecId::H264Baseline), transport)
(Some(preferred_video_codec), transport)
};
crate::emit_call_debug(
&app,
@@ -838,6 +864,8 @@ impl CallEngine {
"t_ms": call_t0.elapsed().as_millis(),
"codec": _negotiated_video_codec.map(|c| format!("{:?}", c)),
"enabled": _negotiated_video_codec.is_some(),
"width": video_width,
"height": video_height,
"direct_p2p": is_direct_p2p,
}),
);
@@ -1496,13 +1524,15 @@ impl CallEngine {
serde_json::json!({
"t_ms": recv_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", codec_id),
"width": 1280,
"height": 720,
"width": video_width,
"height": video_height,
"platform": "android",
}),
);
match wzp_video::factory::create_video_decoder(
codec_id, 1280, 720,
codec_id,
video_width,
video_height,
) {
Ok(d) => {
info!(codec = ?codec_id, "video decoder created (android)");
@@ -2020,8 +2050,8 @@ impl CallEngine {
// Video send task (Android) — mirror of the desktop branch. Only
// spawns when a video codec is available. Relay calls negotiate this
// in the media handshake; direct P2P uses the common H264 baseline
// codec because the relay handshake is intentionally skipped.
// in the media handshake; direct P2P uses the local debug codec
// preference because the relay handshake is intentionally skipped.
let camera_tx = if let Some(vid_codec) = _negotiated_video_codec {
let (tx, mut rx) = tokio::sync::mpsc::channel::<wzp_video::encoder::VideoFrame>(4);
let vid_transport = transport.clone();
@@ -2046,16 +2076,16 @@ impl CallEngine {
serde_json::json!({
"t_ms": vid_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", vid_codec),
"width": 1280,
"height": 720,
"width": video_width,
"height": video_height,
"bitrate_bps": VIDEO_BITRATE_BPS,
"platform": "android",
}),
);
let mut encoder = match wzp_video::factory::create_video_encoder(
vid_codec,
1280,
720,
video_width,
video_height,
VIDEO_BITRATE_BPS,
) {
Ok(e) => {
@@ -2428,19 +2458,28 @@ impl CallEngine {
_app: tauri::AppHandle,
active_quality: Arc<std::sync::Mutex<wzp_proto::QualityProfile>>,
peer_max_quality: Arc<std::sync::Mutex<Option<wzp_proto::QualityProfile>>>,
video_codec_preference: String,
video_width: u32,
video_height: u32,
event_cb: F,
) -> Result<Self, anyhow::Error>
where
F: Fn(&str, &str) + Send + Sync + 'static,
{
let call_t0 = Instant::now();
let preferred_video_codec = parse_video_codec(&video_codec_preference);
let video_width = clamp_video_dimension(video_width, 1280);
let video_height = clamp_video_dimension(video_height, 720);
info!(
%relay, %room, %alias, %quality,
has_reuse = reuse_endpoint.is_some(),
has_pre_connected = pre_connected_transport.is_some(),
is_direct_p2p,
video_codec = ?preferred_video_codec,
video_width,
video_height,
"CallEngine::start (desktop) invoked"
);
let call_t0 = Instant::now();
let _ = rustls::crypto::ring::default_provider().install_default();
let relay_addr: SocketAddr = relay.parse()?;
@@ -2498,13 +2537,17 @@ impl CallEngine {
// PRD lands, media goes plaintext-over-QUIC-TLS to the relay.
let (_negotiated_video_codec, transport): (_, Arc<dyn wzp_proto::MediaTransport>) =
if !is_direct_p2p {
let hs =
wzp_client::handshake::perform_handshake(&*transport, &seed.0, Some(&alias))
.await
.map_err(|e| {
error!("perform_handshake failed: {e}");
e
})?;
let hs = wzp_client::handshake::perform_handshake_with_video_codecs(
&*transport,
&seed.0,
Some(&alias),
vec![preferred_video_codec],
)
.await
.map_err(|e| {
error!("perform_handshake failed: {e}");
e
})?;
crate::emit_call_debug(
&_app,
"connect:handshake_done",
@@ -2518,7 +2561,7 @@ impl CallEngine {
(hs.video_codec, transport)
} else {
info!("direct P2P — skipping relay handshake (QUIC TLS is the encryption layer)");
(Some(wzp_proto::CodecId::H264Baseline), transport)
(Some(preferred_video_codec), transport)
};
crate::emit_call_debug(
&_app,
@@ -2527,6 +2570,8 @@ impl CallEngine {
"t_ms": call_t0.elapsed().as_millis(),
"codec": _negotiated_video_codec.map(|c| format!("{:?}", c)),
"enabled": _negotiated_video_codec.is_some(),
"width": video_width,
"height": video_height,
"direct_p2p": is_direct_p2p,
}),
);
@@ -2970,13 +3015,15 @@ impl CallEngine {
serde_json::json!({
"t_ms": recv_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", codec_id),
"width": 1280,
"height": 720,
"width": video_width,
"height": video_height,
"platform": "desktop",
}),
);
match wzp_video::factory::create_video_decoder(
codec_id, 1280, 720,
codec_id,
video_width,
video_height,
) {
Ok(d) => {
info!(codec = ?codec_id, "video decoder created");
@@ -3339,8 +3386,8 @@ impl CallEngine {
));
// Video send task — active when a video codec is available. Relay calls
// negotiate this in the media handshake; direct P2P uses the common H264
// baseline codec because the relay handshake is intentionally skipped.
// negotiate this in the media handshake; direct P2P uses the local debug
// codec preference because the relay handshake is intentionally skipped.
let camera_tx = if let Some(vid_codec) = _negotiated_video_codec {
let (tx, mut rx) = tokio::sync::mpsc::channel::<wzp_video::encoder::VideoFrame>(4);
let vid_transport = transport.clone();
@@ -3365,16 +3412,16 @@ impl CallEngine {
serde_json::json!({
"t_ms": vid_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", vid_codec),
"width": 1280,
"height": 720,
"width": video_width,
"height": video_height,
"bitrate_bps": VIDEO_BITRATE_BPS,
"platform": "desktop",
}),
);
let mut encoder = match wzp_video::factory::create_video_encoder(
vid_codec,
1280,
720,
video_width,
video_height,
VIDEO_BITRATE_BPS,
) {
Ok(e) => {

View File

@@ -137,13 +137,15 @@ pub(crate) fn i420_to_jpeg_bytes(data: &[u8], width: u32, height: u32) -> Option
let u = data[y_size + uv_idx] as f32 - 128.0;
let v = data[y_size + uv_size + uv_idx] as f32 - 128.0;
let out = (row * w + col) * 3;
rgb[out] = (y + 1.402 * v).clamp(0.0, 255.0) as u8;
rgb[out] = (y + 1.402 * v).clamp(0.0, 255.0) as u8;
rgb[out + 1] = (y - 0.344 * u - 0.714 * v).clamp(0.0, 255.0) as u8;
rgb[out + 2] = (y + 1.772 * u).clamp(0.0, 255.0) as u8;
}
}
let img = DynamicImage::ImageRgb8(ImageBuffer::<Rgb<u8>, Vec<u8>>::from_raw(width, height, rgb)?);
let img = DynamicImage::ImageRgb8(ImageBuffer::<Rgb<u8>, Vec<u8>>::from_raw(
width, height, rgb,
)?);
let mut buf = std::io::Cursor::new(Vec::<u8>::new());
img.write_to(&mut buf, image::ImageFormat::Jpeg).ok()?;
Some(buf.into_inner())
@@ -217,8 +219,10 @@ fn rgb_to_i420(rgb: &[u8], w: usize, h: usize) -> Vec<u8> {
out[row * w + col] = (0.299 * r + 0.587 * g + 0.114 * b).clamp(0.0, 255.0) as u8;
if row % 2 == 0 && col % 2 == 0 {
let uv = (row / 2) * (w / 2) + col / 2;
out[y_size + uv] = (-0.169 * r - 0.331 * g + 0.500 * b + 128.0).clamp(0.0, 255.0) as u8;
out[y_size + uv_size + uv] = (0.500 * r - 0.419 * g - 0.081 * b + 128.0).clamp(0.0, 255.0) as u8;
out[y_size + uv] =
(-0.169 * r - 0.331 * g + 0.500 * b + 128.0).clamp(0.0, 255.0) as u8;
out[y_size + uv_size + uv] =
(0.500 * r - 0.419 * g - 0.081 * b + 128.0).clamp(0.0, 255.0) as u8;
}
}
}
@@ -387,7 +391,7 @@ mod video_tests {
fn solid_rgb_frame(w: usize, h: usize, r: u8, g: u8, b: u8) -> Vec<u8> {
let mut rgb = vec![0u8; w * h * 3];
for i in 0..w * h {
rgb[i * 3] = r;
rgb[i * 3] = r;
rgb[i * 3 + 1] = g;
rgb[i * 3 + 2] = b;
}
@@ -439,8 +443,14 @@ mod video_tests {
let s = b64.unwrap();
assert!(!s.is_empty());
// JPEG base64 starts with '/9j/' (FFD8FF marker).
let decoded = base64::engine::general_purpose::STANDARD.decode(&s).unwrap();
assert_eq!(&decoded[0..2], &[0xFF, 0xD8], "output must start with JPEG SOI marker");
let decoded = base64::engine::general_purpose::STANDARD
.decode(&s)
.unwrap();
assert_eq!(
&decoded[0..2],
&[0xFF, 0xD8],
"output must start with JPEG SOI marker"
);
}
#[test]
@@ -463,13 +473,18 @@ mod video_tests {
let yuv = rgb_to_i420(&rgb, 64, 64);
let b64 = i420_to_jpeg_b64(&yuv, 64, 64).expect("should produce JPEG");
let jpeg = base64::engine::general_purpose::STANDARD.decode(&b64).unwrap();
let jpeg = base64::engine::general_purpose::STANDARD
.decode(&b64)
.unwrap();
let img = image::load_from_memory_with_format(&jpeg, image::ImageFormat::Jpeg).unwrap();
let rgb_img = img.to_rgb8();
let px = rgb_img.get_pixel(32, 32);
let (r, g, b) = (px[0], px[1], px[2]);
assert!(r > g && r > b, "red frame: expected R dominant, got R={r} G={g} B={b}");
assert!(
r > g && r > b,
"red frame: expected R dominant, got R={r} G={g} B={b}"
);
}
#[test]
@@ -746,8 +761,14 @@ async fn connect(
// Enable birthday attack for hard NAT traversal. Adds ~3s to
// call setup when peer has symmetric NAT.
birthday_attack: Option<bool>,
video_codec: Option<String>,
video_width: Option<u32>,
video_height: Option<u32>,
) -> Result<String, String> {
let force_direct = direct_only.unwrap_or(false);
let video_codec = video_codec.unwrap_or_else(|| "h264".to_string());
let video_width = video_width.unwrap_or(1280);
let video_height = video_height.unwrap_or(720);
let enable_birthday = birthday_attack.unwrap_or(false);
emit_call_debug(
&app,
@@ -760,6 +781,9 @@ async fn connect(
"peer_mapped_addr": peer_mapped_addr,
"direct_only": force_direct,
"birthday_attack": enable_birthday,
"video_codec": video_codec,
"video_width": video_width,
"video_height": video_height,
}),
);
let mut engine_lock = state.engine.lock().await;
@@ -1218,6 +1242,9 @@ async fn connect(
app_for_engine,
active_quality,
peer_max_quality,
video_codec,
video_width,
video_height,
move |event_kind, message| {
let _ = app_clone.emit(
"call-event",
@@ -2129,7 +2156,9 @@ fn do_register_signal(
"peer_loss_pct": local_loss_pct, "peer_rtt_ms": local_rtt_ms,
}),
);
if let Err(e) = handle_upgrade_proposal(&*transport, &call_id, &proposal_id).await {
if let Err(e) =
handle_upgrade_proposal(&*transport, &call_id, &proposal_id).await
{
tracing::warn!("failed to send UpgradeResponse: {e}");
}
}
@@ -2150,8 +2179,14 @@ fn do_register_signal(
}),
);
if let Err(e) = handle_upgrade_response(
&*transport, &signal_state, &call_id, &proposal_id, accepted,
).await {
&*transport,
&signal_state,
&call_id,
&proposal_id,
accepted,
)
.await
{
tracing::warn!("failed to handle UpgradeResponse: {e}");
}
}
@@ -3426,7 +3461,9 @@ mod signal_tests {
#[tokio::test]
async fn upgrade_proposal_auto_accepts() {
let transport = LoopbackTransport::new();
handle_upgrade_proposal(&*transport, "c1", "p1").await.unwrap();
handle_upgrade_proposal(&*transport, "c1", "p1")
.await
.unwrap();
let sent = transport.take_sent();
assert_eq!(sent.len(), 1);
@@ -3453,8 +3490,11 @@ mod signal_tests {
let signal_state = empty_signal_state();
{
let sig = signal_state.lock().await;
*sig.pending_upgrade.lock().unwrap() =
Some(("c1".into(), "p1".into(), wzp_proto::QualityProfile::STUDIO_48K));
*sig.pending_upgrade.lock().unwrap() = Some((
"c1".into(),
"p1".into(),
wzp_proto::QualityProfile::STUDIO_48K,
));
}
handle_upgrade_response(&*transport, &signal_state, "c1", "p1", true)

View File

@@ -122,6 +122,8 @@ const sCallDebugCopyBtn = document.getElementById("s-call-debug-copy") as HTMLBu
const sCallDebugShareBtn = document.getElementById("s-call-debug-share") as HTMLButtonElement;
const sQuality = document.getElementById("s-quality") as HTMLInputElement;
const sQualityLabel = document.getElementById("s-quality-label")!;
const sVideoCodec = document.getElementById("s-video-codec") as HTMLSelectElement;
const sVideoResolution = document.getElementById("s-video-resolution") as HTMLSelectElement;
const sFingerprint = document.getElementById("s-fingerprint")!;
const sPublicAddr = document.getElementById("s-public-addr")!;
const sReflectBtn = document.getElementById("s-reflect-btn")!;
@@ -138,6 +140,8 @@ interface Settings {
alias: string;
osAec: boolean;
quality: string;
videoCodec: string;
videoResolution: string;
recentRooms: RecentRoom[];
dredDebugLogs: boolean;
callDebugLogs: boolean;
@@ -151,7 +155,7 @@ function loadSettings(): Settings {
{ name: "Default", address: "193.180.213.68:4433" },
],
selectedRelay: 0, room: "general", alias: "",
osAec: true, quality: "auto", recentRooms: [],
osAec: true, quality: "auto", videoCodec: "h264", videoResolution: "1280x720", recentRooms: [],
dredDebugLogs: false, callDebugLogs: false,
directOnly: false, birthdayAttack: false,
};
@@ -164,6 +168,25 @@ function loadSettings(): Settings {
function saveSettings(s: Settings) {
localStorage.setItem("wzp-settings", JSON.stringify(s));
}
function parseVideoResolution(value: string) {
const [wRaw, hRaw] = (value || "1280x720").split("x");
const width = Number.parseInt(wRaw, 10);
const height = Number.parseInt(hRaw, 10);
if (!Number.isFinite(width) || !Number.isFinite(height)) {
return { width: 1280, height: 720 };
}
return { width, height };
}
function videoConnectOptions(s: Settings) {
const { width, height } = parseVideoResolution(s.videoResolution);
return {
videoCodec: s.videoCodec || "h264",
videoWidth: width,
videoHeight: height,
};
}
function getRelay(): RelayServer | null {
const s = loadSettings();
return s.relays[s.selectedRelay] || s.relays[0] || null;
@@ -466,6 +489,7 @@ joinVoiceBtn.addEventListener("click", async () => {
alias: s.alias || "",
osAec: s.osAec,
quality: s.quality || "auto",
...videoConnectOptions(s),
});
enterVoice(false);
} catch (e: any) {
@@ -494,6 +518,7 @@ joinVideoBtn.addEventListener("click", async () => {
alias: s.alias || "",
osAec: s.osAec,
quality: s.quality || "auto",
...videoConnectOptions(s),
});
enterVoice(false);
startCamera();
@@ -570,8 +595,8 @@ vdSpkBtn.addEventListener("click", async () => {
// ── Camera (Blocker 4 + 5) ────────────────────────────────────────
const camCaptureCanvas = document.createElement("canvas");
const camCaptureCtx = camCaptureCanvas.getContext("2d")!;
const CAMERA_SEND_WIDTH = 1280;
const CAMERA_SEND_HEIGHT = 720;
let cameraSendWidth = 1280;
let cameraSendHeight = 720;
let cameraCaptureFrameNo = 0;
let cameraPushFailures = 0;
const CAMERA_CAPTURE_INTERVAL_MS = 33; // ≈ 30 fps
@@ -582,14 +607,14 @@ function drawCameraFrameForSend() {
const vh = vdLocalVideo.videoHeight || camCaptureCanvas.height;
if (!vw || !vh) return;
const scale = Math.max(CAMERA_SEND_WIDTH / vw, CAMERA_SEND_HEIGHT / vh);
const scale = Math.max(cameraSendWidth / vw, cameraSendHeight / vh);
const dw = vw * scale;
const dh = vh * scale;
const dx = (CAMERA_SEND_WIDTH - dw) / 2;
const dy = (CAMERA_SEND_HEIGHT - dh) / 2;
const dx = (cameraSendWidth - dw) / 2;
const dy = (cameraSendHeight - dh) / 2;
camCaptureCtx.fillStyle = "#000";
camCaptureCtx.fillRect(0, 0, CAMERA_SEND_WIDTH, CAMERA_SEND_HEIGHT);
camCaptureCtx.fillRect(0, 0, cameraSendWidth, cameraSendHeight);
camCaptureCtx.drawImage(vdLocalVideo, dx, dy, dw, dh);
}
@@ -670,8 +695,11 @@ function scheduleCameraFrameCapture() {
async function startCamera() {
if (cameraActive) return;
const videoSize = parseVideoResolution(loadSettings().videoResolution);
cameraSendWidth = videoSize.width;
cameraSendHeight = videoSize.height;
const constraints = {
video: { width: { ideal: 1280 }, height: { ideal: 720 }, facingMode: "user" },
video: { width: { ideal: cameraSendWidth }, height: { ideal: cameraSendHeight }, facingMode: "user" },
audio: false,
};
debugLog("camera:get_user_media_start", { constraints });
@@ -682,8 +710,8 @@ async function startCamera() {
const track = cameraStream.getVideoTracks()[0];
const settings = track.getSettings();
camCaptureCanvas.width = CAMERA_SEND_WIDTH;
camCaptureCanvas.height = CAMERA_SEND_HEIGHT;
camCaptureCanvas.width = cameraSendWidth;
camCaptureCanvas.height = cameraSendHeight;
debugLog("camera:get_user_media_ok", {
width: settings.width ?? null,
height: settings.height ?? null,
@@ -922,6 +950,7 @@ listen("signal-event", (event: any) => {
peerMappedAddr: data.peer_mapped_addr ?? null,
directOnly: s.directOnly || false,
birthdayAttack: s.birthdayAttack || false,
...videoConnectOptions(s),
});
enterVoice(true);
} catch (e: any) {
@@ -1072,6 +1101,8 @@ function openSettings() {
sCallDebug.checked = !!s.callDebugLogs;
sDirectOnly.checked = !!s.directOnly;
sBirthdayAttack.checked = !!s.birthdayAttack;
sVideoCodec.value = s.videoCodec || "h264";
sVideoResolution.value = s.videoResolution || "1280x720";
sCallDebugSection.style.display = s.callDebugLogs ? "" : "none";
renderCallDebugLog();
const qi = qualityToIndex(s.quality || "auto");
@@ -1097,6 +1128,8 @@ settingsSave.addEventListener("click", () => {
s.callDebugLogs = sCallDebug.checked;
s.directOnly = sDirectOnly.checked;
s.birthdayAttack = sBirthdayAttack.checked;
s.videoCodec = sVideoCodec.value || "h264";
s.videoResolution = sVideoResolution.value || "1280x720";
saveSettings(s);
invoke("set_dred_verbose_logs", { enabled: s.dredDebugLogs }).catch(() => {});
invoke("set_call_debug_logs", { enabled: s.callDebugLogs }).catch(() => {});