manawenuz/wz-phone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(video): add codec and resolution controls
Some checks failed
Build Release Binaries / build-amd64 (push) Failing after 3m38s
Details
Mirror to GitHub / mirror (push) Failing after 38s
Details

Browse Source
This commit is contained in:
Siavash Sameni
2026-05-26 10:05:20 +04:00
parent f85efb9576
commit 9a7745978b
7 changed files with 250 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
crates/wzp-client/src/handshake.rs
View File

@@ -73,6 +73,16 @@ pub async fn perform_handshake(
transport: &dyn MediaTransport, transport: &dyn MediaTransport,
seed: &[u8; 32], seed: &[u8; 32],
alias: Option<&str>, alias: Option<&str>,
) -> Result<HandshakeResult, HandshakeError> {
perform_handshake_with_video_codecs(transport, seed, alias, SUPPORTED_VIDEO_CODECS.to_vec())
.await
}
pub async fn perform_handshake_with_video_codecs(
transport: &dyn MediaTransport,
seed: &[u8; 32],
alias: Option<&str>,
video_codecs: Vec<CodecId>,
) -> Result<HandshakeResult, HandshakeError> { ) -> Result<HandshakeResult, HandshakeError> {
// 1. Create key exchange from identity seed // 1. Create key exchange from identity seed
let mut kx = WarzoneKeyExchange::from_identity_seed(seed); let mut kx = WarzoneKeyExchange::from_identity_seed(seed);
@@ -104,7 +114,7 @@ pub async fn perform_handshake(
alias: alias.map(|s| s.to_string()), alias: alias.map(|s| s.to_string()),
protocol_version: 2, protocol_version: 2,
supported_versions: vec![2], supported_versions: vec![2],
video_codecs: SUPPORTED_VIDEO_CODECS.to_vec(), video_codecs,
}; };
transport transport
.send_signal(&offer) .send_signal(&offer)
@@ -112,14 +122,11 @@ pub async fn perform_handshake(
.map_err(HandshakeError::Transport)?; .map_err(HandshakeError::Transport)?;
// 5. Wait for CallAnswer — 10s timeout guards against relay not responding. // 5. Wait for CallAnswer — 10s timeout guards against relay not responding.
let answer = tokio::time::timeout( let answer = tokio::time::timeout(std::time::Duration::from_secs(10), transport.recv_signal())
std::time::Duration::from_secs(10), .await
transport.recv_signal(), .map_err(|_| HandshakeError::Transport(wzp_proto::TransportError::Timeout { ms: 10_000 }))?
) .map_err(HandshakeError::Transport)?
.await .ok_or(HandshakeError::ConnectionClosed)?;
.map_err(|_| HandshakeError::Transport(wzp_proto::TransportError::Timeout { ms: 10_000 }))?
.map_err(HandshakeError::Transport)?
.ok_or(HandshakeError::ConnectionClosed)?;
let (callee_identity_pub, callee_ephemeral_pub, callee_signature, _chosen_profile, video_codec) = let (callee_identity_pub, callee_ephemeral_pub, callee_signature, _chosen_profile, video_codec) =
match answer { match answer {
@@ -130,7 +137,13 @@ pub async fn perform_handshake(
chosen_profile, chosen_profile,
video_codec, video_codec,
.. ..
} => (identity_pub, ephemeral_pub, signature, chosen_profile, video_codec), } => (
identity_pub,
ephemeral_pub,
signature,
chosen_profile,
video_codec,
),
SignalMessage::Hangup { SignalMessage::Hangup {
reason: HangupReason::ProtocolVersionMismatch { server_supported }, reason: HangupReason::ProtocolVersionMismatch { server_supported },
.. ..
@@ -155,7 +168,10 @@ pub async fn perform_handshake(
.derive_session(&callee_ephemeral_pub) .derive_session(&callee_ephemeral_pub)
.map_err(|e| HandshakeError::KeyDerivation(e.to_string()))?; .map_err(|e| HandshakeError::KeyDerivation(e.to_string()))?;
Ok(HandshakeResult { session, video_codec }) Ok(HandshakeResult {
session,
video_codec,
})
} }
#[cfg(test)] #[cfg(test)]
@@ -185,7 +201,10 @@ mod tests {
let mut kx = WarzoneKeyExchange::from_identity_seed(&[0x55; 32]); let mut kx = WarzoneKeyExchange::from_identity_seed(&[0x55; 32]);
kx.generate_ephemeral(); kx.generate_ephemeral();
let session = kx.derive_session(&[0u8; 32]).unwrap(); let session = kx.derive_session(&[0u8; 32]).unwrap();
let hs = HandshakeResult { session, video_codec: None }; let hs = HandshakeResult {
session,
video_codec: None,
};
assert!(hs.video_codec.is_none()); assert!(hs.video_codec.is_none());
let mut kx2 = WarzoneKeyExchange::from_identity_seed(&[0x66; 32]); let mut kx2 = WarzoneKeyExchange::from_identity_seed(&[0x66; 32]);

9
crates/wzp-video/src/mediacodec.rs
View File

@@ -519,11 +519,12 @@ impl VideoEncoder for MediaCodecHevcEncoder {
} }
fn is_keyframe(&self, packet: &[u8]) -> bool { fn is_keyframe(&self, packet: &[u8]) -> bool {
if packet.len() < 2 { let nals = split_annex_b(packet);
return false; if nals.is_empty() {
return packet.len() >= 2 && matches!((packet[0] >> 1) & 0x3F, 19 | 20);
} }
let nal_type = (packet[0] >> 1) & 0x3F; nals.iter()
nal_type == 19 || nal_type == 20 .any(|nal| nal.len() >= 2 && matches!((nal[0] >> 1) & 0x3F, 19 | 20))
} }
} }

12
crates/wzp-video/src/videotoolbox.rs
View File

@@ -164,7 +164,8 @@ impl VideoEncoder for VideoToolboxEncoder {
if nals.is_empty() { if nals.is_empty() {
return (packet[0] & 0x1F) == 5; return (packet[0] & 0x1F) == 5;
} }
nals.iter().any(|nal| !nal.is_empty() && (nal[0] & 0x1F) == 5) nals.iter()
.any(|nal| !nal.is_empty() && (nal[0] & 0x1F) == 5)
} }
} }
@@ -522,12 +523,13 @@ impl VideoEncoder for VideoToolboxHevcEncoder {
} }
fn is_keyframe(&self, packet: &[u8]) -> bool { fn is_keyframe(&self, packet: &[u8]) -> bool {
if packet.len() < 2 { let nals = split_annex_b(packet);
return false; if nals.is_empty() {
return packet.len() >= 2 && matches!((packet[0] >> 1) & 0x3F, 19 | 20);
} }
let nal_type = (packet[0] >> 1) & 0x3F;
// NAL type 19 = IDR_W_RADL, 20 = IDR_N_LP. // NAL type 19 = IDR_W_RADL, 20 = IDR_N_LP.
nal_type == 19 || nal_type == 20 nals.iter()
.any(|nal| nal.len() >= 2 && matches!((nal[0] >> 1) & 0x3F, 19 | 20))
} }
} }

16
desktop/index.html
View File

@@ -174,6 +174,22 @@
OS Echo Cancellation OS Echo Cancellation
</label> </label>
</div> </div>
<div class="settings-section">
<h3>Video</h3>
<label>Codec
<select id="s-video-codec">
<option value="h264">H.264</option>
<option value="h265">H.265 / HEVC</option>
</select>
</label>
<label>Room Resolution
<select id="s-video-resolution">
<option value="640x360">640 x 360</option>
<option value="960x540">960 x 540</option>
<option value="1280x720">1280 x 720</option>
</select>
</label>
</div>
<div class="settings-section"> <div class="settings-section">
<h3>Relays</h3> <h3>Relays</h3>
<div id="s-relay-list"></div> <div id="s-relay-list"></div>

139
desktop/src-tauri/src/engine.rs
View File

@@ -186,6 +186,19 @@ const VIDEO_KEYFRAME_INTERVAL_FRAMES: u32 = 120;
const VIDEO_BITRATE_BPS: u32 = 900_000; const VIDEO_BITRATE_BPS: u32 = 900_000;
const VIDEO_PLI_MIN_INTERVAL_MS: u128 = 250; const VIDEO_PLI_MIN_INTERVAL_MS: u128 = 250;
fn parse_video_codec(codec: &str) -> wzp_proto::CodecId {
match codec.to_ascii_lowercase().as_str() {
"h265" | "h265main" | "hevc" => wzp_proto::CodecId::H265Main,
"av1" | "av1main" => wzp_proto::CodecId::Av1Main,
_ => wzp_proto::CodecId::H264Baseline,
}
}
fn clamp_video_dimension(value: u32, fallback: u32) -> u32 {
let value = if value == 0 { fallback } else { value };
value.clamp(160, 1920) & !1
}
#[derive(Default)] #[derive(Default)]
struct VideoContinuity { struct VideoContinuity {
expected_seq: Option<u32>, expected_seq: Option<u32>,
@@ -684,17 +697,26 @@ impl CallEngine {
app: tauri::AppHandle, app: tauri::AppHandle,
active_quality: Arc<std::sync::Mutex<wzp_proto::QualityProfile>>, active_quality: Arc<std::sync::Mutex<wzp_proto::QualityProfile>>,
peer_max_quality: Arc<std::sync::Mutex<Option<wzp_proto::QualityProfile>>>, peer_max_quality: Arc<std::sync::Mutex<Option<wzp_proto::QualityProfile>>>,
video_codec_preference: String,
video_width: u32,
video_height: u32,
event_cb: F, event_cb: F,
) -> Result<Self, anyhow::Error> ) -> Result<Self, anyhow::Error>
where where
F: Fn(&str, &str) + Send + Sync + 'static, F: Fn(&str, &str) + Send + Sync + 'static,
{ {
let call_t0 = std::time::Instant::now(); let call_t0 = std::time::Instant::now();
let preferred_video_codec = parse_video_codec(&video_codec_preference);
let video_width = clamp_video_dimension(video_width, 1280);
let video_height = clamp_video_dimension(video_height, 720);
info!( info!(
%relay, %room, %alias, %quality, %relay, %room, %alias, %quality,
has_reuse = reuse_endpoint.is_some(), has_reuse = reuse_endpoint.is_some(),
has_pre_connected = pre_connected_transport.is_some(), has_pre_connected = pre_connected_transport.is_some(),
is_direct_p2p, is_direct_p2p,
video_codec = ?preferred_video_codec,
video_width,
video_height,
t_ms = 0u128, t_ms = 0u128,
"CallEngine::start (android) invoked" "CallEngine::start (android) invoked"
); );
@@ -788,24 +810,28 @@ impl CallEngine {
"remote": transport.remote_address().to_string(), "remote": transport.remote_address().to_string(),
}), }),
); );
let hs = let hs = match wzp_client::handshake::perform_handshake_with_video_codecs(
match wzp_client::handshake::perform_handshake(&*transport, &seed.0, Some(&alias)) &*transport,
.await &seed.0,
{ Some(&alias),
Ok(hs) => hs, vec![preferred_video_codec],
Err(e) => { )
error!("perform_handshake failed: {e}"); .await
crate::emit_call_debug( {
&app, Ok(hs) => hs,
"connect:handshake_failed", Err(e) => {
serde_json::json!({ error!("perform_handshake failed: {e}");
"t_ms": call_t0.elapsed().as_millis(), crate::emit_call_debug(
"error": e.to_string(), &app,
}), "connect:handshake_failed",
); serde_json::json!({
return Err(e.into()); "t_ms": call_t0.elapsed().as_millis(),
} "error": e.to_string(),
}; }),
);
return Err(e.into());
}
};
crate::emit_call_debug( crate::emit_call_debug(
&app, &app,
"connect:handshake_done", "connect:handshake_done",
@@ -829,7 +855,7 @@ impl CallEngine {
t_ms = call_t0.elapsed().as_millis(), t_ms = call_t0.elapsed().as_millis(),
"first-join diag: direct P2P — skipping relay handshake (QUIC TLS is the encryption layer)" "first-join diag: direct P2P — skipping relay handshake (QUIC TLS is the encryption layer)"
); );
(Some(wzp_proto::CodecId::H264Baseline), transport) (Some(preferred_video_codec), transport)
}; };
crate::emit_call_debug( crate::emit_call_debug(
&app, &app,
@@ -838,6 +864,8 @@ impl CallEngine {
"t_ms": call_t0.elapsed().as_millis(), "t_ms": call_t0.elapsed().as_millis(),
"codec": _negotiated_video_codec.map(|c| format!("{:?}", c)), "codec": _negotiated_video_codec.map(|c| format!("{:?}", c)),
"enabled": _negotiated_video_codec.is_some(), "enabled": _negotiated_video_codec.is_some(),
"width": video_width,
"height": video_height,
"direct_p2p": is_direct_p2p, "direct_p2p": is_direct_p2p,
}), }),
); );
@@ -1496,13 +1524,15 @@ impl CallEngine {
serde_json::json!({ serde_json::json!({
"t_ms": recv_t0.elapsed().as_millis() as u64, "t_ms": recv_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", codec_id), "codec": format!("{:?}", codec_id),
"width": 1280, "width": video_width,
"height": 720, "height": video_height,
"platform": "android", "platform": "android",
}), }),
); );
match wzp_video::factory::create_video_decoder( match wzp_video::factory::create_video_decoder(
codec_id, 1280, 720, codec_id,
video_width,
video_height,
) { ) {
Ok(d) => { Ok(d) => {
info!(codec = ?codec_id, "video decoder created (android)"); info!(codec = ?codec_id, "video decoder created (android)");
@@ -2020,8 +2050,8 @@ impl CallEngine {
// Video send task (Android) — mirror of the desktop branch. Only // Video send task (Android) — mirror of the desktop branch. Only
// spawns when a video codec is available. Relay calls negotiate this // spawns when a video codec is available. Relay calls negotiate this
// in the media handshake; direct P2P uses the common H264 baseline // in the media handshake; direct P2P uses the local debug codec
// codec because the relay handshake is intentionally skipped. // preference because the relay handshake is intentionally skipped.
let camera_tx = if let Some(vid_codec) = _negotiated_video_codec { let camera_tx = if let Some(vid_codec) = _negotiated_video_codec {
let (tx, mut rx) = tokio::sync::mpsc::channel::<wzp_video::encoder::VideoFrame>(4); let (tx, mut rx) = tokio::sync::mpsc::channel::<wzp_video::encoder::VideoFrame>(4);
let vid_transport = transport.clone(); let vid_transport = transport.clone();
@@ -2046,16 +2076,16 @@ impl CallEngine {
serde_json::json!({ serde_json::json!({
"t_ms": vid_t0.elapsed().as_millis() as u64, "t_ms": vid_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", vid_codec), "codec": format!("{:?}", vid_codec),
"width": 1280, "width": video_width,
"height": 720, "height": video_height,
"bitrate_bps": VIDEO_BITRATE_BPS, "bitrate_bps": VIDEO_BITRATE_BPS,
"platform": "android", "platform": "android",
}), }),
); );
let mut encoder = match wzp_video::factory::create_video_encoder( let mut encoder = match wzp_video::factory::create_video_encoder(
vid_codec, vid_codec,
1280, video_width,
720, video_height,
VIDEO_BITRATE_BPS, VIDEO_BITRATE_BPS,
) { ) {
Ok(e) => { Ok(e) => {
@@ -2428,19 +2458,28 @@ impl CallEngine {
_app: tauri::AppHandle, _app: tauri::AppHandle,
active_quality: Arc<std::sync::Mutex<wzp_proto::QualityProfile>>, active_quality: Arc<std::sync::Mutex<wzp_proto::QualityProfile>>,
peer_max_quality: Arc<std::sync::Mutex<Option<wzp_proto::QualityProfile>>>, peer_max_quality: Arc<std::sync::Mutex<Option<wzp_proto::QualityProfile>>>,
video_codec_preference: String,
video_width: u32,
video_height: u32,
event_cb: F, event_cb: F,
) -> Result<Self, anyhow::Error> ) -> Result<Self, anyhow::Error>
where where
F: Fn(&str, &str) + Send + Sync + 'static, F: Fn(&str, &str) + Send + Sync + 'static,
{ {
let call_t0 = Instant::now();
let preferred_video_codec = parse_video_codec(&video_codec_preference);
let video_width = clamp_video_dimension(video_width, 1280);
let video_height = clamp_video_dimension(video_height, 720);
info!( info!(
%relay, %room, %alias, %quality, %relay, %room, %alias, %quality,
has_reuse = reuse_endpoint.is_some(), has_reuse = reuse_endpoint.is_some(),
has_pre_connected = pre_connected_transport.is_some(), has_pre_connected = pre_connected_transport.is_some(),
is_direct_p2p, is_direct_p2p,
video_codec = ?preferred_video_codec,
video_width,
video_height,
"CallEngine::start (desktop) invoked" "CallEngine::start (desktop) invoked"
); );
let call_t0 = Instant::now();
let _ = rustls::crypto::ring::default_provider().install_default(); let _ = rustls::crypto::ring::default_provider().install_default();
let relay_addr: SocketAddr = relay.parse()?; let relay_addr: SocketAddr = relay.parse()?;
@@ -2498,13 +2537,17 @@ impl CallEngine {
// PRD lands, media goes plaintext-over-QUIC-TLS to the relay. // PRD lands, media goes plaintext-over-QUIC-TLS to the relay.
let (_negotiated_video_codec, transport): (_, Arc<dyn wzp_proto::MediaTransport>) = let (_negotiated_video_codec, transport): (_, Arc<dyn wzp_proto::MediaTransport>) =
if !is_direct_p2p { if !is_direct_p2p {
let hs = let hs = wzp_client::handshake::perform_handshake_with_video_codecs(
wzp_client::handshake::perform_handshake(&*transport, &seed.0, Some(&alias)) &*transport,
.await &seed.0,
.map_err(|e| { Some(&alias),
error!("perform_handshake failed: {e}"); vec![preferred_video_codec],
e )
})?; .await
.map_err(|e| {
error!("perform_handshake failed: {e}");
e
})?;
crate::emit_call_debug( crate::emit_call_debug(
&_app, &_app,
"connect:handshake_done", "connect:handshake_done",
@@ -2518,7 +2561,7 @@ impl CallEngine {
(hs.video_codec, transport) (hs.video_codec, transport)
} else { } else {
info!("direct P2P — skipping relay handshake (QUIC TLS is the encryption layer)"); info!("direct P2P — skipping relay handshake (QUIC TLS is the encryption layer)");
(Some(wzp_proto::CodecId::H264Baseline), transport) (Some(preferred_video_codec), transport)
}; };
crate::emit_call_debug( crate::emit_call_debug(
&_app, &_app,
@@ -2527,6 +2570,8 @@ impl CallEngine {
"t_ms": call_t0.elapsed().as_millis(), "t_ms": call_t0.elapsed().as_millis(),
"codec": _negotiated_video_codec.map(|c| format!("{:?}", c)), "codec": _negotiated_video_codec.map(|c| format!("{:?}", c)),
"enabled": _negotiated_video_codec.is_some(), "enabled": _negotiated_video_codec.is_some(),
"width": video_width,
"height": video_height,
"direct_p2p": is_direct_p2p, "direct_p2p": is_direct_p2p,
}), }),
); );
@@ -2970,13 +3015,15 @@ impl CallEngine {
serde_json::json!({ serde_json::json!({
"t_ms": recv_t0.elapsed().as_millis() as u64, "t_ms": recv_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", codec_id), "codec": format!("{:?}", codec_id),
"width": 1280, "width": video_width,
"height": 720, "height": video_height,
"platform": "desktop", "platform": "desktop",
}), }),
); );
match wzp_video::factory::create_video_decoder( match wzp_video::factory::create_video_decoder(
codec_id, 1280, 720, codec_id,
video_width,
video_height,
) { ) {
Ok(d) => { Ok(d) => {
info!(codec = ?codec_id, "video decoder created"); info!(codec = ?codec_id, "video decoder created");
@@ -3339,8 +3386,8 @@ impl CallEngine {
)); ));
// Video send task — active when a video codec is available. Relay calls // Video send task — active when a video codec is available. Relay calls
// negotiate this in the media handshake; direct P2P uses the common H264 // negotiate this in the media handshake; direct P2P uses the local debug
// baseline codec because the relay handshake is intentionally skipped. // codec preference because the relay handshake is intentionally skipped.
let camera_tx = if let Some(vid_codec) = _negotiated_video_codec { let camera_tx = if let Some(vid_codec) = _negotiated_video_codec {
let (tx, mut rx) = tokio::sync::mpsc::channel::<wzp_video::encoder::VideoFrame>(4); let (tx, mut rx) = tokio::sync::mpsc::channel::<wzp_video::encoder::VideoFrame>(4);
let vid_transport = transport.clone(); let vid_transport = transport.clone();
@@ -3365,16 +3412,16 @@ impl CallEngine {
serde_json::json!({ serde_json::json!({
"t_ms": vid_t0.elapsed().as_millis() as u64, "t_ms": vid_t0.elapsed().as_millis() as u64,
"codec": format!("{:?}", vid_codec), "codec": format!("{:?}", vid_codec),
"width": 1280, "width": video_width,
"height": 720, "height": video_height,
"bitrate_bps": VIDEO_BITRATE_BPS, "bitrate_bps": VIDEO_BITRATE_BPS,
"platform": "desktop", "platform": "desktop",
}), }),
); );
let mut encoder = match wzp_video::factory::create_video_encoder( let mut encoder = match wzp_video::factory::create_video_encoder(
vid_codec, vid_codec,
1280, video_width,
720, video_height,
VIDEO_BITRATE_BPS, VIDEO_BITRATE_BPS,
) { ) {
Ok(e) => { Ok(e) => {

70
desktop/src-tauri/src/lib.rs
View File

@@ -137,13 +137,15 @@ pub(crate) fn i420_to_jpeg_bytes(data: &[u8], width: u32, height: u32) -> Option
let u = data[y_size + uv_idx] as f32 - 128.0; let u = data[y_size + uv_idx] as f32 - 128.0;
let v = data[y_size + uv_size + uv_idx] as f32 - 128.0; let v = data[y_size + uv_size + uv_idx] as f32 - 128.0;
let out = (row * w + col) * 3; let out = (row * w + col) * 3;
rgb[out] = (y + 1.402 * v).clamp(0.0, 255.0) as u8; rgb[out] = (y + 1.402 * v).clamp(0.0, 255.0) as u8;
rgb[out + 1] = (y - 0.344 * u - 0.714 * v).clamp(0.0, 255.0) as u8; rgb[out + 1] = (y - 0.344 * u - 0.714 * v).clamp(0.0, 255.0) as u8;
rgb[out + 2] = (y + 1.772 * u).clamp(0.0, 255.0) as u8; rgb[out + 2] = (y + 1.772 * u).clamp(0.0, 255.0) as u8;
} }
} }
let img = DynamicImage::ImageRgb8(ImageBuffer::<Rgb<u8>, Vec<u8>>::from_raw(width, height, rgb)?); let img = DynamicImage::ImageRgb8(ImageBuffer::<Rgb<u8>, Vec<u8>>::from_raw(
width, height, rgb,
)?);
let mut buf = std::io::Cursor::new(Vec::<u8>::new()); let mut buf = std::io::Cursor::new(Vec::<u8>::new());
img.write_to(&mut buf, image::ImageFormat::Jpeg).ok()?; img.write_to(&mut buf, image::ImageFormat::Jpeg).ok()?;
Some(buf.into_inner()) Some(buf.into_inner())
@@ -217,8 +219,10 @@ fn rgb_to_i420(rgb: &[u8], w: usize, h: usize) -> Vec<u8> {
out[row * w + col] = (0.299 * r + 0.587 * g + 0.114 * b).clamp(0.0, 255.0) as u8; out[row * w + col] = (0.299 * r + 0.587 * g + 0.114 * b).clamp(0.0, 255.0) as u8;
if row % 2 == 0 && col % 2 == 0 { if row % 2 == 0 && col % 2 == 0 {
let uv = (row / 2) * (w / 2) + col / 2; let uv = (row / 2) * (w / 2) + col / 2;
out[y_size + uv] = (-0.169 * r - 0.331 * g + 0.500 * b + 128.0).clamp(0.0, 255.0) as u8; out[y_size + uv] =
out[y_size + uv_size + uv] = (0.500 * r - 0.419 * g - 0.081 * b + 128.0).clamp(0.0, 255.0) as u8; (-0.169 * r - 0.331 * g + 0.500 * b + 128.0).clamp(0.0, 255.0) as u8;
out[y_size + uv_size + uv] =
(0.500 * r - 0.419 * g - 0.081 * b + 128.0).clamp(0.0, 255.0) as u8;
} }
} }
} }
@@ -387,7 +391,7 @@ mod video_tests {
fn solid_rgb_frame(w: usize, h: usize, r: u8, g: u8, b: u8) -> Vec<u8> { fn solid_rgb_frame(w: usize, h: usize, r: u8, g: u8, b: u8) -> Vec<u8> {
let mut rgb = vec![0u8; w * h * 3]; let mut rgb = vec![0u8; w * h * 3];
for i in 0..w * h { for i in 0..w * h {
rgb[i * 3] = r; rgb[i * 3] = r;
rgb[i * 3 + 1] = g; rgb[i * 3 + 1] = g;
rgb[i * 3 + 2] = b; rgb[i * 3 + 2] = b;
} }
@@ -439,8 +443,14 @@ mod video_tests {
let s = b64.unwrap(); let s = b64.unwrap();
assert!(!s.is_empty()); assert!(!s.is_empty());
// JPEG base64 starts with '/9j/' (FFD8FF marker). // JPEG base64 starts with '/9j/' (FFD8FF marker).
let decoded = base64::engine::general_purpose::STANDARD.decode(&s).unwrap(); let decoded = base64::engine::general_purpose::STANDARD
assert_eq!(&decoded[0..2], &[0xFF, 0xD8], "output must start with JPEG SOI marker"); .decode(&s)
.unwrap();
assert_eq!(
&decoded[0..2],
&[0xFF, 0xD8],
"output must start with JPEG SOI marker"
);
} }
#[test] #[test]
@@ -463,13 +473,18 @@ mod video_tests {
let yuv = rgb_to_i420(&rgb, 64, 64); let yuv = rgb_to_i420(&rgb, 64, 64);
let b64 = i420_to_jpeg_b64(&yuv, 64, 64).expect("should produce JPEG"); let b64 = i420_to_jpeg_b64(&yuv, 64, 64).expect("should produce JPEG");
let jpeg = base64::engine::general_purpose::STANDARD.decode(&b64).unwrap(); let jpeg = base64::engine::general_purpose::STANDARD
.decode(&b64)
.unwrap();
let img = image::load_from_memory_with_format(&jpeg, image::ImageFormat::Jpeg).unwrap(); let img = image::load_from_memory_with_format(&jpeg, image::ImageFormat::Jpeg).unwrap();
let rgb_img = img.to_rgb8(); let rgb_img = img.to_rgb8();
let px = rgb_img.get_pixel(32, 32); let px = rgb_img.get_pixel(32, 32);
let (r, g, b) = (px[0], px[1], px[2]); let (r, g, b) = (px[0], px[1], px[2]);
assert!(r > g && r > b, "red frame: expected R dominant, got R={r} G={g} B={b}"); assert!(
r > g && r > b,
"red frame: expected R dominant, got R={r} G={g} B={b}"
);
} }
#[test] #[test]
@@ -746,8 +761,14 @@ async fn connect(
// Enable birthday attack for hard NAT traversal. Adds ~3s to // Enable birthday attack for hard NAT traversal. Adds ~3s to
// call setup when peer has symmetric NAT. // call setup when peer has symmetric NAT.
birthday_attack: Option<bool>, birthday_attack: Option<bool>,
video_codec: Option<String>,
video_width: Option<u32>,
video_height: Option<u32>,
) -> Result<String, String> { ) -> Result<String, String> {
let force_direct = direct_only.unwrap_or(false); let force_direct = direct_only.unwrap_or(false);
let video_codec = video_codec.unwrap_or_else(|| "h264".to_string());
let video_width = video_width.unwrap_or(1280);
let video_height = video_height.unwrap_or(720);
let enable_birthday = birthday_attack.unwrap_or(false); let enable_birthday = birthday_attack.unwrap_or(false);
emit_call_debug( emit_call_debug(
&app, &app,
@@ -760,6 +781,9 @@ async fn connect(
"peer_mapped_addr": peer_mapped_addr, "peer_mapped_addr": peer_mapped_addr,
"direct_only": force_direct, "direct_only": force_direct,
"birthday_attack": enable_birthday, "birthday_attack": enable_birthday,
"video_codec": video_codec,
"video_width": video_width,
"video_height": video_height,
}), }),
); );
let mut engine_lock = state.engine.lock().await; let mut engine_lock = state.engine.lock().await;
@@ -1218,6 +1242,9 @@ async fn connect(
app_for_engine, app_for_engine,
active_quality, active_quality,
peer_max_quality, peer_max_quality,
video_codec,
video_width,
video_height,
move |event_kind, message| { move |event_kind, message| {
let _ = app_clone.emit( let _ = app_clone.emit(
"call-event", "call-event",
@@ -2129,7 +2156,9 @@ fn do_register_signal(
"peer_loss_pct": local_loss_pct, "peer_rtt_ms": local_rtt_ms, "peer_loss_pct": local_loss_pct, "peer_rtt_ms": local_rtt_ms,
}), }),
); );
if let Err(e) = handle_upgrade_proposal(&*transport, &call_id, &proposal_id).await { if let Err(e) =
handle_upgrade_proposal(&*transport, &call_id, &proposal_id).await
{
tracing::warn!("failed to send UpgradeResponse: {e}"); tracing::warn!("failed to send UpgradeResponse: {e}");
} }
} }
@@ -2150,8 +2179,14 @@ fn do_register_signal(
}), }),
); );
if let Err(e) = handle_upgrade_response( if let Err(e) = handle_upgrade_response(
&*transport, &signal_state, &call_id, &proposal_id, accepted, &*transport,
).await { &signal_state,
&call_id,
&proposal_id,
accepted,
)
.await
{
tracing::warn!("failed to handle UpgradeResponse: {e}"); tracing::warn!("failed to handle UpgradeResponse: {e}");
} }
} }
@@ -3426,7 +3461,9 @@ mod signal_tests {
#[tokio::test] #[tokio::test]
async fn upgrade_proposal_auto_accepts() { async fn upgrade_proposal_auto_accepts() {
let transport = LoopbackTransport::new(); let transport = LoopbackTransport::new();
handle_upgrade_proposal(&*transport, "c1", "p1").await.unwrap(); handle_upgrade_proposal(&*transport, "c1", "p1")
.await
.unwrap();
let sent = transport.take_sent(); let sent = transport.take_sent();
assert_eq!(sent.len(), 1); assert_eq!(sent.len(), 1);
@@ -3453,8 +3490,11 @@ mod signal_tests {
let signal_state = empty_signal_state(); let signal_state = empty_signal_state();
{ {
let sig = signal_state.lock().await; let sig = signal_state.lock().await;
*sig.pending_upgrade.lock().unwrap() = *sig.pending_upgrade.lock().unwrap() = Some((
Some(("c1".into(), "p1".into(), wzp_proto::QualityProfile::STUDIO_48K)); "c1".into(),
"p1".into(),
wzp_proto::QualityProfile::STUDIO_48K,
));
} }
handle_upgrade_response(&*transport, &signal_state, "c1", "p1", true) handle_upgrade_response(&*transport, &signal_state, "c1", "p1", true)

53
desktop/src/main.ts
View File

@@ -122,6 +122,8 @@ const sCallDebugCopyBtn = document.getElementById("s-call-debug-copy") as HTMLBu
const sCallDebugShareBtn = document.getElementById("s-call-debug-share") as HTMLButtonElement; const sCallDebugShareBtn = document.getElementById("s-call-debug-share") as HTMLButtonElement;
const sQuality = document.getElementById("s-quality") as HTMLInputElement; const sQuality = document.getElementById("s-quality") as HTMLInputElement;
const sQualityLabel = document.getElementById("s-quality-label")!; const sQualityLabel = document.getElementById("s-quality-label")!;
const sVideoCodec = document.getElementById("s-video-codec") as HTMLSelectElement;
const sVideoResolution = document.getElementById("s-video-resolution") as HTMLSelectElement;
const sFingerprint = document.getElementById("s-fingerprint")!; const sFingerprint = document.getElementById("s-fingerprint")!;
const sPublicAddr = document.getElementById("s-public-addr")!; const sPublicAddr = document.getElementById("s-public-addr")!;
const sReflectBtn = document.getElementById("s-reflect-btn")!; const sReflectBtn = document.getElementById("s-reflect-btn")!;
@@ -138,6 +140,8 @@ interface Settings {
alias: string; alias: string;
osAec: boolean; osAec: boolean;
quality: string; quality: string;
videoCodec: string;
videoResolution: string;
recentRooms: RecentRoom[]; recentRooms: RecentRoom[];
dredDebugLogs: boolean; dredDebugLogs: boolean;
callDebugLogs: boolean; callDebugLogs: boolean;
@@ -151,7 +155,7 @@ function loadSettings(): Settings {
{ name: "Default", address: "193.180.213.68:4433" }, { name: "Default", address: "193.180.213.68:4433" },
], ],
selectedRelay: 0, room: "general", alias: "", selectedRelay: 0, room: "general", alias: "",
osAec: true, quality: "auto", recentRooms: [], osAec: true, quality: "auto", videoCodec: "h264", videoResolution: "1280x720", recentRooms: [],
dredDebugLogs: false, callDebugLogs: false, dredDebugLogs: false, callDebugLogs: false,
directOnly: false, birthdayAttack: false, directOnly: false, birthdayAttack: false,
}; };
@@ -164,6 +168,25 @@ function loadSettings(): Settings {
function saveSettings(s: Settings) { function saveSettings(s: Settings) {
localStorage.setItem("wzp-settings", JSON.stringify(s)); localStorage.setItem("wzp-settings", JSON.stringify(s));
} }
function parseVideoResolution(value: string) {
const [wRaw, hRaw] = (value || "1280x720").split("x");
const width = Number.parseInt(wRaw, 10);
const height = Number.parseInt(hRaw, 10);
if (!Number.isFinite(width) || !Number.isFinite(height)) {
return { width: 1280, height: 720 };
}
return { width, height };
}
function videoConnectOptions(s: Settings) {
const { width, height } = parseVideoResolution(s.videoResolution);
return {
videoCodec: s.videoCodec || "h264",
videoWidth: width,
videoHeight: height,
};
}
function getRelay(): RelayServer | null { function getRelay(): RelayServer | null {
const s = loadSettings(); const s = loadSettings();
return s.relays[s.selectedRelay] || s.relays[0] || null; return s.relays[s.selectedRelay] || s.relays[0] || null;
@@ -466,6 +489,7 @@ joinVoiceBtn.addEventListener("click", async () => {
alias: s.alias || "", alias: s.alias || "",
osAec: s.osAec, osAec: s.osAec,
quality: s.quality || "auto", quality: s.quality || "auto",
...videoConnectOptions(s),
}); });
enterVoice(false); enterVoice(false);
} catch (e: any) { } catch (e: any) {
@@ -494,6 +518,7 @@ joinVideoBtn.addEventListener("click", async () => {
alias: s.alias || "", alias: s.alias || "",
osAec: s.osAec, osAec: s.osAec,
quality: s.quality || "auto", quality: s.quality || "auto",
...videoConnectOptions(s),
}); });
enterVoice(false); enterVoice(false);
startCamera(); startCamera();
@@ -570,8 +595,8 @@ vdSpkBtn.addEventListener("click", async () => {
// ── Camera (Blocker 4 + 5) ──────────────────────────────────────── // ── Camera (Blocker 4 + 5) ────────────────────────────────────────
const camCaptureCanvas = document.createElement("canvas"); const camCaptureCanvas = document.createElement("canvas");
const camCaptureCtx = camCaptureCanvas.getContext("2d")!; const camCaptureCtx = camCaptureCanvas.getContext("2d")!;
const CAMERA_SEND_WIDTH = 1280; let cameraSendWidth = 1280;
const CAMERA_SEND_HEIGHT = 720; let cameraSendHeight = 720;
let cameraCaptureFrameNo = 0; let cameraCaptureFrameNo = 0;
let cameraPushFailures = 0; let cameraPushFailures = 0;
const CAMERA_CAPTURE_INTERVAL_MS = 33; // ≈ 30 fps const CAMERA_CAPTURE_INTERVAL_MS = 33; // ≈ 30 fps
@@ -582,14 +607,14 @@ function drawCameraFrameForSend() {
const vh = vdLocalVideo.videoHeight || camCaptureCanvas.height; const vh = vdLocalVideo.videoHeight || camCaptureCanvas.height;
if (!vw || !vh) return; if (!vw || !vh) return;
const scale = Math.max(CAMERA_SEND_WIDTH / vw, CAMERA_SEND_HEIGHT / vh); const scale = Math.max(cameraSendWidth / vw, cameraSendHeight / vh);
const dw = vw * scale; const dw = vw * scale;
const dh = vh * scale; const dh = vh * scale;
const dx = (CAMERA_SEND_WIDTH - dw) / 2; const dx = (cameraSendWidth - dw) / 2;
const dy = (CAMERA_SEND_HEIGHT - dh) / 2; const dy = (cameraSendHeight - dh) / 2;
camCaptureCtx.fillStyle = "#000"; camCaptureCtx.fillStyle = "#000";
camCaptureCtx.fillRect(0, 0, CAMERA_SEND_WIDTH, CAMERA_SEND_HEIGHT); camCaptureCtx.fillRect(0, 0, cameraSendWidth, cameraSendHeight);
camCaptureCtx.drawImage(vdLocalVideo, dx, dy, dw, dh); camCaptureCtx.drawImage(vdLocalVideo, dx, dy, dw, dh);
} }
@@ -670,8 +695,11 @@ function scheduleCameraFrameCapture() {
async function startCamera() { async function startCamera() {
if (cameraActive) return; if (cameraActive) return;
const videoSize = parseVideoResolution(loadSettings().videoResolution);
cameraSendWidth = videoSize.width;
cameraSendHeight = videoSize.height;
const constraints = { const constraints = {
video: { width: { ideal: 1280 }, height: { ideal: 720 }, facingMode: "user" }, video: { width: { ideal: cameraSendWidth }, height: { ideal: cameraSendHeight }, facingMode: "user" },
audio: false, audio: false,
}; };
debugLog("camera:get_user_media_start", { constraints }); debugLog("camera:get_user_media_start", { constraints });
@@ -682,8 +710,8 @@ async function startCamera() {
const track = cameraStream.getVideoTracks()[0]; const track = cameraStream.getVideoTracks()[0];
const settings = track.getSettings(); const settings = track.getSettings();
camCaptureCanvas.width = CAMERA_SEND_WIDTH; camCaptureCanvas.width = cameraSendWidth;
camCaptureCanvas.height = CAMERA_SEND_HEIGHT; camCaptureCanvas.height = cameraSendHeight;
debugLog("camera:get_user_media_ok", { debugLog("camera:get_user_media_ok", {
width: settings.width ?? null, width: settings.width ?? null,
height: settings.height ?? null, height: settings.height ?? null,
@@ -922,6 +950,7 @@ listen("signal-event", (event: any) => {
peerMappedAddr: data.peer_mapped_addr ?? null, peerMappedAddr: data.peer_mapped_addr ?? null,
directOnly: s.directOnly || false, directOnly: s.directOnly || false,
birthdayAttack: s.birthdayAttack || false, birthdayAttack: s.birthdayAttack || false,
...videoConnectOptions(s),
}); });
enterVoice(true); enterVoice(true);
} catch (e: any) { } catch (e: any) {
@@ -1072,6 +1101,8 @@ function openSettings() {
sCallDebug.checked = !!s.callDebugLogs; sCallDebug.checked = !!s.callDebugLogs;
sDirectOnly.checked = !!s.directOnly; sDirectOnly.checked = !!s.directOnly;
sBirthdayAttack.checked = !!s.birthdayAttack; sBirthdayAttack.checked = !!s.birthdayAttack;
sVideoCodec.value = s.videoCodec || "h264";
sVideoResolution.value = s.videoResolution || "1280x720";
sCallDebugSection.style.display = s.callDebugLogs ? "" : "none"; sCallDebugSection.style.display = s.callDebugLogs ? "" : "none";
renderCallDebugLog(); renderCallDebugLog();
const qi = qualityToIndex(s.quality || "auto"); const qi = qualityToIndex(s.quality || "auto");
@@ -1097,6 +1128,8 @@ settingsSave.addEventListener("click", () => {
s.callDebugLogs = sCallDebug.checked; s.callDebugLogs = sCallDebug.checked;
s.directOnly = sDirectOnly.checked; s.directOnly = sDirectOnly.checked;
s.birthdayAttack = sBirthdayAttack.checked; s.birthdayAttack = sBirthdayAttack.checked;
s.videoCodec = sVideoCodec.value || "h264";
s.videoResolution = sVideoResolution.value || "1280x720";
saveSettings(s); saveSettings(s);
invoke("set_dred_verbose_logs", { enabled: s.dredDebugLogs }).catch(() => {}); invoke("set_dred_verbose_logs", { enabled: s.dredDebugLogs }).catch(() => {});
invoke("set_call_debug_logs", { enabled: s.callDebugLogs }).catch(() => {}); invoke("set_call_debug_logs", { enabled: s.callDebugLogs }).catch(() => {});