Compare commits
38 Commits
experiment
...
video-usab
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
12020b019c | ||
|
|
3ea25a0656 | ||
|
|
112472609e | ||
|
|
9a7745978b | ||
|
|
f85efb9576 | ||
|
|
31b2caa54d | ||
|
|
079e21e174 | ||
|
|
e676641538 | ||
|
|
9713efc404 | ||
|
|
8415804a1a | ||
|
|
f65b399a21 | ||
|
|
3437a6bd11 | ||
|
|
15eb00ed5e | ||
|
|
0c2297a2b7 | ||
|
|
a08a37b5eb | ||
|
|
f6ace54556 | ||
|
|
47baa1a765 | ||
|
|
ee654cd1ef | ||
|
|
d2046060b5 | ||
|
|
0b7bf1b385 | ||
|
|
e8f139588a | ||
|
|
0115b11de7 | ||
|
|
fa812a17d9 | ||
|
|
8d6b168f1b | ||
|
|
ca164ada5c | ||
|
|
2d58bae9ba | ||
|
|
e1ca6ca6e6 | ||
|
|
06d28a9280 | ||
|
|
d57ebe3d2c | ||
|
|
7eca79846f | ||
|
|
25b3278d31 | ||
|
|
cbc3a8d37e | ||
|
|
1329abbeba | ||
|
|
e8cab25eda | ||
|
|
c41ced53e1 | ||
|
|
7fd66be6c8 | ||
|
|
8002acaf09 | ||
|
|
06253fdeeb |
5
.gitignore
vendored
5
.gitignore
vendored
@@ -12,6 +12,11 @@ npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
dev-debug.log
|
||||
|
||||
# Debug frame dump artifacts
|
||||
android-frame-dumps/
|
||||
wzp-frame-dumps.tar
|
||||
|
||||
# Dependency directories
|
||||
node_modules/
|
||||
# Environment variables
|
||||
|
||||
56
Cargo.lock
generated
56
Cargo.lock
generated
@@ -712,6 +712,12 @@ version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder-lite"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.11.1"
|
||||
@@ -2873,6 +2879,20 @@ dependencies = [
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "image"
|
||||
version = "0.25.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104"
|
||||
dependencies = [
|
||||
"bytemuck",
|
||||
"byteorder-lite",
|
||||
"moxcms",
|
||||
"num-traits",
|
||||
"zune-core",
|
||||
"zune-jpeg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "1.9.3"
|
||||
@@ -3365,6 +3385,16 @@ dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "moxcms"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b"
|
||||
dependencies = [
|
||||
"num-traits",
|
||||
"pxfm",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "muda"
|
||||
version = "0.19.1"
|
||||
@@ -4293,6 +4323,12 @@ version = "2.28.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"
|
||||
|
||||
[[package]]
|
||||
name = "pxfm"
|
||||
version = "0.1.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e0c5ccf5294c6ccd63a74f1565028353830a9c2f5eb0c682c355c471726a6e3f"
|
||||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.37.5"
|
||||
@@ -7855,6 +7891,10 @@ name = "wzp-desktop"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"image",
|
||||
"jni",
|
||||
"libloading 0.8.9",
|
||||
"ndk-context",
|
||||
@@ -7874,6 +7914,7 @@ dependencies = [
|
||||
"wzp-fec",
|
||||
"wzp-proto",
|
||||
"wzp-transport",
|
||||
"wzp-video",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8228,6 +8269,21 @@ version = "1.0.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
|
||||
|
||||
[[package]]
|
||||
name = "zune-core"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9"
|
||||
|
||||
[[package]]
|
||||
name = "zune-jpeg"
|
||||
version = "0.5.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296"
|
||||
dependencies = [
|
||||
"zune-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zvariant"
|
||||
version = "5.11.0"
|
||||
|
||||
1
android.sh
Normal file
1
android.sh
Normal file
@@ -0,0 +1 @@
|
||||
./scripts/android-build-async.sh --init
|
||||
@@ -538,6 +538,7 @@ async fn run_call(
|
||||
alias: alias.map(|s| s.to_string()),
|
||||
protocol_version: 2,
|
||||
supported_versions: vec![2],
|
||||
video_codecs: vec![CodecId::H264Baseline],
|
||||
};
|
||||
transport.send_signal(&offer).await?;
|
||||
info!("CallOffer sent, waiting for CallAnswer...");
|
||||
@@ -948,8 +949,8 @@ async fn run_call(
|
||||
}
|
||||
|
||||
let is_repair = pkt.header.is_repair();
|
||||
let pkt_block = pkt.header.fec_block as u8;
|
||||
let pkt_symbol = pkt.header.fec_block >> 8;
|
||||
let pkt_block = pkt.header.fec_block;
|
||||
let pkt_symbol = (pkt.header.fec_block >> 8) as u16;
|
||||
let pkt_is_opus = pkt.header.codec_id.is_opus();
|
||||
|
||||
// Phase 2: Opus packets bypass RaptorQ entirely — DRED
|
||||
|
||||
@@ -137,7 +137,7 @@ impl Pipeline {
|
||||
if header.fec_block != 0 {
|
||||
let is_repair = header.is_repair();
|
||||
if let Err(e) = self.fec_decoder.add_symbol(
|
||||
header.fec_block as u8,
|
||||
header.fec_block,
|
||||
header.fec_block >> 8,
|
||||
is_repair,
|
||||
&packet.payload,
|
||||
|
||||
@@ -15,7 +15,8 @@ use std::time::{Duration, Instant};
|
||||
use clap::Parser;
|
||||
use tracing::info;
|
||||
|
||||
use wzp_proto::{CodecId, MediaPacket, MediaTransport, default_signal_version};
|
||||
use wzp_proto::{CodecId, MediaPacket, MediaTransport, MediaType, default_signal_version};
|
||||
use wzp_video::{VideoDecoder, create_video_decoder, transport::VideoReassembler};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CLI
|
||||
@@ -68,6 +69,14 @@ struct Args {
|
||||
// For now, header-only analysis provides loss%, jitter, codec stats.
|
||||
#[arg(long)]
|
||||
key: Option<String>,
|
||||
|
||||
/// Track video fragmentation, completed frames, keyframes, and decode health.
|
||||
#[arg(long)]
|
||||
video_probe: bool,
|
||||
|
||||
/// Decode completed video frames in --video-probe mode.
|
||||
#[arg(long)]
|
||||
video_decode: bool,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -198,6 +207,305 @@ fn find_or_create_participant(
|
||||
id
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Video probe
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
struct PlaneSample {
|
||||
min: u8,
|
||||
max: u8,
|
||||
mean: f64,
|
||||
}
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
struct I420Sample {
|
||||
y: PlaneSample,
|
||||
u: PlaneSample,
|
||||
v: PlaneSample,
|
||||
valid_i420: bool,
|
||||
}
|
||||
|
||||
struct VideoStreamProbe {
|
||||
id: usize,
|
||||
codec: CodecId,
|
||||
wire_stream_id: u8,
|
||||
packets: u64,
|
||||
lost: u64,
|
||||
last_seq: u32,
|
||||
seq_initialized: bool,
|
||||
frames: u64,
|
||||
keyframes: u64,
|
||||
bytes: u64,
|
||||
max_frame_bytes: usize,
|
||||
first_seen: Instant,
|
||||
last_seen: Instant,
|
||||
last_frame: Option<Instant>,
|
||||
reassembler: VideoReassembler,
|
||||
decoder: Option<Box<dyn VideoDecoder>>,
|
||||
decoder_key: Option<(CodecId, u32, u32)>,
|
||||
decode_ok: u64,
|
||||
decode_pending: u64,
|
||||
decode_err: u64,
|
||||
last_decode_debug: Option<String>,
|
||||
last_i420_sample: Option<I420Sample>,
|
||||
}
|
||||
|
||||
impl VideoStreamProbe {
|
||||
fn new(id: usize, codec: CodecId, wire_stream_id: u8, decode: bool) -> Self {
|
||||
let decoder = if decode {
|
||||
create_video_decoder(codec, 1280, 720).ok()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let now = Instant::now();
|
||||
Self {
|
||||
id,
|
||||
codec,
|
||||
wire_stream_id,
|
||||
packets: 0,
|
||||
lost: 0,
|
||||
last_seq: 0,
|
||||
seq_initialized: false,
|
||||
frames: 0,
|
||||
keyframes: 0,
|
||||
bytes: 0,
|
||||
max_frame_bytes: 0,
|
||||
first_seen: now,
|
||||
last_seen: now,
|
||||
last_frame: None,
|
||||
reassembler: VideoReassembler::new(),
|
||||
decoder,
|
||||
decoder_key: decode.then_some((codec, 1280, 720)),
|
||||
decode_ok: 0,
|
||||
decode_pending: 0,
|
||||
decode_err: 0,
|
||||
last_decode_debug: None,
|
||||
last_i420_sample: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn ingest(&mut self, pkt: &MediaPacket, now: Instant) {
|
||||
self.packets += 1;
|
||||
self.last_seen = now;
|
||||
if pkt.header.codec_id != self.codec {
|
||||
self.codec = pkt.header.codec_id;
|
||||
self.reassembler = VideoReassembler::new();
|
||||
self.decoder = self
|
||||
.decoder
|
||||
.is_some()
|
||||
.then(|| create_video_decoder(self.codec, 1280, 720).ok())
|
||||
.flatten();
|
||||
self.decoder_key = self.decoder.as_ref().map(|_| (self.codec, 1280, 720));
|
||||
}
|
||||
if self.seq_initialized {
|
||||
let expected = self.last_seq.wrapping_add(1);
|
||||
let gap = pkt.header.seq.wrapping_sub(expected);
|
||||
if gap > 0 && gap < 100 {
|
||||
self.lost += gap as u64;
|
||||
}
|
||||
}
|
||||
self.last_seq = pkt.header.seq;
|
||||
self.seq_initialized = true;
|
||||
|
||||
if let Some(frame) = self.reassembler.push(pkt) {
|
||||
self.frames += 1;
|
||||
self.bytes += frame.data.len() as u64;
|
||||
self.max_frame_bytes = self.max_frame_bytes.max(frame.data.len());
|
||||
self.last_frame = Some(now);
|
||||
if frame.is_keyframe {
|
||||
self.keyframes += 1;
|
||||
}
|
||||
if frame.codec_id != self.codec {
|
||||
self.codec = frame.codec_id;
|
||||
}
|
||||
let frame_width = frame.width.unwrap_or(1280) as u32;
|
||||
let frame_height = frame.height.unwrap_or(720) as u32;
|
||||
let decoder_key = (self.codec, frame_width, frame_height);
|
||||
if self.decoder.is_some() && self.decoder_key != Some(decoder_key) {
|
||||
self.decoder = create_video_decoder(self.codec, frame_width, frame_height).ok();
|
||||
self.decoder_key = self.decoder.as_ref().map(|_| decoder_key);
|
||||
}
|
||||
if let Some(decoder) = self.decoder.as_mut() {
|
||||
match decoder.decode(&frame.data) {
|
||||
Ok(Some(decoded)) => {
|
||||
self.decode_ok += 1;
|
||||
self.last_decode_debug = decoder.debug_snapshot();
|
||||
self.last_i420_sample =
|
||||
Some(sample_i420(&decoded.data, decoded.width, decoded.height));
|
||||
}
|
||||
Ok(None) => {
|
||||
self.decode_pending += 1;
|
||||
self.last_decode_debug = decoder.debug_snapshot();
|
||||
}
|
||||
Err(err) => {
|
||||
self.decode_err += 1;
|
||||
self.last_decode_debug = Some(err.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn loss_percent(&self) -> f64 {
|
||||
let total = self.packets + self.lost;
|
||||
if total == 0 {
|
||||
0.0
|
||||
} else {
|
||||
(self.lost as f64 / total as f64) * 100.0
|
||||
}
|
||||
}
|
||||
|
||||
fn avg_frame_bytes(&self) -> u64 {
|
||||
if self.frames == 0 {
|
||||
0
|
||||
} else {
|
||||
self.bytes / self.frames
|
||||
}
|
||||
}
|
||||
|
||||
fn fps(&self) -> f64 {
|
||||
let secs = self.last_seen.duration_since(self.first_seen).as_secs_f64();
|
||||
if secs <= 0.0 {
|
||||
0.0
|
||||
} else {
|
||||
self.frames as f64 / secs
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct VideoProbe {
|
||||
streams: Vec<VideoStreamProbe>,
|
||||
decode: bool,
|
||||
}
|
||||
|
||||
impl VideoProbe {
|
||||
fn new(decode: bool) -> Self {
|
||||
Self {
|
||||
streams: Vec::new(),
|
||||
decode,
|
||||
}
|
||||
}
|
||||
|
||||
fn ingest(&mut self, pkt: &MediaPacket, now: Instant) {
|
||||
if pkt.header.media_type != MediaType::Video {
|
||||
return;
|
||||
}
|
||||
let idx = self.find_or_create_stream(pkt);
|
||||
self.streams[idx].ingest(pkt, now);
|
||||
}
|
||||
|
||||
fn find_or_create_stream(&mut self, pkt: &MediaPacket) -> usize {
|
||||
for (i, s) in self.streams.iter().enumerate() {
|
||||
if s.seq_initialized
|
||||
&& s.wire_stream_id == pkt.header.stream_id
|
||||
&& s.codec == pkt.header.codec_id
|
||||
{
|
||||
let delta = pkt.header.seq.wrapping_sub(s.last_seq);
|
||||
if delta > 0 && delta < 80 {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
}
|
||||
let id = self.streams.len();
|
||||
self.streams.push(VideoStreamProbe::new(
|
||||
id,
|
||||
pkt.header.codec_id,
|
||||
pkt.header.stream_id,
|
||||
self.decode,
|
||||
));
|
||||
id
|
||||
}
|
||||
|
||||
fn print(&self) {
|
||||
if self.streams.is_empty() {
|
||||
eprintln!(" video: no packets yet");
|
||||
return;
|
||||
}
|
||||
for s in &self.streams {
|
||||
let age_ms = s
|
||||
.last_frame
|
||||
.map(|t| t.elapsed().as_millis() as u64)
|
||||
.unwrap_or(u64::MAX);
|
||||
let mut line = format!(
|
||||
" video#{} wire_stream={} {:?}: {} pkts {:.1}% loss | {} frames ({:.1} fps), {} key, avg={}B max={}B, last_frame={}ms",
|
||||
s.id,
|
||||
s.wire_stream_id,
|
||||
s.codec,
|
||||
s.packets,
|
||||
s.loss_percent(),
|
||||
s.frames,
|
||||
s.fps(),
|
||||
s.keyframes,
|
||||
s.avg_frame_bytes(),
|
||||
s.max_frame_bytes,
|
||||
if age_ms == u64::MAX { 0 } else { age_ms },
|
||||
);
|
||||
if s.decoder.is_some() || s.decode_ok > 0 || s.decode_err > 0 {
|
||||
line.push_str(&format!(
|
||||
" | dec ok={} pending={} err={}",
|
||||
s.decode_ok, s.decode_pending, s.decode_err
|
||||
));
|
||||
}
|
||||
if let Some(sample) = &s.last_i420_sample {
|
||||
line.push_str(&format!(
|
||||
" | i420={} y={:.1}/{}/{} u={:.1}/{}/{} v={:.1}/{}/{}",
|
||||
sample.valid_i420,
|
||||
sample.y.mean,
|
||||
sample.y.min,
|
||||
sample.y.max,
|
||||
sample.u.mean,
|
||||
sample.u.min,
|
||||
sample.u.max,
|
||||
sample.v.mean,
|
||||
sample.v.min,
|
||||
sample.v.max,
|
||||
));
|
||||
}
|
||||
if let Some(debug) = &s.last_decode_debug {
|
||||
line.push_str(&format!(" | {debug}"));
|
||||
}
|
||||
eprintln!("{line}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_i420(data: &[u8], width: u32, height: u32) -> I420Sample {
|
||||
let y_len = width as usize * height as usize;
|
||||
let uv_len = y_len / 4;
|
||||
if data.len() < y_len + uv_len * 2 {
|
||||
return I420Sample {
|
||||
valid_i420: false,
|
||||
..I420Sample::default()
|
||||
};
|
||||
}
|
||||
I420Sample {
|
||||
valid_i420: true,
|
||||
y: sample_plane(&data[..y_len]),
|
||||
u: sample_plane(&data[y_len..y_len + uv_len]),
|
||||
v: sample_plane(&data[y_len + uv_len..y_len + uv_len * 2]),
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_plane(data: &[u8]) -> PlaneSample {
|
||||
if data.is_empty() {
|
||||
return PlaneSample::default();
|
||||
}
|
||||
let mut min = u8::MAX;
|
||||
let mut max = u8::MIN;
|
||||
let mut sum: u64 = 0;
|
||||
for &b in data {
|
||||
min = min.min(b);
|
||||
max = max.max(b);
|
||||
sum += b as u64;
|
||||
}
|
||||
PlaneSample {
|
||||
min,
|
||||
max,
|
||||
mean: sum as f64 / data.len() as f64,
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Capture writer (binary packet log for later replay)
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -580,6 +888,7 @@ async fn run_no_tui(
|
||||
total_packets: &mut u64,
|
||||
deadline: Option<Instant>,
|
||||
mut capture_writer: Option<&mut CaptureWriter>,
|
||||
mut video_probe: Option<&mut VideoProbe>,
|
||||
) -> anyhow::Result<()> {
|
||||
let mut print_timer = Instant::now();
|
||||
loop {
|
||||
@@ -594,6 +903,9 @@ async fn run_no_tui(
|
||||
let idx =
|
||||
find_or_create_participant(participants, pkt.header.seq, pkt.header.codec_id);
|
||||
participants[idx].ingest(&pkt, now);
|
||||
if let Some(ref mut probe) = video_probe {
|
||||
probe.ingest(&pkt, now);
|
||||
}
|
||||
*total_packets += 1;
|
||||
if let Some(ref mut w) = capture_writer {
|
||||
w.write_packet(&pkt, now)?;
|
||||
@@ -608,6 +920,9 @@ async fn run_no_tui(
|
||||
}
|
||||
if print_timer.elapsed() >= Duration::from_secs(2) {
|
||||
print_stats(participants, *total_packets);
|
||||
if let Some(ref probe) = video_probe {
|
||||
probe.print();
|
||||
}
|
||||
print_timer = Instant::now();
|
||||
}
|
||||
}
|
||||
@@ -616,7 +931,7 @@ async fn run_no_tui(
|
||||
|
||||
fn print_stats(participants: &[ParticipantStats], total: u64) {
|
||||
eprintln!(
|
||||
"--- {} participants | {} total packets ---",
|
||||
"--- {} packet streams | {} total packets ---",
|
||||
participants.len(),
|
||||
total
|
||||
);
|
||||
@@ -644,6 +959,7 @@ async fn run_tui(
|
||||
start_time: Instant,
|
||||
deadline: Option<Instant>,
|
||||
mut capture_writer: Option<&mut CaptureWriter>,
|
||||
mut video_probe: Option<&mut VideoProbe>,
|
||||
) -> anyhow::Result<()> {
|
||||
crossterm::terminal::enable_raw_mode()?;
|
||||
let mut stdout = std::io::stdout();
|
||||
@@ -684,6 +1000,9 @@ async fn run_tui(
|
||||
pkt.header.codec_id,
|
||||
);
|
||||
participants[idx].ingest(&pkt, now);
|
||||
if let Some(ref mut probe) = video_probe {
|
||||
probe.ingest(&pkt, now);
|
||||
}
|
||||
*total_packets += 1;
|
||||
if let Some(ref mut w) = capture_writer {
|
||||
w.write_packet(&pkt, now)?;
|
||||
@@ -941,6 +1260,17 @@ async fn main() -> anyhow::Result<()> {
|
||||
let mut participants: Vec<ParticipantStats> = Vec::new();
|
||||
let mut total_packets: u64 = 0;
|
||||
let start_time = Instant::now();
|
||||
let mut video_probe = (args.video_probe || args.video_decode).then(|| {
|
||||
eprintln!(
|
||||
"Video probe enabled{}",
|
||||
if args.video_decode {
|
||||
" with decode"
|
||||
} else {
|
||||
""
|
||||
}
|
||||
);
|
||||
VideoProbe::new(args.video_decode)
|
||||
});
|
||||
|
||||
if args.no_tui {
|
||||
run_no_tui(
|
||||
@@ -949,6 +1279,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
&mut total_packets,
|
||||
deadline,
|
||||
capture_writer.as_mut(),
|
||||
video_probe.as_mut(),
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
@@ -959,12 +1290,17 @@ async fn main() -> anyhow::Result<()> {
|
||||
start_time,
|
||||
deadline,
|
||||
capture_writer.as_mut(),
|
||||
video_probe.as_mut(),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
// Print summary
|
||||
print_summary(&participants, total_packets, start_time.elapsed());
|
||||
if let Some(probe) = &video_probe {
|
||||
eprintln!("\n=== Video Probe Summary ===");
|
||||
probe.print();
|
||||
}
|
||||
|
||||
// Clean close
|
||||
transport.close().await?;
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
//! This is the same engine FaceTime and other Apple apps use.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
|
||||
use anyhow::Context;
|
||||
use coreaudio::audio_unit::audio_format::LinearPcmFlags;
|
||||
@@ -28,6 +28,60 @@ pub struct VpioAudio {
|
||||
playout_ring: Arc<AudioRing>,
|
||||
_audio_unit: AudioUnit,
|
||||
running: Arc<AtomicBool>,
|
||||
stats: Arc<VpioStats>,
|
||||
}
|
||||
|
||||
/// Render/capture counters for diagnosing macOS VoiceProcessingIO.
|
||||
///
|
||||
/// These are atomics because CoreAudio callbacks run on realtime audio
|
||||
/// threads. The Tauri engine polls snapshots from a normal async task and
|
||||
/// emits them to the call debug log.
|
||||
#[derive(Default)]
|
||||
pub struct VpioStats {
|
||||
capture_callbacks: AtomicU64,
|
||||
capture_samples: AtomicU64,
|
||||
render_callbacks: AtomicU64,
|
||||
render_requested_samples: AtomicU64,
|
||||
render_read_samples: AtomicU64,
|
||||
render_underrun_callbacks: AtomicU64,
|
||||
render_nonzero_callbacks: AtomicU64,
|
||||
render_last_requested: AtomicU64,
|
||||
render_last_read: AtomicU64,
|
||||
render_last_rms: AtomicU64,
|
||||
render_last_ring_available: AtomicU64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct VpioStatsSnapshot {
|
||||
pub capture_callbacks: u64,
|
||||
pub capture_samples: u64,
|
||||
pub render_callbacks: u64,
|
||||
pub render_requested_samples: u64,
|
||||
pub render_read_samples: u64,
|
||||
pub render_underrun_callbacks: u64,
|
||||
pub render_nonzero_callbacks: u64,
|
||||
pub render_last_requested: u64,
|
||||
pub render_last_read: u64,
|
||||
pub render_last_rms: u64,
|
||||
pub render_last_ring_available: u64,
|
||||
}
|
||||
|
||||
impl VpioStats {
|
||||
pub fn snapshot(&self) -> VpioStatsSnapshot {
|
||||
VpioStatsSnapshot {
|
||||
capture_callbacks: self.capture_callbacks.load(Ordering::Relaxed),
|
||||
capture_samples: self.capture_samples.load(Ordering::Relaxed),
|
||||
render_callbacks: self.render_callbacks.load(Ordering::Relaxed),
|
||||
render_requested_samples: self.render_requested_samples.load(Ordering::Relaxed),
|
||||
render_read_samples: self.render_read_samples.load(Ordering::Relaxed),
|
||||
render_underrun_callbacks: self.render_underrun_callbacks.load(Ordering::Relaxed),
|
||||
render_nonzero_callbacks: self.render_nonzero_callbacks.load(Ordering::Relaxed),
|
||||
render_last_requested: self.render_last_requested.load(Ordering::Relaxed),
|
||||
render_last_read: self.render_last_read.load(Ordering::Relaxed),
|
||||
render_last_rms: self.render_last_rms.load(Ordering::Relaxed),
|
||||
render_last_ring_available: self.render_last_ring_available.load(Ordering::Relaxed),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl VpioAudio {
|
||||
@@ -36,6 +90,7 @@ impl VpioAudio {
|
||||
let capture_ring = Arc::new(AudioRing::new());
|
||||
let playout_ring = Arc::new(AudioRing::new());
|
||||
let running = Arc::new(AtomicBool::new(true));
|
||||
let stats = Arc::new(VpioStats::default());
|
||||
|
||||
let mut au = AudioUnit::new(IOType::VoiceProcessingIO)
|
||||
.context("failed to create VoiceProcessingIO audio unit")?;
|
||||
@@ -98,6 +153,7 @@ impl VpioAudio {
|
||||
// Set up input callback (mic capture with AEC applied)
|
||||
let cap_ring = capture_ring.clone();
|
||||
let cap_running = running.clone();
|
||||
let cap_stats = stats.clone();
|
||||
let logged = Arc::new(AtomicBool::new(false));
|
||||
au.set_input_callback(
|
||||
move |args: render_callback::Args<data::NonInterleaved<f32>>| {
|
||||
@@ -106,6 +162,10 @@ impl VpioAudio {
|
||||
}
|
||||
let mut buffers = args.data.channels();
|
||||
if let Some(ch) = buffers.next() {
|
||||
cap_stats.capture_callbacks.fetch_add(1, Ordering::Relaxed);
|
||||
cap_stats
|
||||
.capture_samples
|
||||
.fetch_add(ch.len() as u64, Ordering::Relaxed);
|
||||
if !logged.swap(true, Ordering::Relaxed) {
|
||||
eprintln!("[vpio] capture callback: {} f32 samples", ch.len());
|
||||
}
|
||||
@@ -125,21 +185,72 @@ impl VpioAudio {
|
||||
|
||||
// Set up output callback (speaker playback — AEC uses this as reference)
|
||||
let play_ring = playout_ring.clone();
|
||||
let render_stats = stats.clone();
|
||||
let logged_render = Arc::new(AtomicBool::new(false));
|
||||
au.set_render_callback(
|
||||
move |mut args: render_callback::Args<data::NonInterleaved<f32>>| {
|
||||
let mut buffers = args.data.channels_mut();
|
||||
if let Some(ch) = buffers.next() {
|
||||
render_stats
|
||||
.render_callbacks
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
render_stats
|
||||
.render_requested_samples
|
||||
.fetch_add(ch.len() as u64, Ordering::Relaxed);
|
||||
render_stats
|
||||
.render_last_requested
|
||||
.store(ch.len() as u64, Ordering::Relaxed);
|
||||
let mut tmp = [0i16; FRAME_SAMPLES];
|
||||
let mut total_read = 0usize;
|
||||
let mut sum_sq = 0u64;
|
||||
let ring_available = play_ring.available();
|
||||
for chunk in ch.chunks_mut(FRAME_SAMPLES) {
|
||||
let n = chunk.len();
|
||||
let read = play_ring.read(&mut tmp[..n]);
|
||||
total_read += read;
|
||||
for i in 0..read {
|
||||
let s = tmp[i] as i64;
|
||||
sum_sq = sum_sq.saturating_add((s * s) as u64);
|
||||
chunk[i] = tmp[i] as f32 / i16::MAX as f32;
|
||||
}
|
||||
for i in read..n {
|
||||
chunk[i] = 0.0;
|
||||
}
|
||||
}
|
||||
render_stats
|
||||
.render_read_samples
|
||||
.fetch_add(total_read as u64, Ordering::Relaxed);
|
||||
render_stats
|
||||
.render_last_read
|
||||
.store(total_read as u64, Ordering::Relaxed);
|
||||
render_stats
|
||||
.render_last_ring_available
|
||||
.store(ring_available as u64, Ordering::Relaxed);
|
||||
if total_read == 0 {
|
||||
render_stats
|
||||
.render_underrun_callbacks
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
let rms = if total_read > 0 {
|
||||
((sum_sq as f64 / total_read as f64).sqrt()) as u64
|
||||
} else {
|
||||
0
|
||||
};
|
||||
render_stats.render_last_rms.store(rms, Ordering::Relaxed);
|
||||
if rms > 0 {
|
||||
render_stats
|
||||
.render_nonzero_callbacks
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
if !logged_render.swap(true, Ordering::Relaxed) {
|
||||
eprintln!(
|
||||
"[vpio] render callback: {} f32 samples, ring_available={}, ring_read={}, rms={}",
|
||||
ch.len(),
|
||||
ring_available,
|
||||
total_read,
|
||||
rms
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
@@ -157,6 +268,7 @@ impl VpioAudio {
|
||||
playout_ring,
|
||||
_audio_unit: au,
|
||||
running,
|
||||
stats,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -168,6 +280,10 @@ impl VpioAudio {
|
||||
&self.playout_ring
|
||||
}
|
||||
|
||||
pub fn stats(&self) -> Arc<VpioStats> {
|
||||
self.stats.clone()
|
||||
}
|
||||
|
||||
pub fn stop(&self) {
|
||||
self.running.store(false, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
@@ -151,7 +151,7 @@ pub fn bench_fec_recovery(loss_pct: f32) -> FecResult {
|
||||
let mut total_repair_bytes = 0usize;
|
||||
|
||||
for block_idx in 0..num_blocks {
|
||||
let block_id = (block_idx % 256) as u8;
|
||||
let block_id = (block_idx % 65536) as u16;
|
||||
|
||||
// Create fresh encoder and decoder for each block
|
||||
let mut fec_enc = RaptorQFecEncoder::new(frames_per_block, 256);
|
||||
|
||||
@@ -565,7 +565,7 @@ impl CallDecoder {
|
||||
// ignored — a graceful mixed-version degradation).
|
||||
if !packet.header.codec_id.is_opus() {
|
||||
let _ = self.fec_dec.add_symbol(
|
||||
(packet.header.fec_block & 0xFF) as u8,
|
||||
packet.header.fec_block,
|
||||
packet.header.fec_block >> 8,
|
||||
packet.header.is_repair(),
|
||||
&packet.payload,
|
||||
|
||||
@@ -388,17 +388,17 @@ async fn main() -> anyhow::Result<()> {
|
||||
}
|
||||
|
||||
// Crypto handshake — establishes verified identity + session key
|
||||
let session = wzp_client::handshake::perform_handshake(
|
||||
let hs = wzp_client::handshake::perform_handshake(
|
||||
&*transport,
|
||||
&seed.0,
|
||||
None, // alias — desktop client doesn't set one yet
|
||||
)
|
||||
.await?;
|
||||
info!("crypto handshake complete");
|
||||
info!(video_codec = ?hs.video_codec, "crypto handshake complete");
|
||||
|
||||
// Wrap the transport so all media I/O goes through AEAD encryption.
|
||||
let enc_transport: Arc<dyn wzp_proto::MediaTransport> = Arc::new(
|
||||
wzp_client::encrypted_transport::EncryptingTransport::new(transport.clone(), session),
|
||||
wzp_client::encrypted_transport::EncryptingTransport::new(transport.clone(), hs.session),
|
||||
);
|
||||
|
||||
if cli.live {
|
||||
@@ -942,7 +942,7 @@ async fn run_signal_mode(
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_session) => {
|
||||
Ok(_hs) => {
|
||||
info!(
|
||||
"media connected — sending tone (press Ctrl+C to hang up)"
|
||||
);
|
||||
|
||||
@@ -164,6 +164,7 @@ mod tests {
|
||||
alias: None,
|
||||
protocol_version: 2,
|
||||
supported_versions: vec![2],
|
||||
video_codecs: vec![],
|
||||
};
|
||||
|
||||
let encoded = encode_call_payload(&signal, Some("relay.example.com:4433"), Some("myroom"));
|
||||
@@ -185,6 +186,7 @@ mod tests {
|
||||
alias: None,
|
||||
protocol_version: 2,
|
||||
supported_versions: vec![2],
|
||||
video_codecs: vec![],
|
||||
};
|
||||
assert!(matches!(signal_to_call_type(&offer), CallSignalType::Offer));
|
||||
|
||||
|
||||
@@ -5,9 +5,18 @@
|
||||
|
||||
use wzp_crypto::{CryptoSession, KeyExchange, WarzoneKeyExchange};
|
||||
use wzp_proto::{
|
||||
HangupReason, MediaTransport, QualityProfile, SignalMessage, default_signal_version,
|
||||
CodecId, HangupReason, MediaTransport, QualityProfile, SignalMessage, default_signal_version,
|
||||
};
|
||||
|
||||
const SUPPORTED_VIDEO_CODECS: &[CodecId] = &[CodecId::H264Baseline];
|
||||
|
||||
/// Result of a successful client-side handshake.
|
||||
pub struct HandshakeResult {
|
||||
pub session: Box<dyn CryptoSession>,
|
||||
/// Video codec agreed with the relay. `None` if peer is audio-only.
|
||||
pub video_codec: Option<CodecId>,
|
||||
}
|
||||
|
||||
/// Errors that can occur during the client-side cryptographic handshake.
|
||||
#[derive(Debug)]
|
||||
pub enum HandshakeError {
|
||||
@@ -64,7 +73,17 @@ pub async fn perform_handshake(
|
||||
transport: &dyn MediaTransport,
|
||||
seed: &[u8; 32],
|
||||
alias: Option<&str>,
|
||||
) -> Result<Box<dyn CryptoSession>, HandshakeError> {
|
||||
) -> Result<HandshakeResult, HandshakeError> {
|
||||
perform_handshake_with_video_codecs(transport, seed, alias, SUPPORTED_VIDEO_CODECS.to_vec())
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn perform_handshake_with_video_codecs(
|
||||
transport: &dyn MediaTransport,
|
||||
seed: &[u8; 32],
|
||||
alias: Option<&str>,
|
||||
video_codecs: Vec<CodecId>,
|
||||
) -> Result<HandshakeResult, HandshakeError> {
|
||||
// 1. Create key exchange from identity seed
|
||||
let mut kx = WarzoneKeyExchange::from_identity_seed(seed);
|
||||
let identity_pub = kx.identity_public_key();
|
||||
@@ -95,6 +114,7 @@ pub async fn perform_handshake(
|
||||
alias: alias.map(|s| s.to_string()),
|
||||
protocol_version: 2,
|
||||
supported_versions: vec![2],
|
||||
video_codecs,
|
||||
};
|
||||
transport
|
||||
.send_signal(&offer)
|
||||
@@ -102,24 +122,28 @@ pub async fn perform_handshake(
|
||||
.map_err(HandshakeError::Transport)?;
|
||||
|
||||
// 5. Wait for CallAnswer — 10s timeout guards against relay not responding.
|
||||
let answer = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(10),
|
||||
transport.recv_signal(),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| HandshakeError::Transport(wzp_proto::TransportError::Timeout { ms: 10_000 }))?
|
||||
.map_err(HandshakeError::Transport)?
|
||||
.ok_or(HandshakeError::ConnectionClosed)?;
|
||||
let answer = tokio::time::timeout(std::time::Duration::from_secs(10), transport.recv_signal())
|
||||
.await
|
||||
.map_err(|_| HandshakeError::Transport(wzp_proto::TransportError::Timeout { ms: 10_000 }))?
|
||||
.map_err(HandshakeError::Transport)?
|
||||
.ok_or(HandshakeError::ConnectionClosed)?;
|
||||
|
||||
let (callee_identity_pub, callee_ephemeral_pub, callee_signature, _chosen_profile) =
|
||||
let (callee_identity_pub, callee_ephemeral_pub, callee_signature, _chosen_profile, video_codec) =
|
||||
match answer {
|
||||
SignalMessage::CallAnswer {
|
||||
identity_pub,
|
||||
ephemeral_pub,
|
||||
signature,
|
||||
chosen_profile,
|
||||
video_codec,
|
||||
..
|
||||
} => (identity_pub, ephemeral_pub, signature, chosen_profile),
|
||||
} => (
|
||||
identity_pub,
|
||||
ephemeral_pub,
|
||||
signature,
|
||||
chosen_profile,
|
||||
video_codec,
|
||||
),
|
||||
SignalMessage::Hangup {
|
||||
reason: HangupReason::ProtocolVersionMismatch { server_supported },
|
||||
..
|
||||
@@ -144,7 +168,10 @@ pub async fn perform_handshake(
|
||||
.derive_session(&callee_ephemeral_pub)
|
||||
.map_err(|e| HandshakeError::KeyDerivation(e.to_string()))?;
|
||||
|
||||
Ok(session)
|
||||
Ok(HandshakeResult {
|
||||
session,
|
||||
video_codec,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -166,4 +193,34 @@ mod tests {
|
||||
&sig,
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn handshake_result_carries_video_codec() {
|
||||
// Verify that HandshakeResult has both fields accessible and that
|
||||
// None is the correct default for audio-only peers.
|
||||
let mut kx = WarzoneKeyExchange::from_identity_seed(&[0x55; 32]);
|
||||
kx.generate_ephemeral();
|
||||
let session = kx.derive_session(&[0u8; 32]).unwrap();
|
||||
let hs = HandshakeResult {
|
||||
session,
|
||||
video_codec: None,
|
||||
};
|
||||
assert!(hs.video_codec.is_none());
|
||||
|
||||
let mut kx2 = WarzoneKeyExchange::from_identity_seed(&[0x66; 32]);
|
||||
kx2.generate_ephemeral();
|
||||
let session2 = kx2.derive_session(&[0u8; 32]).unwrap();
|
||||
let hs2 = HandshakeResult {
|
||||
session: session2,
|
||||
video_codec: Some(CodecId::H264Baseline),
|
||||
};
|
||||
assert_eq!(hs2.video_codec, Some(CodecId::H264Baseline));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn offer_contains_h264_only() {
|
||||
// Keep room video on the common denominator until Android AV1/HEVC
|
||||
// send paths are proven in-device.
|
||||
assert_eq!(SUPPORTED_VIDEO_CODECS, &[CodecId::H264Baseline]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -91,7 +91,7 @@ async fn full_handshake_both_sides_derive_same_session() {
|
||||
wzp_relay::handshake::accept_handshake(relay_transport_clone.as_ref(), &relay_seed),
|
||||
);
|
||||
|
||||
let mut client_session = client_result.expect("client handshake should succeed");
|
||||
let client_hs = client_result.expect("client handshake should succeed");
|
||||
let (mut relay_session, chosen_profile, _caller_fp, _caller_alias) =
|
||||
relay_result.expect("relay handshake should succeed");
|
||||
|
||||
@@ -122,6 +122,7 @@ async fn full_handshake_both_sides_derive_same_session() {
|
||||
let header = make_hdr(0);
|
||||
let plaintext = b"hello from client to relay";
|
||||
|
||||
let mut client_session = client_hs.session;
|
||||
let mut ciphertext = Vec::new();
|
||||
client_session
|
||||
.encrypt(&header, plaintext, &mut ciphertext)
|
||||
@@ -180,6 +181,7 @@ async fn handshake_rejects_tampered_signature() {
|
||||
alias: None,
|
||||
protocol_version: 2,
|
||||
supported_versions: vec![2],
|
||||
video_codecs: vec![],
|
||||
};
|
||||
client_transport_clone
|
||||
.send_signal(&offer)
|
||||
|
||||
@@ -114,11 +114,7 @@ impl EchoCanceller {
|
||||
/// Number of delayed samples available to release.
|
||||
fn delay_available(&self) -> usize {
|
||||
let buffered = self.delay_write - self.delay_read;
|
||||
if buffered > self.delay_samples {
|
||||
buffered - self.delay_samples
|
||||
} else {
|
||||
0
|
||||
}
|
||||
buffered.saturating_sub(self.delay_samples)
|
||||
}
|
||||
|
||||
/// Process a near-end (microphone) frame, removing the estimated echo.
|
||||
@@ -161,8 +157,8 @@ impl EchoCanceller {
|
||||
let mut sum_near_sq: f64 = 0.0;
|
||||
let mut sum_err_sq: f64 = 0.0;
|
||||
|
||||
for i in 0..n {
|
||||
let near_f = nearend[i] as f32;
|
||||
for (i, sample) in nearend.iter_mut().enumerate() {
|
||||
let near_f = *sample as f32;
|
||||
|
||||
// Position of far-end "now" for this near-end sample.
|
||||
let base = (self.far_pos + fl * ((n / fl) + 2) + i - n) % fl;
|
||||
@@ -190,7 +186,7 @@ impl EchoCanceller {
|
||||
}
|
||||
|
||||
let out = error.clamp(-32768.0, 32767.0);
|
||||
nearend[i] = out as i16;
|
||||
*sample = out as i16;
|
||||
|
||||
sum_near_sq += (near_f as f64).powi(2);
|
||||
sum_err_sq += (out as f64).powi(2);
|
||||
|
||||
@@ -45,7 +45,7 @@ impl Codec2Decoder {
|
||||
|
||||
/// Number of compressed bytes per frame.
|
||||
fn bytes_per_frame(&self) -> usize {
|
||||
(self.inner.bits_per_frame() + 7) / 8
|
||||
self.inner.bits_per_frame().div_ceil(8)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -45,7 +45,7 @@ impl Codec2Encoder {
|
||||
|
||||
/// Number of compressed bytes per frame.
|
||||
fn bytes_per_frame(&self) -> usize {
|
||||
(self.inner.bits_per_frame() + 7) / 8
|
||||
self.inner.bits_per_frame().div_ceil(8)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -56,7 +56,7 @@ impl NoiseSupressor {
|
||||
|
||||
// f32 → i16 with clamping
|
||||
for (i, &val) in output.iter().enumerate() {
|
||||
let clamped = val.max(-32768.0).min(32767.0);
|
||||
let clamped = val.clamp(-32768.0, 32767.0);
|
||||
pcm[offset + i] = clamped as i16;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,7 +101,7 @@ pub fn dred_duration_for(codec: CodecId) -> u8 {
|
||||
/// mode; unset or empty leaves DRED enabled.
|
||||
fn read_legacy_fec_env() -> bool {
|
||||
match std::env::var(LEGACY_FEC_ENV) {
|
||||
Ok(v) => !v.is_empty() && v != "0" && v.to_ascii_lowercase() != "false",
|
||||
Ok(v) => !v.is_empty() && v != "0" && !v.eq_ignore_ascii_case("false"),
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
@@ -252,7 +252,7 @@ impl OpusEncoder {
|
||||
let clamped = if self.legacy_fec_mode {
|
||||
loss_pct.min(100)
|
||||
} else {
|
||||
loss_pct.max(DRED_LOSS_FLOOR_PCT).min(100)
|
||||
loss_pct.clamp(DRED_LOSS_FLOOR_PCT, 100)
|
||||
};
|
||||
let _ = self.inner.set_packet_loss(clamped);
|
||||
}
|
||||
|
||||
@@ -48,7 +48,7 @@ fn build_fir_kernel() -> [f64; FIR_TAPS] {
|
||||
let fc = CUTOFF_HZ / SAMPLE_RATE; // normalised cutoff (0..0.5)
|
||||
let beta_denom = bessel_i0(KAISER_BETA);
|
||||
|
||||
for i in 0..FIR_TAPS {
|
||||
for (i, slot) in kernel.iter_mut().enumerate() {
|
||||
// Sinc
|
||||
let n = i as f64 - m / 2.0;
|
||||
let sinc = if n.abs() < 1e-12 {
|
||||
@@ -61,7 +61,7 @@ fn build_fir_kernel() -> [f64; FIR_TAPS] {
|
||||
let t = 2.0 * i as f64 / m - 1.0; // range [-1, 1]
|
||||
let kaiser = bessel_i0(KAISER_BETA * (1.0 - t * t).max(0.0).sqrt()) / beta_denom;
|
||||
|
||||
kernel[i] = sinc * kaiser;
|
||||
*slot = sinc * kaiser;
|
||||
}
|
||||
|
||||
// Normalise to unity DC gain.
|
||||
@@ -180,9 +180,7 @@ impl Upsampler8to48 {
|
||||
work.extend_from_slice(&self.history);
|
||||
for &s in input {
|
||||
work.push(s as f64);
|
||||
for _ in 1..RATIO {
|
||||
work.push(0.0);
|
||||
}
|
||||
work.resize(work.len() + (RATIO - 1), 0.0f64);
|
||||
}
|
||||
|
||||
let out_len = stuffed_len;
|
||||
|
||||
@@ -122,6 +122,7 @@ fn wzp_signal_serializes_into_fc_callsignal_payload() {
|
||||
alias: None,
|
||||
protocol_version: 2,
|
||||
supported_versions: vec![2],
|
||||
video_codecs: vec![],
|
||||
};
|
||||
|
||||
// Encode as featherChat CallSignal payload
|
||||
@@ -186,6 +187,7 @@ fn wzp_answer_round_trips_through_fc_callsignal() {
|
||||
ephemeral_pub: [20u8; 32],
|
||||
signature: vec![30u8; 64],
|
||||
chosen_profile: wzp_proto::QualityProfile::DEGRADED,
|
||||
video_codec: None,
|
||||
};
|
||||
|
||||
let payload = wzp_client::featherchat::encode_call_payload(&answer, None, None);
|
||||
@@ -309,6 +311,7 @@ fn all_signal_types_map_correctly() {
|
||||
alias: None,
|
||||
protocol_version: 2,
|
||||
supported_versions: vec![2],
|
||||
video_codecs: vec![],
|
||||
},
|
||||
"Offer",
|
||||
),
|
||||
@@ -319,6 +322,7 @@ fn all_signal_types_map_correctly() {
|
||||
ephemeral_pub: [0; 32],
|
||||
signature: vec![],
|
||||
chosen_profile: wzp_proto::QualityProfile::GOOD,
|
||||
video_codec: None,
|
||||
},
|
||||
"Answer",
|
||||
),
|
||||
|
||||
@@ -29,9 +29,9 @@ pub enum DecoderBlockState {
|
||||
/// Manages encoder-side block tracking.
|
||||
pub struct EncoderBlockManager {
|
||||
/// Current block ID being built.
|
||||
current_id: u8,
|
||||
current_id: u16,
|
||||
/// State of known blocks.
|
||||
blocks: HashMap<u8, EncoderBlockState>,
|
||||
blocks: HashMap<u16, EncoderBlockState>,
|
||||
}
|
||||
|
||||
impl EncoderBlockManager {
|
||||
@@ -45,7 +45,7 @@ impl EncoderBlockManager {
|
||||
}
|
||||
|
||||
/// Get the next block ID (advances the current building block).
|
||||
pub fn next_block_id(&mut self) -> u8 {
|
||||
pub fn next_block_id(&mut self) -> u16 {
|
||||
let old = self.current_id;
|
||||
// Mark old block as pending.
|
||||
self.blocks.insert(old, EncoderBlockState::Pending);
|
||||
@@ -57,23 +57,23 @@ impl EncoderBlockManager {
|
||||
}
|
||||
|
||||
/// Current block ID being built.
|
||||
pub fn current_id(&self) -> u8 {
|
||||
pub fn current_id(&self) -> u16 {
|
||||
self.current_id
|
||||
}
|
||||
|
||||
/// Mark a block as fully sent.
|
||||
pub fn mark_sent(&mut self, block_id: u8) {
|
||||
pub fn mark_sent(&mut self, block_id: u16) {
|
||||
self.blocks.insert(block_id, EncoderBlockState::Sent);
|
||||
}
|
||||
|
||||
/// Mark a block as acknowledged by the peer.
|
||||
pub fn mark_acknowledged(&mut self, block_id: u8) {
|
||||
pub fn mark_acknowledged(&mut self, block_id: u16) {
|
||||
self.blocks
|
||||
.insert(block_id, EncoderBlockState::Acknowledged);
|
||||
}
|
||||
|
||||
/// Get the state of a block.
|
||||
pub fn state(&self, block_id: u8) -> Option<EncoderBlockState> {
|
||||
pub fn state(&self, block_id: u16) -> Option<EncoderBlockState> {
|
||||
self.blocks.get(&block_id).copied()
|
||||
}
|
||||
|
||||
@@ -93,9 +93,9 @@ impl Default for EncoderBlockManager {
|
||||
/// Manages decoder-side block tracking.
|
||||
pub struct DecoderBlockManager {
|
||||
/// State of known blocks.
|
||||
blocks: HashMap<u8, DecoderBlockState>,
|
||||
blocks: HashMap<u16, DecoderBlockState>,
|
||||
/// Set of completed block IDs.
|
||||
completed: HashSet<u8>,
|
||||
completed: HashSet<u16>,
|
||||
}
|
||||
|
||||
impl DecoderBlockManager {
|
||||
@@ -107,43 +107,43 @@ impl DecoderBlockManager {
|
||||
}
|
||||
|
||||
/// Register that we are receiving symbols for a block.
|
||||
pub fn touch(&mut self, block_id: u8) {
|
||||
pub fn touch(&mut self, block_id: u16) {
|
||||
self.blocks
|
||||
.entry(block_id)
|
||||
.or_insert(DecoderBlockState::Assembling);
|
||||
}
|
||||
|
||||
/// Mark a block as successfully decoded.
|
||||
pub fn mark_complete(&mut self, block_id: u8) {
|
||||
pub fn mark_complete(&mut self, block_id: u16) {
|
||||
self.blocks.insert(block_id, DecoderBlockState::Complete);
|
||||
self.completed.insert(block_id);
|
||||
}
|
||||
|
||||
/// Mark a block as expired.
|
||||
pub fn mark_expired(&mut self, block_id: u8) {
|
||||
pub fn mark_expired(&mut self, block_id: u16) {
|
||||
self.blocks.insert(block_id, DecoderBlockState::Expired);
|
||||
self.completed.remove(&block_id);
|
||||
}
|
||||
|
||||
/// Check if a block has been fully decoded.
|
||||
pub fn is_block_complete(&self, block_id: u8) -> bool {
|
||||
pub fn is_block_complete(&self, block_id: u16) -> bool {
|
||||
self.completed.contains(&block_id)
|
||||
}
|
||||
|
||||
/// Get the state of a block.
|
||||
pub fn state(&self, block_id: u8) -> Option<DecoderBlockState> {
|
||||
pub fn state(&self, block_id: u16) -> Option<DecoderBlockState> {
|
||||
self.blocks.get(&block_id).copied()
|
||||
}
|
||||
|
||||
/// Expire all blocks older than the given block_id (using wrapping distance).
|
||||
pub fn expire_before(&mut self, block_id: u8) {
|
||||
let to_expire: Vec<u8> = self
|
||||
pub fn expire_before(&mut self, block_id: u16) {
|
||||
let to_expire: Vec<u16> = self
|
||||
.blocks
|
||||
.keys()
|
||||
.copied()
|
||||
.filter(|&id| {
|
||||
let distance = block_id.wrapping_sub(id);
|
||||
distance > 0 && distance <= 128
|
||||
distance > 0 && distance <= 32768
|
||||
})
|
||||
.collect();
|
||||
|
||||
@@ -207,7 +207,7 @@ mod tests {
|
||||
#[test]
|
||||
fn decoder_expire_before() {
|
||||
let mut mgr = DecoderBlockManager::new();
|
||||
for i in 0..5u8 {
|
||||
for i in 0..5u16 {
|
||||
mgr.touch(i);
|
||||
}
|
||||
mgr.mark_complete(1);
|
||||
@@ -231,11 +231,11 @@ mod tests {
|
||||
#[test]
|
||||
fn next_block_id_wraps() {
|
||||
let mut mgr = EncoderBlockManager::new();
|
||||
// Start at 0, advance to 255 then wrap
|
||||
for _ in 0..255 {
|
||||
// Start at 0, advance to u16::MAX then wrap
|
||||
for _ in 0..65535 {
|
||||
mgr.next_block_id();
|
||||
}
|
||||
assert_eq!(mgr.current_id(), 255);
|
||||
assert_eq!(mgr.current_id(), u16::MAX);
|
||||
let next = mgr.next_block_id();
|
||||
assert_eq!(next, 0);
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@ struct BlockState {
|
||||
/// RaptorQ-based FEC decoder that handles multiple concurrent blocks.
|
||||
pub struct RaptorQFecDecoder {
|
||||
/// Per-block decoder state, keyed by block_id.
|
||||
blocks: HashMap<u8, BlockState>,
|
||||
blocks: HashMap<u16, BlockState>,
|
||||
/// Symbol size (must match encoder).
|
||||
symbol_size: u16,
|
||||
/// Number of source symbols per block (from encoder config).
|
||||
@@ -57,7 +57,7 @@ impl RaptorQFecDecoder {
|
||||
Self::new(frames_per_block, 256)
|
||||
}
|
||||
|
||||
fn get_or_create_block(&mut self, block_id: u8) -> &mut BlockState {
|
||||
fn get_or_create_block(&mut self, block_id: u16) -> &mut BlockState {
|
||||
self.blocks.entry(block_id).or_insert_with(|| BlockState {
|
||||
num_source_symbols: Some(self.frames_per_block),
|
||||
packets: Vec::new(),
|
||||
@@ -72,7 +72,7 @@ impl RaptorQFecDecoder {
|
||||
impl FecDecoder for RaptorQFecDecoder {
|
||||
fn add_symbol(
|
||||
&mut self,
|
||||
block_id: u8,
|
||||
block_id: u16,
|
||||
symbol_index: u16,
|
||||
_is_repair: bool,
|
||||
data: &[u8],
|
||||
@@ -104,13 +104,13 @@ impl FecDecoder for RaptorQFecDecoder {
|
||||
padded[..len].copy_from_slice(&data[..len]);
|
||||
|
||||
let esi = symbol_index as u32;
|
||||
let packet = EncodingPacket::new(PayloadId::new(block_id, esi), padded);
|
||||
let packet = EncodingPacket::new(PayloadId::new((block_id & 0xFF) as u8, esi), padded);
|
||||
block.packets.push(packet);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn try_decode(&mut self, block_id: u8) -> Result<Option<Vec<Vec<u8>>>, FecError> {
|
||||
fn try_decode(&mut self, block_id: u16) -> Result<Option<Vec<Vec<u8>>>, FecError> {
|
||||
let frames_per_block = self.frames_per_block;
|
||||
let block = match self.blocks.get_mut(&block_id) {
|
||||
Some(b) => b,
|
||||
@@ -125,7 +125,7 @@ impl FecDecoder for RaptorQFecDecoder {
|
||||
let block_length = (num_source as u64) * (block.symbol_size as u64);
|
||||
|
||||
let config = ObjectTransmissionInformation::with_defaults(block_length, block.symbol_size);
|
||||
let mut decoder = SourceBlockDecoder::new(block_id, &config, block_length);
|
||||
let mut decoder = SourceBlockDecoder::new((block_id & 0xFF) as u8, &config, block_length);
|
||||
|
||||
let decoded = decoder.decode(block.packets.clone());
|
||||
|
||||
@@ -156,15 +156,15 @@ impl FecDecoder for RaptorQFecDecoder {
|
||||
}
|
||||
}
|
||||
|
||||
fn expire_before(&mut self, block_id: u8) {
|
||||
fn expire_before(&mut self, block_id: u16) {
|
||||
// Remove blocks with IDs "older" than block_id.
|
||||
// With wrapping u8 IDs, we consider a block old if its distance
|
||||
// (in the forward direction) to block_id is > 128.
|
||||
// With wrapping u16 IDs, we consider a block old if its distance
|
||||
// (in the forward direction) to block_id is > 32768.
|
||||
self.blocks.retain(|&id, _| {
|
||||
let distance = block_id.wrapping_sub(id);
|
||||
// If distance is 0 or > 128, the block is current or "ahead" — keep it.
|
||||
// If distance is 1..=128, the block is behind — remove it.
|
||||
distance == 0 || distance > 128
|
||||
// If distance is 0 or > 32768, the block is current or "ahead" — keep it.
|
||||
// If distance is 1..=32768, the block is behind — remove it.
|
||||
distance == 0 || distance > 32768
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -263,9 +263,9 @@ mod tests {
|
||||
let mut decoder = RaptorQFecDecoder::new(FRAMES_PER_BLOCK, SYMBOL_SIZE);
|
||||
|
||||
// Add symbols to blocks 0, 1, 2
|
||||
for block_id in 0..3u8 {
|
||||
for block_id in 0..3u16 {
|
||||
decoder
|
||||
.add_symbol(block_id, 0, false, &[block_id; 50])
|
||||
.add_symbol(block_id, 0, false, &[block_id as u8; 50])
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
|
||||
@@ -15,8 +15,8 @@ const LEN_PREFIX: usize = 2;
|
||||
/// RaptorQ-based FEC encoder that groups audio frames into blocks
|
||||
/// and generates fountain-code repair symbols.
|
||||
pub struct RaptorQFecEncoder {
|
||||
/// Current block ID (wraps at u8).
|
||||
block_id: u8,
|
||||
/// Current block ID (wraps at u16).
|
||||
block_id: u16,
|
||||
/// Maximum source symbols per block.
|
||||
frames_per_block: usize,
|
||||
/// Accumulated source symbols for the current block.
|
||||
@@ -122,7 +122,7 @@ impl FecEncoder for RaptorQFecEncoder {
|
||||
let block_data = self.build_block_data();
|
||||
let config =
|
||||
ObjectTransmissionInformation::with_defaults(block_data.len() as u64, self.symbol_size);
|
||||
let encoder = SourceBlockEncoder::new(self.block_id, &config, &block_data);
|
||||
let encoder = SourceBlockEncoder::new((self.block_id & 0xFF) as u8, &config, &block_data);
|
||||
|
||||
let num_source = self.source_symbols.len() as u32;
|
||||
let num_repair = ((num_source as f32) * effective_ratio).ceil() as u32;
|
||||
@@ -145,7 +145,7 @@ impl FecEncoder for RaptorQFecEncoder {
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn finalize_block(&mut self) -> Result<u8, FecError> {
|
||||
fn finalize_block(&mut self) -> Result<u16, FecError> {
|
||||
let completed = self.block_id;
|
||||
self.block_id = self.block_id.wrapping_add(1);
|
||||
self.source_symbols.clear();
|
||||
@@ -153,7 +153,7 @@ impl FecEncoder for RaptorQFecEncoder {
|
||||
Ok(completed)
|
||||
}
|
||||
|
||||
fn current_block_id(&self) -> u8 {
|
||||
fn current_block_id(&self) -> u16 {
|
||||
self.block_id
|
||||
}
|
||||
|
||||
@@ -181,7 +181,7 @@ fn build_prefixed_block_data(symbols: &[Vec<u8>], symbol_size: u16) -> Vec<u8> {
|
||||
/// Helper: build source `EncodingPacket`s for a given block. Useful for
|
||||
/// the decoder tests and interleaving.
|
||||
pub fn source_packets_for_block(
|
||||
block_id: u8,
|
||||
block_id: u16,
|
||||
symbols: &[Vec<u8>],
|
||||
symbol_size: u16,
|
||||
) -> Vec<EncodingPacket> {
|
||||
@@ -191,21 +191,21 @@ pub fn source_packets_for_block(
|
||||
.map(|i| {
|
||||
let offset = i * ss;
|
||||
let sym_data = data[offset..offset + ss].to_vec();
|
||||
EncodingPacket::new(PayloadId::new(block_id, i as u32), sym_data)
|
||||
EncodingPacket::new(PayloadId::new((block_id & 0xFF) as u8, i as u32), sym_data)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Helper: generate repair packets for the given source symbols.
|
||||
pub fn repair_packets_for_block(
|
||||
block_id: u8,
|
||||
block_id: u16,
|
||||
symbols: &[Vec<u8>],
|
||||
symbol_size: u16,
|
||||
ratio: f32,
|
||||
) -> Vec<EncodingPacket> {
|
||||
let data = build_prefixed_block_data(symbols, symbol_size);
|
||||
let config = ObjectTransmissionInformation::with_defaults(data.len() as u64, symbol_size);
|
||||
let encoder = SourceBlockEncoder::new(block_id, &config, &data);
|
||||
let encoder = SourceBlockEncoder::new((block_id & 0xFF) as u8, &config, &data);
|
||||
let num_source = symbols.len() as u32;
|
||||
let num_repair = ((num_source as f32) * ratio).ceil() as u32;
|
||||
encoder.repair_packets(0, num_repair)
|
||||
@@ -241,15 +241,21 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn block_id_wraps() {
|
||||
fn block_id_wraps_u16() {
|
||||
let mut enc = RaptorQFecEncoder::with_defaults(1);
|
||||
for expected in 0..=255u8 {
|
||||
// Advance 300 blocks and verify no panic + monotonic increment.
|
||||
for expected in 0..300u16 {
|
||||
assert_eq!(enc.current_block_id(), expected);
|
||||
enc.add_source_symbol(&[expected; 10]).unwrap();
|
||||
enc.add_source_symbol(&[0u8; 10]).unwrap();
|
||||
enc.finalize_block().unwrap();
|
||||
}
|
||||
// After 256 blocks, wraps back to 0
|
||||
assert_eq!(enc.current_block_id(), 0);
|
||||
// Explicitly test wrap at u16 boundary.
|
||||
let mut enc2 = RaptorQFecEncoder::with_defaults(1);
|
||||
enc2.block_id = u16::MAX;
|
||||
enc2.add_source_symbol(&[0u8; 10]).unwrap();
|
||||
let id = enc2.finalize_block().unwrap();
|
||||
assert_eq!(id, u16::MAX);
|
||||
assert_eq!(enc2.current_block_id(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
//! rather than one block fatally.
|
||||
|
||||
/// A symbol ready for transmission: (block_id, symbol_index, is_repair, data).
|
||||
pub type Symbol = (u8, u8, bool, Vec<u8>);
|
||||
pub type Symbol = (u16, u16, bool, Vec<u8>);
|
||||
|
||||
/// Temporal interleaver that mixes symbols across multiple FEC blocks.
|
||||
pub struct Interleaver {
|
||||
@@ -64,13 +64,13 @@ mod tests {
|
||||
let interleaver = Interleaver::with_default_depth();
|
||||
|
||||
let block_a: Vec<Symbol> = (0..3)
|
||||
.map(|i| (0u8, i as u8, false, vec![0xA0 + i as u8]))
|
||||
.map(|i| (0u16, i as u16, false, vec![0xA0 + i as u8]))
|
||||
.collect();
|
||||
let block_b: Vec<Symbol> = (0..3)
|
||||
.map(|i| (1u8, i as u8, false, vec![0xB0 + i as u8]))
|
||||
.map(|i| (1u16, i as u16, false, vec![0xB0 + i as u8]))
|
||||
.collect();
|
||||
let block_c: Vec<Symbol> = (0..3)
|
||||
.map(|i| (2u8, i as u8, false, vec![0xC0 + i as u8]))
|
||||
.map(|i| (2u16, i as u16, false, vec![0xC0 + i as u8]))
|
||||
.collect();
|
||||
|
||||
let result = interleaver.interleave(&[block_a, block_b, block_c]);
|
||||
@@ -96,10 +96,10 @@ mod tests {
|
||||
let interleaver = Interleaver::new(2);
|
||||
|
||||
let block_a: Vec<Symbol> = (0..3)
|
||||
.map(|i| (0u8, i as u8, false, vec![0xA0 + i as u8]))
|
||||
.map(|i| (0u16, i as u16, false, vec![0xA0 + i as u8]))
|
||||
.collect();
|
||||
let block_b: Vec<Symbol> = (0..1)
|
||||
.map(|i| (1u8, i as u8, false, vec![0xB0 + i as u8]))
|
||||
.map(|i| (1u16, i as u16, false, vec![0xB0 + i as u8]))
|
||||
.collect();
|
||||
|
||||
let result = interleaver.interleave(&[block_a, block_b]);
|
||||
@@ -128,7 +128,7 @@ mod tests {
|
||||
let blocks: Vec<Vec<Symbol>> = (0..3)
|
||||
.map(|b| {
|
||||
(0..6)
|
||||
.map(|i| (b as u8, i as u8, false, vec![b as u8; 10]))
|
||||
.map(|i| (b as u16, i as u16, false, vec![b as u8; 10]))
|
||||
.collect()
|
||||
})
|
||||
.collect();
|
||||
|
||||
@@ -574,6 +574,10 @@ pub enum SignalMessage {
|
||||
/// Protocol versions this client supports (default [2]).
|
||||
#[serde(default = "default_supported_versions")]
|
||||
supported_versions: Vec<u8>,
|
||||
/// Video codecs supported by the caller, in preference order.
|
||||
/// Absent on old clients (treated as video-incapable).
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
video_codecs: Vec<crate::CodecId>,
|
||||
},
|
||||
|
||||
/// Call acceptance (analogous to Warzone's WireMessage::CallAnswer).
|
||||
@@ -588,6 +592,10 @@ pub enum SignalMessage {
|
||||
signature: Vec<u8>,
|
||||
/// Chosen quality profile.
|
||||
chosen_profile: crate::QualityProfile,
|
||||
/// Video codec chosen by the callee (None = video declined or peer incapable).
|
||||
/// Absent on old clients (treated as no video).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
video_codec: Option<crate::CodecId>,
|
||||
},
|
||||
|
||||
/// ICE candidate for NAT traversal.
|
||||
|
||||
@@ -85,10 +85,10 @@ pub trait FecEncoder: Send + Sync {
|
||||
|
||||
/// Finalize the current block and start a new one.
|
||||
/// Returns the block ID of the finalized block.
|
||||
fn finalize_block(&mut self) -> Result<u8, FecError>;
|
||||
fn finalize_block(&mut self) -> Result<u16, FecError>;
|
||||
|
||||
/// Current block ID being built.
|
||||
fn current_block_id(&self) -> u8;
|
||||
fn current_block_id(&self) -> u16;
|
||||
|
||||
/// Number of source symbols in the current block.
|
||||
fn current_block_size(&self) -> usize;
|
||||
@@ -99,7 +99,7 @@ pub trait FecDecoder: Send + Sync {
|
||||
/// Feed a received symbol (source or repair) into the decoder.
|
||||
fn add_symbol(
|
||||
&mut self,
|
||||
block_id: u8,
|
||||
block_id: u16,
|
||||
symbol_index: u16,
|
||||
is_repair: bool,
|
||||
data: &[u8],
|
||||
@@ -109,10 +109,10 @@ pub trait FecDecoder: Send + Sync {
|
||||
///
|
||||
/// Returns `None` if not yet decodable (insufficient symbols).
|
||||
/// Returns `Some(Vec<source_frames>)` on success.
|
||||
fn try_decode(&mut self, block_id: u8) -> Result<Option<Vec<Vec<u8>>>, FecError>;
|
||||
fn try_decode(&mut self, block_id: u16) -> Result<Option<Vec<Vec<u8>>>, FecError>;
|
||||
|
||||
/// Drop state for blocks older than `block_id`.
|
||||
fn expire_before(&mut self, block_id: u8);
|
||||
fn expire_before(&mut self, block_id: u16);
|
||||
}
|
||||
|
||||
// ─── Crypto Traits ───────────────────────────────────────────────────────────
|
||||
|
||||
@@ -42,6 +42,7 @@ pub async fn accept_handshake(
|
||||
supported_profiles,
|
||||
caller_alias,
|
||||
protocol_version,
|
||||
caller_video_codecs,
|
||||
) = match offer {
|
||||
SignalMessage::CallOffer {
|
||||
identity_pub,
|
||||
@@ -51,6 +52,7 @@ pub async fn accept_handshake(
|
||||
alias,
|
||||
protocol_version,
|
||||
supported_versions: _,
|
||||
video_codecs,
|
||||
..
|
||||
} => (
|
||||
identity_pub,
|
||||
@@ -59,6 +61,7 @@ pub async fn accept_handshake(
|
||||
supported_profiles,
|
||||
alias,
|
||||
protocol_version,
|
||||
video_codecs,
|
||||
),
|
||||
other => {
|
||||
return Err(anyhow::anyhow!(
|
||||
@@ -108,6 +111,9 @@ pub async fn accept_handshake(
|
||||
// Choose the best supported profile (prefer GOOD > DEGRADED > CATASTROPHIC)
|
||||
let chosen_profile = choose_profile(&supported_profiles);
|
||||
|
||||
// Pick the first video codec the caller supports (relay forwards all video).
|
||||
let video_codec = caller_video_codecs.into_iter().next();
|
||||
|
||||
// 6. Send CallAnswer
|
||||
let answer = SignalMessage::CallAnswer {
|
||||
version: default_signal_version(),
|
||||
@@ -115,6 +121,7 @@ pub async fn accept_handshake(
|
||||
ephemeral_pub,
|
||||
signature,
|
||||
chosen_profile,
|
||||
video_codec,
|
||||
};
|
||||
transport.send_signal(&answer).await?;
|
||||
|
||||
@@ -147,6 +154,7 @@ fn choose_profile(_supported: &[QualityProfile]) -> QualityProfile {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use wzp_proto::CodecId;
|
||||
|
||||
#[test]
|
||||
fn choose_profile_picks_highest_bitrate() {
|
||||
@@ -164,4 +172,35 @@ mod tests {
|
||||
let chosen = choose_profile(&[]);
|
||||
assert_eq!(chosen, QualityProfile::GOOD);
|
||||
}
|
||||
|
||||
// ── Video codec negotiation ───────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn video_codec_picks_first_offered() {
|
||||
let codecs = vec![CodecId::H264Baseline];
|
||||
let chosen: Option<CodecId> = codecs.into_iter().next();
|
||||
assert_eq!(chosen, Some(CodecId::H264Baseline));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_codec_none_when_no_codecs_offered() {
|
||||
let codecs: Vec<CodecId> = vec![];
|
||||
let chosen: Option<CodecId> = codecs.into_iter().next();
|
||||
assert_eq!(chosen, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_codec_single_codec_is_selected() {
|
||||
let codecs = vec![CodecId::H265Main];
|
||||
let chosen: Option<CodecId> = codecs.into_iter().next();
|
||||
assert_eq!(chosen, Some(CodecId::H265Main));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn video_codec_order_is_preserved() {
|
||||
// The relay must pick the FIRST codec as-offered, not sort or re-rank.
|
||||
let codecs = vec![CodecId::H264Baseline, CodecId::Av1Main];
|
||||
let chosen: Option<CodecId> = codecs.into_iter().next();
|
||||
assert_eq!(chosen, Some(CodecId::H264Baseline));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2028,7 +2028,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
(None, None)
|
||||
};
|
||||
|
||||
let media_handle = tokio::spawn(room::run_participant(
|
||||
let mut media_handle = tokio::spawn(room::run_participant(
|
||||
room_mgr.clone(),
|
||||
room_name.clone(),
|
||||
participant_id,
|
||||
@@ -2041,15 +2041,38 @@ async fn main() -> anyhow::Result<()> {
|
||||
federation_room_hash,
|
||||
authenticated_fp.is_some(),
|
||||
));
|
||||
let signal_handle = tokio::spawn(room::run_participant_signals(
|
||||
let mut signal_handle = tokio::spawn(room::run_participant_signals(
|
||||
room_mgr.clone(),
|
||||
room_name.clone(),
|
||||
participant_id,
|
||||
transport.clone(),
|
||||
));
|
||||
tokio::select! {
|
||||
_ = media_handle => {},
|
||||
_ = signal_handle => {},
|
||||
_ = &mut media_handle => {
|
||||
signal_handle.abort();
|
||||
let _ = signal_handle.await;
|
||||
},
|
||||
_ = &mut signal_handle => {
|
||||
close_transport(&*transport, "signal-loop-ended").await;
|
||||
match tokio::time::timeout(Duration::from_secs(2), &mut media_handle).await {
|
||||
Ok(_) => {}
|
||||
Err(_) => {
|
||||
warn!(
|
||||
%addr,
|
||||
room = %room_name,
|
||||
participant = participant_id,
|
||||
"media loop did not exit after signal close; forcing room leave"
|
||||
);
|
||||
media_handle.abort();
|
||||
let _ = media_handle.await;
|
||||
if let Some((update, senders)) =
|
||||
room_mgr.leave(&room_name, participant_id)
|
||||
{
|
||||
room::broadcast_signal(&senders, &update).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
// Participant disconnected — clean up presence + per-session metrics
|
||||
|
||||
@@ -110,7 +110,7 @@ impl RelayPipeline {
|
||||
// Feed packet into FEC decoder
|
||||
let header = &packet.header;
|
||||
let _ = self.fec_decoder.add_symbol(
|
||||
(header.fec_block & 0xFF) as u8,
|
||||
header.fec_block,
|
||||
header.fec_block >> 8,
|
||||
header.is_repair(),
|
||||
&packet.payload,
|
||||
@@ -118,7 +118,7 @@ impl RelayPipeline {
|
||||
|
||||
// Try to decode the FEC block
|
||||
let mut output = Vec::new();
|
||||
if let Ok(Some(frames)) = self.fec_decoder.try_decode((header.fec_block & 0xFF) as u8) {
|
||||
if let Ok(Some(frames)) = self.fec_decoder.try_decode(header.fec_block) {
|
||||
debug!(
|
||||
block = header.fec_block,
|
||||
frames = frames.len(),
|
||||
|
||||
@@ -51,9 +51,13 @@ impl DebugTap {
|
||||
dir = dir,
|
||||
addr = %addr,
|
||||
seq = h.seq,
|
||||
media = ?h.media_type,
|
||||
codec = ?h.codec_id,
|
||||
stream_id = h.stream_id,
|
||||
ts = h.timestamp,
|
||||
fec_block = h.fec_block,
|
||||
keyframe = h.is_keyframe(),
|
||||
frame_end = h.is_frame_end(),
|
||||
repair = h.is_repair(),
|
||||
len = pkt.payload.len(),
|
||||
fan_out,
|
||||
@@ -61,6 +65,35 @@ impl DebugTap {
|
||||
);
|
||||
}
|
||||
|
||||
pub fn log_video_route(
|
||||
&self,
|
||||
room: &str,
|
||||
addr: &std::net::SocketAddr,
|
||||
peer_id: ParticipantId,
|
||||
pkt: &wzp_proto::MediaPacket,
|
||||
selected_layer: u8,
|
||||
forwarded: bool,
|
||||
reason: &str,
|
||||
) {
|
||||
let h = &pkt.header;
|
||||
info!(
|
||||
target: "debug_tap",
|
||||
room = %room,
|
||||
addr = %addr,
|
||||
peer_id,
|
||||
seq = h.seq,
|
||||
stream_id = h.stream_id,
|
||||
selected_layer,
|
||||
codec = ?h.codec_id,
|
||||
keyframe = h.is_keyframe(),
|
||||
frame_end = h.is_frame_end(),
|
||||
len = pkt.payload.len(),
|
||||
forwarded,
|
||||
reason,
|
||||
"TAP VIDEO ROUTE"
|
||||
);
|
||||
}
|
||||
|
||||
pub fn log_signal(&self, room: &str, signal: &wzp_proto::SignalMessage) {
|
||||
match signal {
|
||||
wzp_proto::SignalMessage::RoomUpdate {
|
||||
@@ -295,6 +328,23 @@ impl ReceiverState {
|
||||
}
|
||||
}
|
||||
|
||||
fn video_route_reason(pkt: &wzp_proto::MediaPacket, selected_layer: u8) -> Option<&'static str> {
|
||||
if pkt.header.stream_id == selected_layer {
|
||||
return Some("selected_layer");
|
||||
}
|
||||
|
||||
// Compatibility for the pre-simulcast single-layer H.264 room-video path.
|
||||
// Older clients used video stream 1 while current clients use stream 0 so
|
||||
// they pass through relay defaults. Forward both H.264 single-layer ids.
|
||||
if pkt.header.codec_id == wzp_proto::CodecId::H264Baseline
|
||||
&& (pkt.header.stream_id == 0 || pkt.header.stream_id == 1)
|
||||
{
|
||||
return Some("h264_single_layer_compat");
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Unique participant ID within a room.
|
||||
pub type ParticipantId = u64;
|
||||
|
||||
@@ -304,6 +354,24 @@ fn next_id() -> ParticipantId {
|
||||
NEXT_PARTICIPANT_ID.fetch_add(1, Ordering::Relaxed)
|
||||
}
|
||||
|
||||
fn outbound_video_stream_id(participant_id: ParticipantId) -> u8 {
|
||||
// Reserve stream 0 for the sender's local/simulcast layer id. Forwarded
|
||||
// room video needs a sender-distinct stream id so receivers and analyzers
|
||||
// do not merge independent H264 access-unit sequences.
|
||||
((participant_id.saturating_sub(1) % 250) + 1) as u8
|
||||
}
|
||||
|
||||
fn with_outbound_video_stream_id(
|
||||
pkt: &wzp_proto::MediaPacket,
|
||||
participant_id: ParticipantId,
|
||||
) -> wzp_proto::MediaPacket {
|
||||
let mut out = pkt.clone();
|
||||
if out.header.media_type == wzp_proto::MediaType::Video {
|
||||
out.header.stream_id = outbound_video_stream_id(participant_id);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Events emitted by RoomManager for federation to observe.
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum RoomEvent {
|
||||
@@ -438,6 +506,25 @@ impl Room {
|
||||
);
|
||||
}
|
||||
|
||||
fn remove_by_fingerprint(&mut self, fingerprint: &str) -> Vec<ParticipantId> {
|
||||
let mut removed = Vec::new();
|
||||
self.participants.retain(|p| {
|
||||
let matches = p.fingerprint.as_deref() == Some(fingerprint);
|
||||
if matches {
|
||||
removed.push(p.id);
|
||||
}
|
||||
!matches
|
||||
});
|
||||
for id in &removed {
|
||||
self.qualities.remove(id);
|
||||
}
|
||||
removed
|
||||
}
|
||||
|
||||
fn contains(&self, id: ParticipantId) -> bool {
|
||||
self.participants.iter().any(|p| p.id == id)
|
||||
}
|
||||
|
||||
fn others(&self, exclude_id: ParticipantId) -> Vec<ParticipantSender> {
|
||||
self.participants
|
||||
.iter()
|
||||
@@ -632,6 +719,18 @@ impl RoomManager {
|
||||
.entry(room_name.to_string())
|
||||
.or_insert_with(|| Arc::new(RwLock::new(Room::new())));
|
||||
let mut room = arc.write().unwrap();
|
||||
if let Some(fp) = fingerprint {
|
||||
let removed = room.remove_by_fingerprint(fp);
|
||||
for old_id in removed {
|
||||
warn!(
|
||||
room = room_name,
|
||||
participant = old_id,
|
||||
fingerprint = fp,
|
||||
"replacing existing participant with same fingerprint"
|
||||
);
|
||||
self.clear_participant_state(room_name, old_id);
|
||||
}
|
||||
}
|
||||
let id = room.add(
|
||||
addr,
|
||||
sender,
|
||||
@@ -708,6 +807,7 @@ impl RoomManager {
|
||||
let mut room = arc.write().unwrap();
|
||||
room.qualities.remove(&participant_id);
|
||||
room.remove(participant_id);
|
||||
self.clear_participant_state(room_name, participant_id);
|
||||
if room.is_empty() {
|
||||
drop(room); // release room lock
|
||||
drop(arc); // release DashMap guard
|
||||
@@ -799,7 +899,14 @@ impl RoomManager {
|
||||
self.keyframe_cache
|
||||
.iter()
|
||||
.filter(|e| e.key().0 == room_name)
|
||||
.map(|e| e.value().packets.clone())
|
||||
.map(|e| {
|
||||
let sender_id = e.key().1;
|
||||
e.value()
|
||||
.packets
|
||||
.iter()
|
||||
.map(|pkt| with_outbound_video_stream_id(pkt, sender_id))
|
||||
.collect()
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
@@ -809,6 +916,27 @@ impl RoomManager {
|
||||
self.keyframe_buffer.retain(|k, _| k.0 != room_name);
|
||||
self.pli_state.retain(|k, _| k.0 != room_name);
|
||||
self.stream_owner.retain(|k, _| k.0 != room_name);
|
||||
self.receiver_states.retain(|k, _| k.0 != room_name);
|
||||
}
|
||||
|
||||
fn clear_participant_state(&self, room_name: &str, participant_id: ParticipantId) {
|
||||
self.keyframe_cache
|
||||
.retain(|k, _| !(k.0 == room_name && k.1 == participant_id));
|
||||
self.keyframe_buffer
|
||||
.retain(|k, _| !(k.0 == room_name && k.1 == participant_id));
|
||||
self.pli_state
|
||||
.retain(|k, _| !(k.0 == room_name && k.1 == participant_id));
|
||||
self.stream_owner
|
||||
.retain(|k, owner| !(k.0 == room_name && *owner == participant_id));
|
||||
self.receiver_states
|
||||
.retain(|k, _| !(k.0 == room_name && k.1 == participant_id));
|
||||
}
|
||||
|
||||
pub fn contains_participant(&self, room_name: &str, participant_id: ParticipantId) -> bool {
|
||||
self.rooms
|
||||
.get(room_name)
|
||||
.map(|arc| arc.read().unwrap().contains(participant_id))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
/// PLI suppression window (PRD-video-v1 T4.7).
|
||||
@@ -1142,6 +1270,7 @@ pub async fn run_participant(
|
||||
transport,
|
||||
metrics,
|
||||
session_id,
|
||||
debug_tap,
|
||||
is_authenticated,
|
||||
)
|
||||
.await;
|
||||
@@ -1225,6 +1354,16 @@ async fn run_participant_plain(
|
||||
}
|
||||
};
|
||||
|
||||
if !room_mgr.contains_participant(&room_name, participant_id) {
|
||||
info!(
|
||||
room = %room_name,
|
||||
participant = participant_id,
|
||||
forwarded = packets_forwarded,
|
||||
"stale participant loop stopped"
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
// Cache keyframe packets for fast join-to-first-frame replay.
|
||||
room_mgr.update_keyframe_cache(&room_name, participant_id, &pkt);
|
||||
// Register this participant as the owner of this stream for PLI routing.
|
||||
@@ -1232,6 +1371,12 @@ async fn run_participant_plain(
|
||||
room_mgr
|
||||
.stream_owner
|
||||
.insert((room_name.clone(), pkt.header.stream_id), participant_id);
|
||||
if pkt.header.media_type == wzp_proto::MediaType::Video {
|
||||
room_mgr.stream_owner.insert(
|
||||
(room_name.clone(), outbound_video_stream_id(participant_id)),
|
||||
participant_id,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let recv_gap_ms = last_recv_instant.elapsed().as_millis() as u64;
|
||||
@@ -1275,9 +1420,8 @@ async fn run_participant_plain(
|
||||
room = %room_name,
|
||||
participant = participant_id,
|
||||
seq = pkt.header.seq,
|
||||
"VideoScorer: Abusive verdict — dropping packet"
|
||||
"VideoScorer: Abusive verdict — observe-only"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1324,33 +1468,56 @@ async fn run_participant_plain(
|
||||
broadcast_signal(&all_senders, &directive).await;
|
||||
}
|
||||
|
||||
// Debug tap: log packet metadata + record stats
|
||||
if let Some(ref tap) = debug_tap {
|
||||
if tap.matches(&room_name) {
|
||||
tap.log_packet(&room_name, "in", &addr, &pkt, others.len());
|
||||
}
|
||||
}
|
||||
if let Some(ref mut ts) = tap_stats {
|
||||
ts.record_in(&pkt, others.len());
|
||||
}
|
||||
|
||||
// Forward to all others, applying simulcast layer selection for video.
|
||||
let fwd_start = std::time::Instant::now();
|
||||
let pkt_bytes = pkt.payload.len() as u64;
|
||||
let is_video = pkt.header.media_type == wzp_proto::MediaType::Video;
|
||||
let mut actual_fan_out = 0usize;
|
||||
for (other_id, other) in &others {
|
||||
// Simulcast layer selection (T5.6): video packets are filtered
|
||||
// by the receiver's selected layer. Audio and non-simulcast
|
||||
// traffic pass through unchanged.
|
||||
if is_video {
|
||||
let selected = room_mgr.selected_layer(&room_name, *other_id);
|
||||
if pkt.header.stream_id != selected {
|
||||
let route_reason = video_route_reason(&pkt, selected);
|
||||
if route_reason.is_none() {
|
||||
if let Some(ref tap) = debug_tap {
|
||||
if tap.matches(&room_name) {
|
||||
tap.log_video_route(
|
||||
&room_name,
|
||||
&addr,
|
||||
*other_id,
|
||||
&pkt,
|
||||
selected,
|
||||
false,
|
||||
"simulcast_layer_mismatch",
|
||||
);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if let Some(ref tap) = debug_tap {
|
||||
if tap.matches(&room_name) {
|
||||
tap.log_video_route(
|
||||
&room_name,
|
||||
&addr,
|
||||
*other_id,
|
||||
&pkt,
|
||||
selected,
|
||||
true,
|
||||
route_reason.unwrap_or("selected_layer"),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
match other {
|
||||
ParticipantSender::Quic(t) => {
|
||||
if let Err(e) = t.send_media(&pkt).await {
|
||||
let outbound_pkt = if is_video {
|
||||
with_outbound_video_stream_id(&pkt, participant_id)
|
||||
} else {
|
||||
pkt.clone()
|
||||
};
|
||||
if let Err(e) = t.send_media(&outbound_pkt).await {
|
||||
send_errors += 1;
|
||||
if send_errors <= 5 || send_errors % 100 == 0 {
|
||||
warn!(
|
||||
@@ -1361,14 +1528,28 @@ async fn run_participant_plain(
|
||||
"send_media error: {e}"
|
||||
);
|
||||
}
|
||||
} else {
|
||||
actual_fan_out += 1;
|
||||
}
|
||||
}
|
||||
ParticipantSender::WebSocket(_) => {
|
||||
let _ = other.send_raw(&pkt.payload).await;
|
||||
actual_fan_out += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Debug tap: log packet metadata + record stats after forwarding so
|
||||
// fan_out reflects actual sends after video layer filtering.
|
||||
if let Some(ref tap) = debug_tap {
|
||||
if tap.matches(&room_name) {
|
||||
tap.log_packet(&room_name, "in", &addr, &pkt, actual_fan_out);
|
||||
}
|
||||
}
|
||||
if let Some(ref mut ts) = tap_stats {
|
||||
ts.record_in(&pkt, actual_fan_out);
|
||||
}
|
||||
|
||||
// Federation: forward to active peer relays via channel
|
||||
if let Some(ref fed_tx) = federation_tx {
|
||||
let data = pkt.to_bytes();
|
||||
@@ -1394,7 +1575,7 @@ async fn run_participant_plain(
|
||||
);
|
||||
}
|
||||
|
||||
let fan_out = others.len() as u64;
|
||||
let fan_out = actual_fan_out as u64;
|
||||
metrics.packets_forwarded.inc_by(fan_out);
|
||||
metrics.bytes_forwarded.inc_by(pkt_bytes * fan_out);
|
||||
packets_forwarded += 1;
|
||||
@@ -1457,6 +1638,7 @@ async fn run_participant_trunked(
|
||||
transport: Arc<wzp_transport::QuinnTransport>,
|
||||
metrics: Arc<RelayMetrics>,
|
||||
session_id: String,
|
||||
debug_tap: Option<DebugTap>,
|
||||
_is_authenticated: bool,
|
||||
) {
|
||||
use std::collections::HashMap;
|
||||
@@ -1472,6 +1654,11 @@ async fn run_participant_trunked(
|
||||
ConformanceMeter::with_token_bucket(crate::conformance::TokenBucket::for_audio_session());
|
||||
let mut video_scorer_trunked = VideoScorer::new();
|
||||
let mut last_bwe_kbps_trunked: Option<u32> = None;
|
||||
let mut tap_stats = if debug_tap.as_ref().map_or(false, |t| t.matches(&room_name)) {
|
||||
Some(TapStats::new())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
info!(
|
||||
room = %room_name,
|
||||
@@ -1510,6 +1697,16 @@ async fn run_participant_trunked(
|
||||
}
|
||||
};
|
||||
|
||||
if !room_mgr.contains_participant(&room_name, participant_id) {
|
||||
info!(
|
||||
room = %room_name,
|
||||
participant = participant_id,
|
||||
forwarded = packets_forwarded,
|
||||
"stale participant loop stopped (trunked)"
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
// Cache keyframe packets for fast join-to-first-frame replay.
|
||||
room_mgr.update_keyframe_cache(&room_name, participant_id, &pkt);
|
||||
// Register this participant as the owner of this stream for PLI routing.
|
||||
@@ -1518,6 +1715,15 @@ async fn run_participant_trunked(
|
||||
(room_name.clone(), pkt.header.stream_id),
|
||||
participant_id,
|
||||
);
|
||||
if pkt.header.media_type == wzp_proto::MediaType::Video {
|
||||
room_mgr.stream_owner.insert(
|
||||
(
|
||||
room_name.clone(),
|
||||
outbound_video_stream_id(participant_id),
|
||||
),
|
||||
participant_id,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let recv_gap_ms = last_recv_instant.elapsed().as_millis() as u64;
|
||||
@@ -1560,9 +1766,8 @@ async fn run_participant_trunked(
|
||||
room = %room_name,
|
||||
participant = participant_id,
|
||||
seq = pkt.header.seq,
|
||||
"VideoScorer: Abusive verdict — dropping packet (trunked)"
|
||||
"VideoScorer: Abusive verdict — observe-only (trunked)"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1605,12 +1810,40 @@ async fn run_participant_trunked(
|
||||
let fwd_start = std::time::Instant::now();
|
||||
let pkt_bytes = pkt.payload.len() as u64;
|
||||
let is_video = pkt.header.media_type == wzp_proto::MediaType::Video;
|
||||
let mut actual_fan_out = 0usize;
|
||||
for (other_id, other) in &others {
|
||||
if is_video {
|
||||
let selected = room_mgr.selected_layer(&room_name, *other_id);
|
||||
if pkt.header.stream_id != selected {
|
||||
let route_reason = video_route_reason(&pkt, selected);
|
||||
if route_reason.is_none() {
|
||||
if let Some(ref tap) = debug_tap {
|
||||
if tap.matches(&room_name) {
|
||||
tap.log_video_route(
|
||||
&room_name,
|
||||
&addr,
|
||||
*other_id,
|
||||
&pkt,
|
||||
selected,
|
||||
false,
|
||||
"simulcast_layer_mismatch",
|
||||
);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if let Some(ref tap) = debug_tap {
|
||||
if tap.matches(&room_name) {
|
||||
tap.log_video_route(
|
||||
&room_name,
|
||||
&addr,
|
||||
*other_id,
|
||||
&pkt,
|
||||
selected,
|
||||
true,
|
||||
route_reason.unwrap_or("selected_layer"),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
match other {
|
||||
ParticipantSender::Quic(t) => {
|
||||
@@ -1618,7 +1851,12 @@ async fn run_participant_trunked(
|
||||
let fwd = forwarders
|
||||
.entry(peer_addr)
|
||||
.or_insert_with(|| TrunkedForwarder::new(t.clone(), sid_bytes));
|
||||
if let Err(e) = fwd.send(&pkt).await {
|
||||
let outbound_pkt = if is_video {
|
||||
with_outbound_video_stream_id(&pkt, participant_id)
|
||||
} else {
|
||||
pkt.clone()
|
||||
};
|
||||
if let Err(e) = fwd.send(&outbound_pkt).await {
|
||||
send_errors += 1;
|
||||
if send_errors <= 5 || send_errors % 100 == 0 {
|
||||
warn!(
|
||||
@@ -1629,13 +1867,24 @@ async fn run_participant_trunked(
|
||||
"trunked send error: {e}"
|
||||
);
|
||||
}
|
||||
} else {
|
||||
actual_fan_out += 1;
|
||||
}
|
||||
}
|
||||
ParticipantSender::WebSocket(_) => {
|
||||
let _ = other.send_raw(&pkt.payload).await;
|
||||
actual_fan_out += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(ref tap) = debug_tap {
|
||||
if tap.matches(&room_name) {
|
||||
tap.log_packet(&room_name, "in", &addr, &pkt, actual_fan_out);
|
||||
}
|
||||
}
|
||||
if let Some(ref mut ts) = tap_stats {
|
||||
ts.record_in(&pkt, actual_fan_out);
|
||||
}
|
||||
let fwd_ms = fwd_start.elapsed().as_millis() as u64;
|
||||
if fwd_ms > max_forward_ms {
|
||||
max_forward_ms = fwd_ms;
|
||||
@@ -1645,12 +1894,12 @@ async fn run_participant_trunked(
|
||||
room = %room_name,
|
||||
participant = participant_id,
|
||||
fwd_ms,
|
||||
fan_out = others.len(),
|
||||
fan_out = actual_fan_out,
|
||||
"slow forward (trunked)"
|
||||
);
|
||||
}
|
||||
|
||||
let fan_out = others.len() as u64;
|
||||
let fan_out = actual_fan_out as u64;
|
||||
metrics.packets_forwarded.inc_by(fan_out);
|
||||
metrics.bytes_forwarded.inc_by(pkt_bytes * fan_out);
|
||||
packets_forwarded += 1;
|
||||
@@ -1669,6 +1918,10 @@ async fn run_participant_trunked(
|
||||
send_errors,
|
||||
"participant stats (trunked)"
|
||||
);
|
||||
if let (Some(tap), Some(ts)) = (&debug_tap, &mut tap_stats) {
|
||||
tap.log_stats(&room_name, ts);
|
||||
ts.reset_period();
|
||||
}
|
||||
max_recv_gap_ms = 0;
|
||||
max_forward_ms = 0;
|
||||
last_log_instant = std::time::Instant::now();
|
||||
@@ -1727,6 +1980,72 @@ mod tests {
|
||||
assert!(mgr.list().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn join_replaces_existing_fingerprint_in_same_room() {
|
||||
let mgr = RoomManager::new();
|
||||
let addr: std::net::SocketAddr = "127.0.0.1:10000".parse().unwrap();
|
||||
let (tx1, _rx1) = tokio::sync::mpsc::channel(1);
|
||||
let (tx2, _rx2) = tokio::sync::mpsc::channel(1);
|
||||
|
||||
let (first_id, _, _, _) = mgr
|
||||
.join(
|
||||
"room",
|
||||
addr,
|
||||
ParticipantSender::WebSocket(tx1),
|
||||
Some("fp-a"),
|
||||
Some("old"),
|
||||
)
|
||||
.unwrap();
|
||||
let (second_id, update, _, _) = mgr
|
||||
.join(
|
||||
"room",
|
||||
addr,
|
||||
ParticipantSender::WebSocket(tx2),
|
||||
Some("fp-a"),
|
||||
Some("new"),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_ne!(first_id, second_id);
|
||||
assert!(!mgr.contains_participant("room", first_id));
|
||||
assert!(mgr.contains_participant("room", second_id));
|
||||
assert_eq!(mgr.room_size("room"), 1);
|
||||
if let wzp_proto::SignalMessage::RoomUpdate {
|
||||
count,
|
||||
participants,
|
||||
..
|
||||
} = update
|
||||
{
|
||||
assert_eq!(count, 1);
|
||||
assert_eq!(participants[0].fingerprint, "fp-a");
|
||||
assert_eq!(participants[0].alias.as_deref(), Some("new"));
|
||||
} else {
|
||||
panic!("expected RoomUpdate");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn outbound_video_stream_ids_are_sender_distinct_and_nonzero() {
|
||||
assert_eq!(outbound_video_stream_id(1), 1);
|
||||
assert_eq!(outbound_video_stream_id(2), 2);
|
||||
assert_eq!(outbound_video_stream_id(250), 250);
|
||||
assert_eq!(outbound_video_stream_id(251), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rewrite_only_changes_video_stream_id() {
|
||||
let mut video = make_test_packet(b"video");
|
||||
video.header.media_type = wzp_proto::MediaType::Video;
|
||||
video.header.stream_id = 0;
|
||||
let rewritten = with_outbound_video_stream_id(&video, 42);
|
||||
assert_eq!(rewritten.header.stream_id, 42);
|
||||
assert_eq!(video.header.stream_id, 0);
|
||||
|
||||
let audio = make_test_packet(b"audio");
|
||||
let rewritten_audio = with_outbound_video_stream_id(&audio, 42);
|
||||
assert_eq!(rewritten_audio.header.stream_id, audio.header.stream_id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn acl_open_mode_allows_all() {
|
||||
let mgr = RoomManager::new();
|
||||
|
||||
@@ -87,7 +87,7 @@ async fn handshake_succeeds() {
|
||||
let callee_handle =
|
||||
tokio::spawn(async move { accept_handshake(server_t.as_ref(), &callee_seed).await });
|
||||
|
||||
let caller_session = perform_handshake(client_transport.as_ref(), &caller_seed, None)
|
||||
let caller_hs = perform_handshake(client_transport.as_ref(), &caller_seed, None)
|
||||
.await
|
||||
.expect("perform_handshake should succeed");
|
||||
|
||||
@@ -102,7 +102,7 @@ async fn handshake_succeeds() {
|
||||
let plaintext = b"hello warzone";
|
||||
|
||||
let mut ciphertext = Vec::new();
|
||||
let mut caller_session = caller_session;
|
||||
let mut caller_session = caller_hs.session;
|
||||
let mut callee_session = callee_session;
|
||||
|
||||
caller_session
|
||||
@@ -156,6 +156,7 @@ async fn handshake_rejects_v1_protocol_version() {
|
||||
alias: None,
|
||||
protocol_version: 1,
|
||||
supported_versions: vec![1, 2],
|
||||
video_codecs: vec![],
|
||||
};
|
||||
|
||||
client_transport
|
||||
@@ -221,7 +222,7 @@ async fn handshake_verifies_identity() {
|
||||
let callee_handle =
|
||||
tokio::spawn(async move { accept_handshake(server_t.as_ref(), &callee_seed).await });
|
||||
|
||||
let caller_session = perform_handshake(client_transport.as_ref(), &caller_seed, None)
|
||||
let caller_hs = perform_handshake(client_transport.as_ref(), &caller_seed, None)
|
||||
.await
|
||||
.expect("handshake must succeed even with different identities");
|
||||
|
||||
@@ -235,7 +236,7 @@ async fn handshake_verifies_identity() {
|
||||
let plaintext = b"identity verified";
|
||||
|
||||
let mut ct = Vec::new();
|
||||
let mut caller_session = caller_session;
|
||||
let mut caller_session = caller_hs.session;
|
||||
let mut callee_session = callee_session;
|
||||
|
||||
caller_session
|
||||
@@ -301,7 +302,7 @@ async fn auth_then_handshake() {
|
||||
.await
|
||||
.expect("send AuthToken");
|
||||
|
||||
let caller_session = perform_handshake(client_transport.as_ref(), &caller_seed, None)
|
||||
let caller_hs = perform_handshake(client_transport.as_ref(), &caller_seed, None)
|
||||
.await
|
||||
.expect("perform_handshake after auth");
|
||||
|
||||
@@ -315,7 +316,7 @@ async fn auth_then_handshake() {
|
||||
let plaintext = b"post-auth payload";
|
||||
|
||||
let mut ct = Vec::new();
|
||||
let mut caller_session = caller_session;
|
||||
let mut caller_session = caller_hs.session;
|
||||
let mut callee_session = callee_session;
|
||||
|
||||
caller_session
|
||||
@@ -373,6 +374,7 @@ async fn handshake_rejects_bad_signature() {
|
||||
alias: None,
|
||||
protocol_version: 2,
|
||||
supported_versions: vec![2],
|
||||
video_codecs: vec![],
|
||||
};
|
||||
|
||||
client_transport
|
||||
|
||||
@@ -19,7 +19,7 @@ shiguredo_svt_av1 = "2026.1.0"
|
||||
shiguredo_video_toolbox = "2026.1"
|
||||
|
||||
[target.'cfg(target_os = "android")'.dependencies]
|
||||
ndk = { version = "0.9", features = ["media"] }
|
||||
ndk = { version = "0.9", features = ["api-level-28", "media"] }
|
||||
|
||||
[dev-dependencies]
|
||||
rand = "0.8"
|
||||
|
||||
@@ -12,4 +12,9 @@ pub trait VideoDecoder: Send {
|
||||
/// Returns `Ok(Some(frame))` when a frame is ready, `Ok(None)` if more
|
||||
/// data is needed (e.g., for reordering), or an error.
|
||||
fn decode(&mut self, access_unit: &[u8]) -> Result<Option<VideoFrame>, VideoError>;
|
||||
|
||||
/// Compact implementation-specific state useful for field diagnostics.
|
||||
fn debug_snapshot(&self) -> Option<String> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,6 +49,11 @@ pub trait VideoEncoder: Send {
|
||||
///
|
||||
/// Default implementation is a no-op.
|
||||
fn set_mode(&mut self, _mode: crate::EncoderMode) {}
|
||||
|
||||
/// Optional platform-specific encoder state for debug logs.
|
||||
fn debug_snapshot(&self) -> Option<String> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// Raw video frame input for encoding.
|
||||
|
||||
@@ -16,6 +16,7 @@ pub mod factory;
|
||||
pub mod framer;
|
||||
pub mod mediacodec;
|
||||
pub mod nack;
|
||||
pub mod transport;
|
||||
pub mod simulcast;
|
||||
#[cfg(target_os = "macos")]
|
||||
pub mod svt_av1;
|
||||
|
||||
@@ -27,6 +27,8 @@ pub struct MediaCodecEncoder {
|
||||
width: u32,
|
||||
#[cfg(target_os = "android")]
|
||||
height: u32,
|
||||
#[cfg(target_os = "android")]
|
||||
input_format_logged: bool,
|
||||
force_keyframe: bool,
|
||||
#[cfg(not(target_os = "android"))]
|
||||
_width: u32,
|
||||
@@ -39,12 +41,18 @@ pub struct MediaCodecEncoder {
|
||||
/// Android color format constant: YUV 4:2:0 planar (I420).
|
||||
#[cfg(target_os = "android")]
|
||||
const COLOR_FORMAT_YUV420_PLANAR: i32 = 19;
|
||||
/// Android color format constant: YUV 4:2:0 semiplanar (usually NV12).
|
||||
#[cfg(target_os = "android")]
|
||||
const COLOR_FORMAT_YUV420_SEMIPLANAR: i32 = 21;
|
||||
/// Android MediaCodec CBR bitrate mode (MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR).
|
||||
#[cfg(target_os = "android")]
|
||||
const BITRATE_MODE_CBR: i32 = 2;
|
||||
/// AMediaCodec keyframe buffer flag.
|
||||
#[cfg(target_os = "android")]
|
||||
const AMEDIACODEC_BUFFER_FLAG_KEY_FRAME: u32 = 1;
|
||||
/// MediaCodec encoder parameter key for forcing the next output frame to be a sync frame.
|
||||
#[cfg(target_os = "android")]
|
||||
const MEDIA_CODEC_REQUEST_SYNC_FRAME: &str = "request-sync";
|
||||
|
||||
// AMediaCodec is thread-safe; the NonNull inside MediaCodec suppresses auto-Send.
|
||||
#[cfg(target_os = "android")]
|
||||
@@ -61,8 +69,8 @@ impl MediaCodecEncoder {
|
||||
format.set_i32("height", height as i32);
|
||||
format.set_i32("bitrate", bitrate_bps as i32);
|
||||
format.set_i32("frame-rate", 30);
|
||||
format.set_i32("i-frame-interval", 1);
|
||||
format.set_i32("color-format", COLOR_FORMAT_YUV420_PLANAR);
|
||||
format.set_i32("i-frame-interval", 4);
|
||||
format.set_i32("color-format", COLOR_FORMAT_YUV420_SEMIPLANAR);
|
||||
|
||||
let codec = MediaCodec::from_encoder_type("video/avc").ok_or_else(|| {
|
||||
VideoError::PlatformError("AMediaCodec_createEncoderByType failed".into())
|
||||
@@ -80,6 +88,7 @@ impl MediaCodecEncoder {
|
||||
codec,
|
||||
width,
|
||||
height,
|
||||
input_format_logged: false,
|
||||
force_keyframe: false,
|
||||
})
|
||||
}
|
||||
@@ -114,21 +123,52 @@ impl VideoEncoder for MediaCodecEncoder {
|
||||
.dequeue_input_buffer(std::time::Duration::from_millis(10))
|
||||
{
|
||||
Ok(ndk::media::media_codec::DequeuedInputBufferResult::Buffer(mut buffer)) => {
|
||||
let flags = if self.force_keyframe {
|
||||
AMEDIACODEC_BUFFER_FLAG_KEY_FRAME
|
||||
} else {
|
||||
0
|
||||
};
|
||||
if self.force_keyframe {
|
||||
self.request_sync_frame();
|
||||
}
|
||||
let layout = encoder_input_layout(&self.codec, self.width, self.height);
|
||||
if !self.input_format_logged {
|
||||
self.input_format_logged = true;
|
||||
log_media_codec_input_format("h264_encoder_input", &self.codec, &layout);
|
||||
}
|
||||
let input_capacity = { buffer.buffer_mut().len() };
|
||||
let mut input = i420_to_encoder_input(
|
||||
&frame.data,
|
||||
self.width as usize,
|
||||
self.height as usize,
|
||||
&layout,
|
||||
COLOR_FORMAT_YUV420_SEMIPLANAR,
|
||||
)?;
|
||||
if input.len() > input_capacity {
|
||||
tracing::warn!(
|
||||
target: "wzp_video::mediacodec",
|
||||
padded_len = input.len(),
|
||||
input_capacity,
|
||||
"MediaCodec H.264 input buffer smaller than padded layout; falling back to tight NV12"
|
||||
);
|
||||
let tight_layout = EncoderInputLayout {
|
||||
color_format: layout.color_format,
|
||||
stride: self.width as usize,
|
||||
slice_height: self.height as usize,
|
||||
};
|
||||
input = i420_to_encoder_input(
|
||||
&frame.data,
|
||||
self.width as usize,
|
||||
self.height as usize,
|
||||
&tight_layout,
|
||||
COLOR_FORMAT_YUV420_SEMIPLANAR,
|
||||
)?;
|
||||
}
|
||||
let to_copy = {
|
||||
let buf = buffer.buffer_mut();
|
||||
let n = frame.data.len().min(buf.len());
|
||||
for (d, &s) in buf[..n].iter_mut().zip(frame.data[..n].iter()) {
|
||||
let n = input.len().min(buf.len());
|
||||
for (d, &s) in buf[..n].iter_mut().zip(input[..n].iter()) {
|
||||
d.write(s);
|
||||
}
|
||||
n
|
||||
};
|
||||
self.codec
|
||||
.queue_input_buffer(buffer, 0, to_copy, frame.timestamp_ms as u64 * 1000, flags)
|
||||
.queue_input_buffer(buffer, 0, to_copy, frame.timestamp_ms as u64 * 1000, 0)
|
||||
.map_err(|e| {
|
||||
VideoError::PlatformError(format!("queue_input_buffer failed: {e}"))
|
||||
})?;
|
||||
@@ -160,13 +200,25 @@ impl VideoEncoder for MediaCodecEncoder {
|
||||
if packet.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let nal_type = packet[0] & 0x1F;
|
||||
nal_type == 5
|
||||
let nals = split_annex_b(packet);
|
||||
if nals.is_empty() {
|
||||
return (packet[0] & 0x1F) == 5;
|
||||
}
|
||||
nals.iter()
|
||||
.any(|nal| !nal.is_empty() && (nal[0] & 0x1F) == 5)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "android")]
|
||||
impl MediaCodecEncoder {
|
||||
fn request_sync_frame(&self) {
|
||||
let mut params = MediaFormat::new();
|
||||
params.set_i32(MEDIA_CODEC_REQUEST_SYNC_FRAME, 0);
|
||||
if let Err(e) = self.codec.set_parameters(params) {
|
||||
tracing::warn!(error = %e, "AMediaCodec request sync frame failed");
|
||||
}
|
||||
}
|
||||
|
||||
/// Drain all available output buffers and convert from AVCC to Annex-B.
|
||||
fn drain_output(&mut self) -> Result<Vec<u8>, VideoError> {
|
||||
let mut output = Vec::new();
|
||||
@@ -181,7 +233,7 @@ impl MediaCodecEncoder {
|
||||
if is_keyframe {
|
||||
self.force_keyframe = false;
|
||||
}
|
||||
let data = buffer.buffer().to_vec();
|
||||
let data = output_buffer_payload(&buffer)?;
|
||||
output.extend_from_slice(&avcc_to_annexb(&data));
|
||||
self.codec
|
||||
.release_output_buffer(buffer, false)
|
||||
@@ -191,7 +243,10 @@ impl MediaCodecEncoder {
|
||||
}
|
||||
Ok(
|
||||
ndk::media::media_codec::DequeuedOutputBufferInfoResult::OutputFormatChanged,
|
||||
) => continue,
|
||||
) => {
|
||||
log_media_codec_format("h264_encoder_output", &self.codec);
|
||||
continue;
|
||||
}
|
||||
Ok(
|
||||
ndk::media::media_codec::DequeuedOutputBufferInfoResult::OutputBuffersChanged,
|
||||
) => continue,
|
||||
@@ -266,6 +321,7 @@ impl VideoDecoder for MediaCodecDecoder {
|
||||
format.set_str("mime", "video/avc");
|
||||
format.set_i32("width", self.width as i32);
|
||||
format.set_i32("height", self.height as i32);
|
||||
format.set_i32("color-format", COLOR_FORMAT_YUV420_PLANAR);
|
||||
format.set_buffer("csd-0", &sps);
|
||||
format.set_buffer("csd-1", &pps);
|
||||
|
||||
@@ -318,14 +374,12 @@ impl VideoDecoder for MediaCodecDecoder {
|
||||
// Drain output.
|
||||
match codec.dequeue_output_buffer(std::time::Duration::from_millis(10)) {
|
||||
Ok(ndk::media::media_codec::DequeuedOutputBufferInfoResult::Buffer(buffer)) => {
|
||||
let data = buffer.buffer().to_vec();
|
||||
codec
|
||||
.release_output_buffer(buffer, false)
|
||||
.map_err(|e| {
|
||||
VideoError::PlatformError(format!(
|
||||
"decoder release_output_buffer failed: {e}"
|
||||
))
|
||||
})?;
|
||||
let data = decoded_i420_payload(codec, &buffer, self.width, self.height)?;
|
||||
codec.release_output_buffer(buffer, false).map_err(|e| {
|
||||
VideoError::PlatformError(format!(
|
||||
"decoder release_output_buffer failed: {e}"
|
||||
))
|
||||
})?;
|
||||
Ok(Some(VideoFrame {
|
||||
width: self.width,
|
||||
height: self.height,
|
||||
@@ -333,6 +387,12 @@ impl VideoDecoder for MediaCodecDecoder {
|
||||
timestamp_ms: 0,
|
||||
}))
|
||||
}
|
||||
Ok(
|
||||
ndk::media::media_codec::DequeuedOutputBufferInfoResult::OutputFormatChanged,
|
||||
) => {
|
||||
log_media_codec_format("h264_decoder_output", codec);
|
||||
Ok(None)
|
||||
}
|
||||
Ok(_) => Ok(None),
|
||||
Err(e) => Err(VideoError::PlatformError(format!(
|
||||
"decoder dequeue_output_buffer failed: {e}"
|
||||
@@ -345,6 +405,17 @@ impl VideoDecoder for MediaCodecDecoder {
|
||||
Err(VideoError::NotInitialized)
|
||||
}
|
||||
}
|
||||
|
||||
fn debug_snapshot(&self) -> Option<String> {
|
||||
#[cfg(target_os = "android")]
|
||||
{
|
||||
media_codec_debug_snapshot(self.codec.as_ref())
|
||||
}
|
||||
#[cfg(not(target_os = "android"))]
|
||||
{
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -361,6 +432,8 @@ pub struct MediaCodecHevcEncoder {
|
||||
width: u32,
|
||||
#[cfg(target_os = "android")]
|
||||
height: u32,
|
||||
#[cfg(target_os = "android")]
|
||||
input_format_logged: bool,
|
||||
force_keyframe: bool,
|
||||
#[cfg(not(target_os = "android"))]
|
||||
_width: u32,
|
||||
@@ -383,7 +456,7 @@ impl MediaCodecHevcEncoder {
|
||||
format.set_i32("height", height as i32);
|
||||
format.set_i32("bitrate", bitrate_bps as i32);
|
||||
format.set_i32("frame-rate", 30);
|
||||
format.set_i32("i-frame-interval", 1);
|
||||
format.set_i32("i-frame-interval", 4);
|
||||
format.set_i32("color-format", COLOR_FORMAT_YUV420_PLANAR);
|
||||
|
||||
let codec = MediaCodec::from_encoder_type("video/hevc").ok_or_else(|| {
|
||||
@@ -402,6 +475,7 @@ impl MediaCodecHevcEncoder {
|
||||
codec,
|
||||
width,
|
||||
height,
|
||||
input_format_logged: false,
|
||||
force_keyframe: false,
|
||||
})
|
||||
}
|
||||
@@ -434,17 +508,60 @@ impl VideoEncoder for MediaCodecHevcEncoder {
|
||||
.dequeue_input_buffer(std::time::Duration::from_millis(10))
|
||||
{
|
||||
Ok(ndk::media::media_codec::DequeuedInputBufferResult::Buffer(mut buffer)) => {
|
||||
let flags = if self.force_keyframe { AMEDIACODEC_BUFFER_FLAG_KEY_FRAME } else { 0 };
|
||||
let flags = if self.force_keyframe {
|
||||
AMEDIACODEC_BUFFER_FLAG_KEY_FRAME
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let layout = encoder_input_layout(&self.codec, self.width, self.height);
|
||||
if !self.input_format_logged {
|
||||
self.input_format_logged = true;
|
||||
log_media_codec_input_format("hevc_encoder_input", &self.codec, &layout);
|
||||
}
|
||||
let input_capacity = { buffer.buffer_mut().len() };
|
||||
let mut input = i420_to_encoder_input(
|
||||
&frame.data,
|
||||
self.width as usize,
|
||||
self.height as usize,
|
||||
&layout,
|
||||
COLOR_FORMAT_YUV420_PLANAR,
|
||||
)?;
|
||||
if input.len() > input_capacity {
|
||||
tracing::warn!(
|
||||
target: "wzp_video::mediacodec",
|
||||
padded_len = input.len(),
|
||||
input_capacity,
|
||||
"MediaCodec HEVC input buffer smaller than padded layout; falling back to tight I420"
|
||||
);
|
||||
let tight_layout = EncoderInputLayout {
|
||||
color_format: layout.color_format,
|
||||
stride: self.width as usize,
|
||||
slice_height: self.height as usize,
|
||||
};
|
||||
input = i420_to_encoder_input(
|
||||
&frame.data,
|
||||
self.width as usize,
|
||||
self.height as usize,
|
||||
&tight_layout,
|
||||
COLOR_FORMAT_YUV420_PLANAR,
|
||||
)?;
|
||||
}
|
||||
let to_copy = {
|
||||
let buf = buffer.buffer_mut();
|
||||
let n = frame.data.len().min(buf.len());
|
||||
for (d, &s) in buf[..n].iter_mut().zip(frame.data[..n].iter()) {
|
||||
let n = input.len().min(buf.len());
|
||||
for (d, &s) in buf[..n].iter_mut().zip(input[..n].iter()) {
|
||||
d.write(s);
|
||||
}
|
||||
n
|
||||
};
|
||||
self.codec
|
||||
.queue_input_buffer(buffer, 0, to_copy, frame.timestamp_ms as u64 * 1000, flags)
|
||||
.queue_input_buffer(
|
||||
buffer,
|
||||
0,
|
||||
to_copy,
|
||||
frame.timestamp_ms as u64 * 1000,
|
||||
flags,
|
||||
)
|
||||
.map_err(|e| {
|
||||
VideoError::PlatformError(format!("queue_input_buffer failed: {e}"))
|
||||
})?;
|
||||
@@ -472,11 +589,12 @@ impl VideoEncoder for MediaCodecHevcEncoder {
|
||||
}
|
||||
|
||||
fn is_keyframe(&self, packet: &[u8]) -> bool {
|
||||
if packet.len() < 2 {
|
||||
return false;
|
||||
let nals = split_annex_b(packet);
|
||||
if nals.is_empty() {
|
||||
return packet.len() >= 2 && matches!((packet[0] >> 1) & 0x3F, 19 | 20);
|
||||
}
|
||||
let nal_type = (packet[0] >> 1) & 0x3F;
|
||||
nal_type == 19 || nal_type == 20
|
||||
nals.iter()
|
||||
.any(|nal| nal.len() >= 2 && matches!((nal[0] >> 1) & 0x3F, 19 | 20))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -556,7 +674,11 @@ impl VideoEncoder for MediaCodecAv1Encoder {
|
||||
.dequeue_input_buffer(std::time::Duration::from_millis(0))
|
||||
{
|
||||
Ok(ndk::media::media_codec::DequeuedInputBufferResult::Buffer(mut buffer)) => {
|
||||
let flags = if self.force_keyframe { AMEDIACODEC_BUFFER_FLAG_KEY_FRAME } else { 0 };
|
||||
let flags = if self.force_keyframe {
|
||||
AMEDIACODEC_BUFFER_FLAG_KEY_FRAME
|
||||
} else {
|
||||
0
|
||||
};
|
||||
let to_copy = {
|
||||
let buf = buffer.buffer_mut();
|
||||
let n = frame.data.len().min(buf.len());
|
||||
@@ -566,7 +688,13 @@ impl VideoEncoder for MediaCodecAv1Encoder {
|
||||
n
|
||||
};
|
||||
self.codec
|
||||
.queue_input_buffer(buffer, 0, to_copy, frame.timestamp_ms as u64 * 1000, flags)
|
||||
.queue_input_buffer(
|
||||
buffer,
|
||||
0,
|
||||
to_copy,
|
||||
frame.timestamp_ms as u64 * 1000,
|
||||
flags,
|
||||
)
|
||||
.map_err(|e| {
|
||||
VideoError::PlatformError(format!(
|
||||
"AV1 encoder queue_input_buffer failed: {e}"
|
||||
@@ -615,7 +743,7 @@ impl MediaCodecHevcEncoder {
|
||||
if is_keyframe {
|
||||
self.force_keyframe = false;
|
||||
}
|
||||
let data = buffer.buffer().to_vec();
|
||||
let data = output_buffer_payload(&buffer)?;
|
||||
output.extend_from_slice(&avcc_to_annexb(&data));
|
||||
self.codec
|
||||
.release_output_buffer(buffer, false)
|
||||
@@ -625,7 +753,10 @@ impl MediaCodecHevcEncoder {
|
||||
}
|
||||
Ok(
|
||||
ndk::media::media_codec::DequeuedOutputBufferInfoResult::OutputFormatChanged,
|
||||
) => continue,
|
||||
) => {
|
||||
log_media_codec_format("hevc_encoder_output", &self.codec);
|
||||
continue;
|
||||
}
|
||||
Ok(
|
||||
ndk::media::media_codec::DequeuedOutputBufferInfoResult::OutputBuffersChanged,
|
||||
) => continue,
|
||||
@@ -657,7 +788,7 @@ impl MediaCodecAv1Encoder {
|
||||
self.force_keyframe = false;
|
||||
}
|
||||
// AV1 output from MediaCodec is already in OBU format.
|
||||
let data = buffer.buffer().to_vec();
|
||||
let data = output_buffer_payload(&buffer)?;
|
||||
output.extend_from_slice(&data);
|
||||
self.codec
|
||||
.release_output_buffer(buffer, false)
|
||||
@@ -669,7 +800,10 @@ impl MediaCodecAv1Encoder {
|
||||
}
|
||||
Ok(
|
||||
ndk::media::media_codec::DequeuedOutputBufferInfoResult::OutputFormatChanged,
|
||||
) => continue,
|
||||
) => {
|
||||
log_media_codec_format("av1_encoder_output", &self.codec);
|
||||
continue;
|
||||
}
|
||||
Ok(
|
||||
ndk::media::media_codec::DequeuedOutputBufferInfoResult::OutputBuffersChanged,
|
||||
) => continue,
|
||||
@@ -742,6 +876,7 @@ impl VideoDecoder for MediaCodecHevcDecoder {
|
||||
format.set_str("mime", "video/hevc");
|
||||
format.set_i32("width", self.width as i32);
|
||||
format.set_i32("height", self.height as i32);
|
||||
format.set_i32("color-format", COLOR_FORMAT_YUV420_PLANAR);
|
||||
format.set_buffer("csd-0", &vps);
|
||||
format.set_buffer("csd-1", &sps);
|
||||
format.set_buffer("csd-2", &pps);
|
||||
@@ -795,14 +930,12 @@ impl VideoDecoder for MediaCodecHevcDecoder {
|
||||
|
||||
match codec.dequeue_output_buffer(std::time::Duration::from_millis(10)) {
|
||||
Ok(ndk::media::media_codec::DequeuedOutputBufferInfoResult::Buffer(buffer)) => {
|
||||
let data = buffer.buffer().to_vec();
|
||||
codec
|
||||
.release_output_buffer(buffer, false)
|
||||
.map_err(|e| {
|
||||
VideoError::PlatformError(format!(
|
||||
"decoder release_output_buffer failed: {e}"
|
||||
))
|
||||
})?;
|
||||
let data = decoded_i420_payload(codec, &buffer, self.width, self.height)?;
|
||||
codec.release_output_buffer(buffer, false).map_err(|e| {
|
||||
VideoError::PlatformError(format!(
|
||||
"decoder release_output_buffer failed: {e}"
|
||||
))
|
||||
})?;
|
||||
Ok(Some(VideoFrame {
|
||||
width: self.width,
|
||||
height: self.height,
|
||||
@@ -810,6 +943,12 @@ impl VideoDecoder for MediaCodecHevcDecoder {
|
||||
timestamp_ms: 0,
|
||||
}))
|
||||
}
|
||||
Ok(
|
||||
ndk::media::media_codec::DequeuedOutputBufferInfoResult::OutputFormatChanged,
|
||||
) => {
|
||||
log_media_codec_format("hevc_decoder_output", codec);
|
||||
Ok(None)
|
||||
}
|
||||
Ok(_) => Ok(None),
|
||||
Err(e) => Err(VideoError::PlatformError(format!(
|
||||
"decoder dequeue_output_buffer failed: {e}"
|
||||
@@ -822,6 +961,17 @@ impl VideoDecoder for MediaCodecHevcDecoder {
|
||||
Err(VideoError::NotInitialized)
|
||||
}
|
||||
}
|
||||
|
||||
fn debug_snapshot(&self) -> Option<String> {
|
||||
#[cfg(target_os = "android")]
|
||||
{
|
||||
media_codec_debug_snapshot(self.codec.as_ref())
|
||||
}
|
||||
#[cfg(not(target_os = "android"))]
|
||||
{
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Android MediaCodec AV1 decoder.
|
||||
@@ -881,6 +1031,7 @@ impl VideoDecoder for MediaCodecAv1Decoder {
|
||||
format.set_str("mime", "video/av01");
|
||||
format.set_i32("width", self.width as i32);
|
||||
format.set_i32("height", self.height as i32);
|
||||
format.set_i32("color-format", COLOR_FORMAT_YUV420_PLANAR);
|
||||
format.set_buffer("csd-0", &seq_header);
|
||||
|
||||
let codec = MediaCodec::from_decoder_type("video/av01").ok_or_else(|| {
|
||||
@@ -930,14 +1081,12 @@ impl VideoDecoder for MediaCodecAv1Decoder {
|
||||
|
||||
match codec.dequeue_output_buffer(std::time::Duration::from_millis(10)) {
|
||||
Ok(ndk::media::media_codec::DequeuedOutputBufferInfoResult::Buffer(buffer)) => {
|
||||
let data = buffer.buffer().to_vec();
|
||||
codec
|
||||
.release_output_buffer(buffer, false)
|
||||
.map_err(|e| {
|
||||
VideoError::PlatformError(format!(
|
||||
"AV1 decoder release_output_buffer failed: {e}"
|
||||
))
|
||||
})?;
|
||||
let data = decoded_i420_payload(codec, &buffer, self.width, self.height)?;
|
||||
codec.release_output_buffer(buffer, false).map_err(|e| {
|
||||
VideoError::PlatformError(format!(
|
||||
"AV1 decoder release_output_buffer failed: {e}"
|
||||
))
|
||||
})?;
|
||||
Ok(Some(VideoFrame {
|
||||
width: self.width,
|
||||
height: self.height,
|
||||
@@ -945,6 +1094,12 @@ impl VideoDecoder for MediaCodecAv1Decoder {
|
||||
timestamp_ms: 0,
|
||||
}))
|
||||
}
|
||||
Ok(
|
||||
ndk::media::media_codec::DequeuedOutputBufferInfoResult::OutputFormatChanged,
|
||||
) => {
|
||||
log_media_codec_format("av1_decoder_output", codec);
|
||||
Ok(None)
|
||||
}
|
||||
Ok(_) => Ok(None),
|
||||
Err(e) => Err(VideoError::PlatformError(format!(
|
||||
"AV1 decoder dequeue_output_buffer failed: {e}"
|
||||
@@ -957,6 +1112,453 @@ impl VideoDecoder for MediaCodecAv1Decoder {
|
||||
Err(VideoError::NotInitialized)
|
||||
}
|
||||
}
|
||||
|
||||
fn debug_snapshot(&self) -> Option<String> {
|
||||
#[cfg(target_os = "android")]
|
||||
{
|
||||
media_codec_debug_snapshot(self.codec.as_ref())
|
||||
}
|
||||
#[cfg(not(target_os = "android"))]
|
||||
{
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "android")]
|
||||
fn media_codec_debug_snapshot(codec: Option<&MediaCodec>) -> Option<String> {
|
||||
let codec = codec?;
|
||||
let format = codec.output_format();
|
||||
Some(format!(
|
||||
"color_format={:?} width={:?} height={:?} stride={:?} slice_height={:?} crop=({:?},{:?},{:?},{:?})",
|
||||
format.i32("color-format"),
|
||||
format.i32("width"),
|
||||
format.i32("height"),
|
||||
format.i32("stride"),
|
||||
format.i32("slice-height"),
|
||||
format.i32("crop-left"),
|
||||
format.i32("crop-top"),
|
||||
format.i32("crop-right"),
|
||||
format.i32("crop-bottom"),
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(target_os = "android")]
|
||||
fn output_buffer_payload(
|
||||
buffer: &ndk::media::media_codec::OutputBuffer<'_>,
|
||||
) -> Result<Vec<u8>, VideoError> {
|
||||
let info = buffer.info();
|
||||
let offset = usize::try_from(info.offset()).map_err(|_| {
|
||||
VideoError::PlatformError(format!(
|
||||
"negative MediaCodec output offset: {}",
|
||||
info.offset()
|
||||
))
|
||||
})?;
|
||||
let size = usize::try_from(info.size()).map_err(|_| {
|
||||
VideoError::PlatformError(format!("negative MediaCodec output size: {}", info.size()))
|
||||
})?;
|
||||
let end = offset.checked_add(size).ok_or_else(|| {
|
||||
VideoError::PlatformError(format!(
|
||||
"MediaCodec output range overflow: offset={offset} size={size}"
|
||||
))
|
||||
})?;
|
||||
let raw = buffer.buffer();
|
||||
if end > raw.len() {
|
||||
return Err(VideoError::PlatformError(format!(
|
||||
"MediaCodec output range outside buffer: offset={offset} size={size} buffer_len={}",
|
||||
raw.len()
|
||||
)));
|
||||
}
|
||||
Ok(raw[offset..end].to_vec())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "android")]
|
||||
fn decoded_i420_payload(
|
||||
codec: &MediaCodec,
|
||||
buffer: &ndk::media::media_codec::OutputBuffer<'_>,
|
||||
width: u32,
|
||||
height: u32,
|
||||
) -> Result<Vec<u8>, VideoError> {
|
||||
let payload = output_buffer_payload(buffer)?;
|
||||
let format = codec.output_format();
|
||||
let color_format = format
|
||||
.i32("color-format")
|
||||
.unwrap_or(COLOR_FORMAT_YUV420_PLANAR);
|
||||
let stride = positive_format_usize(&format, "stride").unwrap_or(width as usize);
|
||||
let slice_height = positive_format_usize(&format, "slice-height").unwrap_or(height as usize);
|
||||
|
||||
match color_format {
|
||||
COLOR_FORMAT_YUV420_PLANAR => yuv420_planar_to_tight_i420(
|
||||
&payload,
|
||||
width as usize,
|
||||
height as usize,
|
||||
stride,
|
||||
slice_height,
|
||||
),
|
||||
COLOR_FORMAT_YUV420_SEMIPLANAR => yuv420_semiplanar_to_tight_i420(
|
||||
&payload,
|
||||
width as usize,
|
||||
height as usize,
|
||||
stride,
|
||||
slice_height,
|
||||
),
|
||||
_ => {
|
||||
let expected = i420_len(width as usize, height as usize)?;
|
||||
if payload.len() < expected {
|
||||
return Err(VideoError::PlatformError(format!(
|
||||
"unsupported MediaCodec color format {color_format} produced {} bytes, expected at least {expected}",
|
||||
payload.len()
|
||||
)));
|
||||
}
|
||||
let mut data = payload;
|
||||
data.truncate(expected);
|
||||
Ok(data)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "android")]
|
||||
fn positive_format_usize(format: &MediaFormat, key: &str) -> Option<usize> {
|
||||
let value = format.i32(key)?;
|
||||
(value > 0).then_some(value as usize)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "android")]
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct EncoderInputLayout {
|
||||
color_format: Option<i32>,
|
||||
stride: usize,
|
||||
slice_height: usize,
|
||||
}
|
||||
|
||||
#[cfg(target_os = "android")]
|
||||
fn encoder_input_layout(codec: &MediaCodec, width: u32, height: u32) -> EncoderInputLayout {
|
||||
let format = codec.input_format();
|
||||
let width = width as usize;
|
||||
let height = height as usize;
|
||||
EncoderInputLayout {
|
||||
color_format: format.i32("color-format"),
|
||||
stride: positive_format_usize(&format, "stride")
|
||||
.unwrap_or(width)
|
||||
.max(width),
|
||||
slice_height: positive_format_usize(&format, "slice-height")
|
||||
.unwrap_or(height)
|
||||
.max(height),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "android")]
|
||||
fn log_media_codec_input_format(label: &str, codec: &MediaCodec, layout: &EncoderInputLayout) {
|
||||
let input_format = codec.input_format();
|
||||
let output_format = codec.output_format();
|
||||
tracing::info!(
|
||||
target: "wzp_video::mediacodec",
|
||||
label,
|
||||
input_color_format = input_format.i32("color-format"),
|
||||
input_width = input_format.i32("width"),
|
||||
input_height = input_format.i32("height"),
|
||||
input_stride = input_format.i32("stride"),
|
||||
input_slice_height = input_format.i32("slice-height"),
|
||||
output_color_format = output_format.i32("color-format"),
|
||||
output_width = output_format.i32("width"),
|
||||
output_height = output_format.i32("height"),
|
||||
output_stride = output_format.i32("stride"),
|
||||
output_slice_height = output_format.i32("slice-height"),
|
||||
effective_color_format = layout.color_format,
|
||||
effective_stride = layout.stride,
|
||||
effective_slice_height = layout.slice_height,
|
||||
"MediaCodec input format"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(target_os = "android")]
|
||||
fn i420_to_encoder_input(
|
||||
src: &[u8],
|
||||
width: usize,
|
||||
height: usize,
|
||||
layout: &EncoderInputLayout,
|
||||
default_color_format: i32,
|
||||
) -> Result<Vec<u8>, VideoError> {
|
||||
let color_format = layout.color_format.unwrap_or(default_color_format);
|
||||
match color_format {
|
||||
COLOR_FORMAT_YUV420_PLANAR => {
|
||||
i420_to_padded_planar(src, width, height, layout.stride, layout.slice_height)
|
||||
}
|
||||
COLOR_FORMAT_YUV420_SEMIPLANAR => {
|
||||
i420_to_padded_nv12(src, width, height, layout.stride, layout.slice_height)
|
||||
}
|
||||
other => {
|
||||
tracing::warn!(
|
||||
target: "wzp_video::mediacodec",
|
||||
color_format = other,
|
||||
default_color_format,
|
||||
"unsupported MediaCodec encoder input color format; using requested default"
|
||||
);
|
||||
if default_color_format == COLOR_FORMAT_YUV420_PLANAR {
|
||||
i420_to_padded_planar(src, width, height, layout.stride, layout.slice_height)
|
||||
} else {
|
||||
i420_to_padded_nv12(src, width, height, layout.stride, layout.slice_height)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "android")]
|
||||
fn log_media_codec_format(label: &str, codec: &MediaCodec) {
|
||||
let format = codec.output_format();
|
||||
tracing::info!(
|
||||
target: "wzp_video::mediacodec",
|
||||
label,
|
||||
color_format = format.i32("color-format"),
|
||||
width = format.i32("width"),
|
||||
height = format.i32("height"),
|
||||
stride = format.i32("stride"),
|
||||
slice_height = format.i32("slice-height"),
|
||||
crop_left = format.i32("crop-left"),
|
||||
crop_right = format.i32("crop-right"),
|
||||
crop_top = format.i32("crop-top"),
|
||||
crop_bottom = format.i32("crop-bottom"),
|
||||
"MediaCodec output format changed"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(target_os = "android")]
|
||||
fn i420_len(width: usize, height: usize) -> Result<usize, VideoError> {
|
||||
width
|
||||
.checked_mul(height)
|
||||
.and_then(|y| y.checked_add(y / 2))
|
||||
.ok_or_else(|| {
|
||||
VideoError::InvalidInput(format!("invalid I420 dimensions {width}x{height}"))
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_os = "android")]
|
||||
fn i420_to_padded_nv12(
|
||||
src: &[u8],
|
||||
width: usize,
|
||||
height: usize,
|
||||
stride: usize,
|
||||
slice_height: usize,
|
||||
) -> Result<Vec<u8>, VideoError> {
|
||||
let y_size = width.checked_mul(height).ok_or_else(|| {
|
||||
VideoError::InvalidInput(format!("invalid frame dimensions {width}x{height}"))
|
||||
})?;
|
||||
let uv_size = y_size / 4;
|
||||
let expected = y_size + uv_size * 2;
|
||||
if src.len() < expected {
|
||||
return Err(VideoError::InvalidInput(format!(
|
||||
"I420 frame too small for NV12 conversion: {} bytes, expected {expected}",
|
||||
src.len()
|
||||
)));
|
||||
}
|
||||
|
||||
if stride < width || slice_height < height {
|
||||
return Err(VideoError::InvalidInput(format!(
|
||||
"invalid encoder input layout {stride}x{slice_height} for {width}x{height}"
|
||||
)));
|
||||
}
|
||||
|
||||
let chroma_width = width / 2;
|
||||
let chroma_height = height / 2;
|
||||
let y_stride = stride;
|
||||
let uv_stride = stride;
|
||||
let y_slice_height = slice_height;
|
||||
let uv_slice_height = (slice_height / 2).max(chroma_height);
|
||||
let y_padded_size = y_stride.checked_mul(y_slice_height).ok_or_else(|| {
|
||||
VideoError::InvalidInput(format!(
|
||||
"invalid padded Y layout {y_stride}x{y_slice_height}"
|
||||
))
|
||||
})?;
|
||||
let uv_padded_size = uv_stride.checked_mul(uv_slice_height).ok_or_else(|| {
|
||||
VideoError::InvalidInput(format!(
|
||||
"invalid padded UV layout {uv_stride}x{uv_slice_height}"
|
||||
))
|
||||
})?;
|
||||
let total = y_padded_size
|
||||
.checked_add(uv_padded_size)
|
||||
.ok_or_else(|| VideoError::InvalidInput("padded NV12 size overflow".into()))?;
|
||||
|
||||
let mut out = vec![0u8; total];
|
||||
out[y_padded_size..].fill(128);
|
||||
for row in 0..height {
|
||||
let src_off = row * width;
|
||||
let dst_off = row * y_stride;
|
||||
out[dst_off..dst_off + width].copy_from_slice(&src[src_off..src_off + width]);
|
||||
}
|
||||
|
||||
let u = &src[y_size..y_size + uv_size];
|
||||
let v = &src[y_size + uv_size..y_size + uv_size * 2];
|
||||
for row in 0..chroma_height {
|
||||
let src_row = row * chroma_width;
|
||||
let dst_row = y_padded_size + row * uv_stride;
|
||||
for col in 0..chroma_width {
|
||||
out[dst_row + col * 2] = u[src_row + col];
|
||||
out[dst_row + col * 2 + 1] = v[src_row + col];
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "android")]
|
||||
fn i420_to_padded_planar(
|
||||
src: &[u8],
|
||||
width: usize,
|
||||
height: usize,
|
||||
stride: usize,
|
||||
slice_height: usize,
|
||||
) -> Result<Vec<u8>, VideoError> {
|
||||
let y_size = width.checked_mul(height).ok_or_else(|| {
|
||||
VideoError::InvalidInput(format!("invalid frame dimensions {width}x{height}"))
|
||||
})?;
|
||||
let uv_size = y_size / 4;
|
||||
let expected = y_size + uv_size * 2;
|
||||
if src.len() < expected {
|
||||
return Err(VideoError::InvalidInput(format!(
|
||||
"I420 frame too small for padded planar copy: {} bytes, expected {expected}",
|
||||
src.len()
|
||||
)));
|
||||
}
|
||||
if stride < width || slice_height < height {
|
||||
return Err(VideoError::InvalidInput(format!(
|
||||
"invalid encoder input layout {stride}x{slice_height} for {width}x{height}"
|
||||
)));
|
||||
}
|
||||
|
||||
let chroma_width = width / 2;
|
||||
let chroma_height = height / 2;
|
||||
let y_stride = stride;
|
||||
let chroma_stride = (stride / 2).max(chroma_width);
|
||||
let y_slice_height = slice_height;
|
||||
let chroma_slice_height = (slice_height / 2).max(chroma_height);
|
||||
let y_padded_size = y_stride.checked_mul(y_slice_height).ok_or_else(|| {
|
||||
VideoError::InvalidInput(format!(
|
||||
"invalid padded Y layout {y_stride}x{y_slice_height}"
|
||||
))
|
||||
})?;
|
||||
let chroma_padded_size = chroma_stride
|
||||
.checked_mul(chroma_slice_height)
|
||||
.ok_or_else(|| {
|
||||
VideoError::InvalidInput(format!(
|
||||
"invalid padded chroma layout {chroma_stride}x{chroma_slice_height}"
|
||||
))
|
||||
})?;
|
||||
let chroma_total = chroma_padded_size
|
||||
.checked_mul(2)
|
||||
.ok_or_else(|| VideoError::InvalidInput("padded I420 chroma size overflow".into()))?;
|
||||
let total = y_padded_size
|
||||
.checked_add(chroma_total)
|
||||
.ok_or_else(|| VideoError::InvalidInput("padded I420 size overflow".into()))?;
|
||||
|
||||
let mut out = vec![0u8; total];
|
||||
out[y_padded_size..].fill(128);
|
||||
for row in 0..height {
|
||||
let src_off = row * width;
|
||||
let dst_off = row * y_stride;
|
||||
out[dst_off..dst_off + width].copy_from_slice(&src[src_off..src_off + width]);
|
||||
}
|
||||
|
||||
let src_u = y_size;
|
||||
let src_v = y_size + uv_size;
|
||||
let dst_u = y_padded_size;
|
||||
let dst_v = y_padded_size + chroma_padded_size;
|
||||
for row in 0..chroma_height {
|
||||
let src_off = row * chroma_width;
|
||||
let dst_off = row * chroma_stride;
|
||||
out[dst_u + dst_off..dst_u + dst_off + chroma_width]
|
||||
.copy_from_slice(&src[src_u + src_off..src_u + src_off + chroma_width]);
|
||||
out[dst_v + dst_off..dst_v + dst_off + chroma_width]
|
||||
.copy_from_slice(&src[src_v + src_off..src_v + src_off + chroma_width]);
|
||||
}
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "android")]
|
||||
fn yuv420_planar_to_tight_i420(
|
||||
src: &[u8],
|
||||
width: usize,
|
||||
height: usize,
|
||||
stride: usize,
|
||||
slice_height: usize,
|
||||
) -> Result<Vec<u8>, VideoError> {
|
||||
let y_size = width * height;
|
||||
let chroma_width = width / 2;
|
||||
let chroma_height = height / 2;
|
||||
let chroma_stride = stride / 2;
|
||||
let chroma_slice_height = slice_height / 2;
|
||||
let padded_y_size = stride * slice_height;
|
||||
let padded_chroma_size = chroma_stride * chroma_slice_height;
|
||||
let required = padded_y_size + padded_chroma_size * 2;
|
||||
if src.len() < required {
|
||||
return Err(VideoError::PlatformError(format!(
|
||||
"planar YUV buffer too small: {} < {required} (stride={stride}, slice_height={slice_height})",
|
||||
src.len()
|
||||
)));
|
||||
}
|
||||
|
||||
let mut out = vec![0u8; i420_len(width, height)?];
|
||||
for row in 0..height {
|
||||
let src_start = row * stride;
|
||||
let dst_start = row * width;
|
||||
out[dst_start..dst_start + width].copy_from_slice(&src[src_start..src_start + width]);
|
||||
}
|
||||
|
||||
let src_u = padded_y_size;
|
||||
let src_v = src_u + padded_chroma_size;
|
||||
let dst_u = y_size;
|
||||
let dst_v = dst_u + chroma_width * chroma_height;
|
||||
for row in 0..chroma_height {
|
||||
let src_row = row * chroma_stride;
|
||||
let dst_row = row * chroma_width;
|
||||
out[dst_u + dst_row..dst_u + dst_row + chroma_width]
|
||||
.copy_from_slice(&src[src_u + src_row..src_u + src_row + chroma_width]);
|
||||
out[dst_v + dst_row..dst_v + dst_row + chroma_width]
|
||||
.copy_from_slice(&src[src_v + src_row..src_v + src_row + chroma_width]);
|
||||
}
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "android")]
|
||||
fn yuv420_semiplanar_to_tight_i420(
|
||||
src: &[u8],
|
||||
width: usize,
|
||||
height: usize,
|
||||
stride: usize,
|
||||
slice_height: usize,
|
||||
) -> Result<Vec<u8>, VideoError> {
|
||||
let y_size = width * height;
|
||||
let chroma_width = width / 2;
|
||||
let chroma_height = height / 2;
|
||||
let padded_y_size = stride * slice_height;
|
||||
let required = padded_y_size + stride * chroma_height;
|
||||
if src.len() < required {
|
||||
return Err(VideoError::PlatformError(format!(
|
||||
"semiplanar YUV buffer too small: {} < {required} (stride={stride}, slice_height={slice_height})",
|
||||
src.len()
|
||||
)));
|
||||
}
|
||||
|
||||
let mut out = vec![0u8; i420_len(width, height)?];
|
||||
for row in 0..height {
|
||||
let src_start = row * stride;
|
||||
let dst_start = row * width;
|
||||
out[dst_start..dst_start + width].copy_from_slice(&src[src_start..src_start + width]);
|
||||
}
|
||||
|
||||
let dst_u = y_size;
|
||||
let dst_v = dst_u + chroma_width * chroma_height;
|
||||
for row in 0..chroma_height {
|
||||
let src_row = padded_y_size + row * stride;
|
||||
let dst_row = row * chroma_width;
|
||||
for col in 0..chroma_width {
|
||||
let pair = src_row + col * 2;
|
||||
out[dst_u + dst_row + col] = src[pair];
|
||||
out[dst_v + dst_row + col] = src[pair + 1];
|
||||
}
|
||||
}
|
||||
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Type alias for HEVC parameter-set triple returned by `extract_vps_sps_pps`.
|
||||
@@ -989,8 +1591,13 @@ fn extract_vps_sps_pps(annex_b: &[u8]) -> HevcParameterSets {
|
||||
/// (4-byte start codes `0x00 0x00 0x00 0x01`).
|
||||
#[allow(dead_code)]
|
||||
fn avcc_to_annexb(data: &[u8]) -> Vec<u8> {
|
||||
if starts_with_annex_b_start_code(data) {
|
||||
return data.to_vec();
|
||||
}
|
||||
|
||||
let mut out = Vec::with_capacity(data.len() + data.len() / 4);
|
||||
let mut offset = 0;
|
||||
let mut saw_nal = false;
|
||||
while offset + 4 <= data.len() {
|
||||
let nal_len = u32::from_be_bytes([
|
||||
data[offset],
|
||||
@@ -1000,15 +1607,20 @@ fn avcc_to_annexb(data: &[u8]) -> Vec<u8> {
|
||||
]) as usize;
|
||||
offset += 4;
|
||||
if offset + nal_len > data.len() {
|
||||
break;
|
||||
return if saw_nal { out } else { data.to_vec() };
|
||||
}
|
||||
out.extend_from_slice(&[0x00, 0x00, 0x00, 0x01]);
|
||||
out.extend_from_slice(&data[offset..offset + nal_len]);
|
||||
offset += nal_len;
|
||||
saw_nal = true;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn starts_with_annex_b_start_code(data: &[u8]) -> bool {
|
||||
data.starts_with(&[0x00, 0x00, 0x01]) || data.starts_with(&[0x00, 0x00, 0x00, 0x01])
|
||||
}
|
||||
|
||||
/// Parse an Annex-B access unit and return the first SPS and PPS found.
|
||||
#[allow(dead_code)]
|
||||
fn extract_sps_pps(annex_b: &[u8]) -> (Option<Vec<u8>>, Option<Vec<u8>>) {
|
||||
@@ -1064,7 +1676,7 @@ fn split_annex_b(data: &[u8]) -> Vec<&[u8]> {
|
||||
/// Android MediaCodec `csd-0`.
|
||||
#[allow(dead_code)]
|
||||
fn extract_sequence_header_obu(data: &[u8]) -> Option<Vec<u8>> {
|
||||
use crate::av1_obu::{ObuHeader, read_leb128};
|
||||
use crate::av1_obu::{read_leb128, ObuHeader};
|
||||
let mut i = 0usize;
|
||||
while i < data.len() {
|
||||
let header = ObuHeader::from_byte(data[i]);
|
||||
@@ -1135,6 +1747,11 @@ mod tests {
|
||||
};
|
||||
assert!(enc.is_keyframe(&[0x65, 0x01]));
|
||||
assert!(!enc.is_keyframe(&[0x41, 0x01]));
|
||||
assert!(enc.is_keyframe(&[
|
||||
0x00, 0x00, 0x00, 0x01, 0x67, 0x01, // SPS
|
||||
0x00, 0x00, 0x00, 0x01, 0x68, 0x02, // PPS
|
||||
0x00, 0x00, 0x00, 0x01, 0x65, 0x03, // IDR
|
||||
]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1155,6 +1772,16 @@ mod tests {
|
||||
assert_eq!(annex_b, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn avcc_to_annexb_passes_through_annexb() {
|
||||
let annex_b = vec![
|
||||
0x00, 0x00, 0x00, 0x01, 0x67, 0x42, 0xC0, 0x1E, 0x00, 0x00, 0x00, 0x01, 0x65, 0x88,
|
||||
0x84, 0x21,
|
||||
];
|
||||
|
||||
assert_eq!(avcc_to_annexb(&annex_b), annex_b);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hevc_mediacodec_encoder_returns_not_initialized_on_non_android() {
|
||||
let enc = MediaCodecHevcEncoder::new(1280, 720, 2_000_000);
|
||||
|
||||
341
crates/wzp-video/src/transport.rs
Normal file
341
crates/wzp-video/src/transport.rs
Normal file
@@ -0,0 +1,341 @@
|
||||
//! Video packet serialization and reassembly on top of [`MediaHeaderV2`].
|
||||
//!
|
||||
//! A single encoded video frame may be far larger than one QUIC datagram
|
||||
//! (~1200 bytes after header and AEAD overhead). This module fragments
|
||||
//! frames into `MediaPacket`s on the send side and reassembles them on the
|
||||
//! receive side.
|
||||
//!
|
||||
//! ## Wire layout
|
||||
//!
|
||||
//! Each fragment uses a standard `MediaHeaderV2` with:
|
||||
//! - `media_type = Video`
|
||||
//! - `codec_id` = the negotiated video codec
|
||||
//! - `FLAG_KEYFRAME` set on all fragments of a keyframe
|
||||
//! - `FLAG_FRAME_END` set on the last fragment of a frame
|
||||
//! - `seq` = monotonic packet sequence number (wrapping u32)
|
||||
//! - `fec_block` = `(fragment_index as u8) << 8 | (fragment_count as u8)`
|
||||
//! where fragment_count = total fragments in this frame (1-based)
|
||||
//!
|
||||
//! Max fragments per frame: 255 → max frame size ≈ 255 × 1150 ≈ 293 KB,
|
||||
//! which covers 1080p keyframes at reasonable quality.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use wzp_proto::{CodecId, MediaHeaderV2, MediaPacket, MediaType};
|
||||
|
||||
/// Maximum video payload bytes per QUIC datagram.
|
||||
/// 1200 (QUIC MTU) − 16 (MediaHeaderV2) − 16 (AEAD tag) = 1168.
|
||||
pub const VIDEO_MAX_PAYLOAD: usize = 1168;
|
||||
|
||||
const VIDEO_FRAME_META_MAGIC: [u8; 4] = *b"WZV1";
|
||||
const VIDEO_FRAME_META_LEN: usize = 8;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct VideoFrameMeta {
|
||||
pub width: u16,
|
||||
pub height: u16,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ReassembledVideoFrame {
|
||||
pub codec_id: CodecId,
|
||||
pub is_keyframe: bool,
|
||||
pub width: Option<u16>,
|
||||
pub height: Option<u16>,
|
||||
pub data: Vec<u8>,
|
||||
}
|
||||
|
||||
/// Fragments one encoded video frame into a sequence of [`MediaPacket`]s.
|
||||
///
|
||||
/// Pass each `MediaPacket` to `transport.send_media()`.
|
||||
pub fn packetize_video_frame(
|
||||
frame: &[u8],
|
||||
codec_id: CodecId,
|
||||
is_keyframe: bool,
|
||||
seq: &mut u32,
|
||||
timestamp_ms: u32,
|
||||
width: u32,
|
||||
height: u32,
|
||||
) -> Vec<MediaPacket> {
|
||||
if frame.is_empty() {
|
||||
return vec![];
|
||||
}
|
||||
|
||||
let mut framed = Vec::with_capacity(VIDEO_FRAME_META_LEN + frame.len());
|
||||
framed.extend_from_slice(&VIDEO_FRAME_META_MAGIC);
|
||||
framed.extend_from_slice(&(width.min(u16::MAX as u32) as u16).to_be_bytes());
|
||||
framed.extend_from_slice(&(height.min(u16::MAX as u32) as u16).to_be_bytes());
|
||||
framed.extend_from_slice(frame);
|
||||
|
||||
let chunks: Vec<&[u8]> = framed.chunks(VIDEO_MAX_PAYLOAD).collect();
|
||||
let total = chunks.len().min(255);
|
||||
let mut packets = Vec::with_capacity(total);
|
||||
|
||||
for (i, chunk) in chunks.iter().enumerate().take(255) {
|
||||
let is_last = i + 1 == total;
|
||||
let mut flags = 0u8;
|
||||
if is_keyframe {
|
||||
flags |= MediaHeaderV2::FLAG_KEYFRAME;
|
||||
}
|
||||
if is_last {
|
||||
flags |= MediaHeaderV2::FLAG_FRAME_END;
|
||||
}
|
||||
|
||||
let fec_block = ((i as u16) << 8) | (total as u16);
|
||||
|
||||
let header = MediaHeaderV2 {
|
||||
version: MediaHeaderV2::VERSION,
|
||||
flags,
|
||||
media_type: MediaType::Video,
|
||||
codec_id,
|
||||
// Legacy relays default receivers to video layer 0. Use video stream
|
||||
// 0 for the single-layer room-video path so packets are forwarded
|
||||
// before any receiver quality state exists. Audio is separated by
|
||||
// media_type, so stream_id 0 does not collide with audio packets.
|
||||
stream_id: 0,
|
||||
fec_ratio: 0,
|
||||
seq: *seq,
|
||||
timestamp: timestamp_ms,
|
||||
fec_block,
|
||||
};
|
||||
*seq = seq.wrapping_add(1);
|
||||
|
||||
let mut buf = BytesMut::with_capacity(MediaHeaderV2::WIRE_SIZE + chunk.len());
|
||||
header.write_to(&mut buf);
|
||||
buf.extend_from_slice(chunk);
|
||||
|
||||
packets.push(MediaPacket {
|
||||
header,
|
||||
payload: Bytes::copy_from_slice(chunk),
|
||||
quality_report: None,
|
||||
});
|
||||
}
|
||||
|
||||
packets
|
||||
}
|
||||
|
||||
/// State for one partially-reassembled video frame.
|
||||
#[derive(Default)]
|
||||
struct PendingFrame {
|
||||
fragments: HashMap<u8, Vec<u8>>,
|
||||
total_fragments: u8,
|
||||
is_keyframe: bool,
|
||||
saw_frame_end: bool,
|
||||
codec_id: Option<CodecId>,
|
||||
}
|
||||
|
||||
/// Reassembles fragmented [`MediaPacket`]s back into complete video frames.
|
||||
///
|
||||
/// Call [`VideoReassembler::push`] for every received video `MediaPacket`.
|
||||
/// It returns a complete frame only when the last fragment (`FLAG_FRAME_END`)
|
||||
/// of a frame arrives and all prior fragments are present.
|
||||
pub struct VideoReassembler {
|
||||
/// Keyed by the timestamp of the frame being assembled.
|
||||
pending: HashMap<u32, PendingFrame>,
|
||||
}
|
||||
|
||||
impl VideoReassembler {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
pending: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Push one received video packet.
|
||||
///
|
||||
/// Returns `Some(frame)` when a complete frame is ready, `None` otherwise.
|
||||
pub fn push(&mut self, pkt: &MediaPacket) -> Option<ReassembledVideoFrame> {
|
||||
let hdr = &pkt.header;
|
||||
let fragment_index = (hdr.fec_block >> 8) as u8;
|
||||
let fragment_count = (hdr.fec_block & 0xFF) as u8;
|
||||
let is_keyframe = hdr.is_keyframe();
|
||||
let is_frame_end = hdr.is_frame_end();
|
||||
|
||||
// Use the packet timestamp as the frame identifier.
|
||||
let entry = self.pending.entry(hdr.timestamp).or_default();
|
||||
entry.fragments.insert(fragment_index, pkt.payload.to_vec());
|
||||
if fragment_count > 0 {
|
||||
entry.total_fragments = fragment_count;
|
||||
}
|
||||
if is_keyframe {
|
||||
entry.is_keyframe = true;
|
||||
}
|
||||
if is_frame_end {
|
||||
entry.saw_frame_end = true;
|
||||
}
|
||||
entry.codec_id = Some(hdr.codec_id);
|
||||
|
||||
// Attempt reassembly once we know the frame end has arrived. The end
|
||||
// fragment can arrive before earlier fragments on QUIC/datagram paths,
|
||||
// so retry on every later fragment instead of only on the end packet.
|
||||
if !entry.saw_frame_end {
|
||||
return None;
|
||||
}
|
||||
|
||||
let total = entry.total_fragments as usize;
|
||||
if total == 0 || entry.fragments.len() < total {
|
||||
// Haven't received all fragments yet; keep waiting.
|
||||
return None;
|
||||
}
|
||||
|
||||
// All fragments present — reassemble in order.
|
||||
let pending = self.pending.remove(&hdr.timestamp)?;
|
||||
let codec_id = pending.codec_id?;
|
||||
let mut frame = Vec::new();
|
||||
for i in 0..total as u8 {
|
||||
frame.extend_from_slice(pending.fragments.get(&i)?);
|
||||
}
|
||||
let (meta, data) = split_video_frame_payload(frame);
|
||||
Some(ReassembledVideoFrame {
|
||||
codec_id,
|
||||
is_keyframe: pending.is_keyframe,
|
||||
width: meta.map(|m| m.width),
|
||||
height: meta.map(|m| m.height),
|
||||
data,
|
||||
})
|
||||
}
|
||||
|
||||
/// Evict stale pending frames older than `max_age_ms` milliseconds.
|
||||
///
|
||||
/// Call periodically (e.g. every 2s) to prevent accumulation of frames
|
||||
/// whose first or middle fragments were lost.
|
||||
pub fn evict_stale(&mut self, current_timestamp_ms: u32, max_age_ms: u32) {
|
||||
self.pending
|
||||
.retain(|&ts, _| current_timestamp_ms.wrapping_sub(ts) <= max_age_ms);
|
||||
}
|
||||
}
|
||||
|
||||
fn split_video_frame_payload(mut frame: Vec<u8>) -> (Option<VideoFrameMeta>, Vec<u8>) {
|
||||
if frame.len() < VIDEO_FRAME_META_LEN || frame[..4] != VIDEO_FRAME_META_MAGIC {
|
||||
return (None, frame);
|
||||
}
|
||||
|
||||
let width = u16::from_be_bytes([frame[4], frame[5]]);
|
||||
let height = u16::from_be_bytes([frame[6], frame[7]]);
|
||||
frame.drain(..VIDEO_FRAME_META_LEN);
|
||||
(Some(VideoFrameMeta { width, height }), frame)
|
||||
}
|
||||
|
||||
impl Default for VideoReassembler {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn make_frame(size: usize) -> Vec<u8> {
|
||||
(0..size).map(|i| (i & 0xFF) as u8).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_fragment_roundtrip() {
|
||||
let frame = make_frame(100);
|
||||
let mut seq = 0u32;
|
||||
let pkts = packetize_video_frame(&frame, CodecId::Av1Main, true, &mut seq, 1000, 640, 480);
|
||||
assert_eq!(pkts.len(), 1);
|
||||
assert!(pkts[0].header.is_keyframe());
|
||||
assert!(pkts[0].header.is_frame_end());
|
||||
assert_eq!(pkts[0].header.media_type, MediaType::Video);
|
||||
assert_eq!(pkts[0].header.stream_id, 0);
|
||||
|
||||
let mut reassembler = VideoReassembler::new();
|
||||
let result = reassembler.push(&pkts[0]);
|
||||
assert!(result.is_some());
|
||||
let result = result.unwrap();
|
||||
assert_eq!(result.codec_id, CodecId::Av1Main);
|
||||
assert!(result.is_keyframe);
|
||||
assert_eq!(result.width, Some(640));
|
||||
assert_eq!(result.height, Some(480));
|
||||
assert_eq!(result.data, frame);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multi_fragment_roundtrip() {
|
||||
let frame = make_frame(VIDEO_MAX_PAYLOAD * 3 + 50);
|
||||
let mut seq = 0u32;
|
||||
let pkts = packetize_video_frame(
|
||||
&frame,
|
||||
CodecId::H264Baseline,
|
||||
false,
|
||||
&mut seq,
|
||||
2000,
|
||||
960,
|
||||
540,
|
||||
);
|
||||
assert_eq!(pkts.len(), 4);
|
||||
assert!(!pkts[0].header.is_frame_end());
|
||||
assert!(pkts[3].header.is_frame_end());
|
||||
assert!(!pkts[0].header.is_keyframe());
|
||||
|
||||
let mut reassembler = VideoReassembler::new();
|
||||
let mut result = None;
|
||||
for pkt in &pkts {
|
||||
result = reassembler.push(pkt);
|
||||
}
|
||||
let result = result.unwrap();
|
||||
assert_eq!(result.codec_id, CodecId::H264Baseline);
|
||||
assert!(!result.is_keyframe);
|
||||
assert_eq!(result.width, Some(960));
|
||||
assert_eq!(result.height, Some(540));
|
||||
assert_eq!(result.data, frame);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn out_of_order_delivery() {
|
||||
let frame = make_frame(VIDEO_MAX_PAYLOAD * 2 + 100);
|
||||
let mut seq = 0u32;
|
||||
let pkts = packetize_video_frame(&frame, CodecId::Av1Main, false, &mut seq, 3000, 320, 240);
|
||||
assert_eq!(pkts.len(), 3);
|
||||
|
||||
let mut reassembler = VideoReassembler::new();
|
||||
// Deliver out of order: 2, 0, 1
|
||||
assert!(reassembler.push(&pkts[2]).is_none()); // last arrives first — no total_fragments yet
|
||||
assert!(reassembler.push(&pkts[0]).is_none());
|
||||
let result = reassembler
|
||||
.push(&pkts[1])
|
||||
.expect("last missing fragment completes frame");
|
||||
assert_eq!(result.codec_id, CodecId::Av1Main);
|
||||
assert!(!result.is_keyframe);
|
||||
assert_eq!(result.width, Some(320));
|
||||
assert_eq!(result.height, Some(240));
|
||||
assert_eq!(result.data, frame);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_frame_produces_no_packets() {
|
||||
let mut seq = 0u32;
|
||||
let pkts = packetize_video_frame(&[], CodecId::Av1Main, false, &mut seq, 0, 640, 480);
|
||||
assert!(pkts.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn old_payload_without_meta_still_reassembles() {
|
||||
let payload = Bytes::copy_from_slice(&[0x00, 0x00, 0x00, 0x01, 0x65]);
|
||||
let pkt = MediaPacket {
|
||||
header: MediaHeaderV2 {
|
||||
version: MediaHeaderV2::VERSION,
|
||||
flags: MediaHeaderV2::FLAG_KEYFRAME | MediaHeaderV2::FLAG_FRAME_END,
|
||||
media_type: MediaType::Video,
|
||||
codec_id: CodecId::H264Baseline,
|
||||
stream_id: 0,
|
||||
fec_ratio: 0,
|
||||
seq: 7,
|
||||
timestamp: 123,
|
||||
fec_block: 1,
|
||||
},
|
||||
payload: payload.clone(),
|
||||
quality_report: None,
|
||||
};
|
||||
|
||||
let mut reassembler = VideoReassembler::new();
|
||||
let frame = reassembler.push(&pkt).unwrap();
|
||||
assert_eq!(frame.codec_id, CodecId::H264Baseline);
|
||||
assert_eq!(frame.width, None);
|
||||
assert_eq!(frame.height, None);
|
||||
assert_eq!(frame.data, payload.to_vec());
|
||||
}
|
||||
}
|
||||
@@ -8,13 +8,110 @@ mod imp {
|
||||
pub use shiguredo_video_toolbox::{
|
||||
CodecConfig, DecodedFrame, Decoder, DecoderCodec, DecoderConfig, EncodeOptions, Encoder,
|
||||
EncoderConfig, FrameData, H264EncoderConfig, H264EntropyMode, H264Profile,
|
||||
HevcEncoderConfig, HevcProfile, PixelFormat,
|
||||
HevcEncoderConfig, HevcProfile, I420Frame, PixelFormat,
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
use imp::*;
|
||||
|
||||
/// Copy a VideoToolbox I420 CVPixelBuffer into a tightly-packed I420 byte vector
|
||||
/// of `width * height + 2 * (width/2) * (height/2)` bytes.
|
||||
///
|
||||
/// The per-plane `bytes_per_row` (stride) reported by CoreVideo can be larger
|
||||
/// than the visible plane width (typically aligned to 16/64 bytes). Concatenating
|
||||
/// the raw plane slices without removing that stride padding produces a buffer
|
||||
/// that downstream code — which indexes as tight I420 of `width x height` —
|
||||
/// mis-interprets, producing horizontal green/magenta bands that drift one
|
||||
/// chroma row each time the per-row stride excess accumulates to one full row.
|
||||
///
|
||||
/// `frame_label` is used for one-time tracing of the actual plane dimensions so
|
||||
/// the first decoded frame of a session prints its real layout. The boolean
|
||||
/// flag is flipped to true after the first log so the format string is emitted
|
||||
/// at most once per decoder lifetime.
|
||||
#[cfg(target_os = "macos")]
|
||||
fn i420_frame_to_tight(
|
||||
frame: &I420Frame<'_>,
|
||||
width: u32,
|
||||
height: u32,
|
||||
frame_label: &'static str,
|
||||
logged: &mut bool,
|
||||
) -> Result<Vec<u8>, VideoError> {
|
||||
let w = width as usize;
|
||||
let h = height as usize;
|
||||
if w == 0 || h == 0 {
|
||||
return Err(VideoError::PlatformError(format!(
|
||||
"decoder produced empty frame ({w}x{h})"
|
||||
)));
|
||||
}
|
||||
let cw = w / 2;
|
||||
let ch = h / 2;
|
||||
|
||||
let y = frame.y_plane();
|
||||
let u = frame.u_plane();
|
||||
let v = frame.v_plane();
|
||||
let y_stride = frame.y_stride();
|
||||
let u_stride = frame.u_stride();
|
||||
let v_stride = frame.v_stride();
|
||||
let fw = frame.width();
|
||||
let fh = frame.height();
|
||||
|
||||
if !*logged {
|
||||
*logged = true;
|
||||
tracing::info!(
|
||||
target: "wzp_video::videotoolbox",
|
||||
label = frame_label,
|
||||
configured_width = w,
|
||||
configured_height = h,
|
||||
frame_width = fw,
|
||||
frame_height = fh,
|
||||
y_stride,
|
||||
u_stride,
|
||||
v_stride,
|
||||
y_len = y.len(),
|
||||
u_len = u.len(),
|
||||
v_len = v.len(),
|
||||
"VideoToolbox decoder I420 plane layout"
|
||||
);
|
||||
}
|
||||
|
||||
if y_stride < w || u_stride < cw || v_stride < cw {
|
||||
return Err(VideoError::PlatformError(format!(
|
||||
"decoder plane stride smaller than width: y_stride={y_stride} u_stride={u_stride} v_stride={v_stride} for {w}x{h}"
|
||||
)));
|
||||
}
|
||||
let needed_y = y_stride.checked_mul(h).ok_or_else(|| {
|
||||
VideoError::PlatformError(format!("y plane size overflow {y_stride}x{h}"))
|
||||
})?;
|
||||
let needed_uv = u_stride.checked_mul(ch).ok_or_else(|| {
|
||||
VideoError::PlatformError(format!("uv plane size overflow {u_stride}x{ch}"))
|
||||
})?;
|
||||
if y.len() < needed_y || u.len() < needed_uv || v.len() < v_stride * ch {
|
||||
return Err(VideoError::PlatformError(format!(
|
||||
"decoder plane buffer too small: y_len={} (need {needed_y}) u_len={} (need {needed_uv}) v_len={} (need {})",
|
||||
y.len(),
|
||||
u.len(),
|
||||
v.len(),
|
||||
v_stride * ch,
|
||||
)));
|
||||
}
|
||||
|
||||
let mut data = Vec::with_capacity(w * h + 2 * cw * ch);
|
||||
for row in 0..h {
|
||||
let off = row * y_stride;
|
||||
data.extend_from_slice(&y[off..off + w]);
|
||||
}
|
||||
for row in 0..ch {
|
||||
let off = row * u_stride;
|
||||
data.extend_from_slice(&u[off..off + cw]);
|
||||
}
|
||||
for row in 0..ch {
|
||||
let off = row * v_stride;
|
||||
data.extend_from_slice(&v[off..off + cw]);
|
||||
}
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
/// macOS VideoToolbox H.264 encoder.
|
||||
///
|
||||
/// Wraps `VTCompressionSession`. On non-macOS targets this is a compile-safe
|
||||
@@ -160,9 +257,12 @@ impl VideoEncoder for VideoToolboxEncoder {
|
||||
if packet.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let nal_type = packet[0] & 0x1F;
|
||||
// NAL type 5 = IDR slice (keyframe).
|
||||
nal_type == 5
|
||||
let nals = split_annex_b(packet);
|
||||
if nals.is_empty() {
|
||||
return (packet[0] & 0x1F) == 5;
|
||||
}
|
||||
nals.iter()
|
||||
.any(|nal| !nal.is_empty() && (nal[0] & 0x1F) == 5)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -261,6 +361,8 @@ pub struct VideoToolboxDecoder {
|
||||
width: u32,
|
||||
#[cfg(target_os = "macos")]
|
||||
height: u32,
|
||||
#[cfg(target_os = "macos")]
|
||||
layout_logged: bool,
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
_width: u32,
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
@@ -279,6 +381,7 @@ impl VideoToolboxDecoder {
|
||||
inner: None,
|
||||
width,
|
||||
height,
|
||||
layout_logged: false,
|
||||
})
|
||||
}
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
@@ -357,13 +460,13 @@ impl VideoDecoder for VideoToolboxDecoder {
|
||||
|
||||
match decoded {
|
||||
Some(DecodedFrame::I420(frame)) => {
|
||||
let y = frame.y_plane();
|
||||
let u = frame.u_plane();
|
||||
let v = frame.v_plane();
|
||||
let mut data = Vec::with_capacity(y.len() + u.len() + v.len());
|
||||
data.extend_from_slice(y);
|
||||
data.extend_from_slice(u);
|
||||
data.extend_from_slice(v);
|
||||
let data = i420_frame_to_tight(
|
||||
&frame,
|
||||
self.width,
|
||||
self.height,
|
||||
"h264_decoder",
|
||||
&mut self.layout_logged,
|
||||
)?;
|
||||
Ok(Some(VideoFrame {
|
||||
width: self.width,
|
||||
height: self.height,
|
||||
@@ -520,12 +623,13 @@ impl VideoEncoder for VideoToolboxHevcEncoder {
|
||||
}
|
||||
|
||||
fn is_keyframe(&self, packet: &[u8]) -> bool {
|
||||
if packet.len() < 2 {
|
||||
return false;
|
||||
let nals = split_annex_b(packet);
|
||||
if nals.is_empty() {
|
||||
return packet.len() >= 2 && matches!((packet[0] >> 1) & 0x3F, 19 | 20);
|
||||
}
|
||||
let nal_type = (packet[0] >> 1) & 0x3F;
|
||||
// NAL type 19 = IDR_W_RADL, 20 = IDR_N_LP.
|
||||
nal_type == 19 || nal_type == 20
|
||||
nals.iter()
|
||||
.any(|nal| nal.len() >= 2 && matches!((nal[0] >> 1) & 0x3F, 19 | 20))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -537,6 +641,8 @@ pub struct VideoToolboxHevcDecoder {
|
||||
width: u32,
|
||||
#[cfg(target_os = "macos")]
|
||||
height: u32,
|
||||
#[cfg(target_os = "macos")]
|
||||
layout_logged: bool,
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
_width: u32,
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
@@ -551,6 +657,7 @@ impl VideoToolboxHevcDecoder {
|
||||
inner: None,
|
||||
width,
|
||||
height,
|
||||
layout_logged: false,
|
||||
})
|
||||
}
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
@@ -624,13 +731,13 @@ impl VideoDecoder for VideoToolboxHevcDecoder {
|
||||
|
||||
match decoded {
|
||||
Some(DecodedFrame::I420(frame)) => {
|
||||
let y = frame.y_plane();
|
||||
let u = frame.u_plane();
|
||||
let v = frame.v_plane();
|
||||
let mut data = Vec::with_capacity(y.len() + u.len() + v.len());
|
||||
data.extend_from_slice(y);
|
||||
data.extend_from_slice(u);
|
||||
data.extend_from_slice(v);
|
||||
let data = i420_frame_to_tight(
|
||||
&frame,
|
||||
self.width,
|
||||
self.height,
|
||||
"hevc_decoder",
|
||||
&mut self.layout_logged,
|
||||
)?;
|
||||
Ok(Some(VideoFrame {
|
||||
width: self.width,
|
||||
height: self.height,
|
||||
@@ -660,6 +767,8 @@ pub struct VideoToolboxAv1Decoder {
|
||||
width: u32,
|
||||
#[cfg(target_os = "macos")]
|
||||
height: u32,
|
||||
#[cfg(target_os = "macos")]
|
||||
layout_logged: bool,
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
_width: u32,
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
@@ -679,6 +788,7 @@ impl VideoToolboxAv1Decoder {
|
||||
inner: Some(decoder),
|
||||
width,
|
||||
height,
|
||||
layout_logged: false,
|
||||
}),
|
||||
Err(shiguredo_video_toolbox::Error::UnsupportedCodec { .. }) => {
|
||||
// AV1 decode not supported on this platform (e.g. M1/M2).
|
||||
@@ -686,6 +796,7 @@ impl VideoToolboxAv1Decoder {
|
||||
inner: None,
|
||||
width,
|
||||
height,
|
||||
layout_logged: false,
|
||||
})
|
||||
}
|
||||
Err(e) => Err(VideoError::PlatformError(format!(
|
||||
@@ -717,13 +828,13 @@ impl VideoDecoder for VideoToolboxAv1Decoder {
|
||||
.map_err(|e| VideoError::PlatformError(format!("decode failed: {e}")))?;
|
||||
match decoded {
|
||||
Some(DecodedFrame::I420(frame)) => {
|
||||
let y = frame.y_plane();
|
||||
let u = frame.u_plane();
|
||||
let v = frame.v_plane();
|
||||
let mut data = Vec::with_capacity(y.len() + u.len() + v.len());
|
||||
data.extend_from_slice(y);
|
||||
data.extend_from_slice(u);
|
||||
data.extend_from_slice(v);
|
||||
let data = i420_frame_to_tight(
|
||||
&frame,
|
||||
self.width,
|
||||
self.height,
|
||||
"av1_decoder",
|
||||
&mut self.layout_logged,
|
||||
)?;
|
||||
Ok(Some(VideoFrame {
|
||||
width: self.width,
|
||||
height: self.height,
|
||||
@@ -791,6 +902,11 @@ mod tests {
|
||||
let enc = VideoToolboxEncoder::new(1280, 720, 2_000_000).unwrap();
|
||||
assert!(enc.is_keyframe(&[0x65, 0x01, 0x02]));
|
||||
assert!(!enc.is_keyframe(&[0x41, 0x01, 0x02]));
|
||||
assert!(enc.is_keyframe(&[
|
||||
0x00, 0x00, 0x00, 0x01, 0x67, 0x01, // SPS
|
||||
0x00, 0x00, 0x00, 0x01, 0x68, 0x02, // PPS
|
||||
0x00, 0x00, 0x00, 0x01, 0x65, 0x03, // IDR
|
||||
]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
276
crates/wzp-video/tests/pipeline_roundtrip.rs
Normal file
276
crates/wzp-video/tests/pipeline_roundtrip.rs
Normal file
@@ -0,0 +1,276 @@
|
||||
//! Full-stack video pipeline integration test.
|
||||
//!
|
||||
//! Exercises every layer of the Blocker 1–3 implementation end-to-end:
|
||||
//!
|
||||
//! factory::create_video_encoder
|
||||
//! → encoder.encode()
|
||||
//! → transport::packetize_video_frame
|
||||
//! → VideoReassembler::push
|
||||
//! → factory::create_video_decoder
|
||||
//! → decoder.decode()
|
||||
//!
|
||||
//! Runs only on macOS (VideoToolbox encoders / decoders).
|
||||
|
||||
#![cfg(target_os = "macos")]
|
||||
|
||||
use std::sync::Mutex;
|
||||
use wzp_proto::CodecId;
|
||||
use wzp_video::{
|
||||
factory::{create_video_decoder, create_video_encoder},
|
||||
transport::{packetize_video_frame, VideoReassembler},
|
||||
VideoFrame,
|
||||
};
|
||||
|
||||
/// VideoToolbox has global session registry state — serialise integration tests
|
||||
/// to avoid races when multiple sessions open concurrently.
|
||||
static VT_LOCK: Mutex<()> = Mutex::new(());
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
fn synthetic_i420(width: u32, height: u32, frame_idx: u32) -> VideoFrame {
|
||||
let y_size = (width * height) as usize;
|
||||
let uv_size = y_size / 4;
|
||||
let mut data = vec![0u8; y_size + 2 * uv_size];
|
||||
|
||||
for y in 0..height {
|
||||
for x in 0..width {
|
||||
// Shift the gradient by frame_idx so successive frames differ.
|
||||
let val = (((x + frame_idx) * 255) / width) as u8;
|
||||
data[(y * width + x) as usize] = val;
|
||||
}
|
||||
}
|
||||
data[y_size..y_size + uv_size].fill(128);
|
||||
data[y_size + uv_size..].fill(128);
|
||||
|
||||
VideoFrame {
|
||||
width,
|
||||
height,
|
||||
data,
|
||||
timestamp_ms: frame_idx as u64 * 33,
|
||||
}
|
||||
}
|
||||
|
||||
// ── tests ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Encode → packetize → reassemble → decode round-trip for H.264 Baseline.
|
||||
#[test]
|
||||
fn h264_pipeline_roundtrip() {
|
||||
let _g = VT_LOCK.lock().unwrap();
|
||||
let (w, h) = (640, 360);
|
||||
|
||||
let mut encoder =
|
||||
create_video_encoder(CodecId::H264Baseline, w, h, 1_500_000).expect("H264Baseline encoder");
|
||||
let mut decoder =
|
||||
create_video_decoder(CodecId::H264Baseline, w, h).expect("H264Baseline decoder");
|
||||
|
||||
let mut seq = 0u32;
|
||||
let mut decoded_count = 0usize;
|
||||
|
||||
encoder.request_keyframe();
|
||||
|
||||
for i in 0..30u32 {
|
||||
let frame = synthetic_i420(w, h, i);
|
||||
let encoded = encoder.encode(&frame).expect("encode");
|
||||
if encoded.is_empty() {
|
||||
continue; // codec may buffer
|
||||
}
|
||||
|
||||
let is_keyframe = encoder.is_keyframe(&encoded);
|
||||
let pkts = packetize_video_frame(
|
||||
&encoded,
|
||||
CodecId::H264Baseline,
|
||||
is_keyframe,
|
||||
&mut seq,
|
||||
i * 33,
|
||||
w,
|
||||
h,
|
||||
);
|
||||
assert!(
|
||||
!pkts.is_empty(),
|
||||
"packetize must produce at least one packet"
|
||||
);
|
||||
|
||||
// All fragments for this frame share the same timestamp.
|
||||
let ts = pkts[0].header.timestamp;
|
||||
let total_frags = pkts.len();
|
||||
for (idx, pkt) in pkts.iter().enumerate() {
|
||||
assert_eq!(
|
||||
pkt.header.timestamp, ts,
|
||||
"all fragments of one frame share timestamp"
|
||||
);
|
||||
let frag_idx = (pkt.header.fec_block >> 8) as usize;
|
||||
let frag_total = (pkt.header.fec_block & 0xFF) as usize;
|
||||
assert_eq!(frag_idx, idx, "fragment index must match packet position");
|
||||
assert_eq!(
|
||||
frag_total, total_frags,
|
||||
"all fragments carry the correct total count"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
pkts.last().unwrap().header.is_frame_end(),
|
||||
"last packet must have FLAG_FRAME_END"
|
||||
);
|
||||
|
||||
// Push through reassembler — only the last packet should yield a frame.
|
||||
let mut reassembler = VideoReassembler::new();
|
||||
for (j, pkt) in pkts.iter().enumerate() {
|
||||
let result = reassembler.push(pkt);
|
||||
if j + 1 < pkts.len() {
|
||||
assert!(
|
||||
result.is_none(),
|
||||
"intermediate fragments must not yield a complete frame"
|
||||
);
|
||||
} else {
|
||||
let frame = result.expect("last fragment must complete the frame");
|
||||
assert_eq!(frame.codec_id, CodecId::H264Baseline);
|
||||
assert_eq!(frame.is_keyframe, is_keyframe);
|
||||
assert_eq!(frame.width, Some(w as u16));
|
||||
assert_eq!(frame.height, Some(h as u16));
|
||||
assert_eq!(
|
||||
frame.data, encoded,
|
||||
"reassembled bytes must match original encoded bytes"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Decode the reassembled frame.
|
||||
match decoder.decode(&encoded) {
|
||||
Ok(Some(yuv)) => {
|
||||
assert_eq!(yuv.width, w);
|
||||
assert_eq!(yuv.height, h);
|
||||
let expected_size = (w * h * 3 / 2) as usize;
|
||||
assert!(
|
||||
yuv.data.len() >= expected_size,
|
||||
"decoded I420 too small: {} < {expected_size}",
|
||||
yuv.data.len()
|
||||
);
|
||||
decoded_count += 1;
|
||||
}
|
||||
Ok(None) => {} // pipeline latency — decoder still buffering
|
||||
Err(e) => panic!("decode error: {e}"),
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
decoded_count > 0,
|
||||
"at least one frame must have been decoded"
|
||||
);
|
||||
}
|
||||
|
||||
/// Fragmentation: a frame larger than VIDEO_MAX_PAYLOAD splits into multiple packets,
|
||||
/// all of which reassemble back to the original bytes.
|
||||
#[test]
|
||||
fn large_frame_fragments_and_reassembles() {
|
||||
use wzp_video::transport::VIDEO_MAX_PAYLOAD;
|
||||
|
||||
// Craft a fake "encoded" blob larger than one MTU.
|
||||
let synthetic_encoded: Vec<u8> = (0..VIDEO_MAX_PAYLOAD * 3 + 200)
|
||||
.map(|i| (i & 0xFF) as u8)
|
||||
.collect();
|
||||
|
||||
let mut seq = 0u32;
|
||||
let pkts = packetize_video_frame(
|
||||
&synthetic_encoded,
|
||||
CodecId::H264Baseline,
|
||||
true,
|
||||
&mut seq,
|
||||
9000,
|
||||
1280,
|
||||
720,
|
||||
);
|
||||
|
||||
assert!(pkts.len() >= 4, "large frame must produce ≥4 fragments");
|
||||
assert!(
|
||||
pkts[0].header.is_keyframe(),
|
||||
"keyframe flag propagates to all fragments"
|
||||
);
|
||||
assert!(
|
||||
!pkts[0].header.is_frame_end(),
|
||||
"first packet is not frame end"
|
||||
);
|
||||
assert!(
|
||||
pkts.last().unwrap().header.is_frame_end(),
|
||||
"last packet is frame end"
|
||||
);
|
||||
|
||||
let mut reassembler = VideoReassembler::new();
|
||||
let mut result = None;
|
||||
for pkt in &pkts {
|
||||
result = reassembler.push(pkt);
|
||||
}
|
||||
|
||||
let frame = result.expect("all fragments delivered → complete frame");
|
||||
assert_eq!(frame.width, Some(1280));
|
||||
assert_eq!(frame.height, Some(720));
|
||||
assert_eq!(
|
||||
frame.data, synthetic_encoded,
|
||||
"reassembled bytes must match input exactly"
|
||||
);
|
||||
}
|
||||
|
||||
/// Packet loss: if the first fragment is missing, reassembly cannot complete.
|
||||
#[test]
|
||||
fn missing_fragment_blocks_reassembly() {
|
||||
use wzp_video::transport::VIDEO_MAX_PAYLOAD;
|
||||
|
||||
let frame: Vec<u8> = vec![0xAB; VIDEO_MAX_PAYLOAD * 2 + 50];
|
||||
let mut seq = 0u32;
|
||||
let pkts = packetize_video_frame(&frame, CodecId::Av1Main, false, &mut seq, 1234, 640, 480);
|
||||
assert!(pkts.len() >= 3);
|
||||
|
||||
let mut reassembler = VideoReassembler::new();
|
||||
// Skip fragment 0 — deliver 1 and 2.
|
||||
for pkt in &pkts[1..] {
|
||||
let r = reassembler.push(pkt);
|
||||
assert!(r.is_none(), "incomplete set must not yield a frame");
|
||||
}
|
||||
}
|
||||
|
||||
/// Codec negotiation smoke test: relay picks first offered codec.
|
||||
///
|
||||
/// This keeps codec-selection logic exercised at the transport layer even though
|
||||
/// the real negotiation happens in wzp-relay/wzp-client handshakes.
|
||||
#[test]
|
||||
fn video_codec_selection_semantics() {
|
||||
// The relay's selection rule is: first codec offered by the caller.
|
||||
let offered = vec![CodecId::H264Baseline];
|
||||
let chosen = offered.into_iter().next();
|
||||
assert_eq!(chosen, Some(CodecId::H264Baseline));
|
||||
|
||||
// When no codecs are offered, video is audio-only.
|
||||
let empty: Vec<CodecId> = vec![];
|
||||
assert_eq!(empty.into_iter().next(), None);
|
||||
}
|
||||
|
||||
/// Evict-stale does not panic and removes old frames.
|
||||
#[test]
|
||||
fn evict_stale_removes_aged_frames() {
|
||||
use wzp_video::transport::VIDEO_MAX_PAYLOAD;
|
||||
|
||||
let frame: Vec<u8> = vec![0x55; VIDEO_MAX_PAYLOAD * 2];
|
||||
let mut seq = 0u32;
|
||||
let pkts = packetize_video_frame(
|
||||
&frame,
|
||||
CodecId::H264Baseline,
|
||||
false,
|
||||
&mut seq,
|
||||
500,
|
||||
640,
|
||||
480,
|
||||
);
|
||||
|
||||
let mut reassembler = VideoReassembler::new();
|
||||
// Push only first packet — frame is incomplete.
|
||||
reassembler.push(&pkts[0]);
|
||||
|
||||
// Evict frames older than 1000 ms; current timestamp is 10000.
|
||||
reassembler.evict_stale(10_000, 1_000);
|
||||
|
||||
// Pushing the rest now must not complete a frame (state was evicted).
|
||||
for pkt in &pkts[1..] {
|
||||
let r = reassembler.push(pkt);
|
||||
// May or may not reassemble depending on reassembler's handling
|
||||
// of a new frame with the same timestamp — mainly verify no panic.
|
||||
let _ = r;
|
||||
}
|
||||
}
|
||||
@@ -43,12 +43,16 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Voice join FAB -->
|
||||
<!-- Voice / Video join FABs -->
|
||||
<div class="lobby-fab-row">
|
||||
<button id="join-voice-btn" class="fab" title="Join Voice Chat">
|
||||
<span class="fab-icon">🎧</span>
|
||||
<span class="fab-label">Join Voice</span>
|
||||
</button>
|
||||
<button id="join-video-btn" class="fab fab-video" title="Join with Video">
|
||||
<span class="fab-icon">📹</span>
|
||||
<span class="fab-label">Join Video</span>
|
||||
</button>
|
||||
</div>
|
||||
|
||||
<!-- Incoming call banner -->
|
||||
@@ -84,6 +88,9 @@
|
||||
<button id="vd-spk-btn" class="vd-btn" title="Speaker (s)">
|
||||
<span id="vd-spk-icon">Spk</span>
|
||||
</button>
|
||||
<button id="vd-cam-btn" class="vd-btn" title="Camera (v)">
|
||||
<span id="vd-cam-icon">Cam</span>
|
||||
</button>
|
||||
<button id="vd-end-btn" class="vd-btn vd-end" title="Leave voice (q)">
|
||||
<span>End</span>
|
||||
</button>
|
||||
@@ -99,6 +106,16 @@
|
||||
</div>
|
||||
<div id="vd-stats" class="vd-stats"></div>
|
||||
</div>
|
||||
|
||||
<!-- ═════ Video stage — full-screen overlay above drawer ═════ -->
|
||||
<div id="vd-video-strip" class="vd-video-stage hidden">
|
||||
<canvas id="vd-remote-video" class="vd-remote-stage" width="1280" height="720"></canvas>
|
||||
<div id="vd-remote-placeholder" class="vd-remote-placeholder">
|
||||
<div class="vd-placeholder-text">Waiting for remote video…</div>
|
||||
<div id="vd-remote-counter" class="vd-placeholder-sub">0 frames received</div>
|
||||
</div>
|
||||
<video id="vd-local-video" class="vd-local-pip" autoplay muted playsinline></video>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ═══════════════════════════════════════════════════════
|
||||
@@ -157,6 +174,22 @@
|
||||
OS Echo Cancellation
|
||||
</label>
|
||||
</div>
|
||||
<div class="settings-section">
|
||||
<h3>Video</h3>
|
||||
<label>Codec
|
||||
<select id="s-video-codec">
|
||||
<option value="h264">H.264</option>
|
||||
<option value="h265">H.265 / HEVC</option>
|
||||
</select>
|
||||
</label>
|
||||
<label>Room Resolution
|
||||
<select id="s-video-resolution">
|
||||
<option value="640x360">640 x 360</option>
|
||||
<option value="960x540">960 x 540</option>
|
||||
<option value="1280x720">1280 x 720</option>
|
||||
</select>
|
||||
</label>
|
||||
</div>
|
||||
<div class="settings-section">
|
||||
<h3>Relays</h3>
|
||||
<div id="s-relay-list"></div>
|
||||
|
||||
@@ -44,6 +44,9 @@ tracing = "0.1"
|
||||
tracing-subscriber = "0.3"
|
||||
anyhow = "1"
|
||||
rustls = { version = "0.23", default-features = false, features = ["ring", "std"] }
|
||||
# JPEG encoding for video:frame events (I420 → RGB → JPEG for IPC to WebView)
|
||||
image = { version = "0.25", default-features = false, features = ["jpeg"] }
|
||||
base64 = "0.22"
|
||||
|
||||
# WarzonePhone crates — protocol layer is platform-independent
|
||||
wzp-proto = { path = "../../crates/wzp-proto" }
|
||||
@@ -51,6 +54,7 @@ wzp-codec = { path = "../../crates/wzp-codec" }
|
||||
wzp-fec = { path = "../../crates/wzp-fec" }
|
||||
wzp-crypto = { path = "../../crates/wzp-crypto" }
|
||||
wzp-transport = { path = "../../crates/wzp-transport" }
|
||||
wzp-video = { path = "../../crates/wzp-video" }
|
||||
|
||||
# wzp-client pulls in CPAL on every desktop target and, additionally on
|
||||
# macOS, VoiceProcessingIO (coreaudio-rs behind the "vpio" feature). The
|
||||
@@ -99,6 +103,10 @@ libloading = "0.8"
|
||||
jni = "0.21"
|
||||
ndk-context = "0.1"
|
||||
|
||||
[dev-dependencies]
|
||||
bytes = "1"
|
||||
async-trait = "0.1"
|
||||
|
||||
[features]
|
||||
default = ["custom-protocol"]
|
||||
custom-protocol = ["tauri/custom-protocol"]
|
||||
|
||||
@@ -17,5 +17,7 @@
|
||||
-->
|
||||
<key>NSMicrophoneUsageDescription</key>
|
||||
<string>WarzonePhone needs microphone access to transmit your voice during calls.</string>
|
||||
<key>NSCameraUsageDescription</key>
|
||||
<string>WarzonePhone needs camera access for video calls.</string>
|
||||
</dict>
|
||||
</plist>
|
||||
|
||||
@@ -3,7 +3,9 @@
|
||||
<uses-permission android:name="android.permission.INTERNET" />
|
||||
<uses-permission android:name="android.permission.RECORD_AUDIO" />
|
||||
<uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS" />
|
||||
<uses-permission android:name="android.permission.CAMERA" />
|
||||
<uses-feature android:name="android.hardware.microphone" android:required="true" />
|
||||
<uses-feature android:name="android.hardware.camera" android:required="false" />
|
||||
|
||||
<!-- AndroidTV support -->
|
||||
<uses-feature android:name="android.software.leanback" android:required="false" />
|
||||
|
||||
@@ -16,10 +16,19 @@ class MainActivity : TauriActivity() {
|
||||
private const val AUDIO_PERMISSIONS_REQUEST = 4242
|
||||
private val REQUIRED_AUDIO_PERMISSIONS = arrayOf(
|
||||
Manifest.permission.RECORD_AUDIO,
|
||||
Manifest.permission.MODIFY_AUDIO_SETTINGS
|
||||
Manifest.permission.MODIFY_AUDIO_SETTINGS,
|
||||
Manifest.permission.CAMERA
|
||||
)
|
||||
}
|
||||
|
||||
// NOTE: granting CAMERA at the Android system layer is necessary but NOT
|
||||
// sufficient for video on Android. Tauri/Wry's internal WebChromeClient
|
||||
// does not currently grant `getUserMedia` permission requests, so the
|
||||
// browser-layer getUserMedia call still fails even after the OS grants
|
||||
// CAMERA. Fixing this needs either a Tauri plugin that overrides the
|
||||
// WebChromeClient, or a native Camera2/CameraX capture path that bypasses
|
||||
// the WebView. Tracked as a follow-up.
|
||||
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
enableEdgeToEdge()
|
||||
super.onCreate(savedInstanceState)
|
||||
|
||||
@@ -99,9 +99,7 @@ pub fn set_audio_mode_communication() -> Result<(), String> {
|
||||
/// Run `set_audio_mode_communication` on Tauri's main thread, where the
|
||||
/// Android context is initialized. Calling it from arbitrary Tokio blocking
|
||||
/// workers panics inside `ndk_context::android_context()`.
|
||||
pub async fn set_audio_mode_communication_on_main(
|
||||
app: tauri::AppHandle,
|
||||
) -> Result<(), String> {
|
||||
pub async fn set_audio_mode_communication_on_main(app: tauri::AppHandle) -> Result<(), String> {
|
||||
let (tx, rx) = tokio::sync::oneshot::channel();
|
||||
app.run_on_main_thread(move || {
|
||||
let result = std::panic::catch_unwind(set_audio_mode_communication)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -31,7 +31,7 @@ use engine::CallEngine;
|
||||
|
||||
use serde::Serialize;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use tauri::{Emitter, Manager};
|
||||
use tokio::sync::Mutex;
|
||||
@@ -49,6 +49,12 @@ use wzp_proto::{MediaTransport, default_signal_version};
|
||||
// Mirrors the existing `wzp_codec::dred_verbose_logs` pattern.
|
||||
|
||||
static CALL_DEBUG_LOGS: AtomicBool = AtomicBool::new(false);
|
||||
static CAMERA_PUSH_FRAMES: AtomicU64 = AtomicU64::new(0);
|
||||
static CAMERA_PUSH_DROPS: AtomicU64 = AtomicU64::new(0);
|
||||
static CAMERA_PUSH_NO_ENGINE: AtomicU64 = AtomicU64::new(0);
|
||||
static CAMERA_PUSH_NO_SENDER: AtomicU64 = AtomicU64::new(0);
|
||||
static CAMERA_PUSH_DECODE_ERRORS: AtomicU64 = AtomicU64::new(0);
|
||||
static FRAME_DUMP_WRITES: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
#[inline]
|
||||
fn call_debug_logs_enabled() -> bool {
|
||||
@@ -81,9 +87,470 @@ pub(crate) fn emit_call_debug(app: &tauri::AppHandle, step: &str, details: serde
|
||||
let _ = app.emit("call-debug-log", payload);
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
fn call_debug_log(app: tauri::AppHandle, step: String, details: serde_json::Value) {
|
||||
if step == "camera:get_user_media_start" {
|
||||
CAMERA_PUSH_FRAMES.store(0, Ordering::Relaxed);
|
||||
CAMERA_PUSH_DROPS.store(0, Ordering::Relaxed);
|
||||
CAMERA_PUSH_NO_ENGINE.store(0, Ordering::Relaxed);
|
||||
CAMERA_PUSH_NO_SENDER.store(0, Ordering::Relaxed);
|
||||
CAMERA_PUSH_DECODE_ERRORS.store(0, Ordering::Relaxed);
|
||||
}
|
||||
emit_call_debug(&app, &step, details);
|
||||
}
|
||||
|
||||
/// Short git hash captured at compile time by build.rs.
|
||||
const GIT_HASH: &str = env!("WZP_GIT_HASH");
|
||||
|
||||
// ─── Video helpers ────────────────────────────────────────────────────────────
|
||||
|
||||
/// Convert an I420 frame to a JPEG and base64-encode it for IPC.
|
||||
///
|
||||
/// Returns `None` if the data is too short or encoding fails.
|
||||
/// Called from the video recv task in engine.rs to produce the `jpeg_b64`
|
||||
/// field of every `video:frame` Tauri event.
|
||||
#[cfg_attr(not(test), allow(dead_code))]
|
||||
pub(crate) fn i420_to_jpeg_b64(data: &[u8], width: u32, height: u32) -> Option<String> {
|
||||
use base64::Engine as _;
|
||||
|
||||
let bytes = i420_to_jpeg_bytes(data, width, height)?;
|
||||
Some(base64::engine::general_purpose::STANDARD.encode(bytes))
|
||||
}
|
||||
|
||||
pub(crate) fn i420_to_jpeg_bytes(data: &[u8], width: u32, height: u32) -> Option<Vec<u8>> {
|
||||
use image::{DynamicImage, ImageBuffer, Rgb};
|
||||
|
||||
let w = width as usize;
|
||||
let h = height as usize;
|
||||
let y_size = w * h;
|
||||
let uv_size = w * h / 4;
|
||||
|
||||
if data.len() < y_size + 2 * uv_size {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut rgb = vec![0u8; w * h * 3];
|
||||
for row in 0..h {
|
||||
for col in 0..w {
|
||||
let y = data[row * w + col] as f32;
|
||||
let uv_idx = (row / 2) * (w / 2) + col / 2;
|
||||
let u = data[y_size + uv_idx] as f32 - 128.0;
|
||||
let v = data[y_size + uv_size + uv_idx] as f32 - 128.0;
|
||||
let out = (row * w + col) * 3;
|
||||
rgb[out] = (y + 1.402 * v).clamp(0.0, 255.0) as u8;
|
||||
rgb[out + 1] = (y - 0.344 * u - 0.714 * v).clamp(0.0, 255.0) as u8;
|
||||
rgb[out + 2] = (y + 1.772 * u).clamp(0.0, 255.0) as u8;
|
||||
}
|
||||
}
|
||||
|
||||
let img = DynamicImage::ImageRgb8(ImageBuffer::<Rgb<u8>, Vec<u8>>::from_raw(
|
||||
width, height, rgb,
|
||||
)?);
|
||||
let mut buf = std::io::Cursor::new(Vec::<u8>::new());
|
||||
img.write_to(&mut buf, image::ImageFormat::Jpeg).ok()?;
|
||||
Some(buf.into_inner())
|
||||
}
|
||||
|
||||
fn should_dump_frame(frame_no: u64) -> bool {
|
||||
frame_no <= 5 || frame_no % 30 == 0
|
||||
}
|
||||
|
||||
pub(crate) fn maybe_dump_video_jpeg(
|
||||
app: &tauri::AppHandle,
|
||||
stage: &str,
|
||||
platform: &str,
|
||||
frame_no: u64,
|
||||
jpeg_bytes: &[u8],
|
||||
width: u32,
|
||||
height: u32,
|
||||
) {
|
||||
if !should_dump_frame(frame_no) {
|
||||
return;
|
||||
}
|
||||
let seq = FRAME_DUMP_WRITES.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
let dir = identity_dir().join("frame-dumps");
|
||||
let file_name = format!("{seq:06}_{platform}_{stage}_f{frame_no:06}_{width}x{height}.jpg");
|
||||
let path = dir.join(file_name);
|
||||
let result = std::fs::create_dir_all(&dir).and_then(|_| std::fs::write(&path, jpeg_bytes));
|
||||
|
||||
match result {
|
||||
Ok(()) => emit_call_debug(
|
||||
app,
|
||||
"video:frame_dump",
|
||||
serde_json::json!({
|
||||
"stage": stage,
|
||||
"platform": platform,
|
||||
"frame_no": frame_no,
|
||||
"width": width,
|
||||
"height": height,
|
||||
"jpeg_bytes": jpeg_bytes.len(),
|
||||
"path": path,
|
||||
}),
|
||||
),
|
||||
Err(e) => {
|
||||
if seq <= 5 || seq % 30 == 0 {
|
||||
emit_call_debug(
|
||||
app,
|
||||
"video:frame_dump_failed",
|
||||
serde_json::json!({
|
||||
"stage": stage,
|
||||
"platform": platform,
|
||||
"frame_no": frame_no,
|
||||
"error": e.to_string(),
|
||||
"path": path,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn maybe_dump_video_bytes(
|
||||
app: &tauri::AppHandle,
|
||||
stage: &str,
|
||||
platform: &str,
|
||||
frame_no: u64,
|
||||
bytes: &[u8],
|
||||
codec: wzp_proto::CodecId,
|
||||
) {
|
||||
if !should_dump_frame(frame_no) || bytes.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let ext = match codec {
|
||||
wzp_proto::CodecId::H265Main => "h265",
|
||||
wzp_proto::CodecId::Av1Main => "obu",
|
||||
_ => "h264",
|
||||
};
|
||||
let seq = FRAME_DUMP_WRITES.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
let dir = identity_dir().join("frame-dumps");
|
||||
let file_name = format!("{seq:06}_{platform}_{stage}_f{frame_no:06}.{ext}");
|
||||
let path = dir.join(file_name);
|
||||
let result = std::fs::create_dir_all(&dir).and_then(|_| std::fs::write(&path, bytes));
|
||||
|
||||
match result {
|
||||
Ok(()) => emit_call_debug(
|
||||
app,
|
||||
"video:byte_dump",
|
||||
serde_json::json!({
|
||||
"stage": stage,
|
||||
"platform": platform,
|
||||
"frame_no": frame_no,
|
||||
"codec": format!("{:?}", codec),
|
||||
"bytes": bytes.len(),
|
||||
"path": path,
|
||||
}),
|
||||
),
|
||||
Err(e) => {
|
||||
if seq <= 5 || seq % 30 == 0 {
|
||||
emit_call_debug(
|
||||
app,
|
||||
"video:byte_dump_failed",
|
||||
serde_json::json!({
|
||||
"stage": stage,
|
||||
"platform": platform,
|
||||
"frame_no": frame_no,
|
||||
"codec": format!("{:?}", codec),
|
||||
"error": e.to_string(),
|
||||
"path": path,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// RGB24 → I420 (planar 4:2:0). Layout: Y(w×h) | U(w/2×h/2) | V(w/2×h/2).
|
||||
fn rgb_to_i420(rgb: &[u8], w: usize, h: usize) -> Vec<u8> {
|
||||
let y_size = w * h;
|
||||
let uv_size = (w / 2) * (h / 2);
|
||||
let mut out = vec![0u8; y_size + 2 * uv_size];
|
||||
for row in 0..h {
|
||||
for col in 0..w {
|
||||
let i = (row * w + col) * 3;
|
||||
let r = rgb[i] as f32;
|
||||
let g = rgb[i + 1] as f32;
|
||||
let b = rgb[i + 2] as f32;
|
||||
out[row * w + col] = (0.299 * r + 0.587 * g + 0.114 * b).clamp(0.0, 255.0) as u8;
|
||||
if row % 2 == 0 && col % 2 == 0 {
|
||||
let uv = (row / 2) * (w / 2) + col / 2;
|
||||
out[y_size + uv] =
|
||||
(-0.169 * r - 0.331 * g + 0.500 * b + 128.0).clamp(0.0, 255.0) as u8;
|
||||
out[y_size + uv_size + uv] =
|
||||
(0.500 * r - 0.419 * g - 0.081 * b + 128.0).clamp(0.0, 255.0) as u8;
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Tauri command: receive a JPEG frame from the frontend camera (getUserMedia),
|
||||
/// decode it, convert to I420, and push into the active call's video send task.
|
||||
///
|
||||
/// The frontend calls this at ~15 fps from a canvas.toDataURL() capture loop.
|
||||
#[tauri::command]
|
||||
async fn push_camera_frame(
|
||||
app: tauri::AppHandle,
|
||||
state: tauri::State<'_, Arc<AppState>>,
|
||||
jpeg_b64: String,
|
||||
) -> Result<(), String> {
|
||||
use base64::Engine as _;
|
||||
let jpeg_bytes = match base64::engine::general_purpose::STANDARD.decode(&jpeg_b64) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(e) => {
|
||||
let errs = CAMERA_PUSH_DECODE_ERRORS.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
if errs == 1 || errs % 30 == 0 {
|
||||
emit_call_debug(
|
||||
&app,
|
||||
"camera:jpeg_base64_decode_failed",
|
||||
serde_json::json!({
|
||||
"errors": errs,
|
||||
"error": e.to_string(),
|
||||
"b64_len": jpeg_b64.len(),
|
||||
}),
|
||||
);
|
||||
}
|
||||
return Err(e.to_string());
|
||||
}
|
||||
};
|
||||
|
||||
let dyn_img = match image::load_from_memory_with_format(&jpeg_bytes, image::ImageFormat::Jpeg) {
|
||||
Ok(img) => img,
|
||||
Err(e) => {
|
||||
let errs = CAMERA_PUSH_DECODE_ERRORS.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
if errs == 1 || errs % 30 == 0 {
|
||||
emit_call_debug(
|
||||
&app,
|
||||
"camera:jpeg_decode_failed",
|
||||
serde_json::json!({
|
||||
"errors": errs,
|
||||
"error": e.to_string(),
|
||||
"jpeg_bytes": jpeg_bytes.len(),
|
||||
}),
|
||||
);
|
||||
}
|
||||
return Err(e.to_string());
|
||||
}
|
||||
};
|
||||
let rgb_img = dyn_img.to_rgb8();
|
||||
let w = rgb_img.width() as usize;
|
||||
let h = rgb_img.height() as usize;
|
||||
let yuv = rgb_to_i420(rgb_img.as_raw(), w, h);
|
||||
let frame_no = CAMERA_PUSH_FRAMES.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
maybe_dump_video_jpeg(
|
||||
&app,
|
||||
"camera_jpeg_in",
|
||||
std::env::consts::OS,
|
||||
frame_no,
|
||||
&jpeg_bytes,
|
||||
w as u32,
|
||||
h as u32,
|
||||
);
|
||||
if let Some(converted_jpeg) = i420_to_jpeg_bytes(&yuv, w as u32, h as u32) {
|
||||
maybe_dump_video_jpeg(
|
||||
&app,
|
||||
"camera_i420_roundtrip",
|
||||
std::env::consts::OS,
|
||||
frame_no,
|
||||
&converted_jpeg,
|
||||
w as u32,
|
||||
h as u32,
|
||||
);
|
||||
}
|
||||
if frame_no == 1 || frame_no % 150 == 0 {
|
||||
emit_call_debug(
|
||||
&app,
|
||||
"camera:frame_received",
|
||||
serde_json::json!({
|
||||
"frame_no": frame_no,
|
||||
"width": w,
|
||||
"height": h,
|
||||
"jpeg_bytes": jpeg_bytes.len(),
|
||||
"yuv_bytes": yuv.len(),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
let ts = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_millis() as u64;
|
||||
|
||||
let frame = wzp_video::encoder::VideoFrame {
|
||||
width: w as u32,
|
||||
height: h as u32,
|
||||
data: yuv,
|
||||
timestamp_ms: ts,
|
||||
};
|
||||
|
||||
let engine = state.engine.lock().await;
|
||||
if let Some(ref eng) = *engine {
|
||||
if let Some(ref tx) = eng.camera_tx {
|
||||
match tx.try_send(frame) {
|
||||
Ok(()) => {
|
||||
if frame_no == 1 || frame_no % 150 == 0 {
|
||||
emit_call_debug(
|
||||
&app,
|
||||
"camera:frame_queued",
|
||||
serde_json::json!({ "frame_no": frame_no }),
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
let drops = CAMERA_PUSH_DROPS.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
if drops == 1 || drops % 30 == 0 {
|
||||
emit_call_debug(
|
||||
&app,
|
||||
"camera:frame_drop",
|
||||
serde_json::json!({
|
||||
"frame_no": frame_no,
|
||||
"drops": drops,
|
||||
"reason": e.to_string(),
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let count = CAMERA_PUSH_NO_SENDER.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
if count == 1 || count % 150 == 0 {
|
||||
emit_call_debug(
|
||||
&app,
|
||||
"camera:no_video_sender",
|
||||
serde_json::json!({
|
||||
"count": count,
|
||||
"hint": "video was not negotiated or the encoder task failed before camera_tx was installed",
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let count = CAMERA_PUSH_NO_ENGINE.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
if count == 1 || count % 150 == 0 {
|
||||
emit_call_debug(
|
||||
&app,
|
||||
"camera:no_call_engine",
|
||||
serde_json::json!({ "count": count }),
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ─── Video helper tests ───────────────────────────────────────────────────────
|
||||
#[cfg(test)]
|
||||
mod video_tests {
|
||||
use super::{i420_to_jpeg_b64, rgb_to_i420};
|
||||
use base64::Engine as _;
|
||||
|
||||
fn solid_rgb_frame(w: usize, h: usize, r: u8, g: u8, b: u8) -> Vec<u8> {
|
||||
let mut rgb = vec![0u8; w * h * 3];
|
||||
for i in 0..w * h {
|
||||
rgb[i * 3] = r;
|
||||
rgb[i * 3 + 1] = g;
|
||||
rgb[i * 3 + 2] = b;
|
||||
}
|
||||
rgb
|
||||
}
|
||||
|
||||
fn solid_i420(w: usize, h: usize, y: u8, u: u8, v: u8) -> Vec<u8> {
|
||||
let y_size = w * h;
|
||||
let uv_size = w * h / 4;
|
||||
let mut data = vec![y; y_size + 2 * uv_size];
|
||||
data[y_size..y_size + uv_size].fill(u);
|
||||
data[y_size + uv_size..].fill(v);
|
||||
data
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rgb_to_i420_output_size() {
|
||||
let rgb = solid_rgb_frame(640, 360, 128, 128, 128);
|
||||
let yuv = rgb_to_i420(&rgb, 640, 360);
|
||||
assert_eq!(yuv.len(), 640 * 360 * 3 / 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rgb_to_i420_pure_green_luma() {
|
||||
// Pure green (0, 255, 0) → Y ≈ 150 (0.587 × 255 ≈ 150).
|
||||
let rgb = solid_rgb_frame(4, 4, 0, 255, 0);
|
||||
let yuv = rgb_to_i420(&rgb, 4, 4);
|
||||
let y = yuv[0];
|
||||
assert!(y >= 140 && y <= 160, "pure-green luma out of range: {y}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rgb_to_i420_grey_is_neutral() {
|
||||
// Mid-grey RGB → U and V should both be near 128.
|
||||
let rgb = solid_rgb_frame(4, 4, 128, 128, 128);
|
||||
let yuv = rgb_to_i420(&rgb, 4, 4);
|
||||
let uv_start = 4 * 4;
|
||||
let u = yuv[uv_start];
|
||||
let v = yuv[uv_start + 4]; // 4 = (4/2)*(4/2)
|
||||
assert!((u as i32 - 128).abs() <= 5, "grey U out of range: {u}");
|
||||
assert!((v as i32 - 128).abs() <= 5, "grey V out of range: {v}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn i420_to_jpeg_b64_produces_non_empty_output() {
|
||||
let data = solid_i420(64, 64, 128, 128, 128);
|
||||
let b64 = i420_to_jpeg_b64(&data, 64, 64);
|
||||
assert!(b64.is_some(), "valid I420 must produce Some(b64)");
|
||||
let s = b64.unwrap();
|
||||
assert!(!s.is_empty());
|
||||
// JPEG base64 starts with '/9j/' (FFD8FF marker).
|
||||
let decoded = base64::engine::general_purpose::STANDARD
|
||||
.decode(&s)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
&decoded[0..2],
|
||||
&[0xFF, 0xD8],
|
||||
"output must start with JPEG SOI marker"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn i420_to_jpeg_b64_rejects_undersized_buffer() {
|
||||
// Buffer too short: only Y plane, no chroma.
|
||||
let data = vec![128u8; 64 * 64];
|
||||
let b64 = i420_to_jpeg_b64(&data, 64, 64);
|
||||
assert!(b64.is_none(), "truncated buffer must yield None");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn i420_to_jpeg_b64_color_preservation() {
|
||||
// A red (255, 0, 0) I420 frame should decode to a mostly-red JPEG.
|
||||
// After JPEG lossy compression the exact values drift, so we only
|
||||
// check that the decoded pixel has R > G and R > B.
|
||||
use base64::Engine as _;
|
||||
|
||||
// Convert red RGB → I420.
|
||||
let rgb = solid_rgb_frame(64, 64, 255, 0, 0);
|
||||
let yuv = rgb_to_i420(&rgb, 64, 64);
|
||||
|
||||
let b64 = i420_to_jpeg_b64(&yuv, 64, 64).expect("should produce JPEG");
|
||||
let jpeg = base64::engine::general_purpose::STANDARD
|
||||
.decode(&b64)
|
||||
.unwrap();
|
||||
|
||||
let img = image::load_from_memory_with_format(&jpeg, image::ImageFormat::Jpeg).unwrap();
|
||||
let rgb_img = img.to_rgb8();
|
||||
let px = rgb_img.get_pixel(32, 32);
|
||||
let (r, g, b) = (px[0], px[1], px[2]);
|
||||
assert!(
|
||||
r > g && r > b,
|
||||
"red frame: expected R dominant, got R={r} G={g} B={b}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rgb_i420_conversion_is_deterministic() {
|
||||
let rgb = solid_rgb_frame(8, 8, 200, 100, 50);
|
||||
let yuv1 = rgb_to_i420(&rgb, 8, 8);
|
||||
let yuv2 = rgb_to_i420(&rgb, 8, 8);
|
||||
assert_eq!(yuv1, yuv2, "rgb_to_i420 must be deterministic");
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolved by `setup()` once we have a Tauri AppHandle. Holds the
|
||||
/// platform-correct app data dir (e.g. `/data/data/com.wzp.desktop/files` on
|
||||
/// Android, `~/Library/Application Support/com.wzp.desktop` on macOS).
|
||||
@@ -349,8 +816,14 @@ async fn connect(
|
||||
// Enable birthday attack for hard NAT traversal. Adds ~3s to
|
||||
// call setup when peer has symmetric NAT.
|
||||
birthday_attack: Option<bool>,
|
||||
video_codec: Option<String>,
|
||||
video_width: Option<u32>,
|
||||
video_height: Option<u32>,
|
||||
) -> Result<String, String> {
|
||||
let force_direct = direct_only.unwrap_or(false);
|
||||
let video_codec = video_codec.unwrap_or_else(|| "h264".to_string());
|
||||
let video_width = video_width.unwrap_or(1280);
|
||||
let video_height = video_height.unwrap_or(720);
|
||||
let enable_birthday = birthday_attack.unwrap_or(false);
|
||||
emit_call_debug(
|
||||
&app,
|
||||
@@ -363,6 +836,9 @@ async fn connect(
|
||||
"peer_mapped_addr": peer_mapped_addr,
|
||||
"direct_only": force_direct,
|
||||
"birthday_attack": enable_birthday,
|
||||
"video_codec": video_codec,
|
||||
"video_width": video_width,
|
||||
"video_height": video_height,
|
||||
}),
|
||||
);
|
||||
let mut engine_lock = state.engine.lock().await;
|
||||
@@ -805,6 +1281,10 @@ async fn connect(
|
||||
}),
|
||||
);
|
||||
let app_for_engine = app.clone();
|
||||
let (active_quality, peer_max_quality) = {
|
||||
let sig = state.signal.lock().await;
|
||||
(sig.active_quality.clone(), sig.peer_max_quality.clone())
|
||||
};
|
||||
match CallEngine::start(
|
||||
relay,
|
||||
room,
|
||||
@@ -815,6 +1295,11 @@ async fn connect(
|
||||
pre_connected_transport,
|
||||
is_direct_p2p_agreed,
|
||||
app_for_engine,
|
||||
active_quality,
|
||||
peer_max_quality,
|
||||
video_codec,
|
||||
video_width,
|
||||
video_height,
|
||||
move |event_kind, message| {
|
||||
let _ = app_clone.emit(
|
||||
"call-event",
|
||||
@@ -1157,6 +1642,12 @@ struct SignalState {
|
||||
peer_hard_nat_probe: Option<PeerHardNatInfo>,
|
||||
/// Phase 8.6: peer's birthday attack ports, if received.
|
||||
peer_birthday_ports: Option<PeerBirthdayInfo>,
|
||||
/// Active quality profile for the encoder. Updated by signal upgrade flow.
|
||||
active_quality: Arc<std::sync::Mutex<wzp_proto::QualityProfile>>,
|
||||
/// Peer's reported max quality cap. The encoder clamps to min(active, peer_max).
|
||||
peer_max_quality: Arc<std::sync::Mutex<Option<wzp_proto::QualityProfile>>>,
|
||||
/// Pending outgoing upgrade proposal: (call_id, proposal_id, profile).
|
||||
pending_upgrade: Arc<std::sync::Mutex<Option<(String, String, wzp_proto::QualityProfile)>>>,
|
||||
}
|
||||
|
||||
/// Parsed data from a peer's HardNatBirthdayStart signal.
|
||||
@@ -1720,8 +2211,11 @@ fn do_register_signal(
|
||||
"peer_loss_pct": local_loss_pct, "peer_rtt_ms": local_rtt_ms,
|
||||
}),
|
||||
);
|
||||
// TODO: auto-accept if our own quality supports it,
|
||||
// or surface to UI for manual accept/reject
|
||||
if let Err(e) =
|
||||
handle_upgrade_proposal(&*transport, &call_id, &proposal_id).await
|
||||
{
|
||||
tracing::warn!("failed to send UpgradeResponse: {e}");
|
||||
}
|
||||
}
|
||||
Ok(Some(SignalMessage::UpgradeResponse {
|
||||
call_id,
|
||||
@@ -1739,7 +2233,17 @@ fn do_register_signal(
|
||||
"accepted": accepted, "reason": reason,
|
||||
}),
|
||||
);
|
||||
// TODO: if accepted, send UpgradeConfirm + switch encoder
|
||||
if let Err(e) = handle_upgrade_response(
|
||||
&*transport,
|
||||
&signal_state,
|
||||
&call_id,
|
||||
&proposal_id,
|
||||
accepted,
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::warn!("failed to handle UpgradeResponse: {e}");
|
||||
}
|
||||
}
|
||||
Ok(Some(SignalMessage::UpgradeConfirm {
|
||||
call_id,
|
||||
@@ -1756,7 +2260,7 @@ fn do_register_signal(
|
||||
"confirmed_profile": format!("{confirmed_profile:?}"),
|
||||
}),
|
||||
);
|
||||
// TODO: switch encoder to confirmed_profile at next frame boundary
|
||||
handle_upgrade_confirm(&signal_state, confirmed_profile).await;
|
||||
}
|
||||
Ok(Some(SignalMessage::QualityCapability {
|
||||
call_id,
|
||||
@@ -1775,8 +2279,7 @@ fn do_register_signal(
|
||||
"peer_loss_pct": loss_pct, "peer_rtt_ms": rtt_ms,
|
||||
}),
|
||||
);
|
||||
// TODO: adjust our encoder to not exceed peer's max_profile
|
||||
// (asymmetric quality — each side encodes at its own best)
|
||||
handle_quality_capability(&signal_state, max_profile).await;
|
||||
}
|
||||
Ok(Some(SignalMessage::HardNatBirthdayStart {
|
||||
call_id,
|
||||
@@ -2131,8 +2634,13 @@ async fn place_call(
|
||||
.map(|la| la.port())
|
||||
.unwrap_or(0);
|
||||
if v4_port > 0 {
|
||||
match wzp_client::portmap::acquire_port_mapping(v4_port, None).await {
|
||||
Ok(mapping) => {
|
||||
match tokio::time::timeout(
|
||||
std::time::Duration::from_millis(750),
|
||||
wzp_client::portmap::acquire_port_mapping(v4_port, None),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(mapping)) => {
|
||||
let addr = mapping.external_addr.to_string();
|
||||
tracing::info!(%addr, protocol = ?mapping.protocol, "place_call: port mapping acquired");
|
||||
emit_call_debug(
|
||||
@@ -2144,10 +2652,19 @@ async fn place_call(
|
||||
);
|
||||
Some(addr)
|
||||
}
|
||||
Err(e) => {
|
||||
Ok(Err(e)) => {
|
||||
tracing::debug!(error = %e, "place_call: port mapping unavailable (normal on most networks)");
|
||||
None
|
||||
}
|
||||
Err(_) => {
|
||||
tracing::debug!("place_call: port mapping quick probe timed out");
|
||||
emit_call_debug(
|
||||
&app,
|
||||
"place_call:portmap_timeout",
|
||||
serde_json::json!({ "timeout_ms": 750 }),
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
@@ -2374,8 +2891,13 @@ async fn answer_call(
|
||||
.map(|la| la.port())
|
||||
.unwrap_or(0);
|
||||
if v4_port > 0 {
|
||||
match wzp_client::portmap::acquire_port_mapping(v4_port, None).await {
|
||||
Ok(mapping) => {
|
||||
match tokio::time::timeout(
|
||||
std::time::Duration::from_millis(750),
|
||||
wzp_client::portmap::acquire_port_mapping(v4_port, None),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(mapping)) => {
|
||||
tracing::info!(
|
||||
addr = %mapping.external_addr,
|
||||
protocol = ?mapping.protocol,
|
||||
@@ -2383,10 +2905,19 @@ async fn answer_call(
|
||||
);
|
||||
Some(mapping.external_addr.to_string())
|
||||
}
|
||||
Err(e) => {
|
||||
Ok(Err(e)) => {
|
||||
tracing::debug!(error = %e, "answer_call: port mapping unavailable");
|
||||
None
|
||||
}
|
||||
Err(_) => {
|
||||
tracing::debug!("answer_call: port mapping quick probe timed out");
|
||||
emit_call_debug(
|
||||
&app,
|
||||
"answer_call:portmap_timeout",
|
||||
serde_json::json!({ "timeout_ms": 750 }),
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
} else {
|
||||
None
|
||||
@@ -2505,7 +3036,7 @@ async fn answer_call(
|
||||
/// or temporarily unreachable for reflect but the call can still
|
||||
/// proceed with STUN-discovered addresses.
|
||||
async fn try_reflect_own_addr(state: &Arc<AppState>) -> Result<Option<String>, String> {
|
||||
use wzp_proto::{SignalMessage, default_signal_version};
|
||||
use wzp_proto::SignalMessage;
|
||||
let (tx, rx) = tokio::sync::oneshot::channel::<std::net::SocketAddr>();
|
||||
let transport = {
|
||||
let mut sig = state.signal.lock().await;
|
||||
@@ -2592,7 +3123,7 @@ async fn try_stun_fallback(state: &Arc<AppState>) -> Result<Option<String>, Stri
|
||||
/// with `new URL(...)` / a regex if needed.
|
||||
#[tauri::command]
|
||||
async fn get_reflected_address(state: tauri::State<'_, Arc<AppState>>) -> Result<String, String> {
|
||||
use wzp_proto::{SignalMessage, default_signal_version};
|
||||
use wzp_proto::SignalMessage;
|
||||
let (tx, rx) = tokio::sync::oneshot::channel::<std::net::SocketAddr>();
|
||||
let transport = {
|
||||
let mut sig = state.signal.lock().await;
|
||||
@@ -2850,11 +3381,237 @@ async fn hangup_call(
|
||||
|
||||
// ─── App entry point ─────────────────────────────────────────────────────────
|
||||
|
||||
// ─── Quality upgrade flow handlers (testable) ─────────────────────────────
|
||||
|
||||
async fn handle_upgrade_proposal(
|
||||
transport: &dyn wzp_proto::MediaTransport,
|
||||
call_id: &str,
|
||||
proposal_id: &str,
|
||||
) -> Result<(), wzp_proto::TransportError> {
|
||||
let response = wzp_proto::SignalMessage::UpgradeResponse {
|
||||
version: default_signal_version(),
|
||||
call_id: call_id.to_string(),
|
||||
proposal_id: proposal_id.to_string(),
|
||||
accepted: true,
|
||||
reason: None,
|
||||
};
|
||||
transport.send_signal(&response).await
|
||||
}
|
||||
|
||||
async fn handle_upgrade_response(
|
||||
transport: &dyn wzp_proto::MediaTransport,
|
||||
signal_state: &Arc<tokio::sync::Mutex<SignalState>>,
|
||||
call_id: &str,
|
||||
proposal_id: &str,
|
||||
accepted: bool,
|
||||
) -> Result<(), wzp_proto::TransportError> {
|
||||
if accepted {
|
||||
let maybe_proposal = {
|
||||
let sig = signal_state.lock().await;
|
||||
sig.pending_upgrade.lock().unwrap().take()
|
||||
};
|
||||
if let Some((_cid, pid, profile)) = maybe_proposal {
|
||||
if pid == proposal_id {
|
||||
let confirm = wzp_proto::SignalMessage::UpgradeConfirm {
|
||||
version: default_signal_version(),
|
||||
call_id: call_id.to_string(),
|
||||
proposal_id: proposal_id.to_string(),
|
||||
confirmed_profile: profile.clone(),
|
||||
};
|
||||
transport.send_signal(&confirm).await?;
|
||||
{
|
||||
let sig = signal_state.lock().await;
|
||||
*sig.active_quality.lock().unwrap() = profile;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn handle_upgrade_confirm(
|
||||
signal_state: &Arc<tokio::sync::Mutex<SignalState>>,
|
||||
confirmed_profile: wzp_proto::QualityProfile,
|
||||
) {
|
||||
let sig = signal_state.lock().await;
|
||||
*sig.active_quality.lock().unwrap() = confirmed_profile;
|
||||
}
|
||||
|
||||
async fn handle_quality_capability(
|
||||
signal_state: &Arc<tokio::sync::Mutex<SignalState>>,
|
||||
max_profile: wzp_proto::QualityProfile,
|
||||
) {
|
||||
let sig = signal_state.lock().await;
|
||||
*sig.peer_max_quality.lock().unwrap() = Some(max_profile);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod signal_tests {
|
||||
use super::*;
|
||||
use async_trait::async_trait;
|
||||
use std::sync::Mutex as StdMutex;
|
||||
use wzp_proto::{MediaPacket, MediaTransport, PathQuality, SignalMessage, TransportError};
|
||||
|
||||
struct LoopbackTransport {
|
||||
sent: StdMutex<Vec<SignalMessage>>,
|
||||
}
|
||||
|
||||
impl LoopbackTransport {
|
||||
fn new() -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
sent: StdMutex::new(Vec::new()),
|
||||
})
|
||||
}
|
||||
fn take_sent(&self) -> Vec<SignalMessage> {
|
||||
self.sent.lock().unwrap().drain(..).collect()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl MediaTransport for LoopbackTransport {
|
||||
async fn send_media(&self, _packet: &MediaPacket) -> Result<(), TransportError> {
|
||||
Ok(())
|
||||
}
|
||||
async fn recv_media(&self) -> Result<Option<MediaPacket>, TransportError> {
|
||||
Ok(None)
|
||||
}
|
||||
async fn send_signal(&self, msg: &SignalMessage) -> Result<(), TransportError> {
|
||||
self.sent.lock().unwrap().push(msg.clone());
|
||||
Ok(())
|
||||
}
|
||||
async fn recv_signal(&self) -> Result<Option<SignalMessage>, TransportError> {
|
||||
Ok(None)
|
||||
}
|
||||
fn path_quality(&self) -> PathQuality {
|
||||
PathQuality::default()
|
||||
}
|
||||
async fn close(&self) -> Result<(), TransportError> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn empty_signal_state() -> Arc<tokio::sync::Mutex<SignalState>> {
|
||||
Arc::new(tokio::sync::Mutex::new(SignalState {
|
||||
transport: None,
|
||||
endpoint: None,
|
||||
ipv6_endpoint: None,
|
||||
fingerprint: String::new(),
|
||||
signal_status: "idle".into(),
|
||||
incoming_call_id: None,
|
||||
incoming_caller_fp: None,
|
||||
incoming_caller_alias: None,
|
||||
pending_reflect: None,
|
||||
own_reflex_addr: None,
|
||||
desired_relay_addr: None,
|
||||
reconnect_in_progress: false,
|
||||
pending_path_report: None,
|
||||
peer_hard_nat_probe: None,
|
||||
peer_birthday_ports: None,
|
||||
active_quality: Arc::new(std::sync::Mutex::new(wzp_proto::QualityProfile::GOOD)),
|
||||
peer_max_quality: Arc::new(std::sync::Mutex::new(None)),
|
||||
pending_upgrade: Arc::new(std::sync::Mutex::new(None)),
|
||||
}))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn upgrade_proposal_auto_accepts() {
|
||||
let transport = LoopbackTransport::new();
|
||||
handle_upgrade_proposal(&*transport, "c1", "p1")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let sent = transport.take_sent();
|
||||
assert_eq!(sent.len(), 1);
|
||||
match &sent[0] {
|
||||
SignalMessage::UpgradeResponse {
|
||||
call_id,
|
||||
proposal_id,
|
||||
accepted,
|
||||
reason,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(call_id, "c1");
|
||||
assert_eq!(proposal_id, "p1");
|
||||
assert!(accepted);
|
||||
assert!(reason.is_none());
|
||||
}
|
||||
other => panic!("expected UpgradeResponse, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn upgrade_response_accepted_sends_confirm_and_updates_quality() {
|
||||
let transport = LoopbackTransport::new();
|
||||
let signal_state = empty_signal_state();
|
||||
{
|
||||
let sig = signal_state.lock().await;
|
||||
*sig.pending_upgrade.lock().unwrap() = Some((
|
||||
"c1".into(),
|
||||
"p1".into(),
|
||||
wzp_proto::QualityProfile::STUDIO_48K,
|
||||
));
|
||||
}
|
||||
|
||||
handle_upgrade_response(&*transport, &signal_state, "c1", "p1", true)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let sent = transport.take_sent();
|
||||
assert_eq!(sent.len(), 1);
|
||||
match &sent[0] {
|
||||
SignalMessage::UpgradeConfirm {
|
||||
call_id,
|
||||
proposal_id,
|
||||
confirmed_profile,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(call_id, "c1");
|
||||
assert_eq!(proposal_id, "p1");
|
||||
assert_eq!(*confirmed_profile, wzp_proto::QualityProfile::STUDIO_48K);
|
||||
}
|
||||
other => panic!("expected UpgradeConfirm, got {other:?}"),
|
||||
}
|
||||
|
||||
let sig = signal_state.lock().await;
|
||||
assert_eq!(
|
||||
*sig.active_quality.lock().unwrap(),
|
||||
wzp_proto::QualityProfile::STUDIO_48K
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn upgrade_confirm_updates_active_quality() {
|
||||
let signal_state = empty_signal_state();
|
||||
handle_upgrade_confirm(&signal_state, wzp_proto::QualityProfile::STUDIO_64K).await;
|
||||
|
||||
let sig = signal_state.lock().await;
|
||||
assert_eq!(
|
||||
*sig.active_quality.lock().unwrap(),
|
||||
wzp_proto::QualityProfile::STUDIO_64K
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quality_capability_updates_peer_max() {
|
||||
let signal_state = empty_signal_state();
|
||||
handle_quality_capability(&signal_state, wzp_proto::QualityProfile::GOOD).await;
|
||||
|
||||
let sig = signal_state.lock().await;
|
||||
assert_eq!(
|
||||
sig.peer_max_quality.lock().unwrap().unwrap(),
|
||||
wzp_proto::QualityProfile::GOOD
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared Tauri app builder. Used by the desktop `main.rs` and the mobile
|
||||
/// entry point below.
|
||||
pub fn run() {
|
||||
tracing_subscriber::fmt().init();
|
||||
|
||||
let active_quality = Arc::new(std::sync::Mutex::new(wzp_proto::QualityProfile::GOOD));
|
||||
let peer_max_quality = Arc::new(std::sync::Mutex::new(None));
|
||||
let pending_upgrade = Arc::new(std::sync::Mutex::new(None));
|
||||
let state = Arc::new(AppState {
|
||||
engine: Mutex::new(None),
|
||||
signal: Arc::new(Mutex::new(SignalState {
|
||||
@@ -2873,6 +3630,9 @@ pub fn run() {
|
||||
pending_path_report: None,
|
||||
peer_hard_nat_probe: None,
|
||||
peer_birthday_ports: None,
|
||||
active_quality: active_quality.clone(),
|
||||
peer_max_quality: peer_max_quality.clone(),
|
||||
pending_upgrade: pending_upgrade.clone(),
|
||||
})),
|
||||
});
|
||||
|
||||
@@ -2949,6 +3709,8 @@ pub fn run() {
|
||||
get_dred_verbose_logs,
|
||||
set_call_debug_logs,
|
||||
get_call_debug_logs,
|
||||
call_debug_log,
|
||||
push_camera_frame,
|
||||
])
|
||||
.run(tauri::generate_context!())
|
||||
.expect("error while running WarzonePhone");
|
||||
|
||||
@@ -62,6 +62,7 @@ const lobbyFp = document.getElementById("lobby-fp")!;
|
||||
const lobbyUserList = document.getElementById("lobby-user-list")!;
|
||||
const lobbyUserCount = document.getElementById("lobby-user-count")!;
|
||||
const joinVoiceBtn = document.getElementById("join-voice-btn")!;
|
||||
const joinVideoBtn = document.getElementById("join-video-btn")!;
|
||||
const incomingBanner = document.getElementById("incoming-call-banner")!;
|
||||
const incomingCallerName = document.getElementById("incoming-caller-name")!;
|
||||
const incomingIdenticon = document.getElementById("incoming-identicon")!;
|
||||
@@ -79,6 +80,11 @@ const vdMicIcon = document.getElementById("vd-mic-icon")!;
|
||||
const vdSpkBtn = document.getElementById("vd-spk-btn")!;
|
||||
const vdSpkIcon = document.getElementById("vd-spk-icon")!;
|
||||
const vdEndBtn = document.getElementById("vd-end-btn")!;
|
||||
const vdCamBtn = document.getElementById("vd-cam-btn")!;
|
||||
const vdCamIcon = document.getElementById("vd-cam-icon")!;
|
||||
const vdVideoStrip = document.getElementById("vd-video-strip")!;
|
||||
const vdRemoteVideo = document.getElementById("vd-remote-video") as HTMLCanvasElement;
|
||||
const vdLocalVideo = document.getElementById("vd-local-video") as HTMLVideoElement;
|
||||
const vdDirectInfo = document.getElementById("vd-direct-info")!;
|
||||
const vdDcIdenticon = document.getElementById("vd-dc-identicon")!;
|
||||
const vdDcName = document.getElementById("vd-dc-name")!;
|
||||
@@ -116,6 +122,8 @@ const sCallDebugCopyBtn = document.getElementById("s-call-debug-copy") as HTMLBu
|
||||
const sCallDebugShareBtn = document.getElementById("s-call-debug-share") as HTMLButtonElement;
|
||||
const sQuality = document.getElementById("s-quality") as HTMLInputElement;
|
||||
const sQualityLabel = document.getElementById("s-quality-label")!;
|
||||
const sVideoCodec = document.getElementById("s-video-codec") as HTMLSelectElement;
|
||||
const sVideoResolution = document.getElementById("s-video-resolution") as HTMLSelectElement;
|
||||
const sFingerprint = document.getElementById("s-fingerprint")!;
|
||||
const sPublicAddr = document.getElementById("s-public-addr")!;
|
||||
const sReflectBtn = document.getElementById("s-reflect-btn")!;
|
||||
@@ -132,6 +140,8 @@ interface Settings {
|
||||
alias: string;
|
||||
osAec: boolean;
|
||||
quality: string;
|
||||
videoCodec: string;
|
||||
videoResolution: string;
|
||||
recentRooms: RecentRoom[];
|
||||
dredDebugLogs: boolean;
|
||||
callDebugLogs: boolean;
|
||||
@@ -145,7 +155,7 @@ function loadSettings(): Settings {
|
||||
{ name: "Default", address: "193.180.213.68:4433" },
|
||||
],
|
||||
selectedRelay: 0, room: "general", alias: "",
|
||||
osAec: true, quality: "auto", recentRooms: [],
|
||||
osAec: true, quality: "auto", videoCodec: "h264", videoResolution: "1280x720", recentRooms: [],
|
||||
dredDebugLogs: false, callDebugLogs: false,
|
||||
directOnly: false, birthdayAttack: false,
|
||||
};
|
||||
@@ -158,6 +168,25 @@ function loadSettings(): Settings {
|
||||
function saveSettings(s: Settings) {
|
||||
localStorage.setItem("wzp-settings", JSON.stringify(s));
|
||||
}
|
||||
|
||||
function parseVideoResolution(value: string) {
|
||||
const [wRaw, hRaw] = (value || "1280x720").split("x");
|
||||
const width = Number.parseInt(wRaw, 10);
|
||||
const height = Number.parseInt(hRaw, 10);
|
||||
if (!Number.isFinite(width) || !Number.isFinite(height)) {
|
||||
return { width: 1280, height: 720 };
|
||||
}
|
||||
return { width, height };
|
||||
}
|
||||
|
||||
function videoConnectOptions(s: Settings) {
|
||||
const { width, height } = parseVideoResolution(s.videoResolution);
|
||||
return {
|
||||
videoCodec: s.videoCodec || "h264",
|
||||
videoWidth: width,
|
||||
videoHeight: height,
|
||||
};
|
||||
}
|
||||
function getRelay(): RelayServer | null {
|
||||
const s = loadSettings();
|
||||
return s.relays[s.selectedRelay] || s.relays[0] || null;
|
||||
@@ -170,6 +199,89 @@ let connectPending = false; // guard against double-tap while connect is in-flig
|
||||
let directCallPeer: { fingerprint: string; alias: string | null } | null = null;
|
||||
let pendingCallId: string | null = null;
|
||||
|
||||
// Video / camera state
|
||||
let cameraActive = false;
|
||||
let cameraStream: MediaStream | null = null;
|
||||
let cameraFrameTimer: number | null = null;
|
||||
let cameraFrameCallbackHandle: number | null = null;
|
||||
let cameraCaptureInFlight = false;
|
||||
let lastCameraCaptureAtMs = 0;
|
||||
let remoteVideoActive = false;
|
||||
|
||||
interface FrameCallbackVideoElement extends HTMLVideoElement {
|
||||
requestVideoFrameCallback?: (callback: (now: DOMHighResTimeStamp, metadata: unknown) => void) => number;
|
||||
cancelVideoFrameCallback?: (handle: number) => void;
|
||||
}
|
||||
|
||||
// Keep the local preview out of the video stage stacking context so it can float
|
||||
// above the call drawer and remain draggable on phones.
|
||||
document.body.appendChild(vdLocalVideo);
|
||||
vdLocalVideo.classList.add("hidden");
|
||||
|
||||
function clampNumber(value: number, min: number, max: number) {
|
||||
return Math.min(Math.max(value, min), max);
|
||||
}
|
||||
|
||||
function keepLocalPipInViewport() {
|
||||
if (vdLocalVideo.classList.contains("hidden")) return;
|
||||
const rect = vdLocalVideo.getBoundingClientRect();
|
||||
if (!rect.width || !rect.height) return;
|
||||
const margin = 12;
|
||||
const maxLeft = Math.max(margin, window.innerWidth - rect.width - margin);
|
||||
const maxTop = Math.max(margin, window.innerHeight - rect.height - margin);
|
||||
const left = clampNumber(rect.left, margin, maxLeft);
|
||||
const top = clampNumber(rect.top, margin, maxTop);
|
||||
vdLocalVideo.style.left = `${left}px`;
|
||||
vdLocalVideo.style.top = `${top}px`;
|
||||
vdLocalVideo.style.right = "auto";
|
||||
vdLocalVideo.style.bottom = "auto";
|
||||
}
|
||||
|
||||
function initLocalPipDrag() {
|
||||
let dragPointerId: number | null = null;
|
||||
let dragOffsetX = 0;
|
||||
let dragOffsetY = 0;
|
||||
|
||||
vdLocalVideo.addEventListener("pointerdown", (event) => {
|
||||
if (vdLocalVideo.classList.contains("hidden")) return;
|
||||
dragPointerId = event.pointerId;
|
||||
const rect = vdLocalVideo.getBoundingClientRect();
|
||||
dragOffsetX = event.clientX - rect.left;
|
||||
dragOffsetY = event.clientY - rect.top;
|
||||
vdLocalVideo.classList.add("dragging");
|
||||
vdLocalVideo.setPointerCapture(event.pointerId);
|
||||
event.preventDefault();
|
||||
});
|
||||
|
||||
vdLocalVideo.addEventListener("pointermove", (event) => {
|
||||
if (dragPointerId !== event.pointerId) return;
|
||||
const rect = vdLocalVideo.getBoundingClientRect();
|
||||
const margin = 12;
|
||||
const maxLeft = Math.max(margin, window.innerWidth - rect.width - margin);
|
||||
const maxTop = Math.max(margin, window.innerHeight - rect.height - margin);
|
||||
const left = clampNumber(event.clientX - dragOffsetX, margin, maxLeft);
|
||||
const top = clampNumber(event.clientY - dragOffsetY, margin, maxTop);
|
||||
vdLocalVideo.style.left = `${left}px`;
|
||||
vdLocalVideo.style.top = `${top}px`;
|
||||
vdLocalVideo.style.right = "auto";
|
||||
vdLocalVideo.style.bottom = "auto";
|
||||
event.preventDefault();
|
||||
});
|
||||
|
||||
function endDrag(event: PointerEvent) {
|
||||
if (dragPointerId !== event.pointerId) return;
|
||||
dragPointerId = null;
|
||||
vdLocalVideo.classList.remove("dragging");
|
||||
try { vdLocalVideo.releasePointerCapture(event.pointerId); } catch {}
|
||||
}
|
||||
|
||||
vdLocalVideo.addEventListener("pointerup", endDrag);
|
||||
vdLocalVideo.addEventListener("pointercancel", endDrag);
|
||||
window.addEventListener("resize", keepLocalPipInViewport);
|
||||
}
|
||||
|
||||
initLocalPipDrag();
|
||||
|
||||
function showToast(msg: string, durationMs = 3500) {
|
||||
let el = document.getElementById("wzp-toast");
|
||||
if (!el) {
|
||||
@@ -251,6 +363,10 @@ function renderCallDebugLog() {
|
||||
sCallDebugLogEl.scrollTop = sCallDebugLogEl.scrollHeight;
|
||||
}
|
||||
|
||||
function debugLog(step: string, details: any = {}) {
|
||||
invoke("call_debug_log", { step, details }).catch(() => {});
|
||||
}
|
||||
|
||||
// ── Quality slider ────────────────────────────────────────────────
|
||||
const QUALITY_STEPS = ["studio-64k", "studio-48k", "studio-32k", "auto", "good", "degraded", "codec2-3200", "catastrophic"];
|
||||
const QUALITY_LABELS = ["Studio 64k", "Studio 48k", "Studio 32k", "Auto", "Opus 24k", "Opus 6k", "Codec2 3.2k", "Codec2 1.2k"];
|
||||
@@ -373,6 +489,7 @@ joinVoiceBtn.addEventListener("click", async () => {
|
||||
alias: s.alias || "",
|
||||
osAec: s.osAec,
|
||||
quality: s.quality || "auto",
|
||||
...videoConnectOptions(s),
|
||||
});
|
||||
enterVoice(false);
|
||||
} catch (e: any) {
|
||||
@@ -385,10 +502,41 @@ joinVoiceBtn.addEventListener("click", async () => {
|
||||
}
|
||||
});
|
||||
|
||||
joinVideoBtn.addEventListener("click", async () => {
|
||||
if (inVoice || connectPending) return;
|
||||
const relay = getRelay();
|
||||
const s = loadSettings();
|
||||
if (!relay) { showToast("No relay configured"); return; }
|
||||
connectPending = true;
|
||||
const origText = joinVideoBtn.textContent;
|
||||
joinVideoBtn.textContent = "Connecting…";
|
||||
(joinVideoBtn as HTMLButtonElement).disabled = true;
|
||||
try {
|
||||
await connectWithTimeout({
|
||||
relay: relay.address,
|
||||
room: s.room || "general",
|
||||
alias: s.alias || "",
|
||||
osAec: s.osAec,
|
||||
quality: s.quality || "auto",
|
||||
...videoConnectOptions(s),
|
||||
});
|
||||
enterVoice(false);
|
||||
startCamera();
|
||||
} catch (e: any) {
|
||||
console.error("connect failed:", e);
|
||||
showToast(`Join failed: ${errorMessage(e)}`);
|
||||
} finally {
|
||||
connectPending = false;
|
||||
joinVideoBtn.textContent = origText;
|
||||
(joinVideoBtn as HTMLButtonElement).disabled = false;
|
||||
}
|
||||
});
|
||||
|
||||
function enterVoice(isDirect: boolean) {
|
||||
inVoice = true;
|
||||
const s = loadSettings();
|
||||
joinVoiceBtn.classList.add("hidden");
|
||||
joinVideoBtn.classList.add("hidden");
|
||||
voiceDrawer.classList.remove("hidden");
|
||||
vdRoom.textContent = isDirect && directCallPeer
|
||||
? (directCallPeer.alias || directCallPeer.fingerprint.substring(0, 16))
|
||||
@@ -418,8 +566,17 @@ function leaveVoice() {
|
||||
pendingCallId = null;
|
||||
voiceDrawer.classList.add("hidden");
|
||||
joinVoiceBtn.classList.remove("hidden");
|
||||
joinVideoBtn.classList.remove("hidden");
|
||||
vdLevelBar.style.width = "0%";
|
||||
if (statusInterval) { clearInterval(statusInterval); statusInterval = null; }
|
||||
stopCamera();
|
||||
remoteVideoActive = false;
|
||||
remoteFrameCount = 0;
|
||||
remoteFrameSerial++;
|
||||
vdRemoteCounter.textContent = "0 frames received";
|
||||
vdRemotePlaceholder.classList.remove("hidden");
|
||||
vdVideoStrip.classList.add("hidden");
|
||||
remoteCtx.clearRect(0, 0, vdRemoteVideo.width, vdRemoteVideo.height);
|
||||
}
|
||||
|
||||
// Drawer controls
|
||||
@@ -435,6 +592,247 @@ vdSpkBtn.addEventListener("click", async () => {
|
||||
try { await invoke("toggle_speaker"); } catch {}
|
||||
});
|
||||
|
||||
// ── Camera (Blocker 4 + 5) ────────────────────────────────────────
|
||||
const camCaptureCanvas = document.createElement("canvas");
|
||||
const camCaptureCtx = camCaptureCanvas.getContext("2d")!;
|
||||
let cameraSendWidth = 1280;
|
||||
let cameraSendHeight = 720;
|
||||
let cameraCaptureFrameNo = 0;
|
||||
let cameraPushFailures = 0;
|
||||
const CAMERA_CAPTURE_INTERVAL_MS = 33; // ≈ 30 fps
|
||||
const CAMERA_JPEG_QUALITY = 0.7;
|
||||
|
||||
function drawCameraFrameForSend() {
|
||||
const vw = vdLocalVideo.videoWidth || camCaptureCanvas.width;
|
||||
const vh = vdLocalVideo.videoHeight || camCaptureCanvas.height;
|
||||
if (!vw || !vh) return;
|
||||
|
||||
const scale = Math.min(cameraSendWidth / vw, cameraSendHeight / vh);
|
||||
const dw = vw * scale;
|
||||
const dh = vh * scale;
|
||||
const dx = (cameraSendWidth - dw) / 2;
|
||||
const dy = (cameraSendHeight - dh) / 2;
|
||||
|
||||
camCaptureCtx.fillStyle = "#000";
|
||||
camCaptureCtx.fillRect(0, 0, cameraSendWidth, cameraSendHeight);
|
||||
camCaptureCtx.drawImage(vdLocalVideo, dx, dy, dw, dh);
|
||||
}
|
||||
|
||||
async function captureAndPushCameraFrame() {
|
||||
if (!cameraActive || cameraCaptureInFlight) return;
|
||||
cameraCaptureInFlight = true;
|
||||
cameraCaptureFrameNo++;
|
||||
try {
|
||||
drawCameraFrameForSend();
|
||||
const dataUrl = camCaptureCanvas.toDataURL("image/jpeg", CAMERA_JPEG_QUALITY);
|
||||
const b64 = dataUrl.slice(dataUrl.indexOf(",") + 1);
|
||||
if (cameraCaptureFrameNo === 1 || cameraCaptureFrameNo % 150 === 0) {
|
||||
debugLog("camera:capture_frame", {
|
||||
frame_no: cameraCaptureFrameNo,
|
||||
width: camCaptureCanvas.width,
|
||||
height: camCaptureCanvas.height,
|
||||
source_width: vdLocalVideo.videoWidth || null,
|
||||
source_height: vdLocalVideo.videoHeight || null,
|
||||
jpeg_b64_len: b64.length,
|
||||
capture_clock: getVideoFrameCallbackApi() ? "video_frame_callback" : "interval",
|
||||
});
|
||||
}
|
||||
await invoke("push_camera_frame", { jpegB64: b64 });
|
||||
} catch (e: any) {
|
||||
cameraPushFailures++;
|
||||
if (cameraPushFailures === 1 || cameraPushFailures % 30 === 0) {
|
||||
debugLog("camera:push_failed", {
|
||||
frame_no: cameraCaptureFrameNo,
|
||||
failures: cameraPushFailures,
|
||||
error: errorMessage(e),
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
cameraCaptureInFlight = false;
|
||||
}
|
||||
}
|
||||
|
||||
function getVideoFrameCallbackApi() {
|
||||
const video = vdLocalVideo as FrameCallbackVideoElement;
|
||||
if (typeof video.requestVideoFrameCallback !== "function") return null;
|
||||
return video;
|
||||
}
|
||||
|
||||
function cancelCameraCaptureLoop() {
|
||||
if (cameraFrameTimer != null) {
|
||||
window.clearInterval(cameraFrameTimer);
|
||||
cameraFrameTimer = null;
|
||||
}
|
||||
const video = getVideoFrameCallbackApi();
|
||||
if (video && cameraFrameCallbackHandle != null && typeof video.cancelVideoFrameCallback === "function") {
|
||||
video.cancelVideoFrameCallback(cameraFrameCallbackHandle);
|
||||
}
|
||||
cameraFrameCallbackHandle = null;
|
||||
}
|
||||
|
||||
function scheduleCameraFrameCapture() {
|
||||
cancelCameraCaptureLoop();
|
||||
lastCameraCaptureAtMs = 0;
|
||||
const video = getVideoFrameCallbackApi();
|
||||
if (video) {
|
||||
const onVideoFrame = (now: DOMHighResTimeStamp) => {
|
||||
cameraFrameCallbackHandle = null;
|
||||
if (!cameraActive) return;
|
||||
if (lastCameraCaptureAtMs === 0 || now - lastCameraCaptureAtMs >= CAMERA_CAPTURE_INTERVAL_MS) {
|
||||
lastCameraCaptureAtMs = now;
|
||||
void captureAndPushCameraFrame();
|
||||
}
|
||||
cameraFrameCallbackHandle = video.requestVideoFrameCallback!(onVideoFrame);
|
||||
};
|
||||
cameraFrameCallbackHandle = video.requestVideoFrameCallback(onVideoFrame);
|
||||
debugLog("camera:capture_clock", { mode: "video_frame_callback", interval_ms: CAMERA_CAPTURE_INTERVAL_MS });
|
||||
return;
|
||||
}
|
||||
|
||||
cameraFrameTimer = window.setInterval(() => {
|
||||
void captureAndPushCameraFrame();
|
||||
}, CAMERA_CAPTURE_INTERVAL_MS);
|
||||
debugLog("camera:capture_clock", { mode: "interval", interval_ms: CAMERA_CAPTURE_INTERVAL_MS });
|
||||
}
|
||||
|
||||
async function startCamera() {
|
||||
if (cameraActive) return;
|
||||
const videoSize = parseVideoResolution(loadSettings().videoResolution);
|
||||
cameraSendWidth = videoSize.width;
|
||||
cameraSendHeight = videoSize.height;
|
||||
const constraints = {
|
||||
video: { width: { ideal: cameraSendWidth }, height: { ideal: cameraSendHeight }, facingMode: "user" },
|
||||
audio: false,
|
||||
};
|
||||
debugLog("camera:get_user_media_start", { constraints });
|
||||
try {
|
||||
cameraStream = await navigator.mediaDevices.getUserMedia(constraints);
|
||||
vdLocalVideo.srcObject = cameraStream;
|
||||
vdVideoStrip.classList.remove("hidden");
|
||||
|
||||
const track = cameraStream.getVideoTracks()[0];
|
||||
const settings = track.getSettings();
|
||||
camCaptureCanvas.width = cameraSendWidth;
|
||||
camCaptureCanvas.height = cameraSendHeight;
|
||||
debugLog("camera:get_user_media_ok", {
|
||||
width: settings.width ?? null,
|
||||
height: settings.height ?? null,
|
||||
send_width: camCaptureCanvas.width,
|
||||
send_height: camCaptureCanvas.height,
|
||||
frameRate: settings.frameRate ?? null,
|
||||
deviceId: settings.deviceId ? "present" : null,
|
||||
facingMode: settings.facingMode ?? null,
|
||||
});
|
||||
|
||||
cameraActive = true;
|
||||
cameraCaptureFrameNo = 0;
|
||||
cameraPushFailures = 0;
|
||||
vdCamIcon.textContent = "Cam ✓";
|
||||
vdCamBtn.classList.add("active");
|
||||
vdLocalVideo.classList.remove("hidden");
|
||||
keepLocalPipInViewport();
|
||||
|
||||
scheduleCameraFrameCapture();
|
||||
} catch (e: any) {
|
||||
console.warn("camera access denied or unavailable:", e);
|
||||
debugLog("camera:get_user_media_failed", {
|
||||
name: e?.name ?? null,
|
||||
message: e?.message ?? String(e),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function stopCamera() {
|
||||
if (cameraActive) {
|
||||
debugLog("camera:stopped", { frames: cameraCaptureFrameNo });
|
||||
}
|
||||
cameraActive = false;
|
||||
cancelCameraCaptureLoop();
|
||||
if (cameraStream) { cameraStream.getTracks().forEach(t => t.stop()); cameraStream = null; }
|
||||
vdLocalVideo.srcObject = null;
|
||||
vdLocalVideo.classList.add("hidden");
|
||||
vdCamIcon.textContent = "Cam";
|
||||
vdCamBtn.classList.remove("active");
|
||||
// Hide strip only if remote video is also gone
|
||||
if (!remoteVideoActive) vdVideoStrip.classList.add("hidden");
|
||||
}
|
||||
|
||||
vdCamBtn.addEventListener("click", () => {
|
||||
if (cameraActive) { stopCamera(); } else { startCamera(); }
|
||||
});
|
||||
|
||||
// ── Remote video display (Blocker 5) ─────────────────────────────
|
||||
const remoteCtx = vdRemoteVideo.getContext("2d")!;
|
||||
const vdRemotePlaceholder = document.getElementById("vd-remote-placeholder")!;
|
||||
const vdRemoteCounter = document.getElementById("vd-remote-counter")!;
|
||||
let remoteFrameCount = 0;
|
||||
let remoteFrameSerial = 0;
|
||||
let remoteDrawInFlight = false;
|
||||
let remotePendingFrame: { serial: number; width: number; height: number; jpeg_b64: string } | null = null;
|
||||
|
||||
function nextAnimationFrame() {
|
||||
return new Promise<void>(resolve => requestAnimationFrame(() => resolve()));
|
||||
}
|
||||
|
||||
async function drawRemoteFrame(frame: { serial: number; width: number; height: number; jpeg_b64: string }) {
|
||||
const img = new Image();
|
||||
img.src = `data:image/jpeg;base64,${frame.jpeg_b64}`;
|
||||
if ("decode" in img) {
|
||||
await img.decode();
|
||||
} else {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
img.onload = () => resolve();
|
||||
img.onerror = () => reject(new Error("remote video image decode failed"));
|
||||
});
|
||||
}
|
||||
|
||||
if (frame.serial !== remoteFrameSerial) return;
|
||||
await nextAnimationFrame();
|
||||
if (frame.serial !== remoteFrameSerial) return;
|
||||
if (vdRemoteVideo.width !== frame.width) vdRemoteVideo.width = frame.width;
|
||||
if (vdRemoteVideo.height !== frame.height) vdRemoteVideo.height = frame.height;
|
||||
remoteCtx.drawImage(img, 0, 0, vdRemoteVideo.width, vdRemoteVideo.height);
|
||||
}
|
||||
|
||||
async function pumpRemoteVideoFrames() {
|
||||
if (remoteDrawInFlight) return;
|
||||
remoteDrawInFlight = true;
|
||||
try {
|
||||
while (remotePendingFrame) {
|
||||
const frame = remotePendingFrame;
|
||||
remotePendingFrame = null;
|
||||
try {
|
||||
await drawRemoteFrame(frame);
|
||||
} catch (e) {
|
||||
console.warn("remote video draw failed:", e);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
remoteDrawInFlight = false;
|
||||
if (remotePendingFrame) void pumpRemoteVideoFrames();
|
||||
}
|
||||
}
|
||||
|
||||
listen("video:frame", (event: any) => {
|
||||
const { width, height, jpeg_b64 } = event.payload;
|
||||
if (!jpeg_b64) return;
|
||||
const frameSerial = ++remoteFrameSerial;
|
||||
|
||||
remoteVideoActive = true;
|
||||
vdVideoStrip.classList.remove("hidden");
|
||||
vdRemotePlaceholder.classList.add("hidden");
|
||||
remoteFrameCount++;
|
||||
if (remoteFrameCount === 1) console.log("first remote video frame:", width, "x", height);
|
||||
|
||||
remotePendingFrame = {
|
||||
serial: frameSerial,
|
||||
width: width ?? vdRemoteVideo.width,
|
||||
height: height ?? vdRemoteVideo.height,
|
||||
jpeg_b64,
|
||||
};
|
||||
void pumpRemoteVideoFrames();
|
||||
});
|
||||
|
||||
// ── Poll status ───────────────────────────────────────────────────
|
||||
interface CallStatusI {
|
||||
active: boolean;
|
||||
@@ -554,6 +952,7 @@ listen("signal-event", (event: any) => {
|
||||
peerMappedAddr: data.peer_mapped_addr ?? null,
|
||||
directOnly: s.directOnly || false,
|
||||
birthdayAttack: s.birthdayAttack || false,
|
||||
...videoConnectOptions(s),
|
||||
});
|
||||
enterVoice(true);
|
||||
} catch (e: any) {
|
||||
@@ -704,6 +1103,8 @@ function openSettings() {
|
||||
sCallDebug.checked = !!s.callDebugLogs;
|
||||
sDirectOnly.checked = !!s.directOnly;
|
||||
sBirthdayAttack.checked = !!s.birthdayAttack;
|
||||
sVideoCodec.value = s.videoCodec || "h264";
|
||||
sVideoResolution.value = s.videoResolution || "1280x720";
|
||||
sCallDebugSection.style.display = s.callDebugLogs ? "" : "none";
|
||||
renderCallDebugLog();
|
||||
const qi = qualityToIndex(s.quality || "auto");
|
||||
@@ -729,6 +1130,8 @@ settingsSave.addEventListener("click", () => {
|
||||
s.callDebugLogs = sCallDebug.checked;
|
||||
s.directOnly = sDirectOnly.checked;
|
||||
s.birthdayAttack = sBirthdayAttack.checked;
|
||||
s.videoCodec = sVideoCodec.value || "h264";
|
||||
s.videoResolution = sVideoResolution.value || "1280x720";
|
||||
saveSettings(s);
|
||||
invoke("set_dred_verbose_logs", { enabled: s.dredDebugLogs }).catch(() => {});
|
||||
invoke("set_call_debug_logs", { enabled: s.callDebugLogs }).catch(() => {});
|
||||
@@ -831,6 +1234,7 @@ document.addEventListener("keydown", (e) => {
|
||||
if (e.key === "m") vdMicBtn.click();
|
||||
if (e.key === "q") vdEndBtn.click();
|
||||
if (e.key === "s") vdSpkBtn.click();
|
||||
if (e.key === "v") vdCamBtn.click();
|
||||
if (e.key === "," && (e.metaKey || e.ctrlKey)) { e.preventDefault(); openSettings(); }
|
||||
});
|
||||
|
||||
|
||||
@@ -204,6 +204,16 @@ body {
|
||||
padding: 12px 0;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.fab-video {
|
||||
background: #3b82f6;
|
||||
box-shadow: 0 4px 16px rgba(59, 130, 246, 0.3);
|
||||
}
|
||||
|
||||
.fab-video:hover {
|
||||
box-shadow: 0 6px 20px rgba(59, 130, 246, 0.4);
|
||||
}
|
||||
|
||||
.fab {
|
||||
@@ -248,7 +258,7 @@ body {
|
||||
border-top: 1px solid var(--surface2);
|
||||
padding: 0 16px;
|
||||
padding-bottom: env(safe-area-inset-bottom, 8px);
|
||||
z-index: 50;
|
||||
z-index: 70;
|
||||
animation: drawerUp 0.25s ease-out;
|
||||
box-shadow: 0 -4px 20px rgba(0,0,0,0.4);
|
||||
}
|
||||
@@ -306,6 +316,68 @@ body {
|
||||
padding: 2px 0 4px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
/* Full-screen video stage — overlays lobby/main when video is active */
|
||||
.vd-video-stage {
|
||||
position: fixed;
|
||||
top: 0;
|
||||
left: 0;
|
||||
right: 0;
|
||||
bottom: 96px; /* leave room for voice drawer */
|
||||
background: #000;
|
||||
z-index: 40;
|
||||
overflow: hidden;
|
||||
}
|
||||
.vd-remote-stage {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: contain;
|
||||
background: #000;
|
||||
}
|
||||
.vd-remote-placeholder {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
color: #888;
|
||||
pointer-events: none;
|
||||
z-index: 1;
|
||||
}
|
||||
.vd-remote-placeholder.hidden { display: none; }
|
||||
.vd-placeholder-text { font-size: 18px; margin-bottom: 8px; }
|
||||
.vd-placeholder-sub { font-size: 12px; opacity: 0.7; }
|
||||
.vd-local-pip {
|
||||
position: fixed;
|
||||
right: 18px;
|
||||
bottom: calc(176px + env(safe-area-inset-bottom, 0px));
|
||||
width: min(34vw, 220px);
|
||||
height: auto;
|
||||
aspect-ratio: 16 / 9;
|
||||
border-radius: 8px;
|
||||
background: #111;
|
||||
border: 2px solid rgba(255, 255, 255, 0.2);
|
||||
object-fit: cover;
|
||||
box-shadow: 0 4px 20px rgba(0, 0, 0, 0.5);
|
||||
z-index: 90;
|
||||
cursor: grab;
|
||||
touch-action: none;
|
||||
-webkit-user-drag: none;
|
||||
}
|
||||
.vd-local-pip.dragging {
|
||||
cursor: grabbing;
|
||||
box-shadow: 0 8px 28px rgba(0, 0, 0, 0.65);
|
||||
}
|
||||
@media (max-width: 520px) {
|
||||
.vd-local-pip {
|
||||
width: min(48vw, 190px);
|
||||
right: 12px;
|
||||
bottom: calc(188px + env(safe-area-inset-bottom, 0px));
|
||||
}
|
||||
}
|
||||
|
||||
/* Incoming call banner */
|
||||
.incoming-banner {
|
||||
position: fixed;
|
||||
|
||||
225
docs/PRD/PRD-android-mediacodec-ndk9.md
Normal file
225
docs/PRD/PRD-android-mediacodec-ndk9.md
Normal file
@@ -0,0 +1,225 @@
|
||||
# PRD: Android MediaCodec NDK 0.9 Compatibility
|
||||
|
||||
> **Status:** proposed
|
||||
> **Resolves:** 31 compile errors in `crates/wzp-video/src/mediacodec.rs` blocking all Android video.
|
||||
> **Depends on:** Remote build server `manwe@188.245.59.196` with Docker image `wzp-android-builder:latest`.
|
||||
|
||||
## Problem
|
||||
|
||||
`crates/wzp-video/src/mediacodec.rs` fails to compile for
|
||||
`aarch64-linux-android` against the NDK 0.9 Rust crate. There are 31 errors
|
||||
in 5 categories. Android video is completely blocked.
|
||||
|
||||
The file already compiles for non-Android targets (all Android code is behind
|
||||
`#[cfg(target_os = "android")]`). Only the Android target path needs fixing.
|
||||
|
||||
## Goals
|
||||
|
||||
- `cargo build --target aarch64-linux-android -p wzp-video` produces 0 errors on the remote server.
|
||||
- Each fix category lands in a separate commit so failures can be bisected.
|
||||
- Non-Android compilation is not broken.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Upgrading the NDK Docker image or changing the NDK version.
|
||||
- Fixing video functionality beyond compilation (runtime testing is a separate task).
|
||||
- Any files outside `crates/wzp-video/`.
|
||||
|
||||
## Design
|
||||
|
||||
### Build command (run after each fix)
|
||||
|
||||
```bash
|
||||
ssh manwe@188.245.59.196 'cd ~/wzp-builder/data/source && \
|
||||
git fetch github && git reset --hard github/experimental-ui && \
|
||||
docker run --rm \
|
||||
-v ~/wzp-builder/data/source:/build/source \
|
||||
-v ~/wzp-builder/data/cache/cargo-registry:/home/builder/.cargo/registry \
|
||||
-v ~/wzp-builder/data/cache/cargo-git:/home/builder/.cargo/git \
|
||||
-v ~/wzp-builder/data/cache/target:/build/source/target \
|
||||
wzp-android-builder:latest bash -c \
|
||||
"cd /build/source && cargo build --target aarch64-linux-android -p wzp-video 2>&1 | grep -E \"^error\" | head -30"'
|
||||
```
|
||||
|
||||
### Fix order (commit one per category)
|
||||
|
||||
#### Fix 1 — `E0433`: `ndk_sys` not declared as a dependency
|
||||
|
||||
**Symptom**: `use of undeclared crate or module 'ndk_sys'`
|
||||
|
||||
**File**: `crates/wzp-video/Cargo.toml`
|
||||
|
||||
NDK 0.9 no longer re-exports raw `ndk_sys` symbols; they must be declared as
|
||||
a direct dependency. Add to the `[target.'cfg(target_os = "android")'.dependencies]`
|
||||
section (or create it if absent):
|
||||
|
||||
```toml
|
||||
[target.'cfg(target_os = "android")'.dependencies]
|
||||
ndk = { version = "0.9" }
|
||||
ndk-sys = { version = "0.6" } # ndk 0.9 depends on ndk-sys 0.6
|
||||
```
|
||||
|
||||
If `mediacodec.rs` only uses safe wrappers from the `ndk` crate and the
|
||||
`ndk_sys` imports are not strictly needed, remove the `use ndk_sys::*` lines
|
||||
from `mediacodec.rs` instead — whichever approach results in fewer changes.
|
||||
|
||||
After this fix the `E0433` errors should drop from the build output.
|
||||
|
||||
#### Fix 2 — `E0425`: `BITRATE_MODE_CBR` constant missing
|
||||
|
||||
**Symptom**: `cannot find value 'BITRATE_MODE_CBR' in this scope`
|
||||
|
||||
**File**: `crates/wzp-video/src/mediacodec.rs`
|
||||
|
||||
`BITRATE_MODE_CBR` is already defined as a local constant at line 44:
|
||||
|
||||
```rust
|
||||
#[cfg(target_os = "android")]
|
||||
const BITRATE_MODE_CBR: i32 = 2;
|
||||
```
|
||||
|
||||
If the error persists after Fix 1, the issue is that `ndk_sys` was providing
|
||||
a conflicting symbol. Verify the constant is still at line 44 after Fix 1. If
|
||||
NDK 0.9 moved `BITRATE_MODE_CBR` to an enum, update the usage at line 516
|
||||
(`format.set_i32("bitrate-mode", BITRATE_MODE_CBR)`) to use the integer
|
||||
value directly (`2`) or update the constant's value.
|
||||
|
||||
If `ndk 0.9` defines `MediaCodecBitrateMode::Cbr` as an enum, the call site
|
||||
in `MediaCodecAv1Encoder::new` (line ~516) can be updated to:
|
||||
|
||||
```rust
|
||||
format.set_i32(
|
||||
"bitrate-mode",
|
||||
ndk::media::media_codec::MediaCodecBitrateMode::Cbr as i32,
|
||||
);
|
||||
```
|
||||
|
||||
#### Fix 3 — `E0308`: `InputBuffer` returns `&mut [MaybeUninit<u8>]`
|
||||
|
||||
**Symptom**: `expected &mut [u8], found &mut [MaybeUninit<u8>]`
|
||||
|
||||
**File**: `crates/wzp-video/src/mediacodec.rs`
|
||||
|
||||
NDK 0.9 changed `InputBuffer::buffer_mut()` from `&mut [u8]` to
|
||||
`&mut [MaybeUninit<u8>]`. There are multiple write sites in the file — all
|
||||
follow the same pattern:
|
||||
|
||||
```rust
|
||||
// Before (NDK 0.8):
|
||||
let buf = buffer.buffer_mut(); // &mut [u8]
|
||||
let n = frame.data.len().min(buf.len());
|
||||
buf[..n].copy_from_slice(&frame.data[..n]);
|
||||
```
|
||||
|
||||
```rust
|
||||
// After (NDK 0.9):
|
||||
let buf = buffer.buffer_mut(); // &mut [MaybeUninit<u8>]
|
||||
let n = frame.data.len().min(buf.len());
|
||||
for (d, &s) in buf[..n].iter_mut().zip(frame.data[..n].iter()) {
|
||||
d.write(s);
|
||||
}
|
||||
```
|
||||
|
||||
The file already uses the `d.write(s)` pattern in some places (lines 125–127,
|
||||
297–299, etc.). Search for **every** occurrence of `buffer.buffer_mut()` and
|
||||
`buffer_mut()` and apply the same pattern. Affected structs:
|
||||
`MediaCodecEncoder::encode` (~line 123), `MediaCodecDecoder::decode`
|
||||
(~line 294), `MediaCodecHevcEncoder::encode` (~line 439),
|
||||
`MediaCodecHevcDecoder::decode` (~line 773), `MediaCodecAv1Encoder::encode`
|
||||
(~line 560), `MediaCodecAv1Decoder::decode` (~line 907).
|
||||
|
||||
Do NOT use `unsafe { std::mem::transmute }` — the `d.write(s)` pattern is
|
||||
already present and safe.
|
||||
|
||||
Note: if the file already uses `d.write(s)` everywhere, this category may
|
||||
already be addressed by the existing code. Check the actual error count.
|
||||
|
||||
#### Fix 4 — `E0599`: `.index()` is private
|
||||
|
||||
**Symptom**: `method 'index' is private`
|
||||
|
||||
**File**: `crates/wzp-video/src/mediacodec.rs`
|
||||
|
||||
NDK 0.9 removed the public `.index()` method from `DequeuedInputBuffer` and
|
||||
`DequeuedOutputBuffer`. The pattern that broke:
|
||||
|
||||
```rust
|
||||
// Broken: buffer.index() is private in NDK 0.9
|
||||
let idx = buffer.index();
|
||||
codec.queue_input_buffer_index(idx, ...);
|
||||
```
|
||||
|
||||
In NDK 0.9 the correct API is to pass the buffer object directly to
|
||||
`queue_input_buffer`:
|
||||
|
||||
```rust
|
||||
codec.queue_input_buffer(buffer, offset, size, pts_us, flags)?;
|
||||
```
|
||||
|
||||
The file already uses `codec.queue_input_buffer(buffer, 0, to_copy, ...)` in
|
||||
most places (lines 131, 303, 447, etc.). Search for any remaining `.index()`
|
||||
calls on buffer objects and replace them with the direct-pass pattern shown
|
||||
above.
|
||||
|
||||
#### Fix 5 — `E0277`: `NonNull<AMediaCodec>` is not `Send`
|
||||
|
||||
**Symptom**: `NonNull<AMediaCodec>` cannot be sent between threads safely
|
||||
|
||||
**File**: `crates/wzp-video/src/mediacodec.rs`
|
||||
|
||||
Each codec struct must have an `unsafe impl Send` declaration. Audit all six
|
||||
codec structs:
|
||||
|
||||
| Struct | `unsafe impl Send` present? |
|
||||
|--------|----------------------------|
|
||||
| `MediaCodecEncoder` | Yes (line 51) |
|
||||
| `MediaCodecDecoder` | Yes (line 228) |
|
||||
| `MediaCodecHevcEncoder` | Yes (line 374) |
|
||||
| `MediaCodecHevcDecoder` | Yes (line 705) |
|
||||
| `MediaCodecAv1Encoder` | Yes (line 503) |
|
||||
| `MediaCodecAv1Decoder` | Yes (line 844) |
|
||||
|
||||
If any are missing, add them with a safety comment:
|
||||
|
||||
```rust
|
||||
// SAFETY: AMediaCodec is documented as thread-safe.
|
||||
#[cfg(target_os = "android")]
|
||||
unsafe impl Send for MediaCodecXxxYyy {}
|
||||
```
|
||||
|
||||
This category may already be clean. Confirm with the build output.
|
||||
|
||||
## Implementation steps
|
||||
|
||||
1. Push the current branch to `github/experimental-ui` before starting.
|
||||
2. **Commit 1**: Fix `ndk_sys` dependency (`Cargo.toml`). Push. Run build.
|
||||
Confirm `E0433` errors drop.
|
||||
3. **Commit 2**: Fix `BITRATE_MODE_CBR`. Push. Run build. Confirm `E0425` gone.
|
||||
4. **Commit 3**: Fix `MaybeUninit` write sites. Push. Run build. Confirm
|
||||
`E0308` gone.
|
||||
5. **Commit 4**: Remove any `.index()` calls. Push. Run build. Confirm
|
||||
`E0599` gone.
|
||||
6. **Commit 5**: Add missing `unsafe impl Send` if any. Push. Run build.
|
||||
Confirm `E0277` gone and total error count is 0.
|
||||
|
||||
## Files to read before implementing
|
||||
|
||||
- `crates/wzp-video/src/mediacodec.rs` (full file — 45 KB; read in chunks)
|
||||
- `crates/wzp-video/Cargo.toml` (check existing `[dependencies]` sections)
|
||||
|
||||
## Verify
|
||||
|
||||
Final build command (see Design section). Expected output: no lines matching
|
||||
`^error`.
|
||||
|
||||
Also verify non-Android host still compiles:
|
||||
|
||||
```bash
|
||||
cargo check -p wzp-video
|
||||
```
|
||||
|
||||
## Done when
|
||||
|
||||
`cargo build --target aarch64-linux-android -p wzp-video` on the remote
|
||||
server produces 0 `error[...]` lines. Non-Android `cargo check -p wzp-video`
|
||||
also passes.
|
||||
260
docs/PRD/PRD-clippy-debt.md
Normal file
260
docs/PRD/PRD-clippy-debt.md
Normal file
@@ -0,0 +1,260 @@
|
||||
# PRD: Fix wzp-codec Clippy Lint Debt
|
||||
|
||||
> **Status:** proposed
|
||||
> **Resolves:** 9 pre-existing clippy lints in `crates/wzp-codec/src/` that cause `cargo clippy --workspace -D warnings` to fail, breaking any strict-CI configuration.
|
||||
> **Depends on:** Nothing — all changes are in `crates/wzp-codec/src/`.
|
||||
|
||||
## Problem
|
||||
|
||||
`cargo clippy -p wzp-codec -- -D warnings` fails with 9 lints across 5 files.
|
||||
These are pre-existing code patterns that were never flagged during development
|
||||
because the CI flag was not set. They have no runtime impact today but prevent
|
||||
adding `-D warnings` to CI without first cleaning them up.
|
||||
|
||||
The 3 errors in `deps/featherchat` are in a submodule — do NOT touch them.
|
||||
`warzone_protocol` clippy errors are accepted debt (not our code).
|
||||
|
||||
## Goals
|
||||
|
||||
- `cargo clippy -p wzp-codec -- -D warnings` exits 0.
|
||||
- No behavior changes — every fix is a semantically equivalent rewrite.
|
||||
- No changes outside `crates/wzp-codec/src/`.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Fixing clippy lints in any crate other than `wzp-codec`.
|
||||
- Adding new functionality.
|
||||
- Touching the `deps/featherchat` submodule.
|
||||
|
||||
## Design
|
||||
|
||||
### Lint inventory
|
||||
|
||||
| Lint | Count | File | Approx line | Fix |
|
||||
|------|-------|------|-------------|-----|
|
||||
| `implicit_saturating_sub` | 1 | `aec.rs` | 117–119 | `saturating_sub` |
|
||||
| `needless_range_loop` | 2 | `aec.rs:164`, `resample.rs:51` | — | iterate with `iter().enumerate()` or direct iter |
|
||||
| `manual_div_ceil` | 2 | `codec2_dec.rs:48`, `codec2_enc.rs:48` | — | `div_ceil` |
|
||||
| `manual_clamp` | 2 | `denoise.rs:59`, `opus_enc.rs:250` | — | `.clamp(min, max)` |
|
||||
| `manual_ascii_check` | 1 | `opus_enc.rs:104` | — | `.eq_ignore_ascii_case()` |
|
||||
| `same_item_push` | 1 | `resample.rs:184` | — | `vec.resize` or `extend(repeat)` |
|
||||
|
||||
### Fix details
|
||||
|
||||
#### 1. `implicit_saturating_sub` — `aec.rs` line ~117
|
||||
|
||||
Current code:
|
||||
|
||||
```rust
|
||||
fn delay_available(&self) -> usize {
|
||||
let buffered = self.delay_write - self.delay_read;
|
||||
if buffered > self.delay_samples {
|
||||
buffered - self.delay_samples
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Clippy wants `saturating_sub` because the subtraction can underflow if
|
||||
`buffered < self.delay_samples`:
|
||||
|
||||
```rust
|
||||
fn delay_available(&self) -> usize {
|
||||
let buffered = self.delay_write - self.delay_read;
|
||||
buffered.saturating_sub(self.delay_samples)
|
||||
}
|
||||
```
|
||||
|
||||
This is semantically identical (both return 0 when `buffered <= delay_samples`).
|
||||
|
||||
#### 2a. `needless_range_loop` — `aec.rs` line ~164
|
||||
|
||||
Current code:
|
||||
|
||||
```rust
|
||||
for i in 0..n {
|
||||
let near_f = nearend[i] as f32;
|
||||
let base = (self.far_pos + fl * ((n / fl) + 2) + i - n) % fl;
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
`i` is used both to index `nearend[i]` and in arithmetic (`+ i - n`).
|
||||
Clippy fires because `nearend[i]` could use `.iter().enumerate()`.
|
||||
Convert to `enumerate`:
|
||||
|
||||
```rust
|
||||
for (i, &sample) in nearend.iter().enumerate() {
|
||||
let near_f = sample as f32;
|
||||
let base = (self.far_pos + fl * ((n / fl) + 2) + i - n) % fl;
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
Make sure to update any references to `nearend[i]` inside the loop body
|
||||
to use `sample` (or `near_f` directly). Also update the NLMS adaptation
|
||||
sub-loop if it references `nearend[i]`.
|
||||
|
||||
#### 2b. `needless_range_loop` — `resample.rs` line ~51
|
||||
|
||||
Current code:
|
||||
|
||||
```rust
|
||||
for i in 0..FIR_TAPS {
|
||||
let n = i as f64 - m / 2.0;
|
||||
let sinc = ...;
|
||||
let t = 2.0 * i as f64 / m - 1.0;
|
||||
let kaiser = ...;
|
||||
kernel[i] = sinc * kaiser;
|
||||
}
|
||||
```
|
||||
|
||||
`i` is used both as an index (`kernel[i]`) and in arithmetic. Use
|
||||
`iter_mut().enumerate()`:
|
||||
|
||||
```rust
|
||||
for (i, slot) in kernel.iter_mut().enumerate() {
|
||||
let n = i as f64 - m / 2.0;
|
||||
let sinc = ...;
|
||||
let t = 2.0 * i as f64 / m - 1.0;
|
||||
let kaiser = ...;
|
||||
*slot = sinc * kaiser;
|
||||
}
|
||||
```
|
||||
|
||||
#### 3a. `manual_div_ceil` — `codec2_dec.rs` line ~48
|
||||
|
||||
Current code:
|
||||
|
||||
```rust
|
||||
fn bytes_per_frame(&self) -> usize {
|
||||
(self.inner.bits_per_frame() + 7) / 8
|
||||
}
|
||||
```
|
||||
|
||||
Replace with:
|
||||
|
||||
```rust
|
||||
fn bytes_per_frame(&self) -> usize {
|
||||
self.inner.bits_per_frame().div_ceil(8)
|
||||
}
|
||||
```
|
||||
|
||||
`div_ceil` is stable as of Rust 1.73. The builder uses a recent enough
|
||||
toolchain. If `bits_per_frame()` returns `usize`, the method is available.
|
||||
If it returns a different integer type, cast accordingly.
|
||||
|
||||
#### 3b. `manual_div_ceil` — `codec2_enc.rs` line ~48
|
||||
|
||||
Same pattern, same fix:
|
||||
|
||||
```rust
|
||||
fn bytes_per_frame(&self) -> usize {
|
||||
self.inner.bits_per_frame().div_ceil(8)
|
||||
}
|
||||
```
|
||||
|
||||
#### 4a. `manual_clamp` — `denoise.rs` line ~59
|
||||
|
||||
Current code:
|
||||
|
||||
```rust
|
||||
let clamped = val.max(-32768.0).min(32767.0);
|
||||
```
|
||||
|
||||
Replace with:
|
||||
|
||||
```rust
|
||||
let clamped = val.clamp(-32768.0_f32, 32767.0_f32);
|
||||
```
|
||||
|
||||
Note: `.clamp()` on `f32` requires both bounds to be the same type. If `val`
|
||||
is already `f32`, no extra cast is needed. Verify the type of `val` in
|
||||
context (it is `f32` per the output array type `[f32; 480]`).
|
||||
|
||||
#### 4b. `manual_clamp` — `opus_enc.rs` line ~252
|
||||
|
||||
Read the surrounding code for the exact pattern. It will be something like:
|
||||
|
||||
```rust
|
||||
let v = if x < min_val { min_val } else if x > max_val { max_val } else { x };
|
||||
```
|
||||
|
||||
or the `.max().min()` chain. Replace with `x.clamp(min_val, max_val)`.
|
||||
|
||||
#### 5. `manual_ascii_check` — `opus_enc.rs` line ~104
|
||||
|
||||
Current code:
|
||||
|
||||
```rust
|
||||
Ok(v) => !v.is_empty() && v != "0" && v.to_ascii_lowercase() != "false",
|
||||
```
|
||||
|
||||
Clippy wants `.eq_ignore_ascii_case()` instead of lowercasing the whole string:
|
||||
|
||||
```rust
|
||||
Ok(v) => !v.is_empty() && v != "0" && !v.eq_ignore_ascii_case("false"),
|
||||
```
|
||||
|
||||
#### 6. `same_item_push` — `resample.rs` line ~183
|
||||
|
||||
Current code:
|
||||
|
||||
```rust
|
||||
for _ in 1..RATIO {
|
||||
work.push(0.0);
|
||||
}
|
||||
```
|
||||
|
||||
This pushes the same `0.0` value `(RATIO - 1)` times. Replace with:
|
||||
|
||||
```rust
|
||||
work.resize(work.len() + (RATIO - 1), 0.0f64);
|
||||
```
|
||||
|
||||
Or equivalently:
|
||||
|
||||
```rust
|
||||
work.extend(std::iter::repeat(0.0f64).take(RATIO - 1));
|
||||
```
|
||||
|
||||
Note: `RATIO` is a `const usize`. Verify `work` is `Vec<f64>` in context
|
||||
(it is — `work.push(s as f64)` immediately before).
|
||||
|
||||
## Implementation steps
|
||||
|
||||
1. Read each file at the line numbers listed above to confirm the exact current
|
||||
code before editing (line numbers may shift slightly due to prior edits).
|
||||
2. Apply all 9 fixes. They are independent — no ordering requirement.
|
||||
3. Run `cargo clippy -p wzp-codec -- -D warnings` locally or via the CI
|
||||
command.
|
||||
4. If any lint persists, re-read that file section and adjust.
|
||||
|
||||
## Files to read before implementing
|
||||
|
||||
- `crates/wzp-codec/src/aec.rs` lines 114–200
|
||||
- `crates/wzp-codec/src/resample.rs` lines 45–70 and 178–190
|
||||
- `crates/wzp-codec/src/codec2_dec.rs` lines 40–55
|
||||
- `crates/wzp-codec/src/codec2_enc.rs` lines 40–55
|
||||
- `crates/wzp-codec/src/denoise.rs` lines 45–65
|
||||
- `crates/wzp-codec/src/opus_enc.rs` lines 96–110 and 244–260
|
||||
|
||||
## Verify
|
||||
|
||||
```bash
|
||||
cargo clippy -p wzp-codec -- -D warnings
|
||||
```
|
||||
|
||||
Expected: exits 0 with no warnings.
|
||||
|
||||
Also run to confirm no regressions:
|
||||
|
||||
```bash
|
||||
cargo test -p wzp-codec
|
||||
```
|
||||
|
||||
## Done when
|
||||
|
||||
`cargo clippy -p wzp-codec -- -D warnings` exits 0. All 9 lints are gone.
|
||||
`cargo test -p wzp-codec` passes. No changes outside `crates/wzp-codec/src/`.
|
||||
98
docs/PRD/PRD-e2e-media-encryption.md
Normal file
98
docs/PRD/PRD-e2e-media-encryption.md
Normal file
@@ -0,0 +1,98 @@
|
||||
# PRD: E2E Media Encryption (rewrite)
|
||||
|
||||
> **Status:** proposed (supersedes prior version)
|
||||
> **Resolves:** Real end-to-end media encryption between call participants.
|
||||
> **Replaces:** The prior version of this PRD described wrapping `QuinnTransport` in `EncryptingTransport` using the pairwise client↔relay session. That approach was implemented (commit `52a6f5e`) and **broke voice between any two clients** because the relay does not decrypt+re-encrypt — see "Why the prior fix failed" below. The wrapping was reverted in commit `e8cab25`.
|
||||
|
||||
---
|
||||
|
||||
## Why the prior fix failed
|
||||
|
||||
`wzp_client::handshake::perform_handshake` performs ECDH **between the client and the relay**. Each client in a room ends up with a **different** pairwise session key (key_A for client A, key_B for client B, etc.).
|
||||
|
||||
The relay is an SFU — it forwards `MediaPacket` bytes between participants in a room without inspecting their payloads. The relay does not run a decrypt-then-encrypt step keyed per-recipient.
|
||||
|
||||
Wrapping `QuinnTransport` in `EncryptingTransport` therefore produced:
|
||||
|
||||
```
|
||||
Client A: plaintext --[encrypt key_A]--> ciphertext --> Relay
|
||||
Relay: forwards ciphertext (bytes) --> Client B
|
||||
Client B: ciphertext --[decrypt key_B]--> garbage --> silent audio
|
||||
```
|
||||
|
||||
Result: every recipient saw decryption failures, audio went silent.
|
||||
|
||||
This is **not a bug in `EncryptingTransport`** — the wrapper does exactly what it claims. The bug was thinking the pairwise client-relay session was usable for participant-to-participant media. It isn't.
|
||||
|
||||
## Goals
|
||||
|
||||
A future implementation must satisfy:
|
||||
|
||||
- Two clients in a room can exchange media that the **other client** can decrypt.
|
||||
- The **relay cannot decrypt** any media payload (true E2E), OR alternatively, the relay can decrypt+re-encrypt per recipient (hop-by-hop, sometimes called SFU-trusted).
|
||||
- Joining and leaving the room mid-call rotates keys so departed members can't decrypt subsequent traffic (forward secrecy on membership change).
|
||||
- Compatible with the existing `MediaPacket` wire format (header in plaintext, payload encrypted).
|
||||
|
||||
## Two valid approaches
|
||||
|
||||
### Approach A — MLS group keys (true E2E)
|
||||
|
||||
Use the [MLS protocol](https://datatracker.ietf.org/doc/rfc9420/) (e.g. via the `openmls` crate) to derive a shared **group key** that all room members possess and the relay does not.
|
||||
|
||||
- Relay acts as a **delivery service** for MLS Handshake messages (`Welcome`, `Commit`, `Proposal`) but never sees the group secret.
|
||||
- Every media packet is AEAD-sealed with the current group epoch key.
|
||||
- Group rekey is triggered by:
|
||||
- Member join/leave (forward secrecy on membership)
|
||||
- Periodic (every N seconds or N packets) for post-compromise security
|
||||
- Each room maintains its own MLS group; the relay just stores opaque `mls_blob` payloads in `SignalMessage::MlsHandshake`.
|
||||
|
||||
**Pros:** real E2E. Relay compromise does not leak media.
|
||||
**Cons:** Significant complexity (MLS state machine per room, persistent ratchet trees, key schedule). Adds `openmls` dependency (~30 KLOC). Federation across relays is harder.
|
||||
|
||||
### Approach B — Hop-by-hop re-encryption at the relay
|
||||
|
||||
The relay holds a `CryptoSession` per connected client (which it already does — see `_crypto_session` discarded in `crates/wzp-relay/src/main.rs:1817`). On forward:
|
||||
|
||||
```
|
||||
Relay.recv_media(from A): decrypt with key_A → plaintext
|
||||
Relay.send_media(to B, C, D): for each recipient X, encrypt with key_X
|
||||
```
|
||||
|
||||
This is the same model as Matrix Megolm-without-Megolm — encrypted hop-by-hop but the relay sees plaintext briefly in between.
|
||||
|
||||
**Pros:** Reuses existing per-client `ChaChaSession`. Implementation is ~100 lines in the relay's room forwarding loop. Federation works the same way (each relay-relay hop has its own session).
|
||||
**Cons:** Relay sees plaintext. A compromised relay can record and decrypt all media. This is **not E2E** — but it is strictly stronger than the current state (plaintext-over-QUIC-TLS exposes media to anyone with a TLS-terminating proxy on the relay).
|
||||
|
||||
## Recommendation
|
||||
|
||||
**Ship Approach B first.** It's a small, well-scoped change that closes the relay-operator-can-see-plaintext-in-RAM gap without requiring an MLS rewrite. Then layer Approach A on top when the threat model demands relay-untrusted operation.
|
||||
|
||||
## Out of scope for this PRD
|
||||
|
||||
- Federation gossip key exchange (separate PRD)
|
||||
- SAS (Short Authentication String) verification UX (separate PRD)
|
||||
- Rekey on session compromise (handled by the chosen approach's group/pairwise rekey)
|
||||
|
||||
## Acceptance criteria (Approach B, first iteration)
|
||||
|
||||
1. Relay's room forwarding loop (`crates/wzp-relay/src/room.rs:354` and `:1353`) calls `sender_session.decrypt()` then `recipient_session.encrypt()` per recipient before `send_media`.
|
||||
2. Each `RoomMember` holds its `Box<dyn CryptoSession>` (currently discarded as `_crypto_session` in `main.rs:1817`).
|
||||
3. Client-side: re-add the `EncryptingTransport` wrapping in `desktop/src-tauri/src/engine.rs` (the two sites reverted in `e8cab25`).
|
||||
4. Integration test: two-client mock room exchanges media; verify each recipient gets the sender's plaintext back after the relay double-hop.
|
||||
5. Existing 825 tests still pass.
|
||||
|
||||
## Verification
|
||||
|
||||
`cargo test -p wzp-relay --test multi_client_relay_path` should pass with two simulated clients sending audio in both directions and decrypting each other's frames.
|
||||
|
||||
## Files to touch
|
||||
|
||||
- `crates/wzp-relay/src/main.rs` — keep `crypto_session` per-client (drop the `_` prefix)
|
||||
- `crates/wzp-relay/src/room.rs` — add decrypt/re-encrypt to forward path
|
||||
- `crates/wzp-relay/src/session_mgr.rs` — store sessions keyed by peer
|
||||
- `desktop/src-tauri/src/engine.rs` — restore `EncryptingTransport` wrapping (~2 sites)
|
||||
- `crates/wzp-relay/tests/multi_client_relay_path.rs` — new integration test
|
||||
|
||||
## Risk / rollback
|
||||
|
||||
If multi-client tests fail in CI, the change is contained to the relay forwarding loop and one engine.rs edit — straightforward revert.
|
||||
220
docs/PRD/PRD-quality-upgrade-flow.md
Normal file
220
docs/PRD/PRD-quality-upgrade-flow.md
Normal file
@@ -0,0 +1,220 @@
|
||||
# PRD: Quality Upgrade Flow — UpgradeProposal / Response / Confirm
|
||||
|
||||
> **Status:** proposed
|
||||
> **Resolves:** Four TODO comments in the signal task of `desktop/src-tauri/src/lib.rs` that leave quality upgrade messages unhandled. Audio quality never upgrades mid-call even when the network improves.
|
||||
> **Depends on:** `wzp_proto::SignalMessage::{UpgradeProposal, UpgradeResponse, UpgradeConfirm, QualityCapability}` (already defined in `crates/wzp-proto/src/packet.rs`).
|
||||
|
||||
## Problem
|
||||
|
||||
The signal receive task in `lib.rs` matches `UpgradeProposal`, `UpgradeResponse`,
|
||||
`UpgradeConfirm`, and `QualityCapability` messages from the peer, logs them,
|
||||
then hits a `// TODO` comment and does nothing. The 4 TODOs are at lines
|
||||
1930, 1949, 1966, and 1985 of `desktop/src-tauri/src/lib.rs`.
|
||||
|
||||
Consequence: audio quality is frozen at the profile negotiated at call start.
|
||||
Even when the network improves, the encoder never upgrades.
|
||||
|
||||
## Goals
|
||||
|
||||
1. `UpgradeProposal` auto-accepts and sends `UpgradeResponse { accepted: true }`.
|
||||
2. Accepted `UpgradeResponse` sends `UpgradeConfirm` and switches the local encoder.
|
||||
3. Received `UpgradeConfirm` switches the local encoder.
|
||||
4. Received `QualityCapability` caps the local encoder to the peer's max profile.
|
||||
5. A unit test verifies the accept/confirm round-trip.
|
||||
6. `cargo check --manifest-path desktop/src-tauri/Cargo.toml` passes.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- UI for manual accept/reject of upgrade proposals (auto-accept only).
|
||||
- Sending `UpgradeProposal` from our side (the outgoing path already exists in
|
||||
`lib.rs`; this PRD only handles receiving).
|
||||
- Downgrade negotiation.
|
||||
- Persisting quality profiles across calls.
|
||||
|
||||
## Design
|
||||
|
||||
### New shared state
|
||||
|
||||
Add the following to `AppState` (or as captured variables in the signal task
|
||||
closure — whichever is cleaner given the existing structure):
|
||||
|
||||
```rust
|
||||
/// Pending outgoing upgrade: (call_id, proposal_id, profile).
|
||||
/// Set when we send an UpgradeProposal, consumed when we receive an accepted UpgradeResponse.
|
||||
pending_upgrade: Arc<Mutex<Option<(String, String, QualityProfile)>>>,
|
||||
|
||||
/// Current quality profile for the encoder. The audio send task reads this
|
||||
/// at the start of each encode cycle.
|
||||
active_quality: Arc<Mutex<QualityProfile>>,
|
||||
|
||||
/// Peer's reported maximum quality cap. The audio send task clamps to min(active, peer_max).
|
||||
peer_max_quality: Arc<Mutex<Option<QualityProfile>>>,
|
||||
```
|
||||
|
||||
If `AppState` already holds these fields (check `lib.rs` for the struct
|
||||
definition), reuse them instead of adding duplicates.
|
||||
|
||||
### Handler implementations
|
||||
|
||||
#### 1. `UpgradeProposal` (line ~1930)
|
||||
|
||||
```rust
|
||||
// Replace the TODO comment with:
|
||||
let response = SignalMessage::UpgradeResponse {
|
||||
version: wzp_proto::default_signal_version(),
|
||||
call_id: call_id.clone(),
|
||||
proposal_id: proposal_id.clone(),
|
||||
accepted: true,
|
||||
reason: None,
|
||||
};
|
||||
if let Err(e) = signal_transport.send_signal(&response).await {
|
||||
tracing::warn!("failed to send UpgradeResponse: {e}");
|
||||
}
|
||||
```
|
||||
|
||||
`signal_transport` is whatever variable holds the signal `Arc<dyn MediaTransport>`
|
||||
in scope at that match arm. Inspect the enclosing task to find the right name.
|
||||
|
||||
#### 2. `UpgradeResponse` (line ~1949)
|
||||
|
||||
```rust
|
||||
// Replace the TODO comment with:
|
||||
if accepted {
|
||||
// Retrieve the pending proposal to get the confirmed_profile.
|
||||
let maybe_proposal = pending_upgrade.lock().unwrap().take();
|
||||
if let Some((_cid, pid, profile)) = maybe_proposal {
|
||||
if pid == proposal_id {
|
||||
// Send UpgradeConfirm.
|
||||
let confirm = SignalMessage::UpgradeConfirm {
|
||||
version: wzp_proto::default_signal_version(),
|
||||
call_id: call_id.clone(),
|
||||
proposal_id: proposal_id.clone(),
|
||||
confirmed_profile: profile.clone(),
|
||||
};
|
||||
if let Err(e) = signal_transport.send_signal(&confirm).await {
|
||||
tracing::warn!("failed to send UpgradeConfirm: {e}");
|
||||
}
|
||||
// Switch our encoder.
|
||||
*active_quality.lock().unwrap() = profile;
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
If `pending_upgrade` is a captured `Arc<Mutex<...>>` in the task closure, it
|
||||
can be read/written without going through `AppState`.
|
||||
|
||||
#### 3. `UpgradeConfirm` (line ~1966)
|
||||
|
||||
```rust
|
||||
// Replace the TODO comment with:
|
||||
*active_quality.lock().unwrap() = confirmed_profile;
|
||||
```
|
||||
|
||||
The audio send task (in `engine.rs`) reads `active_quality` at the start of
|
||||
each encode cycle and reconfigures the Opus encoder bitrate accordingly.
|
||||
|
||||
#### 4. `QualityCapability` (line ~1985)
|
||||
|
||||
```rust
|
||||
// Replace the TODO comment with:
|
||||
*peer_max_quality.lock().unwrap() = Some(max_profile);
|
||||
```
|
||||
|
||||
#### 5. Audio send task changes (`engine.rs`)
|
||||
|
||||
The audio send task already runs in a loop. Add a quality-check at the top of
|
||||
each encode iteration:
|
||||
|
||||
```rust
|
||||
// At the start of the encode loop body:
|
||||
let effective_profile = {
|
||||
let active = active_quality.lock().unwrap().clone();
|
||||
let peer_cap = peer_max_quality.lock().unwrap().clone();
|
||||
match peer_cap {
|
||||
Some(cap) if cap.opus_bitrate_bps() < active.opus_bitrate_bps() => cap,
|
||||
_ => active,
|
||||
}
|
||||
};
|
||||
// Pass effective_profile to encoder if it changed since last iteration.
|
||||
```
|
||||
|
||||
`QualityProfile::opus_bitrate_bps()` already exists (check
|
||||
`crates/wzp-proto/src/codec_id.rs`). If `QualityProfile` does not have a
|
||||
direct bitrate accessor, compare using the `PartialOrd` impl or a helper that
|
||||
ranks profiles numerically.
|
||||
|
||||
To avoid calling `encoder.set_bitrate()` every single frame, cache the last
|
||||
applied profile and only reconfigure on change:
|
||||
|
||||
```rust
|
||||
let mut last_applied_profile: Option<QualityProfile> = None;
|
||||
|
||||
// Inside loop:
|
||||
if Some(&effective_profile) != last_applied_profile.as_ref() {
|
||||
encoder.set_bitrate(effective_profile.opus_bitrate_bps());
|
||||
last_applied_profile = Some(effective_profile.clone());
|
||||
}
|
||||
```
|
||||
|
||||
`encoder.set_bitrate(bps: u32)` — add this method to `OpusEncoder` in
|
||||
`crates/wzp-codec/src/opus_enc.rs` if it does not exist. It wraps
|
||||
`opus_encoder_ctl(OPUS_SET_BITRATE_REQUEST, bps)`.
|
||||
|
||||
### Unit tests
|
||||
|
||||
Add a `#[cfg(test)]` module in `lib.rs` (or a dedicated test file) that:
|
||||
|
||||
1. Creates a `LoopbackSignalTransport` stub that records sent `SignalMessage`s.
|
||||
2. Calls the `UpgradeProposal` handler logic directly, asserts that an
|
||||
`UpgradeResponse { accepted: true }` was sent.
|
||||
3. Calls the `UpgradeResponse { accepted: true }` handler with a pre-populated
|
||||
`pending_upgrade`, asserts that `UpgradeConfirm` was sent and
|
||||
`active_quality` was updated.
|
||||
|
||||
These can be pure unit tests (no Tauri or audio), since the handlers are
|
||||
pure async functions over captured state.
|
||||
|
||||
## Implementation steps
|
||||
|
||||
1. Read `desktop/src-tauri/src/lib.rs` lines 1910–1990 (the four TODO blocks)
|
||||
and the surrounding signal task structure to identify the variable names
|
||||
for `signal_transport`, `app_state`, and any existing quality-state fields.
|
||||
2. Read `desktop/src-tauri/src/engine.rs` for `CallEngine` struct fields and
|
||||
the audio send task loop.
|
||||
3. Read `crates/wzp-proto/src/codec_id.rs` for `QualityProfile` methods.
|
||||
4. Add `pending_upgrade`, `active_quality`, `peer_max_quality` to the
|
||||
appropriate shared state (or as closure captures in the signal task).
|
||||
5. Replace the 4 TODO comments with the handlers described above.
|
||||
6. Add `set_bitrate` to `OpusEncoder` if missing.
|
||||
7. Update the audio send task to read `active_quality` / `peer_max_quality`
|
||||
each iteration.
|
||||
8. Add unit tests.
|
||||
9. Run `cargo check --manifest-path desktop/src-tauri/Cargo.toml`.
|
||||
|
||||
## Files to read before implementing
|
||||
|
||||
- `desktop/src-tauri/src/lib.rs` — grep for `UpgradeProposal` to find the
|
||||
exact lines; also read the surrounding signal task for variable names.
|
||||
- `crates/wzp-proto/src/packet.rs` lines 1130–1190 — `UpgradeProposal`,
|
||||
`UpgradeResponse`, `UpgradeConfirm`, `QualityCapability` struct layouts.
|
||||
- `desktop/src-tauri/src/engine.rs` — `CallEngine` struct fields, audio
|
||||
send task loop.
|
||||
- `crates/wzp-proto/src/codec_id.rs` — `QualityProfile` methods.
|
||||
- `crates/wzp-codec/src/opus_enc.rs` — `OpusEncoder` API.
|
||||
|
||||
## Verify
|
||||
|
||||
```bash
|
||||
cargo check --manifest-path desktop/src-tauri/Cargo.toml
|
||||
cargo test -p wzp-desktop 2>/dev/null || cargo test --manifest-path desktop/src-tauri/Cargo.toml
|
||||
```
|
||||
|
||||
Expected: 0 errors; unit tests pass.
|
||||
|
||||
## Done when
|
||||
|
||||
- All 4 TODO comments replaced with real logic.
|
||||
- `cargo check --manifest-path desktop/src-tauri/Cargo.toml` exits 0.
|
||||
- Unit test verifies: `UpgradeProposal` → `UpgradeResponse { accepted: true }` sent;
|
||||
`UpgradeResponse { accepted: true }` → `UpgradeConfirm` sent + `active_quality` updated.
|
||||
242
docs/PRD/PRD-wire-format-hardening.md
Normal file
242
docs/PRD/PRD-wire-format-hardening.md
Normal file
@@ -0,0 +1,242 @@
|
||||
# PRD: Wire Format Hardening — FEC block_id u16, SignalMessage version byte, FEC repair index wrap
|
||||
|
||||
> **Status:** proposed
|
||||
> **Resolves:** Three small wire-format defects (H2, M1, M4) that compound over time into silent data corruption or protocol breakage.
|
||||
> **Depends on:** Nothing — purely mechanical changes to `wzp-fec` and `wzp-proto`.
|
||||
|
||||
## Problem
|
||||
|
||||
Three independent issues:
|
||||
|
||||
**H2 — `fec_block_id` u8 wraps too fast.** The `block_id` field in
|
||||
`RaptorQFecEncoder` (and `RaptorQFecDecoder`) is `u8`. At 5 audio frames
|
||||
per block and 50 fps this wraps every ~51 seconds. A slow receiver or a
|
||||
mid-session join can receive packets from two different blocks with the same
|
||||
`block_id`, silently corrupting FEC recovery.
|
||||
|
||||
**M1 — Some `SignalMessage` variants lack a `version` byte.** Most variants
|
||||
have `#[serde(default = "default_signal_version")] version: u8`. The unit
|
||||
variant `Reflect` (and potentially others added recently) does not. Future
|
||||
protocol changes that key on `version` will silently misparse old messages
|
||||
from peers without the field.
|
||||
|
||||
**M4 — FEC repair index can silently wrap at 255.** In
|
||||
`crates/wzp-fec/src/encoder.rs` line 140:
|
||||
|
||||
```rust
|
||||
let idx = (num_source as u16).wrapping_add(i as u16);
|
||||
```
|
||||
|
||||
(The line was already fixed to `u16` — verify it is `u16`, not `u8`. If it
|
||||
is still `u8`, the fix is below.)
|
||||
|
||||
If the line currently reads `(num_source as u8).wrapping_add(i as u8)`, then
|
||||
when `num_source + repair_count > 255` the repair symbol indices wrap silently,
|
||||
producing incorrect ESI values that the decoder cannot correlate to source
|
||||
blocks.
|
||||
|
||||
## Goals
|
||||
|
||||
- **H2**: Widen `block_id` in encoder and decoder from `u8` to `u16`.
|
||||
Update `finalize_block` return type and `current_block_id` return type in
|
||||
the trait (`wzp-proto`) and implementations (`wzp-fec`).
|
||||
- **M1**: Audit every `SignalMessage` variant; add
|
||||
`#[serde(default = "default_signal_version")] version: u8` to any that
|
||||
are missing it.
|
||||
- **M4**: Confirm the repair index uses `u16`; fix it if it is still `u8`.
|
||||
Update the decoder's `add_symbol` call site if the index type changes.
|
||||
- `cargo test -p wzp-fec -p wzp-proto` passes; no existing tests broken.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Changing the wire encoding of `MediaHeaderV2::fec_block` — it is already
|
||||
`u16` on the wire. This PRD only widens the **internal counter** to match.
|
||||
- Multi-block decode concurrency or block expiry policy.
|
||||
- Any crate outside `wzp-fec` and `wzp-proto`.
|
||||
|
||||
## Design
|
||||
|
||||
### Item A — `fec_block_id` u8 → u16
|
||||
|
||||
**Files**:
|
||||
- `crates/wzp-proto/src/traits.rs` — `FecEncoder` and `FecDecoder` traits
|
||||
- `crates/wzp-fec/src/encoder.rs` — `RaptorQFecEncoder`
|
||||
- `crates/wzp-fec/src/decoder.rs` — `RaptorQFecDecoder`
|
||||
|
||||
**Trait changes** (`traits.rs`):
|
||||
|
||||
```rust
|
||||
// Before:
|
||||
fn finalize_block(&mut self) -> Result<u8, FecError>;
|
||||
fn current_block_id(&self) -> u8;
|
||||
fn add_symbol(&mut self, block_id: u8, ...) -> Result<(), FecError>;
|
||||
fn try_decode(&mut self, block_id: u8) -> Result<...>;
|
||||
fn expire_before(&mut self, block_id: u8);
|
||||
```
|
||||
|
||||
```rust
|
||||
// After:
|
||||
fn finalize_block(&mut self) -> Result<u16, FecError>;
|
||||
fn current_block_id(&self) -> u16;
|
||||
fn add_symbol(&mut self, block_id: u16, ...) -> Result<(), FecError>;
|
||||
fn try_decode(&mut self, block_id: u16) -> Result<...>;
|
||||
fn expire_before(&mut self, block_id: u16);
|
||||
```
|
||||
|
||||
**Encoder changes** (`encoder.rs`):
|
||||
|
||||
- Change `block_id: u8` field to `block_id: u16`.
|
||||
- Update `self.block_id.wrapping_add(1)` (already u16 semantics; keep as is).
|
||||
- Update `finalize_block` to return `u16`.
|
||||
- Update `current_block_id` to return `u16`.
|
||||
- Update all tests that assert `block_id == 0u8` → `== 0u16`, and the
|
||||
wrap test (`block_id_wraps`) to iterate to `u16::MAX` (65535) — or reduce
|
||||
it to 300 iterations to keep it fast, asserting the wrap at 65536.
|
||||
|
||||
The wrap test at 256 iterations (`0..=255u8`) must be updated; a full
|
||||
`u16` wrap test at 65536 iterations is too slow for CI. Change to:
|
||||
|
||||
```rust
|
||||
#[test]
|
||||
fn block_id_wraps_u16() {
|
||||
let mut enc = RaptorQFecEncoder::with_defaults(1);
|
||||
// Advance 300 blocks and verify no panic + monotonic increment.
|
||||
for expected in 0..300u16 {
|
||||
assert_eq!(enc.current_block_id(), expected);
|
||||
enc.add_source_symbol(&[0u8; 10]).unwrap();
|
||||
enc.finalize_block().unwrap();
|
||||
}
|
||||
// Explicitly test wrap at u16 boundary.
|
||||
let mut enc2 = RaptorQFecEncoder::with_defaults(1);
|
||||
enc2.block_id = u16::MAX;
|
||||
enc2.add_source_symbol(&[0u8; 10]).unwrap();
|
||||
let id = enc2.finalize_block().unwrap();
|
||||
assert_eq!(id, u16::MAX);
|
||||
assert_eq!(enc2.current_block_id(), 0);
|
||||
}
|
||||
```
|
||||
|
||||
Note: `block_id` is a private field; expose a test helper or set it in a
|
||||
`#[cfg(test)]` `impl` block.
|
||||
|
||||
**Decoder changes** (`decoder.rs`):
|
||||
|
||||
- Change `blocks: HashMap<u8, BlockState>` to `HashMap<u16, BlockState>`.
|
||||
- Update `get_or_create_block(block_id: u8)` → `get_or_create_block(block_id: u16)`.
|
||||
- Update `add_symbol`, `try_decode`, `expire_before` signatures to `u16`.
|
||||
- The `SourceBlockEncoder::new(self.block_id, ...)` call in `encoder.rs` passes
|
||||
`block_id` to `raptorq`. RaptorQ uses `u8` for source block number internally.
|
||||
Cast it: `(block_id & 0xFF) as u8` or `(block_id % 256) as u8` — the `raptorq`
|
||||
crate's source block ID is a logical identifier within a single object
|
||||
transmission, not a global counter. The u16 is our session counter; truncate
|
||||
to u8 when calling into raptorq.
|
||||
|
||||
### Item B — `SignalMessage` version byte audit
|
||||
|
||||
**File**: `crates/wzp-proto/src/packet.rs`
|
||||
|
||||
Read every variant in the `SignalMessage` enum (lines 555–1241) and check
|
||||
for the presence of:
|
||||
|
||||
```rust
|
||||
#[serde(default = "default_signal_version")]
|
||||
version: u8,
|
||||
```
|
||||
|
||||
The `Reflect` variant at line 974 is a **unit variant** (no fields). Unit
|
||||
variants cannot carry a `version` field without becoming struct variants.
|
||||
Change it to a struct variant:
|
||||
|
||||
```rust
|
||||
// Before:
|
||||
Reflect,
|
||||
|
||||
// After:
|
||||
Reflect {
|
||||
#[serde(default = "default_signal_version")]
|
||||
version: u8,
|
||||
},
|
||||
```
|
||||
|
||||
This is a wire-compatible change: serde JSON struct variants serialize as
|
||||
`{"Reflect": {"version": 1}}` whereas unit variants serialize as
|
||||
`"Reflect"`. These are **not** backward-compatible formats. Since `Reflect`
|
||||
is sent client → relay only and the relay immediately responds, upgrading
|
||||
both sides atomically is acceptable. Add a serde test to confirm round-trip.
|
||||
|
||||
For any other variants missing `version`, follow the same pattern as all
|
||||
existing variants.
|
||||
|
||||
Verify by grepping the enum for variants that do NOT have `version`:
|
||||
|
||||
```bash
|
||||
grep -A3 "^\s*[A-Z][A-Za-z]*\s*{" crates/wzp-proto/src/packet.rs | \
|
||||
grep -B1 -v "serde.*default_signal_version\|version:"
|
||||
```
|
||||
|
||||
### Item C — FEC repair index wrap (M4)
|
||||
|
||||
**File**: `crates/wzp-fec/src/encoder.rs`, line ~140.
|
||||
|
||||
Current code:
|
||||
|
||||
```rust
|
||||
let idx = (num_source as u16).wrapping_add(i as u16);
|
||||
```
|
||||
|
||||
If this line already uses `u16` (as shown in the file at line 140), M4 is
|
||||
already fixed. Verify by reading the current file. If it still reads
|
||||
`u8`, apply:
|
||||
|
||||
```rust
|
||||
let idx = (num_source as u16).wrapping_add(i as u16);
|
||||
```
|
||||
|
||||
**Decoder** (`crates/wzp-fec/src/decoder.rs`): `add_symbol` already accepts
|
||||
`symbol_index: u16` (per the trait). Confirm the parameter flows through to
|
||||
`PayloadId::new(block_id_u8, symbol_index as u32)` without truncation.
|
||||
|
||||
## Implementation steps
|
||||
|
||||
1. Read `crates/wzp-proto/src/traits.rs` lines 60–116 (FecEncoder/FecDecoder
|
||||
trait definitions) to confirm current signatures.
|
||||
2. Read `crates/wzp-fec/src/encoder.rs` and `decoder.rs` (full files).
|
||||
3. Apply Item C fix first (smallest change, easiest to verify).
|
||||
4. Apply Item A: widen `block_id` from u8 to u16 in traits, encoder, decoder.
|
||||
Update all callers by running `cargo check -p wzp-fec -p wzp-proto` and
|
||||
fixing each E0308/E0308 error.
|
||||
5. Apply Item B: read every variant, add missing `version` fields.
|
||||
Change `Reflect` to a struct variant.
|
||||
6. Run tests.
|
||||
|
||||
## Files to read before implementing
|
||||
|
||||
- `crates/wzp-proto/src/traits.rs` lines 60–116 (trait signatures)
|
||||
- `crates/wzp-fec/src/encoder.rs` (full)
|
||||
- `crates/wzp-fec/src/decoder.rs` (full)
|
||||
- `crates/wzp-proto/src/packet.rs` lines 555–1241 (all `SignalMessage` variants)
|
||||
|
||||
## Verify
|
||||
|
||||
```bash
|
||||
cargo test -p wzp-fec -p wzp-proto
|
||||
```
|
||||
|
||||
Expected: all tests pass, 0 failures. Also run:
|
||||
|
||||
```bash
|
||||
cargo check --workspace
|
||||
```
|
||||
|
||||
to catch any call sites outside `wzp-fec` and `wzp-proto` that passed `u8`
|
||||
block IDs to the trait methods.
|
||||
|
||||
## Done when
|
||||
|
||||
- `cargo test -p wzp-fec -p wzp-proto` exits 0.
|
||||
- `block_id` is `u16` in `RaptorQFecEncoder`, `RaptorQFecDecoder`, and the
|
||||
`FecEncoder`/`FecDecoder` traits.
|
||||
- Every non-unit `SignalMessage` variant has a `version: u8` field with
|
||||
`#[serde(default = "default_signal_version")]`.
|
||||
- Repair index in `encoder.rs` is computed with `u16` arithmetic.
|
||||
- No existing tests are broken.
|
||||
165
docs/bugs/002-macos-vpio-playout-silent.md
Normal file
165
docs/bugs/002-macos-vpio-playout-silent.md
Normal file
@@ -0,0 +1,165 @@
|
||||
# BUG-002: macOS VPIO Playout Silent — Audio Decoded But Not Heard
|
||||
|
||||
**Severity:** P0 — outgoing audio (Mac mic → peer) works, but the user hears nothing on the Mac side
|
||||
**Status:** Instrumented on 2026-05-25; awaiting next VPIO vs CPAL repro
|
||||
**Branch:** `experimental-ui`
|
||||
**Build observed:** `01f55ca` (Mac desktop), same-day Android `01f55ca`
|
||||
**Last investigated:** 2026-05-25
|
||||
**Platforms confirmed affected:** macOS desktop (VPIO path)
|
||||
|
||||
---
|
||||
|
||||
## Symptom
|
||||
|
||||
In a relay-forwarded group call between macOS and Android in the same room (`General`, `count:2`):
|
||||
|
||||
- The Mac user can be **heard** on Android (Mac→Android leg works).
|
||||
- The Mac user **hears nothing** when the Android peer speaks (Android→Mac playout silent).
|
||||
- Muting the Android peer's mic results in total silence on both ends — confirming the only audio the user perceived during the call was the Mac→Android leg playing through the Android speaker.
|
||||
|
||||
This was initially misreported as "I hear myself on Android" — the user was hearing their own Mac mic looped through Android playout, not an actual echo bug.
|
||||
|
||||
---
|
||||
|
||||
## Evidence
|
||||
|
||||
### Mac log excerpt (`01f55ca`, fingerprint `63ba…`, 10:31:22)
|
||||
|
||||
```
|
||||
10:31:23 media:room_update {"count":2, participants:[Akbar fa06…, Manwe 63ba…]}
|
||||
10:31:23 media:first_recv {"codec":"Opus24k","payload_bytes":27,"t_ms":933}
|
||||
10:31:25 media:recv_heartbeat {"codec":"Opus24k","decode_errs":0,"decoded_frames":140,"last_written":960,"written_samples":134400}
|
||||
10:31:29 media:recv_heartbeat {"codec":"Opus32k","decoded_frames":338,"last_written":960,"written_samples":324480}
|
||||
10:31:35 media:recv_heartbeat {"codec":"Opus6k","decoded_frames":595,"last_written":1920,"written_samples":618240}
|
||||
…
|
||||
10:31:57 media:recv_heartbeat {"codec":"Opus6k","decoded_frames":1086,"last_written":1920,"written_samples":1560960}
|
||||
```
|
||||
|
||||
Recv path is healthy:
|
||||
- `decode_errs:0` throughout
|
||||
- `decoded_frames` climbs monotonically 140 → 1086
|
||||
- `written_samples` reaches 1.56 M (≈32 s of 48 kHz mono)
|
||||
- `last_written` correctly flips 960 (Opus24k/32k, 20 ms) ↔ 1920 (Opus6k, 40 ms)
|
||||
|
||||
**Conclusion:** packets arrive, decode succeeds, samples are written into `playout_ring`. The breakage is **downstream of the ring write**, i.e. in the macOS playout consumer (the VPIO `set_render_callback`).
|
||||
|
||||
### Mac send path also works
|
||||
`media:send_heartbeat` shows `last_rms` spiking to 168, 477, 867, 1458 in response to speech. Android's recv log for the same window decoded those frames successfully.
|
||||
|
||||
---
|
||||
|
||||
## Suspected Root Cause
|
||||
|
||||
`crates/wzp-client/src/audio_vpio.rs:128–147` — the render (output) callback reads from `playout_ring` in `FRAME_SAMPLES` (960) chunks. Three plausible failure modes:
|
||||
|
||||
### Hypothesis A: Codec-change frame-size mismatch
|
||||
Mid-call codec switches (`Opus24k` → `Opus32k` → `Opus6k`) change the frame size written into the ring (960 ↔ 1920 samples per frame). The render callback reads in fixed 960-sample chunks. The ring is FIFO and should absorb this, but if `AudioRing` semantics drop partial frames or stall on alignment, the consumer side could starve while `written_samples` continues to climb on the producer side.
|
||||
|
||||
`engine.rs:1852` and `engine.rs:1895` write into `playout_ring` directly with the decoder's output length (variable). Worth confirming `AudioRing::read` handles arbitrary chunk sizes vs producer.
|
||||
|
||||
### Hypothesis B: VPIO output element never actually started
|
||||
`audio_vpio.rs:151` calls `au.start()` once on the combined VPIO unit. VPIO is supposed to start both input and output elements together, but if AEC initialization fails silently, output rendering may be suppressed while input still produces callbacks. The `[vpio] capture callback: N f32 samples` log line proves input callbacks fire — but there is **no equivalent log line for the render callback**. We do not know whether the render callback is being invoked at all.
|
||||
|
||||
### Hypothesis C: Output device routing
|
||||
VPIO may have grabbed an unexpected default output (e.g. the previous Bluetooth headset, an HDMI sink, or a virtual device). The render callback runs and pulls samples, but they're sent to a device the user can't hear.
|
||||
|
||||
### Hypothesis D: AEC over-suppression
|
||||
VPIO's AEC uses the render callback as the far-end reference. If the unit decides the far-end and near-end are too correlated (it shouldn't here — different speakers in different rooms), it could attenuate playout. Unlikely to produce 100 % silence but listed for completeness.
|
||||
|
||||
---
|
||||
|
||||
## Instrumentation Added
|
||||
|
||||
As of the current workspace, the desktop client emits VPIO render/capture counters into the normal call debug log when OS AEC is enabled:
|
||||
|
||||
```
|
||||
vpio:render_heartbeat {
|
||||
"capture_callbacks": ...,
|
||||
"capture_samples": ...,
|
||||
"render_callbacks": ...,
|
||||
"render_requested_samples": ...,
|
||||
"render_read_samples": ...,
|
||||
"render_underrun_callbacks": ...,
|
||||
"render_nonzero_callbacks": ...,
|
||||
"render_last_requested": ...,
|
||||
"render_last_read": ...,
|
||||
"render_last_rms": ...,
|
||||
"render_last_ring_available": ...
|
||||
}
|
||||
```
|
||||
|
||||
Interpretation:
|
||||
|
||||
- `render_callbacks == 0`: VPIO output callback is not running. Focus on VPIO initialization / output element start.
|
||||
- `render_callbacks > 0` and `render_read_samples == 0` while `media:recv_heartbeat.written_samples` climbs: VPIO callback runs but the ring it reads is not receiving the same samples the recv task writes, or the callback is draining before writes arrive.
|
||||
- `render_read_samples > 0` and `render_last_rms > 0` while the user hears silence: VPIO is feeding non-zero speaker samples to CoreAudio; focus on output device routing or VoiceProcessingIO suppression.
|
||||
- CPAL fallback test: disable OS AEC in settings. If CPAL playback is audible with the same call, the failure is VPIO-specific.
|
||||
|
||||
## Proposed Diagnostic Steps (Prioritized)
|
||||
|
||||
1. **Reproduce with current instrumentation** and compare `media:recv_heartbeat` to `vpio:render_heartbeat`.
|
||||
|
||||
2. **One-shot render callback stderr log is now present** (`audio_vpio.rs`) mirroring the existing capture-side `eprintln!`:
|
||||
```rust
|
||||
let logged_render = Arc::new(AtomicBool::new(false));
|
||||
…
|
||||
if !logged_render.swap(true, Ordering::Relaxed) {
|
||||
eprintln!("[vpio] render callback: {} f32 samples, ring_read={}", ch.len(), read);
|
||||
}
|
||||
```
|
||||
This will immediately distinguish Hypothesis B (callback never fires) from A/C/D (callback fires but output is silent or misrouted).
|
||||
|
||||
3. **Periodically log render-callback stats** — total samples pulled from ring, samples requested per callback, non-zero render callback count, and last render RMS. Compare against producer-side `written_samples` to confirm consumer is keeping up.
|
||||
|
||||
4. **Verify output device** via `AudioUnitGetProperty(kAudioOutputUnitProperty_CurrentDevice, Output)` immediately after `au.start()`. Log device name. If it doesn't match the user's intended speaker, force-set the default output device.
|
||||
|
||||
5. **Test with codec pinned** — set `WZP_FORCE_CODEC=Opus24k` (or wire a temporary CLI flag) so codec doesn't change mid-call. If audio works with a pinned codec, Hypothesis A is confirmed and `AudioRing` chunk handling needs review.
|
||||
|
||||
6. **Compare CPAL fallback path** — disable OS AEC in settings and reproduce. If CPAL playback works, the bug is VPIO-specific.
|
||||
|
||||
---
|
||||
|
||||
## Open Questions
|
||||
|
||||
- Does the macOS render callback have permission to write to the user's selected output device? Apple changed CoreAudio output-device permission semantics in macOS 14+.
|
||||
- Is `_audio_unit: AudioUnit` being dropped early? It's stored in `VpioAudio` and that struct is boxed into `audio_handle: Box<dyn Any + Send>` in `engine.rs:1573`, which is held by `CallEngine`. Should be alive for the call duration — confirm no early-drop path.
|
||||
- Are there any `os_log` / Console.app warnings from `AudioToolbox` / `CoreAudio` / `AVAudioSession` during the call?
|
||||
|
||||
---
|
||||
|
||||
## Reproduction Steps
|
||||
|
||||
1. Start macOS desktop client (build `01f55ca` or later), join relay `193.180.213.68:4433`, room `General`.
|
||||
2. Start Android client (same build), join same relay + room.
|
||||
3. Confirm `media:room_update count:2` on both ends.
|
||||
4. Speak into Android mic.
|
||||
5. Observe: Mac log shows `decoded_frames` climbing, `decode_errs:0`, `written_samples` increasing. User hears nothing on Mac speakers.
|
||||
6. Speak into Mac mic — Android user hears Mac audio fine, confirming Mac→Android works.
|
||||
|
||||
---
|
||||
|
||||
## Related Files
|
||||
|
||||
- `crates/wzp-client/src/audio_vpio.rs:128–147` — render callback (primary suspect)
|
||||
- `crates/wzp-client/src/audio_vpio.rs:35–161` — full VPIO start sequence
|
||||
- `crates/wzp-client/src/audio_ring.rs` — ring buffer used by both producer and consumer
|
||||
- `desktop/src-tauri/src/engine.rs:1562–1600` — VPIO vs CPAL selection
|
||||
- `desktop/src-tauri/src/engine.rs:1760–1900` — recv task writing into `playout_ring`
|
||||
|
||||
---
|
||||
|
||||
## Fix Plan (Once Diagnosed)
|
||||
|
||||
| Diagnosis | Fix |
|
||||
|-----------|-----|
|
||||
| A — frame-size mismatch | Make `AudioRing` consumer drain variable chunks, or buffer to fixed 960 in recv task before ring write |
|
||||
| B — render callback not firing | Investigate VPIO initialization order; consider separate input + output `AudioUnit` instances |
|
||||
| C — wrong output device | Set `kAudioOutputUnitProperty_CurrentDevice` explicitly to `kAudioObjectSystemObject` default output at start |
|
||||
| D — AEC suppression | Test with VPIO bypass mode (`kAUVoiceIOProperty_BypassVoiceProcessing`) on; if audio returns, file CoreAudio quirk and tune AEC config |
|
||||
|
||||
---
|
||||
|
||||
## Cross-References
|
||||
|
||||
- BUG-001 (Android join-voice hang) — separate issue, already mitigated; current Android build joins room successfully and recv works.
|
||||
- Memory: `project_desktop_client.md` notes the desktop rewrite uses CPAL + VoiceProcessingIO with "direct playout, OS-level AEC" — this bug is the first failure of that path under real call conditions.
|
||||
415
docs/bugs/003-android-to-macos-video-banding.md
Normal file
415
docs/bugs/003-android-to-macos-video-banding.md
Normal file
@@ -0,0 +1,415 @@
|
||||
# BUG-003: Android to macOS Video Banding / Horizontal Lines
|
||||
|
||||
**Severity:** P0/P1 - Android camera video is visibly corrupted on macOS at common resolutions.
|
||||
**Status:** Root cause identified 2026-05-26; candidate fix in `crates/wzp-video/src/videotoolbox.rs`. Awaiting on-device verification.
|
||||
**Branch:** `main`.
|
||||
**Latest build observed:** `3ea25a0` (`fix(android): use MediaCodec input layout for video encode`).
|
||||
**Direction affected:** Android camera -> macOS desktop display.
|
||||
**Direction mostly OK:** macOS camera -> Android display.
|
||||
|
||||
---
|
||||
|
||||
## Root Cause (2026-05-26)
|
||||
|
||||
The Android H.264 bitstream is **valid**: the locally-encoded `.h264` files and
|
||||
the macOS-reassembled `.h264` files both decode cleanly with software ffmpeg.
|
||||
SPS reports the expected `960x540`, `coded_height=544`, `yuv420p`, High profile,
|
||||
level 3.1.
|
||||
|
||||
The corruption appears purely on the macOS receive side. The shiguredo
|
||||
`I420Frame` wrapper around `CVPixelBuffer` exposes each plane as
|
||||
`bytes_per_row * height` bytes — i.e. the raw plane buffer including the
|
||||
per-row stride padding that CoreVideo adds for alignment. `VideoToolboxDecoder`
|
||||
was concatenating those slices verbatim, then handing the buffer downstream
|
||||
tagged as tight I420 of `width x height`. The JPEG-encoding consumer
|
||||
(`i420_to_jpeg_bytes` in `desktop/src-tauri/src/lib.rs`) indexes the buffer
|
||||
with tight strides `width` and `width/2`, so any plane where
|
||||
`bytes_per_row > tight_stride` produces per-row drift in the consumer's reads.
|
||||
|
||||
Numerical confirmation from the corrupted dump
|
||||
`000002_desktop_remote_decoded_f000001_960x540.jpg`:
|
||||
|
||||
- Banding period along the diagonal: exactly **32 luma rows** = 16 chroma rows.
|
||||
- Per-column-slice peak offsets shift by ~5 rows per 230-column step, i.e. the
|
||||
bands are a tilted diagonal, not horizontal — consistent with one chroma row
|
||||
of drift accumulating per 16 chroma rows of consumer read.
|
||||
- Solving `u_stride / (u_stride - chroma_width) = 16` with `chroma_width = 480`
|
||||
yields `u_stride = 512`. That is exactly the 64-byte aligned chroma stride
|
||||
CoreVideo emits for a 480-wide plane.
|
||||
- Luma at 960 wide is already 64-aligned, so `y_stride = 960` and the luma
|
||||
plane is unaffected. This matches the bug doc note that 640x360 looks fine
|
||||
(chroma_width 320 is also 64-aligned, no padding needed).
|
||||
|
||||
## Fix
|
||||
|
||||
`crates/wzp-video/src/videotoolbox.rs` now has an `i420_frame_to_tight` helper
|
||||
that copies each plane row-by-row using its own `bytes_per_row`, producing a
|
||||
genuine tight I420 buffer of `width * height + 2 * (width/2) * (height/2)`
|
||||
bytes. All three decoders (H.264, HEVC, AV1) call the helper instead of
|
||||
concatenating raw plane slices. On the first successful decode each decoder
|
||||
logs the actual plane dimensions and strides (`tracing::info!` at target
|
||||
`wzp_video::videotoolbox`) so future similar bugs are easier to diagnose
|
||||
without re-deriving from band spacing.
|
||||
|
||||
---
|
||||
|
||||
## Symptom
|
||||
|
||||
When Android sends camera video to macOS, the macOS view shows repeated horizontal green/magenta line bands over the decoded picture. The lines cover the whole decoded frame, including black side bars added by the Android portrait-camera contain/crop fix.
|
||||
|
||||
The Android camera crop/zoom problem is fixed now: the Android front camera is no longer cover-cropped into an extreme zoom. The remaining bug is the line/banding corruption.
|
||||
|
||||
The issue is easy to see at H.264 960x540. At 640x360 it has been reported as visually good or much better. HEVC behaves differently: minimum resolution can look good, but 960x540 and 1280x720 tend to pause or deliver only bursts of frames.
|
||||
|
||||
---
|
||||
|
||||
## Current State
|
||||
|
||||
Recent commits relevant to this bug:
|
||||
|
||||
```text
|
||||
3ea25a0 fix(android): use MediaCodec input layout for video encode
|
||||
1124726 fix(video): add frame metadata and Android encode diagnostics
|
||||
9a77459 feat(video): add codec and resolution controls
|
||||
f85efb9 fix(video): improve android stream smoothness
|
||||
31b2caa fix(video): request keyframes after packet loss
|
||||
079e21e fix(video): resync decoder after packet gaps
|
||||
e676641 fix(android): suppress debuggable lint for diagnostic builds
|
||||
9713efc chore(android): add release debuggable build
|
||||
```
|
||||
|
||||
Important behavior:
|
||||
|
||||
- Android source dumps are clean.
|
||||
- Android I420 roundtrip dumps are clean.
|
||||
- macOS decoded remote Android frames are corrupted.
|
||||
- Android receiving macOS video is generally clean.
|
||||
- Transport/reassembly is probably not the primary issue: early Android local encoded `.h264` files match the corresponding macOS remote reassembled `.h264` prefix/length.
|
||||
- The bug is likely in Android MediaCodec encoder input layout/color handling, H.264 non-macroblock-aligned dimensions/cropping, or macOS VideoToolbox interpretation of Android-encoded H.264.
|
||||
|
||||
---
|
||||
|
||||
## Reproduction Build
|
||||
|
||||
Use the Tauri Android pipeline, not the legacy native Android Gradle app.
|
||||
|
||||
```bash
|
||||
cd /Users/manwe/CascadeProjects/warzonePhone
|
||||
git status --short
|
||||
git log -1 --oneline
|
||||
./scripts/android-build-async.sh --release-debuggable --wait
|
||||
```
|
||||
|
||||
The APK lands here:
|
||||
|
||||
```bash
|
||||
/Users/manwe/CascadeProjects/warzonePhone/target/tauri-android-apk/wzp-tauri-arm64.apk
|
||||
```
|
||||
|
||||
Install it:
|
||||
|
||||
```bash
|
||||
adb install -r /Users/manwe/CascadeProjects/warzonePhone/target/tauri-android-apk/wzp-tauri-arm64.apk
|
||||
```
|
||||
|
||||
Use `--release-debuggable` for this bug. Plain debug builds can mask the issue because they run at much lower frame rate and look like a slideshow. Plain release builds are not usable for `run-as` frame-dump retrieval.
|
||||
|
||||
Critical build trap: `scripts/android-build-async.sh` runs `scripts/build-tauri-android.sh`, which SSHes to `SepehrHomeserverdk` and resets the remote source to `origin/$BRANCH`. Uncommitted local changes are ignored by the Android build. Commit and push before building, or the phone may run old code.
|
||||
|
||||
---
|
||||
|
||||
## macOS Build / Run
|
||||
|
||||
For local desktop repro:
|
||||
|
||||
```bash
|
||||
cd /Users/manwe/CascadeProjects/warzonePhone/desktop
|
||||
npm install
|
||||
npm run tauri dev
|
||||
```
|
||||
|
||||
Enable call debug logs in the app settings before starting the call. The in-app call log only keeps the last 200 entries; use the copy/share buttons if preserving textual logs matters.
|
||||
|
||||
---
|
||||
|
||||
## Repro Steps
|
||||
|
||||
1. Start the macOS desktop client.
|
||||
2. Start the Android `--release-debuggable` APK.
|
||||
3. Join the same room, usually `general`.
|
||||
4. Use the same relay as the current manual tests, e.g. `172.16.81.135:4433`, unless testing relay-specific behavior.
|
||||
5. Turn camera on for both clients.
|
||||
6. Set both sides to H.264.
|
||||
7. Set Android send resolution to 960x540. Mac can be 960x540 or higher.
|
||||
8. Observe Android camera video on macOS.
|
||||
|
||||
Expected failure: macOS shows Android video with repeated horizontal green/magenta lines. Android camera source preview and Android frame dumps are clean.
|
||||
|
||||
Useful comparison tests:
|
||||
|
||||
| Codec / resolution | Observed result |
|
||||
|---|---|
|
||||
| H.264 960x540 | Lines/banding on macOS for Android video |
|
||||
| H.264 640x360 | Reported good or much better; smoother |
|
||||
| H.264 1280x720 | Lines/banding and/or worse smoothness |
|
||||
| HEVC 1280x720 | Mac video smooth on Android; Android video on Mac pauses and can look zoomed/corrupt |
|
||||
| HEVC 960x540 | Same pause pattern, shorter pauses |
|
||||
| HEVC minimum resolution | Reported good on both devices |
|
||||
|
||||
---
|
||||
|
||||
## Artifact Collection
|
||||
|
||||
### Clear old dumps before a fresh run
|
||||
|
||||
macOS:
|
||||
|
||||
```bash
|
||||
rm -rf "$HOME/Library/Application Support/com.wzp.desktop/.wzp/frame-dumps"
|
||||
```
|
||||
|
||||
Android:
|
||||
|
||||
```bash
|
||||
adb shell run-as com.wzp.desktop rm -rf .wzp/frame-dumps
|
||||
```
|
||||
|
||||
The Android clear command requires a debuggable build. If `run-as` fails, rebuild with `--release-debuggable`.
|
||||
|
||||
### Pull Android dumps
|
||||
|
||||
```bash
|
||||
cd /Users/manwe/CascadeProjects/warzonePhone
|
||||
./scripts/pull-android-frame-dumps.sh
|
||||
```
|
||||
|
||||
Output directory:
|
||||
|
||||
```text
|
||||
/Users/manwe/CascadeProjects/warzonePhone/android-frame-dumps/frame-dumps
|
||||
```
|
||||
|
||||
The pull script packages files using:
|
||||
|
||||
```bash
|
||||
adb exec-out "run-as com.wzp.desktop tar -C .wzp -cf - frame-dumps"
|
||||
```
|
||||
|
||||
### macOS dump directory
|
||||
|
||||
```text
|
||||
/Users/manwe/Library/Application Support/com.wzp.desktop/.wzp/frame-dumps
|
||||
```
|
||||
|
||||
### Important dump names
|
||||
|
||||
| Dump suffix | Meaning |
|
||||
|---|---|
|
||||
| `android_camera_jpeg_in_fXXXXXX_<WxH>.jpg` | Raw browser/camera JPEG entering Rust from Android WebView |
|
||||
| `android_camera_i420_roundtrip_fXXXXXX_<WxH>.jpg` | Android camera frame after JS/canvas -> Rust I420 conversion, converted back to JPEG |
|
||||
| `android_local_encoded_fXXXXXX.h264` / `.h265` | Encoded Android camera bitstream before packetization |
|
||||
| `desktop_remote_encoded_reassembled_fXXXXXX.h264` / `.h265` | macOS reassembled encoded bitstream received from Android |
|
||||
| `desktop_remote_decoded_fXXXXXX_<WxH>.jpg` | macOS decoded Android video frame, where the lines show |
|
||||
| `android_remote_decoded_fXXXXXX_<WxH>.jpg` | Android decoded macOS video frame |
|
||||
|
||||
Known useful local examples from the latest sessions:
|
||||
|
||||
```text
|
||||
Clean Android source:
|
||||
/Users/manwe/CascadeProjects/warzonePhone/android-frame-dumps/frame-dumps/000407_android_camera_jpeg_in_f000150_960x540.jpg
|
||||
/Users/manwe/CascadeProjects/warzonePhone/android-frame-dumps/frame-dumps/000408_android_camera_i420_roundtrip_f000150_960x540.jpg
|
||||
|
||||
Corrupt macOS decode:
|
||||
/Users/manwe/Library/Application Support/com.wzp.desktop/.wzp/frame-dumps/000236_desktop_remote_decoded_f000030_960x540.jpg
|
||||
/Users/manwe/Library/Application Support/com.wzp.desktop/.wzp/frame-dumps/000241_desktop_remote_decoded_f000060_960x540.jpg
|
||||
/Users/manwe/Library/Application Support/com.wzp.desktop/.wzp/frame-dumps/000244_desktop_remote_decoded_f000090_960x540.jpg
|
||||
|
||||
Encoded bitstream comparison:
|
||||
/Users/manwe/CascadeProjects/warzonePhone/android-frame-dumps/frame-dumps/000005_android_local_encoded_f000001.h264
|
||||
/Users/manwe/Library/Application Support/com.wzp.desktop/.wzp/frame-dumps/000064_desktop_remote_encoded_reassembled_f000001.h264
|
||||
```
|
||||
|
||||
These files are local artifacts, not committed test fixtures.
|
||||
|
||||
---
|
||||
|
||||
## Text Logs
|
||||
|
||||
### In-app call debug log
|
||||
|
||||
Enable `Call debug logs` in settings before joining. The UI buffer is limited to 200 entries. Use the in-app copy/share buttons immediately after the repro.
|
||||
|
||||
Useful events:
|
||||
|
||||
```text
|
||||
camera:get_user_media_ok
|
||||
camera:capture_clock
|
||||
camera:capture_frame
|
||||
video:first_camera_frame
|
||||
video:camera_frame_sample
|
||||
video:encoded_frame
|
||||
video:first_send
|
||||
video:first_recv
|
||||
video:first_reassembled
|
||||
video:reassembled_frame
|
||||
video:decoder_init_start
|
||||
video:first_decoded_frame
|
||||
video:decoded_frame_sample
|
||||
video:frame_dump
|
||||
video:byte_dump
|
||||
```
|
||||
|
||||
The crop fix is active when Android `camera:capture_frame` includes portrait source dimensions with a landscape send frame, for example:
|
||||
|
||||
```text
|
||||
camera:capture_frame {"frame_no":150,"width":960,"height":540,"source_width":540,"source_height":960,...}
|
||||
```
|
||||
|
||||
### Android logcat
|
||||
|
||||
Logcat can be noisy and may not always retain the in-app call debug entries. Still useful commands:
|
||||
|
||||
```bash
|
||||
adb logcat -c
|
||||
adb logcat -v time | rg 'camera:capture_frame|video:frame_dump|video:byte_dump|video:first_camera_frame|video:camera_frame_sample|video:encoded_frame|h264_encoder_input|hevc_encoder_input|MediaCodec input format|decoder_debug'
|
||||
```
|
||||
|
||||
For post-run collection:
|
||||
|
||||
```bash
|
||||
adb logcat -d -v time > /tmp/wzp-android-logcat.txt
|
||||
rg 'camera:|video:|h264_encoder_input|hevc_encoder_input|MediaCodec|decoder_debug' /tmp/wzp-android-logcat.txt
|
||||
```
|
||||
|
||||
If no `h264_encoder_input` / `hevc_encoder_input` entries appear, the current `tracing::info!` path in `crates/wzp-video/src/mediacodec.rs` may not be making it into Android logcat. Convert that diagnostic to `emit_call_debug` from the caller if the next step needs guaranteed visibility.
|
||||
|
||||
---
|
||||
|
||||
## What We Know
|
||||
|
||||
### The Android camera/canvas path is probably clean
|
||||
|
||||
The Android dumps for `android_camera_jpeg_in` and `android_camera_i420_roundtrip` at 960x540 are clean. They show the portrait front camera contained inside a landscape frame with black side bars. This means the former zoom/crop bug is fixed and the current bands are not introduced by CSS, canvas sizing, or the browser camera preview.
|
||||
|
||||
### The corruption appears after encode/decode
|
||||
|
||||
The corrupt lines are present in `desktop_remote_decoded_*`. They cover black bars as well as image content, which points to frame buffer / codec layout corruption rather than a real scene artifact.
|
||||
|
||||
### Transport is not the leading suspect
|
||||
|
||||
`android_local_encoded_f000001.h264` and `desktop_remote_encoded_reassembled_f000001.h264` have matching sizes/prefixes in the latest diagnostic run. That does not fully prove every later packet is perfect, but it makes relay/datagram/reassembly much less likely as the root cause.
|
||||
|
||||
Relays should not need changes for this bug unless the wire format changes. The relay forwards datagrams and does not inspect video frame internals.
|
||||
|
||||
### Resolution alignment is suspicious
|
||||
|
||||
960x540 has a height that is not divisible by 16. H.264 macroblock encoders commonly encode 960x544 and signal cropping to 960x540. The horizontal line bands may be a crop/padding/chroma-plane issue. Testing 960x544 and/or 960x528 is a high-value next step.
|
||||
|
||||
---
|
||||
|
||||
## Code Areas
|
||||
|
||||
Primary suspects:
|
||||
|
||||
- `crates/wzp-video/src/mediacodec.rs` - Android MediaCodec H.264/HEVC encoder and decoder, color format, stride, slice height handling.
|
||||
- `desktop/src-tauri/src/engine.rs` - packet send/receive, decode lifecycle, frame/byte dump calls.
|
||||
- `desktop/src-tauri/src/lib.rs` - `maybe_dump_video_jpeg`, `maybe_dump_video_bytes`, app-data paths, call-debug event plumbing.
|
||||
- `desktop/src/main.ts` - browser camera capture, canvas scaling, codec/resolution settings, UI debug log buffer.
|
||||
- `crates/wzp-video/src/transport.rs` - video packetization/reassembly and `WZV1` metadata header.
|
||||
|
||||
The latest attempted fix in `mediacodec.rs` uses `codec.input_format()` on Android API 28+ to derive encoder input stride/slice/color layout. Since the lines persist, either those fields are not reliable for this encoder, the chosen color format conversion is wrong, or macOS decode/crop interpretation is involved.
|
||||
|
||||
---
|
||||
|
||||
## Recommended Next Debug Steps
|
||||
|
||||
1. Verify whether Android logs the encoder input format on the failing build.
|
||||
|
||||
```bash
|
||||
adb logcat -d -v time | rg 'h264_encoder_input|hevc_encoder_input|input_color_format|effective_stride|effective_slice'
|
||||
```
|
||||
|
||||
If absent, make this an app call-debug event instead of plain tracing so it appears in the copied call log.
|
||||
|
||||
2. Add Android loopback decode of `android_local_encoded_*` before network.
|
||||
|
||||
Dump a new `android_local_decoded_fXXXXXX_<WxH>.jpg` immediately after encoding. If this local Android decode already has bands, the encoder output is bad. If Android local decode is clean but macOS decode is bad, focus on H.264 SPS cropping / VideoToolbox decode assumptions.
|
||||
|
||||
3. Test macroblock-aligned debug resolutions.
|
||||
|
||||
Add or force:
|
||||
|
||||
```text
|
||||
960x544
|
||||
960x528
|
||||
640x368
|
||||
640x352
|
||||
```
|
||||
|
||||
If 960x544 fixes the lines, the bug is almost certainly H.264 crop/padding handling. If 960x528 fixes it but 960x544 does not, inspect bottom padding and crop signaling.
|
||||
|
||||
4. Offline-decode `android_local_encoded_*.h264` with a known-good decoder.
|
||||
|
||||
Example on a machine with working ffmpeg:
|
||||
|
||||
```bash
|
||||
ffmpeg -f h264 -i android-frame-dumps/frame-dumps/000005_android_local_encoded_f000001.h264 -frames:v 1 /tmp/android-local-f1.png
|
||||
ffmpeg -f h264 -i "$HOME/Library/Application Support/com.wzp.desktop/.wzp/frame-dumps/000064_desktop_remote_encoded_reassembled_f000001.h264" -frames:v 1 /tmp/macos-remote-f1.png
|
||||
```
|
||||
|
||||
Note: Homebrew ffmpeg on this Mac was broken during debugging with a missing `libvpx.11.dylib`, so do not assume `/opt/homebrew/bin/ffmpeg` works until fixed.
|
||||
|
||||
5. Try explicit Android encoder input variants.
|
||||
|
||||
Test one variable at a time:
|
||||
|
||||
- Force planar color format `COLOR_FormatYUV420Planar` / value `19` and feed I420.
|
||||
- Force semiplanar and try NV12 vs NV21/VU order.
|
||||
- Use `COLOR_FormatYUV420Flexible` if accepted by this device.
|
||||
- Use `stride = width`, `slice_height = align_up(height, 16)` only.
|
||||
- Use `stride = align_up(width, 16)`, `slice_height = align_up(height, 16)`.
|
||||
|
||||
6. Parse SPS from Android H.264 output.
|
||||
|
||||
Confirm encoded dimensions and frame cropping offsets for 960x540. Compare Android output against macOS output. If SPS says 960x544 with crop to 540, test whether VideoToolbox applies the crop correctly.
|
||||
|
||||
7. Keep relay out of the first debugging loop.
|
||||
|
||||
The relay is unlikely to affect deterministic decoded line bands when local encoded and remote reassembled payloads match. Only redeploy relay if packet framing changes.
|
||||
|
||||
---
|
||||
|
||||
## Verification Criteria For A Fix
|
||||
|
||||
A candidate fix is good when:
|
||||
|
||||
- Android `android_camera_jpeg_in` and `android_camera_i420_roundtrip` remain clean.
|
||||
- Android `android_local_decoded`, if added, is clean.
|
||||
- macOS `desktop_remote_decoded` is clean at H.264 960x540.
|
||||
- 960x540 is smooth enough for normal calls, not a debug-build slideshow.
|
||||
- H.264 1280x720 either works or fails in an understood performance-only way.
|
||||
- HEVC behavior is not regressed from current minimum-resolution success.
|
||||
|
||||
Run at least:
|
||||
|
||||
```bash
|
||||
cargo check -p wzp-video --target aarch64-linux-android
|
||||
cargo check -p wzp-video -p wzp-client -p wzp-desktop
|
||||
```
|
||||
|
||||
Then build Android with:
|
||||
|
||||
```bash
|
||||
./scripts/android-build-async.sh --release-debuggable --wait
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Open Questions
|
||||
|
||||
- Does the failing Android device actually report encoder input `stride`, `slice-height`, and `color-format` after `start()`? The code asks for this, but recent logcat sampling did not show the `h264_encoder_input` tracing lines.
|
||||
- Does Android local decode of its own encoded H.264 reproduce the same lines?
|
||||
- Is 960x540 failing because H.264 encodes a 544-high macroblock frame and macOS crops or interprets chroma padding incorrectly?
|
||||
- Are the green/magenta bands chroma-plane corruption, luma padding leakage, or debug overlay from an encoder surface path? Current pipeline uses byte-buffer input, not surface input.
|
||||
- Is HEVC's pause behavior a separate decoder buffering/keyframe issue or the same layout problem expressed differently?
|
||||
|
||||
@@ -1,122 +1,113 @@
|
||||
#!/usr/bin/env bash
|
||||
# Fire-and-forget Android APK builder.
|
||||
#
|
||||
# Uploads the build script to SepehrHomeserverdk, starts it in a tmux
|
||||
# session so it survives SSH disconnects, then exits immediately.
|
||||
# Progress and the finished APK URL arrive via ntfy.sh/wzp.
|
||||
# Runs ./scripts/build-tauri-android.sh inside a LOCAL tmux session so the
|
||||
# build survives terminal disconnects. The wrapped script SSHes to
|
||||
# SepehrHomeserverdk on its own — we don't try to upload+run anything on
|
||||
# the remote (that would re-SSH from the remote to itself, which fails).
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/android-build-async.sh # build current branch, arm64
|
||||
# ./scripts/android-build-async.sh --init # also run cargo tauri android init
|
||||
# ./scripts/android-build-async.sh --rust # force-clean Rust target cache
|
||||
# ./scripts/android-build-async.sh --no-pull # skip git fetch on remote
|
||||
# ./scripts/android-build-async.sh --wait # block until done, then download APK
|
||||
# ./scripts/android-build-async.sh --debug # debug APK
|
||||
# ./scripts/android-build-async.sh --release-debuggable # release APK with run-as dumps
|
||||
# ./scripts/android-build-async.sh --wait # block until done, then tail status
|
||||
#
|
||||
# When the build finishes, ntfy.sh/wzp will show:
|
||||
# "WZP Tauri arm64 [<hash>] ready! <rustypaste-url>"
|
||||
# or on failure:
|
||||
# "WZP Tauri Android build FAILED [<hash>] (line N) log: <url>"
|
||||
# Progress / completion: ntfy.sh/wzp (handled by build-tauri-android.sh).
|
||||
# Monitor locally: tmux attach -t wzp-android-local
|
||||
# tail -f /tmp/wzp-tauri-build-local.log
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REMOTE_HOST="SepehrHomeserverdk"
|
||||
NTFY_TOPIC="https://ntfy.sh/wzp"
|
||||
LOCAL_OUTPUT="target/tauri-android-apk"
|
||||
TMUX_SESSION="wzp-android"
|
||||
REMOTE_LOG="/tmp/wzp-tauri-build.log"
|
||||
SSH_OPTS="-o ConnectTimeout=15 -o ServerAliveInterval=30 -o ServerAliveCountMax=6 -o LogLevel=ERROR"
|
||||
TMUX_SESSION="wzp-android-local"
|
||||
LOCAL_LOG="/tmp/wzp-tauri-build-local.log"
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
REPO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
BUILD_SCRIPT="$SCRIPT_DIR/build-tauri-android.sh"
|
||||
|
||||
BRANCH="${WZP_BRANCH:-$(git -C "$(dirname "$0")/.." branch --show-current 2>/dev/null || echo "")}"
|
||||
DO_PULL=1
|
||||
DO_INIT=0
|
||||
BUILD_RELEASE=1
|
||||
REBUILD_RUST=0
|
||||
BUILD_ARCH="arm64"
|
||||
DO_WAIT=0
|
||||
if ! command -v tmux >/dev/null 2>&1; then
|
||||
echo "ERROR: tmux is not installed locally. Install with: brew install tmux"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--pull) DO_PULL=1 ;;
|
||||
--no-pull) DO_PULL=0 ;;
|
||||
--init) DO_INIT=1 ;;
|
||||
--debug) BUILD_RELEASE=0 ;;
|
||||
--rust) REBUILD_RUST=1 ;;
|
||||
--wait) DO_WAIT=1 ;;
|
||||
esac
|
||||
done
|
||||
if [ ! -x "$BUILD_SCRIPT" ]; then
|
||||
echo "ERROR: $BUILD_SCRIPT not found or not executable"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
BRANCH="${WZP_BRANCH:-$(git -C "$REPO_DIR" branch --show-current 2>/dev/null || echo "")}"
|
||||
if [ -z "$BRANCH" ]; then
|
||||
echo "ERROR: could not determine branch (detached HEAD?). Set WZP_BRANCH=name."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log() { echo -e "\033[1;36m>>> $*\033[0m"; }
|
||||
err() { echo -e "\033[1;31mERROR: $*\033[0m" >&2; }
|
||||
ssh_q() { ssh $SSH_OPTS "$REMOTE_HOST" "$@"; }
|
||||
DO_WAIT=0
|
||||
PASS_ARGS=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--wait) DO_WAIT=1 ;;
|
||||
*) PASS_ARGS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# ── Step 1: upload the remote build script ──────────────────────────────────
|
||||
log "Uploading build script to $REMOTE_HOST..."
|
||||
# Re-use the existing full build script (it already handles all logic).
|
||||
scp $SSH_OPTS "$(dirname "$0")/build-tauri-android.sh" "$REMOTE_HOST:/tmp/wzp-tauri-build-full.sh"
|
||||
ssh_q "chmod +x /tmp/wzp-tauri-build-full.sh"
|
||||
log() { echo -e "\033[1;36m>>> $*\033[0m"; }
|
||||
|
||||
# ── Step 2: launch in tmux (detached) ──────────────────────────────────────
|
||||
log "Starting build in tmux session '$TMUX_SESSION' on $REMOTE_HOST..."
|
||||
ssh_q "tmux kill-session -t $TMUX_SESSION 2>/dev/null; true"
|
||||
# Kill any prior session that might still be hanging around.
|
||||
tmux kill-session -t "$TMUX_SESSION" 2>/dev/null || true
|
||||
|
||||
# The full script accepts flags directly; pass them through.
|
||||
REMOTE_FLAGS=""
|
||||
[ "$DO_PULL" = "1" ] || REMOTE_FLAGS="$REMOTE_FLAGS --no-pull"
|
||||
[ "$DO_INIT" = "1" ] && REMOTE_FLAGS="$REMOTE_FLAGS --init"
|
||||
[ "$BUILD_RELEASE" = "0" ] && REMOTE_FLAGS="$REMOTE_FLAGS --debug"
|
||||
[ "$REBUILD_RUST" = "1" ] && REMOTE_FLAGS="$REMOTE_FLAGS --rust"
|
||||
# Write a launcher script — avoids fragile quoting inside `tmux new-session`.
|
||||
LAUNCHER="$(mktemp -t wzp-android-launcher.XXXXXX)"
|
||||
chmod +x "$LAUNCHER"
|
||||
{
|
||||
echo "#!/usr/bin/env bash"
|
||||
echo "set -o pipefail"
|
||||
echo "cd $(printf %q "$REPO_DIR")"
|
||||
echo "export WZP_BRANCH=$(printf %q "$BRANCH")"
|
||||
printf 'bash %q' "$BUILD_SCRIPT"
|
||||
for a in "${PASS_ARGS[@]:-}"; do
|
||||
[ -z "$a" ] && continue
|
||||
printf ' %q' "$a"
|
||||
done
|
||||
echo " 2>&1 | tee $(printf %q "$LOCAL_LOG")"
|
||||
echo "echo DONE_EXIT_CODE=\$? >> $(printf %q "$LOCAL_LOG")"
|
||||
} > "$LAUNCHER"
|
||||
|
||||
# Run via WZP_BRANCH so the remote script picks up the right branch
|
||||
# (it calls `git branch --show-current` which would return the remote's
|
||||
# currently checked-out branch, not necessarily the one we want).
|
||||
ssh_q "tmux new-session -d -s $TMUX_SESSION \
|
||||
'WZP_BRANCH=$BRANCH bash /tmp/wzp-tauri-build-full.sh $REMOTE_FLAGS \
|
||||
2>&1 | tee $REMOTE_LOG; echo DONE_EXIT_CODE=\$? >> $REMOTE_LOG'"
|
||||
# Create the log file up front so `tail -f` works immediately.
|
||||
: > "$LOCAL_LOG"
|
||||
|
||||
log "Build dispatched! Notification on ntfy.sh/wzp when done."
|
||||
log "Starting local tmux session '$TMUX_SESSION' (branch: $BRANCH)..."
|
||||
log "Build script: $BUILD_SCRIPT ${PASS_ARGS[*]:-}"
|
||||
log "Launcher: $LAUNCHER"
|
||||
log "Local log: $LOCAL_LOG"
|
||||
|
||||
tmux new-session -d -s "$TMUX_SESSION" -c "$REPO_DIR" "bash $LAUNCHER; exec bash"
|
||||
|
||||
# Verify the session actually started.
|
||||
sleep 1
|
||||
if ! tmux has-session -t "$TMUX_SESSION" 2>/dev/null; then
|
||||
echo "ERROR: tmux session '$TMUX_SESSION' failed to start. Launcher contents:"
|
||||
cat "$LAUNCHER"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log "Build dispatched! ntfy.sh/wzp will notify on completion."
|
||||
echo ""
|
||||
echo " Monitor : ssh $REMOTE_HOST 'tail -f $REMOTE_LOG'"
|
||||
echo " Status : ssh $REMOTE_HOST 'tail -5 $REMOTE_LOG'"
|
||||
echo " Attach : ssh $REMOTE_HOST 'tmux attach -t $TMUX_SESSION'"
|
||||
echo " Monitor : tail -f $LOCAL_LOG"
|
||||
echo " Status : tail -5 $LOCAL_LOG"
|
||||
echo " Attach : tmux attach -t $TMUX_SESSION"
|
||||
echo " Kill : tmux kill-session -t $TMUX_SESSION"
|
||||
echo ""
|
||||
|
||||
# ── Step 3 (optional --wait): block until done, download APK ───────────────
|
||||
if [ "$DO_WAIT" = "0" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log "Waiting for build to finish (monitoring $REMOTE_LOG)..."
|
||||
ssh_q "until grep -qE 'APK_REMOTE_PATH|FAILED|ERROR|DONE_EXIT_CODE' \
|
||||
$REMOTE_LOG 2>/dev/null; do sleep 20; done"
|
||||
|
||||
# Check for failure
|
||||
if ssh_q "grep -q 'FAILED\|ERROR' $REMOTE_LOG 2>/dev/null" && \
|
||||
! ssh_q "grep -q 'APK_REMOTE_PATH' $REMOTE_LOG 2>/dev/null"; then
|
||||
err "Build failed — check ntfy or: ssh $REMOTE_HOST 'cat $REMOTE_LOG'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Grab APK paths from log
|
||||
APK_REMOTES=$(ssh_q "grep '^APK_REMOTE_PATH=' $REMOTE_LOG | cut -d= -f2-")
|
||||
if [ -z "$APK_REMOTES" ]; then
|
||||
err "No APK_REMOTE_PATH in log — build may have failed silently"
|
||||
ssh_q "tail -20 $REMOTE_LOG" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$LOCAL_OUTPUT"
|
||||
echo "$APK_REMOTES" | while IFS= read -r REMOTE_PATH; do
|
||||
[ -z "$REMOTE_PATH" ] && continue
|
||||
APK_NAME=$(basename "$REMOTE_PATH")
|
||||
log "Downloading $APK_NAME..."
|
||||
scp $SSH_OPTS "$REMOTE_HOST:$REMOTE_PATH" "$LOCAL_OUTPUT/$APK_NAME"
|
||||
echo " $LOCAL_OUTPUT/$APK_NAME ($(du -h "$LOCAL_OUTPUT/$APK_NAME" | cut -f1))"
|
||||
log "Waiting for build to finish (watching $LOCAL_LOG)..."
|
||||
until grep -qE 'DONE_EXIT_CODE|APK_REMOTE_PATH=|FAILED' "$LOCAL_LOG" 2>/dev/null; do
|
||||
sleep 20
|
||||
done
|
||||
|
||||
log "Done! APKs in $LOCAL_OUTPUT/"
|
||||
ls -lh "$LOCAL_OUTPUT"/wzp-tauri-*.apk 2>/dev/null || true
|
||||
log "Build session ended. Last 20 lines:"
|
||||
tail -20 "$LOCAL_LOG"
|
||||
|
||||
@@ -17,6 +17,7 @@ set -euo pipefail
|
||||
# Usage:
|
||||
# ./scripts/build-tauri-android.sh # full pipeline (release, arm64 only)
|
||||
# ./scripts/build-tauri-android.sh --debug # debug APK (faster, no optimisation)
|
||||
# ./scripts/build-tauri-android.sh --release-debuggable # release APK with android:debuggable=true
|
||||
# ./scripts/build-tauri-android.sh --no-pull # skip git fetch
|
||||
# ./scripts/build-tauri-android.sh --rust # force-clean rust target
|
||||
# ./scripts/build-tauri-android.sh --init # also run `cargo tauri android init`
|
||||
@@ -39,6 +40,7 @@ REBUILD_RUST=0
|
||||
DO_PULL=1
|
||||
DO_INIT=0
|
||||
BUILD_RELEASE=1
|
||||
RELEASE_DEBUGGABLE=0
|
||||
BUILD_ARCH="arm64"
|
||||
NEXT_IS_ARCH=0
|
||||
for arg in "$@"; do
|
||||
@@ -53,6 +55,7 @@ for arg in "$@"; do
|
||||
--no-pull) DO_PULL=0 ;;
|
||||
--init) DO_INIT=1 ;;
|
||||
--debug) BUILD_RELEASE=0 ;;
|
||||
--release-debuggable) RELEASE_DEBUGGABLE=1 ;;
|
||||
--arch) NEXT_IS_ARCH=1 ;;
|
||||
-h|--help)
|
||||
sed -n '3,32p' "$0"
|
||||
@@ -93,6 +96,7 @@ REBUILD_RUST="${3:-0}"
|
||||
DO_INIT="${4:-0}"
|
||||
BUILD_RELEASE="${5:-0}"
|
||||
BUILD_ARCH="${6:-arm64}"
|
||||
RELEASE_DEBUGGABLE="${7:-0}"
|
||||
|
||||
LOG_FILE=/tmp/wzp-tauri-build.log
|
||||
GIT_HASH="unknown" # populated after fetch
|
||||
@@ -192,6 +196,7 @@ docker run --rm \
|
||||
-e DO_INIT="$DO_INIT" \
|
||||
-e PROFILE_FLAG="$PROFILE_FLAG" \
|
||||
-e BUILD_ARCH="$BUILD_ARCH" \
|
||||
-e RELEASE_DEBUGGABLE="$RELEASE_DEBUGGABLE" \
|
||||
-v "$BASE_DIR/data/source:/build/source" \
|
||||
-v "$BASE_DIR/data/cache/cargo-registry:/home/builder/.cargo/registry" \
|
||||
-v "$BASE_DIR/data/cache/cargo-git:/home/builder/.cargo/git" \
|
||||
@@ -218,6 +223,29 @@ if [ "${DO_INIT}" = "1" ] || [ ! -x gen/android/gradlew ]; then
|
||||
cargo tauri android init 2>&1 | tail -20
|
||||
fi
|
||||
|
||||
if [ "${RELEASE_DEBUGGABLE}" = "1" ]; then
|
||||
MANIFEST="gen/android/app/src/main/AndroidManifest.xml"
|
||||
if [ -f "$MANIFEST" ]; then
|
||||
echo ">>> Marking release APK debuggable for frame-dump run-as access"
|
||||
if ! grep -q "xmlns:tools=" "$MANIFEST"; then
|
||||
perl -0pi -e "s/<manifest\\b/<manifest xmlns:tools=\"http:\\/\\/schemas.android.com\\/tools\"/s" "$MANIFEST"
|
||||
fi
|
||||
if grep -q "android:debuggable=" "$MANIFEST"; then
|
||||
sed -i "s/android:debuggable=\"[^\"]*\"/android:debuggable=\"true\"/" "$MANIFEST"
|
||||
else
|
||||
perl -0pi -e "s/(<application\\b[^>]*)(>)/\$1\\n android:debuggable=\"true\"\$2/s" "$MANIFEST"
|
||||
fi
|
||||
if grep -q "tools:ignore=" "$MANIFEST"; then
|
||||
sed -i "s/tools:ignore=\"[^\"]*\"/tools:ignore=\"HardcodedDebugMode\"/" "$MANIFEST"
|
||||
else
|
||||
perl -0pi -e "s/(<application\\b[^>]*)(>)/\$1\\n tools:ignore=\"HardcodedDebugMode\"\$2/s" "$MANIFEST"
|
||||
fi
|
||||
grep -n "debuggable\\|<application" "$MANIFEST"
|
||||
else
|
||||
echo ">>> WARNING: AndroidManifest.xml not found; release APK will not be debuggable"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ─── Arch list from BUILD_ARCH env var ───────────────────────────────────
|
||||
case "${BUILD_ARCH}" in
|
||||
arm64) ARCHS="arm64" ;;
|
||||
@@ -302,6 +330,7 @@ done
|
||||
|
||||
APK_OUTPUT_DIR="/build/source/target/apk-output"
|
||||
mkdir -p "$APK_OUTPUT_DIR"
|
||||
rm -f "$APK_OUTPUT_DIR"/wzp-tauri-*.apk
|
||||
|
||||
for ARCH in $ARCHS; do
|
||||
TARGET=$(tauri_target "$ARCH")
|
||||
@@ -333,7 +362,9 @@ for ARCH in $ARCHS; do
|
||||
# Re-running Gradle is NOT used here because the Gradle Rust build
|
||||
# task (BuildTask.kt) calls `cargo tauri android android-studio-script`
|
||||
# which requires the full Tauri CLI environment and fails standalone.
|
||||
UNSIGNED_APK_PATH="gen/android/app/build/outputs/apk/universal/release/app-universal-release-unsigned.apk"
|
||||
BUILD_VARIANT="debug"
|
||||
[ -z "${PROFILE_FLAG}" ] && BUILD_VARIANT="release"
|
||||
UNSIGNED_APK_PATH="gen/android/app/build/outputs/apk/universal/${BUILD_VARIANT}/app-universal-${BUILD_VARIANT}-unsigned.apk"
|
||||
if [ -f "$UNSIGNED_APK_PATH" ] && ! unzip -l "$UNSIGNED_APK_PATH" 2>/dev/null | grep -q "assets/index.html"; then
|
||||
echo ">>> frontend assets missing from APK — patching unsigned APK directly"
|
||||
PATCH_DIR="/tmp/apk-frontend-patch-$$"
|
||||
@@ -347,7 +378,7 @@ for ARCH in $ARCHS; do
|
||||
fi
|
||||
|
||||
# Copy produced APK with arch suffix
|
||||
BUILT_APK=$(find gen/android -name "*.apk" -newer "$APK_OUTPUT_DIR" -type f 2>/dev/null | head -1)
|
||||
BUILT_APK=$(find "gen/android/app/build/outputs/apk" -path "*/${BUILD_VARIANT}/*.apk" -type f 2>/dev/null | sort | head -1)
|
||||
if [ -z "$BUILT_APK" ]; then
|
||||
BUILT_APK=$(find gen/android -name "*.apk" -type f 2>/dev/null | sort -t/ -k1 | tail -1)
|
||||
fi
|
||||
@@ -359,6 +390,12 @@ for ARCH in $ARCHS; do
|
||||
# Release builds are unsigned by default. Sign with the release
|
||||
# keystore (checked into the repo at android/keystore/) so the
|
||||
# APK can be installed on real devices.
|
||||
if [ "${BUILD_VARIANT}" = "debug" ]; then
|
||||
echo ">>> Debug APK selected; preserving Gradle debug signing and android:debuggable=true"
|
||||
echo ">>> $ARCH APK: $(ls -lh "$OUT_APK" | awk "{print \$5}")"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Pick keystore + credentials (release preferred, debug fallback)
|
||||
KS_RELEASE="/build/source/android/keystore/wzp-release.jks"
|
||||
KS_DEBUG="/build/source/android/keystore/wzp-debug.jks"
|
||||
@@ -452,11 +489,11 @@ REMOTE_SCRIPT
|
||||
|
||||
ssh_cmd "chmod +x /tmp/wzp-tauri-build.sh"
|
||||
|
||||
notify_local "WZP Tauri Android build dispatched (branch=$BRANCH, arch=$BUILD_ARCH, release=$BUILD_RELEASE)"
|
||||
notify_local "WZP Tauri Android build dispatched (branch=$BRANCH, arch=$BUILD_ARCH, release=$BUILD_RELEASE, release-debuggable=$RELEASE_DEBUGGABLE)"
|
||||
log "Triggering remote build (branch=$BRANCH, arch=$BUILD_ARCH)..."
|
||||
|
||||
# Run; last lines are APK_REMOTE_PATH=... (one per arch)
|
||||
REMOTE_OUTPUT=$(ssh_cmd "/tmp/wzp-tauri-build.sh '$BRANCH' '$DO_PULL' '$REBUILD_RUST' '$DO_INIT' '$BUILD_RELEASE' '$BUILD_ARCH'" || true)
|
||||
REMOTE_OUTPUT=$(ssh_cmd "/tmp/wzp-tauri-build.sh '$BRANCH' '$DO_PULL' '$REBUILD_RUST' '$DO_INIT' '$BUILD_RELEASE' '$BUILD_ARCH' '$RELEASE_DEBUGGABLE'" || true)
|
||||
echo "$REMOTE_OUTPUT" | tail -60
|
||||
|
||||
# Download all produced APKs
|
||||
|
||||
25
scripts/pull-android-frame-dumps.sh
Executable file
25
scripts/pull-android-frame-dumps.sh
Executable file
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
PACKAGE="${1:-com.wzp.desktop}"
|
||||
OUT_DIR="${2:-android-frame-dumps}"
|
||||
LOCAL_TAR="wzp-frame-dumps.tar"
|
||||
APP_DUMP_DIR="${WZP_ANDROID_DUMP_ROOT:-.wzp}"
|
||||
trap 'rm -f "$LOCAL_TAR"' EXIT
|
||||
|
||||
if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then
|
||||
echo "Usage: $0 [package] [out-dir]"
|
||||
echo "Default package: com.wzp.desktop"
|
||||
echo "Default out-dir: android-frame-dumps"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo ">>> Packaging frame dumps from $PACKAGE..."
|
||||
adb exec-out "run-as $PACKAGE tar -C $APP_DUMP_DIR -cf - frame-dumps" > "$LOCAL_TAR"
|
||||
|
||||
rm -rf "$OUT_DIR"
|
||||
mkdir -p "$OUT_DIR"
|
||||
tar -xf "$LOCAL_TAR" -C "$OUT_DIR"
|
||||
|
||||
echo ">>> Pulled dumps:"
|
||||
find "$OUT_DIR" -type f | sort | sed 's#^# #'
|
||||
Reference in New Issue
Block a user