Compare commits
60 Commits
f3e3ee5ed0
...
video-usab
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
12020b019c | ||
|
|
3ea25a0656 | ||
|
|
112472609e | ||
|
|
9a7745978b | ||
|
|
f85efb9576 | ||
|
|
31b2caa54d | ||
|
|
079e21e174 | ||
|
|
e676641538 | ||
|
|
9713efc404 | ||
|
|
8415804a1a | ||
|
|
f65b399a21 | ||
|
|
3437a6bd11 | ||
|
|
15eb00ed5e | ||
|
|
0c2297a2b7 | ||
|
|
a08a37b5eb | ||
|
|
f6ace54556 | ||
|
|
47baa1a765 | ||
|
|
ee654cd1ef | ||
|
|
d2046060b5 | ||
|
|
0b7bf1b385 | ||
|
|
e8f139588a | ||
|
|
0115b11de7 | ||
|
|
fa812a17d9 | ||
|
|
8d6b168f1b | ||
|
|
ca164ada5c | ||
|
|
2d58bae9ba | ||
|
|
e1ca6ca6e6 | ||
|
|
06d28a9280 | ||
|
|
d57ebe3d2c | ||
|
|
7eca79846f | ||
|
|
25b3278d31 | ||
|
|
cbc3a8d37e | ||
|
|
1329abbeba | ||
|
|
e8cab25eda | ||
|
|
c41ced53e1 | ||
|
|
7fd66be6c8 | ||
|
|
8002acaf09 | ||
|
|
06253fdeeb | ||
|
|
01f55caa96 | ||
|
|
0f93a2b745 | ||
|
|
2b93bd4b45 | ||
|
|
bc021517c0 | ||
|
|
739bdaf3ab | ||
|
|
bc1668ed96 | ||
|
|
77b036439b | ||
|
|
0ebc73ab13 | ||
|
|
394987a349 | ||
|
|
2aa6582585 | ||
|
|
ca987d547c | ||
|
|
5a13f12334 | ||
|
|
b0a3b1f18e | ||
|
|
32c07d1b61 | ||
|
|
5d05b021aa | ||
|
|
4ac62d99e0 | ||
|
|
4ebb2dac2d | ||
|
|
52a6f5e048 | ||
|
|
15af58a95d | ||
|
|
ed8a7ae5aa | ||
|
|
12b0d9738f | ||
|
|
f78794f4b6 |
5
.gitignore
vendored
5
.gitignore
vendored
@@ -12,6 +12,11 @@ npm-debug.log*
|
|||||||
yarn-debug.log*
|
yarn-debug.log*
|
||||||
yarn-error.log*
|
yarn-error.log*
|
||||||
dev-debug.log
|
dev-debug.log
|
||||||
|
|
||||||
|
# Debug frame dump artifacts
|
||||||
|
android-frame-dumps/
|
||||||
|
wzp-frame-dumps.tar
|
||||||
|
|
||||||
# Dependency directories
|
# Dependency directories
|
||||||
node_modules/
|
node_modules/
|
||||||
# Environment variables
|
# Environment variables
|
||||||
|
|||||||
56
Cargo.lock
generated
56
Cargo.lock
generated
@@ -712,6 +712,12 @@ version = "1.5.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "byteorder-lite"
|
||||||
|
version = "0.1.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bytes"
|
name = "bytes"
|
||||||
version = "1.11.1"
|
version = "1.11.1"
|
||||||
@@ -2873,6 +2879,20 @@ dependencies = [
|
|||||||
"windows-sys 0.59.0",
|
"windows-sys 0.59.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "image"
|
||||||
|
version = "0.25.10"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104"
|
||||||
|
dependencies = [
|
||||||
|
"bytemuck",
|
||||||
|
"byteorder-lite",
|
||||||
|
"moxcms",
|
||||||
|
"num-traits",
|
||||||
|
"zune-core",
|
||||||
|
"zune-jpeg",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "indexmap"
|
name = "indexmap"
|
||||||
version = "1.9.3"
|
version = "1.9.3"
|
||||||
@@ -3365,6 +3385,16 @@ dependencies = [
|
|||||||
"windows-sys 0.61.2",
|
"windows-sys 0.61.2",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "moxcms"
|
||||||
|
version = "0.8.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b"
|
||||||
|
dependencies = [
|
||||||
|
"num-traits",
|
||||||
|
"pxfm",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "muda"
|
name = "muda"
|
||||||
version = "0.19.1"
|
version = "0.19.1"
|
||||||
@@ -4293,6 +4323,12 @@ version = "2.28.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"
|
checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "pxfm"
|
||||||
|
version = "0.1.29"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e0c5ccf5294c6ccd63a74f1565028353830a9c2f5eb0c682c355c471726a6e3f"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "quick-xml"
|
name = "quick-xml"
|
||||||
version = "0.37.5"
|
version = "0.37.5"
|
||||||
@@ -7855,6 +7891,10 @@ name = "wzp-desktop"
|
|||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
|
"async-trait",
|
||||||
|
"base64 0.22.1",
|
||||||
|
"bytes",
|
||||||
|
"image",
|
||||||
"jni",
|
"jni",
|
||||||
"libloading 0.8.9",
|
"libloading 0.8.9",
|
||||||
"ndk-context",
|
"ndk-context",
|
||||||
@@ -7874,6 +7914,7 @@ dependencies = [
|
|||||||
"wzp-fec",
|
"wzp-fec",
|
||||||
"wzp-proto",
|
"wzp-proto",
|
||||||
"wzp-transport",
|
"wzp-transport",
|
||||||
|
"wzp-video",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -8228,6 +8269,21 @@ version = "1.0.21"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
|
checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zune-core"
|
||||||
|
version = "0.5.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "zune-jpeg"
|
||||||
|
version = "0.5.15"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296"
|
||||||
|
dependencies = [
|
||||||
|
"zune-core",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "zvariant"
|
name = "zvariant"
|
||||||
version = "5.11.0"
|
version = "5.11.0"
|
||||||
|
|||||||
1
android.sh
Normal file
1
android.sh
Normal file
@@ -0,0 +1 @@
|
|||||||
|
./scripts/android-build-async.sh --init
|
||||||
@@ -538,6 +538,7 @@ async fn run_call(
|
|||||||
alias: alias.map(|s| s.to_string()),
|
alias: alias.map(|s| s.to_string()),
|
||||||
protocol_version: 2,
|
protocol_version: 2,
|
||||||
supported_versions: vec![2],
|
supported_versions: vec![2],
|
||||||
|
video_codecs: vec![CodecId::H264Baseline],
|
||||||
};
|
};
|
||||||
transport.send_signal(&offer).await?;
|
transport.send_signal(&offer).await?;
|
||||||
info!("CallOffer sent, waiting for CallAnswer...");
|
info!("CallOffer sent, waiting for CallAnswer...");
|
||||||
@@ -796,7 +797,7 @@ async fn run_call(
|
|||||||
),
|
),
|
||||||
seq: rs,
|
seq: rs,
|
||||||
timestamp: t,
|
timestamp: t,
|
||||||
fec_block: ((sym_idx as u16) << 8) | (block_id as u16),
|
fec_block: (sym_idx << 8) | (block_id as u16),
|
||||||
},
|
},
|
||||||
payload: Bytes::from(repair_data),
|
payload: Bytes::from(repair_data),
|
||||||
quality_report: None,
|
quality_report: None,
|
||||||
@@ -948,8 +949,8 @@ async fn run_call(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let is_repair = pkt.header.is_repair();
|
let is_repair = pkt.header.is_repair();
|
||||||
let pkt_block = pkt.header.fec_block as u8;
|
let pkt_block = pkt.header.fec_block;
|
||||||
let pkt_symbol = (pkt.header.fec_block >> 8) as u8;
|
let pkt_symbol = (pkt.header.fec_block >> 8) as u16;
|
||||||
let pkt_is_opus = pkt.header.codec_id.is_opus();
|
let pkt_is_opus = pkt.header.codec_id.is_opus();
|
||||||
|
|
||||||
// Phase 2: Opus packets bypass RaptorQ entirely — DRED
|
// Phase 2: Opus packets bypass RaptorQ entirely — DRED
|
||||||
|
|||||||
@@ -137,8 +137,8 @@ impl Pipeline {
|
|||||||
if header.fec_block != 0 {
|
if header.fec_block != 0 {
|
||||||
let is_repair = header.is_repair();
|
let is_repair = header.is_repair();
|
||||||
if let Err(e) = self.fec_decoder.add_symbol(
|
if let Err(e) = self.fec_decoder.add_symbol(
|
||||||
header.fec_block as u8,
|
header.fec_block,
|
||||||
(header.fec_block >> 8) as u8,
|
header.fec_block >> 8,
|
||||||
is_repair,
|
is_repair,
|
||||||
&packet.payload,
|
&packet.payload,
|
||||||
) {
|
) {
|
||||||
|
|||||||
@@ -15,7 +15,8 @@ use std::time::{Duration, Instant};
|
|||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
use wzp_proto::{CodecId, MediaPacket, MediaTransport, default_signal_version};
|
use wzp_proto::{CodecId, MediaPacket, MediaTransport, MediaType, default_signal_version};
|
||||||
|
use wzp_video::{VideoDecoder, create_video_decoder, transport::VideoReassembler};
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// CLI
|
// CLI
|
||||||
@@ -68,6 +69,14 @@ struct Args {
|
|||||||
// For now, header-only analysis provides loss%, jitter, codec stats.
|
// For now, header-only analysis provides loss%, jitter, codec stats.
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
key: Option<String>,
|
key: Option<String>,
|
||||||
|
|
||||||
|
/// Track video fragmentation, completed frames, keyframes, and decode health.
|
||||||
|
#[arg(long)]
|
||||||
|
video_probe: bool,
|
||||||
|
|
||||||
|
/// Decode completed video frames in --video-probe mode.
|
||||||
|
#[arg(long)]
|
||||||
|
video_decode: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -198,6 +207,305 @@ fn find_or_create_participant(
|
|||||||
id
|
id
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Video probe
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[derive(Default, Clone)]
|
||||||
|
struct PlaneSample {
|
||||||
|
min: u8,
|
||||||
|
max: u8,
|
||||||
|
mean: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default, Clone)]
|
||||||
|
struct I420Sample {
|
||||||
|
y: PlaneSample,
|
||||||
|
u: PlaneSample,
|
||||||
|
v: PlaneSample,
|
||||||
|
valid_i420: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct VideoStreamProbe {
|
||||||
|
id: usize,
|
||||||
|
codec: CodecId,
|
||||||
|
wire_stream_id: u8,
|
||||||
|
packets: u64,
|
||||||
|
lost: u64,
|
||||||
|
last_seq: u32,
|
||||||
|
seq_initialized: bool,
|
||||||
|
frames: u64,
|
||||||
|
keyframes: u64,
|
||||||
|
bytes: u64,
|
||||||
|
max_frame_bytes: usize,
|
||||||
|
first_seen: Instant,
|
||||||
|
last_seen: Instant,
|
||||||
|
last_frame: Option<Instant>,
|
||||||
|
reassembler: VideoReassembler,
|
||||||
|
decoder: Option<Box<dyn VideoDecoder>>,
|
||||||
|
decoder_key: Option<(CodecId, u32, u32)>,
|
||||||
|
decode_ok: u64,
|
||||||
|
decode_pending: u64,
|
||||||
|
decode_err: u64,
|
||||||
|
last_decode_debug: Option<String>,
|
||||||
|
last_i420_sample: Option<I420Sample>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl VideoStreamProbe {
|
||||||
|
fn new(id: usize, codec: CodecId, wire_stream_id: u8, decode: bool) -> Self {
|
||||||
|
let decoder = if decode {
|
||||||
|
create_video_decoder(codec, 1280, 720).ok()
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
let now = Instant::now();
|
||||||
|
Self {
|
||||||
|
id,
|
||||||
|
codec,
|
||||||
|
wire_stream_id,
|
||||||
|
packets: 0,
|
||||||
|
lost: 0,
|
||||||
|
last_seq: 0,
|
||||||
|
seq_initialized: false,
|
||||||
|
frames: 0,
|
||||||
|
keyframes: 0,
|
||||||
|
bytes: 0,
|
||||||
|
max_frame_bytes: 0,
|
||||||
|
first_seen: now,
|
||||||
|
last_seen: now,
|
||||||
|
last_frame: None,
|
||||||
|
reassembler: VideoReassembler::new(),
|
||||||
|
decoder,
|
||||||
|
decoder_key: decode.then_some((codec, 1280, 720)),
|
||||||
|
decode_ok: 0,
|
||||||
|
decode_pending: 0,
|
||||||
|
decode_err: 0,
|
||||||
|
last_decode_debug: None,
|
||||||
|
last_i420_sample: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ingest(&mut self, pkt: &MediaPacket, now: Instant) {
|
||||||
|
self.packets += 1;
|
||||||
|
self.last_seen = now;
|
||||||
|
if pkt.header.codec_id != self.codec {
|
||||||
|
self.codec = pkt.header.codec_id;
|
||||||
|
self.reassembler = VideoReassembler::new();
|
||||||
|
self.decoder = self
|
||||||
|
.decoder
|
||||||
|
.is_some()
|
||||||
|
.then(|| create_video_decoder(self.codec, 1280, 720).ok())
|
||||||
|
.flatten();
|
||||||
|
self.decoder_key = self.decoder.as_ref().map(|_| (self.codec, 1280, 720));
|
||||||
|
}
|
||||||
|
if self.seq_initialized {
|
||||||
|
let expected = self.last_seq.wrapping_add(1);
|
||||||
|
let gap = pkt.header.seq.wrapping_sub(expected);
|
||||||
|
if gap > 0 && gap < 100 {
|
||||||
|
self.lost += gap as u64;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.last_seq = pkt.header.seq;
|
||||||
|
self.seq_initialized = true;
|
||||||
|
|
||||||
|
if let Some(frame) = self.reassembler.push(pkt) {
|
||||||
|
self.frames += 1;
|
||||||
|
self.bytes += frame.data.len() as u64;
|
||||||
|
self.max_frame_bytes = self.max_frame_bytes.max(frame.data.len());
|
||||||
|
self.last_frame = Some(now);
|
||||||
|
if frame.is_keyframe {
|
||||||
|
self.keyframes += 1;
|
||||||
|
}
|
||||||
|
if frame.codec_id != self.codec {
|
||||||
|
self.codec = frame.codec_id;
|
||||||
|
}
|
||||||
|
let frame_width = frame.width.unwrap_or(1280) as u32;
|
||||||
|
let frame_height = frame.height.unwrap_or(720) as u32;
|
||||||
|
let decoder_key = (self.codec, frame_width, frame_height);
|
||||||
|
if self.decoder.is_some() && self.decoder_key != Some(decoder_key) {
|
||||||
|
self.decoder = create_video_decoder(self.codec, frame_width, frame_height).ok();
|
||||||
|
self.decoder_key = self.decoder.as_ref().map(|_| decoder_key);
|
||||||
|
}
|
||||||
|
if let Some(decoder) = self.decoder.as_mut() {
|
||||||
|
match decoder.decode(&frame.data) {
|
||||||
|
Ok(Some(decoded)) => {
|
||||||
|
self.decode_ok += 1;
|
||||||
|
self.last_decode_debug = decoder.debug_snapshot();
|
||||||
|
self.last_i420_sample =
|
||||||
|
Some(sample_i420(&decoded.data, decoded.width, decoded.height));
|
||||||
|
}
|
||||||
|
Ok(None) => {
|
||||||
|
self.decode_pending += 1;
|
||||||
|
self.last_decode_debug = decoder.debug_snapshot();
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
self.decode_err += 1;
|
||||||
|
self.last_decode_debug = Some(err.to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn loss_percent(&self) -> f64 {
|
||||||
|
let total = self.packets + self.lost;
|
||||||
|
if total == 0 {
|
||||||
|
0.0
|
||||||
|
} else {
|
||||||
|
(self.lost as f64 / total as f64) * 100.0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn avg_frame_bytes(&self) -> u64 {
|
||||||
|
if self.frames == 0 {
|
||||||
|
0
|
||||||
|
} else {
|
||||||
|
self.bytes / self.frames
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn fps(&self) -> f64 {
|
||||||
|
let secs = self.last_seen.duration_since(self.first_seen).as_secs_f64();
|
||||||
|
if secs <= 0.0 {
|
||||||
|
0.0
|
||||||
|
} else {
|
||||||
|
self.frames as f64 / secs
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct VideoProbe {
|
||||||
|
streams: Vec<VideoStreamProbe>,
|
||||||
|
decode: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl VideoProbe {
|
||||||
|
fn new(decode: bool) -> Self {
|
||||||
|
Self {
|
||||||
|
streams: Vec::new(),
|
||||||
|
decode,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ingest(&mut self, pkt: &MediaPacket, now: Instant) {
|
||||||
|
if pkt.header.media_type != MediaType::Video {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let idx = self.find_or_create_stream(pkt);
|
||||||
|
self.streams[idx].ingest(pkt, now);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn find_or_create_stream(&mut self, pkt: &MediaPacket) -> usize {
|
||||||
|
for (i, s) in self.streams.iter().enumerate() {
|
||||||
|
if s.seq_initialized
|
||||||
|
&& s.wire_stream_id == pkt.header.stream_id
|
||||||
|
&& s.codec == pkt.header.codec_id
|
||||||
|
{
|
||||||
|
let delta = pkt.header.seq.wrapping_sub(s.last_seq);
|
||||||
|
if delta > 0 && delta < 80 {
|
||||||
|
return i;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let id = self.streams.len();
|
||||||
|
self.streams.push(VideoStreamProbe::new(
|
||||||
|
id,
|
||||||
|
pkt.header.codec_id,
|
||||||
|
pkt.header.stream_id,
|
||||||
|
self.decode,
|
||||||
|
));
|
||||||
|
id
|
||||||
|
}
|
||||||
|
|
||||||
|
fn print(&self) {
|
||||||
|
if self.streams.is_empty() {
|
||||||
|
eprintln!(" video: no packets yet");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for s in &self.streams {
|
||||||
|
let age_ms = s
|
||||||
|
.last_frame
|
||||||
|
.map(|t| t.elapsed().as_millis() as u64)
|
||||||
|
.unwrap_or(u64::MAX);
|
||||||
|
let mut line = format!(
|
||||||
|
" video#{} wire_stream={} {:?}: {} pkts {:.1}% loss | {} frames ({:.1} fps), {} key, avg={}B max={}B, last_frame={}ms",
|
||||||
|
s.id,
|
||||||
|
s.wire_stream_id,
|
||||||
|
s.codec,
|
||||||
|
s.packets,
|
||||||
|
s.loss_percent(),
|
||||||
|
s.frames,
|
||||||
|
s.fps(),
|
||||||
|
s.keyframes,
|
||||||
|
s.avg_frame_bytes(),
|
||||||
|
s.max_frame_bytes,
|
||||||
|
if age_ms == u64::MAX { 0 } else { age_ms },
|
||||||
|
);
|
||||||
|
if s.decoder.is_some() || s.decode_ok > 0 || s.decode_err > 0 {
|
||||||
|
line.push_str(&format!(
|
||||||
|
" | dec ok={} pending={} err={}",
|
||||||
|
s.decode_ok, s.decode_pending, s.decode_err
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if let Some(sample) = &s.last_i420_sample {
|
||||||
|
line.push_str(&format!(
|
||||||
|
" | i420={} y={:.1}/{}/{} u={:.1}/{}/{} v={:.1}/{}/{}",
|
||||||
|
sample.valid_i420,
|
||||||
|
sample.y.mean,
|
||||||
|
sample.y.min,
|
||||||
|
sample.y.max,
|
||||||
|
sample.u.mean,
|
||||||
|
sample.u.min,
|
||||||
|
sample.u.max,
|
||||||
|
sample.v.mean,
|
||||||
|
sample.v.min,
|
||||||
|
sample.v.max,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if let Some(debug) = &s.last_decode_debug {
|
||||||
|
line.push_str(&format!(" | {debug}"));
|
||||||
|
}
|
||||||
|
eprintln!("{line}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sample_i420(data: &[u8], width: u32, height: u32) -> I420Sample {
|
||||||
|
let y_len = width as usize * height as usize;
|
||||||
|
let uv_len = y_len / 4;
|
||||||
|
if data.len() < y_len + uv_len * 2 {
|
||||||
|
return I420Sample {
|
||||||
|
valid_i420: false,
|
||||||
|
..I420Sample::default()
|
||||||
|
};
|
||||||
|
}
|
||||||
|
I420Sample {
|
||||||
|
valid_i420: true,
|
||||||
|
y: sample_plane(&data[..y_len]),
|
||||||
|
u: sample_plane(&data[y_len..y_len + uv_len]),
|
||||||
|
v: sample_plane(&data[y_len + uv_len..y_len + uv_len * 2]),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sample_plane(data: &[u8]) -> PlaneSample {
|
||||||
|
if data.is_empty() {
|
||||||
|
return PlaneSample::default();
|
||||||
|
}
|
||||||
|
let mut min = u8::MAX;
|
||||||
|
let mut max = u8::MIN;
|
||||||
|
let mut sum: u64 = 0;
|
||||||
|
for &b in data {
|
||||||
|
min = min.min(b);
|
||||||
|
max = max.max(b);
|
||||||
|
sum += b as u64;
|
||||||
|
}
|
||||||
|
PlaneSample {
|
||||||
|
min,
|
||||||
|
max,
|
||||||
|
mean: sum as f64 / data.len() as f64,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Capture writer (binary packet log for later replay)
|
// Capture writer (binary packet log for later replay)
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -580,6 +888,7 @@ async fn run_no_tui(
|
|||||||
total_packets: &mut u64,
|
total_packets: &mut u64,
|
||||||
deadline: Option<Instant>,
|
deadline: Option<Instant>,
|
||||||
mut capture_writer: Option<&mut CaptureWriter>,
|
mut capture_writer: Option<&mut CaptureWriter>,
|
||||||
|
mut video_probe: Option<&mut VideoProbe>,
|
||||||
) -> anyhow::Result<()> {
|
) -> anyhow::Result<()> {
|
||||||
let mut print_timer = Instant::now();
|
let mut print_timer = Instant::now();
|
||||||
loop {
|
loop {
|
||||||
@@ -594,6 +903,9 @@ async fn run_no_tui(
|
|||||||
let idx =
|
let idx =
|
||||||
find_or_create_participant(participants, pkt.header.seq, pkt.header.codec_id);
|
find_or_create_participant(participants, pkt.header.seq, pkt.header.codec_id);
|
||||||
participants[idx].ingest(&pkt, now);
|
participants[idx].ingest(&pkt, now);
|
||||||
|
if let Some(ref mut probe) = video_probe {
|
||||||
|
probe.ingest(&pkt, now);
|
||||||
|
}
|
||||||
*total_packets += 1;
|
*total_packets += 1;
|
||||||
if let Some(ref mut w) = capture_writer {
|
if let Some(ref mut w) = capture_writer {
|
||||||
w.write_packet(&pkt, now)?;
|
w.write_packet(&pkt, now)?;
|
||||||
@@ -608,6 +920,9 @@ async fn run_no_tui(
|
|||||||
}
|
}
|
||||||
if print_timer.elapsed() >= Duration::from_secs(2) {
|
if print_timer.elapsed() >= Duration::from_secs(2) {
|
||||||
print_stats(participants, *total_packets);
|
print_stats(participants, *total_packets);
|
||||||
|
if let Some(ref probe) = video_probe {
|
||||||
|
probe.print();
|
||||||
|
}
|
||||||
print_timer = Instant::now();
|
print_timer = Instant::now();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -616,7 +931,7 @@ async fn run_no_tui(
|
|||||||
|
|
||||||
fn print_stats(participants: &[ParticipantStats], total: u64) {
|
fn print_stats(participants: &[ParticipantStats], total: u64) {
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"--- {} participants | {} total packets ---",
|
"--- {} packet streams | {} total packets ---",
|
||||||
participants.len(),
|
participants.len(),
|
||||||
total
|
total
|
||||||
);
|
);
|
||||||
@@ -644,6 +959,7 @@ async fn run_tui(
|
|||||||
start_time: Instant,
|
start_time: Instant,
|
||||||
deadline: Option<Instant>,
|
deadline: Option<Instant>,
|
||||||
mut capture_writer: Option<&mut CaptureWriter>,
|
mut capture_writer: Option<&mut CaptureWriter>,
|
||||||
|
mut video_probe: Option<&mut VideoProbe>,
|
||||||
) -> anyhow::Result<()> {
|
) -> anyhow::Result<()> {
|
||||||
crossterm::terminal::enable_raw_mode()?;
|
crossterm::terminal::enable_raw_mode()?;
|
||||||
let mut stdout = std::io::stdout();
|
let mut stdout = std::io::stdout();
|
||||||
@@ -684,6 +1000,9 @@ async fn run_tui(
|
|||||||
pkt.header.codec_id,
|
pkt.header.codec_id,
|
||||||
);
|
);
|
||||||
participants[idx].ingest(&pkt, now);
|
participants[idx].ingest(&pkt, now);
|
||||||
|
if let Some(ref mut probe) = video_probe {
|
||||||
|
probe.ingest(&pkt, now);
|
||||||
|
}
|
||||||
*total_packets += 1;
|
*total_packets += 1;
|
||||||
if let Some(ref mut w) = capture_writer {
|
if let Some(ref mut w) = capture_writer {
|
||||||
w.write_packet(&pkt, now)?;
|
w.write_packet(&pkt, now)?;
|
||||||
@@ -941,6 +1260,17 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
let mut participants: Vec<ParticipantStats> = Vec::new();
|
let mut participants: Vec<ParticipantStats> = Vec::new();
|
||||||
let mut total_packets: u64 = 0;
|
let mut total_packets: u64 = 0;
|
||||||
let start_time = Instant::now();
|
let start_time = Instant::now();
|
||||||
|
let mut video_probe = (args.video_probe || args.video_decode).then(|| {
|
||||||
|
eprintln!(
|
||||||
|
"Video probe enabled{}",
|
||||||
|
if args.video_decode {
|
||||||
|
" with decode"
|
||||||
|
} else {
|
||||||
|
""
|
||||||
|
}
|
||||||
|
);
|
||||||
|
VideoProbe::new(args.video_decode)
|
||||||
|
});
|
||||||
|
|
||||||
if args.no_tui {
|
if args.no_tui {
|
||||||
run_no_tui(
|
run_no_tui(
|
||||||
@@ -949,6 +1279,7 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
&mut total_packets,
|
&mut total_packets,
|
||||||
deadline,
|
deadline,
|
||||||
capture_writer.as_mut(),
|
capture_writer.as_mut(),
|
||||||
|
video_probe.as_mut(),
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
} else {
|
} else {
|
||||||
@@ -959,12 +1290,17 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
start_time,
|
start_time,
|
||||||
deadline,
|
deadline,
|
||||||
capture_writer.as_mut(),
|
capture_writer.as_mut(),
|
||||||
|
video_probe.as_mut(),
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Print summary
|
// Print summary
|
||||||
print_summary(&participants, total_packets, start_time.elapsed());
|
print_summary(&participants, total_packets, start_time.elapsed());
|
||||||
|
if let Some(probe) = &video_probe {
|
||||||
|
eprintln!("\n=== Video Probe Summary ===");
|
||||||
|
probe.print();
|
||||||
|
}
|
||||||
|
|
||||||
// Clean close
|
// Clean close
|
||||||
transport.close().await?;
|
transport.close().await?;
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
//! This is the same engine FaceTime and other Apple apps use.
|
//! This is the same engine FaceTime and other Apple apps use.
|
||||||
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::atomic::{AtomicBool, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||||
|
|
||||||
use anyhow::Context;
|
use anyhow::Context;
|
||||||
use coreaudio::audio_unit::audio_format::LinearPcmFlags;
|
use coreaudio::audio_unit::audio_format::LinearPcmFlags;
|
||||||
@@ -28,6 +28,60 @@ pub struct VpioAudio {
|
|||||||
playout_ring: Arc<AudioRing>,
|
playout_ring: Arc<AudioRing>,
|
||||||
_audio_unit: AudioUnit,
|
_audio_unit: AudioUnit,
|
||||||
running: Arc<AtomicBool>,
|
running: Arc<AtomicBool>,
|
||||||
|
stats: Arc<VpioStats>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Render/capture counters for diagnosing macOS VoiceProcessingIO.
|
||||||
|
///
|
||||||
|
/// These are atomics because CoreAudio callbacks run on realtime audio
|
||||||
|
/// threads. The Tauri engine polls snapshots from a normal async task and
|
||||||
|
/// emits them to the call debug log.
|
||||||
|
#[derive(Default)]
|
||||||
|
pub struct VpioStats {
|
||||||
|
capture_callbacks: AtomicU64,
|
||||||
|
capture_samples: AtomicU64,
|
||||||
|
render_callbacks: AtomicU64,
|
||||||
|
render_requested_samples: AtomicU64,
|
||||||
|
render_read_samples: AtomicU64,
|
||||||
|
render_underrun_callbacks: AtomicU64,
|
||||||
|
render_nonzero_callbacks: AtomicU64,
|
||||||
|
render_last_requested: AtomicU64,
|
||||||
|
render_last_read: AtomicU64,
|
||||||
|
render_last_rms: AtomicU64,
|
||||||
|
render_last_ring_available: AtomicU64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug)]
|
||||||
|
pub struct VpioStatsSnapshot {
|
||||||
|
pub capture_callbacks: u64,
|
||||||
|
pub capture_samples: u64,
|
||||||
|
pub render_callbacks: u64,
|
||||||
|
pub render_requested_samples: u64,
|
||||||
|
pub render_read_samples: u64,
|
||||||
|
pub render_underrun_callbacks: u64,
|
||||||
|
pub render_nonzero_callbacks: u64,
|
||||||
|
pub render_last_requested: u64,
|
||||||
|
pub render_last_read: u64,
|
||||||
|
pub render_last_rms: u64,
|
||||||
|
pub render_last_ring_available: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl VpioStats {
|
||||||
|
pub fn snapshot(&self) -> VpioStatsSnapshot {
|
||||||
|
VpioStatsSnapshot {
|
||||||
|
capture_callbacks: self.capture_callbacks.load(Ordering::Relaxed),
|
||||||
|
capture_samples: self.capture_samples.load(Ordering::Relaxed),
|
||||||
|
render_callbacks: self.render_callbacks.load(Ordering::Relaxed),
|
||||||
|
render_requested_samples: self.render_requested_samples.load(Ordering::Relaxed),
|
||||||
|
render_read_samples: self.render_read_samples.load(Ordering::Relaxed),
|
||||||
|
render_underrun_callbacks: self.render_underrun_callbacks.load(Ordering::Relaxed),
|
||||||
|
render_nonzero_callbacks: self.render_nonzero_callbacks.load(Ordering::Relaxed),
|
||||||
|
render_last_requested: self.render_last_requested.load(Ordering::Relaxed),
|
||||||
|
render_last_read: self.render_last_read.load(Ordering::Relaxed),
|
||||||
|
render_last_rms: self.render_last_rms.load(Ordering::Relaxed),
|
||||||
|
render_last_ring_available: self.render_last_ring_available.load(Ordering::Relaxed),
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl VpioAudio {
|
impl VpioAudio {
|
||||||
@@ -36,6 +90,7 @@ impl VpioAudio {
|
|||||||
let capture_ring = Arc::new(AudioRing::new());
|
let capture_ring = Arc::new(AudioRing::new());
|
||||||
let playout_ring = Arc::new(AudioRing::new());
|
let playout_ring = Arc::new(AudioRing::new());
|
||||||
let running = Arc::new(AtomicBool::new(true));
|
let running = Arc::new(AtomicBool::new(true));
|
||||||
|
let stats = Arc::new(VpioStats::default());
|
||||||
|
|
||||||
let mut au = AudioUnit::new(IOType::VoiceProcessingIO)
|
let mut au = AudioUnit::new(IOType::VoiceProcessingIO)
|
||||||
.context("failed to create VoiceProcessingIO audio unit")?;
|
.context("failed to create VoiceProcessingIO audio unit")?;
|
||||||
@@ -98,6 +153,7 @@ impl VpioAudio {
|
|||||||
// Set up input callback (mic capture with AEC applied)
|
// Set up input callback (mic capture with AEC applied)
|
||||||
let cap_ring = capture_ring.clone();
|
let cap_ring = capture_ring.clone();
|
||||||
let cap_running = running.clone();
|
let cap_running = running.clone();
|
||||||
|
let cap_stats = stats.clone();
|
||||||
let logged = Arc::new(AtomicBool::new(false));
|
let logged = Arc::new(AtomicBool::new(false));
|
||||||
au.set_input_callback(
|
au.set_input_callback(
|
||||||
move |args: render_callback::Args<data::NonInterleaved<f32>>| {
|
move |args: render_callback::Args<data::NonInterleaved<f32>>| {
|
||||||
@@ -106,6 +162,10 @@ impl VpioAudio {
|
|||||||
}
|
}
|
||||||
let mut buffers = args.data.channels();
|
let mut buffers = args.data.channels();
|
||||||
if let Some(ch) = buffers.next() {
|
if let Some(ch) = buffers.next() {
|
||||||
|
cap_stats.capture_callbacks.fetch_add(1, Ordering::Relaxed);
|
||||||
|
cap_stats
|
||||||
|
.capture_samples
|
||||||
|
.fetch_add(ch.len() as u64, Ordering::Relaxed);
|
||||||
if !logged.swap(true, Ordering::Relaxed) {
|
if !logged.swap(true, Ordering::Relaxed) {
|
||||||
eprintln!("[vpio] capture callback: {} f32 samples", ch.len());
|
eprintln!("[vpio] capture callback: {} f32 samples", ch.len());
|
||||||
}
|
}
|
||||||
@@ -125,21 +185,72 @@ impl VpioAudio {
|
|||||||
|
|
||||||
// Set up output callback (speaker playback — AEC uses this as reference)
|
// Set up output callback (speaker playback — AEC uses this as reference)
|
||||||
let play_ring = playout_ring.clone();
|
let play_ring = playout_ring.clone();
|
||||||
|
let render_stats = stats.clone();
|
||||||
|
let logged_render = Arc::new(AtomicBool::new(false));
|
||||||
au.set_render_callback(
|
au.set_render_callback(
|
||||||
move |mut args: render_callback::Args<data::NonInterleaved<f32>>| {
|
move |mut args: render_callback::Args<data::NonInterleaved<f32>>| {
|
||||||
let mut buffers = args.data.channels_mut();
|
let mut buffers = args.data.channels_mut();
|
||||||
if let Some(ch) = buffers.next() {
|
if let Some(ch) = buffers.next() {
|
||||||
|
render_stats
|
||||||
|
.render_callbacks
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
render_stats
|
||||||
|
.render_requested_samples
|
||||||
|
.fetch_add(ch.len() as u64, Ordering::Relaxed);
|
||||||
|
render_stats
|
||||||
|
.render_last_requested
|
||||||
|
.store(ch.len() as u64, Ordering::Relaxed);
|
||||||
let mut tmp = [0i16; FRAME_SAMPLES];
|
let mut tmp = [0i16; FRAME_SAMPLES];
|
||||||
|
let mut total_read = 0usize;
|
||||||
|
let mut sum_sq = 0u64;
|
||||||
|
let ring_available = play_ring.available();
|
||||||
for chunk in ch.chunks_mut(FRAME_SAMPLES) {
|
for chunk in ch.chunks_mut(FRAME_SAMPLES) {
|
||||||
let n = chunk.len();
|
let n = chunk.len();
|
||||||
let read = play_ring.read(&mut tmp[..n]);
|
let read = play_ring.read(&mut tmp[..n]);
|
||||||
|
total_read += read;
|
||||||
for i in 0..read {
|
for i in 0..read {
|
||||||
|
let s = tmp[i] as i64;
|
||||||
|
sum_sq = sum_sq.saturating_add((s * s) as u64);
|
||||||
chunk[i] = tmp[i] as f32 / i16::MAX as f32;
|
chunk[i] = tmp[i] as f32 / i16::MAX as f32;
|
||||||
}
|
}
|
||||||
for i in read..n {
|
for i in read..n {
|
||||||
chunk[i] = 0.0;
|
chunk[i] = 0.0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
render_stats
|
||||||
|
.render_read_samples
|
||||||
|
.fetch_add(total_read as u64, Ordering::Relaxed);
|
||||||
|
render_stats
|
||||||
|
.render_last_read
|
||||||
|
.store(total_read as u64, Ordering::Relaxed);
|
||||||
|
render_stats
|
||||||
|
.render_last_ring_available
|
||||||
|
.store(ring_available as u64, Ordering::Relaxed);
|
||||||
|
if total_read == 0 {
|
||||||
|
render_stats
|
||||||
|
.render_underrun_callbacks
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
let rms = if total_read > 0 {
|
||||||
|
((sum_sq as f64 / total_read as f64).sqrt()) as u64
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
};
|
||||||
|
render_stats.render_last_rms.store(rms, Ordering::Relaxed);
|
||||||
|
if rms > 0 {
|
||||||
|
render_stats
|
||||||
|
.render_nonzero_callbacks
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
if !logged_render.swap(true, Ordering::Relaxed) {
|
||||||
|
eprintln!(
|
||||||
|
"[vpio] render callback: {} f32 samples, ring_available={}, ring_read={}, rms={}",
|
||||||
|
ch.len(),
|
||||||
|
ring_available,
|
||||||
|
total_read,
|
||||||
|
rms
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
},
|
},
|
||||||
@@ -157,6 +268,7 @@ impl VpioAudio {
|
|||||||
playout_ring,
|
playout_ring,
|
||||||
_audio_unit: au,
|
_audio_unit: au,
|
||||||
running,
|
running,
|
||||||
|
stats,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -168,6 +280,10 @@ impl VpioAudio {
|
|||||||
&self.playout_ring
|
&self.playout_ring
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn stats(&self) -> Arc<VpioStats> {
|
||||||
|
self.stats.clone()
|
||||||
|
}
|
||||||
|
|
||||||
pub fn stop(&self) {
|
pub fn stop(&self) {
|
||||||
self.running.store(false, Ordering::Relaxed);
|
self.running.store(false, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -151,7 +151,7 @@ pub fn bench_fec_recovery(loss_pct: f32) -> FecResult {
|
|||||||
let mut total_repair_bytes = 0usize;
|
let mut total_repair_bytes = 0usize;
|
||||||
|
|
||||||
for block_idx in 0..num_blocks {
|
for block_idx in 0..num_blocks {
|
||||||
let block_id = (block_idx % 256) as u8;
|
let block_id = (block_idx % 65536) as u16;
|
||||||
|
|
||||||
// Create fresh encoder and decoder for each block
|
// Create fresh encoder and decoder for each block
|
||||||
let mut fec_enc = RaptorQFecEncoder::new(frames_per_block, 256);
|
let mut fec_enc = RaptorQFecEncoder::new(frames_per_block, 256);
|
||||||
@@ -170,7 +170,7 @@ pub fn bench_fec_recovery(loss_pct: f32) -> FecResult {
|
|||||||
|
|
||||||
// Collect all symbols: source + repair
|
// Collect all symbols: source + repair
|
||||||
struct Symbol {
|
struct Symbol {
|
||||||
index: u8,
|
index: u16,
|
||||||
is_repair: bool,
|
is_repair: bool,
|
||||||
data: Vec<u8>,
|
data: Vec<u8>,
|
||||||
}
|
}
|
||||||
@@ -180,7 +180,7 @@ pub fn bench_fec_recovery(loss_pct: f32) -> FecResult {
|
|||||||
// For add_symbol we need to provide the raw data; the decoder pads internally
|
// For add_symbol we need to provide the raw data; the decoder pads internally
|
||||||
total_source_bytes += sym.len();
|
total_source_bytes += sym.len();
|
||||||
all_symbols.push(Symbol {
|
all_symbols.push(Symbol {
|
||||||
index: i as u8,
|
index: i as u16,
|
||||||
is_repair: false,
|
is_repair: false,
|
||||||
data: sym.clone(),
|
data: sym.clone(),
|
||||||
});
|
});
|
||||||
@@ -263,17 +263,36 @@ pub fn bench_encrypt_decrypt() -> CryptoResult {
|
|||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let header = b"bench-header";
|
// Build valid v2 MediaHeader bytes — encrypt/decrypt now derive nonces from
|
||||||
|
// header.seq and require a parseable MediaHeader (WIRE_SIZE bytes minimum).
|
||||||
|
use wzp_proto::packet::MediaHeader;
|
||||||
|
use wzp_proto::{CodecId, MediaType};
|
||||||
let mut total_bytes: usize = 0;
|
let mut total_bytes: usize = 0;
|
||||||
|
|
||||||
let start = Instant::now();
|
let start = Instant::now();
|
||||||
for payload in &payloads {
|
for (i, payload) in payloads.iter().enumerate() {
|
||||||
|
let hdr = MediaHeader {
|
||||||
|
version: 2,
|
||||||
|
flags: 0,
|
||||||
|
media_type: MediaType::Audio,
|
||||||
|
codec_id: CodecId::Opus24k,
|
||||||
|
stream_id: 0,
|
||||||
|
fec_ratio: 0,
|
||||||
|
seq: i as u32,
|
||||||
|
timestamp: (i as u32).wrapping_mul(20),
|
||||||
|
fec_block: 0,
|
||||||
|
};
|
||||||
|
let mut header_bytes = Vec::with_capacity(MediaHeader::WIRE_SIZE);
|
||||||
|
hdr.write_to(&mut header_bytes);
|
||||||
|
|
||||||
let mut ciphertext = Vec::with_capacity(payload.len() + 16);
|
let mut ciphertext = Vec::with_capacity(payload.len() + 16);
|
||||||
encryptor.encrypt(header, payload, &mut ciphertext).unwrap();
|
encryptor
|
||||||
|
.encrypt(&header_bytes, payload, &mut ciphertext)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
let mut plaintext = Vec::with_capacity(payload.len());
|
let mut plaintext = Vec::with_capacity(payload.len());
|
||||||
decryptor
|
decryptor
|
||||||
.decrypt(header, &ciphertext, &mut plaintext)
|
.decrypt(&header_bytes, &ciphertext, &mut plaintext)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
total_bytes += payload.len();
|
total_bytes += payload.len();
|
||||||
|
|||||||
@@ -409,7 +409,7 @@ impl CallEncoder {
|
|||||||
fec_ratio: MediaHeader::encode_fec_ratio(self.profile.fec_ratio),
|
fec_ratio: MediaHeader::encode_fec_ratio(self.profile.fec_ratio),
|
||||||
seq: self.seq,
|
seq: self.seq,
|
||||||
timestamp: self.timestamp_ms,
|
timestamp: self.timestamp_ms,
|
||||||
fec_block: u16::from(self.block_id) | (u16::from(sym_idx) << 8),
|
fec_block: u16::from(self.block_id) | (sym_idx << 8),
|
||||||
},
|
},
|
||||||
payload: Bytes::from(repair_data),
|
payload: Bytes::from(repair_data),
|
||||||
quality_report: None,
|
quality_report: None,
|
||||||
@@ -565,8 +565,8 @@ impl CallDecoder {
|
|||||||
// ignored — a graceful mixed-version degradation).
|
// ignored — a graceful mixed-version degradation).
|
||||||
if !packet.header.codec_id.is_opus() {
|
if !packet.header.codec_id.is_opus() {
|
||||||
let _ = self.fec_dec.add_symbol(
|
let _ = self.fec_dec.add_symbol(
|
||||||
(packet.header.fec_block & 0xFF) as u8,
|
packet.header.fec_block,
|
||||||
(packet.header.fec_block >> 8) as u8,
|
packet.header.fec_block >> 8,
|
||||||
packet.header.is_repair(),
|
packet.header.is_repair(),
|
||||||
&packet.payload,
|
&packet.payload,
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -388,18 +388,23 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Crypto handshake — establishes verified identity + session key
|
// Crypto handshake — establishes verified identity + session key
|
||||||
let _crypto_session = wzp_client::handshake::perform_handshake(
|
let hs = wzp_client::handshake::perform_handshake(
|
||||||
&*transport,
|
&*transport,
|
||||||
&seed.0,
|
&seed.0,
|
||||||
None, // alias — desktop client doesn't set one yet
|
None, // alias — desktop client doesn't set one yet
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
info!("crypto handshake complete");
|
info!(video_codec = ?hs.video_codec, "crypto handshake complete");
|
||||||
|
|
||||||
|
// Wrap the transport so all media I/O goes through AEAD encryption.
|
||||||
|
let enc_transport: Arc<dyn wzp_proto::MediaTransport> = Arc::new(
|
||||||
|
wzp_client::encrypted_transport::EncryptingTransport::new(transport.clone(), hs.session),
|
||||||
|
);
|
||||||
|
|
||||||
if cli.live {
|
if cli.live {
|
||||||
#[cfg(feature = "audio")]
|
#[cfg(feature = "audio")]
|
||||||
{
|
{
|
||||||
return run_live(transport).await;
|
return run_live(enc_transport).await;
|
||||||
}
|
}
|
||||||
#[cfg(not(feature = "audio"))]
|
#[cfg(not(feature = "audio"))]
|
||||||
{
|
{
|
||||||
@@ -423,19 +428,19 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
} else if cli.send_tone_secs.is_some() || cli.send_file.is_some() || cli.record_file.is_some() {
|
} else if cli.send_tone_secs.is_some() || cli.send_file.is_some() || cli.record_file.is_some() {
|
||||||
run_file_mode(
|
run_file_mode(
|
||||||
transport,
|
enc_transport,
|
||||||
cli.send_tone_secs,
|
cli.send_tone_secs,
|
||||||
cli.send_file,
|
cli.send_file,
|
||||||
cli.record_file,
|
cli.record_file,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
} else {
|
} else {
|
||||||
run_silence(transport).await
|
run_silence(enc_transport).await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Send silence frames (connectivity test).
|
/// Send silence frames (connectivity test).
|
||||||
async fn run_silence(transport: Arc<wzp_transport::QuinnTransport>) -> anyhow::Result<()> {
|
async fn run_silence(transport: Arc<dyn wzp_proto::MediaTransport>) -> anyhow::Result<()> {
|
||||||
let config = CallConfig::default();
|
let config = CallConfig::default();
|
||||||
let mut encoder = CallEncoder::new(&config);
|
let mut encoder = CallEncoder::new(&config);
|
||||||
|
|
||||||
@@ -485,7 +490,7 @@ async fn run_silence(transport: Arc<wzp_transport::QuinnTransport>) -> anyhow::R
|
|||||||
|
|
||||||
/// File/tone mode: send a test tone or audio file, and/or record received audio.
|
/// File/tone mode: send a test tone or audio file, and/or record received audio.
|
||||||
async fn run_file_mode(
|
async fn run_file_mode(
|
||||||
transport: Arc<wzp_transport::QuinnTransport>,
|
transport: Arc<dyn wzp_proto::MediaTransport>,
|
||||||
send_tone_secs: Option<u32>,
|
send_tone_secs: Option<u32>,
|
||||||
send_file: Option<String>,
|
send_file: Option<String>,
|
||||||
record_file: Option<String>,
|
record_file: Option<String>,
|
||||||
@@ -674,7 +679,7 @@ async fn run_file_mode(
|
|||||||
|
|
||||||
/// Live mode: capture from mic, encode, send; receive, decode, play.
|
/// Live mode: capture from mic, encode, send; receive, decode, play.
|
||||||
#[cfg(feature = "audio")]
|
#[cfg(feature = "audio")]
|
||||||
async fn run_live(transport: Arc<wzp_transport::QuinnTransport>) -> anyhow::Result<()> {
|
async fn run_live(transport: Arc<dyn wzp_proto::MediaTransport>) -> anyhow::Result<()> {
|
||||||
use wzp_client::audio_io::{AudioCapture, AudioPlayback};
|
use wzp_client::audio_io::{AudioCapture, AudioPlayback};
|
||||||
|
|
||||||
let capture = AudioCapture::start()?;
|
let capture = AudioCapture::start()?;
|
||||||
@@ -937,7 +942,7 @@ async fn run_signal_mode(
|
|||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(_session) => {
|
Ok(_hs) => {
|
||||||
info!(
|
info!(
|
||||||
"media connected — sending tone (press Ctrl+C to hang up)"
|
"media connected — sending tone (press Ctrl+C to hang up)"
|
||||||
);
|
);
|
||||||
|
|||||||
213
crates/wzp-client/src/encrypted_transport.rs
Normal file
213
crates/wzp-client/src/encrypted_transport.rs
Normal file
@@ -0,0 +1,213 @@
|
|||||||
|
//! `EncryptingTransport` — wraps any `MediaTransport` with a `CryptoSession`.
|
||||||
|
//!
|
||||||
|
//! All outbound `send_media` calls encrypt the payload before handing off to
|
||||||
|
//! the inner transport; all inbound `recv_media` calls decrypt after receiving.
|
||||||
|
//! Signal, quality, and close are forwarded unchanged.
|
||||||
|
//!
|
||||||
|
//! The quality report travels in plaintext so the relay can make QoS decisions
|
||||||
|
//! without being able to decrypt media content.
|
||||||
|
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use bytes::Bytes;
|
||||||
|
use wzp_proto::{
|
||||||
|
CryptoSession, MediaHeader, MediaPacket, MediaTransport, PathQuality, SignalMessage,
|
||||||
|
TransportError,
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Wraps a `MediaTransport` and applies AEAD encryption/decryption to media payloads.
|
||||||
|
pub struct EncryptingTransport {
|
||||||
|
inner: Arc<dyn MediaTransport>,
|
||||||
|
session: Mutex<Box<dyn CryptoSession>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl EncryptingTransport {
|
||||||
|
pub fn new(inner: Arc<dyn MediaTransport>, session: Box<dyn CryptoSession>) -> Self {
|
||||||
|
Self {
|
||||||
|
inner,
|
||||||
|
session: Mutex::new(session),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl MediaTransport for EncryptingTransport {
|
||||||
|
async fn send_media(&self, packet: &MediaPacket) -> Result<(), TransportError> {
|
||||||
|
let mut header_bytes = Vec::with_capacity(MediaHeader::WIRE_SIZE);
|
||||||
|
packet.header.write_to(&mut header_bytes);
|
||||||
|
|
||||||
|
let mut ciphertext = Vec::new();
|
||||||
|
self.session
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.encrypt(&header_bytes, &packet.payload, &mut ciphertext)
|
||||||
|
.map_err(|e| TransportError::Internal(format!("encrypt: {e}")))?;
|
||||||
|
|
||||||
|
let encrypted = MediaPacket {
|
||||||
|
header: packet.header,
|
||||||
|
payload: Bytes::from(ciphertext),
|
||||||
|
quality_report: packet.quality_report.clone(),
|
||||||
|
};
|
||||||
|
self.inner.send_media(&encrypted).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn recv_media(&self) -> Result<Option<MediaPacket>, TransportError> {
|
||||||
|
let packet = match self.inner.recv_media().await? {
|
||||||
|
Some(p) => p,
|
||||||
|
None => return Ok(None),
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut header_bytes = Vec::with_capacity(MediaHeader::WIRE_SIZE);
|
||||||
|
packet.header.write_to(&mut header_bytes);
|
||||||
|
|
||||||
|
let mut plaintext = Vec::new();
|
||||||
|
self.session
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.decrypt(&header_bytes, &packet.payload, &mut plaintext)
|
||||||
|
.map_err(|e| TransportError::Internal(format!("decrypt: {e}")))?;
|
||||||
|
|
||||||
|
Ok(Some(MediaPacket {
|
||||||
|
header: packet.header,
|
||||||
|
payload: Bytes::from(plaintext),
|
||||||
|
quality_report: packet.quality_report,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn send_signal(&self, msg: &SignalMessage) -> Result<(), TransportError> {
|
||||||
|
self.inner.send_signal(msg).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn recv_signal(&self) -> Result<Option<SignalMessage>, TransportError> {
|
||||||
|
self.inner.recv_signal().await
|
||||||
|
}
|
||||||
|
|
||||||
|
fn path_quality(&self) -> PathQuality {
|
||||||
|
self.inner.path_quality()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn close(&self) -> Result<(), TransportError> {
|
||||||
|
self.inner.close().await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use std::sync::Mutex as StdMutex;
|
||||||
|
use wzp_crypto::ChaChaSession;
|
||||||
|
use wzp_proto::{CodecId, MediaType};
|
||||||
|
|
||||||
|
struct LoopbackTransport {
|
||||||
|
sent: StdMutex<Vec<MediaPacket>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LoopbackTransport {
|
||||||
|
fn new() -> Arc<Self> {
|
||||||
|
Arc::new(Self {
|
||||||
|
sent: StdMutex::new(Vec::new()),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
fn take_sent(&self) -> Vec<MediaPacket> {
|
||||||
|
self.sent.lock().unwrap().drain(..).collect()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl MediaTransport for LoopbackTransport {
|
||||||
|
async fn send_media(&self, packet: &MediaPacket) -> Result<(), TransportError> {
|
||||||
|
self.sent.lock().unwrap().push(packet.clone());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
async fn recv_media(&self) -> Result<Option<MediaPacket>, TransportError> {
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
async fn send_signal(&self, _msg: &SignalMessage) -> Result<(), TransportError> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
async fn recv_signal(&self) -> Result<Option<SignalMessage>, TransportError> {
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
fn path_quality(&self) -> PathQuality {
|
||||||
|
PathQuality::default()
|
||||||
|
}
|
||||||
|
async fn close(&self) -> Result<(), TransportError> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn make_header(seq: u32) -> MediaHeader {
|
||||||
|
MediaHeader {
|
||||||
|
version: 2,
|
||||||
|
flags: 0,
|
||||||
|
media_type: MediaType::Audio,
|
||||||
|
codec_id: CodecId::Opus24k,
|
||||||
|
stream_id: 0,
|
||||||
|
fec_ratio: 0,
|
||||||
|
seq,
|
||||||
|
timestamp: seq * 20,
|
||||||
|
fec_block: 0,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn payload_is_encrypted_on_wire() {
|
||||||
|
let key = [0x42u8; 32];
|
||||||
|
let session: Box<dyn CryptoSession> = Box::new(ChaChaSession::new(key));
|
||||||
|
let loopback = LoopbackTransport::new();
|
||||||
|
let enc = EncryptingTransport::new(loopback.clone(), session);
|
||||||
|
|
||||||
|
let header = make_header(1);
|
||||||
|
let plaintext = b"secret audio frame";
|
||||||
|
let pkt = MediaPacket {
|
||||||
|
header,
|
||||||
|
payload: Bytes::from_static(plaintext),
|
||||||
|
quality_report: None,
|
||||||
|
};
|
||||||
|
|
||||||
|
enc.send_media(&pkt).await.unwrap();
|
||||||
|
|
||||||
|
let sent = loopback.take_sent();
|
||||||
|
assert_eq!(sent.len(), 1);
|
||||||
|
assert_eq!(sent[0].header, header, "header must be preserved");
|
||||||
|
assert_ne!(
|
||||||
|
sent[0].payload.as_ref(),
|
||||||
|
plaintext.as_ref(),
|
||||||
|
"plaintext must not appear on wire"
|
||||||
|
);
|
||||||
|
// Ciphertext is longer by exactly the AEAD tag (16 bytes)
|
||||||
|
assert_eq!(sent[0].payload.len(), plaintext.len() + 16);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn encrypt_then_decrypt_roundtrip() {
|
||||||
|
let key = [0x42u8; 32];
|
||||||
|
let send_session: Box<dyn CryptoSession> = Box::new(ChaChaSession::new(key));
|
||||||
|
let mut recv_session = ChaChaSession::new(key);
|
||||||
|
|
||||||
|
let loopback = LoopbackTransport::new();
|
||||||
|
let enc = EncryptingTransport::new(loopback.clone(), send_session);
|
||||||
|
|
||||||
|
let header = make_header(5);
|
||||||
|
let plaintext = b"hello encrypted world";
|
||||||
|
let pkt = MediaPacket {
|
||||||
|
header,
|
||||||
|
payload: Bytes::from_static(plaintext),
|
||||||
|
quality_report: None,
|
||||||
|
};
|
||||||
|
|
||||||
|
enc.send_media(&pkt).await.unwrap();
|
||||||
|
|
||||||
|
let sent = loopback.take_sent();
|
||||||
|
let wire_pkt = &sent[0];
|
||||||
|
|
||||||
|
let mut header_bytes = Vec::new();
|
||||||
|
header.write_to(&mut header_bytes);
|
||||||
|
let mut decrypted = Vec::new();
|
||||||
|
recv_session
|
||||||
|
.decrypt(&header_bytes, &wire_pkt.payload, &mut decrypted)
|
||||||
|
.expect("decrypt should succeed with matching key");
|
||||||
|
assert_eq!(&decrypted[..], plaintext);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -99,12 +99,12 @@ pub fn signal_to_call_type(signal: &SignalMessage) -> CallSignalType {
|
|||||||
SignalMessage::LossRecoveryUpdate { .. } => CallSignalType::Offer, // reuse (telemetry)
|
SignalMessage::LossRecoveryUpdate { .. } => CallSignalType::Offer, // reuse (telemetry)
|
||||||
SignalMessage::Ping { .. } | SignalMessage::Pong { .. } => CallSignalType::Offer,
|
SignalMessage::Ping { .. } | SignalMessage::Pong { .. } => CallSignalType::Offer,
|
||||||
SignalMessage::AuthToken { .. } => CallSignalType::Offer,
|
SignalMessage::AuthToken { .. } => CallSignalType::Offer,
|
||||||
SignalMessage::Hold => CallSignalType::Hold,
|
SignalMessage::Hold { .. } => CallSignalType::Hold,
|
||||||
SignalMessage::Unhold => CallSignalType::Unhold,
|
SignalMessage::Unhold { .. } => CallSignalType::Unhold,
|
||||||
SignalMessage::Mute => CallSignalType::Mute,
|
SignalMessage::Mute { .. } => CallSignalType::Mute,
|
||||||
SignalMessage::Unmute => CallSignalType::Unmute,
|
SignalMessage::Unmute { .. } => CallSignalType::Unmute,
|
||||||
SignalMessage::Transfer { .. } => CallSignalType::Transfer,
|
SignalMessage::Transfer { .. } => CallSignalType::Transfer,
|
||||||
SignalMessage::TransferAck => CallSignalType::Offer, // reuse
|
SignalMessage::TransferAck { .. } => CallSignalType::Offer, // reuse
|
||||||
SignalMessage::PresenceUpdate { .. } => CallSignalType::Offer, // reuse
|
SignalMessage::PresenceUpdate { .. } => CallSignalType::Offer, // reuse
|
||||||
SignalMessage::RouteQuery { .. } => CallSignalType::Offer, // reuse
|
SignalMessage::RouteQuery { .. } => CallSignalType::Offer, // reuse
|
||||||
SignalMessage::TransportFeedback { .. } => CallSignalType::Offer, // reuse (BWE)
|
SignalMessage::TransportFeedback { .. } => CallSignalType::Offer, // reuse (BWE)
|
||||||
@@ -164,6 +164,7 @@ mod tests {
|
|||||||
alias: None,
|
alias: None,
|
||||||
protocol_version: 2,
|
protocol_version: 2,
|
||||||
supported_versions: vec![2],
|
supported_versions: vec![2],
|
||||||
|
video_codecs: vec![],
|
||||||
};
|
};
|
||||||
|
|
||||||
let encoded = encode_call_payload(&signal, Some("relay.example.com:4433"), Some("myroom"));
|
let encoded = encode_call_payload(&signal, Some("relay.example.com:4433"), Some("myroom"));
|
||||||
@@ -185,6 +186,7 @@ mod tests {
|
|||||||
alias: None,
|
alias: None,
|
||||||
protocol_version: 2,
|
protocol_version: 2,
|
||||||
supported_versions: vec![2],
|
supported_versions: vec![2],
|
||||||
|
video_codecs: vec![],
|
||||||
};
|
};
|
||||||
assert!(matches!(signal_to_call_type(&offer), CallSignalType::Offer));
|
assert!(matches!(signal_to_call_type(&offer), CallSignalType::Offer));
|
||||||
|
|
||||||
@@ -199,19 +201,19 @@ mod tests {
|
|||||||
));
|
));
|
||||||
|
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
signal_to_call_type(&SignalMessage::Hold),
|
signal_to_call_type(&SignalMessage::Hold { version: default_signal_version() }),
|
||||||
CallSignalType::Hold
|
CallSignalType::Hold
|
||||||
));
|
));
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
signal_to_call_type(&SignalMessage::Unhold),
|
signal_to_call_type(&SignalMessage::Unhold { version: default_signal_version() }),
|
||||||
CallSignalType::Unhold
|
CallSignalType::Unhold
|
||||||
));
|
));
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
signal_to_call_type(&SignalMessage::Mute),
|
signal_to_call_type(&SignalMessage::Mute { version: default_signal_version() }),
|
||||||
CallSignalType::Mute
|
CallSignalType::Mute
|
||||||
));
|
));
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
signal_to_call_type(&SignalMessage::Unmute),
|
signal_to_call_type(&SignalMessage::Unmute { version: default_signal_version() }),
|
||||||
CallSignalType::Unmute
|
CallSignalType::Unmute
|
||||||
));
|
));
|
||||||
|
|
||||||
|
|||||||
@@ -5,9 +5,18 @@
|
|||||||
|
|
||||||
use wzp_crypto::{CryptoSession, KeyExchange, WarzoneKeyExchange};
|
use wzp_crypto::{CryptoSession, KeyExchange, WarzoneKeyExchange};
|
||||||
use wzp_proto::{
|
use wzp_proto::{
|
||||||
HangupReason, MediaTransport, QualityProfile, SignalMessage, default_signal_version,
|
CodecId, HangupReason, MediaTransport, QualityProfile, SignalMessage, default_signal_version,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const SUPPORTED_VIDEO_CODECS: &[CodecId] = &[CodecId::H264Baseline];
|
||||||
|
|
||||||
|
/// Result of a successful client-side handshake.
|
||||||
|
pub struct HandshakeResult {
|
||||||
|
pub session: Box<dyn CryptoSession>,
|
||||||
|
/// Video codec agreed with the relay. `None` if peer is audio-only.
|
||||||
|
pub video_codec: Option<CodecId>,
|
||||||
|
}
|
||||||
|
|
||||||
/// Errors that can occur during the client-side cryptographic handshake.
|
/// Errors that can occur during the client-side cryptographic handshake.
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub enum HandshakeError {
|
pub enum HandshakeError {
|
||||||
@@ -64,7 +73,17 @@ pub async fn perform_handshake(
|
|||||||
transport: &dyn MediaTransport,
|
transport: &dyn MediaTransport,
|
||||||
seed: &[u8; 32],
|
seed: &[u8; 32],
|
||||||
alias: Option<&str>,
|
alias: Option<&str>,
|
||||||
) -> Result<Box<dyn CryptoSession>, HandshakeError> {
|
) -> Result<HandshakeResult, HandshakeError> {
|
||||||
|
perform_handshake_with_video_codecs(transport, seed, alias, SUPPORTED_VIDEO_CODECS.to_vec())
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn perform_handshake_with_video_codecs(
|
||||||
|
transport: &dyn MediaTransport,
|
||||||
|
seed: &[u8; 32],
|
||||||
|
alias: Option<&str>,
|
||||||
|
video_codecs: Vec<CodecId>,
|
||||||
|
) -> Result<HandshakeResult, HandshakeError> {
|
||||||
// 1. Create key exchange from identity seed
|
// 1. Create key exchange from identity seed
|
||||||
let mut kx = WarzoneKeyExchange::from_identity_seed(seed);
|
let mut kx = WarzoneKeyExchange::from_identity_seed(seed);
|
||||||
let identity_pub = kx.identity_public_key();
|
let identity_pub = kx.identity_public_key();
|
||||||
@@ -95,28 +114,36 @@ pub async fn perform_handshake(
|
|||||||
alias: alias.map(|s| s.to_string()),
|
alias: alias.map(|s| s.to_string()),
|
||||||
protocol_version: 2,
|
protocol_version: 2,
|
||||||
supported_versions: vec![2],
|
supported_versions: vec![2],
|
||||||
|
video_codecs,
|
||||||
};
|
};
|
||||||
transport
|
transport
|
||||||
.send_signal(&offer)
|
.send_signal(&offer)
|
||||||
.await
|
.await
|
||||||
.map_err(HandshakeError::Transport)?;
|
.map_err(HandshakeError::Transport)?;
|
||||||
|
|
||||||
// 5. Wait for CallAnswer
|
// 5. Wait for CallAnswer — 10s timeout guards against relay not responding.
|
||||||
let answer = transport
|
let answer = tokio::time::timeout(std::time::Duration::from_secs(10), transport.recv_signal())
|
||||||
.recv_signal()
|
|
||||||
.await
|
.await
|
||||||
|
.map_err(|_| HandshakeError::Transport(wzp_proto::TransportError::Timeout { ms: 10_000 }))?
|
||||||
.map_err(HandshakeError::Transport)?
|
.map_err(HandshakeError::Transport)?
|
||||||
.ok_or(HandshakeError::ConnectionClosed)?;
|
.ok_or(HandshakeError::ConnectionClosed)?;
|
||||||
|
|
||||||
let (callee_identity_pub, callee_ephemeral_pub, callee_signature, _chosen_profile) =
|
let (callee_identity_pub, callee_ephemeral_pub, callee_signature, _chosen_profile, video_codec) =
|
||||||
match answer {
|
match answer {
|
||||||
SignalMessage::CallAnswer {
|
SignalMessage::CallAnswer {
|
||||||
identity_pub,
|
identity_pub,
|
||||||
ephemeral_pub,
|
ephemeral_pub,
|
||||||
signature,
|
signature,
|
||||||
chosen_profile,
|
chosen_profile,
|
||||||
|
video_codec,
|
||||||
..
|
..
|
||||||
} => (identity_pub, ephemeral_pub, signature, chosen_profile),
|
} => (
|
||||||
|
identity_pub,
|
||||||
|
ephemeral_pub,
|
||||||
|
signature,
|
||||||
|
chosen_profile,
|
||||||
|
video_codec,
|
||||||
|
),
|
||||||
SignalMessage::Hangup {
|
SignalMessage::Hangup {
|
||||||
reason: HangupReason::ProtocolVersionMismatch { server_supported },
|
reason: HangupReason::ProtocolVersionMismatch { server_supported },
|
||||||
..
|
..
|
||||||
@@ -141,7 +168,10 @@ pub async fn perform_handshake(
|
|||||||
.derive_session(&callee_ephemeral_pub)
|
.derive_session(&callee_ephemeral_pub)
|
||||||
.map_err(|e| HandshakeError::KeyDerivation(e.to_string()))?;
|
.map_err(|e| HandshakeError::KeyDerivation(e.to_string()))?;
|
||||||
|
|
||||||
Ok(session)
|
Ok(HandshakeResult {
|
||||||
|
session,
|
||||||
|
video_codec,
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -163,4 +193,34 @@ mod tests {
|
|||||||
&sig,
|
&sig,
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn handshake_result_carries_video_codec() {
|
||||||
|
// Verify that HandshakeResult has both fields accessible and that
|
||||||
|
// None is the correct default for audio-only peers.
|
||||||
|
let mut kx = WarzoneKeyExchange::from_identity_seed(&[0x55; 32]);
|
||||||
|
kx.generate_ephemeral();
|
||||||
|
let session = kx.derive_session(&[0u8; 32]).unwrap();
|
||||||
|
let hs = HandshakeResult {
|
||||||
|
session,
|
||||||
|
video_codec: None,
|
||||||
|
};
|
||||||
|
assert!(hs.video_codec.is_none());
|
||||||
|
|
||||||
|
let mut kx2 = WarzoneKeyExchange::from_identity_seed(&[0x66; 32]);
|
||||||
|
kx2.generate_ephemeral();
|
||||||
|
let session2 = kx2.derive_session(&[0u8; 32]).unwrap();
|
||||||
|
let hs2 = HandshakeResult {
|
||||||
|
session: session2,
|
||||||
|
video_codec: Some(CodecId::H264Baseline),
|
||||||
|
};
|
||||||
|
assert_eq!(hs2.video_codec, Some(CodecId::H264Baseline));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn offer_contains_h264_only() {
|
||||||
|
// Keep room video on the common denominator until Android AV1/HEVC
|
||||||
|
// send paths are proven in-device.
|
||||||
|
assert_eq!(SUPPORTED_VIDEO_CODECS, &[CodecId::H264Baseline]);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ pub mod audio_linux_aec;
|
|||||||
pub mod bench;
|
pub mod bench;
|
||||||
pub mod birthday;
|
pub mod birthday;
|
||||||
pub mod call;
|
pub mod call;
|
||||||
|
pub mod encrypted_transport;
|
||||||
pub mod drift_test;
|
pub mod drift_test;
|
||||||
pub mod dual_path;
|
pub mod dual_path;
|
||||||
pub mod echo_test;
|
pub mod echo_test;
|
||||||
|
|||||||
@@ -91,7 +91,7 @@ async fn full_handshake_both_sides_derive_same_session() {
|
|||||||
wzp_relay::handshake::accept_handshake(relay_transport_clone.as_ref(), &relay_seed),
|
wzp_relay::handshake::accept_handshake(relay_transport_clone.as_ref(), &relay_seed),
|
||||||
);
|
);
|
||||||
|
|
||||||
let mut client_session = client_result.expect("client handshake should succeed");
|
let client_hs = client_result.expect("client handshake should succeed");
|
||||||
let (mut relay_session, chosen_profile, _caller_fp, _caller_alias) =
|
let (mut relay_session, chosen_profile, _caller_fp, _caller_alias) =
|
||||||
relay_result.expect("relay handshake should succeed");
|
relay_result.expect("relay handshake should succeed");
|
||||||
|
|
||||||
@@ -99,31 +99,53 @@ async fn full_handshake_both_sides_derive_same_session() {
|
|||||||
assert_eq!(chosen_profile, wzp_proto::QualityProfile::GOOD);
|
assert_eq!(chosen_profile, wzp_proto::QualityProfile::GOOD);
|
||||||
|
|
||||||
// Verify both sides can communicate: client encrypts, relay decrypts.
|
// Verify both sides can communicate: client encrypts, relay decrypts.
|
||||||
let header = b"test-header";
|
// encrypt/decrypt derive nonces from MediaHeader.seq, so we need valid headers.
|
||||||
|
use wzp_proto::packet::MediaHeader;
|
||||||
|
use wzp_proto::{CodecId, MediaType};
|
||||||
|
let make_hdr = |seq: u32| {
|
||||||
|
let h = MediaHeader {
|
||||||
|
version: 2,
|
||||||
|
flags: 0,
|
||||||
|
media_type: MediaType::Audio,
|
||||||
|
codec_id: CodecId::Opus24k,
|
||||||
|
stream_id: 0,
|
||||||
|
fec_ratio: 0,
|
||||||
|
seq,
|
||||||
|
timestamp: seq.wrapping_mul(20),
|
||||||
|
fec_block: 0,
|
||||||
|
};
|
||||||
|
let mut b = Vec::new();
|
||||||
|
h.write_to(&mut b);
|
||||||
|
b
|
||||||
|
};
|
||||||
|
|
||||||
|
let header = make_hdr(0);
|
||||||
let plaintext = b"hello from client to relay";
|
let plaintext = b"hello from client to relay";
|
||||||
|
|
||||||
|
let mut client_session = client_hs.session;
|
||||||
let mut ciphertext = Vec::new();
|
let mut ciphertext = Vec::new();
|
||||||
client_session
|
client_session
|
||||||
.encrypt(header, plaintext, &mut ciphertext)
|
.encrypt(&header, plaintext, &mut ciphertext)
|
||||||
.expect("client encrypt should succeed");
|
.expect("client encrypt should succeed");
|
||||||
|
|
||||||
let mut decrypted = Vec::new();
|
let mut decrypted = Vec::new();
|
||||||
relay_session
|
relay_session
|
||||||
.decrypt(header, &ciphertext, &mut decrypted)
|
.decrypt(&header, &ciphertext, &mut decrypted)
|
||||||
.expect("relay decrypt should succeed");
|
.expect("relay decrypt should succeed");
|
||||||
|
|
||||||
assert_eq!(&decrypted[..], plaintext);
|
assert_eq!(&decrypted[..], plaintext);
|
||||||
|
|
||||||
// Verify reverse direction: relay encrypts, client decrypts.
|
// Verify reverse direction: relay encrypts, client decrypts.
|
||||||
|
let header2 = make_hdr(0); // relay's send_seq starts at 0
|
||||||
let plaintext2 = b"hello from relay to client";
|
let plaintext2 = b"hello from relay to client";
|
||||||
let mut ciphertext2 = Vec::new();
|
let mut ciphertext2 = Vec::new();
|
||||||
relay_session
|
relay_session
|
||||||
.encrypt(header, plaintext2, &mut ciphertext2)
|
.encrypt(&header2, plaintext2, &mut ciphertext2)
|
||||||
.expect("relay encrypt should succeed");
|
.expect("relay encrypt should succeed");
|
||||||
|
|
||||||
let mut decrypted2 = Vec::new();
|
let mut decrypted2 = Vec::new();
|
||||||
client_session
|
client_session
|
||||||
.decrypt(header, &ciphertext2, &mut decrypted2)
|
.decrypt(&header2, &ciphertext2, &mut decrypted2)
|
||||||
.expect("client decrypt should succeed");
|
.expect("client decrypt should succeed");
|
||||||
|
|
||||||
assert_eq!(&decrypted2[..], plaintext2);
|
assert_eq!(&decrypted2[..], plaintext2);
|
||||||
@@ -159,6 +181,7 @@ async fn handshake_rejects_tampered_signature() {
|
|||||||
alias: None,
|
alias: None,
|
||||||
protocol_version: 2,
|
protocol_version: 2,
|
||||||
supported_versions: vec![2],
|
supported_versions: vec![2],
|
||||||
|
video_codecs: vec![],
|
||||||
};
|
};
|
||||||
client_transport_clone
|
client_transport_clone
|
||||||
.send_signal(&offer)
|
.send_signal(&offer)
|
||||||
|
|||||||
@@ -114,11 +114,7 @@ impl EchoCanceller {
|
|||||||
/// Number of delayed samples available to release.
|
/// Number of delayed samples available to release.
|
||||||
fn delay_available(&self) -> usize {
|
fn delay_available(&self) -> usize {
|
||||||
let buffered = self.delay_write - self.delay_read;
|
let buffered = self.delay_write - self.delay_read;
|
||||||
if buffered > self.delay_samples {
|
buffered.saturating_sub(self.delay_samples)
|
||||||
buffered - self.delay_samples
|
|
||||||
} else {
|
|
||||||
0
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Process a near-end (microphone) frame, removing the estimated echo.
|
/// Process a near-end (microphone) frame, removing the estimated echo.
|
||||||
@@ -161,8 +157,8 @@ impl EchoCanceller {
|
|||||||
let mut sum_near_sq: f64 = 0.0;
|
let mut sum_near_sq: f64 = 0.0;
|
||||||
let mut sum_err_sq: f64 = 0.0;
|
let mut sum_err_sq: f64 = 0.0;
|
||||||
|
|
||||||
for i in 0..n {
|
for (i, sample) in nearend.iter_mut().enumerate() {
|
||||||
let near_f = nearend[i] as f32;
|
let near_f = *sample as f32;
|
||||||
|
|
||||||
// Position of far-end "now" for this near-end sample.
|
// Position of far-end "now" for this near-end sample.
|
||||||
let base = (self.far_pos + fl * ((n / fl) + 2) + i - n) % fl;
|
let base = (self.far_pos + fl * ((n / fl) + 2) + i - n) % fl;
|
||||||
@@ -190,7 +186,7 @@ impl EchoCanceller {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let out = error.clamp(-32768.0, 32767.0);
|
let out = error.clamp(-32768.0, 32767.0);
|
||||||
nearend[i] = out as i16;
|
*sample = out as i16;
|
||||||
|
|
||||||
sum_near_sq += (near_f as f64).powi(2);
|
sum_near_sq += (near_f as f64).powi(2);
|
||||||
sum_err_sq += (out as f64).powi(2);
|
sum_err_sq += (out as f64).powi(2);
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ impl Codec2Decoder {
|
|||||||
|
|
||||||
/// Number of compressed bytes per frame.
|
/// Number of compressed bytes per frame.
|
||||||
fn bytes_per_frame(&self) -> usize {
|
fn bytes_per_frame(&self) -> usize {
|
||||||
(self.inner.bits_per_frame() + 7) / 8
|
self.inner.bits_per_frame().div_ceil(8)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -45,7 +45,7 @@ impl Codec2Encoder {
|
|||||||
|
|
||||||
/// Number of compressed bytes per frame.
|
/// Number of compressed bytes per frame.
|
||||||
fn bytes_per_frame(&self) -> usize {
|
fn bytes_per_frame(&self) -> usize {
|
||||||
(self.inner.bits_per_frame() + 7) / 8
|
self.inner.bits_per_frame().div_ceil(8)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -56,7 +56,7 @@ impl NoiseSupressor {
|
|||||||
|
|
||||||
// f32 → i16 with clamping
|
// f32 → i16 with clamping
|
||||||
for (i, &val) in output.iter().enumerate() {
|
for (i, &val) in output.iter().enumerate() {
|
||||||
let clamped = val.max(-32768.0).min(32767.0);
|
let clamped = val.clamp(-32768.0, 32767.0);
|
||||||
pcm[offset + i] = clamped as i16;
|
pcm[offset + i] = clamped as i16;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -101,7 +101,7 @@ pub fn dred_duration_for(codec: CodecId) -> u8 {
|
|||||||
/// mode; unset or empty leaves DRED enabled.
|
/// mode; unset or empty leaves DRED enabled.
|
||||||
fn read_legacy_fec_env() -> bool {
|
fn read_legacy_fec_env() -> bool {
|
||||||
match std::env::var(LEGACY_FEC_ENV) {
|
match std::env::var(LEGACY_FEC_ENV) {
|
||||||
Ok(v) => !v.is_empty() && v != "0" && v.to_ascii_lowercase() != "false",
|
Ok(v) => !v.is_empty() && v != "0" && !v.eq_ignore_ascii_case("false"),
|
||||||
Err(_) => false,
|
Err(_) => false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -252,7 +252,7 @@ impl OpusEncoder {
|
|||||||
let clamped = if self.legacy_fec_mode {
|
let clamped = if self.legacy_fec_mode {
|
||||||
loss_pct.min(100)
|
loss_pct.min(100)
|
||||||
} else {
|
} else {
|
||||||
loss_pct.max(DRED_LOSS_FLOOR_PCT).min(100)
|
loss_pct.clamp(DRED_LOSS_FLOOR_PCT, 100)
|
||||||
};
|
};
|
||||||
let _ = self.inner.set_packet_loss(clamped);
|
let _ = self.inner.set_packet_loss(clamped);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -48,7 +48,7 @@ fn build_fir_kernel() -> [f64; FIR_TAPS] {
|
|||||||
let fc = CUTOFF_HZ / SAMPLE_RATE; // normalised cutoff (0..0.5)
|
let fc = CUTOFF_HZ / SAMPLE_RATE; // normalised cutoff (0..0.5)
|
||||||
let beta_denom = bessel_i0(KAISER_BETA);
|
let beta_denom = bessel_i0(KAISER_BETA);
|
||||||
|
|
||||||
for i in 0..FIR_TAPS {
|
for (i, slot) in kernel.iter_mut().enumerate() {
|
||||||
// Sinc
|
// Sinc
|
||||||
let n = i as f64 - m / 2.0;
|
let n = i as f64 - m / 2.0;
|
||||||
let sinc = if n.abs() < 1e-12 {
|
let sinc = if n.abs() < 1e-12 {
|
||||||
@@ -61,7 +61,7 @@ fn build_fir_kernel() -> [f64; FIR_TAPS] {
|
|||||||
let t = 2.0 * i as f64 / m - 1.0; // range [-1, 1]
|
let t = 2.0 * i as f64 / m - 1.0; // range [-1, 1]
|
||||||
let kaiser = bessel_i0(KAISER_BETA * (1.0 - t * t).max(0.0).sqrt()) / beta_denom;
|
let kaiser = bessel_i0(KAISER_BETA * (1.0 - t * t).max(0.0).sqrt()) / beta_denom;
|
||||||
|
|
||||||
kernel[i] = sinc * kaiser;
|
*slot = sinc * kaiser;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Normalise to unity DC gain.
|
// Normalise to unity DC gain.
|
||||||
@@ -180,9 +180,7 @@ impl Upsampler8to48 {
|
|||||||
work.extend_from_slice(&self.history);
|
work.extend_from_slice(&self.history);
|
||||||
for &s in input {
|
for &s in input {
|
||||||
work.push(s as f64);
|
work.push(s as f64);
|
||||||
for _ in 1..RATIO {
|
work.resize(work.len() + (RATIO - 1), 0.0f64);
|
||||||
work.push(0.0);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let out_len = stuffed_len;
|
let out_len = stuffed_len;
|
||||||
|
|||||||
@@ -209,18 +209,34 @@ mod tests {
|
|||||||
let mut alice_session = alice.derive_session(&bob_eph_pub).unwrap();
|
let mut alice_session = alice.derive_session(&bob_eph_pub).unwrap();
|
||||||
let mut bob_session = bob.derive_session(&alice_eph_pub).unwrap();
|
let mut bob_session = bob.derive_session(&alice_eph_pub).unwrap();
|
||||||
|
|
||||||
// Verify they can communicate: Alice encrypts, Bob decrypts
|
// Verify they can communicate: Alice encrypts, Bob decrypts.
|
||||||
let header = b"call-header";
|
// Use a valid v2 MediaHeader — encrypt/decrypt now derive the nonce from
|
||||||
|
// header.seq and will reject raw byte slices shorter than WIRE_SIZE.
|
||||||
|
use wzp_proto::{CodecId, MediaHeader, MediaType};
|
||||||
|
let header = MediaHeader {
|
||||||
|
version: 2,
|
||||||
|
flags: 0,
|
||||||
|
media_type: MediaType::Audio,
|
||||||
|
codec_id: CodecId::Opus24k,
|
||||||
|
stream_id: 0,
|
||||||
|
fec_ratio: 0,
|
||||||
|
seq: 0,
|
||||||
|
timestamp: 0,
|
||||||
|
fec_block: 0,
|
||||||
|
};
|
||||||
|
let mut header_bytes = Vec::new();
|
||||||
|
header.write_to(&mut header_bytes);
|
||||||
|
|
||||||
let plaintext = b"hello from alice";
|
let plaintext = b"hello from alice";
|
||||||
|
|
||||||
let mut ciphertext = Vec::new();
|
let mut ciphertext = Vec::new();
|
||||||
alice_session
|
alice_session
|
||||||
.encrypt(header, plaintext, &mut ciphertext)
|
.encrypt(&header_bytes, plaintext, &mut ciphertext)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let mut decrypted = Vec::new();
|
let mut decrypted = Vec::new();
|
||||||
bob_session
|
bob_session
|
||||||
.decrypt(header, &ciphertext, &mut decrypted)
|
.decrypt(&header_bytes, &ciphertext, &mut decrypted)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
assert_eq!(&decrypted, plaintext);
|
assert_eq!(&decrypted, plaintext);
|
||||||
|
|||||||
@@ -33,6 +33,8 @@ pub struct ChaChaSession {
|
|||||||
sas_code: Option<u32>,
|
sas_code: Option<u32>,
|
||||||
/// Per-stream anti-replay windows, keyed by (stream_id, media_type).
|
/// Per-stream anti-replay windows, keyed by (stream_id, media_type).
|
||||||
anti_replay: HashMap<(u8, MediaType), AntiReplayWindow>,
|
anti_replay: HashMap<(u8, MediaType), AntiReplayWindow>,
|
||||||
|
/// Last timestamp seen in encrypt() — used to assert monotonicity across rekeys.
|
||||||
|
last_encrypt_timestamp: Option<u32>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ChaChaSession {
|
impl ChaChaSession {
|
||||||
@@ -55,6 +57,7 @@ impl ChaChaSession {
|
|||||||
pending_rekey_secret: None,
|
pending_rekey_secret: None,
|
||||||
sas_code: None,
|
sas_code: None,
|
||||||
anti_replay: HashMap::new(),
|
anti_replay: HashMap::new(),
|
||||||
|
last_encrypt_timestamp: None,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -101,10 +104,14 @@ impl CryptoSession for ChaChaSession {
|
|||||||
plaintext: &[u8],
|
plaintext: &[u8],
|
||||||
out: &mut Vec<u8>,
|
out: &mut Vec<u8>,
|
||||||
) -> Result<(), CryptoError> {
|
) -> Result<(), CryptoError> {
|
||||||
let nonce_bytes = nonce::build_nonce(&self.session_id, self.send_seq, Direction::Send);
|
// Derive nonce from the wire-level seq in the header, not from an
|
||||||
|
// internal counter. This ensures the receiver can reconstruct the
|
||||||
|
// same nonce using the header it receives, regardless of delivery order.
|
||||||
|
let header = parse_header(header_bytes)
|
||||||
|
.ok_or_else(|| CryptoError::Internal("header too short to derive nonce".into()))?;
|
||||||
|
let nonce_bytes = nonce::build_nonce(&self.session_id, header.seq, Direction::Send);
|
||||||
let nonce = Nonce::from_slice(&nonce_bytes);
|
let nonce = Nonce::from_slice(&nonce_bytes);
|
||||||
|
|
||||||
// Encrypt with AAD
|
|
||||||
use chacha20poly1305::aead::Payload;
|
use chacha20poly1305::aead::Payload;
|
||||||
let payload = Payload {
|
let payload = Payload {
|
||||||
msg: plaintext,
|
msg: plaintext,
|
||||||
@@ -117,7 +124,19 @@ impl CryptoSession for ChaChaSession {
|
|||||||
.map_err(|_| CryptoError::Internal("encryption failed".into()))?;
|
.map_err(|_| CryptoError::Internal("encryption failed".into()))?;
|
||||||
|
|
||||||
out.extend_from_slice(&ciphertext);
|
out.extend_from_slice(&ciphertext);
|
||||||
self.send_seq = self.send_seq.wrapping_add(1);
|
self.send_seq = self.send_seq.wrapping_add(1); // packet counter for rekey trigger only
|
||||||
|
|
||||||
|
// M5: assert timestamp_ms is non-decreasing across calls (including post-rekey).
|
||||||
|
// Timestamps are u32 and wrap at 2^32 ms (~49 days); allow wrapping.
|
||||||
|
debug_assert!(
|
||||||
|
self.last_encrypt_timestamp
|
||||||
|
.map_or(true, |last| header.timestamp.wrapping_sub(last) < u32::MAX / 2),
|
||||||
|
"encrypt: timestamp must not decrease (last={:?}, now={})",
|
||||||
|
self.last_encrypt_timestamp,
|
||||||
|
header.timestamp,
|
||||||
|
);
|
||||||
|
self.last_encrypt_timestamp = Some(header.timestamp);
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -127,9 +146,14 @@ impl CryptoSession for ChaChaSession {
|
|||||||
ciphertext: &[u8],
|
ciphertext: &[u8],
|
||||||
out: &mut Vec<u8>,
|
out: &mut Vec<u8>,
|
||||||
) -> Result<(), CryptoError> {
|
) -> Result<(), CryptoError> {
|
||||||
// Use Direction::Send to match the sender's nonce construction.
|
// Parse header before decryption — needed for nonce derivation.
|
||||||
// The recv_seq counter tracks which packet from the peer we're decrypting.
|
// Using header.seq (not recv_seq) means the nonce is always derived
|
||||||
let nonce_bytes = nonce::build_nonce(&self.session_id, self.recv_seq, Direction::Send);
|
// from the same wire field as the sender, surviving out-of-order delivery.
|
||||||
|
// A recv_seq counter diverges from the sender's send_seq on any reorder,
|
||||||
|
// causing every subsequent decryption to fail for the rest of the session.
|
||||||
|
let header = parse_header(header_bytes)
|
||||||
|
.ok_or_else(|| CryptoError::Internal("header too short to derive nonce".into()))?;
|
||||||
|
let nonce_bytes = nonce::build_nonce(&self.session_id, header.seq, Direction::Send);
|
||||||
let nonce = Nonce::from_slice(&nonce_bytes);
|
let nonce = Nonce::from_slice(&nonce_bytes);
|
||||||
|
|
||||||
use chacha20poly1305::aead::Payload;
|
use chacha20poly1305::aead::Payload;
|
||||||
@@ -145,20 +169,17 @@ impl CryptoSession for ChaChaSession {
|
|||||||
|
|
||||||
let plaintext_len = plaintext.len();
|
let plaintext_len = plaintext.len();
|
||||||
out.extend_from_slice(&plaintext);
|
out.extend_from_slice(&plaintext);
|
||||||
self.recv_seq = self.recv_seq.wrapping_add(1);
|
self.recv_seq = self.recv_seq.wrapping_add(1); // packet counter for rekey trigger only
|
||||||
|
|
||||||
// Anti-replay check: if header parses as a v2 MediaHeader, verify seq
|
// Anti-replay check: header already parsed above.
|
||||||
// is not a replay for this (stream_id, media_type).
|
let window = self
|
||||||
if let Some(header) = parse_header(header_bytes) {
|
.anti_replay
|
||||||
let window = self
|
.entry((header.stream_id, header.media_type))
|
||||||
.anti_replay
|
.or_insert_with(|| default_window_for_media_type(header.media_type));
|
||||||
.entry((header.stream_id, header.media_type))
|
if let Err(e) = window.check_and_update(header.seq) {
|
||||||
.or_insert_with(|| default_window_for_media_type(header.media_type));
|
// Roll back the plaintext we just appended.
|
||||||
if let Err(e) = window.check_and_update(header.seq) {
|
out.truncate(out.len() - plaintext_len);
|
||||||
// Roll back the plaintext we just appended.
|
return Err(e);
|
||||||
out.truncate(out.len() - plaintext_len);
|
|
||||||
return Err(e);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -183,7 +204,9 @@ impl CryptoSession for ChaChaSession {
|
|||||||
.perform_rekey(peer_ephemeral_pub, secret, total_packets);
|
.perform_rekey(peer_ephemeral_pub, secret, total_packets);
|
||||||
self.install_key(new_key);
|
self.install_key(new_key);
|
||||||
|
|
||||||
// Reset sequence counters after rekey for nonce uniqueness
|
// Reset sequence counters after rekey for nonce uniqueness.
|
||||||
|
// last_encrypt_timestamp is intentionally NOT reset — spec requires
|
||||||
|
// timestamp_ms to be monotonic across rekeys.
|
||||||
self.send_seq = 0;
|
self.send_seq = 0;
|
||||||
self.recv_seq = 0;
|
self.recv_seq = 0;
|
||||||
|
|
||||||
@@ -198,24 +221,42 @@ impl CryptoSession for ChaChaSession {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use wzp_proto::{CodecId, MediaType};
|
||||||
|
|
||||||
fn make_session_pair() -> (ChaChaSession, ChaChaSession) {
|
fn make_session_pair() -> (ChaChaSession, ChaChaSession) {
|
||||||
let key = [0x42u8; 32];
|
let key = [0x42u8; 32];
|
||||||
(ChaChaSession::new(key), ChaChaSession::new(key))
|
(ChaChaSession::new(key), ChaChaSession::new(key))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Build a minimal valid v2 MediaHeader serialised to bytes.
|
||||||
|
fn make_header_bytes(seq: u32) -> Vec<u8> {
|
||||||
|
let header = MediaHeader {
|
||||||
|
version: 2,
|
||||||
|
flags: 0,
|
||||||
|
media_type: MediaType::Audio,
|
||||||
|
codec_id: CodecId::Opus24k,
|
||||||
|
stream_id: 0,
|
||||||
|
fec_ratio: 0,
|
||||||
|
seq,
|
||||||
|
timestamp: seq.wrapping_mul(20),
|
||||||
|
fec_block: 0,
|
||||||
|
};
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
header.write_to(&mut bytes);
|
||||||
|
bytes
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn encrypt_decrypt_roundtrip() {
|
fn encrypt_decrypt_roundtrip() {
|
||||||
let (mut alice, mut bob) = make_session_pair();
|
let (mut alice, mut bob) = make_session_pair();
|
||||||
let header = b"test-header";
|
let header = make_header_bytes(0);
|
||||||
let plaintext = b"hello warzone";
|
let plaintext = b"hello warzone";
|
||||||
|
|
||||||
let mut ciphertext = Vec::new();
|
let mut ciphertext = Vec::new();
|
||||||
alice.encrypt(header, plaintext, &mut ciphertext).unwrap();
|
alice.encrypt(&header, plaintext, &mut ciphertext).unwrap();
|
||||||
|
|
||||||
// Bob decrypts (his recv matches Alice's send)
|
|
||||||
let mut decrypted = Vec::new();
|
let mut decrypted = Vec::new();
|
||||||
bob.decrypt(header, &ciphertext, &mut decrypted).unwrap();
|
bob.decrypt(&header, &ciphertext, &mut decrypted).unwrap();
|
||||||
|
|
||||||
assert_eq!(&decrypted, plaintext);
|
assert_eq!(&decrypted, plaintext);
|
||||||
}
|
}
|
||||||
@@ -223,14 +264,18 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn decrypt_wrong_aad_fails() {
|
fn decrypt_wrong_aad_fails() {
|
||||||
let (mut alice, mut bob) = make_session_pair();
|
let (mut alice, mut bob) = make_session_pair();
|
||||||
let header = b"correct-header";
|
let correct_header = make_header_bytes(0);
|
||||||
|
// Different seq → different nonce AND different AAD bytes: decryption must fail.
|
||||||
|
let wrong_header = make_header_bytes(1);
|
||||||
let plaintext = b"secret data";
|
let plaintext = b"secret data";
|
||||||
|
|
||||||
let mut ciphertext = Vec::new();
|
let mut ciphertext = Vec::new();
|
||||||
alice.encrypt(header, plaintext, &mut ciphertext).unwrap();
|
alice
|
||||||
|
.encrypt(&correct_header, plaintext, &mut ciphertext)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
let mut decrypted = Vec::new();
|
let mut decrypted = Vec::new();
|
||||||
let result = bob.decrypt(b"wrong-header", &ciphertext, &mut decrypted);
|
let result = bob.decrypt(&wrong_header, &ciphertext, &mut decrypted);
|
||||||
assert!(result.is_err());
|
assert!(result.is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -239,29 +284,29 @@ mod tests {
|
|||||||
let mut alice = ChaChaSession::new([0xAA; 32]);
|
let mut alice = ChaChaSession::new([0xAA; 32]);
|
||||||
let mut eve = ChaChaSession::new([0xBB; 32]);
|
let mut eve = ChaChaSession::new([0xBB; 32]);
|
||||||
|
|
||||||
let header = b"hdr";
|
let header = make_header_bytes(0);
|
||||||
let plaintext = b"secret";
|
let plaintext = b"secret";
|
||||||
|
|
||||||
let mut ciphertext = Vec::new();
|
let mut ciphertext = Vec::new();
|
||||||
alice.encrypt(header, plaintext, &mut ciphertext).unwrap();
|
alice.encrypt(&header, plaintext, &mut ciphertext).unwrap();
|
||||||
|
|
||||||
let mut decrypted = Vec::new();
|
let mut decrypted = Vec::new();
|
||||||
let result = eve.decrypt(header, &ciphertext, &mut decrypted);
|
let result = eve.decrypt(&header, &ciphertext, &mut decrypted);
|
||||||
assert!(result.is_err());
|
assert!(result.is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn multiple_packets_roundtrip() {
|
fn multiple_packets_roundtrip() {
|
||||||
let (mut alice, mut bob) = make_session_pair();
|
let (mut alice, mut bob) = make_session_pair();
|
||||||
let header = b"hdr";
|
|
||||||
|
|
||||||
for i in 0..100 {
|
for i in 0..100u32 {
|
||||||
|
let header = make_header_bytes(i);
|
||||||
let msg = format!("message {}", i);
|
let msg = format!("message {}", i);
|
||||||
let mut ct = Vec::new();
|
let mut ct = Vec::new();
|
||||||
alice.encrypt(header, msg.as_bytes(), &mut ct).unwrap();
|
alice.encrypt(&header, msg.as_bytes(), &mut ct).unwrap();
|
||||||
|
|
||||||
let mut pt = Vec::new();
|
let mut pt = Vec::new();
|
||||||
bob.decrypt(header, &ct, &mut pt).unwrap();
|
bob.decrypt(&header, &ct, &mut pt).unwrap();
|
||||||
assert_eq!(pt, msg.as_bytes());
|
assert_eq!(pt, msg.as_bytes());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -281,6 +326,57 @@ mod tests {
|
|||||||
assert_eq!(alice.send_seq, 0);
|
assert_eq!(alice.send_seq, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn decrypt_survives_out_of_order_delivery() {
|
||||||
|
// Regression test for nonce derivation using recv_seq instead of
|
||||||
|
// MediaHeader.seq. If nonces are tied to a local counter, any reorder
|
||||||
|
// causes the counter to diverge from the sender's seq and every
|
||||||
|
// subsequent packet fails decryption permanently.
|
||||||
|
use wzp_proto::{CodecId, MediaType};
|
||||||
|
|
||||||
|
let key = [0x55u8; 32];
|
||||||
|
let mut alice = ChaChaSession::new(key);
|
||||||
|
let mut bob = ChaChaSession::new(key);
|
||||||
|
|
||||||
|
let plaintext = b"audio payload";
|
||||||
|
|
||||||
|
// Encrypt 5 packets in order (seqs 10, 11, 12, 13, 14).
|
||||||
|
let seqs = [10u32, 11, 12, 13, 14];
|
||||||
|
let mut ciphertexts: Vec<(Vec<u8>, Vec<u8>)> = Vec::new();
|
||||||
|
for &seq in &seqs {
|
||||||
|
let header = MediaHeader {
|
||||||
|
version: 2,
|
||||||
|
flags: 0,
|
||||||
|
media_type: MediaType::Audio,
|
||||||
|
codec_id: CodecId::Opus24k,
|
||||||
|
stream_id: 0,
|
||||||
|
fec_ratio: 0,
|
||||||
|
seq,
|
||||||
|
timestamp: seq * 20,
|
||||||
|
fec_block: 0,
|
||||||
|
};
|
||||||
|
let mut header_bytes = Vec::new();
|
||||||
|
header.write_to(&mut header_bytes);
|
||||||
|
let mut ct = Vec::new();
|
||||||
|
alice.encrypt(&header_bytes, plaintext, &mut ct).unwrap();
|
||||||
|
ciphertexts.push((header_bytes, ct));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Bob receives them out of order: 0, 2, 1, 4, 3
|
||||||
|
let delivery_order = [0usize, 2, 1, 4, 3];
|
||||||
|
for &idx in &delivery_order {
|
||||||
|
let (ref hdr, ref ct) = ciphertexts[idx];
|
||||||
|
let mut pt = Vec::new();
|
||||||
|
let result = bob.decrypt(hdr, ct, &mut pt);
|
||||||
|
assert!(
|
||||||
|
result.is_ok(),
|
||||||
|
"out-of-order packet (original idx={idx}, seq={}) must decrypt successfully",
|
||||||
|
seqs[idx]
|
||||||
|
);
|
||||||
|
assert_eq!(&pt, plaintext);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn per_stream_anti_replay_rejects_duplicate() {
|
fn per_stream_anti_replay_rejects_duplicate() {
|
||||||
use wzp_proto::{CodecId, MediaType};
|
use wzp_proto::{CodecId, MediaType};
|
||||||
|
|||||||
@@ -122,6 +122,7 @@ fn wzp_signal_serializes_into_fc_callsignal_payload() {
|
|||||||
alias: None,
|
alias: None,
|
||||||
protocol_version: 2,
|
protocol_version: 2,
|
||||||
supported_versions: vec![2],
|
supported_versions: vec![2],
|
||||||
|
video_codecs: vec![],
|
||||||
};
|
};
|
||||||
|
|
||||||
// Encode as featherChat CallSignal payload
|
// Encode as featherChat CallSignal payload
|
||||||
@@ -186,6 +187,7 @@ fn wzp_answer_round_trips_through_fc_callsignal() {
|
|||||||
ephemeral_pub: [20u8; 32],
|
ephemeral_pub: [20u8; 32],
|
||||||
signature: vec![30u8; 64],
|
signature: vec![30u8; 64],
|
||||||
chosen_profile: wzp_proto::QualityProfile::DEGRADED,
|
chosen_profile: wzp_proto::QualityProfile::DEGRADED,
|
||||||
|
video_codec: None,
|
||||||
};
|
};
|
||||||
|
|
||||||
let payload = wzp_client::featherchat::encode_call_payload(&answer, None, None);
|
let payload = wzp_client::featherchat::encode_call_payload(&answer, None, None);
|
||||||
@@ -309,6 +311,7 @@ fn all_signal_types_map_correctly() {
|
|||||||
alias: None,
|
alias: None,
|
||||||
protocol_version: 2,
|
protocol_version: 2,
|
||||||
supported_versions: vec![2],
|
supported_versions: vec![2],
|
||||||
|
video_codecs: vec![],
|
||||||
},
|
},
|
||||||
"Offer",
|
"Offer",
|
||||||
),
|
),
|
||||||
@@ -319,6 +322,7 @@ fn all_signal_types_map_correctly() {
|
|||||||
ephemeral_pub: [0; 32],
|
ephemeral_pub: [0; 32],
|
||||||
signature: vec![],
|
signature: vec![],
|
||||||
chosen_profile: wzp_proto::QualityProfile::GOOD,
|
chosen_profile: wzp_proto::QualityProfile::GOOD,
|
||||||
|
video_codec: None,
|
||||||
},
|
},
|
||||||
"Answer",
|
"Answer",
|
||||||
),
|
),
|
||||||
|
|||||||
@@ -29,9 +29,9 @@ pub enum DecoderBlockState {
|
|||||||
/// Manages encoder-side block tracking.
|
/// Manages encoder-side block tracking.
|
||||||
pub struct EncoderBlockManager {
|
pub struct EncoderBlockManager {
|
||||||
/// Current block ID being built.
|
/// Current block ID being built.
|
||||||
current_id: u8,
|
current_id: u16,
|
||||||
/// State of known blocks.
|
/// State of known blocks.
|
||||||
blocks: HashMap<u8, EncoderBlockState>,
|
blocks: HashMap<u16, EncoderBlockState>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl EncoderBlockManager {
|
impl EncoderBlockManager {
|
||||||
@@ -45,7 +45,7 @@ impl EncoderBlockManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Get the next block ID (advances the current building block).
|
/// Get the next block ID (advances the current building block).
|
||||||
pub fn next_block_id(&mut self) -> u8 {
|
pub fn next_block_id(&mut self) -> u16 {
|
||||||
let old = self.current_id;
|
let old = self.current_id;
|
||||||
// Mark old block as pending.
|
// Mark old block as pending.
|
||||||
self.blocks.insert(old, EncoderBlockState::Pending);
|
self.blocks.insert(old, EncoderBlockState::Pending);
|
||||||
@@ -57,23 +57,23 @@ impl EncoderBlockManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Current block ID being built.
|
/// Current block ID being built.
|
||||||
pub fn current_id(&self) -> u8 {
|
pub fn current_id(&self) -> u16 {
|
||||||
self.current_id
|
self.current_id
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Mark a block as fully sent.
|
/// Mark a block as fully sent.
|
||||||
pub fn mark_sent(&mut self, block_id: u8) {
|
pub fn mark_sent(&mut self, block_id: u16) {
|
||||||
self.blocks.insert(block_id, EncoderBlockState::Sent);
|
self.blocks.insert(block_id, EncoderBlockState::Sent);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Mark a block as acknowledged by the peer.
|
/// Mark a block as acknowledged by the peer.
|
||||||
pub fn mark_acknowledged(&mut self, block_id: u8) {
|
pub fn mark_acknowledged(&mut self, block_id: u16) {
|
||||||
self.blocks
|
self.blocks
|
||||||
.insert(block_id, EncoderBlockState::Acknowledged);
|
.insert(block_id, EncoderBlockState::Acknowledged);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get the state of a block.
|
/// Get the state of a block.
|
||||||
pub fn state(&self, block_id: u8) -> Option<EncoderBlockState> {
|
pub fn state(&self, block_id: u16) -> Option<EncoderBlockState> {
|
||||||
self.blocks.get(&block_id).copied()
|
self.blocks.get(&block_id).copied()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -93,9 +93,9 @@ impl Default for EncoderBlockManager {
|
|||||||
/// Manages decoder-side block tracking.
|
/// Manages decoder-side block tracking.
|
||||||
pub struct DecoderBlockManager {
|
pub struct DecoderBlockManager {
|
||||||
/// State of known blocks.
|
/// State of known blocks.
|
||||||
blocks: HashMap<u8, DecoderBlockState>,
|
blocks: HashMap<u16, DecoderBlockState>,
|
||||||
/// Set of completed block IDs.
|
/// Set of completed block IDs.
|
||||||
completed: HashSet<u8>,
|
completed: HashSet<u16>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl DecoderBlockManager {
|
impl DecoderBlockManager {
|
||||||
@@ -107,43 +107,43 @@ impl DecoderBlockManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Register that we are receiving symbols for a block.
|
/// Register that we are receiving symbols for a block.
|
||||||
pub fn touch(&mut self, block_id: u8) {
|
pub fn touch(&mut self, block_id: u16) {
|
||||||
self.blocks
|
self.blocks
|
||||||
.entry(block_id)
|
.entry(block_id)
|
||||||
.or_insert(DecoderBlockState::Assembling);
|
.or_insert(DecoderBlockState::Assembling);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Mark a block as successfully decoded.
|
/// Mark a block as successfully decoded.
|
||||||
pub fn mark_complete(&mut self, block_id: u8) {
|
pub fn mark_complete(&mut self, block_id: u16) {
|
||||||
self.blocks.insert(block_id, DecoderBlockState::Complete);
|
self.blocks.insert(block_id, DecoderBlockState::Complete);
|
||||||
self.completed.insert(block_id);
|
self.completed.insert(block_id);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Mark a block as expired.
|
/// Mark a block as expired.
|
||||||
pub fn mark_expired(&mut self, block_id: u8) {
|
pub fn mark_expired(&mut self, block_id: u16) {
|
||||||
self.blocks.insert(block_id, DecoderBlockState::Expired);
|
self.blocks.insert(block_id, DecoderBlockState::Expired);
|
||||||
self.completed.remove(&block_id);
|
self.completed.remove(&block_id);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Check if a block has been fully decoded.
|
/// Check if a block has been fully decoded.
|
||||||
pub fn is_block_complete(&self, block_id: u8) -> bool {
|
pub fn is_block_complete(&self, block_id: u16) -> bool {
|
||||||
self.completed.contains(&block_id)
|
self.completed.contains(&block_id)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Get the state of a block.
|
/// Get the state of a block.
|
||||||
pub fn state(&self, block_id: u8) -> Option<DecoderBlockState> {
|
pub fn state(&self, block_id: u16) -> Option<DecoderBlockState> {
|
||||||
self.blocks.get(&block_id).copied()
|
self.blocks.get(&block_id).copied()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Expire all blocks older than the given block_id (using wrapping distance).
|
/// Expire all blocks older than the given block_id (using wrapping distance).
|
||||||
pub fn expire_before(&mut self, block_id: u8) {
|
pub fn expire_before(&mut self, block_id: u16) {
|
||||||
let to_expire: Vec<u8> = self
|
let to_expire: Vec<u16> = self
|
||||||
.blocks
|
.blocks
|
||||||
.keys()
|
.keys()
|
||||||
.copied()
|
.copied()
|
||||||
.filter(|&id| {
|
.filter(|&id| {
|
||||||
let distance = block_id.wrapping_sub(id);
|
let distance = block_id.wrapping_sub(id);
|
||||||
distance > 0 && distance <= 128
|
distance > 0 && distance <= 32768
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
@@ -207,7 +207,7 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn decoder_expire_before() {
|
fn decoder_expire_before() {
|
||||||
let mut mgr = DecoderBlockManager::new();
|
let mut mgr = DecoderBlockManager::new();
|
||||||
for i in 0..5u8 {
|
for i in 0..5u16 {
|
||||||
mgr.touch(i);
|
mgr.touch(i);
|
||||||
}
|
}
|
||||||
mgr.mark_complete(1);
|
mgr.mark_complete(1);
|
||||||
@@ -231,11 +231,11 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn next_block_id_wraps() {
|
fn next_block_id_wraps() {
|
||||||
let mut mgr = EncoderBlockManager::new();
|
let mut mgr = EncoderBlockManager::new();
|
||||||
// Start at 0, advance to 255 then wrap
|
// Start at 0, advance to u16::MAX then wrap
|
||||||
for _ in 0..255 {
|
for _ in 0..65535 {
|
||||||
mgr.next_block_id();
|
mgr.next_block_id();
|
||||||
}
|
}
|
||||||
assert_eq!(mgr.current_id(), 255);
|
assert_eq!(mgr.current_id(), u16::MAX);
|
||||||
let next = mgr.next_block_id();
|
let next = mgr.next_block_id();
|
||||||
assert_eq!(next, 0);
|
assert_eq!(next, 0);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -32,7 +32,7 @@ struct BlockState {
|
|||||||
/// RaptorQ-based FEC decoder that handles multiple concurrent blocks.
|
/// RaptorQ-based FEC decoder that handles multiple concurrent blocks.
|
||||||
pub struct RaptorQFecDecoder {
|
pub struct RaptorQFecDecoder {
|
||||||
/// Per-block decoder state, keyed by block_id.
|
/// Per-block decoder state, keyed by block_id.
|
||||||
blocks: HashMap<u8, BlockState>,
|
blocks: HashMap<u16, BlockState>,
|
||||||
/// Symbol size (must match encoder).
|
/// Symbol size (must match encoder).
|
||||||
symbol_size: u16,
|
symbol_size: u16,
|
||||||
/// Number of source symbols per block (from encoder config).
|
/// Number of source symbols per block (from encoder config).
|
||||||
@@ -57,7 +57,7 @@ impl RaptorQFecDecoder {
|
|||||||
Self::new(frames_per_block, 256)
|
Self::new(frames_per_block, 256)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_or_create_block(&mut self, block_id: u8) -> &mut BlockState {
|
fn get_or_create_block(&mut self, block_id: u16) -> &mut BlockState {
|
||||||
self.blocks.entry(block_id).or_insert_with(|| BlockState {
|
self.blocks.entry(block_id).or_insert_with(|| BlockState {
|
||||||
num_source_symbols: Some(self.frames_per_block),
|
num_source_symbols: Some(self.frames_per_block),
|
||||||
packets: Vec::new(),
|
packets: Vec::new(),
|
||||||
@@ -72,8 +72,8 @@ impl RaptorQFecDecoder {
|
|||||||
impl FecDecoder for RaptorQFecDecoder {
|
impl FecDecoder for RaptorQFecDecoder {
|
||||||
fn add_symbol(
|
fn add_symbol(
|
||||||
&mut self,
|
&mut self,
|
||||||
block_id: u8,
|
block_id: u16,
|
||||||
symbol_index: u8,
|
symbol_index: u16,
|
||||||
_is_repair: bool,
|
_is_repair: bool,
|
||||||
data: &[u8],
|
data: &[u8],
|
||||||
) -> Result<(), FecError> {
|
) -> Result<(), FecError> {
|
||||||
@@ -104,13 +104,13 @@ impl FecDecoder for RaptorQFecDecoder {
|
|||||||
padded[..len].copy_from_slice(&data[..len]);
|
padded[..len].copy_from_slice(&data[..len]);
|
||||||
|
|
||||||
let esi = symbol_index as u32;
|
let esi = symbol_index as u32;
|
||||||
let packet = EncodingPacket::new(PayloadId::new(block_id, esi), padded);
|
let packet = EncodingPacket::new(PayloadId::new((block_id & 0xFF) as u8, esi), padded);
|
||||||
block.packets.push(packet);
|
block.packets.push(packet);
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn try_decode(&mut self, block_id: u8) -> Result<Option<Vec<Vec<u8>>>, FecError> {
|
fn try_decode(&mut self, block_id: u16) -> Result<Option<Vec<Vec<u8>>>, FecError> {
|
||||||
let frames_per_block = self.frames_per_block;
|
let frames_per_block = self.frames_per_block;
|
||||||
let block = match self.blocks.get_mut(&block_id) {
|
let block = match self.blocks.get_mut(&block_id) {
|
||||||
Some(b) => b,
|
Some(b) => b,
|
||||||
@@ -125,7 +125,7 @@ impl FecDecoder for RaptorQFecDecoder {
|
|||||||
let block_length = (num_source as u64) * (block.symbol_size as u64);
|
let block_length = (num_source as u64) * (block.symbol_size as u64);
|
||||||
|
|
||||||
let config = ObjectTransmissionInformation::with_defaults(block_length, block.symbol_size);
|
let config = ObjectTransmissionInformation::with_defaults(block_length, block.symbol_size);
|
||||||
let mut decoder = SourceBlockDecoder::new(block_id, &config, block_length);
|
let mut decoder = SourceBlockDecoder::new((block_id & 0xFF) as u8, &config, block_length);
|
||||||
|
|
||||||
let decoded = decoder.decode(block.packets.clone());
|
let decoded = decoder.decode(block.packets.clone());
|
||||||
|
|
||||||
@@ -156,15 +156,15 @@ impl FecDecoder for RaptorQFecDecoder {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn expire_before(&mut self, block_id: u8) {
|
fn expire_before(&mut self, block_id: u16) {
|
||||||
// Remove blocks with IDs "older" than block_id.
|
// Remove blocks with IDs "older" than block_id.
|
||||||
// With wrapping u8 IDs, we consider a block old if its distance
|
// With wrapping u16 IDs, we consider a block old if its distance
|
||||||
// (in the forward direction) to block_id is > 128.
|
// (in the forward direction) to block_id is > 32768.
|
||||||
self.blocks.retain(|&id, _| {
|
self.blocks.retain(|&id, _| {
|
||||||
let distance = block_id.wrapping_sub(id);
|
let distance = block_id.wrapping_sub(id);
|
||||||
// If distance is 0 or > 128, the block is current or "ahead" — keep it.
|
// If distance is 0 or > 32768, the block is current or "ahead" — keep it.
|
||||||
// If distance is 1..=128, the block is behind — remove it.
|
// If distance is 1..=32768, the block is behind — remove it.
|
||||||
distance == 0 || distance > 128
|
distance == 0 || distance > 32768
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -195,7 +195,7 @@ mod tests {
|
|||||||
|
|
||||||
// Feed all source symbols (using the length-prefixed padded data).
|
// Feed all source symbols (using the length-prefixed padded data).
|
||||||
for (i, pkt) in source_pkts.iter().enumerate() {
|
for (i, pkt) in source_pkts.iter().enumerate() {
|
||||||
decoder.add_symbol(0, i as u8, false, pkt.data()).unwrap();
|
decoder.add_symbol(0, i as u16, false, pkt.data()).unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
let result = decoder.try_decode(0).unwrap();
|
let result = decoder.try_decode(0).unwrap();
|
||||||
@@ -263,9 +263,9 @@ mod tests {
|
|||||||
let mut decoder = RaptorQFecDecoder::new(FRAMES_PER_BLOCK, SYMBOL_SIZE);
|
let mut decoder = RaptorQFecDecoder::new(FRAMES_PER_BLOCK, SYMBOL_SIZE);
|
||||||
|
|
||||||
// Add symbols to blocks 0, 1, 2
|
// Add symbols to blocks 0, 1, 2
|
||||||
for block_id in 0..3u8 {
|
for block_id in 0..3u16 {
|
||||||
decoder
|
decoder
|
||||||
.add_symbol(block_id, 0, false, &[block_id; 50])
|
.add_symbol(block_id, 0, false, &[block_id as u8; 50])
|
||||||
.unwrap();
|
.unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -293,10 +293,10 @@ mod tests {
|
|||||||
// Interleave symbols from block 0 and block 1
|
// Interleave symbols from block 0 and block 1
|
||||||
for i in 0..FRAMES_PER_BLOCK {
|
for i in 0..FRAMES_PER_BLOCK {
|
||||||
decoder
|
decoder
|
||||||
.add_symbol(0, i as u8, false, pkts_a[i].data())
|
.add_symbol(0, i as u16, false, pkts_a[i].data())
|
||||||
.unwrap();
|
.unwrap();
|
||||||
decoder
|
decoder
|
||||||
.add_symbol(1, i as u8, false, pkts_b[i].data())
|
.add_symbol(1, i as u16, false, pkts_b[i].data())
|
||||||
.unwrap();
|
.unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -15,8 +15,8 @@ const LEN_PREFIX: usize = 2;
|
|||||||
/// RaptorQ-based FEC encoder that groups audio frames into blocks
|
/// RaptorQ-based FEC encoder that groups audio frames into blocks
|
||||||
/// and generates fountain-code repair symbols.
|
/// and generates fountain-code repair symbols.
|
||||||
pub struct RaptorQFecEncoder {
|
pub struct RaptorQFecEncoder {
|
||||||
/// Current block ID (wraps at u8).
|
/// Current block ID (wraps at u16).
|
||||||
block_id: u8,
|
block_id: u16,
|
||||||
/// Maximum source symbols per block.
|
/// Maximum source symbols per block.
|
||||||
frames_per_block: usize,
|
frames_per_block: usize,
|
||||||
/// Accumulated source symbols for the current block.
|
/// Accumulated source symbols for the current block.
|
||||||
@@ -108,7 +108,7 @@ impl FecEncoder for RaptorQFecEncoder {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn generate_repair(&mut self, ratio: f32) -> Result<Vec<(u8, Vec<u8>)>, FecError> {
|
fn generate_repair(&mut self, ratio: f32) -> Result<Vec<(u16, Vec<u8>)>, FecError> {
|
||||||
if self.source_symbols.is_empty() {
|
if self.source_symbols.is_empty() {
|
||||||
return Ok(vec![]);
|
return Ok(vec![]);
|
||||||
}
|
}
|
||||||
@@ -122,7 +122,7 @@ impl FecEncoder for RaptorQFecEncoder {
|
|||||||
let block_data = self.build_block_data();
|
let block_data = self.build_block_data();
|
||||||
let config =
|
let config =
|
||||||
ObjectTransmissionInformation::with_defaults(block_data.len() as u64, self.symbol_size);
|
ObjectTransmissionInformation::with_defaults(block_data.len() as u64, self.symbol_size);
|
||||||
let encoder = SourceBlockEncoder::new(self.block_id, &config, &block_data);
|
let encoder = SourceBlockEncoder::new((self.block_id & 0xFF) as u8, &config, &block_data);
|
||||||
|
|
||||||
let num_source = self.source_symbols.len() as u32;
|
let num_source = self.source_symbols.len() as u32;
|
||||||
let num_repair = ((num_source as f32) * effective_ratio).ceil() as u32;
|
let num_repair = ((num_source as f32) * effective_ratio).ceil() as u32;
|
||||||
@@ -133,11 +133,11 @@ impl FecEncoder for RaptorQFecEncoder {
|
|||||||
// Generate repair packets starting from offset 0 (ESIs begin at num_source).
|
// Generate repair packets starting from offset 0 (ESIs begin at num_source).
|
||||||
let repair_packets: Vec<EncodingPacket> = encoder.repair_packets(0, num_repair);
|
let repair_packets: Vec<EncodingPacket> = encoder.repair_packets(0, num_repair);
|
||||||
|
|
||||||
let result: Vec<(u8, Vec<u8>)> = repair_packets
|
let result: Vec<(u16, Vec<u8>)> = repair_packets
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.enumerate()
|
.enumerate()
|
||||||
.map(|(i, pkt): (usize, EncodingPacket)| {
|
.map(|(i, pkt): (usize, EncodingPacket)| {
|
||||||
let idx = (num_source as u8).wrapping_add(i as u8);
|
let idx = (num_source as u16).wrapping_add(i as u16);
|
||||||
(idx, pkt.data().to_vec())
|
(idx, pkt.data().to_vec())
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
@@ -145,7 +145,7 @@ impl FecEncoder for RaptorQFecEncoder {
|
|||||||
Ok(result)
|
Ok(result)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn finalize_block(&mut self) -> Result<u8, FecError> {
|
fn finalize_block(&mut self) -> Result<u16, FecError> {
|
||||||
let completed = self.block_id;
|
let completed = self.block_id;
|
||||||
self.block_id = self.block_id.wrapping_add(1);
|
self.block_id = self.block_id.wrapping_add(1);
|
||||||
self.source_symbols.clear();
|
self.source_symbols.clear();
|
||||||
@@ -153,7 +153,7 @@ impl FecEncoder for RaptorQFecEncoder {
|
|||||||
Ok(completed)
|
Ok(completed)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn current_block_id(&self) -> u8 {
|
fn current_block_id(&self) -> u16 {
|
||||||
self.block_id
|
self.block_id
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -181,7 +181,7 @@ fn build_prefixed_block_data(symbols: &[Vec<u8>], symbol_size: u16) -> Vec<u8> {
|
|||||||
/// Helper: build source `EncodingPacket`s for a given block. Useful for
|
/// Helper: build source `EncodingPacket`s for a given block. Useful for
|
||||||
/// the decoder tests and interleaving.
|
/// the decoder tests and interleaving.
|
||||||
pub fn source_packets_for_block(
|
pub fn source_packets_for_block(
|
||||||
block_id: u8,
|
block_id: u16,
|
||||||
symbols: &[Vec<u8>],
|
symbols: &[Vec<u8>],
|
||||||
symbol_size: u16,
|
symbol_size: u16,
|
||||||
) -> Vec<EncodingPacket> {
|
) -> Vec<EncodingPacket> {
|
||||||
@@ -191,21 +191,21 @@ pub fn source_packets_for_block(
|
|||||||
.map(|i| {
|
.map(|i| {
|
||||||
let offset = i * ss;
|
let offset = i * ss;
|
||||||
let sym_data = data[offset..offset + ss].to_vec();
|
let sym_data = data[offset..offset + ss].to_vec();
|
||||||
EncodingPacket::new(PayloadId::new(block_id, i as u32), sym_data)
|
EncodingPacket::new(PayloadId::new((block_id & 0xFF) as u8, i as u32), sym_data)
|
||||||
})
|
})
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Helper: generate repair packets for the given source symbols.
|
/// Helper: generate repair packets for the given source symbols.
|
||||||
pub fn repair_packets_for_block(
|
pub fn repair_packets_for_block(
|
||||||
block_id: u8,
|
block_id: u16,
|
||||||
symbols: &[Vec<u8>],
|
symbols: &[Vec<u8>],
|
||||||
symbol_size: u16,
|
symbol_size: u16,
|
||||||
ratio: f32,
|
ratio: f32,
|
||||||
) -> Vec<EncodingPacket> {
|
) -> Vec<EncodingPacket> {
|
||||||
let data = build_prefixed_block_data(symbols, symbol_size);
|
let data = build_prefixed_block_data(symbols, symbol_size);
|
||||||
let config = ObjectTransmissionInformation::with_defaults(data.len() as u64, symbol_size);
|
let config = ObjectTransmissionInformation::with_defaults(data.len() as u64, symbol_size);
|
||||||
let encoder = SourceBlockEncoder::new(block_id, &config, &data);
|
let encoder = SourceBlockEncoder::new((block_id & 0xFF) as u8, &config, &data);
|
||||||
let num_source = symbols.len() as u32;
|
let num_source = symbols.len() as u32;
|
||||||
let num_repair = ((num_source as f32) * ratio).ceil() as u32;
|
let num_repair = ((num_source as f32) * ratio).ceil() as u32;
|
||||||
encoder.repair_packets(0, num_repair)
|
encoder.repair_packets(0, num_repair)
|
||||||
@@ -241,15 +241,21 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn block_id_wraps() {
|
fn block_id_wraps_u16() {
|
||||||
let mut enc = RaptorQFecEncoder::with_defaults(1);
|
let mut enc = RaptorQFecEncoder::with_defaults(1);
|
||||||
for expected in 0..=255u8 {
|
// Advance 300 blocks and verify no panic + monotonic increment.
|
||||||
|
for expected in 0..300u16 {
|
||||||
assert_eq!(enc.current_block_id(), expected);
|
assert_eq!(enc.current_block_id(), expected);
|
||||||
enc.add_source_symbol(&[expected; 10]).unwrap();
|
enc.add_source_symbol(&[0u8; 10]).unwrap();
|
||||||
enc.finalize_block().unwrap();
|
enc.finalize_block().unwrap();
|
||||||
}
|
}
|
||||||
// After 256 blocks, wraps back to 0
|
// Explicitly test wrap at u16 boundary.
|
||||||
assert_eq!(enc.current_block_id(), 0);
|
let mut enc2 = RaptorQFecEncoder::with_defaults(1);
|
||||||
|
enc2.block_id = u16::MAX;
|
||||||
|
enc2.add_source_symbol(&[0u8; 10]).unwrap();
|
||||||
|
let id = enc2.finalize_block().unwrap();
|
||||||
|
assert_eq!(id, u16::MAX);
|
||||||
|
assert_eq!(enc2.current_block_id(), 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
//! rather than one block fatally.
|
//! rather than one block fatally.
|
||||||
|
|
||||||
/// A symbol ready for transmission: (block_id, symbol_index, is_repair, data).
|
/// A symbol ready for transmission: (block_id, symbol_index, is_repair, data).
|
||||||
pub type Symbol = (u8, u8, bool, Vec<u8>);
|
pub type Symbol = (u16, u16, bool, Vec<u8>);
|
||||||
|
|
||||||
/// Temporal interleaver that mixes symbols across multiple FEC blocks.
|
/// Temporal interleaver that mixes symbols across multiple FEC blocks.
|
||||||
pub struct Interleaver {
|
pub struct Interleaver {
|
||||||
@@ -64,13 +64,13 @@ mod tests {
|
|||||||
let interleaver = Interleaver::with_default_depth();
|
let interleaver = Interleaver::with_default_depth();
|
||||||
|
|
||||||
let block_a: Vec<Symbol> = (0..3)
|
let block_a: Vec<Symbol> = (0..3)
|
||||||
.map(|i| (0u8, i as u8, false, vec![0xA0 + i as u8]))
|
.map(|i| (0u16, i as u16, false, vec![0xA0 + i as u8]))
|
||||||
.collect();
|
.collect();
|
||||||
let block_b: Vec<Symbol> = (0..3)
|
let block_b: Vec<Symbol> = (0..3)
|
||||||
.map(|i| (1u8, i as u8, false, vec![0xB0 + i as u8]))
|
.map(|i| (1u16, i as u16, false, vec![0xB0 + i as u8]))
|
||||||
.collect();
|
.collect();
|
||||||
let block_c: Vec<Symbol> = (0..3)
|
let block_c: Vec<Symbol> = (0..3)
|
||||||
.map(|i| (2u8, i as u8, false, vec![0xC0 + i as u8]))
|
.map(|i| (2u16, i as u16, false, vec![0xC0 + i as u8]))
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let result = interleaver.interleave(&[block_a, block_b, block_c]);
|
let result = interleaver.interleave(&[block_a, block_b, block_c]);
|
||||||
@@ -96,10 +96,10 @@ mod tests {
|
|||||||
let interleaver = Interleaver::new(2);
|
let interleaver = Interleaver::new(2);
|
||||||
|
|
||||||
let block_a: Vec<Symbol> = (0..3)
|
let block_a: Vec<Symbol> = (0..3)
|
||||||
.map(|i| (0u8, i as u8, false, vec![0xA0 + i as u8]))
|
.map(|i| (0u16, i as u16, false, vec![0xA0 + i as u8]))
|
||||||
.collect();
|
.collect();
|
||||||
let block_b: Vec<Symbol> = (0..1)
|
let block_b: Vec<Symbol> = (0..1)
|
||||||
.map(|i| (1u8, i as u8, false, vec![0xB0 + i as u8]))
|
.map(|i| (1u16, i as u16, false, vec![0xB0 + i as u8]))
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
let result = interleaver.interleave(&[block_a, block_b]);
|
let result = interleaver.interleave(&[block_a, block_b]);
|
||||||
@@ -128,7 +128,7 @@ mod tests {
|
|||||||
let blocks: Vec<Vec<Symbol>> = (0..3)
|
let blocks: Vec<Vec<Symbol>> = (0..3)
|
||||||
.map(|b| {
|
.map(|b| {
|
||||||
(0..6)
|
(0..6)
|
||||||
.map(|i| (b as u8, i as u8, false, vec![b as u8; 10]))
|
.map(|i| (b as u16, i as u16, false, vec![b as u8; 10]))
|
||||||
.collect()
|
.collect()
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|||||||
@@ -404,12 +404,14 @@ int wzp_oboe_start(const WzpOboeConfig* config, const WzpOboeRings* rings) {
|
|||||||
{
|
{
|
||||||
auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(2000);
|
auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(2000);
|
||||||
int poll_count = 0;
|
int poll_count = 0;
|
||||||
|
bool streams_started = false;
|
||||||
while (std::chrono::steady_clock::now() < deadline) {
|
while (std::chrono::steady_clock::now() < deadline) {
|
||||||
auto cap_state = g_capture_stream->getState();
|
auto cap_state = g_capture_stream->getState();
|
||||||
auto play_state = g_playout_stream->getState();
|
auto play_state = g_playout_stream->getState();
|
||||||
if (cap_state == oboe::StreamState::Started &&
|
if (cap_state == oboe::StreamState::Started &&
|
||||||
play_state == oboe::StreamState::Started) {
|
play_state == oboe::StreamState::Started) {
|
||||||
LOGI("both streams Started after %d polls", poll_count);
|
LOGI("both streams Started after %d polls", poll_count);
|
||||||
|
streams_started = true;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
poll_count++;
|
poll_count++;
|
||||||
@@ -420,6 +422,18 @@ int wzp_oboe_start(const WzpOboeConfig* config, const WzpOboeRings* rings) {
|
|||||||
(int)g_capture_stream->getState(),
|
(int)g_capture_stream->getState(),
|
||||||
(int)g_playout_stream->getState(),
|
(int)g_playout_stream->getState(),
|
||||||
poll_count);
|
poll_count);
|
||||||
|
if (!streams_started) {
|
||||||
|
LOGE("Timed out waiting for Oboe streams to reach Started state");
|
||||||
|
g_running.store(false, std::memory_order_release);
|
||||||
|
g_rings_valid.store(false, std::memory_order_release);
|
||||||
|
g_capture_stream->requestStop();
|
||||||
|
g_playout_stream->requestStop();
|
||||||
|
g_capture_stream->close();
|
||||||
|
g_playout_stream->close();
|
||||||
|
g_capture_stream.reset();
|
||||||
|
g_playout_stream.reset();
|
||||||
|
return -6;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
LOGI("Oboe started: sr=%d burst=%d ch=%d",
|
LOGI("Oboe started: sr=%d burst=%d ch=%d",
|
||||||
|
|||||||
@@ -574,6 +574,10 @@ pub enum SignalMessage {
|
|||||||
/// Protocol versions this client supports (default [2]).
|
/// Protocol versions this client supports (default [2]).
|
||||||
#[serde(default = "default_supported_versions")]
|
#[serde(default = "default_supported_versions")]
|
||||||
supported_versions: Vec<u8>,
|
supported_versions: Vec<u8>,
|
||||||
|
/// Video codecs supported by the caller, in preference order.
|
||||||
|
/// Absent on old clients (treated as video-incapable).
|
||||||
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
|
video_codecs: Vec<crate::CodecId>,
|
||||||
},
|
},
|
||||||
|
|
||||||
/// Call acceptance (analogous to Warzone's WireMessage::CallAnswer).
|
/// Call acceptance (analogous to Warzone's WireMessage::CallAnswer).
|
||||||
@@ -588,6 +592,10 @@ pub enum SignalMessage {
|
|||||||
signature: Vec<u8>,
|
signature: Vec<u8>,
|
||||||
/// Chosen quality profile.
|
/// Chosen quality profile.
|
||||||
chosen_profile: crate::QualityProfile,
|
chosen_profile: crate::QualityProfile,
|
||||||
|
/// Video codec chosen by the callee (None = video declined or peer incapable).
|
||||||
|
/// Absent on old clients (treated as no video).
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
video_codec: Option<crate::CodecId>,
|
||||||
},
|
},
|
||||||
|
|
||||||
/// ICE candidate for NAT traversal.
|
/// ICE candidate for NAT traversal.
|
||||||
@@ -669,13 +677,25 @@ pub enum SignalMessage {
|
|||||||
},
|
},
|
||||||
|
|
||||||
/// Put the call on hold (stop sending media, keep session alive).
|
/// Put the call on hold (stop sending media, keep session alive).
|
||||||
Hold,
|
Hold {
|
||||||
|
#[serde(default = "default_signal_version")]
|
||||||
|
version: u8,
|
||||||
|
},
|
||||||
/// Resume a held call.
|
/// Resume a held call.
|
||||||
Unhold,
|
Unhold {
|
||||||
|
#[serde(default = "default_signal_version")]
|
||||||
|
version: u8,
|
||||||
|
},
|
||||||
/// Mute request from the remote side (server-initiated mute, like IAX2 QUELCH).
|
/// Mute request from the remote side (server-initiated mute, like IAX2 QUELCH).
|
||||||
Mute,
|
Mute {
|
||||||
|
#[serde(default = "default_signal_version")]
|
||||||
|
version: u8,
|
||||||
|
},
|
||||||
/// Unmute request from the remote side (like IAX2 UNQUELCH).
|
/// Unmute request from the remote side (like IAX2 UNQUELCH).
|
||||||
Unmute,
|
Unmute {
|
||||||
|
#[serde(default = "default_signal_version")]
|
||||||
|
version: u8,
|
||||||
|
},
|
||||||
/// Transfer the call to another peer.
|
/// Transfer the call to another peer.
|
||||||
Transfer {
|
Transfer {
|
||||||
#[serde(default = "default_signal_version")]
|
#[serde(default = "default_signal_version")]
|
||||||
@@ -685,7 +705,10 @@ pub enum SignalMessage {
|
|||||||
relay_addr: Option<String>,
|
relay_addr: Option<String>,
|
||||||
},
|
},
|
||||||
/// Acknowledge a transfer request.
|
/// Acknowledge a transfer request.
|
||||||
TransferAck,
|
TransferAck {
|
||||||
|
#[serde(default = "default_signal_version")]
|
||||||
|
version: u8,
|
||||||
|
},
|
||||||
|
|
||||||
/// Presence update from a peer relay (gossip protocol).
|
/// Presence update from a peer relay (gossip protocol).
|
||||||
/// Sent periodically over probe connections to share which fingerprints
|
/// Sent periodically over probe connections to share which fingerprints
|
||||||
@@ -1729,7 +1752,7 @@ mod tests {
|
|||||||
version: default_signal_version(),
|
version: default_signal_version(),
|
||||||
timestamp_ms: 12345,
|
timestamp_ms: 12345,
|
||||||
},
|
},
|
||||||
SignalMessage::Hold,
|
SignalMessage::Hold { version: default_signal_version() },
|
||||||
SignalMessage::Hangup {
|
SignalMessage::Hangup {
|
||||||
version: default_signal_version(),
|
version: default_signal_version(),
|
||||||
reason: HangupReason::Normal,
|
reason: HangupReason::Normal,
|
||||||
@@ -1750,28 +1773,28 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn hold_unhold_serialize() {
|
fn hold_unhold_serialize() {
|
||||||
let hold = SignalMessage::Hold;
|
let hold = SignalMessage::Hold { version: default_signal_version() };
|
||||||
let json = serde_json::to_string(&hold).unwrap();
|
let json = serde_json::to_string(&hold).unwrap();
|
||||||
let decoded: SignalMessage = serde_json::from_str(&json).unwrap();
|
let decoded: SignalMessage = serde_json::from_str(&json).unwrap();
|
||||||
assert!(matches!(decoded, SignalMessage::Hold));
|
assert!(matches!(decoded, SignalMessage::Hold { .. }));
|
||||||
|
|
||||||
let unhold = SignalMessage::Unhold;
|
let unhold = SignalMessage::Unhold { version: default_signal_version() };
|
||||||
let json = serde_json::to_string(&unhold).unwrap();
|
let json = serde_json::to_string(&unhold).unwrap();
|
||||||
let decoded: SignalMessage = serde_json::from_str(&json).unwrap();
|
let decoded: SignalMessage = serde_json::from_str(&json).unwrap();
|
||||||
assert!(matches!(decoded, SignalMessage::Unhold));
|
assert!(matches!(decoded, SignalMessage::Unhold { .. }));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn mute_unmute_serialize() {
|
fn mute_unmute_serialize() {
|
||||||
let mute = SignalMessage::Mute;
|
let mute = SignalMessage::Mute { version: default_signal_version() };
|
||||||
let json = serde_json::to_string(&mute).unwrap();
|
let json = serde_json::to_string(&mute).unwrap();
|
||||||
let decoded: SignalMessage = serde_json::from_str(&json).unwrap();
|
let decoded: SignalMessage = serde_json::from_str(&json).unwrap();
|
||||||
assert!(matches!(decoded, SignalMessage::Mute));
|
assert!(matches!(decoded, SignalMessage::Mute { .. }));
|
||||||
|
|
||||||
let unmute = SignalMessage::Unmute;
|
let unmute = SignalMessage::Unmute { version: default_signal_version() };
|
||||||
let json = serde_json::to_string(&unmute).unwrap();
|
let json = serde_json::to_string(&unmute).unwrap();
|
||||||
let decoded: SignalMessage = serde_json::from_str(&json).unwrap();
|
let decoded: SignalMessage = serde_json::from_str(&json).unwrap();
|
||||||
assert!(matches!(decoded, SignalMessage::Unmute));
|
assert!(matches!(decoded, SignalMessage::Unmute { .. }));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1818,10 +1841,10 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn transfer_ack_serialize() {
|
fn transfer_ack_serialize() {
|
||||||
let ack = SignalMessage::TransferAck;
|
let ack = SignalMessage::TransferAck { version: default_signal_version() };
|
||||||
let json = serde_json::to_string(&ack).unwrap();
|
let json = serde_json::to_string(&ack).unwrap();
|
||||||
let decoded: SignalMessage = serde_json::from_str(&json).unwrap();
|
let decoded: SignalMessage = serde_json::from_str(&json).unwrap();
|
||||||
assert!(matches!(decoded, SignalMessage::TransferAck));
|
assert!(matches!(decoded, SignalMessage::TransferAck { .. }));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -81,14 +81,14 @@ pub trait FecEncoder: Send + Sync {
|
|||||||
///
|
///
|
||||||
/// `ratio` is the repair overhead (e.g., 0.5 = 50% more symbols than source).
|
/// `ratio` is the repair overhead (e.g., 0.5 = 50% more symbols than source).
|
||||||
/// Returns `(fec_symbol_index, repair_data)` pairs.
|
/// Returns `(fec_symbol_index, repair_data)` pairs.
|
||||||
fn generate_repair(&mut self, ratio: f32) -> Result<Vec<(u8, Vec<u8>)>, FecError>;
|
fn generate_repair(&mut self, ratio: f32) -> Result<Vec<(u16, Vec<u8>)>, FecError>;
|
||||||
|
|
||||||
/// Finalize the current block and start a new one.
|
/// Finalize the current block and start a new one.
|
||||||
/// Returns the block ID of the finalized block.
|
/// Returns the block ID of the finalized block.
|
||||||
fn finalize_block(&mut self) -> Result<u8, FecError>;
|
fn finalize_block(&mut self) -> Result<u16, FecError>;
|
||||||
|
|
||||||
/// Current block ID being built.
|
/// Current block ID being built.
|
||||||
fn current_block_id(&self) -> u8;
|
fn current_block_id(&self) -> u16;
|
||||||
|
|
||||||
/// Number of source symbols in the current block.
|
/// Number of source symbols in the current block.
|
||||||
fn current_block_size(&self) -> usize;
|
fn current_block_size(&self) -> usize;
|
||||||
@@ -99,8 +99,8 @@ pub trait FecDecoder: Send + Sync {
|
|||||||
/// Feed a received symbol (source or repair) into the decoder.
|
/// Feed a received symbol (source or repair) into the decoder.
|
||||||
fn add_symbol(
|
fn add_symbol(
|
||||||
&mut self,
|
&mut self,
|
||||||
block_id: u8,
|
block_id: u16,
|
||||||
symbol_index: u8,
|
symbol_index: u16,
|
||||||
is_repair: bool,
|
is_repair: bool,
|
||||||
data: &[u8],
|
data: &[u8],
|
||||||
) -> Result<(), FecError>;
|
) -> Result<(), FecError>;
|
||||||
@@ -109,10 +109,10 @@ pub trait FecDecoder: Send + Sync {
|
|||||||
///
|
///
|
||||||
/// Returns `None` if not yet decodable (insufficient symbols).
|
/// Returns `None` if not yet decodable (insufficient symbols).
|
||||||
/// Returns `Some(Vec<source_frames>)` on success.
|
/// Returns `Some(Vec<source_frames>)` on success.
|
||||||
fn try_decode(&mut self, block_id: u8) -> Result<Option<Vec<Vec<u8>>>, FecError>;
|
fn try_decode(&mut self, block_id: u16) -> Result<Option<Vec<Vec<u8>>>, FecError>;
|
||||||
|
|
||||||
/// Drop state for blocks older than `block_id`.
|
/// Drop state for blocks older than `block_id`.
|
||||||
fn expire_before(&mut self, block_id: u8);
|
fn expire_before(&mut self, block_id: u16);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ─── Crypto Traits ───────────────────────────────────────────────────────────
|
// ─── Crypto Traits ───────────────────────────────────────────────────────────
|
||||||
|
|||||||
@@ -42,6 +42,7 @@ pub async fn accept_handshake(
|
|||||||
supported_profiles,
|
supported_profiles,
|
||||||
caller_alias,
|
caller_alias,
|
||||||
protocol_version,
|
protocol_version,
|
||||||
|
caller_video_codecs,
|
||||||
) = match offer {
|
) = match offer {
|
||||||
SignalMessage::CallOffer {
|
SignalMessage::CallOffer {
|
||||||
identity_pub,
|
identity_pub,
|
||||||
@@ -51,6 +52,7 @@ pub async fn accept_handshake(
|
|||||||
alias,
|
alias,
|
||||||
protocol_version,
|
protocol_version,
|
||||||
supported_versions: _,
|
supported_versions: _,
|
||||||
|
video_codecs,
|
||||||
..
|
..
|
||||||
} => (
|
} => (
|
||||||
identity_pub,
|
identity_pub,
|
||||||
@@ -59,6 +61,7 @@ pub async fn accept_handshake(
|
|||||||
supported_profiles,
|
supported_profiles,
|
||||||
alias,
|
alias,
|
||||||
protocol_version,
|
protocol_version,
|
||||||
|
video_codecs,
|
||||||
),
|
),
|
||||||
other => {
|
other => {
|
||||||
return Err(anyhow::anyhow!(
|
return Err(anyhow::anyhow!(
|
||||||
@@ -108,6 +111,9 @@ pub async fn accept_handshake(
|
|||||||
// Choose the best supported profile (prefer GOOD > DEGRADED > CATASTROPHIC)
|
// Choose the best supported profile (prefer GOOD > DEGRADED > CATASTROPHIC)
|
||||||
let chosen_profile = choose_profile(&supported_profiles);
|
let chosen_profile = choose_profile(&supported_profiles);
|
||||||
|
|
||||||
|
// Pick the first video codec the caller supports (relay forwards all video).
|
||||||
|
let video_codec = caller_video_codecs.into_iter().next();
|
||||||
|
|
||||||
// 6. Send CallAnswer
|
// 6. Send CallAnswer
|
||||||
let answer = SignalMessage::CallAnswer {
|
let answer = SignalMessage::CallAnswer {
|
||||||
version: default_signal_version(),
|
version: default_signal_version(),
|
||||||
@@ -115,6 +121,7 @@ pub async fn accept_handshake(
|
|||||||
ephemeral_pub,
|
ephemeral_pub,
|
||||||
signature,
|
signature,
|
||||||
chosen_profile,
|
chosen_profile,
|
||||||
|
video_codec,
|
||||||
};
|
};
|
||||||
transport.send_signal(&answer).await?;
|
transport.send_signal(&answer).await?;
|
||||||
|
|
||||||
@@ -147,6 +154,7 @@ fn choose_profile(_supported: &[QualityProfile]) -> QualityProfile {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use wzp_proto::CodecId;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn choose_profile_picks_highest_bitrate() {
|
fn choose_profile_picks_highest_bitrate() {
|
||||||
@@ -164,4 +172,35 @@ mod tests {
|
|||||||
let chosen = choose_profile(&[]);
|
let chosen = choose_profile(&[]);
|
||||||
assert_eq!(chosen, QualityProfile::GOOD);
|
assert_eq!(chosen, QualityProfile::GOOD);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Video codec negotiation ───────────────────────────────────────
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn video_codec_picks_first_offered() {
|
||||||
|
let codecs = vec![CodecId::H264Baseline];
|
||||||
|
let chosen: Option<CodecId> = codecs.into_iter().next();
|
||||||
|
assert_eq!(chosen, Some(CodecId::H264Baseline));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn video_codec_none_when_no_codecs_offered() {
|
||||||
|
let codecs: Vec<CodecId> = vec![];
|
||||||
|
let chosen: Option<CodecId> = codecs.into_iter().next();
|
||||||
|
assert_eq!(chosen, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn video_codec_single_codec_is_selected() {
|
||||||
|
let codecs = vec![CodecId::H265Main];
|
||||||
|
let chosen: Option<CodecId> = codecs.into_iter().next();
|
||||||
|
assert_eq!(chosen, Some(CodecId::H265Main));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn video_codec_order_is_preserved() {
|
||||||
|
// The relay must pick the FIRST codec as-offered, not sort or re-rank.
|
||||||
|
let codecs = vec![CodecId::H264Baseline, CodecId::Av1Main];
|
||||||
|
let chosen: Option<CodecId> = codecs.into_iter().next();
|
||||||
|
assert_eq!(chosen, Some(CodecId::H264Baseline));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2028,7 +2028,7 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
(None, None)
|
(None, None)
|
||||||
};
|
};
|
||||||
|
|
||||||
let media_handle = tokio::spawn(room::run_participant(
|
let mut media_handle = tokio::spawn(room::run_participant(
|
||||||
room_mgr.clone(),
|
room_mgr.clone(),
|
||||||
room_name.clone(),
|
room_name.clone(),
|
||||||
participant_id,
|
participant_id,
|
||||||
@@ -2041,15 +2041,38 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
federation_room_hash,
|
federation_room_hash,
|
||||||
authenticated_fp.is_some(),
|
authenticated_fp.is_some(),
|
||||||
));
|
));
|
||||||
let signal_handle = tokio::spawn(room::run_participant_signals(
|
let mut signal_handle = tokio::spawn(room::run_participant_signals(
|
||||||
room_mgr.clone(),
|
room_mgr.clone(),
|
||||||
room_name.clone(),
|
room_name.clone(),
|
||||||
participant_id,
|
participant_id,
|
||||||
transport.clone(),
|
transport.clone(),
|
||||||
));
|
));
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
_ = media_handle => {},
|
_ = &mut media_handle => {
|
||||||
_ = signal_handle => {},
|
signal_handle.abort();
|
||||||
|
let _ = signal_handle.await;
|
||||||
|
},
|
||||||
|
_ = &mut signal_handle => {
|
||||||
|
close_transport(&*transport, "signal-loop-ended").await;
|
||||||
|
match tokio::time::timeout(Duration::from_secs(2), &mut media_handle).await {
|
||||||
|
Ok(_) => {}
|
||||||
|
Err(_) => {
|
||||||
|
warn!(
|
||||||
|
%addr,
|
||||||
|
room = %room_name,
|
||||||
|
participant = participant_id,
|
||||||
|
"media loop did not exit after signal close; forcing room leave"
|
||||||
|
);
|
||||||
|
media_handle.abort();
|
||||||
|
let _ = media_handle.await;
|
||||||
|
if let Some((update, senders)) =
|
||||||
|
room_mgr.leave(&room_name, participant_id)
|
||||||
|
{
|
||||||
|
room::broadcast_signal(&senders, &update).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
// Participant disconnected — clean up presence + per-session metrics
|
// Participant disconnected — clean up presence + per-session metrics
|
||||||
|
|||||||
@@ -110,15 +110,15 @@ impl RelayPipeline {
|
|||||||
// Feed packet into FEC decoder
|
// Feed packet into FEC decoder
|
||||||
let header = &packet.header;
|
let header = &packet.header;
|
||||||
let _ = self.fec_decoder.add_symbol(
|
let _ = self.fec_decoder.add_symbol(
|
||||||
(header.fec_block & 0xFF) as u8,
|
header.fec_block,
|
||||||
(header.fec_block >> 8) as u8,
|
header.fec_block >> 8,
|
||||||
header.is_repair(),
|
header.is_repair(),
|
||||||
&packet.payload,
|
&packet.payload,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Try to decode the FEC block
|
// Try to decode the FEC block
|
||||||
let mut output = Vec::new();
|
let mut output = Vec::new();
|
||||||
if let Ok(Some(frames)) = self.fec_decoder.try_decode((header.fec_block & 0xFF) as u8) {
|
if let Ok(Some(frames)) = self.fec_decoder.try_decode(header.fec_block) {
|
||||||
debug!(
|
debug!(
|
||||||
block = header.fec_block,
|
block = header.fec_block,
|
||||||
frames = frames.len(),
|
frames = frames.len(),
|
||||||
|
|||||||
@@ -21,6 +21,8 @@ use wzp_proto::{MediaTransport, default_signal_version};
|
|||||||
use crate::conformance::ConformanceMeter;
|
use crate::conformance::ConformanceMeter;
|
||||||
use crate::metrics::RelayMetrics;
|
use crate::metrics::RelayMetrics;
|
||||||
use crate::trunk::TrunkBatcher;
|
use crate::trunk::TrunkBatcher;
|
||||||
|
use crate::verdict::Verdict;
|
||||||
|
use crate::video_scorer::VideoScorer;
|
||||||
|
|
||||||
/// Debug tap: logs packet metadata for matching rooms.
|
/// Debug tap: logs packet metadata for matching rooms.
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
@@ -49,9 +51,13 @@ impl DebugTap {
|
|||||||
dir = dir,
|
dir = dir,
|
||||||
addr = %addr,
|
addr = %addr,
|
||||||
seq = h.seq,
|
seq = h.seq,
|
||||||
|
media = ?h.media_type,
|
||||||
codec = ?h.codec_id,
|
codec = ?h.codec_id,
|
||||||
|
stream_id = h.stream_id,
|
||||||
ts = h.timestamp,
|
ts = h.timestamp,
|
||||||
fec_block = h.fec_block,
|
fec_block = h.fec_block,
|
||||||
|
keyframe = h.is_keyframe(),
|
||||||
|
frame_end = h.is_frame_end(),
|
||||||
repair = h.is_repair(),
|
repair = h.is_repair(),
|
||||||
len = pkt.payload.len(),
|
len = pkt.payload.len(),
|
||||||
fan_out,
|
fan_out,
|
||||||
@@ -59,6 +65,35 @@ impl DebugTap {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn log_video_route(
|
||||||
|
&self,
|
||||||
|
room: &str,
|
||||||
|
addr: &std::net::SocketAddr,
|
||||||
|
peer_id: ParticipantId,
|
||||||
|
pkt: &wzp_proto::MediaPacket,
|
||||||
|
selected_layer: u8,
|
||||||
|
forwarded: bool,
|
||||||
|
reason: &str,
|
||||||
|
) {
|
||||||
|
let h = &pkt.header;
|
||||||
|
info!(
|
||||||
|
target: "debug_tap",
|
||||||
|
room = %room,
|
||||||
|
addr = %addr,
|
||||||
|
peer_id,
|
||||||
|
seq = h.seq,
|
||||||
|
stream_id = h.stream_id,
|
||||||
|
selected_layer,
|
||||||
|
codec = ?h.codec_id,
|
||||||
|
keyframe = h.is_keyframe(),
|
||||||
|
frame_end = h.is_frame_end(),
|
||||||
|
len = pkt.payload.len(),
|
||||||
|
forwarded,
|
||||||
|
reason,
|
||||||
|
"TAP VIDEO ROUTE"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
pub fn log_signal(&self, room: &str, signal: &wzp_proto::SignalMessage) {
|
pub fn log_signal(&self, room: &str, signal: &wzp_proto::SignalMessage) {
|
||||||
match signal {
|
match signal {
|
||||||
wzp_proto::SignalMessage::RoomUpdate {
|
wzp_proto::SignalMessage::RoomUpdate {
|
||||||
@@ -293,6 +328,23 @@ impl ReceiverState {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn video_route_reason(pkt: &wzp_proto::MediaPacket, selected_layer: u8) -> Option<&'static str> {
|
||||||
|
if pkt.header.stream_id == selected_layer {
|
||||||
|
return Some("selected_layer");
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compatibility for the pre-simulcast single-layer H.264 room-video path.
|
||||||
|
// Older clients used video stream 1 while current clients use stream 0 so
|
||||||
|
// they pass through relay defaults. Forward both H.264 single-layer ids.
|
||||||
|
if pkt.header.codec_id == wzp_proto::CodecId::H264Baseline
|
||||||
|
&& (pkt.header.stream_id == 0 || pkt.header.stream_id == 1)
|
||||||
|
{
|
||||||
|
return Some("h264_single_layer_compat");
|
||||||
|
}
|
||||||
|
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
/// Unique participant ID within a room.
|
/// Unique participant ID within a room.
|
||||||
pub type ParticipantId = u64;
|
pub type ParticipantId = u64;
|
||||||
|
|
||||||
@@ -302,6 +354,24 @@ fn next_id() -> ParticipantId {
|
|||||||
NEXT_PARTICIPANT_ID.fetch_add(1, Ordering::Relaxed)
|
NEXT_PARTICIPANT_ID.fetch_add(1, Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn outbound_video_stream_id(participant_id: ParticipantId) -> u8 {
|
||||||
|
// Reserve stream 0 for the sender's local/simulcast layer id. Forwarded
|
||||||
|
// room video needs a sender-distinct stream id so receivers and analyzers
|
||||||
|
// do not merge independent H264 access-unit sequences.
|
||||||
|
((participant_id.saturating_sub(1) % 250) + 1) as u8
|
||||||
|
}
|
||||||
|
|
||||||
|
fn with_outbound_video_stream_id(
|
||||||
|
pkt: &wzp_proto::MediaPacket,
|
||||||
|
participant_id: ParticipantId,
|
||||||
|
) -> wzp_proto::MediaPacket {
|
||||||
|
let mut out = pkt.clone();
|
||||||
|
if out.header.media_type == wzp_proto::MediaType::Video {
|
||||||
|
out.header.stream_id = outbound_video_stream_id(participant_id);
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
/// Events emitted by RoomManager for federation to observe.
|
/// Events emitted by RoomManager for federation to observe.
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub enum RoomEvent {
|
pub enum RoomEvent {
|
||||||
@@ -436,6 +506,25 @@ impl Room {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn remove_by_fingerprint(&mut self, fingerprint: &str) -> Vec<ParticipantId> {
|
||||||
|
let mut removed = Vec::new();
|
||||||
|
self.participants.retain(|p| {
|
||||||
|
let matches = p.fingerprint.as_deref() == Some(fingerprint);
|
||||||
|
if matches {
|
||||||
|
removed.push(p.id);
|
||||||
|
}
|
||||||
|
!matches
|
||||||
|
});
|
||||||
|
for id in &removed {
|
||||||
|
self.qualities.remove(id);
|
||||||
|
}
|
||||||
|
removed
|
||||||
|
}
|
||||||
|
|
||||||
|
fn contains(&self, id: ParticipantId) -> bool {
|
||||||
|
self.participants.iter().any(|p| p.id == id)
|
||||||
|
}
|
||||||
|
|
||||||
fn others(&self, exclude_id: ParticipantId) -> Vec<ParticipantSender> {
|
fn others(&self, exclude_id: ParticipantId) -> Vec<ParticipantSender> {
|
||||||
self.participants
|
self.participants
|
||||||
.iter()
|
.iter()
|
||||||
@@ -630,6 +719,18 @@ impl RoomManager {
|
|||||||
.entry(room_name.to_string())
|
.entry(room_name.to_string())
|
||||||
.or_insert_with(|| Arc::new(RwLock::new(Room::new())));
|
.or_insert_with(|| Arc::new(RwLock::new(Room::new())));
|
||||||
let mut room = arc.write().unwrap();
|
let mut room = arc.write().unwrap();
|
||||||
|
if let Some(fp) = fingerprint {
|
||||||
|
let removed = room.remove_by_fingerprint(fp);
|
||||||
|
for old_id in removed {
|
||||||
|
warn!(
|
||||||
|
room = room_name,
|
||||||
|
participant = old_id,
|
||||||
|
fingerprint = fp,
|
||||||
|
"replacing existing participant with same fingerprint"
|
||||||
|
);
|
||||||
|
self.clear_participant_state(room_name, old_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
let id = room.add(
|
let id = room.add(
|
||||||
addr,
|
addr,
|
||||||
sender,
|
sender,
|
||||||
@@ -706,6 +807,7 @@ impl RoomManager {
|
|||||||
let mut room = arc.write().unwrap();
|
let mut room = arc.write().unwrap();
|
||||||
room.qualities.remove(&participant_id);
|
room.qualities.remove(&participant_id);
|
||||||
room.remove(participant_id);
|
room.remove(participant_id);
|
||||||
|
self.clear_participant_state(room_name, participant_id);
|
||||||
if room.is_empty() {
|
if room.is_empty() {
|
||||||
drop(room); // release room lock
|
drop(room); // release room lock
|
||||||
drop(arc); // release DashMap guard
|
drop(arc); // release DashMap guard
|
||||||
@@ -797,7 +899,14 @@ impl RoomManager {
|
|||||||
self.keyframe_cache
|
self.keyframe_cache
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|e| e.key().0 == room_name)
|
.filter(|e| e.key().0 == room_name)
|
||||||
.map(|e| e.value().packets.clone())
|
.map(|e| {
|
||||||
|
let sender_id = e.key().1;
|
||||||
|
e.value()
|
||||||
|
.packets
|
||||||
|
.iter()
|
||||||
|
.map(|pkt| with_outbound_video_stream_id(pkt, sender_id))
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
.collect()
|
.collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -807,6 +916,27 @@ impl RoomManager {
|
|||||||
self.keyframe_buffer.retain(|k, _| k.0 != room_name);
|
self.keyframe_buffer.retain(|k, _| k.0 != room_name);
|
||||||
self.pli_state.retain(|k, _| k.0 != room_name);
|
self.pli_state.retain(|k, _| k.0 != room_name);
|
||||||
self.stream_owner.retain(|k, _| k.0 != room_name);
|
self.stream_owner.retain(|k, _| k.0 != room_name);
|
||||||
|
self.receiver_states.retain(|k, _| k.0 != room_name);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn clear_participant_state(&self, room_name: &str, participant_id: ParticipantId) {
|
||||||
|
self.keyframe_cache
|
||||||
|
.retain(|k, _| !(k.0 == room_name && k.1 == participant_id));
|
||||||
|
self.keyframe_buffer
|
||||||
|
.retain(|k, _| !(k.0 == room_name && k.1 == participant_id));
|
||||||
|
self.pli_state
|
||||||
|
.retain(|k, _| !(k.0 == room_name && k.1 == participant_id));
|
||||||
|
self.stream_owner
|
||||||
|
.retain(|k, owner| !(k.0 == room_name && *owner == participant_id));
|
||||||
|
self.receiver_states
|
||||||
|
.retain(|k, _| !(k.0 == room_name && k.1 == participant_id));
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn contains_participant(&self, room_name: &str, participant_id: ParticipantId) -> bool {
|
||||||
|
self.rooms
|
||||||
|
.get(room_name)
|
||||||
|
.map(|arc| arc.read().unwrap().contains(participant_id))
|
||||||
|
.unwrap_or(false)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// PLI suppression window (PRD-video-v1 T4.7).
|
/// PLI suppression window (PRD-video-v1 T4.7).
|
||||||
@@ -1140,6 +1270,7 @@ pub async fn run_participant(
|
|||||||
transport,
|
transport,
|
||||||
metrics,
|
metrics,
|
||||||
session_id,
|
session_id,
|
||||||
|
debug_tap,
|
||||||
is_authenticated,
|
is_authenticated,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
@@ -1194,6 +1325,9 @@ async fn run_participant_plain(
|
|||||||
None
|
None
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let mut video_scorer = VideoScorer::new();
|
||||||
|
let mut last_bwe_kbps: Option<u32> = None;
|
||||||
|
|
||||||
info!(
|
info!(
|
||||||
room = %room_name,
|
room = %room_name,
|
||||||
participant = participant_id,
|
participant = participant_id,
|
||||||
@@ -1220,6 +1354,16 @@ async fn run_participant_plain(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
if !room_mgr.contains_participant(&room_name, participant_id) {
|
||||||
|
info!(
|
||||||
|
room = %room_name,
|
||||||
|
participant = participant_id,
|
||||||
|
forwarded = packets_forwarded,
|
||||||
|
"stale participant loop stopped"
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
// Cache keyframe packets for fast join-to-first-frame replay.
|
// Cache keyframe packets for fast join-to-first-frame replay.
|
||||||
room_mgr.update_keyframe_cache(&room_name, participant_id, &pkt);
|
room_mgr.update_keyframe_cache(&room_name, participant_id, &pkt);
|
||||||
// Register this participant as the owner of this stream for PLI routing.
|
// Register this participant as the owner of this stream for PLI routing.
|
||||||
@@ -1227,6 +1371,12 @@ async fn run_participant_plain(
|
|||||||
room_mgr
|
room_mgr
|
||||||
.stream_owner
|
.stream_owner
|
||||||
.insert((room_name.clone(), pkt.header.stream_id), participant_id);
|
.insert((room_name.clone(), pkt.header.stream_id), participant_id);
|
||||||
|
if pkt.header.media_type == wzp_proto::MediaType::Video {
|
||||||
|
room_mgr.stream_owner.insert(
|
||||||
|
(room_name.clone(), outbound_video_stream_id(participant_id)),
|
||||||
|
participant_id,
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let recv_gap_ms = last_recv_instant.elapsed().as_millis() as u64;
|
let recv_gap_ms = last_recv_instant.elapsed().as_millis() as u64;
|
||||||
@@ -1261,10 +1411,19 @@ async fn run_participant_plain(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO(T6.2-follow-up): feed video packets to VideoScorer here.
|
// Feed video packets to VideoScorer; drop if verdict is Abusive.
|
||||||
// if pkt.header.media_type == MediaType::Video {
|
if pkt.header.media_type == wzp_proto::MediaType::Video {
|
||||||
// video_scorer.observe(&pkt.header, pkt.payload.len(), now, bwe_kbps);
|
let now = std::time::Instant::now();
|
||||||
// }
|
video_scorer.observe(&pkt.header, pkt.payload.len(), now, last_bwe_kbps);
|
||||||
|
if let Some(Verdict::Abusive) = video_scorer.verdict() {
|
||||||
|
warn!(
|
||||||
|
room = %room_name,
|
||||||
|
participant = participant_id,
|
||||||
|
seq = pkt.header.seq,
|
||||||
|
"VideoScorer: Abusive verdict — observe-only"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Update per-session quality metrics if a quality report is present
|
// Update per-session quality metrics if a quality report is present
|
||||||
if let Some(ref report) = pkt.quality_report {
|
if let Some(ref report) = pkt.quality_report {
|
||||||
@@ -1274,6 +1433,7 @@ async fn run_participant_plain(
|
|||||||
// Update receiver state from this participant's quality report (if present).
|
// Update receiver state from this participant's quality report (if present).
|
||||||
if let Some(ref report) = pkt.quality_report {
|
if let Some(ref report) = pkt.quality_report {
|
||||||
let bwe_kbps = report.bitrate_cap_kbps as u32;
|
let bwe_kbps = report.bitrate_cap_kbps as u32;
|
||||||
|
last_bwe_kbps = Some(bwe_kbps);
|
||||||
room_mgr.update_receiver_state(&room_name, participant_id, bwe_kbps, report.loss_pct);
|
room_mgr.update_receiver_state(&room_name, participant_id, bwe_kbps, report.loss_pct);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1308,33 +1468,56 @@ async fn run_participant_plain(
|
|||||||
broadcast_signal(&all_senders, &directive).await;
|
broadcast_signal(&all_senders, &directive).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Debug tap: log packet metadata + record stats
|
|
||||||
if let Some(ref tap) = debug_tap {
|
|
||||||
if tap.matches(&room_name) {
|
|
||||||
tap.log_packet(&room_name, "in", &addr, &pkt, others.len());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if let Some(ref mut ts) = tap_stats {
|
|
||||||
ts.record_in(&pkt, others.len());
|
|
||||||
}
|
|
||||||
|
|
||||||
// Forward to all others, applying simulcast layer selection for video.
|
// Forward to all others, applying simulcast layer selection for video.
|
||||||
let fwd_start = std::time::Instant::now();
|
let fwd_start = std::time::Instant::now();
|
||||||
let pkt_bytes = pkt.payload.len() as u64;
|
let pkt_bytes = pkt.payload.len() as u64;
|
||||||
let is_video = pkt.header.media_type == wzp_proto::MediaType::Video;
|
let is_video = pkt.header.media_type == wzp_proto::MediaType::Video;
|
||||||
|
let mut actual_fan_out = 0usize;
|
||||||
for (other_id, other) in &others {
|
for (other_id, other) in &others {
|
||||||
// Simulcast layer selection (T5.6): video packets are filtered
|
// Simulcast layer selection (T5.6): video packets are filtered
|
||||||
// by the receiver's selected layer. Audio and non-simulcast
|
// by the receiver's selected layer. Audio and non-simulcast
|
||||||
// traffic pass through unchanged.
|
// traffic pass through unchanged.
|
||||||
if is_video {
|
if is_video {
|
||||||
let selected = room_mgr.selected_layer(&room_name, *other_id);
|
let selected = room_mgr.selected_layer(&room_name, *other_id);
|
||||||
if pkt.header.stream_id != selected {
|
let route_reason = video_route_reason(&pkt, selected);
|
||||||
|
if route_reason.is_none() {
|
||||||
|
if let Some(ref tap) = debug_tap {
|
||||||
|
if tap.matches(&room_name) {
|
||||||
|
tap.log_video_route(
|
||||||
|
&room_name,
|
||||||
|
&addr,
|
||||||
|
*other_id,
|
||||||
|
&pkt,
|
||||||
|
selected,
|
||||||
|
false,
|
||||||
|
"simulcast_layer_mismatch",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if let Some(ref tap) = debug_tap {
|
||||||
|
if tap.matches(&room_name) {
|
||||||
|
tap.log_video_route(
|
||||||
|
&room_name,
|
||||||
|
&addr,
|
||||||
|
*other_id,
|
||||||
|
&pkt,
|
||||||
|
selected,
|
||||||
|
true,
|
||||||
|
route_reason.unwrap_or("selected_layer"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
match other {
|
match other {
|
||||||
ParticipantSender::Quic(t) => {
|
ParticipantSender::Quic(t) => {
|
||||||
if let Err(e) = t.send_media(&pkt).await {
|
let outbound_pkt = if is_video {
|
||||||
|
with_outbound_video_stream_id(&pkt, participant_id)
|
||||||
|
} else {
|
||||||
|
pkt.clone()
|
||||||
|
};
|
||||||
|
if let Err(e) = t.send_media(&outbound_pkt).await {
|
||||||
send_errors += 1;
|
send_errors += 1;
|
||||||
if send_errors <= 5 || send_errors % 100 == 0 {
|
if send_errors <= 5 || send_errors % 100 == 0 {
|
||||||
warn!(
|
warn!(
|
||||||
@@ -1345,14 +1528,28 @@ async fn run_participant_plain(
|
|||||||
"send_media error: {e}"
|
"send_media error: {e}"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
actual_fan_out += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
ParticipantSender::WebSocket(_) => {
|
ParticipantSender::WebSocket(_) => {
|
||||||
let _ = other.send_raw(&pkt.payload).await;
|
let _ = other.send_raw(&pkt.payload).await;
|
||||||
|
actual_fan_out += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Debug tap: log packet metadata + record stats after forwarding so
|
||||||
|
// fan_out reflects actual sends after video layer filtering.
|
||||||
|
if let Some(ref tap) = debug_tap {
|
||||||
|
if tap.matches(&room_name) {
|
||||||
|
tap.log_packet(&room_name, "in", &addr, &pkt, actual_fan_out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(ref mut ts) = tap_stats {
|
||||||
|
ts.record_in(&pkt, actual_fan_out);
|
||||||
|
}
|
||||||
|
|
||||||
// Federation: forward to active peer relays via channel
|
// Federation: forward to active peer relays via channel
|
||||||
if let Some(ref fed_tx) = federation_tx {
|
if let Some(ref fed_tx) = federation_tx {
|
||||||
let data = pkt.to_bytes();
|
let data = pkt.to_bytes();
|
||||||
@@ -1378,7 +1575,7 @@ async fn run_participant_plain(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
let fan_out = others.len() as u64;
|
let fan_out = actual_fan_out as u64;
|
||||||
metrics.packets_forwarded.inc_by(fan_out);
|
metrics.packets_forwarded.inc_by(fan_out);
|
||||||
metrics.bytes_forwarded.inc_by(pkt_bytes * fan_out);
|
metrics.bytes_forwarded.inc_by(pkt_bytes * fan_out);
|
||||||
packets_forwarded += 1;
|
packets_forwarded += 1;
|
||||||
@@ -1441,6 +1638,7 @@ async fn run_participant_trunked(
|
|||||||
transport: Arc<wzp_transport::QuinnTransport>,
|
transport: Arc<wzp_transport::QuinnTransport>,
|
||||||
metrics: Arc<RelayMetrics>,
|
metrics: Arc<RelayMetrics>,
|
||||||
session_id: String,
|
session_id: String,
|
||||||
|
debug_tap: Option<DebugTap>,
|
||||||
_is_authenticated: bool,
|
_is_authenticated: bool,
|
||||||
) {
|
) {
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
@@ -1454,6 +1652,13 @@ async fn run_participant_trunked(
|
|||||||
let mut last_log_instant = std::time::Instant::now();
|
let mut last_log_instant = std::time::Instant::now();
|
||||||
let mut conformance =
|
let mut conformance =
|
||||||
ConformanceMeter::with_token_bucket(crate::conformance::TokenBucket::for_audio_session());
|
ConformanceMeter::with_token_bucket(crate::conformance::TokenBucket::for_audio_session());
|
||||||
|
let mut video_scorer_trunked = VideoScorer::new();
|
||||||
|
let mut last_bwe_kbps_trunked: Option<u32> = None;
|
||||||
|
let mut tap_stats = if debug_tap.as_ref().map_or(false, |t| t.matches(&room_name)) {
|
||||||
|
Some(TapStats::new())
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
info!(
|
info!(
|
||||||
room = %room_name,
|
room = %room_name,
|
||||||
@@ -1492,6 +1697,16 @@ async fn run_participant_trunked(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
if !room_mgr.contains_participant(&room_name, participant_id) {
|
||||||
|
info!(
|
||||||
|
room = %room_name,
|
||||||
|
participant = participant_id,
|
||||||
|
forwarded = packets_forwarded,
|
||||||
|
"stale participant loop stopped (trunked)"
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
// Cache keyframe packets for fast join-to-first-frame replay.
|
// Cache keyframe packets for fast join-to-first-frame replay.
|
||||||
room_mgr.update_keyframe_cache(&room_name, participant_id, &pkt);
|
room_mgr.update_keyframe_cache(&room_name, participant_id, &pkt);
|
||||||
// Register this participant as the owner of this stream for PLI routing.
|
// Register this participant as the owner of this stream for PLI routing.
|
||||||
@@ -1500,6 +1715,15 @@ async fn run_participant_trunked(
|
|||||||
(room_name.clone(), pkt.header.stream_id),
|
(room_name.clone(), pkt.header.stream_id),
|
||||||
participant_id,
|
participant_id,
|
||||||
);
|
);
|
||||||
|
if pkt.header.media_type == wzp_proto::MediaType::Video {
|
||||||
|
room_mgr.stream_owner.insert(
|
||||||
|
(
|
||||||
|
room_name.clone(),
|
||||||
|
outbound_video_stream_id(participant_id),
|
||||||
|
),
|
||||||
|
participant_id,
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let recv_gap_ms = last_recv_instant.elapsed().as_millis() as u64;
|
let recv_gap_ms = last_recv_instant.elapsed().as_millis() as u64;
|
||||||
@@ -1533,9 +1757,24 @@ async fn run_participant_trunked(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Feed video packets to VideoScorer; drop if verdict is Abusive.
|
||||||
|
if pkt.header.media_type == wzp_proto::MediaType::Video {
|
||||||
|
let now = std::time::Instant::now();
|
||||||
|
video_scorer_trunked.observe(&pkt.header, pkt.payload.len(), now, last_bwe_kbps_trunked);
|
||||||
|
if let Some(Verdict::Abusive) = video_scorer_trunked.verdict() {
|
||||||
|
warn!(
|
||||||
|
room = %room_name,
|
||||||
|
participant = participant_id,
|
||||||
|
seq = pkt.header.seq,
|
||||||
|
"VideoScorer: Abusive verdict — observe-only (trunked)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Update receiver state from this participant's quality report.
|
// Update receiver state from this participant's quality report.
|
||||||
if let Some(ref report) = pkt.quality_report {
|
if let Some(ref report) = pkt.quality_report {
|
||||||
let bwe_kbps = report.bitrate_cap_kbps as u32;
|
let bwe_kbps = report.bitrate_cap_kbps as u32;
|
||||||
|
last_bwe_kbps_trunked = Some(bwe_kbps);
|
||||||
room_mgr.update_receiver_state(&room_name, participant_id, bwe_kbps, report.loss_pct);
|
room_mgr.update_receiver_state(&room_name, participant_id, bwe_kbps, report.loss_pct);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1571,12 +1810,40 @@ async fn run_participant_trunked(
|
|||||||
let fwd_start = std::time::Instant::now();
|
let fwd_start = std::time::Instant::now();
|
||||||
let pkt_bytes = pkt.payload.len() as u64;
|
let pkt_bytes = pkt.payload.len() as u64;
|
||||||
let is_video = pkt.header.media_type == wzp_proto::MediaType::Video;
|
let is_video = pkt.header.media_type == wzp_proto::MediaType::Video;
|
||||||
|
let mut actual_fan_out = 0usize;
|
||||||
for (other_id, other) in &others {
|
for (other_id, other) in &others {
|
||||||
if is_video {
|
if is_video {
|
||||||
let selected = room_mgr.selected_layer(&room_name, *other_id);
|
let selected = room_mgr.selected_layer(&room_name, *other_id);
|
||||||
if pkt.header.stream_id != selected {
|
let route_reason = video_route_reason(&pkt, selected);
|
||||||
|
if route_reason.is_none() {
|
||||||
|
if let Some(ref tap) = debug_tap {
|
||||||
|
if tap.matches(&room_name) {
|
||||||
|
tap.log_video_route(
|
||||||
|
&room_name,
|
||||||
|
&addr,
|
||||||
|
*other_id,
|
||||||
|
&pkt,
|
||||||
|
selected,
|
||||||
|
false,
|
||||||
|
"simulcast_layer_mismatch",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
if let Some(ref tap) = debug_tap {
|
||||||
|
if tap.matches(&room_name) {
|
||||||
|
tap.log_video_route(
|
||||||
|
&room_name,
|
||||||
|
&addr,
|
||||||
|
*other_id,
|
||||||
|
&pkt,
|
||||||
|
selected,
|
||||||
|
true,
|
||||||
|
route_reason.unwrap_or("selected_layer"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
match other {
|
match other {
|
||||||
ParticipantSender::Quic(t) => {
|
ParticipantSender::Quic(t) => {
|
||||||
@@ -1584,7 +1851,12 @@ async fn run_participant_trunked(
|
|||||||
let fwd = forwarders
|
let fwd = forwarders
|
||||||
.entry(peer_addr)
|
.entry(peer_addr)
|
||||||
.or_insert_with(|| TrunkedForwarder::new(t.clone(), sid_bytes));
|
.or_insert_with(|| TrunkedForwarder::new(t.clone(), sid_bytes));
|
||||||
if let Err(e) = fwd.send(&pkt).await {
|
let outbound_pkt = if is_video {
|
||||||
|
with_outbound_video_stream_id(&pkt, participant_id)
|
||||||
|
} else {
|
||||||
|
pkt.clone()
|
||||||
|
};
|
||||||
|
if let Err(e) = fwd.send(&outbound_pkt).await {
|
||||||
send_errors += 1;
|
send_errors += 1;
|
||||||
if send_errors <= 5 || send_errors % 100 == 0 {
|
if send_errors <= 5 || send_errors % 100 == 0 {
|
||||||
warn!(
|
warn!(
|
||||||
@@ -1595,13 +1867,24 @@ async fn run_participant_trunked(
|
|||||||
"trunked send error: {e}"
|
"trunked send error: {e}"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
} else {
|
||||||
|
actual_fan_out += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
ParticipantSender::WebSocket(_) => {
|
ParticipantSender::WebSocket(_) => {
|
||||||
let _ = other.send_raw(&pkt.payload).await;
|
let _ = other.send_raw(&pkt.payload).await;
|
||||||
|
actual_fan_out += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if let Some(ref tap) = debug_tap {
|
||||||
|
if tap.matches(&room_name) {
|
||||||
|
tap.log_packet(&room_name, "in", &addr, &pkt, actual_fan_out);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(ref mut ts) = tap_stats {
|
||||||
|
ts.record_in(&pkt, actual_fan_out);
|
||||||
|
}
|
||||||
let fwd_ms = fwd_start.elapsed().as_millis() as u64;
|
let fwd_ms = fwd_start.elapsed().as_millis() as u64;
|
||||||
if fwd_ms > max_forward_ms {
|
if fwd_ms > max_forward_ms {
|
||||||
max_forward_ms = fwd_ms;
|
max_forward_ms = fwd_ms;
|
||||||
@@ -1611,12 +1894,12 @@ async fn run_participant_trunked(
|
|||||||
room = %room_name,
|
room = %room_name,
|
||||||
participant = participant_id,
|
participant = participant_id,
|
||||||
fwd_ms,
|
fwd_ms,
|
||||||
fan_out = others.len(),
|
fan_out = actual_fan_out,
|
||||||
"slow forward (trunked)"
|
"slow forward (trunked)"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
let fan_out = others.len() as u64;
|
let fan_out = actual_fan_out as u64;
|
||||||
metrics.packets_forwarded.inc_by(fan_out);
|
metrics.packets_forwarded.inc_by(fan_out);
|
||||||
metrics.bytes_forwarded.inc_by(pkt_bytes * fan_out);
|
metrics.bytes_forwarded.inc_by(pkt_bytes * fan_out);
|
||||||
packets_forwarded += 1;
|
packets_forwarded += 1;
|
||||||
@@ -1635,6 +1918,10 @@ async fn run_participant_trunked(
|
|||||||
send_errors,
|
send_errors,
|
||||||
"participant stats (trunked)"
|
"participant stats (trunked)"
|
||||||
);
|
);
|
||||||
|
if let (Some(tap), Some(ts)) = (&debug_tap, &mut tap_stats) {
|
||||||
|
tap.log_stats(&room_name, ts);
|
||||||
|
ts.reset_period();
|
||||||
|
}
|
||||||
max_recv_gap_ms = 0;
|
max_recv_gap_ms = 0;
|
||||||
max_forward_ms = 0;
|
max_forward_ms = 0;
|
||||||
last_log_instant = std::time::Instant::now();
|
last_log_instant = std::time::Instant::now();
|
||||||
@@ -1693,6 +1980,72 @@ mod tests {
|
|||||||
assert!(mgr.list().is_empty());
|
assert!(mgr.list().is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn join_replaces_existing_fingerprint_in_same_room() {
|
||||||
|
let mgr = RoomManager::new();
|
||||||
|
let addr: std::net::SocketAddr = "127.0.0.1:10000".parse().unwrap();
|
||||||
|
let (tx1, _rx1) = tokio::sync::mpsc::channel(1);
|
||||||
|
let (tx2, _rx2) = tokio::sync::mpsc::channel(1);
|
||||||
|
|
||||||
|
let (first_id, _, _, _) = mgr
|
||||||
|
.join(
|
||||||
|
"room",
|
||||||
|
addr,
|
||||||
|
ParticipantSender::WebSocket(tx1),
|
||||||
|
Some("fp-a"),
|
||||||
|
Some("old"),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let (second_id, update, _, _) = mgr
|
||||||
|
.join(
|
||||||
|
"room",
|
||||||
|
addr,
|
||||||
|
ParticipantSender::WebSocket(tx2),
|
||||||
|
Some("fp-a"),
|
||||||
|
Some("new"),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_ne!(first_id, second_id);
|
||||||
|
assert!(!mgr.contains_participant("room", first_id));
|
||||||
|
assert!(mgr.contains_participant("room", second_id));
|
||||||
|
assert_eq!(mgr.room_size("room"), 1);
|
||||||
|
if let wzp_proto::SignalMessage::RoomUpdate {
|
||||||
|
count,
|
||||||
|
participants,
|
||||||
|
..
|
||||||
|
} = update
|
||||||
|
{
|
||||||
|
assert_eq!(count, 1);
|
||||||
|
assert_eq!(participants[0].fingerprint, "fp-a");
|
||||||
|
assert_eq!(participants[0].alias.as_deref(), Some("new"));
|
||||||
|
} else {
|
||||||
|
panic!("expected RoomUpdate");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn outbound_video_stream_ids_are_sender_distinct_and_nonzero() {
|
||||||
|
assert_eq!(outbound_video_stream_id(1), 1);
|
||||||
|
assert_eq!(outbound_video_stream_id(2), 2);
|
||||||
|
assert_eq!(outbound_video_stream_id(250), 250);
|
||||||
|
assert_eq!(outbound_video_stream_id(251), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rewrite_only_changes_video_stream_id() {
|
||||||
|
let mut video = make_test_packet(b"video");
|
||||||
|
video.header.media_type = wzp_proto::MediaType::Video;
|
||||||
|
video.header.stream_id = 0;
|
||||||
|
let rewritten = with_outbound_video_stream_id(&video, 42);
|
||||||
|
assert_eq!(rewritten.header.stream_id, 42);
|
||||||
|
assert_eq!(video.header.stream_id, 0);
|
||||||
|
|
||||||
|
let audio = make_test_packet(b"audio");
|
||||||
|
let rewritten_audio = with_outbound_video_stream_id(&audio, 42);
|
||||||
|
assert_eq!(rewritten_audio.header.stream_id, audio.header.stream_id);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn acl_open_mode_allows_all() {
|
fn acl_open_mode_allows_all() {
|
||||||
let mgr = RoomManager::new();
|
let mgr = RoomManager::new();
|
||||||
|
|||||||
@@ -9,10 +9,29 @@ use std::sync::Arc;
|
|||||||
|
|
||||||
use wzp_client::perform_handshake;
|
use wzp_client::perform_handshake;
|
||||||
use wzp_crypto::{KeyExchange, WarzoneKeyExchange};
|
use wzp_crypto::{KeyExchange, WarzoneKeyExchange};
|
||||||
use wzp_proto::{MediaTransport, SignalMessage, default_signal_version};
|
use wzp_proto::packet::MediaHeader;
|
||||||
|
use wzp_proto::{CodecId, MediaTransport, MediaType, SignalMessage, default_signal_version};
|
||||||
use wzp_relay::handshake::accept_handshake;
|
use wzp_relay::handshake::accept_handshake;
|
||||||
use wzp_transport::{QuinnTransport, client_config, create_endpoint, server_config};
|
use wzp_transport::{QuinnTransport, client_config, create_endpoint, server_config};
|
||||||
|
|
||||||
|
/// Build valid v2 MediaHeader bytes for use in encrypt/decrypt test calls.
|
||||||
|
fn test_header(seq: u32) -> Vec<u8> {
|
||||||
|
let h = MediaHeader {
|
||||||
|
version: 2,
|
||||||
|
flags: 0,
|
||||||
|
media_type: MediaType::Audio,
|
||||||
|
codec_id: CodecId::Opus24k,
|
||||||
|
stream_id: 0,
|
||||||
|
fec_ratio: 0,
|
||||||
|
seq,
|
||||||
|
timestamp: seq.wrapping_mul(20),
|
||||||
|
fec_block: 0,
|
||||||
|
};
|
||||||
|
let mut b = Vec::new();
|
||||||
|
h.write_to(&mut b);
|
||||||
|
b
|
||||||
|
}
|
||||||
|
|
||||||
/// Establish a QUIC connection and wrap both sides in `QuinnTransport`.
|
/// Establish a QUIC connection and wrap both sides in `QuinnTransport`.
|
||||||
///
|
///
|
||||||
/// Returns (client_transport, server_transport, _endpoints) where the endpoint
|
/// Returns (client_transport, server_transport, _endpoints) where the endpoint
|
||||||
@@ -68,7 +87,7 @@ async fn handshake_succeeds() {
|
|||||||
let callee_handle =
|
let callee_handle =
|
||||||
tokio::spawn(async move { accept_handshake(server_t.as_ref(), &callee_seed).await });
|
tokio::spawn(async move { accept_handshake(server_t.as_ref(), &callee_seed).await });
|
||||||
|
|
||||||
let caller_session = perform_handshake(client_transport.as_ref(), &caller_seed, None)
|
let caller_hs = perform_handshake(client_transport.as_ref(), &caller_seed, None)
|
||||||
.await
|
.await
|
||||||
.expect("perform_handshake should succeed");
|
.expect("perform_handshake should succeed");
|
||||||
|
|
||||||
@@ -79,20 +98,20 @@ async fn handshake_succeeds() {
|
|||||||
|
|
||||||
// Both sides should have derived a working CryptoSession.
|
// Both sides should have derived a working CryptoSession.
|
||||||
// Verify by encrypting on one side and decrypting on the other.
|
// Verify by encrypting on one side and decrypting on the other.
|
||||||
let header = b"test-header";
|
let header = test_header(0);
|
||||||
let plaintext = b"hello warzone";
|
let plaintext = b"hello warzone";
|
||||||
|
|
||||||
let mut ciphertext = Vec::new();
|
let mut ciphertext = Vec::new();
|
||||||
let mut caller_session = caller_session;
|
let mut caller_session = caller_hs.session;
|
||||||
let mut callee_session = callee_session;
|
let mut callee_session = callee_session;
|
||||||
|
|
||||||
caller_session
|
caller_session
|
||||||
.encrypt(header, plaintext, &mut ciphertext)
|
.encrypt(&header, plaintext, &mut ciphertext)
|
||||||
.expect("encrypt");
|
.expect("encrypt");
|
||||||
|
|
||||||
let mut decrypted = Vec::new();
|
let mut decrypted = Vec::new();
|
||||||
callee_session
|
callee_session
|
||||||
.decrypt(header, &ciphertext, &mut decrypted)
|
.decrypt(&header, &ciphertext, &mut decrypted)
|
||||||
.expect("decrypt");
|
.expect("decrypt");
|
||||||
|
|
||||||
assert_eq!(&decrypted, plaintext);
|
assert_eq!(&decrypted, plaintext);
|
||||||
@@ -137,6 +156,7 @@ async fn handshake_rejects_v1_protocol_version() {
|
|||||||
alias: None,
|
alias: None,
|
||||||
protocol_version: 1,
|
protocol_version: 1,
|
||||||
supported_versions: vec![1, 2],
|
supported_versions: vec![1, 2],
|
||||||
|
video_codecs: vec![],
|
||||||
};
|
};
|
||||||
|
|
||||||
client_transport
|
client_transport
|
||||||
@@ -202,7 +222,7 @@ async fn handshake_verifies_identity() {
|
|||||||
let callee_handle =
|
let callee_handle =
|
||||||
tokio::spawn(async move { accept_handshake(server_t.as_ref(), &callee_seed).await });
|
tokio::spawn(async move { accept_handshake(server_t.as_ref(), &callee_seed).await });
|
||||||
|
|
||||||
let caller_session = perform_handshake(client_transport.as_ref(), &caller_seed, None)
|
let caller_hs = perform_handshake(client_transport.as_ref(), &caller_seed, None)
|
||||||
.await
|
.await
|
||||||
.expect("handshake must succeed even with different identities");
|
.expect("handshake must succeed even with different identities");
|
||||||
|
|
||||||
@@ -212,20 +232,20 @@ async fn handshake_verifies_identity() {
|
|||||||
.expect("accept_handshake must succeed");
|
.expect("accept_handshake must succeed");
|
||||||
|
|
||||||
// Cross-encrypt/decrypt to prove the shared session works.
|
// Cross-encrypt/decrypt to prove the shared session works.
|
||||||
let header = b"id-test";
|
let header = test_header(0);
|
||||||
let plaintext = b"identity verified";
|
let plaintext = b"identity verified";
|
||||||
|
|
||||||
let mut ct = Vec::new();
|
let mut ct = Vec::new();
|
||||||
let mut caller_session = caller_session;
|
let mut caller_session = caller_hs.session;
|
||||||
let mut callee_session = callee_session;
|
let mut callee_session = callee_session;
|
||||||
|
|
||||||
caller_session
|
caller_session
|
||||||
.encrypt(header, plaintext, &mut ct)
|
.encrypt(&header, plaintext, &mut ct)
|
||||||
.expect("encrypt");
|
.expect("encrypt");
|
||||||
|
|
||||||
let mut pt = Vec::new();
|
let mut pt = Vec::new();
|
||||||
callee_session
|
callee_session
|
||||||
.decrypt(header, &ct, &mut pt)
|
.decrypt(&header, &ct, &mut pt)
|
||||||
.expect("decrypt");
|
.expect("decrypt");
|
||||||
|
|
||||||
assert_eq!(&pt, plaintext);
|
assert_eq!(&pt, plaintext);
|
||||||
@@ -282,7 +302,7 @@ async fn auth_then_handshake() {
|
|||||||
.await
|
.await
|
||||||
.expect("send AuthToken");
|
.expect("send AuthToken");
|
||||||
|
|
||||||
let caller_session = perform_handshake(client_transport.as_ref(), &caller_seed, None)
|
let caller_hs = perform_handshake(client_transport.as_ref(), &caller_seed, None)
|
||||||
.await
|
.await
|
||||||
.expect("perform_handshake after auth");
|
.expect("perform_handshake after auth");
|
||||||
|
|
||||||
@@ -292,20 +312,20 @@ async fn auth_then_handshake() {
|
|||||||
assert_eq!(received_token, "bearer-test-token-12345");
|
assert_eq!(received_token, "bearer-test-token-12345");
|
||||||
|
|
||||||
// Verify the crypto session works after the auth preamble.
|
// Verify the crypto session works after the auth preamble.
|
||||||
let header = b"auth-hdr";
|
let header = test_header(0);
|
||||||
let plaintext = b"post-auth payload";
|
let plaintext = b"post-auth payload";
|
||||||
|
|
||||||
let mut ct = Vec::new();
|
let mut ct = Vec::new();
|
||||||
let mut caller_session = caller_session;
|
let mut caller_session = caller_hs.session;
|
||||||
let mut callee_session = callee_session;
|
let mut callee_session = callee_session;
|
||||||
|
|
||||||
caller_session
|
caller_session
|
||||||
.encrypt(header, plaintext, &mut ct)
|
.encrypt(&header, plaintext, &mut ct)
|
||||||
.expect("encrypt");
|
.expect("encrypt");
|
||||||
|
|
||||||
let mut pt = Vec::new();
|
let mut pt = Vec::new();
|
||||||
callee_session
|
callee_session
|
||||||
.decrypt(header, &ct, &mut pt)
|
.decrypt(&header, &ct, &mut pt)
|
||||||
.expect("decrypt");
|
.expect("decrypt");
|
||||||
|
|
||||||
assert_eq!(&pt, plaintext);
|
assert_eq!(&pt, plaintext);
|
||||||
@@ -354,6 +374,7 @@ async fn handshake_rejects_bad_signature() {
|
|||||||
alias: None,
|
alias: None,
|
||||||
protocol_version: 2,
|
protocol_version: 2,
|
||||||
supported_versions: vec![2],
|
supported_versions: vec![2],
|
||||||
|
video_codecs: vec![],
|
||||||
};
|
};
|
||||||
|
|
||||||
client_transport
|
client_transport
|
||||||
|
|||||||
@@ -10,17 +10,16 @@ bytes = { workspace = true }
|
|||||||
tracing = { workspace = true }
|
tracing = { workspace = true }
|
||||||
wzp-proto = { path = "../wzp-proto" }
|
wzp-proto = { path = "../wzp-proto" }
|
||||||
|
|
||||||
# AV1 SW codecs do not support Android target (build.rs panics on
|
# AV1 SW codecs: shiguredo crates download prebuilt binaries at build time.
|
||||||
# aarch64-linux-android). Android uses MediaCodec for AV1 instead.
|
# Prebuilts are available for macOS only; Android uses MediaCodec; Linux will
|
||||||
[target.'cfg(not(target_os = "android"))'.dependencies]
|
# use system/vendored libs when that path is wired up (TODO).
|
||||||
|
[target.'cfg(target_os = "macos")'.dependencies]
|
||||||
shiguredo_dav1d = "2026.1.0"
|
shiguredo_dav1d = "2026.1.0"
|
||||||
shiguredo_svt_av1 = "2026.1.0"
|
shiguredo_svt_av1 = "2026.1.0"
|
||||||
|
|
||||||
[target.'cfg(target_os = "macos")'.dependencies]
|
|
||||||
shiguredo_video_toolbox = "2026.1"
|
shiguredo_video_toolbox = "2026.1"
|
||||||
|
|
||||||
[target.'cfg(target_os = "android")'.dependencies]
|
[target.'cfg(target_os = "android")'.dependencies]
|
||||||
ndk = { version = "0.9", features = ["media"] }
|
ndk = { version = "0.9", features = ["api-level-28", "media"] }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
rand = "0.8"
|
rand = "0.8"
|
||||||
|
|||||||
@@ -12,4 +12,9 @@ pub trait VideoDecoder: Send {
|
|||||||
/// Returns `Ok(Some(frame))` when a frame is ready, `Ok(None)` if more
|
/// Returns `Ok(Some(frame))` when a frame is ready, `Ok(None)` if more
|
||||||
/// data is needed (e.g., for reordering), or an error.
|
/// data is needed (e.g., for reordering), or an error.
|
||||||
fn decode(&mut self, access_unit: &[u8]) -> Result<Option<VideoFrame>, VideoError>;
|
fn decode(&mut self, access_unit: &[u8]) -> Result<Option<VideoFrame>, VideoError>;
|
||||||
|
|
||||||
|
/// Compact implementation-specific state useful for field diagnostics.
|
||||||
|
fn debug_snapshot(&self) -> Option<String> {
|
||||||
|
None
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -49,6 +49,11 @@ pub trait VideoEncoder: Send {
|
|||||||
///
|
///
|
||||||
/// Default implementation is a no-op.
|
/// Default implementation is a no-op.
|
||||||
fn set_mode(&mut self, _mode: crate::EncoderMode) {}
|
fn set_mode(&mut self, _mode: crate::EncoderMode) {}
|
||||||
|
|
||||||
|
/// Optional platform-specific encoder state for debug logs.
|
||||||
|
fn debug_snapshot(&self) -> Option<String> {
|
||||||
|
None
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Raw video frame input for encoding.
|
/// Raw video frame input for encoding.
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ use crate::encoder::{VideoEncoder, VideoError};
|
|||||||
/// **Encoder dispatch:**
|
/// **Encoder dispatch:**
|
||||||
/// - `H264Baseline` → `VideoToolboxEncoder` (macOS) / `MediaCodecEncoder` (Android)
|
/// - `H264Baseline` → `VideoToolboxEncoder` (macOS) / `MediaCodecEncoder` (Android)
|
||||||
/// - `H265Main` → `VideoToolboxHevcEncoder` (macOS) / `MediaCodecHevcEncoder` (Android)
|
/// - `H265Main` → `VideoToolboxHevcEncoder` (macOS) / `MediaCodecHevcEncoder` (Android)
|
||||||
/// - `Av1Main` → `SvtAv1Encoder` (all platforms — universal SW fallback)
|
/// - `Av1Main` → `SvtAv1Encoder` (macOS only — SW fallback)
|
||||||
///
|
///
|
||||||
/// Non-video codecs return [`VideoError::InvalidInput`].
|
/// Non-video codecs return [`VideoError::InvalidInput`].
|
||||||
pub fn create_video_encoder(
|
pub fn create_video_encoder(
|
||||||
@@ -78,10 +78,15 @@ pub fn create_video_encoder(
|
|||||||
#[allow(clippy::needless_return)]
|
#[allow(clippy::needless_return)]
|
||||||
return Err(VideoError::NotInitialized);
|
return Err(VideoError::NotInitialized);
|
||||||
}
|
}
|
||||||
#[cfg(not(target_os = "android"))]
|
#[cfg(target_os = "macos")]
|
||||||
{
|
{
|
||||||
Ok(Box::new(crate::svt_av1::SvtAv1Encoder::new(width, height)?))
|
Ok(Box::new(crate::svt_av1::SvtAv1Encoder::new(width, height)?))
|
||||||
}
|
}
|
||||||
|
#[cfg(not(any(target_os = "macos", target_os = "android")))]
|
||||||
|
{
|
||||||
|
let _ = (width, height);
|
||||||
|
Err(VideoError::NotInitialized)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
_ => Err(VideoError::InvalidInput("not a video codec".into())),
|
_ => Err(VideoError::InvalidInput("not a video codec".into())),
|
||||||
}
|
}
|
||||||
@@ -92,7 +97,7 @@ pub fn create_video_encoder(
|
|||||||
/// **Decoder dispatch:**
|
/// **Decoder dispatch:**
|
||||||
/// - `H264Baseline` → `VideoToolboxDecoder` (macOS) / `MediaCodecDecoder` (Android)
|
/// - `H264Baseline` → `VideoToolboxDecoder` (macOS) / `MediaCodecDecoder` (Android)
|
||||||
/// - `H265Main` → `VideoToolboxHevcDecoder` (macOS) / `MediaCodecHevcDecoder` (Android)
|
/// - `H265Main` → `VideoToolboxHevcDecoder` (macOS) / `MediaCodecHevcDecoder` (Android)
|
||||||
/// - `Av1Main` → `VideoToolboxAv1Decoder` (macOS M3+) → `Dav1dDecoder` (fallback, all platforms)
|
/// - `Av1Main` → `VideoToolboxAv1Decoder` (macOS M3+) → `Dav1dDecoder` (macOS SW fallback)
|
||||||
///
|
///
|
||||||
/// Non-video codecs return [`VideoError::InvalidInput`].
|
/// Non-video codecs return [`VideoError::InvalidInput`].
|
||||||
pub fn create_video_decoder(
|
pub fn create_video_decoder(
|
||||||
@@ -154,10 +159,15 @@ pub fn create_video_decoder(
|
|||||||
return crate::mediacodec::MediaCodecAv1Decoder::new(width, height)
|
return crate::mediacodec::MediaCodecAv1Decoder::new(width, height)
|
||||||
.map(|d| Box::new(d) as Box<dyn VideoDecoder>);
|
.map(|d| Box::new(d) as Box<dyn VideoDecoder>);
|
||||||
}
|
}
|
||||||
#[cfg(not(target_os = "android"))]
|
#[cfg(target_os = "macos")]
|
||||||
{
|
{
|
||||||
Ok(Box::new(crate::dav1d::Dav1dDecoder::new()?))
|
Ok(Box::new(crate::dav1d::Dav1dDecoder::new()?))
|
||||||
}
|
}
|
||||||
|
#[cfg(not(any(target_os = "macos", target_os = "android")))]
|
||||||
|
{
|
||||||
|
let _ = (width, height);
|
||||||
|
Err(VideoError::NotInitialized)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
_ => Err(VideoError::InvalidInput("not a video codec".into())),
|
_ => Err(VideoError::InvalidInput("not a video codec".into())),
|
||||||
}
|
}
|
||||||
@@ -170,30 +180,24 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn av1_encoder_factory_creates_svt_av1() {
|
fn av1_encoder_factory_creates_svt_av1() {
|
||||||
let enc = create_video_encoder(CodecId::Av1Main, 640, 480, 2_000_000);
|
let enc = create_video_encoder(CodecId::Av1Main, 640, 480, 2_000_000);
|
||||||
#[cfg(target_os = "android")]
|
#[cfg(target_os = "macos")]
|
||||||
|
assert!(enc.is_ok(), "AV1 encoder factory should succeed on macOS");
|
||||||
|
#[cfg(not(target_os = "macos"))]
|
||||||
assert!(
|
assert!(
|
||||||
matches!(enc, Err(VideoError::NotInitialized)),
|
matches!(enc, Err(VideoError::NotInitialized)),
|
||||||
"AV1 SW encoder is unavailable on Android (no shiguredo_svt_av1)"
|
"AV1 SW encoder is unavailable on Android/Linux (no shiguredo_svt_av1)"
|
||||||
);
|
|
||||||
#[cfg(not(target_os = "android"))]
|
|
||||||
assert!(
|
|
||||||
enc.is_ok(),
|
|
||||||
"AV1 encoder factory should succeed on non-Android platforms"
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn av1_decoder_factory_creates_decoder() {
|
fn av1_decoder_factory_creates_decoder() {
|
||||||
let dec = create_video_decoder(CodecId::Av1Main, 640, 480);
|
let dec = create_video_decoder(CodecId::Av1Main, 640, 480);
|
||||||
#[cfg(target_os = "android")]
|
#[cfg(target_os = "macos")]
|
||||||
|
assert!(dec.is_ok(), "AV1 decoder factory should succeed on macOS (dav1d fallback)");
|
||||||
|
#[cfg(not(target_os = "macos"))]
|
||||||
assert!(
|
assert!(
|
||||||
matches!(dec, Err(VideoError::NotInitialized)),
|
matches!(dec, Err(VideoError::NotInitialized)),
|
||||||
"AV1 decoder requires MediaCodec on Android; non-Android device returns NotInitialized"
|
"AV1 decoder unavailable on Android/Linux (no shiguredo_dav1d)"
|
||||||
);
|
|
||||||
#[cfg(not(target_os = "android"))]
|
|
||||||
assert!(
|
|
||||||
dec.is_ok(),
|
|
||||||
"AV1 decoder factory should succeed on non-Android (dav1d SW fallback)"
|
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
pub mod av1_obu;
|
pub mod av1_obu;
|
||||||
pub mod controller;
|
pub mod controller;
|
||||||
#[cfg(not(target_os = "android"))]
|
#[cfg(target_os = "macos")]
|
||||||
pub mod dav1d;
|
pub mod dav1d;
|
||||||
pub mod decoder;
|
pub mod decoder;
|
||||||
pub mod depacketizer;
|
pub mod depacketizer;
|
||||||
@@ -16,14 +16,15 @@ pub mod factory;
|
|||||||
pub mod framer;
|
pub mod framer;
|
||||||
pub mod mediacodec;
|
pub mod mediacodec;
|
||||||
pub mod nack;
|
pub mod nack;
|
||||||
|
pub mod transport;
|
||||||
pub mod simulcast;
|
pub mod simulcast;
|
||||||
#[cfg(not(target_os = "android"))]
|
#[cfg(target_os = "macos")]
|
||||||
pub mod svt_av1;
|
pub mod svt_av1;
|
||||||
pub mod videotoolbox;
|
pub mod videotoolbox;
|
||||||
|
|
||||||
pub use av1_obu::{Av1Depacketizer, Av1ObuFramer, is_keyframe_obu};
|
pub use av1_obu::{Av1Depacketizer, Av1ObuFramer, is_keyframe_obu};
|
||||||
pub use controller::{VideoQualityController, VideoTarget};
|
pub use controller::{VideoQualityController, VideoTarget};
|
||||||
#[cfg(not(target_os = "android"))]
|
#[cfg(target_os = "macos")]
|
||||||
pub use dav1d::Dav1dDecoder;
|
pub use dav1d::Dav1dDecoder;
|
||||||
pub use decoder::VideoDecoder;
|
pub use decoder::VideoDecoder;
|
||||||
pub use depacketizer::H264Depacketizer;
|
pub use depacketizer::H264Depacketizer;
|
||||||
@@ -37,7 +38,7 @@ pub use mediacodec::{
|
|||||||
};
|
};
|
||||||
pub use nack::{CachedPacket, NackAction, NackReceiver, NackSender};
|
pub use nack::{CachedPacket, NackAction, NackReceiver, NackSender};
|
||||||
pub use simulcast::{LayerPacket, LayerTarget, SimulcastEncoder, SimulcastLayer};
|
pub use simulcast::{LayerPacket, LayerTarget, SimulcastEncoder, SimulcastLayer};
|
||||||
#[cfg(not(target_os = "android"))]
|
#[cfg(target_os = "macos")]
|
||||||
pub use svt_av1::SvtAv1Encoder;
|
pub use svt_av1::SvtAv1Encoder;
|
||||||
pub use videotoolbox::{
|
pub use videotoolbox::{
|
||||||
VideoToolboxAv1Decoder, VideoToolboxDecoder, VideoToolboxEncoder, VideoToolboxHevcDecoder,
|
VideoToolboxAv1Decoder, VideoToolboxDecoder, VideoToolboxEncoder, VideoToolboxHevcDecoder,
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
341
crates/wzp-video/src/transport.rs
Normal file
341
crates/wzp-video/src/transport.rs
Normal file
@@ -0,0 +1,341 @@
|
|||||||
|
//! Video packet serialization and reassembly on top of [`MediaHeaderV2`].
|
||||||
|
//!
|
||||||
|
//! A single encoded video frame may be far larger than one QUIC datagram
|
||||||
|
//! (~1200 bytes after header and AEAD overhead). This module fragments
|
||||||
|
//! frames into `MediaPacket`s on the send side and reassembles them on the
|
||||||
|
//! receive side.
|
||||||
|
//!
|
||||||
|
//! ## Wire layout
|
||||||
|
//!
|
||||||
|
//! Each fragment uses a standard `MediaHeaderV2` with:
|
||||||
|
//! - `media_type = Video`
|
||||||
|
//! - `codec_id` = the negotiated video codec
|
||||||
|
//! - `FLAG_KEYFRAME` set on all fragments of a keyframe
|
||||||
|
//! - `FLAG_FRAME_END` set on the last fragment of a frame
|
||||||
|
//! - `seq` = monotonic packet sequence number (wrapping u32)
|
||||||
|
//! - `fec_block` = `(fragment_index as u8) << 8 | (fragment_count as u8)`
|
||||||
|
//! where fragment_count = total fragments in this frame (1-based)
|
||||||
|
//!
|
||||||
|
//! Max fragments per frame: 255 → max frame size ≈ 255 × 1150 ≈ 293 KB,
|
||||||
|
//! which covers 1080p keyframes at reasonable quality.
|
||||||
|
|
||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
|
use bytes::{Bytes, BytesMut};
|
||||||
|
use wzp_proto::{CodecId, MediaHeaderV2, MediaPacket, MediaType};
|
||||||
|
|
||||||
|
/// Maximum video payload bytes per QUIC datagram.
|
||||||
|
/// 1200 (QUIC MTU) − 16 (MediaHeaderV2) − 16 (AEAD tag) = 1168.
|
||||||
|
pub const VIDEO_MAX_PAYLOAD: usize = 1168;
|
||||||
|
|
||||||
|
const VIDEO_FRAME_META_MAGIC: [u8; 4] = *b"WZV1";
|
||||||
|
const VIDEO_FRAME_META_LEN: usize = 8;
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub struct VideoFrameMeta {
|
||||||
|
pub width: u16,
|
||||||
|
pub height: u16,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub struct ReassembledVideoFrame {
|
||||||
|
pub codec_id: CodecId,
|
||||||
|
pub is_keyframe: bool,
|
||||||
|
pub width: Option<u16>,
|
||||||
|
pub height: Option<u16>,
|
||||||
|
pub data: Vec<u8>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fragments one encoded video frame into a sequence of [`MediaPacket`]s.
|
||||||
|
///
|
||||||
|
/// Pass each `MediaPacket` to `transport.send_media()`.
|
||||||
|
pub fn packetize_video_frame(
|
||||||
|
frame: &[u8],
|
||||||
|
codec_id: CodecId,
|
||||||
|
is_keyframe: bool,
|
||||||
|
seq: &mut u32,
|
||||||
|
timestamp_ms: u32,
|
||||||
|
width: u32,
|
||||||
|
height: u32,
|
||||||
|
) -> Vec<MediaPacket> {
|
||||||
|
if frame.is_empty() {
|
||||||
|
return vec![];
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut framed = Vec::with_capacity(VIDEO_FRAME_META_LEN + frame.len());
|
||||||
|
framed.extend_from_slice(&VIDEO_FRAME_META_MAGIC);
|
||||||
|
framed.extend_from_slice(&(width.min(u16::MAX as u32) as u16).to_be_bytes());
|
||||||
|
framed.extend_from_slice(&(height.min(u16::MAX as u32) as u16).to_be_bytes());
|
||||||
|
framed.extend_from_slice(frame);
|
||||||
|
|
||||||
|
let chunks: Vec<&[u8]> = framed.chunks(VIDEO_MAX_PAYLOAD).collect();
|
||||||
|
let total = chunks.len().min(255);
|
||||||
|
let mut packets = Vec::with_capacity(total);
|
||||||
|
|
||||||
|
for (i, chunk) in chunks.iter().enumerate().take(255) {
|
||||||
|
let is_last = i + 1 == total;
|
||||||
|
let mut flags = 0u8;
|
||||||
|
if is_keyframe {
|
||||||
|
flags |= MediaHeaderV2::FLAG_KEYFRAME;
|
||||||
|
}
|
||||||
|
if is_last {
|
||||||
|
flags |= MediaHeaderV2::FLAG_FRAME_END;
|
||||||
|
}
|
||||||
|
|
||||||
|
let fec_block = ((i as u16) << 8) | (total as u16);
|
||||||
|
|
||||||
|
let header = MediaHeaderV2 {
|
||||||
|
version: MediaHeaderV2::VERSION,
|
||||||
|
flags,
|
||||||
|
media_type: MediaType::Video,
|
||||||
|
codec_id,
|
||||||
|
// Legacy relays default receivers to video layer 0. Use video stream
|
||||||
|
// 0 for the single-layer room-video path so packets are forwarded
|
||||||
|
// before any receiver quality state exists. Audio is separated by
|
||||||
|
// media_type, so stream_id 0 does not collide with audio packets.
|
||||||
|
stream_id: 0,
|
||||||
|
fec_ratio: 0,
|
||||||
|
seq: *seq,
|
||||||
|
timestamp: timestamp_ms,
|
||||||
|
fec_block,
|
||||||
|
};
|
||||||
|
*seq = seq.wrapping_add(1);
|
||||||
|
|
||||||
|
let mut buf = BytesMut::with_capacity(MediaHeaderV2::WIRE_SIZE + chunk.len());
|
||||||
|
header.write_to(&mut buf);
|
||||||
|
buf.extend_from_slice(chunk);
|
||||||
|
|
||||||
|
packets.push(MediaPacket {
|
||||||
|
header,
|
||||||
|
payload: Bytes::copy_from_slice(chunk),
|
||||||
|
quality_report: None,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
packets
|
||||||
|
}
|
||||||
|
|
||||||
|
/// State for one partially-reassembled video frame.
|
||||||
|
#[derive(Default)]
|
||||||
|
struct PendingFrame {
|
||||||
|
fragments: HashMap<u8, Vec<u8>>,
|
||||||
|
total_fragments: u8,
|
||||||
|
is_keyframe: bool,
|
||||||
|
saw_frame_end: bool,
|
||||||
|
codec_id: Option<CodecId>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reassembles fragmented [`MediaPacket`]s back into complete video frames.
|
||||||
|
///
|
||||||
|
/// Call [`VideoReassembler::push`] for every received video `MediaPacket`.
|
||||||
|
/// It returns a complete frame only when the last fragment (`FLAG_FRAME_END`)
|
||||||
|
/// of a frame arrives and all prior fragments are present.
|
||||||
|
pub struct VideoReassembler {
|
||||||
|
/// Keyed by the timestamp of the frame being assembled.
|
||||||
|
pending: HashMap<u32, PendingFrame>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl VideoReassembler {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
pending: HashMap::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Push one received video packet.
|
||||||
|
///
|
||||||
|
/// Returns `Some(frame)` when a complete frame is ready, `None` otherwise.
|
||||||
|
pub fn push(&mut self, pkt: &MediaPacket) -> Option<ReassembledVideoFrame> {
|
||||||
|
let hdr = &pkt.header;
|
||||||
|
let fragment_index = (hdr.fec_block >> 8) as u8;
|
||||||
|
let fragment_count = (hdr.fec_block & 0xFF) as u8;
|
||||||
|
let is_keyframe = hdr.is_keyframe();
|
||||||
|
let is_frame_end = hdr.is_frame_end();
|
||||||
|
|
||||||
|
// Use the packet timestamp as the frame identifier.
|
||||||
|
let entry = self.pending.entry(hdr.timestamp).or_default();
|
||||||
|
entry.fragments.insert(fragment_index, pkt.payload.to_vec());
|
||||||
|
if fragment_count > 0 {
|
||||||
|
entry.total_fragments = fragment_count;
|
||||||
|
}
|
||||||
|
if is_keyframe {
|
||||||
|
entry.is_keyframe = true;
|
||||||
|
}
|
||||||
|
if is_frame_end {
|
||||||
|
entry.saw_frame_end = true;
|
||||||
|
}
|
||||||
|
entry.codec_id = Some(hdr.codec_id);
|
||||||
|
|
||||||
|
// Attempt reassembly once we know the frame end has arrived. The end
|
||||||
|
// fragment can arrive before earlier fragments on QUIC/datagram paths,
|
||||||
|
// so retry on every later fragment instead of only on the end packet.
|
||||||
|
if !entry.saw_frame_end {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let total = entry.total_fragments as usize;
|
||||||
|
if total == 0 || entry.fragments.len() < total {
|
||||||
|
// Haven't received all fragments yet; keep waiting.
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
// All fragments present — reassemble in order.
|
||||||
|
let pending = self.pending.remove(&hdr.timestamp)?;
|
||||||
|
let codec_id = pending.codec_id?;
|
||||||
|
let mut frame = Vec::new();
|
||||||
|
for i in 0..total as u8 {
|
||||||
|
frame.extend_from_slice(pending.fragments.get(&i)?);
|
||||||
|
}
|
||||||
|
let (meta, data) = split_video_frame_payload(frame);
|
||||||
|
Some(ReassembledVideoFrame {
|
||||||
|
codec_id,
|
||||||
|
is_keyframe: pending.is_keyframe,
|
||||||
|
width: meta.map(|m| m.width),
|
||||||
|
height: meta.map(|m| m.height),
|
||||||
|
data,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Evict stale pending frames older than `max_age_ms` milliseconds.
|
||||||
|
///
|
||||||
|
/// Call periodically (e.g. every 2s) to prevent accumulation of frames
|
||||||
|
/// whose first or middle fragments were lost.
|
||||||
|
pub fn evict_stale(&mut self, current_timestamp_ms: u32, max_age_ms: u32) {
|
||||||
|
self.pending
|
||||||
|
.retain(|&ts, _| current_timestamp_ms.wrapping_sub(ts) <= max_age_ms);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn split_video_frame_payload(mut frame: Vec<u8>) -> (Option<VideoFrameMeta>, Vec<u8>) {
|
||||||
|
if frame.len() < VIDEO_FRAME_META_LEN || frame[..4] != VIDEO_FRAME_META_MAGIC {
|
||||||
|
return (None, frame);
|
||||||
|
}
|
||||||
|
|
||||||
|
let width = u16::from_be_bytes([frame[4], frame[5]]);
|
||||||
|
let height = u16::from_be_bytes([frame[6], frame[7]]);
|
||||||
|
frame.drain(..VIDEO_FRAME_META_LEN);
|
||||||
|
(Some(VideoFrameMeta { width, height }), frame)
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for VideoReassembler {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self::new()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn make_frame(size: usize) -> Vec<u8> {
|
||||||
|
(0..size).map(|i| (i & 0xFF) as u8).collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn single_fragment_roundtrip() {
|
||||||
|
let frame = make_frame(100);
|
||||||
|
let mut seq = 0u32;
|
||||||
|
let pkts = packetize_video_frame(&frame, CodecId::Av1Main, true, &mut seq, 1000, 640, 480);
|
||||||
|
assert_eq!(pkts.len(), 1);
|
||||||
|
assert!(pkts[0].header.is_keyframe());
|
||||||
|
assert!(pkts[0].header.is_frame_end());
|
||||||
|
assert_eq!(pkts[0].header.media_type, MediaType::Video);
|
||||||
|
assert_eq!(pkts[0].header.stream_id, 0);
|
||||||
|
|
||||||
|
let mut reassembler = VideoReassembler::new();
|
||||||
|
let result = reassembler.push(&pkts[0]);
|
||||||
|
assert!(result.is_some());
|
||||||
|
let result = result.unwrap();
|
||||||
|
assert_eq!(result.codec_id, CodecId::Av1Main);
|
||||||
|
assert!(result.is_keyframe);
|
||||||
|
assert_eq!(result.width, Some(640));
|
||||||
|
assert_eq!(result.height, Some(480));
|
||||||
|
assert_eq!(result.data, frame);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn multi_fragment_roundtrip() {
|
||||||
|
let frame = make_frame(VIDEO_MAX_PAYLOAD * 3 + 50);
|
||||||
|
let mut seq = 0u32;
|
||||||
|
let pkts = packetize_video_frame(
|
||||||
|
&frame,
|
||||||
|
CodecId::H264Baseline,
|
||||||
|
false,
|
||||||
|
&mut seq,
|
||||||
|
2000,
|
||||||
|
960,
|
||||||
|
540,
|
||||||
|
);
|
||||||
|
assert_eq!(pkts.len(), 4);
|
||||||
|
assert!(!pkts[0].header.is_frame_end());
|
||||||
|
assert!(pkts[3].header.is_frame_end());
|
||||||
|
assert!(!pkts[0].header.is_keyframe());
|
||||||
|
|
||||||
|
let mut reassembler = VideoReassembler::new();
|
||||||
|
let mut result = None;
|
||||||
|
for pkt in &pkts {
|
||||||
|
result = reassembler.push(pkt);
|
||||||
|
}
|
||||||
|
let result = result.unwrap();
|
||||||
|
assert_eq!(result.codec_id, CodecId::H264Baseline);
|
||||||
|
assert!(!result.is_keyframe);
|
||||||
|
assert_eq!(result.width, Some(960));
|
||||||
|
assert_eq!(result.height, Some(540));
|
||||||
|
assert_eq!(result.data, frame);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn out_of_order_delivery() {
|
||||||
|
let frame = make_frame(VIDEO_MAX_PAYLOAD * 2 + 100);
|
||||||
|
let mut seq = 0u32;
|
||||||
|
let pkts = packetize_video_frame(&frame, CodecId::Av1Main, false, &mut seq, 3000, 320, 240);
|
||||||
|
assert_eq!(pkts.len(), 3);
|
||||||
|
|
||||||
|
let mut reassembler = VideoReassembler::new();
|
||||||
|
// Deliver out of order: 2, 0, 1
|
||||||
|
assert!(reassembler.push(&pkts[2]).is_none()); // last arrives first — no total_fragments yet
|
||||||
|
assert!(reassembler.push(&pkts[0]).is_none());
|
||||||
|
let result = reassembler
|
||||||
|
.push(&pkts[1])
|
||||||
|
.expect("last missing fragment completes frame");
|
||||||
|
assert_eq!(result.codec_id, CodecId::Av1Main);
|
||||||
|
assert!(!result.is_keyframe);
|
||||||
|
assert_eq!(result.width, Some(320));
|
||||||
|
assert_eq!(result.height, Some(240));
|
||||||
|
assert_eq!(result.data, frame);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn empty_frame_produces_no_packets() {
|
||||||
|
let mut seq = 0u32;
|
||||||
|
let pkts = packetize_video_frame(&[], CodecId::Av1Main, false, &mut seq, 0, 640, 480);
|
||||||
|
assert!(pkts.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn old_payload_without_meta_still_reassembles() {
|
||||||
|
let payload = Bytes::copy_from_slice(&[0x00, 0x00, 0x00, 0x01, 0x65]);
|
||||||
|
let pkt = MediaPacket {
|
||||||
|
header: MediaHeaderV2 {
|
||||||
|
version: MediaHeaderV2::VERSION,
|
||||||
|
flags: MediaHeaderV2::FLAG_KEYFRAME | MediaHeaderV2::FLAG_FRAME_END,
|
||||||
|
media_type: MediaType::Video,
|
||||||
|
codec_id: CodecId::H264Baseline,
|
||||||
|
stream_id: 0,
|
||||||
|
fec_ratio: 0,
|
||||||
|
seq: 7,
|
||||||
|
timestamp: 123,
|
||||||
|
fec_block: 1,
|
||||||
|
},
|
||||||
|
payload: payload.clone(),
|
||||||
|
quality_report: None,
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut reassembler = VideoReassembler::new();
|
||||||
|
let frame = reassembler.push(&pkt).unwrap();
|
||||||
|
assert_eq!(frame.codec_id, CodecId::H264Baseline);
|
||||||
|
assert_eq!(frame.width, None);
|
||||||
|
assert_eq!(frame.height, None);
|
||||||
|
assert_eq!(frame.data, payload.to_vec());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -8,13 +8,110 @@ mod imp {
|
|||||||
pub use shiguredo_video_toolbox::{
|
pub use shiguredo_video_toolbox::{
|
||||||
CodecConfig, DecodedFrame, Decoder, DecoderCodec, DecoderConfig, EncodeOptions, Encoder,
|
CodecConfig, DecodedFrame, Decoder, DecoderCodec, DecoderConfig, EncodeOptions, Encoder,
|
||||||
EncoderConfig, FrameData, H264EncoderConfig, H264EntropyMode, H264Profile,
|
EncoderConfig, FrameData, H264EncoderConfig, H264EntropyMode, H264Profile,
|
||||||
HevcEncoderConfig, HevcProfile, PixelFormat,
|
HevcEncoderConfig, HevcProfile, I420Frame, PixelFormat,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(target_os = "macos")]
|
#[cfg(target_os = "macos")]
|
||||||
use imp::*;
|
use imp::*;
|
||||||
|
|
||||||
|
/// Copy a VideoToolbox I420 CVPixelBuffer into a tightly-packed I420 byte vector
|
||||||
|
/// of `width * height + 2 * (width/2) * (height/2)` bytes.
|
||||||
|
///
|
||||||
|
/// The per-plane `bytes_per_row` (stride) reported by CoreVideo can be larger
|
||||||
|
/// than the visible plane width (typically aligned to 16/64 bytes). Concatenating
|
||||||
|
/// the raw plane slices without removing that stride padding produces a buffer
|
||||||
|
/// that downstream code — which indexes as tight I420 of `width x height` —
|
||||||
|
/// mis-interprets, producing horizontal green/magenta bands that drift one
|
||||||
|
/// chroma row each time the per-row stride excess accumulates to one full row.
|
||||||
|
///
|
||||||
|
/// `frame_label` is used for one-time tracing of the actual plane dimensions so
|
||||||
|
/// the first decoded frame of a session prints its real layout. The boolean
|
||||||
|
/// flag is flipped to true after the first log so the format string is emitted
|
||||||
|
/// at most once per decoder lifetime.
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
fn i420_frame_to_tight(
|
||||||
|
frame: &I420Frame<'_>,
|
||||||
|
width: u32,
|
||||||
|
height: u32,
|
||||||
|
frame_label: &'static str,
|
||||||
|
logged: &mut bool,
|
||||||
|
) -> Result<Vec<u8>, VideoError> {
|
||||||
|
let w = width as usize;
|
||||||
|
let h = height as usize;
|
||||||
|
if w == 0 || h == 0 {
|
||||||
|
return Err(VideoError::PlatformError(format!(
|
||||||
|
"decoder produced empty frame ({w}x{h})"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
let cw = w / 2;
|
||||||
|
let ch = h / 2;
|
||||||
|
|
||||||
|
let y = frame.y_plane();
|
||||||
|
let u = frame.u_plane();
|
||||||
|
let v = frame.v_plane();
|
||||||
|
let y_stride = frame.y_stride();
|
||||||
|
let u_stride = frame.u_stride();
|
||||||
|
let v_stride = frame.v_stride();
|
||||||
|
let fw = frame.width();
|
||||||
|
let fh = frame.height();
|
||||||
|
|
||||||
|
if !*logged {
|
||||||
|
*logged = true;
|
||||||
|
tracing::info!(
|
||||||
|
target: "wzp_video::videotoolbox",
|
||||||
|
label = frame_label,
|
||||||
|
configured_width = w,
|
||||||
|
configured_height = h,
|
||||||
|
frame_width = fw,
|
||||||
|
frame_height = fh,
|
||||||
|
y_stride,
|
||||||
|
u_stride,
|
||||||
|
v_stride,
|
||||||
|
y_len = y.len(),
|
||||||
|
u_len = u.len(),
|
||||||
|
v_len = v.len(),
|
||||||
|
"VideoToolbox decoder I420 plane layout"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if y_stride < w || u_stride < cw || v_stride < cw {
|
||||||
|
return Err(VideoError::PlatformError(format!(
|
||||||
|
"decoder plane stride smaller than width: y_stride={y_stride} u_stride={u_stride} v_stride={v_stride} for {w}x{h}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
let needed_y = y_stride.checked_mul(h).ok_or_else(|| {
|
||||||
|
VideoError::PlatformError(format!("y plane size overflow {y_stride}x{h}"))
|
||||||
|
})?;
|
||||||
|
let needed_uv = u_stride.checked_mul(ch).ok_or_else(|| {
|
||||||
|
VideoError::PlatformError(format!("uv plane size overflow {u_stride}x{ch}"))
|
||||||
|
})?;
|
||||||
|
if y.len() < needed_y || u.len() < needed_uv || v.len() < v_stride * ch {
|
||||||
|
return Err(VideoError::PlatformError(format!(
|
||||||
|
"decoder plane buffer too small: y_len={} (need {needed_y}) u_len={} (need {needed_uv}) v_len={} (need {})",
|
||||||
|
y.len(),
|
||||||
|
u.len(),
|
||||||
|
v.len(),
|
||||||
|
v_stride * ch,
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut data = Vec::with_capacity(w * h + 2 * cw * ch);
|
||||||
|
for row in 0..h {
|
||||||
|
let off = row * y_stride;
|
||||||
|
data.extend_from_slice(&y[off..off + w]);
|
||||||
|
}
|
||||||
|
for row in 0..ch {
|
||||||
|
let off = row * u_stride;
|
||||||
|
data.extend_from_slice(&u[off..off + cw]);
|
||||||
|
}
|
||||||
|
for row in 0..ch {
|
||||||
|
let off = row * v_stride;
|
||||||
|
data.extend_from_slice(&v[off..off + cw]);
|
||||||
|
}
|
||||||
|
Ok(data)
|
||||||
|
}
|
||||||
|
|
||||||
/// macOS VideoToolbox H.264 encoder.
|
/// macOS VideoToolbox H.264 encoder.
|
||||||
///
|
///
|
||||||
/// Wraps `VTCompressionSession`. On non-macOS targets this is a compile-safe
|
/// Wraps `VTCompressionSession`. On non-macOS targets this is a compile-safe
|
||||||
@@ -160,9 +257,12 @@ impl VideoEncoder for VideoToolboxEncoder {
|
|||||||
if packet.is_empty() {
|
if packet.is_empty() {
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
let nal_type = packet[0] & 0x1F;
|
let nals = split_annex_b(packet);
|
||||||
// NAL type 5 = IDR slice (keyframe).
|
if nals.is_empty() {
|
||||||
nal_type == 5
|
return (packet[0] & 0x1F) == 5;
|
||||||
|
}
|
||||||
|
nals.iter()
|
||||||
|
.any(|nal| !nal.is_empty() && (nal[0] & 0x1F) == 5)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -261,6 +361,8 @@ pub struct VideoToolboxDecoder {
|
|||||||
width: u32,
|
width: u32,
|
||||||
#[cfg(target_os = "macos")]
|
#[cfg(target_os = "macos")]
|
||||||
height: u32,
|
height: u32,
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
layout_logged: bool,
|
||||||
#[cfg(not(target_os = "macos"))]
|
#[cfg(not(target_os = "macos"))]
|
||||||
_width: u32,
|
_width: u32,
|
||||||
#[cfg(not(target_os = "macos"))]
|
#[cfg(not(target_os = "macos"))]
|
||||||
@@ -279,6 +381,7 @@ impl VideoToolboxDecoder {
|
|||||||
inner: None,
|
inner: None,
|
||||||
width,
|
width,
|
||||||
height,
|
height,
|
||||||
|
layout_logged: false,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
#[cfg(not(target_os = "macos"))]
|
#[cfg(not(target_os = "macos"))]
|
||||||
@@ -357,13 +460,13 @@ impl VideoDecoder for VideoToolboxDecoder {
|
|||||||
|
|
||||||
match decoded {
|
match decoded {
|
||||||
Some(DecodedFrame::I420(frame)) => {
|
Some(DecodedFrame::I420(frame)) => {
|
||||||
let y = frame.y_plane();
|
let data = i420_frame_to_tight(
|
||||||
let u = frame.u_plane();
|
&frame,
|
||||||
let v = frame.v_plane();
|
self.width,
|
||||||
let mut data = Vec::with_capacity(y.len() + u.len() + v.len());
|
self.height,
|
||||||
data.extend_from_slice(y);
|
"h264_decoder",
|
||||||
data.extend_from_slice(u);
|
&mut self.layout_logged,
|
||||||
data.extend_from_slice(v);
|
)?;
|
||||||
Ok(Some(VideoFrame {
|
Ok(Some(VideoFrame {
|
||||||
width: self.width,
|
width: self.width,
|
||||||
height: self.height,
|
height: self.height,
|
||||||
@@ -520,12 +623,13 @@ impl VideoEncoder for VideoToolboxHevcEncoder {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn is_keyframe(&self, packet: &[u8]) -> bool {
|
fn is_keyframe(&self, packet: &[u8]) -> bool {
|
||||||
if packet.len() < 2 {
|
let nals = split_annex_b(packet);
|
||||||
return false;
|
if nals.is_empty() {
|
||||||
|
return packet.len() >= 2 && matches!((packet[0] >> 1) & 0x3F, 19 | 20);
|
||||||
}
|
}
|
||||||
let nal_type = (packet[0] >> 1) & 0x3F;
|
|
||||||
// NAL type 19 = IDR_W_RADL, 20 = IDR_N_LP.
|
// NAL type 19 = IDR_W_RADL, 20 = IDR_N_LP.
|
||||||
nal_type == 19 || nal_type == 20
|
nals.iter()
|
||||||
|
.any(|nal| nal.len() >= 2 && matches!((nal[0] >> 1) & 0x3F, 19 | 20))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -537,6 +641,8 @@ pub struct VideoToolboxHevcDecoder {
|
|||||||
width: u32,
|
width: u32,
|
||||||
#[cfg(target_os = "macos")]
|
#[cfg(target_os = "macos")]
|
||||||
height: u32,
|
height: u32,
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
layout_logged: bool,
|
||||||
#[cfg(not(target_os = "macos"))]
|
#[cfg(not(target_os = "macos"))]
|
||||||
_width: u32,
|
_width: u32,
|
||||||
#[cfg(not(target_os = "macos"))]
|
#[cfg(not(target_os = "macos"))]
|
||||||
@@ -551,6 +657,7 @@ impl VideoToolboxHevcDecoder {
|
|||||||
inner: None,
|
inner: None,
|
||||||
width,
|
width,
|
||||||
height,
|
height,
|
||||||
|
layout_logged: false,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
#[cfg(not(target_os = "macos"))]
|
#[cfg(not(target_os = "macos"))]
|
||||||
@@ -624,13 +731,13 @@ impl VideoDecoder for VideoToolboxHevcDecoder {
|
|||||||
|
|
||||||
match decoded {
|
match decoded {
|
||||||
Some(DecodedFrame::I420(frame)) => {
|
Some(DecodedFrame::I420(frame)) => {
|
||||||
let y = frame.y_plane();
|
let data = i420_frame_to_tight(
|
||||||
let u = frame.u_plane();
|
&frame,
|
||||||
let v = frame.v_plane();
|
self.width,
|
||||||
let mut data = Vec::with_capacity(y.len() + u.len() + v.len());
|
self.height,
|
||||||
data.extend_from_slice(y);
|
"hevc_decoder",
|
||||||
data.extend_from_slice(u);
|
&mut self.layout_logged,
|
||||||
data.extend_from_slice(v);
|
)?;
|
||||||
Ok(Some(VideoFrame {
|
Ok(Some(VideoFrame {
|
||||||
width: self.width,
|
width: self.width,
|
||||||
height: self.height,
|
height: self.height,
|
||||||
@@ -660,6 +767,8 @@ pub struct VideoToolboxAv1Decoder {
|
|||||||
width: u32,
|
width: u32,
|
||||||
#[cfg(target_os = "macos")]
|
#[cfg(target_os = "macos")]
|
||||||
height: u32,
|
height: u32,
|
||||||
|
#[cfg(target_os = "macos")]
|
||||||
|
layout_logged: bool,
|
||||||
#[cfg(not(target_os = "macos"))]
|
#[cfg(not(target_os = "macos"))]
|
||||||
_width: u32,
|
_width: u32,
|
||||||
#[cfg(not(target_os = "macos"))]
|
#[cfg(not(target_os = "macos"))]
|
||||||
@@ -679,6 +788,7 @@ impl VideoToolboxAv1Decoder {
|
|||||||
inner: Some(decoder),
|
inner: Some(decoder),
|
||||||
width,
|
width,
|
||||||
height,
|
height,
|
||||||
|
layout_logged: false,
|
||||||
}),
|
}),
|
||||||
Err(shiguredo_video_toolbox::Error::UnsupportedCodec { .. }) => {
|
Err(shiguredo_video_toolbox::Error::UnsupportedCodec { .. }) => {
|
||||||
// AV1 decode not supported on this platform (e.g. M1/M2).
|
// AV1 decode not supported on this platform (e.g. M1/M2).
|
||||||
@@ -686,6 +796,7 @@ impl VideoToolboxAv1Decoder {
|
|||||||
inner: None,
|
inner: None,
|
||||||
width,
|
width,
|
||||||
height,
|
height,
|
||||||
|
layout_logged: false,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
Err(e) => Err(VideoError::PlatformError(format!(
|
Err(e) => Err(VideoError::PlatformError(format!(
|
||||||
@@ -717,13 +828,13 @@ impl VideoDecoder for VideoToolboxAv1Decoder {
|
|||||||
.map_err(|e| VideoError::PlatformError(format!("decode failed: {e}")))?;
|
.map_err(|e| VideoError::PlatformError(format!("decode failed: {e}")))?;
|
||||||
match decoded {
|
match decoded {
|
||||||
Some(DecodedFrame::I420(frame)) => {
|
Some(DecodedFrame::I420(frame)) => {
|
||||||
let y = frame.y_plane();
|
let data = i420_frame_to_tight(
|
||||||
let u = frame.u_plane();
|
&frame,
|
||||||
let v = frame.v_plane();
|
self.width,
|
||||||
let mut data = Vec::with_capacity(y.len() + u.len() + v.len());
|
self.height,
|
||||||
data.extend_from_slice(y);
|
"av1_decoder",
|
||||||
data.extend_from_slice(u);
|
&mut self.layout_logged,
|
||||||
data.extend_from_slice(v);
|
)?;
|
||||||
Ok(Some(VideoFrame {
|
Ok(Some(VideoFrame {
|
||||||
width: self.width,
|
width: self.width,
|
||||||
height: self.height,
|
height: self.height,
|
||||||
@@ -791,6 +902,11 @@ mod tests {
|
|||||||
let enc = VideoToolboxEncoder::new(1280, 720, 2_000_000).unwrap();
|
let enc = VideoToolboxEncoder::new(1280, 720, 2_000_000).unwrap();
|
||||||
assert!(enc.is_keyframe(&[0x65, 0x01, 0x02]));
|
assert!(enc.is_keyframe(&[0x65, 0x01, 0x02]));
|
||||||
assert!(!enc.is_keyframe(&[0x41, 0x01, 0x02]));
|
assert!(!enc.is_keyframe(&[0x41, 0x01, 0x02]));
|
||||||
|
assert!(enc.is_keyframe(&[
|
||||||
|
0x00, 0x00, 0x00, 0x01, 0x67, 0x01, // SPS
|
||||||
|
0x00, 0x00, 0x00, 0x01, 0x68, 0x02, // PPS
|
||||||
|
0x00, 0x00, 0x00, 0x01, 0x65, 0x03, // IDR
|
||||||
|
]));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
276
crates/wzp-video/tests/pipeline_roundtrip.rs
Normal file
276
crates/wzp-video/tests/pipeline_roundtrip.rs
Normal file
@@ -0,0 +1,276 @@
|
|||||||
|
//! Full-stack video pipeline integration test.
|
||||||
|
//!
|
||||||
|
//! Exercises every layer of the Blocker 1–3 implementation end-to-end:
|
||||||
|
//!
|
||||||
|
//! factory::create_video_encoder
|
||||||
|
//! → encoder.encode()
|
||||||
|
//! → transport::packetize_video_frame
|
||||||
|
//! → VideoReassembler::push
|
||||||
|
//! → factory::create_video_decoder
|
||||||
|
//! → decoder.decode()
|
||||||
|
//!
|
||||||
|
//! Runs only on macOS (VideoToolbox encoders / decoders).
|
||||||
|
|
||||||
|
#![cfg(target_os = "macos")]
|
||||||
|
|
||||||
|
use std::sync::Mutex;
|
||||||
|
use wzp_proto::CodecId;
|
||||||
|
use wzp_video::{
|
||||||
|
factory::{create_video_decoder, create_video_encoder},
|
||||||
|
transport::{packetize_video_frame, VideoReassembler},
|
||||||
|
VideoFrame,
|
||||||
|
};
|
||||||
|
|
||||||
|
/// VideoToolbox has global session registry state — serialise integration tests
|
||||||
|
/// to avoid races when multiple sessions open concurrently.
|
||||||
|
static VT_LOCK: Mutex<()> = Mutex::new(());
|
||||||
|
|
||||||
|
// ── helpers ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
fn synthetic_i420(width: u32, height: u32, frame_idx: u32) -> VideoFrame {
|
||||||
|
let y_size = (width * height) as usize;
|
||||||
|
let uv_size = y_size / 4;
|
||||||
|
let mut data = vec![0u8; y_size + 2 * uv_size];
|
||||||
|
|
||||||
|
for y in 0..height {
|
||||||
|
for x in 0..width {
|
||||||
|
// Shift the gradient by frame_idx so successive frames differ.
|
||||||
|
let val = (((x + frame_idx) * 255) / width) as u8;
|
||||||
|
data[(y * width + x) as usize] = val;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
data[y_size..y_size + uv_size].fill(128);
|
||||||
|
data[y_size + uv_size..].fill(128);
|
||||||
|
|
||||||
|
VideoFrame {
|
||||||
|
width,
|
||||||
|
height,
|
||||||
|
data,
|
||||||
|
timestamp_ms: frame_idx as u64 * 33,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── tests ─────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Encode → packetize → reassemble → decode round-trip for H.264 Baseline.
|
||||||
|
#[test]
|
||||||
|
fn h264_pipeline_roundtrip() {
|
||||||
|
let _g = VT_LOCK.lock().unwrap();
|
||||||
|
let (w, h) = (640, 360);
|
||||||
|
|
||||||
|
let mut encoder =
|
||||||
|
create_video_encoder(CodecId::H264Baseline, w, h, 1_500_000).expect("H264Baseline encoder");
|
||||||
|
let mut decoder =
|
||||||
|
create_video_decoder(CodecId::H264Baseline, w, h).expect("H264Baseline decoder");
|
||||||
|
|
||||||
|
let mut seq = 0u32;
|
||||||
|
let mut decoded_count = 0usize;
|
||||||
|
|
||||||
|
encoder.request_keyframe();
|
||||||
|
|
||||||
|
for i in 0..30u32 {
|
||||||
|
let frame = synthetic_i420(w, h, i);
|
||||||
|
let encoded = encoder.encode(&frame).expect("encode");
|
||||||
|
if encoded.is_empty() {
|
||||||
|
continue; // codec may buffer
|
||||||
|
}
|
||||||
|
|
||||||
|
let is_keyframe = encoder.is_keyframe(&encoded);
|
||||||
|
let pkts = packetize_video_frame(
|
||||||
|
&encoded,
|
||||||
|
CodecId::H264Baseline,
|
||||||
|
is_keyframe,
|
||||||
|
&mut seq,
|
||||||
|
i * 33,
|
||||||
|
w,
|
||||||
|
h,
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!pkts.is_empty(),
|
||||||
|
"packetize must produce at least one packet"
|
||||||
|
);
|
||||||
|
|
||||||
|
// All fragments for this frame share the same timestamp.
|
||||||
|
let ts = pkts[0].header.timestamp;
|
||||||
|
let total_frags = pkts.len();
|
||||||
|
for (idx, pkt) in pkts.iter().enumerate() {
|
||||||
|
assert_eq!(
|
||||||
|
pkt.header.timestamp, ts,
|
||||||
|
"all fragments of one frame share timestamp"
|
||||||
|
);
|
||||||
|
let frag_idx = (pkt.header.fec_block >> 8) as usize;
|
||||||
|
let frag_total = (pkt.header.fec_block & 0xFF) as usize;
|
||||||
|
assert_eq!(frag_idx, idx, "fragment index must match packet position");
|
||||||
|
assert_eq!(
|
||||||
|
frag_total, total_frags,
|
||||||
|
"all fragments carry the correct total count"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
pkts.last().unwrap().header.is_frame_end(),
|
||||||
|
"last packet must have FLAG_FRAME_END"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Push through reassembler — only the last packet should yield a frame.
|
||||||
|
let mut reassembler = VideoReassembler::new();
|
||||||
|
for (j, pkt) in pkts.iter().enumerate() {
|
||||||
|
let result = reassembler.push(pkt);
|
||||||
|
if j + 1 < pkts.len() {
|
||||||
|
assert!(
|
||||||
|
result.is_none(),
|
||||||
|
"intermediate fragments must not yield a complete frame"
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
let frame = result.expect("last fragment must complete the frame");
|
||||||
|
assert_eq!(frame.codec_id, CodecId::H264Baseline);
|
||||||
|
assert_eq!(frame.is_keyframe, is_keyframe);
|
||||||
|
assert_eq!(frame.width, Some(w as u16));
|
||||||
|
assert_eq!(frame.height, Some(h as u16));
|
||||||
|
assert_eq!(
|
||||||
|
frame.data, encoded,
|
||||||
|
"reassembled bytes must match original encoded bytes"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decode the reassembled frame.
|
||||||
|
match decoder.decode(&encoded) {
|
||||||
|
Ok(Some(yuv)) => {
|
||||||
|
assert_eq!(yuv.width, w);
|
||||||
|
assert_eq!(yuv.height, h);
|
||||||
|
let expected_size = (w * h * 3 / 2) as usize;
|
||||||
|
assert!(
|
||||||
|
yuv.data.len() >= expected_size,
|
||||||
|
"decoded I420 too small: {} < {expected_size}",
|
||||||
|
yuv.data.len()
|
||||||
|
);
|
||||||
|
decoded_count += 1;
|
||||||
|
}
|
||||||
|
Ok(None) => {} // pipeline latency — decoder still buffering
|
||||||
|
Err(e) => panic!("decode error: {e}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
decoded_count > 0,
|
||||||
|
"at least one frame must have been decoded"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fragmentation: a frame larger than VIDEO_MAX_PAYLOAD splits into multiple packets,
|
||||||
|
/// all of which reassemble back to the original bytes.
|
||||||
|
#[test]
|
||||||
|
fn large_frame_fragments_and_reassembles() {
|
||||||
|
use wzp_video::transport::VIDEO_MAX_PAYLOAD;
|
||||||
|
|
||||||
|
// Craft a fake "encoded" blob larger than one MTU.
|
||||||
|
let synthetic_encoded: Vec<u8> = (0..VIDEO_MAX_PAYLOAD * 3 + 200)
|
||||||
|
.map(|i| (i & 0xFF) as u8)
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
let mut seq = 0u32;
|
||||||
|
let pkts = packetize_video_frame(
|
||||||
|
&synthetic_encoded,
|
||||||
|
CodecId::H264Baseline,
|
||||||
|
true,
|
||||||
|
&mut seq,
|
||||||
|
9000,
|
||||||
|
1280,
|
||||||
|
720,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(pkts.len() >= 4, "large frame must produce ≥4 fragments");
|
||||||
|
assert!(
|
||||||
|
pkts[0].header.is_keyframe(),
|
||||||
|
"keyframe flag propagates to all fragments"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!pkts[0].header.is_frame_end(),
|
||||||
|
"first packet is not frame end"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
pkts.last().unwrap().header.is_frame_end(),
|
||||||
|
"last packet is frame end"
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut reassembler = VideoReassembler::new();
|
||||||
|
let mut result = None;
|
||||||
|
for pkt in &pkts {
|
||||||
|
result = reassembler.push(pkt);
|
||||||
|
}
|
||||||
|
|
||||||
|
let frame = result.expect("all fragments delivered → complete frame");
|
||||||
|
assert_eq!(frame.width, Some(1280));
|
||||||
|
assert_eq!(frame.height, Some(720));
|
||||||
|
assert_eq!(
|
||||||
|
frame.data, synthetic_encoded,
|
||||||
|
"reassembled bytes must match input exactly"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Packet loss: if the first fragment is missing, reassembly cannot complete.
|
||||||
|
#[test]
|
||||||
|
fn missing_fragment_blocks_reassembly() {
|
||||||
|
use wzp_video::transport::VIDEO_MAX_PAYLOAD;
|
||||||
|
|
||||||
|
let frame: Vec<u8> = vec![0xAB; VIDEO_MAX_PAYLOAD * 2 + 50];
|
||||||
|
let mut seq = 0u32;
|
||||||
|
let pkts = packetize_video_frame(&frame, CodecId::Av1Main, false, &mut seq, 1234, 640, 480);
|
||||||
|
assert!(pkts.len() >= 3);
|
||||||
|
|
||||||
|
let mut reassembler = VideoReassembler::new();
|
||||||
|
// Skip fragment 0 — deliver 1 and 2.
|
||||||
|
for pkt in &pkts[1..] {
|
||||||
|
let r = reassembler.push(pkt);
|
||||||
|
assert!(r.is_none(), "incomplete set must not yield a frame");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Codec negotiation smoke test: relay picks first offered codec.
|
||||||
|
///
|
||||||
|
/// This keeps codec-selection logic exercised at the transport layer even though
|
||||||
|
/// the real negotiation happens in wzp-relay/wzp-client handshakes.
|
||||||
|
#[test]
|
||||||
|
fn video_codec_selection_semantics() {
|
||||||
|
// The relay's selection rule is: first codec offered by the caller.
|
||||||
|
let offered = vec![CodecId::H264Baseline];
|
||||||
|
let chosen = offered.into_iter().next();
|
||||||
|
assert_eq!(chosen, Some(CodecId::H264Baseline));
|
||||||
|
|
||||||
|
// When no codecs are offered, video is audio-only.
|
||||||
|
let empty: Vec<CodecId> = vec![];
|
||||||
|
assert_eq!(empty.into_iter().next(), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Evict-stale does not panic and removes old frames.
|
||||||
|
#[test]
|
||||||
|
fn evict_stale_removes_aged_frames() {
|
||||||
|
use wzp_video::transport::VIDEO_MAX_PAYLOAD;
|
||||||
|
|
||||||
|
let frame: Vec<u8> = vec![0x55; VIDEO_MAX_PAYLOAD * 2];
|
||||||
|
let mut seq = 0u32;
|
||||||
|
let pkts = packetize_video_frame(
|
||||||
|
&frame,
|
||||||
|
CodecId::H264Baseline,
|
||||||
|
false,
|
||||||
|
&mut seq,
|
||||||
|
500,
|
||||||
|
640,
|
||||||
|
480,
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut reassembler = VideoReassembler::new();
|
||||||
|
// Push only first packet — frame is incomplete.
|
||||||
|
reassembler.push(&pkts[0]);
|
||||||
|
|
||||||
|
// Evict frames older than 1000 ms; current timestamp is 10000.
|
||||||
|
reassembler.evict_stale(10_000, 1_000);
|
||||||
|
|
||||||
|
// Pushing the rest now must not complete a frame (state was evicted).
|
||||||
|
for pkt in &pkts[1..] {
|
||||||
|
let r = reassembler.push(pkt);
|
||||||
|
// May or may not reassemble depending on reassembler's handling
|
||||||
|
// of a new frame with the same timestamp — mainly verify no panic.
|
||||||
|
let _ = r;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -43,12 +43,16 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<!-- Voice join FAB -->
|
<!-- Voice / Video join FABs -->
|
||||||
<div class="lobby-fab-row">
|
<div class="lobby-fab-row">
|
||||||
<button id="join-voice-btn" class="fab" title="Join Voice Chat">
|
<button id="join-voice-btn" class="fab" title="Join Voice Chat">
|
||||||
<span class="fab-icon">🎧</span>
|
<span class="fab-icon">🎧</span>
|
||||||
<span class="fab-label">Join Voice</span>
|
<span class="fab-label">Join Voice</span>
|
||||||
</button>
|
</button>
|
||||||
|
<button id="join-video-btn" class="fab fab-video" title="Join with Video">
|
||||||
|
<span class="fab-icon">📹</span>
|
||||||
|
<span class="fab-label">Join Video</span>
|
||||||
|
</button>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<!-- Incoming call banner -->
|
<!-- Incoming call banner -->
|
||||||
@@ -84,6 +88,9 @@
|
|||||||
<button id="vd-spk-btn" class="vd-btn" title="Speaker (s)">
|
<button id="vd-spk-btn" class="vd-btn" title="Speaker (s)">
|
||||||
<span id="vd-spk-icon">Spk</span>
|
<span id="vd-spk-icon">Spk</span>
|
||||||
</button>
|
</button>
|
||||||
|
<button id="vd-cam-btn" class="vd-btn" title="Camera (v)">
|
||||||
|
<span id="vd-cam-icon">Cam</span>
|
||||||
|
</button>
|
||||||
<button id="vd-end-btn" class="vd-btn vd-end" title="Leave voice (q)">
|
<button id="vd-end-btn" class="vd-btn vd-end" title="Leave voice (q)">
|
||||||
<span>End</span>
|
<span>End</span>
|
||||||
</button>
|
</button>
|
||||||
@@ -99,6 +106,16 @@
|
|||||||
</div>
|
</div>
|
||||||
<div id="vd-stats" class="vd-stats"></div>
|
<div id="vd-stats" class="vd-stats"></div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<!-- ═════ Video stage — full-screen overlay above drawer ═════ -->
|
||||||
|
<div id="vd-video-strip" class="vd-video-stage hidden">
|
||||||
|
<canvas id="vd-remote-video" class="vd-remote-stage" width="1280" height="720"></canvas>
|
||||||
|
<div id="vd-remote-placeholder" class="vd-remote-placeholder">
|
||||||
|
<div class="vd-placeholder-text">Waiting for remote video…</div>
|
||||||
|
<div id="vd-remote-counter" class="vd-placeholder-sub">0 frames received</div>
|
||||||
|
</div>
|
||||||
|
<video id="vd-local-video" class="vd-local-pip" autoplay muted playsinline></video>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<!-- ═══════════════════════════════════════════════════════
|
<!-- ═══════════════════════════════════════════════════════
|
||||||
@@ -157,6 +174,22 @@
|
|||||||
OS Echo Cancellation
|
OS Echo Cancellation
|
||||||
</label>
|
</label>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="settings-section">
|
||||||
|
<h3>Video</h3>
|
||||||
|
<label>Codec
|
||||||
|
<select id="s-video-codec">
|
||||||
|
<option value="h264">H.264</option>
|
||||||
|
<option value="h265">H.265 / HEVC</option>
|
||||||
|
</select>
|
||||||
|
</label>
|
||||||
|
<label>Room Resolution
|
||||||
|
<select id="s-video-resolution">
|
||||||
|
<option value="640x360">640 x 360</option>
|
||||||
|
<option value="960x540">960 x 540</option>
|
||||||
|
<option value="1280x720">1280 x 720</option>
|
||||||
|
</select>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
<div class="settings-section">
|
<div class="settings-section">
|
||||||
<h3>Relays</h3>
|
<h3>Relays</h3>
|
||||||
<div id="s-relay-list"></div>
|
<div id="s-relay-list"></div>
|
||||||
|
|||||||
@@ -9,7 +9,7 @@
|
|||||||
"tauri": "tauri"
|
"tauri": "tauri"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@tauri-apps/api": "^2"
|
"@tauri-apps/api": "^2.11"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"typescript": "^5",
|
"typescript": "^5",
|
||||||
|
|||||||
@@ -44,6 +44,9 @@ tracing = "0.1"
|
|||||||
tracing-subscriber = "0.3"
|
tracing-subscriber = "0.3"
|
||||||
anyhow = "1"
|
anyhow = "1"
|
||||||
rustls = { version = "0.23", default-features = false, features = ["ring", "std"] }
|
rustls = { version = "0.23", default-features = false, features = ["ring", "std"] }
|
||||||
|
# JPEG encoding for video:frame events (I420 → RGB → JPEG for IPC to WebView)
|
||||||
|
image = { version = "0.25", default-features = false, features = ["jpeg"] }
|
||||||
|
base64 = "0.22"
|
||||||
|
|
||||||
# WarzonePhone crates — protocol layer is platform-independent
|
# WarzonePhone crates — protocol layer is platform-independent
|
||||||
wzp-proto = { path = "../../crates/wzp-proto" }
|
wzp-proto = { path = "../../crates/wzp-proto" }
|
||||||
@@ -51,6 +54,7 @@ wzp-codec = { path = "../../crates/wzp-codec" }
|
|||||||
wzp-fec = { path = "../../crates/wzp-fec" }
|
wzp-fec = { path = "../../crates/wzp-fec" }
|
||||||
wzp-crypto = { path = "../../crates/wzp-crypto" }
|
wzp-crypto = { path = "../../crates/wzp-crypto" }
|
||||||
wzp-transport = { path = "../../crates/wzp-transport" }
|
wzp-transport = { path = "../../crates/wzp-transport" }
|
||||||
|
wzp-video = { path = "../../crates/wzp-video" }
|
||||||
|
|
||||||
# wzp-client pulls in CPAL on every desktop target and, additionally on
|
# wzp-client pulls in CPAL on every desktop target and, additionally on
|
||||||
# macOS, VoiceProcessingIO (coreaudio-rs behind the "vpio" feature). The
|
# macOS, VoiceProcessingIO (coreaudio-rs behind the "vpio" feature). The
|
||||||
@@ -99,6 +103,10 @@ libloading = "0.8"
|
|||||||
jni = "0.21"
|
jni = "0.21"
|
||||||
ndk-context = "0.1"
|
ndk-context = "0.1"
|
||||||
|
|
||||||
|
[dev-dependencies]
|
||||||
|
bytes = "1"
|
||||||
|
async-trait = "0.1"
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
default = ["custom-protocol"]
|
default = ["custom-protocol"]
|
||||||
custom-protocol = ["tauri/custom-protocol"]
|
custom-protocol = ["tauri/custom-protocol"]
|
||||||
|
|||||||
@@ -17,5 +17,7 @@
|
|||||||
-->
|
-->
|
||||||
<key>NSMicrophoneUsageDescription</key>
|
<key>NSMicrophoneUsageDescription</key>
|
||||||
<string>WarzonePhone needs microphone access to transmit your voice during calls.</string>
|
<string>WarzonePhone needs microphone access to transmit your voice during calls.</string>
|
||||||
|
<key>NSCameraUsageDescription</key>
|
||||||
|
<string>WarzonePhone needs camera access for video calls.</string>
|
||||||
</dict>
|
</dict>
|
||||||
</plist>
|
</plist>
|
||||||
|
|||||||
@@ -3,7 +3,9 @@
|
|||||||
<uses-permission android:name="android.permission.INTERNET" />
|
<uses-permission android:name="android.permission.INTERNET" />
|
||||||
<uses-permission android:name="android.permission.RECORD_AUDIO" />
|
<uses-permission android:name="android.permission.RECORD_AUDIO" />
|
||||||
<uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS" />
|
<uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS" />
|
||||||
|
<uses-permission android:name="android.permission.CAMERA" />
|
||||||
<uses-feature android:name="android.hardware.microphone" android:required="true" />
|
<uses-feature android:name="android.hardware.microphone" android:required="true" />
|
||||||
|
<uses-feature android:name="android.hardware.camera" android:required="false" />
|
||||||
|
|
||||||
<!-- AndroidTV support -->
|
<!-- AndroidTV support -->
|
||||||
<uses-feature android:name="android.software.leanback" android:required="false" />
|
<uses-feature android:name="android.software.leanback" android:required="false" />
|
||||||
|
|||||||
@@ -16,10 +16,19 @@ class MainActivity : TauriActivity() {
|
|||||||
private const val AUDIO_PERMISSIONS_REQUEST = 4242
|
private const val AUDIO_PERMISSIONS_REQUEST = 4242
|
||||||
private val REQUIRED_AUDIO_PERMISSIONS = arrayOf(
|
private val REQUIRED_AUDIO_PERMISSIONS = arrayOf(
|
||||||
Manifest.permission.RECORD_AUDIO,
|
Manifest.permission.RECORD_AUDIO,
|
||||||
Manifest.permission.MODIFY_AUDIO_SETTINGS
|
Manifest.permission.MODIFY_AUDIO_SETTINGS,
|
||||||
|
Manifest.permission.CAMERA
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NOTE: granting CAMERA at the Android system layer is necessary but NOT
|
||||||
|
// sufficient for video on Android. Tauri/Wry's internal WebChromeClient
|
||||||
|
// does not currently grant `getUserMedia` permission requests, so the
|
||||||
|
// browser-layer getUserMedia call still fails even after the OS grants
|
||||||
|
// CAMERA. Fixing this needs either a Tauri plugin that overrides the
|
||||||
|
// WebChromeClient, or a native Camera2/CameraX capture path that bypasses
|
||||||
|
// the WebView. Tracked as a follow-up.
|
||||||
|
|
||||||
override fun onCreate(savedInstanceState: Bundle?) {
|
override fun onCreate(savedInstanceState: Bundle?) {
|
||||||
enableEdgeToEdge()
|
enableEdgeToEdge()
|
||||||
super.onCreate(savedInstanceState)
|
super.onCreate(savedInstanceState)
|
||||||
|
|||||||
@@ -56,6 +56,30 @@ fn audio_manager<'local>(
|
|||||||
Ok(am)
|
Ok(am)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn has_permission(permission: &str) -> Result<bool, String> {
|
||||||
|
let (vm, activity) = jvm_and_activity()?;
|
||||||
|
let mut env = vm
|
||||||
|
.attach_current_thread()
|
||||||
|
.map_err(|e| format!("attach_current_thread: {e}"))?;
|
||||||
|
let permission = env
|
||||||
|
.new_string(permission)
|
||||||
|
.map_err(|e| format!("new_string(permission): {e}"))?;
|
||||||
|
let result = env
|
||||||
|
.call_method(
|
||||||
|
&activity,
|
||||||
|
"checkSelfPermission",
|
||||||
|
"(Ljava/lang/String;)I",
|
||||||
|
&[JValue::Object(&permission)],
|
||||||
|
)
|
||||||
|
.and_then(|v| v.i())
|
||||||
|
.map_err(|e| format!("checkSelfPermission: {e}"))?;
|
||||||
|
Ok(result == 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn has_record_audio_permission() -> Result<bool, String> {
|
||||||
|
has_permission("android.permission.RECORD_AUDIO")
|
||||||
|
}
|
||||||
|
|
||||||
/// Set `AudioManager.MODE_IN_COMMUNICATION`. Call when a VoIP call starts.
|
/// Set `AudioManager.MODE_IN_COMMUNICATION`. Call when a VoIP call starts.
|
||||||
/// This tells the audio policy to route through the communication device
|
/// This tells the audio policy to route through the communication device
|
||||||
/// path (earpiece/BT SCO) instead of the media path (speaker/BT A2DP).
|
/// path (earpiece/BT SCO) instead of the media path (speaker/BT A2DP).
|
||||||
@@ -72,6 +96,33 @@ pub fn set_audio_mode_communication() -> Result<(), String> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Run `set_audio_mode_communication` on Tauri's main thread, where the
|
||||||
|
/// Android context is initialized. Calling it from arbitrary Tokio blocking
|
||||||
|
/// workers panics inside `ndk_context::android_context()`.
|
||||||
|
pub async fn set_audio_mode_communication_on_main(app: tauri::AppHandle) -> Result<(), String> {
|
||||||
|
let (tx, rx) = tokio::sync::oneshot::channel();
|
||||||
|
app.run_on_main_thread(move || {
|
||||||
|
let result = std::panic::catch_unwind(set_audio_mode_communication)
|
||||||
|
.map_err(|panic| {
|
||||||
|
if let Some(s) = panic.downcast_ref::<&str>() {
|
||||||
|
format!("panic: {s}")
|
||||||
|
} else if let Some(s) = panic.downcast_ref::<String>() {
|
||||||
|
format!("panic: {s}")
|
||||||
|
} else {
|
||||||
|
"panic: unknown".to_string()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.and_then(|r| r);
|
||||||
|
let _ = tx.send(result);
|
||||||
|
})
|
||||||
|
.map_err(|e| format!("run_on_main_thread: {e}"))?;
|
||||||
|
|
||||||
|
tokio::time::timeout(std::time::Duration::from_secs(2), rx)
|
||||||
|
.await
|
||||||
|
.map_err(|_| "set_audio_mode_communication timed out after 2s".to_string())?
|
||||||
|
.map_err(|_| "set_audio_mode_communication result channel closed".to_string())?
|
||||||
|
}
|
||||||
|
|
||||||
/// Restore `AudioManager.MODE_NORMAL`. Call when a VoIP call ends.
|
/// Restore `AudioManager.MODE_NORMAL`. Call when a VoIP call ends.
|
||||||
pub fn set_audio_mode_normal() -> Result<(), String> {
|
pub fn set_audio_mode_normal() -> Result<(), String> {
|
||||||
let (vm, activity) = jvm_and_activity()?;
|
let (vm, activity) = jvm_and_activity()?;
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -31,7 +31,7 @@ use engine::CallEngine;
|
|||||||
|
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::sync::atomic::{AtomicBool, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||||
use std::sync::{Arc, OnceLock};
|
use std::sync::{Arc, OnceLock};
|
||||||
use tauri::{Emitter, Manager};
|
use tauri::{Emitter, Manager};
|
||||||
use tokio::sync::Mutex;
|
use tokio::sync::Mutex;
|
||||||
@@ -49,6 +49,12 @@ use wzp_proto::{MediaTransport, default_signal_version};
|
|||||||
// Mirrors the existing `wzp_codec::dred_verbose_logs` pattern.
|
// Mirrors the existing `wzp_codec::dred_verbose_logs` pattern.
|
||||||
|
|
||||||
static CALL_DEBUG_LOGS: AtomicBool = AtomicBool::new(false);
|
static CALL_DEBUG_LOGS: AtomicBool = AtomicBool::new(false);
|
||||||
|
static CAMERA_PUSH_FRAMES: AtomicU64 = AtomicU64::new(0);
|
||||||
|
static CAMERA_PUSH_DROPS: AtomicU64 = AtomicU64::new(0);
|
||||||
|
static CAMERA_PUSH_NO_ENGINE: AtomicU64 = AtomicU64::new(0);
|
||||||
|
static CAMERA_PUSH_NO_SENDER: AtomicU64 = AtomicU64::new(0);
|
||||||
|
static CAMERA_PUSH_DECODE_ERRORS: AtomicU64 = AtomicU64::new(0);
|
||||||
|
static FRAME_DUMP_WRITES: AtomicU64 = AtomicU64::new(0);
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
fn call_debug_logs_enabled() -> bool {
|
fn call_debug_logs_enabled() -> bool {
|
||||||
@@ -59,13 +65,15 @@ fn set_call_debug_logs_internal(on: bool) {
|
|||||||
CALL_DEBUG_LOGS.store(on, Ordering::Relaxed);
|
CALL_DEBUG_LOGS.store(on, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Emit a `call-debug-log` event to the JS side IF the flag is on.
|
/// Emit a `call-debug-log` event to the JS side.
|
||||||
/// Also mirrors to `tracing::info!` so logcat keeps its copy
|
/// Also mirrors to `tracing::info!` so logcat keeps its copy
|
||||||
/// regardless of the flag — the toggle only controls the GUI
|
/// regardless of the flag. Connect/register steps are always emitted
|
||||||
/// overlay, not the underlying Android log stream.
|
/// because they are needed to diagnose failed joins after app data is
|
||||||
|
/// cleared and the GUI debug toggle is back to its default false value.
|
||||||
pub(crate) fn emit_call_debug(app: &tauri::AppHandle, step: &str, details: serde_json::Value) {
|
pub(crate) fn emit_call_debug(app: &tauri::AppHandle, step: &str, details: serde_json::Value) {
|
||||||
tracing::info!(step, ?details, "call-debug");
|
tracing::info!(step, ?details, "call-debug");
|
||||||
if !call_debug_logs_enabled() {
|
let force_emit = step.starts_with("connect:") || step.starts_with("register_signal:");
|
||||||
|
if !force_emit && !call_debug_logs_enabled() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let payload = serde_json::json!({
|
let payload = serde_json::json!({
|
||||||
@@ -79,9 +87,470 @@ pub(crate) fn emit_call_debug(app: &tauri::AppHandle, step: &str, details: serde
|
|||||||
let _ = app.emit("call-debug-log", payload);
|
let _ = app.emit("call-debug-log", payload);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tauri::command]
|
||||||
|
fn call_debug_log(app: tauri::AppHandle, step: String, details: serde_json::Value) {
|
||||||
|
if step == "camera:get_user_media_start" {
|
||||||
|
CAMERA_PUSH_FRAMES.store(0, Ordering::Relaxed);
|
||||||
|
CAMERA_PUSH_DROPS.store(0, Ordering::Relaxed);
|
||||||
|
CAMERA_PUSH_NO_ENGINE.store(0, Ordering::Relaxed);
|
||||||
|
CAMERA_PUSH_NO_SENDER.store(0, Ordering::Relaxed);
|
||||||
|
CAMERA_PUSH_DECODE_ERRORS.store(0, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
emit_call_debug(&app, &step, details);
|
||||||
|
}
|
||||||
|
|
||||||
/// Short git hash captured at compile time by build.rs.
|
/// Short git hash captured at compile time by build.rs.
|
||||||
const GIT_HASH: &str = env!("WZP_GIT_HASH");
|
const GIT_HASH: &str = env!("WZP_GIT_HASH");
|
||||||
|
|
||||||
|
// ─── Video helpers ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
/// Convert an I420 frame to a JPEG and base64-encode it for IPC.
|
||||||
|
///
|
||||||
|
/// Returns `None` if the data is too short or encoding fails.
|
||||||
|
/// Called from the video recv task in engine.rs to produce the `jpeg_b64`
|
||||||
|
/// field of every `video:frame` Tauri event.
|
||||||
|
#[cfg_attr(not(test), allow(dead_code))]
|
||||||
|
pub(crate) fn i420_to_jpeg_b64(data: &[u8], width: u32, height: u32) -> Option<String> {
|
||||||
|
use base64::Engine as _;
|
||||||
|
|
||||||
|
let bytes = i420_to_jpeg_bytes(data, width, height)?;
|
||||||
|
Some(base64::engine::general_purpose::STANDARD.encode(bytes))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn i420_to_jpeg_bytes(data: &[u8], width: u32, height: u32) -> Option<Vec<u8>> {
|
||||||
|
use image::{DynamicImage, ImageBuffer, Rgb};
|
||||||
|
|
||||||
|
let w = width as usize;
|
||||||
|
let h = height as usize;
|
||||||
|
let y_size = w * h;
|
||||||
|
let uv_size = w * h / 4;
|
||||||
|
|
||||||
|
if data.len() < y_size + 2 * uv_size {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut rgb = vec![0u8; w * h * 3];
|
||||||
|
for row in 0..h {
|
||||||
|
for col in 0..w {
|
||||||
|
let y = data[row * w + col] as f32;
|
||||||
|
let uv_idx = (row / 2) * (w / 2) + col / 2;
|
||||||
|
let u = data[y_size + uv_idx] as f32 - 128.0;
|
||||||
|
let v = data[y_size + uv_size + uv_idx] as f32 - 128.0;
|
||||||
|
let out = (row * w + col) * 3;
|
||||||
|
rgb[out] = (y + 1.402 * v).clamp(0.0, 255.0) as u8;
|
||||||
|
rgb[out + 1] = (y - 0.344 * u - 0.714 * v).clamp(0.0, 255.0) as u8;
|
||||||
|
rgb[out + 2] = (y + 1.772 * u).clamp(0.0, 255.0) as u8;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let img = DynamicImage::ImageRgb8(ImageBuffer::<Rgb<u8>, Vec<u8>>::from_raw(
|
||||||
|
width, height, rgb,
|
||||||
|
)?);
|
||||||
|
let mut buf = std::io::Cursor::new(Vec::<u8>::new());
|
||||||
|
img.write_to(&mut buf, image::ImageFormat::Jpeg).ok()?;
|
||||||
|
Some(buf.into_inner())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn should_dump_frame(frame_no: u64) -> bool {
|
||||||
|
frame_no <= 5 || frame_no % 30 == 0
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn maybe_dump_video_jpeg(
|
||||||
|
app: &tauri::AppHandle,
|
||||||
|
stage: &str,
|
||||||
|
platform: &str,
|
||||||
|
frame_no: u64,
|
||||||
|
jpeg_bytes: &[u8],
|
||||||
|
width: u32,
|
||||||
|
height: u32,
|
||||||
|
) {
|
||||||
|
if !should_dump_frame(frame_no) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let seq = FRAME_DUMP_WRITES.fetch_add(1, Ordering::Relaxed) + 1;
|
||||||
|
let dir = identity_dir().join("frame-dumps");
|
||||||
|
let file_name = format!("{seq:06}_{platform}_{stage}_f{frame_no:06}_{width}x{height}.jpg");
|
||||||
|
let path = dir.join(file_name);
|
||||||
|
let result = std::fs::create_dir_all(&dir).and_then(|_| std::fs::write(&path, jpeg_bytes));
|
||||||
|
|
||||||
|
match result {
|
||||||
|
Ok(()) => emit_call_debug(
|
||||||
|
app,
|
||||||
|
"video:frame_dump",
|
||||||
|
serde_json::json!({
|
||||||
|
"stage": stage,
|
||||||
|
"platform": platform,
|
||||||
|
"frame_no": frame_no,
|
||||||
|
"width": width,
|
||||||
|
"height": height,
|
||||||
|
"jpeg_bytes": jpeg_bytes.len(),
|
||||||
|
"path": path,
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
Err(e) => {
|
||||||
|
if seq <= 5 || seq % 30 == 0 {
|
||||||
|
emit_call_debug(
|
||||||
|
app,
|
||||||
|
"video:frame_dump_failed",
|
||||||
|
serde_json::json!({
|
||||||
|
"stage": stage,
|
||||||
|
"platform": platform,
|
||||||
|
"frame_no": frame_no,
|
||||||
|
"error": e.to_string(),
|
||||||
|
"path": path,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn maybe_dump_video_bytes(
|
||||||
|
app: &tauri::AppHandle,
|
||||||
|
stage: &str,
|
||||||
|
platform: &str,
|
||||||
|
frame_no: u64,
|
||||||
|
bytes: &[u8],
|
||||||
|
codec: wzp_proto::CodecId,
|
||||||
|
) {
|
||||||
|
if !should_dump_frame(frame_no) || bytes.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let ext = match codec {
|
||||||
|
wzp_proto::CodecId::H265Main => "h265",
|
||||||
|
wzp_proto::CodecId::Av1Main => "obu",
|
||||||
|
_ => "h264",
|
||||||
|
};
|
||||||
|
let seq = FRAME_DUMP_WRITES.fetch_add(1, Ordering::Relaxed) + 1;
|
||||||
|
let dir = identity_dir().join("frame-dumps");
|
||||||
|
let file_name = format!("{seq:06}_{platform}_{stage}_f{frame_no:06}.{ext}");
|
||||||
|
let path = dir.join(file_name);
|
||||||
|
let result = std::fs::create_dir_all(&dir).and_then(|_| std::fs::write(&path, bytes));
|
||||||
|
|
||||||
|
match result {
|
||||||
|
Ok(()) => emit_call_debug(
|
||||||
|
app,
|
||||||
|
"video:byte_dump",
|
||||||
|
serde_json::json!({
|
||||||
|
"stage": stage,
|
||||||
|
"platform": platform,
|
||||||
|
"frame_no": frame_no,
|
||||||
|
"codec": format!("{:?}", codec),
|
||||||
|
"bytes": bytes.len(),
|
||||||
|
"path": path,
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
Err(e) => {
|
||||||
|
if seq <= 5 || seq % 30 == 0 {
|
||||||
|
emit_call_debug(
|
||||||
|
app,
|
||||||
|
"video:byte_dump_failed",
|
||||||
|
serde_json::json!({
|
||||||
|
"stage": stage,
|
||||||
|
"platform": platform,
|
||||||
|
"frame_no": frame_no,
|
||||||
|
"codec": format!("{:?}", codec),
|
||||||
|
"error": e.to_string(),
|
||||||
|
"path": path,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// RGB24 → I420 (planar 4:2:0). Layout: Y(w×h) | U(w/2×h/2) | V(w/2×h/2).
|
||||||
|
fn rgb_to_i420(rgb: &[u8], w: usize, h: usize) -> Vec<u8> {
|
||||||
|
let y_size = w * h;
|
||||||
|
let uv_size = (w / 2) * (h / 2);
|
||||||
|
let mut out = vec![0u8; y_size + 2 * uv_size];
|
||||||
|
for row in 0..h {
|
||||||
|
for col in 0..w {
|
||||||
|
let i = (row * w + col) * 3;
|
||||||
|
let r = rgb[i] as f32;
|
||||||
|
let g = rgb[i + 1] as f32;
|
||||||
|
let b = rgb[i + 2] as f32;
|
||||||
|
out[row * w + col] = (0.299 * r + 0.587 * g + 0.114 * b).clamp(0.0, 255.0) as u8;
|
||||||
|
if row % 2 == 0 && col % 2 == 0 {
|
||||||
|
let uv = (row / 2) * (w / 2) + col / 2;
|
||||||
|
out[y_size + uv] =
|
||||||
|
(-0.169 * r - 0.331 * g + 0.500 * b + 128.0).clamp(0.0, 255.0) as u8;
|
||||||
|
out[y_size + uv_size + uv] =
|
||||||
|
(0.500 * r - 0.419 * g - 0.081 * b + 128.0).clamp(0.0, 255.0) as u8;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Tauri command: receive a JPEG frame from the frontend camera (getUserMedia),
|
||||||
|
/// decode it, convert to I420, and push into the active call's video send task.
|
||||||
|
///
|
||||||
|
/// The frontend calls this at ~15 fps from a canvas.toDataURL() capture loop.
|
||||||
|
#[tauri::command]
|
||||||
|
async fn push_camera_frame(
|
||||||
|
app: tauri::AppHandle,
|
||||||
|
state: tauri::State<'_, Arc<AppState>>,
|
||||||
|
jpeg_b64: String,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
use base64::Engine as _;
|
||||||
|
let jpeg_bytes = match base64::engine::general_purpose::STANDARD.decode(&jpeg_b64) {
|
||||||
|
Ok(bytes) => bytes,
|
||||||
|
Err(e) => {
|
||||||
|
let errs = CAMERA_PUSH_DECODE_ERRORS.fetch_add(1, Ordering::Relaxed) + 1;
|
||||||
|
if errs == 1 || errs % 30 == 0 {
|
||||||
|
emit_call_debug(
|
||||||
|
&app,
|
||||||
|
"camera:jpeg_base64_decode_failed",
|
||||||
|
serde_json::json!({
|
||||||
|
"errors": errs,
|
||||||
|
"error": e.to_string(),
|
||||||
|
"b64_len": jpeg_b64.len(),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return Err(e.to_string());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let dyn_img = match image::load_from_memory_with_format(&jpeg_bytes, image::ImageFormat::Jpeg) {
|
||||||
|
Ok(img) => img,
|
||||||
|
Err(e) => {
|
||||||
|
let errs = CAMERA_PUSH_DECODE_ERRORS.fetch_add(1, Ordering::Relaxed) + 1;
|
||||||
|
if errs == 1 || errs % 30 == 0 {
|
||||||
|
emit_call_debug(
|
||||||
|
&app,
|
||||||
|
"camera:jpeg_decode_failed",
|
||||||
|
serde_json::json!({
|
||||||
|
"errors": errs,
|
||||||
|
"error": e.to_string(),
|
||||||
|
"jpeg_bytes": jpeg_bytes.len(),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return Err(e.to_string());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let rgb_img = dyn_img.to_rgb8();
|
||||||
|
let w = rgb_img.width() as usize;
|
||||||
|
let h = rgb_img.height() as usize;
|
||||||
|
let yuv = rgb_to_i420(rgb_img.as_raw(), w, h);
|
||||||
|
let frame_no = CAMERA_PUSH_FRAMES.fetch_add(1, Ordering::Relaxed) + 1;
|
||||||
|
maybe_dump_video_jpeg(
|
||||||
|
&app,
|
||||||
|
"camera_jpeg_in",
|
||||||
|
std::env::consts::OS,
|
||||||
|
frame_no,
|
||||||
|
&jpeg_bytes,
|
||||||
|
w as u32,
|
||||||
|
h as u32,
|
||||||
|
);
|
||||||
|
if let Some(converted_jpeg) = i420_to_jpeg_bytes(&yuv, w as u32, h as u32) {
|
||||||
|
maybe_dump_video_jpeg(
|
||||||
|
&app,
|
||||||
|
"camera_i420_roundtrip",
|
||||||
|
std::env::consts::OS,
|
||||||
|
frame_no,
|
||||||
|
&converted_jpeg,
|
||||||
|
w as u32,
|
||||||
|
h as u32,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if frame_no == 1 || frame_no % 150 == 0 {
|
||||||
|
emit_call_debug(
|
||||||
|
&app,
|
||||||
|
"camera:frame_received",
|
||||||
|
serde_json::json!({
|
||||||
|
"frame_no": frame_no,
|
||||||
|
"width": w,
|
||||||
|
"height": h,
|
||||||
|
"jpeg_bytes": jpeg_bytes.len(),
|
||||||
|
"yuv_bytes": yuv.len(),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let ts = std::time::SystemTime::now()
|
||||||
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_millis() as u64;
|
||||||
|
|
||||||
|
let frame = wzp_video::encoder::VideoFrame {
|
||||||
|
width: w as u32,
|
||||||
|
height: h as u32,
|
||||||
|
data: yuv,
|
||||||
|
timestamp_ms: ts,
|
||||||
|
};
|
||||||
|
|
||||||
|
let engine = state.engine.lock().await;
|
||||||
|
if let Some(ref eng) = *engine {
|
||||||
|
if let Some(ref tx) = eng.camera_tx {
|
||||||
|
match tx.try_send(frame) {
|
||||||
|
Ok(()) => {
|
||||||
|
if frame_no == 1 || frame_no % 150 == 0 {
|
||||||
|
emit_call_debug(
|
||||||
|
&app,
|
||||||
|
"camera:frame_queued",
|
||||||
|
serde_json::json!({ "frame_no": frame_no }),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
let drops = CAMERA_PUSH_DROPS.fetch_add(1, Ordering::Relaxed) + 1;
|
||||||
|
if drops == 1 || drops % 30 == 0 {
|
||||||
|
emit_call_debug(
|
||||||
|
&app,
|
||||||
|
"camera:frame_drop",
|
||||||
|
serde_json::json!({
|
||||||
|
"frame_no": frame_no,
|
||||||
|
"drops": drops,
|
||||||
|
"reason": e.to_string(),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
let count = CAMERA_PUSH_NO_SENDER.fetch_add(1, Ordering::Relaxed) + 1;
|
||||||
|
if count == 1 || count % 150 == 0 {
|
||||||
|
emit_call_debug(
|
||||||
|
&app,
|
||||||
|
"camera:no_video_sender",
|
||||||
|
serde_json::json!({
|
||||||
|
"count": count,
|
||||||
|
"hint": "video was not negotiated or the encoder task failed before camera_tx was installed",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
let count = CAMERA_PUSH_NO_ENGINE.fetch_add(1, Ordering::Relaxed) + 1;
|
||||||
|
if count == 1 || count % 150 == 0 {
|
||||||
|
emit_call_debug(
|
||||||
|
&app,
|
||||||
|
"camera:no_call_engine",
|
||||||
|
serde_json::json!({ "count": count }),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// ─── Video helper tests ───────────────────────────────────────────────────────
|
||||||
|
#[cfg(test)]
|
||||||
|
mod video_tests {
|
||||||
|
use super::{i420_to_jpeg_b64, rgb_to_i420};
|
||||||
|
use base64::Engine as _;
|
||||||
|
|
||||||
|
fn solid_rgb_frame(w: usize, h: usize, r: u8, g: u8, b: u8) -> Vec<u8> {
|
||||||
|
let mut rgb = vec![0u8; w * h * 3];
|
||||||
|
for i in 0..w * h {
|
||||||
|
rgb[i * 3] = r;
|
||||||
|
rgb[i * 3 + 1] = g;
|
||||||
|
rgb[i * 3 + 2] = b;
|
||||||
|
}
|
||||||
|
rgb
|
||||||
|
}
|
||||||
|
|
||||||
|
fn solid_i420(w: usize, h: usize, y: u8, u: u8, v: u8) -> Vec<u8> {
|
||||||
|
let y_size = w * h;
|
||||||
|
let uv_size = w * h / 4;
|
||||||
|
let mut data = vec![y; y_size + 2 * uv_size];
|
||||||
|
data[y_size..y_size + uv_size].fill(u);
|
||||||
|
data[y_size + uv_size..].fill(v);
|
||||||
|
data
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rgb_to_i420_output_size() {
|
||||||
|
let rgb = solid_rgb_frame(640, 360, 128, 128, 128);
|
||||||
|
let yuv = rgb_to_i420(&rgb, 640, 360);
|
||||||
|
assert_eq!(yuv.len(), 640 * 360 * 3 / 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rgb_to_i420_pure_green_luma() {
|
||||||
|
// Pure green (0, 255, 0) → Y ≈ 150 (0.587 × 255 ≈ 150).
|
||||||
|
let rgb = solid_rgb_frame(4, 4, 0, 255, 0);
|
||||||
|
let yuv = rgb_to_i420(&rgb, 4, 4);
|
||||||
|
let y = yuv[0];
|
||||||
|
assert!(y >= 140 && y <= 160, "pure-green luma out of range: {y}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rgb_to_i420_grey_is_neutral() {
|
||||||
|
// Mid-grey RGB → U and V should both be near 128.
|
||||||
|
let rgb = solid_rgb_frame(4, 4, 128, 128, 128);
|
||||||
|
let yuv = rgb_to_i420(&rgb, 4, 4);
|
||||||
|
let uv_start = 4 * 4;
|
||||||
|
let u = yuv[uv_start];
|
||||||
|
let v = yuv[uv_start + 4]; // 4 = (4/2)*(4/2)
|
||||||
|
assert!((u as i32 - 128).abs() <= 5, "grey U out of range: {u}");
|
||||||
|
assert!((v as i32 - 128).abs() <= 5, "grey V out of range: {v}");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn i420_to_jpeg_b64_produces_non_empty_output() {
|
||||||
|
let data = solid_i420(64, 64, 128, 128, 128);
|
||||||
|
let b64 = i420_to_jpeg_b64(&data, 64, 64);
|
||||||
|
assert!(b64.is_some(), "valid I420 must produce Some(b64)");
|
||||||
|
let s = b64.unwrap();
|
||||||
|
assert!(!s.is_empty());
|
||||||
|
// JPEG base64 starts with '/9j/' (FFD8FF marker).
|
||||||
|
let decoded = base64::engine::general_purpose::STANDARD
|
||||||
|
.decode(&s)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
&decoded[0..2],
|
||||||
|
&[0xFF, 0xD8],
|
||||||
|
"output must start with JPEG SOI marker"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn i420_to_jpeg_b64_rejects_undersized_buffer() {
|
||||||
|
// Buffer too short: only Y plane, no chroma.
|
||||||
|
let data = vec![128u8; 64 * 64];
|
||||||
|
let b64 = i420_to_jpeg_b64(&data, 64, 64);
|
||||||
|
assert!(b64.is_none(), "truncated buffer must yield None");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn i420_to_jpeg_b64_color_preservation() {
|
||||||
|
// A red (255, 0, 0) I420 frame should decode to a mostly-red JPEG.
|
||||||
|
// After JPEG lossy compression the exact values drift, so we only
|
||||||
|
// check that the decoded pixel has R > G and R > B.
|
||||||
|
use base64::Engine as _;
|
||||||
|
|
||||||
|
// Convert red RGB → I420.
|
||||||
|
let rgb = solid_rgb_frame(64, 64, 255, 0, 0);
|
||||||
|
let yuv = rgb_to_i420(&rgb, 64, 64);
|
||||||
|
|
||||||
|
let b64 = i420_to_jpeg_b64(&yuv, 64, 64).expect("should produce JPEG");
|
||||||
|
let jpeg = base64::engine::general_purpose::STANDARD
|
||||||
|
.decode(&b64)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let img = image::load_from_memory_with_format(&jpeg, image::ImageFormat::Jpeg).unwrap();
|
||||||
|
let rgb_img = img.to_rgb8();
|
||||||
|
let px = rgb_img.get_pixel(32, 32);
|
||||||
|
let (r, g, b) = (px[0], px[1], px[2]);
|
||||||
|
assert!(
|
||||||
|
r > g && r > b,
|
||||||
|
"red frame: expected R dominant, got R={r} G={g} B={b}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rgb_i420_conversion_is_deterministic() {
|
||||||
|
let rgb = solid_rgb_frame(8, 8, 200, 100, 50);
|
||||||
|
let yuv1 = rgb_to_i420(&rgb, 8, 8);
|
||||||
|
let yuv2 = rgb_to_i420(&rgb, 8, 8);
|
||||||
|
assert_eq!(yuv1, yuv2, "rgb_to_i420 must be deterministic");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Resolved by `setup()` once we have a Tauri AppHandle. Holds the
|
/// Resolved by `setup()` once we have a Tauri AppHandle. Holds the
|
||||||
/// platform-correct app data dir (e.g. `/data/data/com.wzp.desktop/files` on
|
/// platform-correct app data dir (e.g. `/data/data/com.wzp.desktop/files` on
|
||||||
/// Android, `~/Library/Application Support/com.wzp.desktop` on macOS).
|
/// Android, `~/Library/Application Support/com.wzp.desktop` on macOS).
|
||||||
@@ -347,8 +816,14 @@ async fn connect(
|
|||||||
// Enable birthday attack for hard NAT traversal. Adds ~3s to
|
// Enable birthday attack for hard NAT traversal. Adds ~3s to
|
||||||
// call setup when peer has symmetric NAT.
|
// call setup when peer has symmetric NAT.
|
||||||
birthday_attack: Option<bool>,
|
birthday_attack: Option<bool>,
|
||||||
|
video_codec: Option<String>,
|
||||||
|
video_width: Option<u32>,
|
||||||
|
video_height: Option<u32>,
|
||||||
) -> Result<String, String> {
|
) -> Result<String, String> {
|
||||||
let force_direct = direct_only.unwrap_or(false);
|
let force_direct = direct_only.unwrap_or(false);
|
||||||
|
let video_codec = video_codec.unwrap_or_else(|| "h264".to_string());
|
||||||
|
let video_width = video_width.unwrap_or(1280);
|
||||||
|
let video_height = video_height.unwrap_or(720);
|
||||||
let enable_birthday = birthday_attack.unwrap_or(false);
|
let enable_birthday = birthday_attack.unwrap_or(false);
|
||||||
emit_call_debug(
|
emit_call_debug(
|
||||||
&app,
|
&app,
|
||||||
@@ -361,6 +836,9 @@ async fn connect(
|
|||||||
"peer_mapped_addr": peer_mapped_addr,
|
"peer_mapped_addr": peer_mapped_addr,
|
||||||
"direct_only": force_direct,
|
"direct_only": force_direct,
|
||||||
"birthday_attack": enable_birthday,
|
"birthday_attack": enable_birthday,
|
||||||
|
"video_codec": video_codec,
|
||||||
|
"video_width": video_width,
|
||||||
|
"video_height": video_height,
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
let mut engine_lock = state.engine.lock().await;
|
let mut engine_lock = state.engine.lock().await;
|
||||||
@@ -772,6 +1250,18 @@ async fn connect(
|
|||||||
if reuse_endpoint.is_some() && pre_connected_transport.is_none() {
|
if reuse_endpoint.is_some() && pre_connected_transport.is_none() {
|
||||||
tracing::info!("connect: reusing existing signal endpoint for media connection");
|
tracing::info!("connect: reusing existing signal endpoint for media connection");
|
||||||
}
|
}
|
||||||
|
emit_call_debug(
|
||||||
|
&app,
|
||||||
|
"connect:reuse_endpoint",
|
||||||
|
serde_json::json!({
|
||||||
|
"has_reuse_endpoint": reuse_endpoint.is_some(),
|
||||||
|
"reuse_local_addr": reuse_endpoint
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|ep| ep.local_addr().ok())
|
||||||
|
.map(|addr| addr.to_string()),
|
||||||
|
"has_pre_connected_transport": pre_connected_transport.is_some(),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
let app_clone = app.clone();
|
let app_clone = app.clone();
|
||||||
// Log transport details for debugging direct P2P media issues
|
// Log transport details for debugging direct P2P media issues
|
||||||
@@ -791,6 +1281,10 @@ async fn connect(
|
|||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
let app_for_engine = app.clone();
|
let app_for_engine = app.clone();
|
||||||
|
let (active_quality, peer_max_quality) = {
|
||||||
|
let sig = state.signal.lock().await;
|
||||||
|
(sig.active_quality.clone(), sig.peer_max_quality.clone())
|
||||||
|
};
|
||||||
match CallEngine::start(
|
match CallEngine::start(
|
||||||
relay,
|
relay,
|
||||||
room,
|
room,
|
||||||
@@ -801,6 +1295,11 @@ async fn connect(
|
|||||||
pre_connected_transport,
|
pre_connected_transport,
|
||||||
is_direct_p2p_agreed,
|
is_direct_p2p_agreed,
|
||||||
app_for_engine,
|
app_for_engine,
|
||||||
|
active_quality,
|
||||||
|
peer_max_quality,
|
||||||
|
video_codec,
|
||||||
|
video_width,
|
||||||
|
video_height,
|
||||||
move |event_kind, message| {
|
move |event_kind, message| {
|
||||||
let _ = app_clone.emit(
|
let _ = app_clone.emit(
|
||||||
"call-event",
|
"call-event",
|
||||||
@@ -1143,6 +1642,12 @@ struct SignalState {
|
|||||||
peer_hard_nat_probe: Option<PeerHardNatInfo>,
|
peer_hard_nat_probe: Option<PeerHardNatInfo>,
|
||||||
/// Phase 8.6: peer's birthday attack ports, if received.
|
/// Phase 8.6: peer's birthday attack ports, if received.
|
||||||
peer_birthday_ports: Option<PeerBirthdayInfo>,
|
peer_birthday_ports: Option<PeerBirthdayInfo>,
|
||||||
|
/// Active quality profile for the encoder. Updated by signal upgrade flow.
|
||||||
|
active_quality: Arc<std::sync::Mutex<wzp_proto::QualityProfile>>,
|
||||||
|
/// Peer's reported max quality cap. The encoder clamps to min(active, peer_max).
|
||||||
|
peer_max_quality: Arc<std::sync::Mutex<Option<wzp_proto::QualityProfile>>>,
|
||||||
|
/// Pending outgoing upgrade proposal: (call_id, proposal_id, profile).
|
||||||
|
pending_upgrade: Arc<std::sync::Mutex<Option<(String, String, wzp_proto::QualityProfile)>>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Parsed data from a peer's HardNatBirthdayStart signal.
|
/// Parsed data from a peer's HardNatBirthdayStart signal.
|
||||||
@@ -1706,8 +2211,11 @@ fn do_register_signal(
|
|||||||
"peer_loss_pct": local_loss_pct, "peer_rtt_ms": local_rtt_ms,
|
"peer_loss_pct": local_loss_pct, "peer_rtt_ms": local_rtt_ms,
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
// TODO: auto-accept if our own quality supports it,
|
if let Err(e) =
|
||||||
// or surface to UI for manual accept/reject
|
handle_upgrade_proposal(&*transport, &call_id, &proposal_id).await
|
||||||
|
{
|
||||||
|
tracing::warn!("failed to send UpgradeResponse: {e}");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Ok(Some(SignalMessage::UpgradeResponse {
|
Ok(Some(SignalMessage::UpgradeResponse {
|
||||||
call_id,
|
call_id,
|
||||||
@@ -1725,7 +2233,17 @@ fn do_register_signal(
|
|||||||
"accepted": accepted, "reason": reason,
|
"accepted": accepted, "reason": reason,
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
// TODO: if accepted, send UpgradeConfirm + switch encoder
|
if let Err(e) = handle_upgrade_response(
|
||||||
|
&*transport,
|
||||||
|
&signal_state,
|
||||||
|
&call_id,
|
||||||
|
&proposal_id,
|
||||||
|
accepted,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
tracing::warn!("failed to handle UpgradeResponse: {e}");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Ok(Some(SignalMessage::UpgradeConfirm {
|
Ok(Some(SignalMessage::UpgradeConfirm {
|
||||||
call_id,
|
call_id,
|
||||||
@@ -1742,7 +2260,7 @@ fn do_register_signal(
|
|||||||
"confirmed_profile": format!("{confirmed_profile:?}"),
|
"confirmed_profile": format!("{confirmed_profile:?}"),
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
// TODO: switch encoder to confirmed_profile at next frame boundary
|
handle_upgrade_confirm(&signal_state, confirmed_profile).await;
|
||||||
}
|
}
|
||||||
Ok(Some(SignalMessage::QualityCapability {
|
Ok(Some(SignalMessage::QualityCapability {
|
||||||
call_id,
|
call_id,
|
||||||
@@ -1761,8 +2279,7 @@ fn do_register_signal(
|
|||||||
"peer_loss_pct": loss_pct, "peer_rtt_ms": rtt_ms,
|
"peer_loss_pct": loss_pct, "peer_rtt_ms": rtt_ms,
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
// TODO: adjust our encoder to not exceed peer's max_profile
|
handle_quality_capability(&signal_state, max_profile).await;
|
||||||
// (asymmetric quality — each side encodes at its own best)
|
|
||||||
}
|
}
|
||||||
Ok(Some(SignalMessage::HardNatBirthdayStart {
|
Ok(Some(SignalMessage::HardNatBirthdayStart {
|
||||||
call_id,
|
call_id,
|
||||||
@@ -2117,8 +2634,13 @@ async fn place_call(
|
|||||||
.map(|la| la.port())
|
.map(|la| la.port())
|
||||||
.unwrap_or(0);
|
.unwrap_or(0);
|
||||||
if v4_port > 0 {
|
if v4_port > 0 {
|
||||||
match wzp_client::portmap::acquire_port_mapping(v4_port, None).await {
|
match tokio::time::timeout(
|
||||||
Ok(mapping) => {
|
std::time::Duration::from_millis(750),
|
||||||
|
wzp_client::portmap::acquire_port_mapping(v4_port, None),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(Ok(mapping)) => {
|
||||||
let addr = mapping.external_addr.to_string();
|
let addr = mapping.external_addr.to_string();
|
||||||
tracing::info!(%addr, protocol = ?mapping.protocol, "place_call: port mapping acquired");
|
tracing::info!(%addr, protocol = ?mapping.protocol, "place_call: port mapping acquired");
|
||||||
emit_call_debug(
|
emit_call_debug(
|
||||||
@@ -2130,10 +2652,19 @@ async fn place_call(
|
|||||||
);
|
);
|
||||||
Some(addr)
|
Some(addr)
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Ok(Err(e)) => {
|
||||||
tracing::debug!(error = %e, "place_call: port mapping unavailable (normal on most networks)");
|
tracing::debug!(error = %e, "place_call: port mapping unavailable (normal on most networks)");
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
Err(_) => {
|
||||||
|
tracing::debug!("place_call: port mapping quick probe timed out");
|
||||||
|
emit_call_debug(
|
||||||
|
&app,
|
||||||
|
"place_call:portmap_timeout",
|
||||||
|
serde_json::json!({ "timeout_ms": 750 }),
|
||||||
|
);
|
||||||
|
None
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
@@ -2360,8 +2891,13 @@ async fn answer_call(
|
|||||||
.map(|la| la.port())
|
.map(|la| la.port())
|
||||||
.unwrap_or(0);
|
.unwrap_or(0);
|
||||||
if v4_port > 0 {
|
if v4_port > 0 {
|
||||||
match wzp_client::portmap::acquire_port_mapping(v4_port, None).await {
|
match tokio::time::timeout(
|
||||||
Ok(mapping) => {
|
std::time::Duration::from_millis(750),
|
||||||
|
wzp_client::portmap::acquire_port_mapping(v4_port, None),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(Ok(mapping)) => {
|
||||||
tracing::info!(
|
tracing::info!(
|
||||||
addr = %mapping.external_addr,
|
addr = %mapping.external_addr,
|
||||||
protocol = ?mapping.protocol,
|
protocol = ?mapping.protocol,
|
||||||
@@ -2369,10 +2905,19 @@ async fn answer_call(
|
|||||||
);
|
);
|
||||||
Some(mapping.external_addr.to_string())
|
Some(mapping.external_addr.to_string())
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Ok(Err(e)) => {
|
||||||
tracing::debug!(error = %e, "answer_call: port mapping unavailable");
|
tracing::debug!(error = %e, "answer_call: port mapping unavailable");
|
||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
Err(_) => {
|
||||||
|
tracing::debug!("answer_call: port mapping quick probe timed out");
|
||||||
|
emit_call_debug(
|
||||||
|
&app,
|
||||||
|
"answer_call:portmap_timeout",
|
||||||
|
serde_json::json!({ "timeout_ms": 750 }),
|
||||||
|
);
|
||||||
|
None
|
||||||
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
@@ -2491,7 +3036,7 @@ async fn answer_call(
|
|||||||
/// or temporarily unreachable for reflect but the call can still
|
/// or temporarily unreachable for reflect but the call can still
|
||||||
/// proceed with STUN-discovered addresses.
|
/// proceed with STUN-discovered addresses.
|
||||||
async fn try_reflect_own_addr(state: &Arc<AppState>) -> Result<Option<String>, String> {
|
async fn try_reflect_own_addr(state: &Arc<AppState>) -> Result<Option<String>, String> {
|
||||||
use wzp_proto::{SignalMessage, default_signal_version};
|
use wzp_proto::SignalMessage;
|
||||||
let (tx, rx) = tokio::sync::oneshot::channel::<std::net::SocketAddr>();
|
let (tx, rx) = tokio::sync::oneshot::channel::<std::net::SocketAddr>();
|
||||||
let transport = {
|
let transport = {
|
||||||
let mut sig = state.signal.lock().await;
|
let mut sig = state.signal.lock().await;
|
||||||
@@ -2578,7 +3123,7 @@ async fn try_stun_fallback(state: &Arc<AppState>) -> Result<Option<String>, Stri
|
|||||||
/// with `new URL(...)` / a regex if needed.
|
/// with `new URL(...)` / a regex if needed.
|
||||||
#[tauri::command]
|
#[tauri::command]
|
||||||
async fn get_reflected_address(state: tauri::State<'_, Arc<AppState>>) -> Result<String, String> {
|
async fn get_reflected_address(state: tauri::State<'_, Arc<AppState>>) -> Result<String, String> {
|
||||||
use wzp_proto::{SignalMessage, default_signal_version};
|
use wzp_proto::SignalMessage;
|
||||||
let (tx, rx) = tokio::sync::oneshot::channel::<std::net::SocketAddr>();
|
let (tx, rx) = tokio::sync::oneshot::channel::<std::net::SocketAddr>();
|
||||||
let transport = {
|
let transport = {
|
||||||
let mut sig = state.signal.lock().await;
|
let mut sig = state.signal.lock().await;
|
||||||
@@ -2836,11 +3381,237 @@ async fn hangup_call(
|
|||||||
|
|
||||||
// ─── App entry point ─────────────────────────────────────────────────────────
|
// ─── App entry point ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
// ─── Quality upgrade flow handlers (testable) ─────────────────────────────
|
||||||
|
|
||||||
|
async fn handle_upgrade_proposal(
|
||||||
|
transport: &dyn wzp_proto::MediaTransport,
|
||||||
|
call_id: &str,
|
||||||
|
proposal_id: &str,
|
||||||
|
) -> Result<(), wzp_proto::TransportError> {
|
||||||
|
let response = wzp_proto::SignalMessage::UpgradeResponse {
|
||||||
|
version: default_signal_version(),
|
||||||
|
call_id: call_id.to_string(),
|
||||||
|
proposal_id: proposal_id.to_string(),
|
||||||
|
accepted: true,
|
||||||
|
reason: None,
|
||||||
|
};
|
||||||
|
transport.send_signal(&response).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_upgrade_response(
|
||||||
|
transport: &dyn wzp_proto::MediaTransport,
|
||||||
|
signal_state: &Arc<tokio::sync::Mutex<SignalState>>,
|
||||||
|
call_id: &str,
|
||||||
|
proposal_id: &str,
|
||||||
|
accepted: bool,
|
||||||
|
) -> Result<(), wzp_proto::TransportError> {
|
||||||
|
if accepted {
|
||||||
|
let maybe_proposal = {
|
||||||
|
let sig = signal_state.lock().await;
|
||||||
|
sig.pending_upgrade.lock().unwrap().take()
|
||||||
|
};
|
||||||
|
if let Some((_cid, pid, profile)) = maybe_proposal {
|
||||||
|
if pid == proposal_id {
|
||||||
|
let confirm = wzp_proto::SignalMessage::UpgradeConfirm {
|
||||||
|
version: default_signal_version(),
|
||||||
|
call_id: call_id.to_string(),
|
||||||
|
proposal_id: proposal_id.to_string(),
|
||||||
|
confirmed_profile: profile.clone(),
|
||||||
|
};
|
||||||
|
transport.send_signal(&confirm).await?;
|
||||||
|
{
|
||||||
|
let sig = signal_state.lock().await;
|
||||||
|
*sig.active_quality.lock().unwrap() = profile;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_upgrade_confirm(
|
||||||
|
signal_state: &Arc<tokio::sync::Mutex<SignalState>>,
|
||||||
|
confirmed_profile: wzp_proto::QualityProfile,
|
||||||
|
) {
|
||||||
|
let sig = signal_state.lock().await;
|
||||||
|
*sig.active_quality.lock().unwrap() = confirmed_profile;
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_quality_capability(
|
||||||
|
signal_state: &Arc<tokio::sync::Mutex<SignalState>>,
|
||||||
|
max_profile: wzp_proto::QualityProfile,
|
||||||
|
) {
|
||||||
|
let sig = signal_state.lock().await;
|
||||||
|
*sig.peer_max_quality.lock().unwrap() = Some(max_profile);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod signal_tests {
|
||||||
|
use super::*;
|
||||||
|
use async_trait::async_trait;
|
||||||
|
use std::sync::Mutex as StdMutex;
|
||||||
|
use wzp_proto::{MediaPacket, MediaTransport, PathQuality, SignalMessage, TransportError};
|
||||||
|
|
||||||
|
struct LoopbackTransport {
|
||||||
|
sent: StdMutex<Vec<SignalMessage>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LoopbackTransport {
|
||||||
|
fn new() -> Arc<Self> {
|
||||||
|
Arc::new(Self {
|
||||||
|
sent: StdMutex::new(Vec::new()),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
fn take_sent(&self) -> Vec<SignalMessage> {
|
||||||
|
self.sent.lock().unwrap().drain(..).collect()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl MediaTransport for LoopbackTransport {
|
||||||
|
async fn send_media(&self, _packet: &MediaPacket) -> Result<(), TransportError> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
async fn recv_media(&self) -> Result<Option<MediaPacket>, TransportError> {
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
async fn send_signal(&self, msg: &SignalMessage) -> Result<(), TransportError> {
|
||||||
|
self.sent.lock().unwrap().push(msg.clone());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
async fn recv_signal(&self) -> Result<Option<SignalMessage>, TransportError> {
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
fn path_quality(&self) -> PathQuality {
|
||||||
|
PathQuality::default()
|
||||||
|
}
|
||||||
|
async fn close(&self) -> Result<(), TransportError> {
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn empty_signal_state() -> Arc<tokio::sync::Mutex<SignalState>> {
|
||||||
|
Arc::new(tokio::sync::Mutex::new(SignalState {
|
||||||
|
transport: None,
|
||||||
|
endpoint: None,
|
||||||
|
ipv6_endpoint: None,
|
||||||
|
fingerprint: String::new(),
|
||||||
|
signal_status: "idle".into(),
|
||||||
|
incoming_call_id: None,
|
||||||
|
incoming_caller_fp: None,
|
||||||
|
incoming_caller_alias: None,
|
||||||
|
pending_reflect: None,
|
||||||
|
own_reflex_addr: None,
|
||||||
|
desired_relay_addr: None,
|
||||||
|
reconnect_in_progress: false,
|
||||||
|
pending_path_report: None,
|
||||||
|
peer_hard_nat_probe: None,
|
||||||
|
peer_birthday_ports: None,
|
||||||
|
active_quality: Arc::new(std::sync::Mutex::new(wzp_proto::QualityProfile::GOOD)),
|
||||||
|
peer_max_quality: Arc::new(std::sync::Mutex::new(None)),
|
||||||
|
pending_upgrade: Arc::new(std::sync::Mutex::new(None)),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn upgrade_proposal_auto_accepts() {
|
||||||
|
let transport = LoopbackTransport::new();
|
||||||
|
handle_upgrade_proposal(&*transport, "c1", "p1")
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let sent = transport.take_sent();
|
||||||
|
assert_eq!(sent.len(), 1);
|
||||||
|
match &sent[0] {
|
||||||
|
SignalMessage::UpgradeResponse {
|
||||||
|
call_id,
|
||||||
|
proposal_id,
|
||||||
|
accepted,
|
||||||
|
reason,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
|
assert_eq!(call_id, "c1");
|
||||||
|
assert_eq!(proposal_id, "p1");
|
||||||
|
assert!(accepted);
|
||||||
|
assert!(reason.is_none());
|
||||||
|
}
|
||||||
|
other => panic!("expected UpgradeResponse, got {other:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn upgrade_response_accepted_sends_confirm_and_updates_quality() {
|
||||||
|
let transport = LoopbackTransport::new();
|
||||||
|
let signal_state = empty_signal_state();
|
||||||
|
{
|
||||||
|
let sig = signal_state.lock().await;
|
||||||
|
*sig.pending_upgrade.lock().unwrap() = Some((
|
||||||
|
"c1".into(),
|
||||||
|
"p1".into(),
|
||||||
|
wzp_proto::QualityProfile::STUDIO_48K,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
handle_upgrade_response(&*transport, &signal_state, "c1", "p1", true)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let sent = transport.take_sent();
|
||||||
|
assert_eq!(sent.len(), 1);
|
||||||
|
match &sent[0] {
|
||||||
|
SignalMessage::UpgradeConfirm {
|
||||||
|
call_id,
|
||||||
|
proposal_id,
|
||||||
|
confirmed_profile,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
|
assert_eq!(call_id, "c1");
|
||||||
|
assert_eq!(proposal_id, "p1");
|
||||||
|
assert_eq!(*confirmed_profile, wzp_proto::QualityProfile::STUDIO_48K);
|
||||||
|
}
|
||||||
|
other => panic!("expected UpgradeConfirm, got {other:?}"),
|
||||||
|
}
|
||||||
|
|
||||||
|
let sig = signal_state.lock().await;
|
||||||
|
assert_eq!(
|
||||||
|
*sig.active_quality.lock().unwrap(),
|
||||||
|
wzp_proto::QualityProfile::STUDIO_48K
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn upgrade_confirm_updates_active_quality() {
|
||||||
|
let signal_state = empty_signal_state();
|
||||||
|
handle_upgrade_confirm(&signal_state, wzp_proto::QualityProfile::STUDIO_64K).await;
|
||||||
|
|
||||||
|
let sig = signal_state.lock().await;
|
||||||
|
assert_eq!(
|
||||||
|
*sig.active_quality.lock().unwrap(),
|
||||||
|
wzp_proto::QualityProfile::STUDIO_64K
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn quality_capability_updates_peer_max() {
|
||||||
|
let signal_state = empty_signal_state();
|
||||||
|
handle_quality_capability(&signal_state, wzp_proto::QualityProfile::GOOD).await;
|
||||||
|
|
||||||
|
let sig = signal_state.lock().await;
|
||||||
|
assert_eq!(
|
||||||
|
sig.peer_max_quality.lock().unwrap().unwrap(),
|
||||||
|
wzp_proto::QualityProfile::GOOD
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Shared Tauri app builder. Used by the desktop `main.rs` and the mobile
|
/// Shared Tauri app builder. Used by the desktop `main.rs` and the mobile
|
||||||
/// entry point below.
|
/// entry point below.
|
||||||
pub fn run() {
|
pub fn run() {
|
||||||
tracing_subscriber::fmt().init();
|
tracing_subscriber::fmt().init();
|
||||||
|
|
||||||
|
let active_quality = Arc::new(std::sync::Mutex::new(wzp_proto::QualityProfile::GOOD));
|
||||||
|
let peer_max_quality = Arc::new(std::sync::Mutex::new(None));
|
||||||
|
let pending_upgrade = Arc::new(std::sync::Mutex::new(None));
|
||||||
let state = Arc::new(AppState {
|
let state = Arc::new(AppState {
|
||||||
engine: Mutex::new(None),
|
engine: Mutex::new(None),
|
||||||
signal: Arc::new(Mutex::new(SignalState {
|
signal: Arc::new(Mutex::new(SignalState {
|
||||||
@@ -2859,6 +3630,9 @@ pub fn run() {
|
|||||||
pending_path_report: None,
|
pending_path_report: None,
|
||||||
peer_hard_nat_probe: None,
|
peer_hard_nat_probe: None,
|
||||||
peer_birthday_ports: None,
|
peer_birthday_ports: None,
|
||||||
|
active_quality: active_quality.clone(),
|
||||||
|
peer_max_quality: peer_max_quality.clone(),
|
||||||
|
pending_upgrade: pending_upgrade.clone(),
|
||||||
})),
|
})),
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -2935,6 +3709,8 @@ pub fn run() {
|
|||||||
get_dred_verbose_logs,
|
get_dred_verbose_logs,
|
||||||
set_call_debug_logs,
|
set_call_debug_logs,
|
||||||
get_call_debug_logs,
|
get_call_debug_logs,
|
||||||
|
call_debug_log,
|
||||||
|
push_camera_frame,
|
||||||
])
|
])
|
||||||
.run(tauri::generate_context!())
|
.run(tauri::generate_context!())
|
||||||
.expect("error while running WarzonePhone");
|
.expect("error while running WarzonePhone");
|
||||||
|
|||||||
@@ -62,6 +62,7 @@ const lobbyFp = document.getElementById("lobby-fp")!;
|
|||||||
const lobbyUserList = document.getElementById("lobby-user-list")!;
|
const lobbyUserList = document.getElementById("lobby-user-list")!;
|
||||||
const lobbyUserCount = document.getElementById("lobby-user-count")!;
|
const lobbyUserCount = document.getElementById("lobby-user-count")!;
|
||||||
const joinVoiceBtn = document.getElementById("join-voice-btn")!;
|
const joinVoiceBtn = document.getElementById("join-voice-btn")!;
|
||||||
|
const joinVideoBtn = document.getElementById("join-video-btn")!;
|
||||||
const incomingBanner = document.getElementById("incoming-call-banner")!;
|
const incomingBanner = document.getElementById("incoming-call-banner")!;
|
||||||
const incomingCallerName = document.getElementById("incoming-caller-name")!;
|
const incomingCallerName = document.getElementById("incoming-caller-name")!;
|
||||||
const incomingIdenticon = document.getElementById("incoming-identicon")!;
|
const incomingIdenticon = document.getElementById("incoming-identicon")!;
|
||||||
@@ -79,6 +80,11 @@ const vdMicIcon = document.getElementById("vd-mic-icon")!;
|
|||||||
const vdSpkBtn = document.getElementById("vd-spk-btn")!;
|
const vdSpkBtn = document.getElementById("vd-spk-btn")!;
|
||||||
const vdSpkIcon = document.getElementById("vd-spk-icon")!;
|
const vdSpkIcon = document.getElementById("vd-spk-icon")!;
|
||||||
const vdEndBtn = document.getElementById("vd-end-btn")!;
|
const vdEndBtn = document.getElementById("vd-end-btn")!;
|
||||||
|
const vdCamBtn = document.getElementById("vd-cam-btn")!;
|
||||||
|
const vdCamIcon = document.getElementById("vd-cam-icon")!;
|
||||||
|
const vdVideoStrip = document.getElementById("vd-video-strip")!;
|
||||||
|
const vdRemoteVideo = document.getElementById("vd-remote-video") as HTMLCanvasElement;
|
||||||
|
const vdLocalVideo = document.getElementById("vd-local-video") as HTMLVideoElement;
|
||||||
const vdDirectInfo = document.getElementById("vd-direct-info")!;
|
const vdDirectInfo = document.getElementById("vd-direct-info")!;
|
||||||
const vdDcIdenticon = document.getElementById("vd-dc-identicon")!;
|
const vdDcIdenticon = document.getElementById("vd-dc-identicon")!;
|
||||||
const vdDcName = document.getElementById("vd-dc-name")!;
|
const vdDcName = document.getElementById("vd-dc-name")!;
|
||||||
@@ -116,6 +122,8 @@ const sCallDebugCopyBtn = document.getElementById("s-call-debug-copy") as HTMLBu
|
|||||||
const sCallDebugShareBtn = document.getElementById("s-call-debug-share") as HTMLButtonElement;
|
const sCallDebugShareBtn = document.getElementById("s-call-debug-share") as HTMLButtonElement;
|
||||||
const sQuality = document.getElementById("s-quality") as HTMLInputElement;
|
const sQuality = document.getElementById("s-quality") as HTMLInputElement;
|
||||||
const sQualityLabel = document.getElementById("s-quality-label")!;
|
const sQualityLabel = document.getElementById("s-quality-label")!;
|
||||||
|
const sVideoCodec = document.getElementById("s-video-codec") as HTMLSelectElement;
|
||||||
|
const sVideoResolution = document.getElementById("s-video-resolution") as HTMLSelectElement;
|
||||||
const sFingerprint = document.getElementById("s-fingerprint")!;
|
const sFingerprint = document.getElementById("s-fingerprint")!;
|
||||||
const sPublicAddr = document.getElementById("s-public-addr")!;
|
const sPublicAddr = document.getElementById("s-public-addr")!;
|
||||||
const sReflectBtn = document.getElementById("s-reflect-btn")!;
|
const sReflectBtn = document.getElementById("s-reflect-btn")!;
|
||||||
@@ -132,6 +140,8 @@ interface Settings {
|
|||||||
alias: string;
|
alias: string;
|
||||||
osAec: boolean;
|
osAec: boolean;
|
||||||
quality: string;
|
quality: string;
|
||||||
|
videoCodec: string;
|
||||||
|
videoResolution: string;
|
||||||
recentRooms: RecentRoom[];
|
recentRooms: RecentRoom[];
|
||||||
dredDebugLogs: boolean;
|
dredDebugLogs: boolean;
|
||||||
callDebugLogs: boolean;
|
callDebugLogs: boolean;
|
||||||
@@ -145,7 +155,7 @@ function loadSettings(): Settings {
|
|||||||
{ name: "Default", address: "193.180.213.68:4433" },
|
{ name: "Default", address: "193.180.213.68:4433" },
|
||||||
],
|
],
|
||||||
selectedRelay: 0, room: "general", alias: "",
|
selectedRelay: 0, room: "general", alias: "",
|
||||||
osAec: true, quality: "auto", recentRooms: [],
|
osAec: true, quality: "auto", videoCodec: "h264", videoResolution: "1280x720", recentRooms: [],
|
||||||
dredDebugLogs: false, callDebugLogs: false,
|
dredDebugLogs: false, callDebugLogs: false,
|
||||||
directOnly: false, birthdayAttack: false,
|
directOnly: false, birthdayAttack: false,
|
||||||
};
|
};
|
||||||
@@ -158,6 +168,25 @@ function loadSettings(): Settings {
|
|||||||
function saveSettings(s: Settings) {
|
function saveSettings(s: Settings) {
|
||||||
localStorage.setItem("wzp-settings", JSON.stringify(s));
|
localStorage.setItem("wzp-settings", JSON.stringify(s));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function parseVideoResolution(value: string) {
|
||||||
|
const [wRaw, hRaw] = (value || "1280x720").split("x");
|
||||||
|
const width = Number.parseInt(wRaw, 10);
|
||||||
|
const height = Number.parseInt(hRaw, 10);
|
||||||
|
if (!Number.isFinite(width) || !Number.isFinite(height)) {
|
||||||
|
return { width: 1280, height: 720 };
|
||||||
|
}
|
||||||
|
return { width, height };
|
||||||
|
}
|
||||||
|
|
||||||
|
function videoConnectOptions(s: Settings) {
|
||||||
|
const { width, height } = parseVideoResolution(s.videoResolution);
|
||||||
|
return {
|
||||||
|
videoCodec: s.videoCodec || "h264",
|
||||||
|
videoWidth: width,
|
||||||
|
videoHeight: height,
|
||||||
|
};
|
||||||
|
}
|
||||||
function getRelay(): RelayServer | null {
|
function getRelay(): RelayServer | null {
|
||||||
const s = loadSettings();
|
const s = loadSettings();
|
||||||
return s.relays[s.selectedRelay] || s.relays[0] || null;
|
return s.relays[s.selectedRelay] || s.relays[0] || null;
|
||||||
@@ -166,9 +195,140 @@ function getRelay(): RelayServer | null {
|
|||||||
let myFingerprint = "";
|
let myFingerprint = "";
|
||||||
let statusInterval: number | null = null;
|
let statusInterval: number | null = null;
|
||||||
let inVoice = false;
|
let inVoice = false;
|
||||||
|
let connectPending = false; // guard against double-tap while connect is in-flight
|
||||||
let directCallPeer: { fingerprint: string; alias: string | null } | null = null;
|
let directCallPeer: { fingerprint: string; alias: string | null } | null = null;
|
||||||
let pendingCallId: string | null = null;
|
let pendingCallId: string | null = null;
|
||||||
|
|
||||||
|
// Video / camera state
|
||||||
|
let cameraActive = false;
|
||||||
|
let cameraStream: MediaStream | null = null;
|
||||||
|
let cameraFrameTimer: number | null = null;
|
||||||
|
let cameraFrameCallbackHandle: number | null = null;
|
||||||
|
let cameraCaptureInFlight = false;
|
||||||
|
let lastCameraCaptureAtMs = 0;
|
||||||
|
let remoteVideoActive = false;
|
||||||
|
|
||||||
|
interface FrameCallbackVideoElement extends HTMLVideoElement {
|
||||||
|
requestVideoFrameCallback?: (callback: (now: DOMHighResTimeStamp, metadata: unknown) => void) => number;
|
||||||
|
cancelVideoFrameCallback?: (handle: number) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keep the local preview out of the video stage stacking context so it can float
|
||||||
|
// above the call drawer and remain draggable on phones.
|
||||||
|
document.body.appendChild(vdLocalVideo);
|
||||||
|
vdLocalVideo.classList.add("hidden");
|
||||||
|
|
||||||
|
function clampNumber(value: number, min: number, max: number) {
|
||||||
|
return Math.min(Math.max(value, min), max);
|
||||||
|
}
|
||||||
|
|
||||||
|
function keepLocalPipInViewport() {
|
||||||
|
if (vdLocalVideo.classList.contains("hidden")) return;
|
||||||
|
const rect = vdLocalVideo.getBoundingClientRect();
|
||||||
|
if (!rect.width || !rect.height) return;
|
||||||
|
const margin = 12;
|
||||||
|
const maxLeft = Math.max(margin, window.innerWidth - rect.width - margin);
|
||||||
|
const maxTop = Math.max(margin, window.innerHeight - rect.height - margin);
|
||||||
|
const left = clampNumber(rect.left, margin, maxLeft);
|
||||||
|
const top = clampNumber(rect.top, margin, maxTop);
|
||||||
|
vdLocalVideo.style.left = `${left}px`;
|
||||||
|
vdLocalVideo.style.top = `${top}px`;
|
||||||
|
vdLocalVideo.style.right = "auto";
|
||||||
|
vdLocalVideo.style.bottom = "auto";
|
||||||
|
}
|
||||||
|
|
||||||
|
function initLocalPipDrag() {
|
||||||
|
let dragPointerId: number | null = null;
|
||||||
|
let dragOffsetX = 0;
|
||||||
|
let dragOffsetY = 0;
|
||||||
|
|
||||||
|
vdLocalVideo.addEventListener("pointerdown", (event) => {
|
||||||
|
if (vdLocalVideo.classList.contains("hidden")) return;
|
||||||
|
dragPointerId = event.pointerId;
|
||||||
|
const rect = vdLocalVideo.getBoundingClientRect();
|
||||||
|
dragOffsetX = event.clientX - rect.left;
|
||||||
|
dragOffsetY = event.clientY - rect.top;
|
||||||
|
vdLocalVideo.classList.add("dragging");
|
||||||
|
vdLocalVideo.setPointerCapture(event.pointerId);
|
||||||
|
event.preventDefault();
|
||||||
|
});
|
||||||
|
|
||||||
|
vdLocalVideo.addEventListener("pointermove", (event) => {
|
||||||
|
if (dragPointerId !== event.pointerId) return;
|
||||||
|
const rect = vdLocalVideo.getBoundingClientRect();
|
||||||
|
const margin = 12;
|
||||||
|
const maxLeft = Math.max(margin, window.innerWidth - rect.width - margin);
|
||||||
|
const maxTop = Math.max(margin, window.innerHeight - rect.height - margin);
|
||||||
|
const left = clampNumber(event.clientX - dragOffsetX, margin, maxLeft);
|
||||||
|
const top = clampNumber(event.clientY - dragOffsetY, margin, maxTop);
|
||||||
|
vdLocalVideo.style.left = `${left}px`;
|
||||||
|
vdLocalVideo.style.top = `${top}px`;
|
||||||
|
vdLocalVideo.style.right = "auto";
|
||||||
|
vdLocalVideo.style.bottom = "auto";
|
||||||
|
event.preventDefault();
|
||||||
|
});
|
||||||
|
|
||||||
|
function endDrag(event: PointerEvent) {
|
||||||
|
if (dragPointerId !== event.pointerId) return;
|
||||||
|
dragPointerId = null;
|
||||||
|
vdLocalVideo.classList.remove("dragging");
|
||||||
|
try { vdLocalVideo.releasePointerCapture(event.pointerId); } catch {}
|
||||||
|
}
|
||||||
|
|
||||||
|
vdLocalVideo.addEventListener("pointerup", endDrag);
|
||||||
|
vdLocalVideo.addEventListener("pointercancel", endDrag);
|
||||||
|
window.addEventListener("resize", keepLocalPipInViewport);
|
||||||
|
}
|
||||||
|
|
||||||
|
initLocalPipDrag();
|
||||||
|
|
||||||
|
function showToast(msg: string, durationMs = 3500) {
|
||||||
|
let el = document.getElementById("wzp-toast");
|
||||||
|
if (!el) {
|
||||||
|
el = document.createElement("div");
|
||||||
|
el.id = "wzp-toast";
|
||||||
|
el.style.cssText = "position:fixed;bottom:80px;left:50%;transform:translateX(-50%);" +
|
||||||
|
"background:#1e1e2e;color:#cdd6f4;border:1px solid #45475a;border-radius:8px;" +
|
||||||
|
"padding:10px 18px;font-size:13px;z-index:9999;pointer-events:none;opacity:0;transition:opacity .2s";
|
||||||
|
document.body.appendChild(el);
|
||||||
|
}
|
||||||
|
el.textContent = msg;
|
||||||
|
el.style.opacity = "1";
|
||||||
|
clearTimeout((el as any)._timer);
|
||||||
|
(el as any)._timer = setTimeout(() => { el!.style.opacity = "0"; }, durationMs);
|
||||||
|
}
|
||||||
|
|
||||||
|
function errorMessage(e: unknown): string {
|
||||||
|
if (typeof e === "string") return e;
|
||||||
|
if (e && typeof e === "object" && "message" in e) {
|
||||||
|
const msg = (e as { message?: unknown }).message;
|
||||||
|
if (typeof msg === "string") return msg;
|
||||||
|
}
|
||||||
|
return String(e);
|
||||||
|
}
|
||||||
|
|
||||||
|
function connectDebugSummary(entry: CallDebugEntry | null): string {
|
||||||
|
if (!entry) return "no native connect event received";
|
||||||
|
const details = entry.details && typeof entry.details === "object"
|
||||||
|
? JSON.stringify(entry.details)
|
||||||
|
: String(entry.details ?? "");
|
||||||
|
return `${entry.step}${details ? ` ${details}` : ""}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
let lastConnectDebug: CallDebugEntry | null = null;
|
||||||
|
|
||||||
|
function connectWithTimeout(args: Record<string, unknown>, timeoutMs = 45000) {
|
||||||
|
lastConnectDebug = null;
|
||||||
|
return Promise.race([
|
||||||
|
invoke("connect", args),
|
||||||
|
new Promise<never>((_, reject) =>
|
||||||
|
setTimeout(() => reject(new Error(
|
||||||
|
`connect timed out (${Math.round(timeoutMs / 1000)}s); last native step: ${connectDebugSummary(lastConnectDebug)}`
|
||||||
|
)), timeoutMs)
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
// Known users in the room (from RoomUpdate or signal presence)
|
// Known users in the room (from RoomUpdate or signal presence)
|
||||||
interface LobbyUser {
|
interface LobbyUser {
|
||||||
fingerprint: string;
|
fingerprint: string;
|
||||||
@@ -186,6 +346,7 @@ const CALL_DEBUG_MAX = 200;
|
|||||||
listen("call-debug-log", (event: any) => {
|
listen("call-debug-log", (event: any) => {
|
||||||
const entry: CallDebugEntry = event.payload;
|
const entry: CallDebugEntry = event.payload;
|
||||||
callDebugBuffer.push(entry);
|
callDebugBuffer.push(entry);
|
||||||
|
if (entry.step?.startsWith("connect:")) lastConnectDebug = entry;
|
||||||
if (callDebugBuffer.length > CALL_DEBUG_MAX) callDebugBuffer.shift();
|
if (callDebugBuffer.length > CALL_DEBUG_MAX) callDebugBuffer.shift();
|
||||||
renderCallDebugLog();
|
renderCallDebugLog();
|
||||||
});
|
});
|
||||||
@@ -202,6 +363,10 @@ function renderCallDebugLog() {
|
|||||||
sCallDebugLogEl.scrollTop = sCallDebugLogEl.scrollHeight;
|
sCallDebugLogEl.scrollTop = sCallDebugLogEl.scrollHeight;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function debugLog(step: string, details: any = {}) {
|
||||||
|
invoke("call_debug_log", { step, details }).catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
// ── Quality slider ────────────────────────────────────────────────
|
// ── Quality slider ────────────────────────────────────────────────
|
||||||
const QUALITY_STEPS = ["studio-64k", "studio-48k", "studio-32k", "auto", "good", "degraded", "codec2-3200", "catastrophic"];
|
const QUALITY_STEPS = ["studio-64k", "studio-48k", "studio-32k", "auto", "good", "degraded", "codec2-3200", "catastrophic"];
|
||||||
const QUALITY_LABELS = ["Studio 64k", "Studio 48k", "Studio 32k", "Auto", "Opus 24k", "Opus 6k", "Codec2 3.2k", "Codec2 1.2k"];
|
const QUALITY_LABELS = ["Studio 64k", "Studio 48k", "Studio 32k", "Auto", "Opus 24k", "Opus 6k", "Codec2 3.2k", "Codec2 1.2k"];
|
||||||
@@ -309,21 +474,61 @@ ctxCallBtn.addEventListener("click", async () => {
|
|||||||
|
|
||||||
// ── Voice join/leave (drawer-based) ───────────────────────────────
|
// ── Voice join/leave (drawer-based) ───────────────────────────────
|
||||||
joinVoiceBtn.addEventListener("click", async () => {
|
joinVoiceBtn.addEventListener("click", async () => {
|
||||||
if (inVoice) return;
|
if (inVoice || connectPending) return;
|
||||||
const relay = getRelay();
|
const relay = getRelay();
|
||||||
const s = loadSettings();
|
const s = loadSettings();
|
||||||
if (!relay) return;
|
if (!relay) { showToast("No relay configured"); return; }
|
||||||
|
connectPending = true;
|
||||||
|
const origText = joinVoiceBtn.textContent;
|
||||||
|
joinVoiceBtn.textContent = "Connecting…";
|
||||||
|
(joinVoiceBtn as HTMLButtonElement).disabled = true;
|
||||||
try {
|
try {
|
||||||
await invoke("connect", {
|
await connectWithTimeout({
|
||||||
relay: relay.address,
|
relay: relay.address,
|
||||||
room: s.room || "general",
|
room: s.room || "general",
|
||||||
alias: s.alias || "",
|
alias: s.alias || "",
|
||||||
osAec: s.osAec,
|
osAec: s.osAec,
|
||||||
quality: s.quality || "auto",
|
quality: s.quality || "auto",
|
||||||
|
...videoConnectOptions(s),
|
||||||
});
|
});
|
||||||
enterVoice(false);
|
enterVoice(false);
|
||||||
} catch (e: any) {
|
} catch (e: any) {
|
||||||
console.error("connect failed:", e);
|
console.error("connect failed:", e);
|
||||||
|
showToast(`Join failed: ${errorMessage(e)}`);
|
||||||
|
} finally {
|
||||||
|
connectPending = false;
|
||||||
|
joinVoiceBtn.textContent = origText;
|
||||||
|
(joinVoiceBtn as HTMLButtonElement).disabled = false;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
joinVideoBtn.addEventListener("click", async () => {
|
||||||
|
if (inVoice || connectPending) return;
|
||||||
|
const relay = getRelay();
|
||||||
|
const s = loadSettings();
|
||||||
|
if (!relay) { showToast("No relay configured"); return; }
|
||||||
|
connectPending = true;
|
||||||
|
const origText = joinVideoBtn.textContent;
|
||||||
|
joinVideoBtn.textContent = "Connecting…";
|
||||||
|
(joinVideoBtn as HTMLButtonElement).disabled = true;
|
||||||
|
try {
|
||||||
|
await connectWithTimeout({
|
||||||
|
relay: relay.address,
|
||||||
|
room: s.room || "general",
|
||||||
|
alias: s.alias || "",
|
||||||
|
osAec: s.osAec,
|
||||||
|
quality: s.quality || "auto",
|
||||||
|
...videoConnectOptions(s),
|
||||||
|
});
|
||||||
|
enterVoice(false);
|
||||||
|
startCamera();
|
||||||
|
} catch (e: any) {
|
||||||
|
console.error("connect failed:", e);
|
||||||
|
showToast(`Join failed: ${errorMessage(e)}`);
|
||||||
|
} finally {
|
||||||
|
connectPending = false;
|
||||||
|
joinVideoBtn.textContent = origText;
|
||||||
|
(joinVideoBtn as HTMLButtonElement).disabled = false;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -331,6 +536,7 @@ function enterVoice(isDirect: boolean) {
|
|||||||
inVoice = true;
|
inVoice = true;
|
||||||
const s = loadSettings();
|
const s = loadSettings();
|
||||||
joinVoiceBtn.classList.add("hidden");
|
joinVoiceBtn.classList.add("hidden");
|
||||||
|
joinVideoBtn.classList.add("hidden");
|
||||||
voiceDrawer.classList.remove("hidden");
|
voiceDrawer.classList.remove("hidden");
|
||||||
vdRoom.textContent = isDirect && directCallPeer
|
vdRoom.textContent = isDirect && directCallPeer
|
||||||
? (directCallPeer.alias || directCallPeer.fingerprint.substring(0, 16))
|
? (directCallPeer.alias || directCallPeer.fingerprint.substring(0, 16))
|
||||||
@@ -360,8 +566,17 @@ function leaveVoice() {
|
|||||||
pendingCallId = null;
|
pendingCallId = null;
|
||||||
voiceDrawer.classList.add("hidden");
|
voiceDrawer.classList.add("hidden");
|
||||||
joinVoiceBtn.classList.remove("hidden");
|
joinVoiceBtn.classList.remove("hidden");
|
||||||
|
joinVideoBtn.classList.remove("hidden");
|
||||||
vdLevelBar.style.width = "0%";
|
vdLevelBar.style.width = "0%";
|
||||||
if (statusInterval) { clearInterval(statusInterval); statusInterval = null; }
|
if (statusInterval) { clearInterval(statusInterval); statusInterval = null; }
|
||||||
|
stopCamera();
|
||||||
|
remoteVideoActive = false;
|
||||||
|
remoteFrameCount = 0;
|
||||||
|
remoteFrameSerial++;
|
||||||
|
vdRemoteCounter.textContent = "0 frames received";
|
||||||
|
vdRemotePlaceholder.classList.remove("hidden");
|
||||||
|
vdVideoStrip.classList.add("hidden");
|
||||||
|
remoteCtx.clearRect(0, 0, vdRemoteVideo.width, vdRemoteVideo.height);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Drawer controls
|
// Drawer controls
|
||||||
@@ -377,6 +592,247 @@ vdSpkBtn.addEventListener("click", async () => {
|
|||||||
try { await invoke("toggle_speaker"); } catch {}
|
try { await invoke("toggle_speaker"); } catch {}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// ── Camera (Blocker 4 + 5) ────────────────────────────────────────
|
||||||
|
const camCaptureCanvas = document.createElement("canvas");
|
||||||
|
const camCaptureCtx = camCaptureCanvas.getContext("2d")!;
|
||||||
|
let cameraSendWidth = 1280;
|
||||||
|
let cameraSendHeight = 720;
|
||||||
|
let cameraCaptureFrameNo = 0;
|
||||||
|
let cameraPushFailures = 0;
|
||||||
|
const CAMERA_CAPTURE_INTERVAL_MS = 33; // ≈ 30 fps
|
||||||
|
const CAMERA_JPEG_QUALITY = 0.7;
|
||||||
|
|
||||||
|
function drawCameraFrameForSend() {
|
||||||
|
const vw = vdLocalVideo.videoWidth || camCaptureCanvas.width;
|
||||||
|
const vh = vdLocalVideo.videoHeight || camCaptureCanvas.height;
|
||||||
|
if (!vw || !vh) return;
|
||||||
|
|
||||||
|
const scale = Math.min(cameraSendWidth / vw, cameraSendHeight / vh);
|
||||||
|
const dw = vw * scale;
|
||||||
|
const dh = vh * scale;
|
||||||
|
const dx = (cameraSendWidth - dw) / 2;
|
||||||
|
const dy = (cameraSendHeight - dh) / 2;
|
||||||
|
|
||||||
|
camCaptureCtx.fillStyle = "#000";
|
||||||
|
camCaptureCtx.fillRect(0, 0, cameraSendWidth, cameraSendHeight);
|
||||||
|
camCaptureCtx.drawImage(vdLocalVideo, dx, dy, dw, dh);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function captureAndPushCameraFrame() {
|
||||||
|
if (!cameraActive || cameraCaptureInFlight) return;
|
||||||
|
cameraCaptureInFlight = true;
|
||||||
|
cameraCaptureFrameNo++;
|
||||||
|
try {
|
||||||
|
drawCameraFrameForSend();
|
||||||
|
const dataUrl = camCaptureCanvas.toDataURL("image/jpeg", CAMERA_JPEG_QUALITY);
|
||||||
|
const b64 = dataUrl.slice(dataUrl.indexOf(",") + 1);
|
||||||
|
if (cameraCaptureFrameNo === 1 || cameraCaptureFrameNo % 150 === 0) {
|
||||||
|
debugLog("camera:capture_frame", {
|
||||||
|
frame_no: cameraCaptureFrameNo,
|
||||||
|
width: camCaptureCanvas.width,
|
||||||
|
height: camCaptureCanvas.height,
|
||||||
|
source_width: vdLocalVideo.videoWidth || null,
|
||||||
|
source_height: vdLocalVideo.videoHeight || null,
|
||||||
|
jpeg_b64_len: b64.length,
|
||||||
|
capture_clock: getVideoFrameCallbackApi() ? "video_frame_callback" : "interval",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
await invoke("push_camera_frame", { jpegB64: b64 });
|
||||||
|
} catch (e: any) {
|
||||||
|
cameraPushFailures++;
|
||||||
|
if (cameraPushFailures === 1 || cameraPushFailures % 30 === 0) {
|
||||||
|
debugLog("camera:push_failed", {
|
||||||
|
frame_no: cameraCaptureFrameNo,
|
||||||
|
failures: cameraPushFailures,
|
||||||
|
error: errorMessage(e),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
cameraCaptureInFlight = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function getVideoFrameCallbackApi() {
|
||||||
|
const video = vdLocalVideo as FrameCallbackVideoElement;
|
||||||
|
if (typeof video.requestVideoFrameCallback !== "function") return null;
|
||||||
|
return video;
|
||||||
|
}
|
||||||
|
|
||||||
|
function cancelCameraCaptureLoop() {
|
||||||
|
if (cameraFrameTimer != null) {
|
||||||
|
window.clearInterval(cameraFrameTimer);
|
||||||
|
cameraFrameTimer = null;
|
||||||
|
}
|
||||||
|
const video = getVideoFrameCallbackApi();
|
||||||
|
if (video && cameraFrameCallbackHandle != null && typeof video.cancelVideoFrameCallback === "function") {
|
||||||
|
video.cancelVideoFrameCallback(cameraFrameCallbackHandle);
|
||||||
|
}
|
||||||
|
cameraFrameCallbackHandle = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function scheduleCameraFrameCapture() {
|
||||||
|
cancelCameraCaptureLoop();
|
||||||
|
lastCameraCaptureAtMs = 0;
|
||||||
|
const video = getVideoFrameCallbackApi();
|
||||||
|
if (video) {
|
||||||
|
const onVideoFrame = (now: DOMHighResTimeStamp) => {
|
||||||
|
cameraFrameCallbackHandle = null;
|
||||||
|
if (!cameraActive) return;
|
||||||
|
if (lastCameraCaptureAtMs === 0 || now - lastCameraCaptureAtMs >= CAMERA_CAPTURE_INTERVAL_MS) {
|
||||||
|
lastCameraCaptureAtMs = now;
|
||||||
|
void captureAndPushCameraFrame();
|
||||||
|
}
|
||||||
|
cameraFrameCallbackHandle = video.requestVideoFrameCallback!(onVideoFrame);
|
||||||
|
};
|
||||||
|
cameraFrameCallbackHandle = video.requestVideoFrameCallback(onVideoFrame);
|
||||||
|
debugLog("camera:capture_clock", { mode: "video_frame_callback", interval_ms: CAMERA_CAPTURE_INTERVAL_MS });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
cameraFrameTimer = window.setInterval(() => {
|
||||||
|
void captureAndPushCameraFrame();
|
||||||
|
}, CAMERA_CAPTURE_INTERVAL_MS);
|
||||||
|
debugLog("camera:capture_clock", { mode: "interval", interval_ms: CAMERA_CAPTURE_INTERVAL_MS });
|
||||||
|
}
|
||||||
|
|
||||||
|
async function startCamera() {
|
||||||
|
if (cameraActive) return;
|
||||||
|
const videoSize = parseVideoResolution(loadSettings().videoResolution);
|
||||||
|
cameraSendWidth = videoSize.width;
|
||||||
|
cameraSendHeight = videoSize.height;
|
||||||
|
const constraints = {
|
||||||
|
video: { width: { ideal: cameraSendWidth }, height: { ideal: cameraSendHeight }, facingMode: "user" },
|
||||||
|
audio: false,
|
||||||
|
};
|
||||||
|
debugLog("camera:get_user_media_start", { constraints });
|
||||||
|
try {
|
||||||
|
cameraStream = await navigator.mediaDevices.getUserMedia(constraints);
|
||||||
|
vdLocalVideo.srcObject = cameraStream;
|
||||||
|
vdVideoStrip.classList.remove("hidden");
|
||||||
|
|
||||||
|
const track = cameraStream.getVideoTracks()[0];
|
||||||
|
const settings = track.getSettings();
|
||||||
|
camCaptureCanvas.width = cameraSendWidth;
|
||||||
|
camCaptureCanvas.height = cameraSendHeight;
|
||||||
|
debugLog("camera:get_user_media_ok", {
|
||||||
|
width: settings.width ?? null,
|
||||||
|
height: settings.height ?? null,
|
||||||
|
send_width: camCaptureCanvas.width,
|
||||||
|
send_height: camCaptureCanvas.height,
|
||||||
|
frameRate: settings.frameRate ?? null,
|
||||||
|
deviceId: settings.deviceId ? "present" : null,
|
||||||
|
facingMode: settings.facingMode ?? null,
|
||||||
|
});
|
||||||
|
|
||||||
|
cameraActive = true;
|
||||||
|
cameraCaptureFrameNo = 0;
|
||||||
|
cameraPushFailures = 0;
|
||||||
|
vdCamIcon.textContent = "Cam ✓";
|
||||||
|
vdCamBtn.classList.add("active");
|
||||||
|
vdLocalVideo.classList.remove("hidden");
|
||||||
|
keepLocalPipInViewport();
|
||||||
|
|
||||||
|
scheduleCameraFrameCapture();
|
||||||
|
} catch (e: any) {
|
||||||
|
console.warn("camera access denied or unavailable:", e);
|
||||||
|
debugLog("camera:get_user_media_failed", {
|
||||||
|
name: e?.name ?? null,
|
||||||
|
message: e?.message ?? String(e),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function stopCamera() {
|
||||||
|
if (cameraActive) {
|
||||||
|
debugLog("camera:stopped", { frames: cameraCaptureFrameNo });
|
||||||
|
}
|
||||||
|
cameraActive = false;
|
||||||
|
cancelCameraCaptureLoop();
|
||||||
|
if (cameraStream) { cameraStream.getTracks().forEach(t => t.stop()); cameraStream = null; }
|
||||||
|
vdLocalVideo.srcObject = null;
|
||||||
|
vdLocalVideo.classList.add("hidden");
|
||||||
|
vdCamIcon.textContent = "Cam";
|
||||||
|
vdCamBtn.classList.remove("active");
|
||||||
|
// Hide strip only if remote video is also gone
|
||||||
|
if (!remoteVideoActive) vdVideoStrip.classList.add("hidden");
|
||||||
|
}
|
||||||
|
|
||||||
|
vdCamBtn.addEventListener("click", () => {
|
||||||
|
if (cameraActive) { stopCamera(); } else { startCamera(); }
|
||||||
|
});
|
||||||
|
|
||||||
|
// ── Remote video display (Blocker 5) ─────────────────────────────
|
||||||
|
const remoteCtx = vdRemoteVideo.getContext("2d")!;
|
||||||
|
const vdRemotePlaceholder = document.getElementById("vd-remote-placeholder")!;
|
||||||
|
const vdRemoteCounter = document.getElementById("vd-remote-counter")!;
|
||||||
|
let remoteFrameCount = 0;
|
||||||
|
let remoteFrameSerial = 0;
|
||||||
|
let remoteDrawInFlight = false;
|
||||||
|
let remotePendingFrame: { serial: number; width: number; height: number; jpeg_b64: string } | null = null;
|
||||||
|
|
||||||
|
function nextAnimationFrame() {
|
||||||
|
return new Promise<void>(resolve => requestAnimationFrame(() => resolve()));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function drawRemoteFrame(frame: { serial: number; width: number; height: number; jpeg_b64: string }) {
|
||||||
|
const img = new Image();
|
||||||
|
img.src = `data:image/jpeg;base64,${frame.jpeg_b64}`;
|
||||||
|
if ("decode" in img) {
|
||||||
|
await img.decode();
|
||||||
|
} else {
|
||||||
|
await new Promise<void>((resolve, reject) => {
|
||||||
|
img.onload = () => resolve();
|
||||||
|
img.onerror = () => reject(new Error("remote video image decode failed"));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (frame.serial !== remoteFrameSerial) return;
|
||||||
|
await nextAnimationFrame();
|
||||||
|
if (frame.serial !== remoteFrameSerial) return;
|
||||||
|
if (vdRemoteVideo.width !== frame.width) vdRemoteVideo.width = frame.width;
|
||||||
|
if (vdRemoteVideo.height !== frame.height) vdRemoteVideo.height = frame.height;
|
||||||
|
remoteCtx.drawImage(img, 0, 0, vdRemoteVideo.width, vdRemoteVideo.height);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function pumpRemoteVideoFrames() {
|
||||||
|
if (remoteDrawInFlight) return;
|
||||||
|
remoteDrawInFlight = true;
|
||||||
|
try {
|
||||||
|
while (remotePendingFrame) {
|
||||||
|
const frame = remotePendingFrame;
|
||||||
|
remotePendingFrame = null;
|
||||||
|
try {
|
||||||
|
await drawRemoteFrame(frame);
|
||||||
|
} catch (e) {
|
||||||
|
console.warn("remote video draw failed:", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
remoteDrawInFlight = false;
|
||||||
|
if (remotePendingFrame) void pumpRemoteVideoFrames();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
listen("video:frame", (event: any) => {
|
||||||
|
const { width, height, jpeg_b64 } = event.payload;
|
||||||
|
if (!jpeg_b64) return;
|
||||||
|
const frameSerial = ++remoteFrameSerial;
|
||||||
|
|
||||||
|
remoteVideoActive = true;
|
||||||
|
vdVideoStrip.classList.remove("hidden");
|
||||||
|
vdRemotePlaceholder.classList.add("hidden");
|
||||||
|
remoteFrameCount++;
|
||||||
|
if (remoteFrameCount === 1) console.log("first remote video frame:", width, "x", height);
|
||||||
|
|
||||||
|
remotePendingFrame = {
|
||||||
|
serial: frameSerial,
|
||||||
|
width: width ?? vdRemoteVideo.width,
|
||||||
|
height: height ?? vdRemoteVideo.height,
|
||||||
|
jpeg_b64,
|
||||||
|
};
|
||||||
|
void pumpRemoteVideoFrames();
|
||||||
|
});
|
||||||
|
|
||||||
// ── Poll status ───────────────────────────────────────────────────
|
// ── Poll status ───────────────────────────────────────────────────
|
||||||
interface CallStatusI {
|
interface CallStatusI {
|
||||||
active: boolean;
|
active: boolean;
|
||||||
@@ -481,9 +937,11 @@ listen("signal-event", (event: any) => {
|
|||||||
incomingBanner.classList.add("hidden");
|
incomingBanner.classList.add("hidden");
|
||||||
// Auto-connect to the call
|
// Auto-connect to the call
|
||||||
(async () => {
|
(async () => {
|
||||||
|
if (connectPending) return;
|
||||||
|
connectPending = true;
|
||||||
const s = loadSettings();
|
const s = loadSettings();
|
||||||
try {
|
try {
|
||||||
await invoke("connect", {
|
await connectWithTimeout({
|
||||||
relay: data.relay_addr,
|
relay: data.relay_addr,
|
||||||
room: data.room,
|
room: data.room,
|
||||||
alias: s.alias || "",
|
alias: s.alias || "",
|
||||||
@@ -494,10 +952,14 @@ listen("signal-event", (event: any) => {
|
|||||||
peerMappedAddr: data.peer_mapped_addr ?? null,
|
peerMappedAddr: data.peer_mapped_addr ?? null,
|
||||||
directOnly: s.directOnly || false,
|
directOnly: s.directOnly || false,
|
||||||
birthdayAttack: s.birthdayAttack || false,
|
birthdayAttack: s.birthdayAttack || false,
|
||||||
|
...videoConnectOptions(s),
|
||||||
});
|
});
|
||||||
enterVoice(true);
|
enterVoice(true);
|
||||||
} catch (e: any) {
|
} catch (e: any) {
|
||||||
console.error("connect failed:", e);
|
console.error("connect failed:", e);
|
||||||
|
showToast(`Call failed to connect: ${errorMessage(e)}`);
|
||||||
|
} finally {
|
||||||
|
connectPending = false;
|
||||||
}
|
}
|
||||||
})();
|
})();
|
||||||
break;
|
break;
|
||||||
@@ -641,6 +1103,8 @@ function openSettings() {
|
|||||||
sCallDebug.checked = !!s.callDebugLogs;
|
sCallDebug.checked = !!s.callDebugLogs;
|
||||||
sDirectOnly.checked = !!s.directOnly;
|
sDirectOnly.checked = !!s.directOnly;
|
||||||
sBirthdayAttack.checked = !!s.birthdayAttack;
|
sBirthdayAttack.checked = !!s.birthdayAttack;
|
||||||
|
sVideoCodec.value = s.videoCodec || "h264";
|
||||||
|
sVideoResolution.value = s.videoResolution || "1280x720";
|
||||||
sCallDebugSection.style.display = s.callDebugLogs ? "" : "none";
|
sCallDebugSection.style.display = s.callDebugLogs ? "" : "none";
|
||||||
renderCallDebugLog();
|
renderCallDebugLog();
|
||||||
const qi = qualityToIndex(s.quality || "auto");
|
const qi = qualityToIndex(s.quality || "auto");
|
||||||
@@ -666,6 +1130,8 @@ settingsSave.addEventListener("click", () => {
|
|||||||
s.callDebugLogs = sCallDebug.checked;
|
s.callDebugLogs = sCallDebug.checked;
|
||||||
s.directOnly = sDirectOnly.checked;
|
s.directOnly = sDirectOnly.checked;
|
||||||
s.birthdayAttack = sBirthdayAttack.checked;
|
s.birthdayAttack = sBirthdayAttack.checked;
|
||||||
|
s.videoCodec = sVideoCodec.value || "h264";
|
||||||
|
s.videoResolution = sVideoResolution.value || "1280x720";
|
||||||
saveSettings(s);
|
saveSettings(s);
|
||||||
invoke("set_dred_verbose_logs", { enabled: s.dredDebugLogs }).catch(() => {});
|
invoke("set_dred_verbose_logs", { enabled: s.dredDebugLogs }).catch(() => {});
|
||||||
invoke("set_call_debug_logs", { enabled: s.callDebugLogs }).catch(() => {});
|
invoke("set_call_debug_logs", { enabled: s.callDebugLogs }).catch(() => {});
|
||||||
@@ -768,6 +1234,7 @@ document.addEventListener("keydown", (e) => {
|
|||||||
if (e.key === "m") vdMicBtn.click();
|
if (e.key === "m") vdMicBtn.click();
|
||||||
if (e.key === "q") vdEndBtn.click();
|
if (e.key === "q") vdEndBtn.click();
|
||||||
if (e.key === "s") vdSpkBtn.click();
|
if (e.key === "s") vdSpkBtn.click();
|
||||||
|
if (e.key === "v") vdCamBtn.click();
|
||||||
if (e.key === "," && (e.metaKey || e.ctrlKey)) { e.preventDefault(); openSettings(); }
|
if (e.key === "," && (e.metaKey || e.ctrlKey)) { e.preventDefault(); openSettings(); }
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -204,6 +204,16 @@ body {
|
|||||||
padding: 12px 0;
|
padding: 12px 0;
|
||||||
display: flex;
|
display: flex;
|
||||||
justify-content: center;
|
justify-content: center;
|
||||||
|
gap: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.fab-video {
|
||||||
|
background: #3b82f6;
|
||||||
|
box-shadow: 0 4px 16px rgba(59, 130, 246, 0.3);
|
||||||
|
}
|
||||||
|
|
||||||
|
.fab-video:hover {
|
||||||
|
box-shadow: 0 6px 20px rgba(59, 130, 246, 0.4);
|
||||||
}
|
}
|
||||||
|
|
||||||
.fab {
|
.fab {
|
||||||
@@ -248,7 +258,7 @@ body {
|
|||||||
border-top: 1px solid var(--surface2);
|
border-top: 1px solid var(--surface2);
|
||||||
padding: 0 16px;
|
padding: 0 16px;
|
||||||
padding-bottom: env(safe-area-inset-bottom, 8px);
|
padding-bottom: env(safe-area-inset-bottom, 8px);
|
||||||
z-index: 50;
|
z-index: 70;
|
||||||
animation: drawerUp 0.25s ease-out;
|
animation: drawerUp 0.25s ease-out;
|
||||||
box-shadow: 0 -4px 20px rgba(0,0,0,0.4);
|
box-shadow: 0 -4px 20px rgba(0,0,0,0.4);
|
||||||
}
|
}
|
||||||
@@ -306,6 +316,68 @@ body {
|
|||||||
padding: 2px 0 4px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;
|
padding: 2px 0 4px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* Full-screen video stage — overlays lobby/main when video is active */
|
||||||
|
.vd-video-stage {
|
||||||
|
position: fixed;
|
||||||
|
top: 0;
|
||||||
|
left: 0;
|
||||||
|
right: 0;
|
||||||
|
bottom: 96px; /* leave room for voice drawer */
|
||||||
|
background: #000;
|
||||||
|
z-index: 40;
|
||||||
|
overflow: hidden;
|
||||||
|
}
|
||||||
|
.vd-remote-stage {
|
||||||
|
position: absolute;
|
||||||
|
inset: 0;
|
||||||
|
width: 100%;
|
||||||
|
height: 100%;
|
||||||
|
object-fit: contain;
|
||||||
|
background: #000;
|
||||||
|
}
|
||||||
|
.vd-remote-placeholder {
|
||||||
|
position: absolute;
|
||||||
|
inset: 0;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
color: #888;
|
||||||
|
pointer-events: none;
|
||||||
|
z-index: 1;
|
||||||
|
}
|
||||||
|
.vd-remote-placeholder.hidden { display: none; }
|
||||||
|
.vd-placeholder-text { font-size: 18px; margin-bottom: 8px; }
|
||||||
|
.vd-placeholder-sub { font-size: 12px; opacity: 0.7; }
|
||||||
|
.vd-local-pip {
|
||||||
|
position: fixed;
|
||||||
|
right: 18px;
|
||||||
|
bottom: calc(176px + env(safe-area-inset-bottom, 0px));
|
||||||
|
width: min(34vw, 220px);
|
||||||
|
height: auto;
|
||||||
|
aspect-ratio: 16 / 9;
|
||||||
|
border-radius: 8px;
|
||||||
|
background: #111;
|
||||||
|
border: 2px solid rgba(255, 255, 255, 0.2);
|
||||||
|
object-fit: cover;
|
||||||
|
box-shadow: 0 4px 20px rgba(0, 0, 0, 0.5);
|
||||||
|
z-index: 90;
|
||||||
|
cursor: grab;
|
||||||
|
touch-action: none;
|
||||||
|
-webkit-user-drag: none;
|
||||||
|
}
|
||||||
|
.vd-local-pip.dragging {
|
||||||
|
cursor: grabbing;
|
||||||
|
box-shadow: 0 8px 28px rgba(0, 0, 0, 0.65);
|
||||||
|
}
|
||||||
|
@media (max-width: 520px) {
|
||||||
|
.vd-local-pip {
|
||||||
|
width: min(48vw, 190px);
|
||||||
|
right: 12px;
|
||||||
|
bottom: calc(188px + env(safe-area-inset-bottom, 0px));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/* Incoming call banner */
|
/* Incoming call banner */
|
||||||
.incoming-banner {
|
.incoming-banner {
|
||||||
position: fixed;
|
position: fixed;
|
||||||
|
|||||||
@@ -59,6 +59,7 @@ graph TD
|
|||||||
FEC["wzp-fec<br/>RaptorQ FEC"]
|
FEC["wzp-fec<br/>RaptorQ FEC"]
|
||||||
CRYPTO["wzp-crypto<br/>ChaCha20 + Identity"]
|
CRYPTO["wzp-crypto<br/>ChaCha20 + Identity"]
|
||||||
TRANSPORT["wzp-transport<br/>QUIC / Quinn"]
|
TRANSPORT["wzp-transport<br/>QUIC / Quinn"]
|
||||||
|
VIDEO["wzp-video<br/>H.264 + H.265 + AV1"]
|
||||||
|
|
||||||
RELAY["wzp-relay<br/>Relay Daemon"]
|
RELAY["wzp-relay<br/>Relay Daemon"]
|
||||||
CLIENT["wzp-client<br/>CLI + Call Engine"]
|
CLIENT["wzp-client<br/>CLI + Call Engine"]
|
||||||
@@ -68,16 +69,19 @@ graph TD
|
|||||||
PROTO --> FEC
|
PROTO --> FEC
|
||||||
PROTO --> CRYPTO
|
PROTO --> CRYPTO
|
||||||
PROTO --> TRANSPORT
|
PROTO --> TRANSPORT
|
||||||
|
PROTO --> VIDEO
|
||||||
|
|
||||||
CODEC --> CLIENT
|
CODEC --> CLIENT
|
||||||
FEC --> CLIENT
|
FEC --> CLIENT
|
||||||
CRYPTO --> CLIENT
|
CRYPTO --> CLIENT
|
||||||
TRANSPORT --> CLIENT
|
TRANSPORT --> CLIENT
|
||||||
|
VIDEO --> CLIENT
|
||||||
|
|
||||||
CODEC --> RELAY
|
CODEC --> RELAY
|
||||||
FEC --> RELAY
|
FEC --> RELAY
|
||||||
CRYPTO --> RELAY
|
CRYPTO --> RELAY
|
||||||
TRANSPORT --> RELAY
|
TRANSPORT --> RELAY
|
||||||
|
VIDEO --> RELAY
|
||||||
|
|
||||||
CLIENT --> WEB
|
CLIENT --> WEB
|
||||||
TRANSPORT --> WEB
|
TRANSPORT --> WEB
|
||||||
@@ -90,9 +94,10 @@ graph TD
|
|||||||
style CLIENT fill:#00b894,color:#fff
|
style CLIENT fill:#00b894,color:#fff
|
||||||
style WEB fill:#0984e3,color:#fff
|
style WEB fill:#0984e3,color:#fff
|
||||||
style FC fill:#fd79a8,color:#fff
|
style FC fill:#fd79a8,color:#fff
|
||||||
|
style VIDEO fill:#a29bfe,color:#fff
|
||||||
```
|
```
|
||||||
|
|
||||||
**Star pattern**: Each leaf crate (`wzp-codec`, `wzp-fec`, `wzp-crypto`, `wzp-transport`) depends only on `wzp-proto`. No leaf depends on another leaf. Integration crates (`wzp-relay`, `wzp-client`, `wzp-web`) depend on all leaves.
|
**Star pattern**: Each leaf crate (`wzp-codec`, `wzp-fec`, `wzp-crypto`, `wzp-transport`, `wzp-video`) depends only on `wzp-proto`. No leaf depends on another leaf. Integration crates (`wzp-relay`, `wzp-client`, `wzp-web`) depend on all leaves.
|
||||||
|
|
||||||
## Audio Encode Pipeline
|
## Audio Encode Pipeline
|
||||||
|
|
||||||
@@ -106,7 +111,7 @@ sequenceDiagram
|
|||||||
participant DT as DredTuner<br/>(wzp-proto)
|
participant DT as DredTuner<br/>(wzp-proto)
|
||||||
participant FEC as RaptorQ FEC
|
participant FEC as RaptorQ FEC
|
||||||
participant INT as Interleaver<br/>(depth=3)
|
participant INT as Interleaver<br/>(depth=3)
|
||||||
participant HDR as MediaHeader<br/>(12B or Mini 4B)
|
participant HDR as MediaHeader<br/>(16B or Mini 5B)
|
||||||
participant Enc as ChaCha20-Poly1305
|
participant Enc as ChaCha20-Poly1305
|
||||||
participant QUIC as QUIC Datagram
|
participant QUIC as QUIC Datagram
|
||||||
participant QPS as QuinnPathSnapshot
|
participant QPS as QuinnPathSnapshot
|
||||||
@@ -144,7 +149,7 @@ sequenceDiagram
|
|||||||
- RNNoise processes **2 x 480** samples (ML-based noise suppression via nnnoiseless)
|
- RNNoise processes **2 x 480** samples (ML-based noise suppression via nnnoiseless)
|
||||||
- Silence detection uses VAD + 100ms hangover before switching to ComfortNoise
|
- Silence detection uses VAD + 100ms hangover before switching to ComfortNoise
|
||||||
- FEC symbols are padded to **256 bytes** with a 2-byte LE length prefix
|
- FEC symbols are padded to **256 bytes** with a 2-byte LE length prefix
|
||||||
- MiniHeaders (4 bytes) replace full headers (12 bytes) for 49 of every 50 frames
|
- MiniHeaders (5 bytes) replace full headers (16 bytes) for 49 of every 50 audio frames; video always uses full headers
|
||||||
- DRED tuner polls quinn path stats every 25 frames (~500ms) and adjusts DRED lookback duration continuously
|
- DRED tuner polls quinn path stats every 25 frames (~500ms) and adjusts DRED lookback duration continuously
|
||||||
- Opus tiers bypass RaptorQ entirely -- DRED handles loss recovery at the codec layer
|
- Opus tiers bypass RaptorQ entirely -- DRED handles loss recovery at the codec layer
|
||||||
- Opus6k DRED window: 1040ms (maximum libopus allows)
|
- Opus6k DRED window: 1040ms (maximum libopus allows)
|
||||||
@@ -324,35 +329,29 @@ sequenceDiagram
|
|||||||
|
|
||||||
## Wire Formats
|
## Wire Formats
|
||||||
|
|
||||||
### MediaHeader (12 bytes)
|
### `MediaHeader` v2 (16 bytes, byte-aligned)
|
||||||
|
|
||||||
```
|
```
|
||||||
Byte 0: [V:1][T:1][CodecID:4][Q:1][FecRatioHi:1]
|
Byte 0: version (u8) 0x02
|
||||||
Byte 1: [FecRatioLo:6][unused:2]
|
Byte 1: flags (u8) [T:1][Q:1][KeyFrame:1][FrameEnd:1][reserved:4]
|
||||||
Bytes 2-3: sequence (u16 BE)
|
T = FEC repair, Q = QualityReport trailer
|
||||||
Bytes 4-7: timestamp_ms (u32 BE)
|
KeyFrame = packet belongs to an I-frame (video)
|
||||||
Byte 8: fec_block_id (u8)
|
FrameEnd = last packet of an access unit (video)
|
||||||
Byte 9: fec_symbol_idx (u8)
|
Byte 2: media_type (u8) 0=audio, 1=video, 2=data, 3=control
|
||||||
Byte 10: reserved
|
Byte 3: codec_id (u8) widened from 4-bit (room for 256 codec IDs)
|
||||||
Byte 11: csrc_count
|
Byte 4: stream_id (u8) simulcast layer; 0=base
|
||||||
|
Byte 5: fec_ratio (u8) 0..200 → 0.0..2.0
|
||||||
|
Bytes 6-9: sequence (u32 BE) wrapping packet sequence number
|
||||||
|
Bytes 10-13: timestamp_ms (u32 BE) milliseconds since session start
|
||||||
|
Bytes 14-15: fec_block_id (u16 BE)
|
||||||
|
audio: low 8 bits = block_id, high 8 bits = symbol_idx
|
||||||
|
video: full u16 block_id (large blocks for I-frames)
|
||||||
```
|
```
|
||||||
|
|
||||||
| Field | Bits | Description |
|
|
||||||
|-------|------|-------------|
|
|
||||||
| V (version) | 1 | Protocol version (0 = v1) |
|
|
||||||
| T (is_repair) | 1 | 1 = FEC repair packet, 0 = source media |
|
|
||||||
| CodecID | 4 | Codec identifier (0-8, see table below) |
|
|
||||||
| Q | 1 | 1 = QualityReport trailer appended |
|
|
||||||
| FecRatio | 7 | FEC ratio encoded as 0-127 mapping to 0.0-2.0 |
|
|
||||||
| sequence | 16 | Wrapping packet sequence number |
|
|
||||||
| timestamp_ms | 32 | Milliseconds since session start |
|
|
||||||
| fec_block_id | 8 | FEC source block ID (wrapping) |
|
|
||||||
| fec_symbol_idx | 8 | Symbol index within FEC block |
|
|
||||||
| reserved | 8 | Reserved flags |
|
|
||||||
| csrc_count | 8 | Contributing source count (future mixing) |
|
|
||||||
|
|
||||||
#### CodecID Values
|
#### CodecID Values
|
||||||
|
|
||||||
|
**Audio codecs (media_type = 0)**
|
||||||
|
|
||||||
| Value | Codec | Bitrate | Sample Rate | Frame Duration |
|
| Value | Codec | Bitrate | Sample Rate | Frame Duration |
|
||||||
|-------|-------|---------|-------------|---------------|
|
|-------|-------|---------|-------------|---------------|
|
||||||
| 0 | Opus 24k | 24 kbps | 48 kHz | 20ms |
|
| 0 | Opus 24k | 24 kbps | 48 kHz | 20ms |
|
||||||
@@ -365,15 +364,25 @@ Byte 11: csrc_count
|
|||||||
| 7 | Opus 48k | 48 kbps | 48 kHz | 20ms |
|
| 7 | Opus 48k | 48 kbps | 48 kHz | 20ms |
|
||||||
| 8 | Opus 64k | 64 kbps | 48 kHz | 20ms |
|
| 8 | Opus 64k | 64 kbps | 48 kHz | 20ms |
|
||||||
|
|
||||||
### MiniHeader (4 bytes, compressed)
|
**Video codecs (media_type = 1)**
|
||||||
|
|
||||||
|
| Value | Codec | Notes |
|
||||||
|
|-------|-------|-------|
|
||||||
|
| 9 | H.264 Baseline | Universal HW encode coverage |
|
||||||
|
| 10 | H.264 Main | Slight quality win over baseline |
|
||||||
|
| 11 | H.265 Main | Apple A10+, Snapdragon ~2017, NVENC GTX 9xx+; ~30% better than H.264 |
|
||||||
|
| 12 | AV1 Main | Apple M3/A17+, Snapdragon 8 Gen 3+, RTX 40+; best efficiency, narrow HW |
|
||||||
|
|
||||||
|
### `MiniHeader` v2 (5 bytes)
|
||||||
|
|
||||||
```
|
```
|
||||||
[FRAME_TYPE_MINI: 0x01]
|
[FRAME_TYPE_MINI = 0x01]
|
||||||
Bytes 0-1: timestamp_delta_ms (u16 BE)
|
Byte 0: seq_delta (u8) delta from last full header's seq
|
||||||
Bytes 2-3: payload_len (u16 BE)
|
Bytes 1-2: timestamp_delta_ms (u16 BE)
|
||||||
|
Bytes 3-4: payload_len (u16 BE)
|
||||||
```
|
```
|
||||||
|
|
||||||
Used for 49 of every 50 frames (~1s cycle). Saves 8 bytes per packet (67% header reduction). Full header is sent every 50th frame to resynchronize state.
|
Used for audio only (49 of every 50 frames). Saves 11 bytes per audio packet vs the full 16B header. Full header is sent every 50th frame to resynchronize state. Video always uses full 16B headers.
|
||||||
|
|
||||||
### TrunkFrame (batched datagrams)
|
### TrunkFrame (batched datagrams)
|
||||||
|
|
||||||
@@ -482,9 +491,12 @@ sequenceDiagram
|
|||||||
|
|
||||||
### Shared State & Locking
|
### Shared State & Locking
|
||||||
|
|
||||||
|
The `RoomManager` stores `DashMap<String, Arc<RwLock<Room>>>`. The DashMap guard is held only long enough to clone the `Arc`; all per-room operations then acquire the room-level `RwLock`. Concurrent fan-out calls share a read lock; join/leave acquire write lock.
|
||||||
|
|
||||||
| Lock | Protected Data | Hold Duration | Contention |
|
| Lock | Protected Data | Hold Duration | Contention |
|
||||||
|------|---------------|---------------|------------|
|
|------|---------------|---------------|------------|
|
||||||
| `RoomManager` (Mutex) | Rooms, participants, quality tiers | ~1ms/packet | O(N) per room |
|
| `DashMap<room_id, Arc<RwLock<Room>>>` | Room registry | Instant (clone Arc only) | Near-zero |
|
||||||
|
| `Room` (RwLock) | Participants, quality tiers | ~1ms/packet (read); ~1ms (write on join/leave) | Low (concurrent reads) |
|
||||||
| `PresenceRegistry` (Mutex) | Fingerprint registrations | ~1ms | Low (join/leave only) |
|
| `PresenceRegistry` (Mutex) | Fingerprint registrations | ~1ms | Low (join/leave only) |
|
||||||
| `SessionManager` (Mutex) | Active session tracking | ~1ms | Low |
|
| `SessionManager` (Mutex) | Active session tracking | ~1ms | Low |
|
||||||
| `FederationManager.peer_links` (Mutex) | Peer connections | ~10ms during forward | Per-federation-packet |
|
| `FederationManager.peer_links` (Mutex) | Peer connections | ~10ms during forward | Per-federation-packet |
|
||||||
@@ -492,15 +504,9 @@ sequenceDiagram
|
|||||||
### Scaling Characteristics
|
### Scaling Characteristics
|
||||||
|
|
||||||
- **Many small rooms**: Scales well across all cores (rooms are independent)
|
- **Many small rooms**: Scales well across all cores (rooms are independent)
|
||||||
- **Large single room (100+ participants)**: Serialized by RoomManager lock
|
- **Large single room (100+ participants)**: Fan-out reads share RwLock (non-blocking); only join/leave serializes
|
||||||
- **Federation**: Per-peer tasks scale; `peer_links` lock held during send loop
|
- **Federation**: Per-peer tasks scale; `peer_links` lock held during send loop
|
||||||
|
|
||||||
### Primary Bottleneck
|
|
||||||
|
|
||||||
The RoomManager Mutex is acquired per-packet by every participant to get the fan-out peer list. Lock is released before I/O (sends happen outside lock), but packet processing is serialized through the lock within a room.
|
|
||||||
|
|
||||||
Future optimization: per-room locks or lock-free participant lists via `DashMap`.
|
|
||||||
|
|
||||||
## Client Architecture
|
## Client Architecture
|
||||||
|
|
||||||
### Desktop Engine (Tauri)
|
### Desktop Engine (Tauri)
|
||||||
@@ -553,6 +559,8 @@ Key design decisions:
|
|||||||
|
|
||||||
### Android Engine (Kotlin + JNI)
|
### Android Engine (Kotlin + JNI)
|
||||||
|
|
||||||
|
> **Note (2026-05-12):** The Kotlin+JNI Android app (`android/app/`) described below is superseded by the **Tauri 2.x mobile build** (`desktop/src-tauri/` + `crates/wzp-native/`). The Tauri approach uses the same Rust call engine as desktop, with Oboe audio via `wzp-native` cdylib. The Kotlin codebase is maintained for reference but the Tauri build is the live production app.
|
||||||
|
|
||||||
```mermaid
|
```mermaid
|
||||||
graph TB
|
graph TB
|
||||||
subgraph "Compose UI"
|
subgraph "Compose UI"
|
||||||
@@ -902,6 +910,20 @@ warzonePhone/
|
|||||||
│ │ └── rekey.rs # Forward secrecy rekeying
|
│ │ └── rekey.rs # Forward secrecy rekeying
|
||||||
│ ├── wzp-transport/ # QUIC transport layer
|
│ ├── wzp-transport/ # QUIC transport layer
|
||||||
│ │ └── src/lib.rs # QuinnTransport, send/recv media/signal/trunk
|
│ │ └── src/lib.rs # QuinnTransport, send/recv media/signal/trunk
|
||||||
|
│ ├── wzp-video/ # Video codecs + framer
|
||||||
|
│ │ └── src/
|
||||||
|
│ │ ├── factory.rs # VideoEncoder factory (platform dispatch)
|
||||||
|
│ │ ├── framer.rs # NAL fragmentation (H.264/H.265)
|
||||||
|
│ │ ├── depacketizer.rs # NAL reassembly, access unit emit
|
||||||
|
│ │ ├── controller.rs # VideoQualityController
|
||||||
|
│ │ ├── simulcast.rs # Simulcast layer management
|
||||||
|
│ │ ├── encoder_mode.rs # Encoder mode selection
|
||||||
|
│ │ ├── av1_obu.rs # AV1 OBU framing + depacketizer
|
||||||
|
│ │ ├── dav1d.rs # dav1d AV1 software decoder
|
||||||
|
│ │ ├── svt_av1.rs # SVT-AV1 software encoder (non-Android)
|
||||||
|
│ │ ├── videotoolbox.rs # VideoToolbox H.265 + AV1 (macOS)
|
||||||
|
│ │ ├── mediacodec.rs # MediaCodec H.264/H.265/AV1 (Android, NDK 0.9 migration pending)
|
||||||
|
│ │ └── nack.rs # NACK sender/receiver framework
|
||||||
│ ├── wzp-relay/ # Relay daemon
|
│ ├── wzp-relay/ # Relay daemon
|
||||||
│ │ └── src/
|
│ │ └── src/
|
||||||
│ │ ├── main.rs # CLI, connection loop, auth + handshake
|
│ │ ├── main.rs # CLI, connection loop, auth + handshake
|
||||||
@@ -917,6 +939,10 @@ warzonePhone/
|
|||||||
│ │ ├── presence.rs # PresenceRegistry
|
│ │ ├── presence.rs # PresenceRegistry
|
||||||
│ │ ├── route.rs # RouteResolver
|
│ │ ├── route.rs # RouteResolver
|
||||||
│ │ ├── trunk.rs # TrunkBatcher
|
│ │ ├── trunk.rs # TrunkBatcher
|
||||||
|
│ │ ├── audio_scorer.rs # Per-stream audio quality scoring
|
||||||
|
│ │ ├── response_policy.rs # Relay response policy (rate-limit, drop)
|
||||||
|
│ │ ├── verdict.rs # Verdict enum (Allow/RateLimit/Drop/Malicious)
|
||||||
|
│ │ ├── video_scorer.rs # VideoScorer (legitimacy scoring, keyframe regularity)
|
||||||
│ │ └── ws.rs # WebSocket handler for browser clients
|
│ │ └── ws.rs # WebSocket handler for browser clients
|
||||||
│ ├── wzp-client/ # Call engine + CLI
|
│ ├── wzp-client/ # Call engine + CLI
|
||||||
│ │ └── src/
|
│ │ └── src/
|
||||||
@@ -956,7 +982,7 @@ warzonePhone/
|
|||||||
|
|
||||||
## Test Coverage
|
## Test Coverage
|
||||||
|
|
||||||
571 tests across all crates, 0 failures:
|
702 tests across all crates (excluding wzp-android), 0 failures:
|
||||||
|
|
||||||
| Crate | Tests | Key Coverage |
|
| Crate | Tests | Key Coverage |
|
||||||
|-------|-------|-------------|
|
|-------|-------|-------------|
|
||||||
@@ -965,7 +991,8 @@ warzonePhone/
|
|||||||
| wzp-fec | 21 | RaptorQ encode/decode, loss recovery, interleaving |
|
| wzp-fec | 21 | RaptorQ encode/decode, loss recovery, interleaving |
|
||||||
| wzp-crypto | 64 | Encrypt/decrypt, handshake, anti-replay, featherChat identity |
|
| wzp-crypto | 64 | Encrypt/decrypt, handshake, anti-replay, featherChat identity |
|
||||||
| wzp-transport | 11 | QUIC connection setup, path monitoring |
|
| wzp-transport | 11 | QUIC connection setup, path monitoring |
|
||||||
| wzp-relay | 122 | Room ACL, session mgmt, metrics, probes, mesh, trunking |
|
| wzp-relay | 137 | Room ACL, session mgmt, metrics, probes, mesh, trunking, scoring, verdict |
|
||||||
|
| wzp-video | 88 | NAL framing, AV1 OBU, simulcast, quality controller, NACK |
|
||||||
| wzp-client | 170 | Encoder/decoder, quality adapter, silence, drift, sweep |
|
| wzp-client | 170 | Encoder/decoder, quality adapter, silence, drift, sweep |
|
||||||
| wzp-web | 2 | Metrics |
|
| wzp-web | 2 | Metrics |
|
||||||
| wzp-native | 0 | Native platform bindings (no unit tests) |
|
| wzp-native | 0 | Native platform bindings (no unit tests) |
|
||||||
|
|||||||
231
docs/AUDIT-2026-05-25.md
Normal file
231
docs/AUDIT-2026-05-25.md
Normal file
@@ -0,0 +1,231 @@
|
|||||||
|
# WarzonePhone Protocol Audit — 2026-05-25
|
||||||
|
|
||||||
|
**Auditor:** Claude Sonnet 4.6 (assisted)
|
||||||
|
**Branch:** `experimental-ui` @ `f3e3ee5`
|
||||||
|
**Scope:** All workspace crates (`wzp-proto`, `wzp-codec`, `wzp-fec`, `wzp-crypto`, `wzp-transport`, `wzp-relay`, `wzp-client`, `wzp-android`, `wzp-native`, `wzp-video`)
|
||||||
|
**Test baseline:** 702 passing (excludes `wzp-android`)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Executive Summary
|
||||||
|
|
||||||
|
The audio call path is functionally correct and cryptographically sound on clean network paths. **There is a session-breaking bug in the crypto nonce derivation (C1) that will cause a permanent decryption failure on any out-of-order UDP delivery.** This is the single highest-priority fix — it will manifest as periodic session crashes under normal internet conditions. Video has a solid architectural foundation but three hard blockers remain before shipping: the AEAD coverage gap (C2), dead video scorer (C3), and Android MediaCodec compile failure (C4).
|
||||||
|
|
||||||
|
The project is in good shape overall. The crypto design (X25519, HKDF, ChaCha20-Poly1305, Ed25519 identity, SAS verification) is sound. The SFU-never-decrypts architecture is rare and valuable. The codec adaptation (Opus DRED + Codec2 RaptorQ split) is genuinely innovative. The eight issues below are fixable in ~12 engineer-hours.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Critical
|
||||||
|
|
||||||
|
### C1 — Nonce derives from `recv_seq` counter, not `MediaHeader.seq`
|
||||||
|
|
||||||
|
**File:** `crates/wzp-crypto/src/session.rs:132`
|
||||||
|
**Severity:** Critical — session-breaking on any packet reorder
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// decrypt()
|
||||||
|
let nonce_bytes = nonce::build_nonce(&self.session_id, self.recv_seq, Direction::Send);
|
||||||
|
// ...
|
||||||
|
self.recv_seq = self.recv_seq.wrapping_add(1); // line 148
|
||||||
|
```
|
||||||
|
|
||||||
|
`recv_seq` increments once per successful `decrypt()` call. The sender's `send_seq` also increments once per `encrypt()` call (line 120). In perfect in-order delivery they stay synchronized. With any reorder or mid-stream packet loss they permanently diverge. Once diverged, every subsequent packet uses the wrong nonce → AEAD tag mismatch → every packet fails for the rest of the session.
|
||||||
|
|
||||||
|
This isn't a low-probability edge case. UDP over any internet path reorders packets routinely. The `multiple_packets_roundtrip` test (line 254) only exercises in-order delivery. HANDOFF-2026-05-12.md acknowledges this as a known latent item: *"AEAD nonce derivation: switch to `MediaHeader::seq`"*.
|
||||||
|
|
||||||
|
The anti-replay check at lines 152–161 already parses `MediaHeader` and has `header.seq` available. The fix is one line in `decrypt()`:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// Use sender's wire-level seq as nonce input, not a local counter.
|
||||||
|
// This survives reordering because both sides derive the same nonce from
|
||||||
|
// the same field. recv_seq was wrong: it diverged from send_seq on any
|
||||||
|
// reorder, breaking all subsequent decryptions for the session.
|
||||||
|
let header = parse_header(header_bytes)
|
||||||
|
.ok_or_else(|| CryptoError::Internal("header parse failed".into()))?;
|
||||||
|
let nonce_bytes = nonce::build_nonce(&self.session_id, header.seq, Direction::Send);
|
||||||
|
```
|
||||||
|
|
||||||
|
Remove `recv_seq` field from `ChaChaSession` (it's now redundant — anti-replay uses `header.seq` directly). On the encrypt side, verify that `self.send_seq` equals the `seq` written into the `MediaHeader` at the call site.
|
||||||
|
|
||||||
|
**Estimated effort:** ~1 hour including test coverage for out-of-order delivery.
|
||||||
|
|
||||||
|
> **Note on rekey seq reset:** The agent initially flagged `send_seq/recv_seq = 0` in `complete_rekey()` as a separate critical issue. This is a false positive — `install_key()` rotates `session_id` (hash of new key), so pre-/post-rekey nonces live in distinct namespaces. The reset is intentional and cryptographically safe.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### C2 — AEAD not wired to every QUIC datagram send path
|
||||||
|
|
||||||
|
**File:** `crates/wzp-client/src/analyzer.rs:363` (only confirmed decrypt call site)
|
||||||
|
**Severity:** Critical — potential plaintext media leakage
|
||||||
|
|
||||||
|
The HANDOFF document explicitly flags this: *"Encryption is implemented in `wzp-crypto` but not yet on every QUIC datagram path."* The `analyzer.rs` path decrypts inbound packets. What needs verification: every outbound `send_datagram()` / `write_datagram()` call across `wzp-client` and `wzp-transport` must pass through `ChaChaSession::encrypt()`.
|
||||||
|
|
||||||
|
**Required action:** Grep every `send_datagram` call site. Confirm each path encrypts before transmit. Add a CI-level test or `#[forbid(dead_code)]`-style assertion that makes a plaintext send path impossible to merge. Until this is verified, the E2E security claim cannot be made.
|
||||||
|
|
||||||
|
**Estimated effort:** ~1 hour audit + test.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### C3 — `VideoScorer::observe()` never called — scorer is dead code
|
||||||
|
|
||||||
|
**File:** `crates/wzp-relay/src/room.rs:1263–1266`
|
||||||
|
**Severity:** Critical — relay abuse control for video is completely absent
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// T6.2-follow-up: feed video packets to VideoScorer here.
|
||||||
|
// video_scorer.observe(&pkt.header, pkt.payload.len(), now, bwe_kbps);
|
||||||
|
```
|
||||||
|
|
||||||
|
`video_scorer.rs` was delivered in T6.2 with legitimacy scoring, keyframe regularity checks, I/P ratio analysis, and a verdict enum. The observe call was never wired into the packet forwarding loop. The scorer compiles but accumulates no data. Any participant can flood the room with malformed video or synthetic keyframe bursts and the relay will forward everything without challenge.
|
||||||
|
|
||||||
|
**Fix:** Wire `video_scorer.observe(...)` at the TODO marker and integrate `legitimacy_score()` into the forwarding decision (drop or rate-limit streams with `Verdict::Malicious`). Add an integration test: synthetic high-frequency keyframe bursts should trigger a `Malicious` verdict within 2 seconds.
|
||||||
|
|
||||||
|
**Estimated effort:** ~2 hours.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### C4 — `wzp-video` Android target fails to compile (31 errors)
|
||||||
|
|
||||||
|
**File:** `crates/wzp-video/src/mediacodec.rs`
|
||||||
|
**Severity:** Critical — Android video is completely blocked
|
||||||
|
|
||||||
|
Five error categories from the NDK 0.9 API migration, all documented in HANDOFF-2026-05-12.md. `dav1d`/`svt-av1` were cfg-gated off Android in `f3e3ee5`; these 31 errors are the remaining MediaCodec API mismatch.
|
||||||
|
|
||||||
|
| Error | Count | Root cause | Fix |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `E0277` `NonNull<AMediaCodec>` not `Send` | ~3 | Raw pointer held across `tokio::spawn` boundary | `struct SendMediaCodec(NonNull<…>); unsafe impl Send for SendMediaCodec {}` — or use `ndk::media::MediaCodec` owned type (already `Send`) |
|
||||||
|
| `E0308` `&[MaybeUninit<u8>]` vs `&[u8]` | many | NDK 0.9 returns uninit slices | `MaybeUninit::write_slice` or transmute pattern |
|
||||||
|
| `E0425` missing `BITRATE_MODE_CBR` | 1+ | Constant renamed in NDK 0.9 | Check `ndk` crate docs for current name |
|
||||||
|
| `E0433` `ndk_sys` not a dep | several | Direct `ndk_sys` import; only `ndk = "0.9"` declared | Add `ndk-sys` as explicit dep or use safe `ndk` wrappers |
|
||||||
|
| `E0599` `InputBuffer::index()` / `OutputBuffer::index()` private | 2 | API changed in NDK 0.9 | Use buffer through safe queue/dequeue API |
|
||||||
|
|
||||||
|
Nothing live is blocked today — `wzp-video` is not yet consumed by Tauri Android. But video on Android cannot progress until this compiles.
|
||||||
|
|
||||||
|
**Reproduce:**
|
||||||
|
```bash
|
||||||
|
ssh -i ~/CascadeProjects/wzp manwe@manwehs \
|
||||||
|
'cd ~/wzp-builder/data/source && \
|
||||||
|
docker run --rm \
|
||||||
|
-v ~/wzp-builder/data/source:/build/source \
|
||||||
|
-v ~/wzp-builder/data/cache/cargo-registry:/home/builder/.cargo/registry \
|
||||||
|
-v ~/wzp-builder/data/cache/cargo-git:/home/builder/.cargo/git \
|
||||||
|
-v ~/wzp-builder/data/cache/target:/build/source/target \
|
||||||
|
wzp-android-builder:latest \
|
||||||
|
bash -c "cd /build/source && cargo build --target aarch64-linux-android -p wzp-video 2>&1 | tail -60"'
|
||||||
|
```
|
||||||
|
|
||||||
|
**Estimated effort:** ~2 hours (one commit per error category).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## High
|
||||||
|
|
||||||
|
### H1 — AV1 call engine wiring missing
|
||||||
|
|
||||||
|
**Source:** HANDOFF-2026-05-12.md (T6.1.2 open item)
|
||||||
|
**File:** `crates/wzp-video/src/factory.rs`
|
||||||
|
|
||||||
|
`factory.rs` and step tables landed in commit `086d0a4`. No caller yet invokes `create_video_encoder(Av1Main, ...)`. The entire AV1 path is reachable only from tests. Video on macOS/Linux desktop requires wiring `create_video_encoder` into the call engine's media negotiation path.
|
||||||
|
|
||||||
|
**Estimated effort:** ~1–2 hours.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### H2 — `fec_block_id: u8` wraps every ~25 seconds
|
||||||
|
|
||||||
|
**File:** `crates/wzp-fec/src/encoder.rs` (`block_id.wrapping_add(1)` on u8)
|
||||||
|
**Reference:** PROTOCOL-AUDIT.md W2 (deferred P2)
|
||||||
|
|
||||||
|
At 5 frames/block (Codec2), u8 ID wraps at block 256 ≈ 25 seconds. A slow reconstructor or late-joining peer will collide block IDs with in-flight blocks. The window distance check in `block_manager.rs` partially mitigates this but can't prevent all collisions. Widen to `u16` in the next wire-format revision.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Medium
|
||||||
|
|
||||||
|
### M1 — `SignalMessage` has no version byte
|
||||||
|
|
||||||
|
**File:** `crates/wzp-proto/src/session.rs` (SignalMessage enum)
|
||||||
|
**Reference:** PROTOCOL-AUDIT.md W12
|
||||||
|
|
||||||
|
`bincode + serde(default)` handles field additions but not variant removal or semantic changes. Any variant deprecation is silent at the wire level. This becomes a correctness risk when federation routes `SignalMessage`s across relay versions. Add `version: u8` as a leading field to all variants before federation ships.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### M2 — BWE not consumed by `AdaptiveQualityController`
|
||||||
|
|
||||||
|
**Reference:** PROTOCOL-AUDIT.md W6, deferred to Phase V2
|
||||||
|
|
||||||
|
Quinn exposes `cwnd` and `bytes_in_flight`, but `AdaptiveQualityController` does not consume them. Loss + RTT adaptation works for audio. For video, without bandwidth estimation the encoder cannot detect available uplink capacity and will either oscillate or permanently under-utilize bandwidth. Mandatory before video production.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### M3 — PLI suppression window hardcoded at 200ms
|
||||||
|
|
||||||
|
**File:** `crates/wzp-relay/src/room.rs:1060`
|
||||||
|
|
||||||
|
Not adaptive to link speed. On slow links 200ms may allow multiple keyframe requests. Accept for Phase 1; make configurable in Phase 2.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### M4 — Repair packet index wrapping in FEC encoder
|
||||||
|
|
||||||
|
**File:** `crates/wzp-fec/src/encoder.rs:140`
|
||||||
|
|
||||||
|
```rust
|
||||||
|
let idx = (num_source as u8).wrapping_add(i as u8);
|
||||||
|
```
|
||||||
|
|
||||||
|
If `num_source + repair_count > 255`, indices wrap silently. In practice bounded by `frames_per_block` (5–10), so max sum is ~20. Low risk today; widen to u16 when `fec_block_id` is widened (H2).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### M5 — `timestamp_ms` monotonicity after rekey not enforced
|
||||||
|
|
||||||
|
**Reference:** PROTOCOL-AUDIT.md W3
|
||||||
|
|
||||||
|
Spec: `timestamp_ms` must not reset on rekey. The code correctly does not reset it, but there is no assertion to prevent regression. Add a debug assert in `complete_rekey()` that `new_session.next_timestamp >= old_session.last_timestamp`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Low / Accepted Debt
|
||||||
|
|
||||||
|
| ID | Description | File | Accepted in |
|
||||||
|
|---|---|---|---|
|
||||||
|
| L1 | 9 pre-existing clippy lints in `wzp-codec` | `aec.rs`, `denoise.rs`, `opus_enc.rs`, `codec2_{enc,dec}.rs`, `resample.rs` | PROTOCOL-AUDIT.md |
|
||||||
|
| L2 | 3 clippy errors in `deps/featherchat` submodule | `ratchet.rs`, `types.rs` | PROTOCOL-AUDIT.md |
|
||||||
|
| L3 | Audio anti-replay window 64 packets | `wzp-crypto/src/session.rs:89` | Accepted — jitter buffer + PLC masks loss |
|
||||||
|
| L4 | Debug tap logs at INFO with no rate limiting | `wzp-relay/src/room.rs:46–59` | Safe in dev; add 1:100 sampling for prod |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## What Was Not Found
|
||||||
|
|
||||||
|
These are explicitly confirmed sound after code-level verification:
|
||||||
|
|
||||||
|
- **Anti-replay bitmap** — correct u32 wrapping, per-stream isolation, window sizing by `MediaType`
|
||||||
|
- **HKDF + X25519 + Ed25519 key agreement** — standard construction, no gaps
|
||||||
|
- **SAS code derivation** — SHA-256(shared_secret)[:4] as 4-digit voice verification code
|
||||||
|
- **Rekey forward secrecy** — `session_id` rotation on rekey isolates nonce namespaces; seq counter reset is intentional and safe
|
||||||
|
- **MiniHeader v2 `seq_delta`** — fully implemented at `wzp-proto/src/packet.rs:469–526` with tests; PROTOCOL-AUDIT resolution table is accurate
|
||||||
|
- **SFU E2E preservation** — relay ciphertext passthrough, no plaintext access
|
||||||
|
- **RaptorQ for Codec2** — correct tool for the bitrate regime
|
||||||
|
- **DRED continuous tuning** — better than discrete tiers; 15% loss floor is empirically grounded
|
||||||
|
- **Jitter buffer** — BTreeMap with wrapping-aware comparisons, EWMA adaptive playout delay, solid
|
||||||
|
- **Quinn QUIC datagram transport** — correct primitives for unreliable media
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Fix Priority Table
|
||||||
|
|
||||||
|
| # | Issue | Category | Effort | Blocks |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| 1 | C1: nonce → `MediaHeader.seq` | Crypto | 1h | All sessions on lossy paths |
|
||||||
|
| 2 | C2: verify AEAD on all datagram send paths | Crypto | 1h | E2E security claim |
|
||||||
|
| 3 | C3: wire `VideoScorer::observe()` into room | Relay | 2h | Relay abuse control for video |
|
||||||
|
| 4 | C4: NDK 0.9 `mediacodec.rs` migration (5 categories) | Android | 2h | Android video |
|
||||||
|
| 5 | H1: wire AV1 factory into call engine | Video | 2h | Desktop video |
|
||||||
|
| 6 | H2: widen `fec_block_id` to `u16` | FEC/Wire | 30min | Next protocol release |
|
||||||
|
| 7 | M1: `SignalMessage` version byte | Proto | 1h | Federation correctness |
|
||||||
|
| 8 | M2: BWE into `AdaptiveQualityController` | Transport | 2–3 days | Video production quality |
|
||||||
|
|
||||||
|
**Total for C1–H1 (items 1–5):** ~8 hours focused engineering.
|
||||||
166
docs/HANDOFF-2026-05-12.md
Normal file
166
docs/HANDOFF-2026-05-12.md
Normal file
@@ -0,0 +1,166 @@
|
|||||||
|
# Handoff — 2026-05-12 EOD
|
||||||
|
|
||||||
|
## TL;DR
|
||||||
|
|
||||||
|
Wave 5 (Phase 5) and Wave 6 (Phase 6) implementation is complete and approved on the board. Stopping for the night with one open issue: `wzp-video` does not target-compile for `aarch64-linux-android` and needs a focused `ndk = "0.9"` API migration session (~1–2 h). Nothing live is blocked — Tauri Android does not yet consume `wzp-video`.
|
||||||
|
|
||||||
|
**Branch state:** local `experimental-ui` HEAD `f3e3ee5`, pushed to `github` only. **Not yet on `fj`** (deploy key was read-only). Build server (`manwe@manwehs`) is up to date via github fetch.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## What landed today
|
||||||
|
|
||||||
|
| Wave | Tasks approved | New crates / files | Test delta |
|
||||||
|
|---|---|---|---|
|
||||||
|
| 5 | T5.1, T5.1.1, T5.2, T5.3, T5.4, T5.5, T5.6, T5.7, T5.7.1, T5.8 | `crates/wzp-relay/src/audio_scorer.rs`, `response_policy.rs`, `verdict.rs`; `wzp-video/src/controller.rs`, `simulcast.rs`, `encoder_mode.rs`; H.265 path in VT + MediaCodec | wzp-relay 99→127, wzp-video 43→71 |
|
||||||
|
| 6 | T6.1 (+ rework), T6.1.2, T6.2 | `wzp-video/src/av1_obu.rs`, `dav1d.rs`, `svt_av1.rs`, `factory.rs`; VT AV1 decoder; MediaCodec AV1; `wzp-relay/src/video_scorer.rs` | wzp-video 76→88, wzp-relay 127→137 |
|
||||||
|
|
||||||
|
Total: ~30 task units approved across the two waves. Workspace tests at 702 passing (excluding `wzp-android`).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Open / next-up
|
||||||
|
|
||||||
|
### Top of queue
|
||||||
|
|
||||||
|
- **T4.3.1.1 (deferred → in-progress, blocked)** — Android target-compile of `wzp-video`. We started this tonight and hit 31 errors in `crates/wzp-video/src/mediacodec.rs` against the actual `ndk = "0.9"` API. Error categories captured below; resume with one fix-per-category commit, then attempt device instrumentation.
|
||||||
|
- **T6.3 — federated reputation gossip.** Design exploration committed (`1e729e4`, `docs/PRD/PRD-relay-federation-gossip.md`). **Decision made: Approach 3 (Ban-List Distribution).** My answers to the 6 blocker questions are in the chat thread, awaiting conversion to a real Files/Steps/Verify/Done-when task spec for the agent. The user opted not to run the agent immediately; the task spec is a write-then-park.
|
||||||
|
- **T5.1.1 follow-ups** — none. T5.1.1 closed clean.
|
||||||
|
|
||||||
|
### Latent follow-ups from earlier waves
|
||||||
|
|
||||||
|
These pre-date wave 6 and are still open:
|
||||||
|
|
||||||
|
- **AEAD wired into prod send/recv path** (referenced in T1.5 / T1.6 reports). Encryption is implemented in `wzp-crypto` but not yet on every QUIC datagram path.
|
||||||
|
- **AEAD nonce derivation: switch to `MediaHeader::seq`** (cited in T1.5.x reports). Current scheme works but isn't tied to wire-level seq.
|
||||||
|
- **`wzp-codec` clippy debt sprint** — 9 errors documented as known debt in `docs/PROTOCOL-AUDIT.md`.
|
||||||
|
- **T6.1.2 — wire AV1 into actual call engine.** The factory + step tables landed (commit `086d0a4`); no caller invokes `create_video_encoder(Av1Main, …)` yet. Real video sender wiring (the originally-blocked task) is unstarted.
|
||||||
|
- **T6.2-follow-up — wire `VideoScorer::observe()` into the packet path.** TODO marker at `crates/wzp-relay/src/room.rs:1263`.
|
||||||
|
|
||||||
|
### Permanently deferred
|
||||||
|
|
||||||
|
- **T6.1.1 — Android MediaCodec AV1 device validation.** Deferred indefinitely: the user does not own an AV1-encode-capable Android or iPhone, and AV1 hardware will not be widespread for years. Revisit when devices land.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## The T4.3.1.1 Android build situation
|
||||||
|
|
||||||
|
What we did tonight:
|
||||||
|
|
||||||
|
1. Pushed `experimental-ui` to `github` (deploy key on `fj` is read-only).
|
||||||
|
2. Added `github` as a remote on `manwe@manwehs:~/wzp-builder/data/source/` and checked out `experimental-ui`.
|
||||||
|
3. Ran `cargo build --target aarch64-linux-android -p wzp-video` inside the `wzp-android-builder:latest` docker image.
|
||||||
|
4. First failure: `shiguredo_dav1d` and `shiguredo_svt_av1` build scripts panic with `unsupported target: os=android, arch=aarch64`. Fixed in commit `f3e3ee5` (`fix(wzp-video): cfg-gate dav1d + svt-av1 off Android target`) — those crates now live under `[target.'cfg(not(target_os = "android"))'.dependencies]`, since Android uses MediaCodec for AV1 anyway.
|
||||||
|
5. Re-ran the build → 31 errors in `mediacodec.rs`. **Stopped here.**
|
||||||
|
|
||||||
|
### Error categories to fix tomorrow
|
||||||
|
|
||||||
|
Run the same docker invocation and tackle these one fix-commit per category:
|
||||||
|
|
||||||
|
| Error | Count | Root cause | Likely fix |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `E0277` `NonNull<AMediaCodec>` not `Send` | ~3 | Raw pointer field on a struct held across `tokio::spawn`-able boundaries | Wrap in `struct SendMediaCodec(NonNull<…>); unsafe impl Send for SendMediaCodec {}` or use the `ndk` crate's owned `MediaCodec` type which already implements `Send` |
|
||||||
|
| `E0308` `&[MaybeUninit<u8>]` vs `&[u8]` | many | `ndk 0.9` returns uninitialized buffer slices; agent wrote into them as if initialized | Use `MaybeUninit::write_slice` or transmute pattern; pattern matches what `InputBuffer::write` expects |
|
||||||
|
| `E0425` missing `BITRATE_MODE_CBR` | 1+ | Constant moved/renamed in `ndk 0.9` | Search `ndk` crate docs for current constant name (likely under `MediaCodec::set_parameters` enum) |
|
||||||
|
| `E0433` `ndk_sys` not linked | several | Agent imported `ndk_sys` directly; it's not a dep, only `ndk = "0.9"` is | Replace direct `ndk_sys` calls with safe wrappers from the `ndk` crate, or add `ndk_sys` as an explicit dep |
|
||||||
|
| `E0599` `InputBuffer::index()` / `OutputBuffer::index()` private | 2 | Both are private fields in `ndk 0.9`; were public methods in older versions | Either use the buffer through its safe API (queue/dequeue by handle) or expose index via a different accessor — read the `ndk` source for current API |
|
||||||
|
|
||||||
|
### Reproduce the build
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ssh -i ~/CascadeProjects/wzp manwe@manwehs \
|
||||||
|
'cd ~/wzp-builder/data/source && \
|
||||||
|
docker run --rm \
|
||||||
|
-v ~/wzp-builder/data/source:/build/source \
|
||||||
|
-v ~/wzp-builder/data/cache/cargo-registry:/home/builder/.cargo/registry \
|
||||||
|
-v ~/wzp-builder/data/cache/cargo-git:/home/builder/.cargo/git \
|
||||||
|
-v ~/wzp-builder/data/cache/target:/build/source/target \
|
||||||
|
wzp-android-builder:latest \
|
||||||
|
bash -c "cd /build/source && cargo build --target aarch64-linux-android -p wzp-video 2>&1 | tail -100"'
|
||||||
|
```
|
||||||
|
|
||||||
|
After local fixes:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git push github experimental-ui && \
|
||||||
|
ssh -i ~/CascadeProjects/wzp manwe@manwehs \
|
||||||
|
'cd ~/wzp-builder/data/source && git fetch github && git reset --hard github/experimental-ui'
|
||||||
|
# then re-run the docker build
|
||||||
|
```
|
||||||
|
|
||||||
|
### Device instrumentation half (post-compile)
|
||||||
|
|
||||||
|
User has a physical Android device. Once `cargo build --target aarch64-linux-android -p wzp-video` is clean:
|
||||||
|
|
||||||
|
- Build a minimal test harness binary (probably under `wzp-video/examples/` or a new `wzp-android-test/` crate) that does encode → decode of a synthetic frame via MediaCodec.
|
||||||
|
- Use `adb push` and `adb shell run` to exercise it.
|
||||||
|
- Compare output bytes against the dav1d/SVT-AV1 SW roundtrip from `crates/wzp-video/src/svt_av1.rs:101 svt_av1_dav1d_roundtrip_10_frames`.
|
||||||
|
|
||||||
|
Out of scope for tomorrow if the API migration eats the whole session.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## T6.3 — Approach 3 decision
|
||||||
|
|
||||||
|
User picked Approach 3 (Ban-List Distribution) from `docs/PRD/PRD-relay-federation-gossip.md`. My answers to the 6 open questions:
|
||||||
|
|
||||||
|
1. **Trust model:** Single admin key (user). Strongest Sybil resistance, lowest complexity.
|
||||||
|
2. **Key infra:** Reuse `wzp-crypto` Ed25519. Admin pubkey in relay config; relays verify list signatures.
|
||||||
|
3. **Fingerprint scope:** Ed25519 pubkey, not IP. Resistant to NAT rebind evasion.
|
||||||
|
4. **Privacy:** Publish `SHA-256(pubkey)` hashes, not raw pubkeys. Relays compute `H(observed)` and match. 256-bit space makes brute-force infeasible; loses some audit trail.
|
||||||
|
5. **TTL:** 30-day per-entry auto-expiry. Forces ops to actively re-publish persistent bans; prevents forever-by-mistake.
|
||||||
|
6. **Rate limiting:** N/A under Approach 3 (no gossip channel; relays poll a signed list at configurable interval, that interval is the rate limit).
|
||||||
|
|
||||||
|
Next step: turn these into a Files/Steps/Verify/Done-when task spec in `docs/PRD/TASKS.md` and move T6.3 from `Blocked` → `Open` ready for the agent to claim. User did not want this kicked off tonight.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Build / sync state
|
||||||
|
|
||||||
|
| Location | Branch | HEAD |
|
||||||
|
|---|---|---|
|
||||||
|
| Local (Mac) | `experimental-ui` | `f3e3ee5 fix(wzp-video): cfg-gate dav1d + svt-av1 off Android target` |
|
||||||
|
| `github` remote | `experimental-ui` | `f3e3ee5` (pushed) |
|
||||||
|
| `fj` remote | `experimental-ui` | **not pushed** (deploy key read-only on `fj`) |
|
||||||
|
| `origin` (git.manko.yoga) | `experimental-ui` | **not pushed** |
|
||||||
|
| Build server `~/wzp-builder/data/source` | `experimental-ui` | `f3e3ee5` |
|
||||||
|
|
||||||
|
If you want everything on `fj` / `origin` too, get the deploy key write-privileged or push from a different identity.
|
||||||
|
|
||||||
|
`fj/main` and `github/main` have one commit (`9ae9441 fix(audio): check capture ring available...`) that doesn't exist on `experimental-ui` — a small audio fix from May 11. Cherry-pick or merge before merging `experimental-ui` back into `main`.
|
||||||
|
|
||||||
|
### Gitleaks allowlist
|
||||||
|
|
||||||
|
Added `.gitleaks.toml` in commit `f28f39d` to allowlist 4 pre-existing historical findings. Two are real tokens (paste.tbs.amn.gg and paste.dk.manko.yoga `Authorization` headers in `scripts/build*.sh`). **Rotate those tokens if those endpoints still authenticate** — the allowlist only silences the pre-push hook; the secrets are still in git history.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Agent process notes for tomorrow
|
||||||
|
|
||||||
|
The Kimi Code CLI agent on this project has a **stable, well-documented fabrication tic** — one verifiable detail per report is wrong (SHA, "updated X in same commit", fmt/clippy passes, etc.). Pattern survived an explicit CR on T6.1.
|
||||||
|
|
||||||
|
**Updated policy** (in `memory/feedback_kimi_report_fabrication.md`):
|
||||||
|
|
||||||
|
1. **Always verify the SHA** in the report header against `git log`.
|
||||||
|
2. **Always run** `cargo fmt --check` and `cargo clippy -- -D warnings` yourself — don't trust the report's claims.
|
||||||
|
3. **Don't CR fabrications anymore** — the T6.1 CR didn't change the behavior. Reviewer-fix the detail, note on the board, move on. Reserve CRs for substance issues.
|
||||||
|
|
||||||
|
The substance of the code has been consistently good. Don't let the fabrication tic bias review of the code itself.
|
||||||
|
|
||||||
|
### Rebase tic
|
||||||
|
|
||||||
|
Agent has twice rewritten already-pushed commits to address CR feedback (T5.7.1 `d3b2da6` → `517d0eb`; T6.1 `0de9522` → `9334aa5`). Forward fix commits are the rule; rebasing wasn't asked for and breaks reviewer references. Mention this only if it happens a third time.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Tomorrow's suggested checklist
|
||||||
|
|
||||||
|
1. **(20 min)** Read this doc, the `feedback_kimi_report_fabrication.md` memory, and the T6.1 / T6.2 / T6.1.2 board rows on `docs/PRD/TASKS.md` to reload context.
|
||||||
|
2. **(1–2 h)** Resume T4.3.1.1: ndk-0.9 API migration in `crates/wzp-video/src/mediacodec.rs`. One commit per error category.
|
||||||
|
3. **(30 min)** If migration lands clean, attempt the minimal device test on the user's Android phone.
|
||||||
|
4. **(20 min, optional)** Convert the T6.3 design answers into a task spec block in `TASKS.md`, leave it `Open` for the agent. Don't kick off the agent unless asked.
|
||||||
|
5. **(parking lot)** AEAD prod wiring + nonce switch + wzp-codec clippy sprint — none urgent.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
*Generated 2026-05-12, end of Wave 6 push.*
|
||||||
225
docs/PRD/PRD-android-mediacodec-ndk9.md
Normal file
225
docs/PRD/PRD-android-mediacodec-ndk9.md
Normal file
@@ -0,0 +1,225 @@
|
|||||||
|
# PRD: Android MediaCodec NDK 0.9 Compatibility
|
||||||
|
|
||||||
|
> **Status:** proposed
|
||||||
|
> **Resolves:** 31 compile errors in `crates/wzp-video/src/mediacodec.rs` blocking all Android video.
|
||||||
|
> **Depends on:** Remote build server `manwe@188.245.59.196` with Docker image `wzp-android-builder:latest`.
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
`crates/wzp-video/src/mediacodec.rs` fails to compile for
|
||||||
|
`aarch64-linux-android` against the NDK 0.9 Rust crate. There are 31 errors
|
||||||
|
in 5 categories. Android video is completely blocked.
|
||||||
|
|
||||||
|
The file already compiles for non-Android targets (all Android code is behind
|
||||||
|
`#[cfg(target_os = "android")]`). Only the Android target path needs fixing.
|
||||||
|
|
||||||
|
## Goals
|
||||||
|
|
||||||
|
- `cargo build --target aarch64-linux-android -p wzp-video` produces 0 errors on the remote server.
|
||||||
|
- Each fix category lands in a separate commit so failures can be bisected.
|
||||||
|
- Non-Android compilation is not broken.
|
||||||
|
|
||||||
|
## Non-goals
|
||||||
|
|
||||||
|
- Upgrading the NDK Docker image or changing the NDK version.
|
||||||
|
- Fixing video functionality beyond compilation (runtime testing is a separate task).
|
||||||
|
- Any files outside `crates/wzp-video/`.
|
||||||
|
|
||||||
|
## Design
|
||||||
|
|
||||||
|
### Build command (run after each fix)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ssh manwe@188.245.59.196 'cd ~/wzp-builder/data/source && \
|
||||||
|
git fetch github && git reset --hard github/experimental-ui && \
|
||||||
|
docker run --rm \
|
||||||
|
-v ~/wzp-builder/data/source:/build/source \
|
||||||
|
-v ~/wzp-builder/data/cache/cargo-registry:/home/builder/.cargo/registry \
|
||||||
|
-v ~/wzp-builder/data/cache/cargo-git:/home/builder/.cargo/git \
|
||||||
|
-v ~/wzp-builder/data/cache/target:/build/source/target \
|
||||||
|
wzp-android-builder:latest bash -c \
|
||||||
|
"cd /build/source && cargo build --target aarch64-linux-android -p wzp-video 2>&1 | grep -E \"^error\" | head -30"'
|
||||||
|
```
|
||||||
|
|
||||||
|
### Fix order (commit one per category)
|
||||||
|
|
||||||
|
#### Fix 1 — `E0433`: `ndk_sys` not declared as a dependency
|
||||||
|
|
||||||
|
**Symptom**: `use of undeclared crate or module 'ndk_sys'`
|
||||||
|
|
||||||
|
**File**: `crates/wzp-video/Cargo.toml`
|
||||||
|
|
||||||
|
NDK 0.9 no longer re-exports raw `ndk_sys` symbols; they must be declared as
|
||||||
|
a direct dependency. Add to the `[target.'cfg(target_os = "android")'.dependencies]`
|
||||||
|
section (or create it if absent):
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[target.'cfg(target_os = "android")'.dependencies]
|
||||||
|
ndk = { version = "0.9" }
|
||||||
|
ndk-sys = { version = "0.6" } # ndk 0.9 depends on ndk-sys 0.6
|
||||||
|
```
|
||||||
|
|
||||||
|
If `mediacodec.rs` only uses safe wrappers from the `ndk` crate and the
|
||||||
|
`ndk_sys` imports are not strictly needed, remove the `use ndk_sys::*` lines
|
||||||
|
from `mediacodec.rs` instead — whichever approach results in fewer changes.
|
||||||
|
|
||||||
|
After this fix the `E0433` errors should drop from the build output.
|
||||||
|
|
||||||
|
#### Fix 2 — `E0425`: `BITRATE_MODE_CBR` constant missing
|
||||||
|
|
||||||
|
**Symptom**: `cannot find value 'BITRATE_MODE_CBR' in this scope`
|
||||||
|
|
||||||
|
**File**: `crates/wzp-video/src/mediacodec.rs`
|
||||||
|
|
||||||
|
`BITRATE_MODE_CBR` is already defined as a local constant at line 44:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
#[cfg(target_os = "android")]
|
||||||
|
const BITRATE_MODE_CBR: i32 = 2;
|
||||||
|
```
|
||||||
|
|
||||||
|
If the error persists after Fix 1, the issue is that `ndk_sys` was providing
|
||||||
|
a conflicting symbol. Verify the constant is still at line 44 after Fix 1. If
|
||||||
|
NDK 0.9 moved `BITRATE_MODE_CBR` to an enum, update the usage at line 516
|
||||||
|
(`format.set_i32("bitrate-mode", BITRATE_MODE_CBR)`) to use the integer
|
||||||
|
value directly (`2`) or update the constant's value.
|
||||||
|
|
||||||
|
If `ndk 0.9` defines `MediaCodecBitrateMode::Cbr` as an enum, the call site
|
||||||
|
in `MediaCodecAv1Encoder::new` (line ~516) can be updated to:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
format.set_i32(
|
||||||
|
"bitrate-mode",
|
||||||
|
ndk::media::media_codec::MediaCodecBitrateMode::Cbr as i32,
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
|
#### Fix 3 — `E0308`: `InputBuffer` returns `&mut [MaybeUninit<u8>]`
|
||||||
|
|
||||||
|
**Symptom**: `expected &mut [u8], found &mut [MaybeUninit<u8>]`
|
||||||
|
|
||||||
|
**File**: `crates/wzp-video/src/mediacodec.rs`
|
||||||
|
|
||||||
|
NDK 0.9 changed `InputBuffer::buffer_mut()` from `&mut [u8]` to
|
||||||
|
`&mut [MaybeUninit<u8>]`. There are multiple write sites in the file — all
|
||||||
|
follow the same pattern:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// Before (NDK 0.8):
|
||||||
|
let buf = buffer.buffer_mut(); // &mut [u8]
|
||||||
|
let n = frame.data.len().min(buf.len());
|
||||||
|
buf[..n].copy_from_slice(&frame.data[..n]);
|
||||||
|
```
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// After (NDK 0.9):
|
||||||
|
let buf = buffer.buffer_mut(); // &mut [MaybeUninit<u8>]
|
||||||
|
let n = frame.data.len().min(buf.len());
|
||||||
|
for (d, &s) in buf[..n].iter_mut().zip(frame.data[..n].iter()) {
|
||||||
|
d.write(s);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The file already uses the `d.write(s)` pattern in some places (lines 125–127,
|
||||||
|
297–299, etc.). Search for **every** occurrence of `buffer.buffer_mut()` and
|
||||||
|
`buffer_mut()` and apply the same pattern. Affected structs:
|
||||||
|
`MediaCodecEncoder::encode` (~line 123), `MediaCodecDecoder::decode`
|
||||||
|
(~line 294), `MediaCodecHevcEncoder::encode` (~line 439),
|
||||||
|
`MediaCodecHevcDecoder::decode` (~line 773), `MediaCodecAv1Encoder::encode`
|
||||||
|
(~line 560), `MediaCodecAv1Decoder::decode` (~line 907).
|
||||||
|
|
||||||
|
Do NOT use `unsafe { std::mem::transmute }` — the `d.write(s)` pattern is
|
||||||
|
already present and safe.
|
||||||
|
|
||||||
|
Note: if the file already uses `d.write(s)` everywhere, this category may
|
||||||
|
already be addressed by the existing code. Check the actual error count.
|
||||||
|
|
||||||
|
#### Fix 4 — `E0599`: `.index()` is private
|
||||||
|
|
||||||
|
**Symptom**: `method 'index' is private`
|
||||||
|
|
||||||
|
**File**: `crates/wzp-video/src/mediacodec.rs`
|
||||||
|
|
||||||
|
NDK 0.9 removed the public `.index()` method from `DequeuedInputBuffer` and
|
||||||
|
`DequeuedOutputBuffer`. The pattern that broke:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// Broken: buffer.index() is private in NDK 0.9
|
||||||
|
let idx = buffer.index();
|
||||||
|
codec.queue_input_buffer_index(idx, ...);
|
||||||
|
```
|
||||||
|
|
||||||
|
In NDK 0.9 the correct API is to pass the buffer object directly to
|
||||||
|
`queue_input_buffer`:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
codec.queue_input_buffer(buffer, offset, size, pts_us, flags)?;
|
||||||
|
```
|
||||||
|
|
||||||
|
The file already uses `codec.queue_input_buffer(buffer, 0, to_copy, ...)` in
|
||||||
|
most places (lines 131, 303, 447, etc.). Search for any remaining `.index()`
|
||||||
|
calls on buffer objects and replace them with the direct-pass pattern shown
|
||||||
|
above.
|
||||||
|
|
||||||
|
#### Fix 5 — `E0277`: `NonNull<AMediaCodec>` is not `Send`
|
||||||
|
|
||||||
|
**Symptom**: `NonNull<AMediaCodec>` cannot be sent between threads safely
|
||||||
|
|
||||||
|
**File**: `crates/wzp-video/src/mediacodec.rs`
|
||||||
|
|
||||||
|
Each codec struct must have an `unsafe impl Send` declaration. Audit all six
|
||||||
|
codec structs:
|
||||||
|
|
||||||
|
| Struct | `unsafe impl Send` present? |
|
||||||
|
|--------|----------------------------|
|
||||||
|
| `MediaCodecEncoder` | Yes (line 51) |
|
||||||
|
| `MediaCodecDecoder` | Yes (line 228) |
|
||||||
|
| `MediaCodecHevcEncoder` | Yes (line 374) |
|
||||||
|
| `MediaCodecHevcDecoder` | Yes (line 705) |
|
||||||
|
| `MediaCodecAv1Encoder` | Yes (line 503) |
|
||||||
|
| `MediaCodecAv1Decoder` | Yes (line 844) |
|
||||||
|
|
||||||
|
If any are missing, add them with a safety comment:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// SAFETY: AMediaCodec is documented as thread-safe.
|
||||||
|
#[cfg(target_os = "android")]
|
||||||
|
unsafe impl Send for MediaCodecXxxYyy {}
|
||||||
|
```
|
||||||
|
|
||||||
|
This category may already be clean. Confirm with the build output.
|
||||||
|
|
||||||
|
## Implementation steps
|
||||||
|
|
||||||
|
1. Push the current branch to `github/experimental-ui` before starting.
|
||||||
|
2. **Commit 1**: Fix `ndk_sys` dependency (`Cargo.toml`). Push. Run build.
|
||||||
|
Confirm `E0433` errors drop.
|
||||||
|
3. **Commit 2**: Fix `BITRATE_MODE_CBR`. Push. Run build. Confirm `E0425` gone.
|
||||||
|
4. **Commit 3**: Fix `MaybeUninit` write sites. Push. Run build. Confirm
|
||||||
|
`E0308` gone.
|
||||||
|
5. **Commit 4**: Remove any `.index()` calls. Push. Run build. Confirm
|
||||||
|
`E0599` gone.
|
||||||
|
6. **Commit 5**: Add missing `unsafe impl Send` if any. Push. Run build.
|
||||||
|
Confirm `E0277` gone and total error count is 0.
|
||||||
|
|
||||||
|
## Files to read before implementing
|
||||||
|
|
||||||
|
- `crates/wzp-video/src/mediacodec.rs` (full file — 45 KB; read in chunks)
|
||||||
|
- `crates/wzp-video/Cargo.toml` (check existing `[dependencies]` sections)
|
||||||
|
|
||||||
|
## Verify
|
||||||
|
|
||||||
|
Final build command (see Design section). Expected output: no lines matching
|
||||||
|
`^error`.
|
||||||
|
|
||||||
|
Also verify non-Android host still compiles:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo check -p wzp-video
|
||||||
|
```
|
||||||
|
|
||||||
|
## Done when
|
||||||
|
|
||||||
|
`cargo build --target aarch64-linux-android -p wzp-video` on the remote
|
||||||
|
server produces 0 `error[...]` lines. Non-Android `cargo check -p wzp-video`
|
||||||
|
also passes.
|
||||||
260
docs/PRD/PRD-clippy-debt.md
Normal file
260
docs/PRD/PRD-clippy-debt.md
Normal file
@@ -0,0 +1,260 @@
|
|||||||
|
# PRD: Fix wzp-codec Clippy Lint Debt
|
||||||
|
|
||||||
|
> **Status:** proposed
|
||||||
|
> **Resolves:** 9 pre-existing clippy lints in `crates/wzp-codec/src/` that cause `cargo clippy --workspace -D warnings` to fail, breaking any strict-CI configuration.
|
||||||
|
> **Depends on:** Nothing — all changes are in `crates/wzp-codec/src/`.
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
`cargo clippy -p wzp-codec -- -D warnings` fails with 9 lints across 5 files.
|
||||||
|
These are pre-existing code patterns that were never flagged during development
|
||||||
|
because the CI flag was not set. They have no runtime impact today but prevent
|
||||||
|
adding `-D warnings` to CI without first cleaning them up.
|
||||||
|
|
||||||
|
The 3 errors in `deps/featherchat` are in a submodule — do NOT touch them.
|
||||||
|
`warzone_protocol` clippy errors are accepted debt (not our code).
|
||||||
|
|
||||||
|
## Goals
|
||||||
|
|
||||||
|
- `cargo clippy -p wzp-codec -- -D warnings` exits 0.
|
||||||
|
- No behavior changes — every fix is a semantically equivalent rewrite.
|
||||||
|
- No changes outside `crates/wzp-codec/src/`.
|
||||||
|
|
||||||
|
## Non-goals
|
||||||
|
|
||||||
|
- Fixing clippy lints in any crate other than `wzp-codec`.
|
||||||
|
- Adding new functionality.
|
||||||
|
- Touching the `deps/featherchat` submodule.
|
||||||
|
|
||||||
|
## Design
|
||||||
|
|
||||||
|
### Lint inventory
|
||||||
|
|
||||||
|
| Lint | Count | File | Approx line | Fix |
|
||||||
|
|------|-------|------|-------------|-----|
|
||||||
|
| `implicit_saturating_sub` | 1 | `aec.rs` | 117–119 | `saturating_sub` |
|
||||||
|
| `needless_range_loop` | 2 | `aec.rs:164`, `resample.rs:51` | — | iterate with `iter().enumerate()` or direct iter |
|
||||||
|
| `manual_div_ceil` | 2 | `codec2_dec.rs:48`, `codec2_enc.rs:48` | — | `div_ceil` |
|
||||||
|
| `manual_clamp` | 2 | `denoise.rs:59`, `opus_enc.rs:250` | — | `.clamp(min, max)` |
|
||||||
|
| `manual_ascii_check` | 1 | `opus_enc.rs:104` | — | `.eq_ignore_ascii_case()` |
|
||||||
|
| `same_item_push` | 1 | `resample.rs:184` | — | `vec.resize` or `extend(repeat)` |
|
||||||
|
|
||||||
|
### Fix details
|
||||||
|
|
||||||
|
#### 1. `implicit_saturating_sub` — `aec.rs` line ~117
|
||||||
|
|
||||||
|
Current code:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
fn delay_available(&self) -> usize {
|
||||||
|
let buffered = self.delay_write - self.delay_read;
|
||||||
|
if buffered > self.delay_samples {
|
||||||
|
buffered - self.delay_samples
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Clippy wants `saturating_sub` because the subtraction can underflow if
|
||||||
|
`buffered < self.delay_samples`:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
fn delay_available(&self) -> usize {
|
||||||
|
let buffered = self.delay_write - self.delay_read;
|
||||||
|
buffered.saturating_sub(self.delay_samples)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
This is semantically identical (both return 0 when `buffered <= delay_samples`).
|
||||||
|
|
||||||
|
#### 2a. `needless_range_loop` — `aec.rs` line ~164
|
||||||
|
|
||||||
|
Current code:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
for i in 0..n {
|
||||||
|
let near_f = nearend[i] as f32;
|
||||||
|
let base = (self.far_pos + fl * ((n / fl) + 2) + i - n) % fl;
|
||||||
|
...
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`i` is used both to index `nearend[i]` and in arithmetic (`+ i - n`).
|
||||||
|
Clippy fires because `nearend[i]` could use `.iter().enumerate()`.
|
||||||
|
Convert to `enumerate`:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
for (i, &sample) in nearend.iter().enumerate() {
|
||||||
|
let near_f = sample as f32;
|
||||||
|
let base = (self.far_pos + fl * ((n / fl) + 2) + i - n) % fl;
|
||||||
|
...
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Make sure to update any references to `nearend[i]` inside the loop body
|
||||||
|
to use `sample` (or `near_f` directly). Also update the NLMS adaptation
|
||||||
|
sub-loop if it references `nearend[i]`.
|
||||||
|
|
||||||
|
#### 2b. `needless_range_loop` — `resample.rs` line ~51
|
||||||
|
|
||||||
|
Current code:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
for i in 0..FIR_TAPS {
|
||||||
|
let n = i as f64 - m / 2.0;
|
||||||
|
let sinc = ...;
|
||||||
|
let t = 2.0 * i as f64 / m - 1.0;
|
||||||
|
let kaiser = ...;
|
||||||
|
kernel[i] = sinc * kaiser;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`i` is used both as an index (`kernel[i]`) and in arithmetic. Use
|
||||||
|
`iter_mut().enumerate()`:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
for (i, slot) in kernel.iter_mut().enumerate() {
|
||||||
|
let n = i as f64 - m / 2.0;
|
||||||
|
let sinc = ...;
|
||||||
|
let t = 2.0 * i as f64 / m - 1.0;
|
||||||
|
let kaiser = ...;
|
||||||
|
*slot = sinc * kaiser;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 3a. `manual_div_ceil` — `codec2_dec.rs` line ~48
|
||||||
|
|
||||||
|
Current code:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
fn bytes_per_frame(&self) -> usize {
|
||||||
|
(self.inner.bits_per_frame() + 7) / 8
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Replace with:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
fn bytes_per_frame(&self) -> usize {
|
||||||
|
self.inner.bits_per_frame().div_ceil(8)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`div_ceil` is stable as of Rust 1.73. The builder uses a recent enough
|
||||||
|
toolchain. If `bits_per_frame()` returns `usize`, the method is available.
|
||||||
|
If it returns a different integer type, cast accordingly.
|
||||||
|
|
||||||
|
#### 3b. `manual_div_ceil` — `codec2_enc.rs` line ~48
|
||||||
|
|
||||||
|
Same pattern, same fix:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
fn bytes_per_frame(&self) -> usize {
|
||||||
|
self.inner.bits_per_frame().div_ceil(8)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 4a. `manual_clamp` — `denoise.rs` line ~59
|
||||||
|
|
||||||
|
Current code:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
let clamped = val.max(-32768.0).min(32767.0);
|
||||||
|
```
|
||||||
|
|
||||||
|
Replace with:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
let clamped = val.clamp(-32768.0_f32, 32767.0_f32);
|
||||||
|
```
|
||||||
|
|
||||||
|
Note: `.clamp()` on `f32` requires both bounds to be the same type. If `val`
|
||||||
|
is already `f32`, no extra cast is needed. Verify the type of `val` in
|
||||||
|
context (it is `f32` per the output array type `[f32; 480]`).
|
||||||
|
|
||||||
|
#### 4b. `manual_clamp` — `opus_enc.rs` line ~252
|
||||||
|
|
||||||
|
Read the surrounding code for the exact pattern. It will be something like:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
let v = if x < min_val { min_val } else if x > max_val { max_val } else { x };
|
||||||
|
```
|
||||||
|
|
||||||
|
or the `.max().min()` chain. Replace with `x.clamp(min_val, max_val)`.
|
||||||
|
|
||||||
|
#### 5. `manual_ascii_check` — `opus_enc.rs` line ~104
|
||||||
|
|
||||||
|
Current code:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
Ok(v) => !v.is_empty() && v != "0" && v.to_ascii_lowercase() != "false",
|
||||||
|
```
|
||||||
|
|
||||||
|
Clippy wants `.eq_ignore_ascii_case()` instead of lowercasing the whole string:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
Ok(v) => !v.is_empty() && v != "0" && !v.eq_ignore_ascii_case("false"),
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 6. `same_item_push` — `resample.rs` line ~183
|
||||||
|
|
||||||
|
Current code:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
for _ in 1..RATIO {
|
||||||
|
work.push(0.0);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
This pushes the same `0.0` value `(RATIO - 1)` times. Replace with:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
work.resize(work.len() + (RATIO - 1), 0.0f64);
|
||||||
|
```
|
||||||
|
|
||||||
|
Or equivalently:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
work.extend(std::iter::repeat(0.0f64).take(RATIO - 1));
|
||||||
|
```
|
||||||
|
|
||||||
|
Note: `RATIO` is a `const usize`. Verify `work` is `Vec<f64>` in context
|
||||||
|
(it is — `work.push(s as f64)` immediately before).
|
||||||
|
|
||||||
|
## Implementation steps
|
||||||
|
|
||||||
|
1. Read each file at the line numbers listed above to confirm the exact current
|
||||||
|
code before editing (line numbers may shift slightly due to prior edits).
|
||||||
|
2. Apply all 9 fixes. They are independent — no ordering requirement.
|
||||||
|
3. Run `cargo clippy -p wzp-codec -- -D warnings` locally or via the CI
|
||||||
|
command.
|
||||||
|
4. If any lint persists, re-read that file section and adjust.
|
||||||
|
|
||||||
|
## Files to read before implementing
|
||||||
|
|
||||||
|
- `crates/wzp-codec/src/aec.rs` lines 114–200
|
||||||
|
- `crates/wzp-codec/src/resample.rs` lines 45–70 and 178–190
|
||||||
|
- `crates/wzp-codec/src/codec2_dec.rs` lines 40–55
|
||||||
|
- `crates/wzp-codec/src/codec2_enc.rs` lines 40–55
|
||||||
|
- `crates/wzp-codec/src/denoise.rs` lines 45–65
|
||||||
|
- `crates/wzp-codec/src/opus_enc.rs` lines 96–110 and 244–260
|
||||||
|
|
||||||
|
## Verify
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo clippy -p wzp-codec -- -D warnings
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: exits 0 with no warnings.
|
||||||
|
|
||||||
|
Also run to confirm no regressions:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo test -p wzp-codec
|
||||||
|
```
|
||||||
|
|
||||||
|
## Done when
|
||||||
|
|
||||||
|
`cargo clippy -p wzp-codec -- -D warnings` exits 0. All 9 lints are gone.
|
||||||
|
`cargo test -p wzp-codec` passes. No changes outside `crates/wzp-codec/src/`.
|
||||||
98
docs/PRD/PRD-e2e-media-encryption.md
Normal file
98
docs/PRD/PRD-e2e-media-encryption.md
Normal file
@@ -0,0 +1,98 @@
|
|||||||
|
# PRD: E2E Media Encryption (rewrite)
|
||||||
|
|
||||||
|
> **Status:** proposed (supersedes prior version)
|
||||||
|
> **Resolves:** Real end-to-end media encryption between call participants.
|
||||||
|
> **Replaces:** The prior version of this PRD described wrapping `QuinnTransport` in `EncryptingTransport` using the pairwise client↔relay session. That approach was implemented (commit `52a6f5e`) and **broke voice between any two clients** because the relay does not decrypt+re-encrypt — see "Why the prior fix failed" below. The wrapping was reverted in commit `e8cab25`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Why the prior fix failed
|
||||||
|
|
||||||
|
`wzp_client::handshake::perform_handshake` performs ECDH **between the client and the relay**. Each client in a room ends up with a **different** pairwise session key (key_A for client A, key_B for client B, etc.).
|
||||||
|
|
||||||
|
The relay is an SFU — it forwards `MediaPacket` bytes between participants in a room without inspecting their payloads. The relay does not run a decrypt-then-encrypt step keyed per-recipient.
|
||||||
|
|
||||||
|
Wrapping `QuinnTransport` in `EncryptingTransport` therefore produced:
|
||||||
|
|
||||||
|
```
|
||||||
|
Client A: plaintext --[encrypt key_A]--> ciphertext --> Relay
|
||||||
|
Relay: forwards ciphertext (bytes) --> Client B
|
||||||
|
Client B: ciphertext --[decrypt key_B]--> garbage --> silent audio
|
||||||
|
```
|
||||||
|
|
||||||
|
Result: every recipient saw decryption failures, audio went silent.
|
||||||
|
|
||||||
|
This is **not a bug in `EncryptingTransport`** — the wrapper does exactly what it claims. The bug was thinking the pairwise client-relay session was usable for participant-to-participant media. It isn't.
|
||||||
|
|
||||||
|
## Goals
|
||||||
|
|
||||||
|
A future implementation must satisfy:
|
||||||
|
|
||||||
|
- Two clients in a room can exchange media that the **other client** can decrypt.
|
||||||
|
- The **relay cannot decrypt** any media payload (true E2E), OR alternatively, the relay can decrypt+re-encrypt per recipient (hop-by-hop, sometimes called SFU-trusted).
|
||||||
|
- Joining and leaving the room mid-call rotates keys so departed members can't decrypt subsequent traffic (forward secrecy on membership change).
|
||||||
|
- Compatible with the existing `MediaPacket` wire format (header in plaintext, payload encrypted).
|
||||||
|
|
||||||
|
## Two valid approaches
|
||||||
|
|
||||||
|
### Approach A — MLS group keys (true E2E)
|
||||||
|
|
||||||
|
Use the [MLS protocol](https://datatracker.ietf.org/doc/rfc9420/) (e.g. via the `openmls` crate) to derive a shared **group key** that all room members possess and the relay does not.
|
||||||
|
|
||||||
|
- Relay acts as a **delivery service** for MLS Handshake messages (`Welcome`, `Commit`, `Proposal`) but never sees the group secret.
|
||||||
|
- Every media packet is AEAD-sealed with the current group epoch key.
|
||||||
|
- Group rekey is triggered by:
|
||||||
|
- Member join/leave (forward secrecy on membership)
|
||||||
|
- Periodic (every N seconds or N packets) for post-compromise security
|
||||||
|
- Each room maintains its own MLS group; the relay just stores opaque `mls_blob` payloads in `SignalMessage::MlsHandshake`.
|
||||||
|
|
||||||
|
**Pros:** real E2E. Relay compromise does not leak media.
|
||||||
|
**Cons:** Significant complexity (MLS state machine per room, persistent ratchet trees, key schedule). Adds `openmls` dependency (~30 KLOC). Federation across relays is harder.
|
||||||
|
|
||||||
|
### Approach B — Hop-by-hop re-encryption at the relay
|
||||||
|
|
||||||
|
The relay holds a `CryptoSession` per connected client (which it already does — see `_crypto_session` discarded in `crates/wzp-relay/src/main.rs:1817`). On forward:
|
||||||
|
|
||||||
|
```
|
||||||
|
Relay.recv_media(from A): decrypt with key_A → plaintext
|
||||||
|
Relay.send_media(to B, C, D): for each recipient X, encrypt with key_X
|
||||||
|
```
|
||||||
|
|
||||||
|
This is the same model as Matrix Megolm-without-Megolm — encrypted hop-by-hop but the relay sees plaintext briefly in between.
|
||||||
|
|
||||||
|
**Pros:** Reuses existing per-client `ChaChaSession`. Implementation is ~100 lines in the relay's room forwarding loop. Federation works the same way (each relay-relay hop has its own session).
|
||||||
|
**Cons:** Relay sees plaintext. A compromised relay can record and decrypt all media. This is **not E2E** — but it is strictly stronger than the current state (plaintext-over-QUIC-TLS exposes media to anyone with a TLS-terminating proxy on the relay).
|
||||||
|
|
||||||
|
## Recommendation
|
||||||
|
|
||||||
|
**Ship Approach B first.** It's a small, well-scoped change that closes the relay-operator-can-see-plaintext-in-RAM gap without requiring an MLS rewrite. Then layer Approach A on top when the threat model demands relay-untrusted operation.
|
||||||
|
|
||||||
|
## Out of scope for this PRD
|
||||||
|
|
||||||
|
- Federation gossip key exchange (separate PRD)
|
||||||
|
- SAS (Short Authentication String) verification UX (separate PRD)
|
||||||
|
- Rekey on session compromise (handled by the chosen approach's group/pairwise rekey)
|
||||||
|
|
||||||
|
## Acceptance criteria (Approach B, first iteration)
|
||||||
|
|
||||||
|
1. Relay's room forwarding loop (`crates/wzp-relay/src/room.rs:354` and `:1353`) calls `sender_session.decrypt()` then `recipient_session.encrypt()` per recipient before `send_media`.
|
||||||
|
2. Each `RoomMember` holds its `Box<dyn CryptoSession>` (currently discarded as `_crypto_session` in `main.rs:1817`).
|
||||||
|
3. Client-side: re-add the `EncryptingTransport` wrapping in `desktop/src-tauri/src/engine.rs` (the two sites reverted in `e8cab25`).
|
||||||
|
4. Integration test: two-client mock room exchanges media; verify each recipient gets the sender's plaintext back after the relay double-hop.
|
||||||
|
5. Existing 825 tests still pass.
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
`cargo test -p wzp-relay --test multi_client_relay_path` should pass with two simulated clients sending audio in both directions and decrypting each other's frames.
|
||||||
|
|
||||||
|
## Files to touch
|
||||||
|
|
||||||
|
- `crates/wzp-relay/src/main.rs` — keep `crypto_session` per-client (drop the `_` prefix)
|
||||||
|
- `crates/wzp-relay/src/room.rs` — add decrypt/re-encrypt to forward path
|
||||||
|
- `crates/wzp-relay/src/session_mgr.rs` — store sessions keyed by peer
|
||||||
|
- `desktop/src-tauri/src/engine.rs` — restore `EncryptingTransport` wrapping (~2 sites)
|
||||||
|
- `crates/wzp-relay/tests/multi_client_relay_path.rs` — new integration test
|
||||||
|
|
||||||
|
## Risk / rollback
|
||||||
|
|
||||||
|
If multi-client tests fail in CI, the change is contained to the relay forwarding loop and one engine.rs edit — straightforward revert.
|
||||||
220
docs/PRD/PRD-quality-upgrade-flow.md
Normal file
220
docs/PRD/PRD-quality-upgrade-flow.md
Normal file
@@ -0,0 +1,220 @@
|
|||||||
|
# PRD: Quality Upgrade Flow — UpgradeProposal / Response / Confirm
|
||||||
|
|
||||||
|
> **Status:** proposed
|
||||||
|
> **Resolves:** Four TODO comments in the signal task of `desktop/src-tauri/src/lib.rs` that leave quality upgrade messages unhandled. Audio quality never upgrades mid-call even when the network improves.
|
||||||
|
> **Depends on:** `wzp_proto::SignalMessage::{UpgradeProposal, UpgradeResponse, UpgradeConfirm, QualityCapability}` (already defined in `crates/wzp-proto/src/packet.rs`).
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
The signal receive task in `lib.rs` matches `UpgradeProposal`, `UpgradeResponse`,
|
||||||
|
`UpgradeConfirm`, and `QualityCapability` messages from the peer, logs them,
|
||||||
|
then hits a `// TODO` comment and does nothing. The 4 TODOs are at lines
|
||||||
|
1930, 1949, 1966, and 1985 of `desktop/src-tauri/src/lib.rs`.
|
||||||
|
|
||||||
|
Consequence: audio quality is frozen at the profile negotiated at call start.
|
||||||
|
Even when the network improves, the encoder never upgrades.
|
||||||
|
|
||||||
|
## Goals
|
||||||
|
|
||||||
|
1. `UpgradeProposal` auto-accepts and sends `UpgradeResponse { accepted: true }`.
|
||||||
|
2. Accepted `UpgradeResponse` sends `UpgradeConfirm` and switches the local encoder.
|
||||||
|
3. Received `UpgradeConfirm` switches the local encoder.
|
||||||
|
4. Received `QualityCapability` caps the local encoder to the peer's max profile.
|
||||||
|
5. A unit test verifies the accept/confirm round-trip.
|
||||||
|
6. `cargo check --manifest-path desktop/src-tauri/Cargo.toml` passes.
|
||||||
|
|
||||||
|
## Non-goals
|
||||||
|
|
||||||
|
- UI for manual accept/reject of upgrade proposals (auto-accept only).
|
||||||
|
- Sending `UpgradeProposal` from our side (the outgoing path already exists in
|
||||||
|
`lib.rs`; this PRD only handles receiving).
|
||||||
|
- Downgrade negotiation.
|
||||||
|
- Persisting quality profiles across calls.
|
||||||
|
|
||||||
|
## Design
|
||||||
|
|
||||||
|
### New shared state
|
||||||
|
|
||||||
|
Add the following to `AppState` (or as captured variables in the signal task
|
||||||
|
closure — whichever is cleaner given the existing structure):
|
||||||
|
|
||||||
|
```rust
|
||||||
|
/// Pending outgoing upgrade: (call_id, proposal_id, profile).
|
||||||
|
/// Set when we send an UpgradeProposal, consumed when we receive an accepted UpgradeResponse.
|
||||||
|
pending_upgrade: Arc<Mutex<Option<(String, String, QualityProfile)>>>,
|
||||||
|
|
||||||
|
/// Current quality profile for the encoder. The audio send task reads this
|
||||||
|
/// at the start of each encode cycle.
|
||||||
|
active_quality: Arc<Mutex<QualityProfile>>,
|
||||||
|
|
||||||
|
/// Peer's reported maximum quality cap. The audio send task clamps to min(active, peer_max).
|
||||||
|
peer_max_quality: Arc<Mutex<Option<QualityProfile>>>,
|
||||||
|
```
|
||||||
|
|
||||||
|
If `AppState` already holds these fields (check `lib.rs` for the struct
|
||||||
|
definition), reuse them instead of adding duplicates.
|
||||||
|
|
||||||
|
### Handler implementations
|
||||||
|
|
||||||
|
#### 1. `UpgradeProposal` (line ~1930)
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// Replace the TODO comment with:
|
||||||
|
let response = SignalMessage::UpgradeResponse {
|
||||||
|
version: wzp_proto::default_signal_version(),
|
||||||
|
call_id: call_id.clone(),
|
||||||
|
proposal_id: proposal_id.clone(),
|
||||||
|
accepted: true,
|
||||||
|
reason: None,
|
||||||
|
};
|
||||||
|
if let Err(e) = signal_transport.send_signal(&response).await {
|
||||||
|
tracing::warn!("failed to send UpgradeResponse: {e}");
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`signal_transport` is whatever variable holds the signal `Arc<dyn MediaTransport>`
|
||||||
|
in scope at that match arm. Inspect the enclosing task to find the right name.
|
||||||
|
|
||||||
|
#### 2. `UpgradeResponse` (line ~1949)
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// Replace the TODO comment with:
|
||||||
|
if accepted {
|
||||||
|
// Retrieve the pending proposal to get the confirmed_profile.
|
||||||
|
let maybe_proposal = pending_upgrade.lock().unwrap().take();
|
||||||
|
if let Some((_cid, pid, profile)) = maybe_proposal {
|
||||||
|
if pid == proposal_id {
|
||||||
|
// Send UpgradeConfirm.
|
||||||
|
let confirm = SignalMessage::UpgradeConfirm {
|
||||||
|
version: wzp_proto::default_signal_version(),
|
||||||
|
call_id: call_id.clone(),
|
||||||
|
proposal_id: proposal_id.clone(),
|
||||||
|
confirmed_profile: profile.clone(),
|
||||||
|
};
|
||||||
|
if let Err(e) = signal_transport.send_signal(&confirm).await {
|
||||||
|
tracing::warn!("failed to send UpgradeConfirm: {e}");
|
||||||
|
}
|
||||||
|
// Switch our encoder.
|
||||||
|
*active_quality.lock().unwrap() = profile;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
If `pending_upgrade` is a captured `Arc<Mutex<...>>` in the task closure, it
|
||||||
|
can be read/written without going through `AppState`.
|
||||||
|
|
||||||
|
#### 3. `UpgradeConfirm` (line ~1966)
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// Replace the TODO comment with:
|
||||||
|
*active_quality.lock().unwrap() = confirmed_profile;
|
||||||
|
```
|
||||||
|
|
||||||
|
The audio send task (in `engine.rs`) reads `active_quality` at the start of
|
||||||
|
each encode cycle and reconfigures the Opus encoder bitrate accordingly.
|
||||||
|
|
||||||
|
#### 4. `QualityCapability` (line ~1985)
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// Replace the TODO comment with:
|
||||||
|
*peer_max_quality.lock().unwrap() = Some(max_profile);
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 5. Audio send task changes (`engine.rs`)
|
||||||
|
|
||||||
|
The audio send task already runs in a loop. Add a quality-check at the top of
|
||||||
|
each encode iteration:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// At the start of the encode loop body:
|
||||||
|
let effective_profile = {
|
||||||
|
let active = active_quality.lock().unwrap().clone();
|
||||||
|
let peer_cap = peer_max_quality.lock().unwrap().clone();
|
||||||
|
match peer_cap {
|
||||||
|
Some(cap) if cap.opus_bitrate_bps() < active.opus_bitrate_bps() => cap,
|
||||||
|
_ => active,
|
||||||
|
}
|
||||||
|
};
|
||||||
|
// Pass effective_profile to encoder if it changed since last iteration.
|
||||||
|
```
|
||||||
|
|
||||||
|
`QualityProfile::opus_bitrate_bps()` already exists (check
|
||||||
|
`crates/wzp-proto/src/codec_id.rs`). If `QualityProfile` does not have a
|
||||||
|
direct bitrate accessor, compare using the `PartialOrd` impl or a helper that
|
||||||
|
ranks profiles numerically.
|
||||||
|
|
||||||
|
To avoid calling `encoder.set_bitrate()` every single frame, cache the last
|
||||||
|
applied profile and only reconfigure on change:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
let mut last_applied_profile: Option<QualityProfile> = None;
|
||||||
|
|
||||||
|
// Inside loop:
|
||||||
|
if Some(&effective_profile) != last_applied_profile.as_ref() {
|
||||||
|
encoder.set_bitrate(effective_profile.opus_bitrate_bps());
|
||||||
|
last_applied_profile = Some(effective_profile.clone());
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`encoder.set_bitrate(bps: u32)` — add this method to `OpusEncoder` in
|
||||||
|
`crates/wzp-codec/src/opus_enc.rs` if it does not exist. It wraps
|
||||||
|
`opus_encoder_ctl(OPUS_SET_BITRATE_REQUEST, bps)`.
|
||||||
|
|
||||||
|
### Unit tests
|
||||||
|
|
||||||
|
Add a `#[cfg(test)]` module in `lib.rs` (or a dedicated test file) that:
|
||||||
|
|
||||||
|
1. Creates a `LoopbackSignalTransport` stub that records sent `SignalMessage`s.
|
||||||
|
2. Calls the `UpgradeProposal` handler logic directly, asserts that an
|
||||||
|
`UpgradeResponse { accepted: true }` was sent.
|
||||||
|
3. Calls the `UpgradeResponse { accepted: true }` handler with a pre-populated
|
||||||
|
`pending_upgrade`, asserts that `UpgradeConfirm` was sent and
|
||||||
|
`active_quality` was updated.
|
||||||
|
|
||||||
|
These can be pure unit tests (no Tauri or audio), since the handlers are
|
||||||
|
pure async functions over captured state.
|
||||||
|
|
||||||
|
## Implementation steps
|
||||||
|
|
||||||
|
1. Read `desktop/src-tauri/src/lib.rs` lines 1910–1990 (the four TODO blocks)
|
||||||
|
and the surrounding signal task structure to identify the variable names
|
||||||
|
for `signal_transport`, `app_state`, and any existing quality-state fields.
|
||||||
|
2. Read `desktop/src-tauri/src/engine.rs` for `CallEngine` struct fields and
|
||||||
|
the audio send task loop.
|
||||||
|
3. Read `crates/wzp-proto/src/codec_id.rs` for `QualityProfile` methods.
|
||||||
|
4. Add `pending_upgrade`, `active_quality`, `peer_max_quality` to the
|
||||||
|
appropriate shared state (or as closure captures in the signal task).
|
||||||
|
5. Replace the 4 TODO comments with the handlers described above.
|
||||||
|
6. Add `set_bitrate` to `OpusEncoder` if missing.
|
||||||
|
7. Update the audio send task to read `active_quality` / `peer_max_quality`
|
||||||
|
each iteration.
|
||||||
|
8. Add unit tests.
|
||||||
|
9. Run `cargo check --manifest-path desktop/src-tauri/Cargo.toml`.
|
||||||
|
|
||||||
|
## Files to read before implementing
|
||||||
|
|
||||||
|
- `desktop/src-tauri/src/lib.rs` — grep for `UpgradeProposal` to find the
|
||||||
|
exact lines; also read the surrounding signal task for variable names.
|
||||||
|
- `crates/wzp-proto/src/packet.rs` lines 1130–1190 — `UpgradeProposal`,
|
||||||
|
`UpgradeResponse`, `UpgradeConfirm`, `QualityCapability` struct layouts.
|
||||||
|
- `desktop/src-tauri/src/engine.rs` — `CallEngine` struct fields, audio
|
||||||
|
send task loop.
|
||||||
|
- `crates/wzp-proto/src/codec_id.rs` — `QualityProfile` methods.
|
||||||
|
- `crates/wzp-codec/src/opus_enc.rs` — `OpusEncoder` API.
|
||||||
|
|
||||||
|
## Verify
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo check --manifest-path desktop/src-tauri/Cargo.toml
|
||||||
|
cargo test -p wzp-desktop 2>/dev/null || cargo test --manifest-path desktop/src-tauri/Cargo.toml
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: 0 errors; unit tests pass.
|
||||||
|
|
||||||
|
## Done when
|
||||||
|
|
||||||
|
- All 4 TODO comments replaced with real logic.
|
||||||
|
- `cargo check --manifest-path desktop/src-tauri/Cargo.toml` exits 0.
|
||||||
|
- Unit test verifies: `UpgradeProposal` → `UpgradeResponse { accepted: true }` sent;
|
||||||
|
`UpgradeResponse { accepted: true }` → `UpgradeConfirm` sent + `active_quality` updated.
|
||||||
242
docs/PRD/PRD-wire-format-hardening.md
Normal file
242
docs/PRD/PRD-wire-format-hardening.md
Normal file
@@ -0,0 +1,242 @@
|
|||||||
|
# PRD: Wire Format Hardening — FEC block_id u16, SignalMessage version byte, FEC repair index wrap
|
||||||
|
|
||||||
|
> **Status:** proposed
|
||||||
|
> **Resolves:** Three small wire-format defects (H2, M1, M4) that compound over time into silent data corruption or protocol breakage.
|
||||||
|
> **Depends on:** Nothing — purely mechanical changes to `wzp-fec` and `wzp-proto`.
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
Three independent issues:
|
||||||
|
|
||||||
|
**H2 — `fec_block_id` u8 wraps too fast.** The `block_id` field in
|
||||||
|
`RaptorQFecEncoder` (and `RaptorQFecDecoder`) is `u8`. At 5 audio frames
|
||||||
|
per block and 50 fps this wraps every ~51 seconds. A slow receiver or a
|
||||||
|
mid-session join can receive packets from two different blocks with the same
|
||||||
|
`block_id`, silently corrupting FEC recovery.
|
||||||
|
|
||||||
|
**M1 — Some `SignalMessage` variants lack a `version` byte.** Most variants
|
||||||
|
have `#[serde(default = "default_signal_version")] version: u8`. The unit
|
||||||
|
variant `Reflect` (and potentially others added recently) does not. Future
|
||||||
|
protocol changes that key on `version` will silently misparse old messages
|
||||||
|
from peers without the field.
|
||||||
|
|
||||||
|
**M4 — FEC repair index can silently wrap at 255.** In
|
||||||
|
`crates/wzp-fec/src/encoder.rs` line 140:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
let idx = (num_source as u16).wrapping_add(i as u16);
|
||||||
|
```
|
||||||
|
|
||||||
|
(The line was already fixed to `u16` — verify it is `u16`, not `u8`. If it
|
||||||
|
is still `u8`, the fix is below.)
|
||||||
|
|
||||||
|
If the line currently reads `(num_source as u8).wrapping_add(i as u8)`, then
|
||||||
|
when `num_source + repair_count > 255` the repair symbol indices wrap silently,
|
||||||
|
producing incorrect ESI values that the decoder cannot correlate to source
|
||||||
|
blocks.
|
||||||
|
|
||||||
|
## Goals
|
||||||
|
|
||||||
|
- **H2**: Widen `block_id` in encoder and decoder from `u8` to `u16`.
|
||||||
|
Update `finalize_block` return type and `current_block_id` return type in
|
||||||
|
the trait (`wzp-proto`) and implementations (`wzp-fec`).
|
||||||
|
- **M1**: Audit every `SignalMessage` variant; add
|
||||||
|
`#[serde(default = "default_signal_version")] version: u8` to any that
|
||||||
|
are missing it.
|
||||||
|
- **M4**: Confirm the repair index uses `u16`; fix it if it is still `u8`.
|
||||||
|
Update the decoder's `add_symbol` call site if the index type changes.
|
||||||
|
- `cargo test -p wzp-fec -p wzp-proto` passes; no existing tests broken.
|
||||||
|
|
||||||
|
## Non-goals
|
||||||
|
|
||||||
|
- Changing the wire encoding of `MediaHeaderV2::fec_block` — it is already
|
||||||
|
`u16` on the wire. This PRD only widens the **internal counter** to match.
|
||||||
|
- Multi-block decode concurrency or block expiry policy.
|
||||||
|
- Any crate outside `wzp-fec` and `wzp-proto`.
|
||||||
|
|
||||||
|
## Design
|
||||||
|
|
||||||
|
### Item A — `fec_block_id` u8 → u16
|
||||||
|
|
||||||
|
**Files**:
|
||||||
|
- `crates/wzp-proto/src/traits.rs` — `FecEncoder` and `FecDecoder` traits
|
||||||
|
- `crates/wzp-fec/src/encoder.rs` — `RaptorQFecEncoder`
|
||||||
|
- `crates/wzp-fec/src/decoder.rs` — `RaptorQFecDecoder`
|
||||||
|
|
||||||
|
**Trait changes** (`traits.rs`):
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// Before:
|
||||||
|
fn finalize_block(&mut self) -> Result<u8, FecError>;
|
||||||
|
fn current_block_id(&self) -> u8;
|
||||||
|
fn add_symbol(&mut self, block_id: u8, ...) -> Result<(), FecError>;
|
||||||
|
fn try_decode(&mut self, block_id: u8) -> Result<...>;
|
||||||
|
fn expire_before(&mut self, block_id: u8);
|
||||||
|
```
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// After:
|
||||||
|
fn finalize_block(&mut self) -> Result<u16, FecError>;
|
||||||
|
fn current_block_id(&self) -> u16;
|
||||||
|
fn add_symbol(&mut self, block_id: u16, ...) -> Result<(), FecError>;
|
||||||
|
fn try_decode(&mut self, block_id: u16) -> Result<...>;
|
||||||
|
fn expire_before(&mut self, block_id: u16);
|
||||||
|
```
|
||||||
|
|
||||||
|
**Encoder changes** (`encoder.rs`):
|
||||||
|
|
||||||
|
- Change `block_id: u8` field to `block_id: u16`.
|
||||||
|
- Update `self.block_id.wrapping_add(1)` (already u16 semantics; keep as is).
|
||||||
|
- Update `finalize_block` to return `u16`.
|
||||||
|
- Update `current_block_id` to return `u16`.
|
||||||
|
- Update all tests that assert `block_id == 0u8` → `== 0u16`, and the
|
||||||
|
wrap test (`block_id_wraps`) to iterate to `u16::MAX` (65535) — or reduce
|
||||||
|
it to 300 iterations to keep it fast, asserting the wrap at 65536.
|
||||||
|
|
||||||
|
The wrap test at 256 iterations (`0..=255u8`) must be updated; a full
|
||||||
|
`u16` wrap test at 65536 iterations is too slow for CI. Change to:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
#[test]
|
||||||
|
fn block_id_wraps_u16() {
|
||||||
|
let mut enc = RaptorQFecEncoder::with_defaults(1);
|
||||||
|
// Advance 300 blocks and verify no panic + monotonic increment.
|
||||||
|
for expected in 0..300u16 {
|
||||||
|
assert_eq!(enc.current_block_id(), expected);
|
||||||
|
enc.add_source_symbol(&[0u8; 10]).unwrap();
|
||||||
|
enc.finalize_block().unwrap();
|
||||||
|
}
|
||||||
|
// Explicitly test wrap at u16 boundary.
|
||||||
|
let mut enc2 = RaptorQFecEncoder::with_defaults(1);
|
||||||
|
enc2.block_id = u16::MAX;
|
||||||
|
enc2.add_source_symbol(&[0u8; 10]).unwrap();
|
||||||
|
let id = enc2.finalize_block().unwrap();
|
||||||
|
assert_eq!(id, u16::MAX);
|
||||||
|
assert_eq!(enc2.current_block_id(), 0);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Note: `block_id` is a private field; expose a test helper or set it in a
|
||||||
|
`#[cfg(test)]` `impl` block.
|
||||||
|
|
||||||
|
**Decoder changes** (`decoder.rs`):
|
||||||
|
|
||||||
|
- Change `blocks: HashMap<u8, BlockState>` to `HashMap<u16, BlockState>`.
|
||||||
|
- Update `get_or_create_block(block_id: u8)` → `get_or_create_block(block_id: u16)`.
|
||||||
|
- Update `add_symbol`, `try_decode`, `expire_before` signatures to `u16`.
|
||||||
|
- The `SourceBlockEncoder::new(self.block_id, ...)` call in `encoder.rs` passes
|
||||||
|
`block_id` to `raptorq`. RaptorQ uses `u8` for source block number internally.
|
||||||
|
Cast it: `(block_id & 0xFF) as u8` or `(block_id % 256) as u8` — the `raptorq`
|
||||||
|
crate's source block ID is a logical identifier within a single object
|
||||||
|
transmission, not a global counter. The u16 is our session counter; truncate
|
||||||
|
to u8 when calling into raptorq.
|
||||||
|
|
||||||
|
### Item B — `SignalMessage` version byte audit
|
||||||
|
|
||||||
|
**File**: `crates/wzp-proto/src/packet.rs`
|
||||||
|
|
||||||
|
Read every variant in the `SignalMessage` enum (lines 555–1241) and check
|
||||||
|
for the presence of:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
#[serde(default = "default_signal_version")]
|
||||||
|
version: u8,
|
||||||
|
```
|
||||||
|
|
||||||
|
The `Reflect` variant at line 974 is a **unit variant** (no fields). Unit
|
||||||
|
variants cannot carry a `version` field without becoming struct variants.
|
||||||
|
Change it to a struct variant:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// Before:
|
||||||
|
Reflect,
|
||||||
|
|
||||||
|
// After:
|
||||||
|
Reflect {
|
||||||
|
#[serde(default = "default_signal_version")]
|
||||||
|
version: u8,
|
||||||
|
},
|
||||||
|
```
|
||||||
|
|
||||||
|
This is a wire-compatible change: serde JSON struct variants serialize as
|
||||||
|
`{"Reflect": {"version": 1}}` whereas unit variants serialize as
|
||||||
|
`"Reflect"`. These are **not** backward-compatible formats. Since `Reflect`
|
||||||
|
is sent client → relay only and the relay immediately responds, upgrading
|
||||||
|
both sides atomically is acceptable. Add a serde test to confirm round-trip.
|
||||||
|
|
||||||
|
For any other variants missing `version`, follow the same pattern as all
|
||||||
|
existing variants.
|
||||||
|
|
||||||
|
Verify by grepping the enum for variants that do NOT have `version`:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
grep -A3 "^\s*[A-Z][A-Za-z]*\s*{" crates/wzp-proto/src/packet.rs | \
|
||||||
|
grep -B1 -v "serde.*default_signal_version\|version:"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Item C — FEC repair index wrap (M4)
|
||||||
|
|
||||||
|
**File**: `crates/wzp-fec/src/encoder.rs`, line ~140.
|
||||||
|
|
||||||
|
Current code:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
let idx = (num_source as u16).wrapping_add(i as u16);
|
||||||
|
```
|
||||||
|
|
||||||
|
If this line already uses `u16` (as shown in the file at line 140), M4 is
|
||||||
|
already fixed. Verify by reading the current file. If it still reads
|
||||||
|
`u8`, apply:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
let idx = (num_source as u16).wrapping_add(i as u16);
|
||||||
|
```
|
||||||
|
|
||||||
|
**Decoder** (`crates/wzp-fec/src/decoder.rs`): `add_symbol` already accepts
|
||||||
|
`symbol_index: u16` (per the trait). Confirm the parameter flows through to
|
||||||
|
`PayloadId::new(block_id_u8, symbol_index as u32)` without truncation.
|
||||||
|
|
||||||
|
## Implementation steps
|
||||||
|
|
||||||
|
1. Read `crates/wzp-proto/src/traits.rs` lines 60–116 (FecEncoder/FecDecoder
|
||||||
|
trait definitions) to confirm current signatures.
|
||||||
|
2. Read `crates/wzp-fec/src/encoder.rs` and `decoder.rs` (full files).
|
||||||
|
3. Apply Item C fix first (smallest change, easiest to verify).
|
||||||
|
4. Apply Item A: widen `block_id` from u8 to u16 in traits, encoder, decoder.
|
||||||
|
Update all callers by running `cargo check -p wzp-fec -p wzp-proto` and
|
||||||
|
fixing each E0308/E0308 error.
|
||||||
|
5. Apply Item B: read every variant, add missing `version` fields.
|
||||||
|
Change `Reflect` to a struct variant.
|
||||||
|
6. Run tests.
|
||||||
|
|
||||||
|
## Files to read before implementing
|
||||||
|
|
||||||
|
- `crates/wzp-proto/src/traits.rs` lines 60–116 (trait signatures)
|
||||||
|
- `crates/wzp-fec/src/encoder.rs` (full)
|
||||||
|
- `crates/wzp-fec/src/decoder.rs` (full)
|
||||||
|
- `crates/wzp-proto/src/packet.rs` lines 555–1241 (all `SignalMessage` variants)
|
||||||
|
|
||||||
|
## Verify
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo test -p wzp-fec -p wzp-proto
|
||||||
|
```
|
||||||
|
|
||||||
|
Expected: all tests pass, 0 failures. Also run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo check --workspace
|
||||||
|
```
|
||||||
|
|
||||||
|
to catch any call sites outside `wzp-fec` and `wzp-proto` that passed `u8`
|
||||||
|
block IDs to the trait methods.
|
||||||
|
|
||||||
|
## Done when
|
||||||
|
|
||||||
|
- `cargo test -p wzp-fec -p wzp-proto` exits 0.
|
||||||
|
- `block_id` is `u16` in `RaptorQFecEncoder`, `RaptorQFecDecoder`, and the
|
||||||
|
`FecEncoder`/`FecDecoder` traits.
|
||||||
|
- Every non-unit `SignalMessage` variant has a `version: u8` field with
|
||||||
|
`#[serde(default = "default_signal_version")]`.
|
||||||
|
- Repair index in `encoder.rs` is computed with `u16` arithmetic.
|
||||||
|
- No existing tests are broken.
|
||||||
104
docs/PROGRESS.md
104
docs/PROGRESS.md
@@ -389,3 +389,107 @@ Run with `wzp-bench --all`. Representative results (Apple M-series, single core)
|
|||||||
- `RegisterPresenceAck` populates `relay_region` from config, `available_relays` from federation peers
|
- `RegisterPresenceAck` populates `relay_region` from config, `available_relays` from federation peers
|
||||||
- Desktop `place_call`/`answer_call` call `acquire_port_mapping()` and fill mapped addr fields
|
- Desktop `place_call`/`answer_call` call `acquire_port_mapping()` and fill mapped addr fields
|
||||||
- Legacy `build-android-docker.sh` renamed to `build-android-docker-LEGACY.sh` to prevent accidental use
|
- Legacy `build-android-docker.sh` renamed to `build-android-docker-LEGACY.sh` to prevent accidental use
|
||||||
|
|
||||||
|
## Wave 5: Video Infrastructure (2026-05-12)
|
||||||
|
|
||||||
|
**Tasks completed:** T5.1, T5.1.1, T5.2, T5.3, T5.4, T5.5, T5.6, T5.7, T5.7.1, T5.8
|
||||||
|
|
||||||
|
### Relay: Audio + Video Scoring
|
||||||
|
|
||||||
|
New files in `crates/wzp-relay/src/`:
|
||||||
|
|
||||||
|
- `audio_scorer.rs` — per-stream audio quality scorer tracking packet loss, codec consistency, bitrate stability
|
||||||
|
- `response_policy.rs` — relay response policy engine mapping scores to action thresholds
|
||||||
|
- `verdict.rs` — `Verdict` enum: `Allow`, `RateLimit`, `Drop`, `Malicious`
|
||||||
|
- `video_scorer.rs` — `VideoScorer` with legitimacy scoring: keyframe regularity, I/P ratio, bandwidth responsiveness. **Note: wired but `observe()` not yet called from room forwarding path — T6.2 follow-up open.**
|
||||||
|
|
||||||
|
### Video: H.265 + Quality Controller
|
||||||
|
|
||||||
|
New files in `crates/wzp-video/src/`:
|
||||||
|
|
||||||
|
- `controller.rs` — `VideoQualityController`: maps (bwe_bps, loss_pct, rtt_ms, priority_mode) to (target_bitrate, target_fps, target_resolution, simulcast_layer)
|
||||||
|
- `simulcast.rs` — simulcast layer management (base + enhancement layers)
|
||||||
|
- `encoder_mode.rs` — encoder mode selection (CBR/VBR, keyframe intervals, quality presets)
|
||||||
|
|
||||||
|
H.265 encode/decode path added to:
|
||||||
|
- `videotoolbox.rs` — VideoToolbox H.265 encoder + decoder (macOS/iOS)
|
||||||
|
- `mediacodec.rs` — MediaCodec H.265 encoder + decoder (Android; NDK 0.9 compile errors pending in T4.3.1.1)
|
||||||
|
|
||||||
|
**Test delta:** wzp-relay 99→127, wzp-video 43→71
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Wave 6: AV1 + Federation Gossip Design (2026-05-12)
|
||||||
|
|
||||||
|
**Tasks completed:** T6.1, T6.1.2, T6.2
|
||||||
|
|
||||||
|
### Video: AV1 Codec Support
|
||||||
|
|
||||||
|
New files in `crates/wzp-video/src/`:
|
||||||
|
|
||||||
|
- `av1_obu.rs` — AV1 OBU (Open Bitstream Unit) framing and depacketizer
|
||||||
|
- `dav1d.rs` — dav1d AV1 software decoder (non-Android; gated via cfg)
|
||||||
|
- `svt_av1.rs` — SVT-AV1 software encoder (non-Android; gated via cfg)
|
||||||
|
|
||||||
|
Updated files:
|
||||||
|
- `videotoolbox.rs` — VideoToolbox AV1 decoder + encoder (macOS M3+, iOS A17+)
|
||||||
|
- `mediacodec.rs` — MediaCodec AV1 (Android; compile errors pending)
|
||||||
|
- `factory.rs` — `create_video_encoder(codec, platform)` dispatcher added; H.264, H.265, AV1 wired
|
||||||
|
|
||||||
|
**T6.1.2 follow-up open:** `create_video_encoder(Av1Main, ...)` has no caller in the call engine yet — wiring step is unstarted.
|
||||||
|
|
||||||
|
### Relay: Federation Reputation Gossip (Design Phase)
|
||||||
|
|
||||||
|
- T6.3 design exploration committed at `1e729e4`
|
||||||
|
- `docs/PRD/PRD-relay-federation-gossip.md` — Ban-List Distribution approach selected (Approach 3)
|
||||||
|
- Implementation not started; task spec pending conversion
|
||||||
|
|
||||||
|
### Test Counts
|
||||||
|
|
||||||
|
**Test delta Wave 6:** wzp-video 76→88, wzp-relay 127→137
|
||||||
|
|
||||||
|
**Total workspace tests: 702** (excluding `wzp-android`)
|
||||||
|
|
||||||
|
| Crate | Tests |
|
||||||
|
|---|---|
|
||||||
|
| wzp-proto | 112 |
|
||||||
|
| wzp-codec | 69 |
|
||||||
|
| wzp-fec | 21 |
|
||||||
|
| wzp-crypto | 64 |
|
||||||
|
| wzp-transport | 11 |
|
||||||
|
| wzp-relay | 137 |
|
||||||
|
| wzp-client | 200 |
|
||||||
|
| wzp-video | 88 |
|
||||||
|
| wzp-web | 2 |
|
||||||
|
| wzp-native | 0 |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Current Status (2026-05-25)
|
||||||
|
|
||||||
|
### What Works (Audio)
|
||||||
|
|
||||||
|
All audio path items from previous status section remain working. Additionally:
|
||||||
|
|
||||||
|
- MediaHeader v2 (16 bytes) deployed across all paths
|
||||||
|
- MiniHeader v2 (5 bytes with seq_delta) deployed
|
||||||
|
- Anti-replay windows per stream with media-type-aware sizing (audio 64, video 1024)
|
||||||
|
- Relay DashMap + RwLock concurrency model (T3.1 resolved the Mutex bottleneck)
|
||||||
|
|
||||||
|
### What Works (Video — partial)
|
||||||
|
|
||||||
|
- H.264 framer/depacketizer with FU-A fragmentation handling
|
||||||
|
- H.264, H.265, AV1 VideoToolbox encode/decode (macOS)
|
||||||
|
- AV1 dav1d + SVT-AV1 software path (non-Android)
|
||||||
|
- Video quality controller, simulcast, encoder mode selection (controller only; no active call wiring yet)
|
||||||
|
- Video scorer (scoring logic complete; not yet wired into relay forwarding)
|
||||||
|
- NACK framework (`nack.rs`; not yet wired into room forwarding)
|
||||||
|
|
||||||
|
### Open Blockers
|
||||||
|
|
||||||
|
- **Android video:** `mediacodec.rs` has 31 NDK 0.9 compile errors (T4.3.1.1 in progress)
|
||||||
|
- **AV1 call wiring:** `create_video_encoder(Av1Main, ...)` has no caller (T6.1.2 follow-up)
|
||||||
|
- **VideoScorer wiring:** `VideoScorer::observe()` commented out at `room.rs:1263` (T6.2 follow-up)
|
||||||
|
- **NACK wiring:** NACK path not wired into room forwarding (Phase V2/V4)
|
||||||
|
- **BWE:** `AdaptiveQualityController` does not consume `cwnd`/`bytes_in_flight` (Phase V2)
|
||||||
|
- **Crypto nonce bug:** `decrypt()` uses `recv_seq` instead of `MediaHeader.seq` (see AUDIT-2026-05-25.md C1)
|
||||||
|
|||||||
@@ -12,6 +12,36 @@ The transport, crypto, session, federation, and SFU layers are codec-agnostic. T
|
|||||||
4. Keyframe semantics (PLI, NACK, keyframe cache at SFU)
|
4. Keyframe semantics (PLI, NACK, keyframe cache at SFU)
|
||||||
5. Capture / encode pipeline (VideoToolbox / MediaCodec / NVENC)
|
5. Capture / encode pipeline (VideoToolbox / MediaCodec / NVENC)
|
||||||
|
|
||||||
|
## Implementation Status (as of 2026-05-25)
|
||||||
|
|
||||||
|
| Phase | Description | Status |
|
||||||
|
|---|---|---|
|
||||||
|
| V1 — Wire format | 16B MediaHeader v2, 5B MiniHeader v2, MediaType, u32 seq, 8-bit CodecID | ✅ Complete (T1.x) |
|
||||||
|
| V2 — Transport additions | BWE, NACK loop, TransportFeedback, dynamic FEC boost on I-frames | 🔲 Not started |
|
||||||
|
| V3 — `wzp-video` crate | H.264 baseline framer/depacketizer, VideoToolbox/MediaCodec/dav1d encoders | ✅ Substantially complete (T4.x, T5.x, T6.x) |
|
||||||
|
| V3 — H.264 Baseline | Single-layer H.264 | ✅ Complete |
|
||||||
|
| V3 — H.265 | VideoToolbox + MediaCodec H.265 | ✅ Complete (T5.x) |
|
||||||
|
| V3 — AV1 | dav1d + SVT-AV1 (non-Android), VideoToolbox AV1 (macOS M3+) | ✅ Complete; Android MediaCodec AV1 compile errors pending (T4.3.1.1) |
|
||||||
|
| V3 — Android MediaCodec | NDK 0.9 API migration for `mediacodec.rs` | 🔴 Blocked (31 compile errors) |
|
||||||
|
| V3 — Call engine wiring | `create_video_encoder()` integrated into active call negotiation | 🔴 Not started (T6.1.2 follow-up) |
|
||||||
|
| V4 — Keyframe & loss policy | NACK path, PLI, keyframe cache at SFU | 🟡 Framework present (`nack.rs`); not wired |
|
||||||
|
| V5 — Video adaptive controller | `VideoQualityController` + `PriorityMode` | 🟡 Controller built (`controller.rs`); not wired into call |
|
||||||
|
| V5 — Simulcast | Simulcast layer management | 🟡 `simulcast.rs` present; not wired |
|
||||||
|
| V6 — SFU changes | Keyframe cache, per-receiver layer selection, PLI suppression | 🟡 PLI suppression wired; keyframe cache + layer selection not started |
|
||||||
|
| V6 — Video scorer | `VideoScorer` legitimacy detection | 🟡 Built (`video_scorer.rs`); `observe()` not wired into room forwarding |
|
||||||
|
| V7 — Capture pipeline | Camera capture (AVCaptureSession, Camera2, NVENC) | 🔲 Not started |
|
||||||
|
|
||||||
|
**Legend:** ✅ Complete · 🟡 Partial/Framework only · 🔴 Blocked · 🔲 Not started
|
||||||
|
|
||||||
|
### Critical path to first video call
|
||||||
|
|
||||||
|
1. Fix Android MediaCodec compile errors (T4.3.1.1) — ~2h
|
||||||
|
2. Wire `create_video_encoder()` into call engine codec negotiation (T6.1.2) — ~2h
|
||||||
|
3. Fix crypto nonce bug (`decrypt()` must use `MediaHeader.seq`) — see `AUDIT-2026-05-25.md` C1 — ~1h
|
||||||
|
4. Wire `VideoScorer::observe()` into relay room forwarding (T6.2 follow-up) — ~2h
|
||||||
|
5. Implement Phase V2 BWE (mandatory for usable video) — ~3–4 days
|
||||||
|
6. Implement capture pipeline for at least one platform (V7) — ~1 week
|
||||||
|
|
||||||
## Phase V1 — Wire format & negotiation (no new code paths yet)
|
## Phase V1 — Wire format & negotiation (no new code paths yet)
|
||||||
|
|
||||||
Bump protocol version. Land all wire changes together so compat breaks exactly once.
|
Bump protocol version. Land all wire changes together so compat breaks exactly once.
|
||||||
|
|||||||
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
> Distilled from `docs/ARCHITECTURE.md` and the `wzp-proto` crate. Authoritative wire details live in `crates/wzp-proto/src/packet.rs`.
|
> Distilled from `docs/ARCHITECTURE.md` and the `wzp-proto` crate. Authoritative wire details live in `crates/wzp-proto/src/packet.rs`.
|
||||||
>
|
>
|
||||||
> **Status:** v1 (audio-only) is the deployed protocol. v2 (audio + video, 16 B header, MediaType, u32 seq, etc.) is specified in `ROAD-TO-VIDEO.md` Phase V1 and supersedes this document when implemented.
|
> **Status:** v2 is the deployed protocol (audio + video, 16 B header, MediaType, u32 seq). v1 clients are rejected with `Hangup::ProtocolVersionMismatch`.
|
||||||
|
|
||||||
## Layer summary
|
## Layer summary
|
||||||
|
|
||||||
@@ -16,42 +16,47 @@
|
|||||||
| Loss recovery | **RaptorQ FEC + Opus DRED + classical PLC** | NACK / PLI + reference-picture selection |
|
| Loss recovery | **RaptorQ FEC + Opus DRED + classical PLC** | NACK / PLI + reference-picture selection |
|
||||||
| Adaptive | 3-tier hysteresis (Good / Degraded / Catastrophic) + continuous DRED tuner | Per-frame bitrate ladder |
|
| Adaptive | 3-tier hysteresis (Good / Degraded / Catastrophic) + continuous DRED tuner | Per-frame bitrate ladder |
|
||||||
| Topology | SFU rooms + inter-relay federation + P2P via ICE | Mesh ≤ ~3, SFU above, Apple relays |
|
| Topology | SFU rooms + inter-relay federation + P2P via ICE | Mesh ≤ ~3, SFU above, Apple relays |
|
||||||
| Header | 12 B `MediaHeader` / 4 B `MiniHeader` (49 of 50), 4 B `QualityReport` trailer | RTP 12 B + extensions |
|
| Header | 16 B `MediaHeader` v2 / 5 B `MiniHeader` (49 of 50), 4 B `QualityReport` trailer | RTP 12 B + extensions |
|
||||||
|
|
||||||
## Distinctive choices
|
## Distinctive choices
|
||||||
|
|
||||||
- **QUIC datagrams instead of raw UDP + SRTP.** Brings TLS 1.3, PLPMTUD, path migration, and ACK-based RTT/loss estimation for free.
|
- **QUIC datagrams instead of raw UDP + SRTP.** Brings TLS 1.3, PLPMTUD, path migration, and ACK-based RTT/loss estimation for free.
|
||||||
- **Continuous DRED tuning.** Maps live `(loss%, RTT, jitter)` to a continuous Opus DRED lookback window. Most stacks treat DRED as discrete tiers.
|
- **Continuous DRED tuning.** Maps live `(loss%, RTT, jitter)` to a continuous Opus DRED lookback window. Most stacks treat DRED as discrete tiers.
|
||||||
- **MiniHeader (4 B for 49/50 packets).** Saves ~8 B/packet ≈ 400 B/s/stream at 50 pps.
|
- **MiniHeader (5 B for 49/50 packets).** Saves ~11 B/packet ≈ 550 B/s/stream at 50 pps vs. the full 16 B header.
|
||||||
- **E2E-preserving SFU.** The relay forwards encrypted datagrams; it never decrypts media. Room membership uses SNI = `hash(room_name)`.
|
- **E2E-preserving SFU.** The relay forwards encrypted datagrams; it never decrypts media. Room membership uses SNI = `hash(room_name)`.
|
||||||
- **Codec coordination via `QualityReport` trailer.** Receivers attach 4-byte loss/RTT/jitter/cap to media packets; the SFU broadcasts `QualityDirective` so all senders in a room converge on the same tier.
|
- **Codec coordination via `QualityReport` trailer.** Receivers attach 4-byte loss/RTT/jitter/cap to media packets; the SFU broadcasts `QualityDirective` so all senders in a room converge on the same tier.
|
||||||
|
|
||||||
## Wire format (current — v1)
|
## Wire format (current — v2)
|
||||||
|
|
||||||
### `MediaHeader` (12 bytes)
|
### `MediaHeader` v2 (16 bytes, byte-aligned)
|
||||||
|
|
||||||
```
|
```
|
||||||
Byte 0: [V:1][T:1][CodecID:4][Q:1][FecRatioHi:1]
|
Byte 0: version (u8) 0x02
|
||||||
Byte 1: [FecRatioLo:6][unused:2]
|
Byte 1: flags (u8) [T:1][Q:1][KeyFrame:1][FrameEnd:1][reserved:4]
|
||||||
Bytes 2-3: sequence (u16 BE)
|
Byte 2: media_type (u8) 0=audio, 1=video, 2=data, 3=control
|
||||||
Bytes 4-7: timestamp_ms (u32 BE)
|
Byte 3: codec_id (u8) 0-255 (see codec table)
|
||||||
Byte 8: fec_block_id (u8)
|
Byte 4: stream_id (u8) simulcast layer; 0=base
|
||||||
Byte 9: fec_symbol_idx (u8)
|
Byte 5: fec_ratio (u8) 0..200 → 0.0..2.0
|
||||||
Byte 10: reserved
|
Bytes 6-9: sequence (u32 BE)
|
||||||
Byte 11: csrc_count
|
Bytes 10-13: timestamp_ms (u32 BE)
|
||||||
|
Bytes 14-15: fec_block_id (u16 BE)
|
||||||
```
|
```
|
||||||
|
|
||||||
| Field | Bits | Meaning |
|
| Field | Bits | Meaning |
|
||||||
|---|---|---|
|
|---|---|---|
|
||||||
| V | 1 | Protocol version |
|
| version | 8 | Must be `0x02`; v1 clients receive `Hangup::ProtocolVersionMismatch` |
|
||||||
| T | 1 | 1 = FEC repair packet |
|
| T (bit 7 of flags) | 1 | 1 = FEC repair packet |
|
||||||
| CodecID | 4 | See codec table |
|
| Q (bit 6 of flags) | 1 | QualityReport trailer present |
|
||||||
| Q | 1 | QualityReport trailer present |
|
| KeyFrame (bit 5 of flags) | 1 | Packet belongs to a video I-frame |
|
||||||
| FecRatio | 7 | 0–127 → 0.0–2.0 |
|
| FrameEnd (bit 4 of flags) | 1 | Last packet of an access unit |
|
||||||
| sequence | 16 | Wrapping packet seq |
|
| reserved (bits 3-0 of flags) | 4 | Must be zero |
|
||||||
|
| media_type | 8 | 0=audio, 1=video, 2=data, 3=control |
|
||||||
|
| codec_id | 8 | See codec table (widened from v1's 4-bit field) |
|
||||||
|
| stream_id | 8 | Simulcast layer; 0=base layer |
|
||||||
|
| fec_ratio | 8 | 0..200 → 0.0..2.0 |
|
||||||
|
| sequence | 32 | Monotonically increasing packet seq (not reset by rekey) |
|
||||||
| timestamp_ms | 32 | ms since session start. Monotonic across the full session; **not reset by rekey** |
|
| timestamp_ms | 32 | ms since session start. Monotonic across the full session; **not reset by rekey** |
|
||||||
| fec_block_id | 8 | FEC source block ID |
|
| fec_block_id | 16 | FEC source block ID |
|
||||||
| fec_symbol_idx | 8 | Symbol index in block |
|
|
||||||
|
|
||||||
### Codec table
|
### Codec table
|
||||||
|
|
||||||
@@ -66,13 +71,18 @@ Byte 11: csrc_count
|
|||||||
| 6 | Opus 32k | 32 kbps | 48 kHz | 20 ms |
|
| 6 | Opus 32k | 32 kbps | 48 kHz | 20 ms |
|
||||||
| 7 | Opus 48k | 48 kbps | 48 kHz | 20 ms |
|
| 7 | Opus 48k | 48 kbps | 48 kHz | 20 ms |
|
||||||
| 8 | Opus 64k | 64 kbps | 48 kHz | 20 ms |
|
| 8 | Opus 64k | 64 kbps | 48 kHz | 20 ms |
|
||||||
|
| 9 | H.264 Baseline | — | — | — |
|
||||||
|
| 10 | H.264 Main | — | — | — |
|
||||||
|
| 11 | H.265 Main | — | — | — |
|
||||||
|
| 12 | AV1 Main | — | — | — |
|
||||||
|
|
||||||
### `MiniHeader` (4 bytes, compressed — 49 of every 50 packets)
|
### `MiniHeader` v2 (5 bytes, compressed — 49 of every 50 packets)
|
||||||
|
|
||||||
```
|
```
|
||||||
[FRAME_TYPE_MINI = 0x01]
|
[FRAME_TYPE_MINI = 0x01]
|
||||||
Bytes 0-1: timestamp_delta_ms (u16 BE)
|
Byte 0: seq_delta (u8)
|
||||||
Bytes 2-3: payload_len (u16 BE)
|
Bytes 1-2: timestamp_delta_ms (u16 BE)
|
||||||
|
Bytes 3-4: payload_len (u16 BE)
|
||||||
```
|
```
|
||||||
|
|
||||||
Full header sent every 50th packet to resync.
|
Full header sent every 50th packet to resync.
|
||||||
@@ -95,6 +105,12 @@ Byte 2: jitter_ms (0-255 ms)
|
|||||||
Byte 3: bitrate_cap_kbps (0-255 kbps)
|
Byte 3: bitrate_cap_kbps (0-255 kbps)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Version negotiation
|
||||||
|
|
||||||
|
- `version=0x02` in `MediaHeader` is a hard switch — there is no fallback negotiation.
|
||||||
|
- Both endpoints must speak v2. A v1 peer receives `Hangup::ProtocolVersionMismatch` immediately.
|
||||||
|
- Relays inspect only `version` and `media_type`; they never downgrade or translate between versions.
|
||||||
|
|
||||||
## Session lifecycle
|
## Session lifecycle
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|||||||
192
docs/bugs/001-android-join-voice-hang.md
Normal file
192
docs/bugs/001-android-join-voice-hang.md
Normal file
@@ -0,0 +1,192 @@
|
|||||||
|
# BUG-001: Android "Connecting…" Hangs / Join Voice Never Completes
|
||||||
|
|
||||||
|
**Severity:** P0 — renders the app non-functional for room joins on a fresh install
|
||||||
|
**Status:** Partially mitigated (5a13f12), narrowed by static review; Android repro/logcat still needed
|
||||||
|
**Branch:** `experimental-ui`
|
||||||
|
**Last investigated:** 2026-05-25
|
||||||
|
**Device confirmed affected:** Nothing Phone A059 (Android 15)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Symptom
|
||||||
|
|
||||||
|
User taps "Join Voice". Button changes to "Connecting…" and stays there indefinitely. No error toast, no drawer, no progress. The only recovery is force-quitting the app.
|
||||||
|
|
||||||
|
## 2026-05-25 Static Review Update
|
||||||
|
|
||||||
|
The exact indefinite "Connecting…" symptom most likely came from an APK older than `5a13f12`, because current `desktop/src/main.ts` has a 15s JS-side timeout for manual room joins. The current branch can still produce closely related failures:
|
||||||
|
|
||||||
|
1. Native Oboe start can report false success when Android leaves capture/playout in `Starting` for 2s. That manifests as "joined but silent/dead audio", not a true JS hang.
|
||||||
|
2. First-run microphone permission can still race the first `openStream(Direction::Input)`, especially when the user joins immediately after granting permission.
|
||||||
|
3. Direct-call auto-connect did not have the 15s JS timeout even after `5a13f12`.
|
||||||
|
4. Toasts used `${e}`, so object-shaped Tauri errors could appear as `[object Object]`.
|
||||||
|
|
||||||
|
Working-tree diagnostic changes applied during this investigation:
|
||||||
|
|
||||||
|
- `crates/wzp-native/cpp/oboe_bridge.cpp`: return `-6` if both streams do not reach `Started` before the 2s poll deadline. This turns Oboe false-success into a visible Rust/JS error.
|
||||||
|
- `desktop/src/main.ts`: shared `connectWithTimeout()` for room joins and direct-call auto-connect; shared `errorMessage()` for useful toast text.
|
||||||
|
- `desktop/src-tauri/src/engine.rs`: emit `connect:handshake_*`, `connect:android_audio_preflight`, `connect:audio_*` markers around each Android-only join step.
|
||||||
|
- `desktop/src-tauri/src/lib.rs`: emit `connect:reuse_endpoint` so we can see whether the room join is sharing the signal QUIC endpoint.
|
||||||
|
|
||||||
|
Next Android repro should distinguish:
|
||||||
|
|
||||||
|
| Toast / log | Meaning |
|
||||||
|
|---|---|
|
||||||
|
| `Join failed: wzp_native_audio_start failed: code -2` | mic permission / capture open failure |
|
||||||
|
| `Join failed: wzp_native_audio_start failed: code -6` | Oboe streams opened/requested start, but HAL never transitioned both to `Started` |
|
||||||
|
| `Join failed: transport: timeout after 10000ms` or similar after `connect:handshake_start` | QUIC connected, but relay media handshake did not return `CallAnswer` |
|
||||||
|
| `Join failed: connect timed out (15s) - check audio permissions` | Tauri command did not resolve to JS; collect Rust/Tauri logs around `connect:call_engine_starting` |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Root Cause Chain
|
||||||
|
|
||||||
|
The `invoke("connect")` Tauri command runs the full `CallEngine::start` coroutine on Android. Execution order:
|
||||||
|
|
||||||
|
1. Parse relay address → QUIC dial → crypto handshake (~200ms, works — relay logs confirm room join succeeds)
|
||||||
|
2. `audio_stop()` (no-op on first launch)
|
||||||
|
3. `tokio::time::sleep(50ms)`
|
||||||
|
4. `set_audio_mode_communication()` (JNI into Kotlin)
|
||||||
|
5. **`tokio::task::spawn_blocking(crate::wzp_native::audio_start)`** ← primary hang point
|
||||||
|
|
||||||
|
`audio_start` calls `wzp_oboe_start()` (C++ FFI in `crates/wzp-native/cpp/oboe_bridge.cpp`), which:
|
||||||
|
- Opens capture stream (`captureBuilder.openStream`)
|
||||||
|
- Opens playout stream (`playoutBuilder.openStream`)
|
||||||
|
- `g_capture_stream->requestStart()`
|
||||||
|
- `g_playout_stream->requestStart()`
|
||||||
|
- **Polls up to 2 seconds** in a `std::this_thread::sleep_for(10ms)` busy-wait loop waiting for both streams to reach `Started` state (`oboe_bridge.cpp:404–423`)
|
||||||
|
|
||||||
|
Before the working-tree `-6` diagnostic change, if the HAL never transitioned to `Started`, `wzp_oboe_start` returned 0 (success!) after the 2s timeout even though streams were not functional. Rust saw `ret == 0`, considered it success, and `CallEngine::start` returned `Ok`.
|
||||||
|
|
||||||
|
The `invoke("connect")` promise resolves successfully, `enterVoice(false)` is called, the voice drawer appears — but audio streams are dead. The send task reads silence, the playout ring never drains.
|
||||||
|
|
||||||
|
**However**, relay log evidence shows the connection is established and then dropped 166ms later with `forwarded=0`, which means `CallEngine::start` did return to the `connect` command. If the user still sees "Connecting…" at that point, the JS `await connectRace` is not resolving — suggesting either the Rust command returned an error (which should show as a toast) or the `invoke` promise is hanging for a different reason.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Evidence
|
||||||
|
|
||||||
|
**Relay log (pangolin, session at 06:40:04 UTC):**
|
||||||
|
```
|
||||||
|
room "general" join accepted
|
||||||
|
crypto handshake complete t=+184ms
|
||||||
|
connection dropped t=+350ms forwarded=0
|
||||||
|
```
|
||||||
|
|
||||||
|
The relay sees a clean connection that self-terminates in ~350ms total. `forwarded=0` means no media was exchanged. Consistent with audio_start failing or the call task throwing before media loops start.
|
||||||
|
|
||||||
|
**Four rapid connects at 06:40:04** in the relay log suggest multiple taps (no `connectPending` guard in the APK installed at that time, or user was on an older build).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Fixes Applied in `5a13f12`
|
||||||
|
|
||||||
|
| # | Problem | Fix | File |
|
||||||
|
|---|---------|-----|------|
|
||||||
|
| 1 | `wzp_oboe_start` called directly on tokio worker thread → froze entire runtime including timeouts | Changed to `spawn_blocking` | `desktop/src-tauri/src/engine.rs:609` |
|
||||||
|
| 2 | No JS-side timeout → "Connecting…" hangs forever if Rust never returns | Added 15s `Promise.race` | `desktop/src/main.ts:338` |
|
||||||
|
| 3 | No error feedback to user | Added `showToast()` in `catch` block | `desktop/src/main.ts:352` |
|
||||||
|
| 4 | Button disappeared on click | Changed to `disabled + "Connecting…"` text | `desktop/src/main.ts:335` |
|
||||||
|
| 5 | Handshake could hang forever waiting for `CallAnswer` | Added 10s `tokio::time::timeout` | `crates/wzp-client/src/handshake.rs:105` |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Open Issues (Not Yet Fixed)
|
||||||
|
|
||||||
|
### Issue A: `g_running` flag race between `audio_stop` and `audio_start`
|
||||||
|
|
||||||
|
**Current status:** likely fixed in current branch. `crates/wzp-native/cpp/oboe_bridge.cpp:430` now clears `g_running` at the top of `wzp_oboe_stop`.
|
||||||
|
|
||||||
|
`oboe_bridge.cpp:244` checks `g_running.load()` at entry to `wzp_oboe_start`. The engine calls `audio_stop()` then waits 50ms then calls `audio_start()`. If `wzp_oboe_stop` does not synchronously clear `g_running` before returning, the next `wzp_oboe_start` sees `g_running == true` and returns `-1` immediately (line 246–247).
|
||||||
|
|
||||||
|
With `5a13f12`, Rust now propagates this as `"wzp_native_audio_start failed: code -1"` → toast. Confirm via logcat.
|
||||||
|
|
||||||
|
### Issue B: Mic permission granted at runtime causes audio HAL delay
|
||||||
|
|
||||||
|
After clearing app data, Android prompts for mic permission. The OS grants it but the audio HAL may not immediately honor it. The first `openStream(Direction::Input)` within ~1s of permission grant can fail with `ErrorPermissionDenied` → Oboe returns `-2`.
|
||||||
|
|
||||||
|
With `5a13f12` this should surface as toast: `"Join failed: wzp_native_audio_start failed: code -2"`.
|
||||||
|
|
||||||
|
### Issue C: `wzp_oboe_start` 2s poll timeout returns 0 (false success)
|
||||||
|
|
||||||
|
`oboe_bridge.cpp:404–423`: if streams don't reach `Started` state within 2s, the poll loop exits with no error — `wzp_oboe_start` returns 0. Rust treats this as success. The drawer appears but audio is dead. This is the "joined but silent" failure mode, distinct from "stuck on Connecting…".
|
||||||
|
|
||||||
|
**Fix:** return a distinct error code (e.g. `-6`) from `wzp_oboe_start` when the poll times out without both streams reaching `Started`.
|
||||||
|
|
||||||
|
**Working-tree status:** implemented as `-6`; needs Android NDK/device validation.
|
||||||
|
|
||||||
|
### Issue D: Error object serialization in JS toast
|
||||||
|
|
||||||
|
The `connect` command returns `Result<String, String>`. Tauri wraps the `Err` as a JS exception. If `e` in the `catch` block is a Tauri error object rather than a plain string, `${e}` renders as `"[object Object]"`. Should use `e?.message ?? String(e)` for robust stringification.
|
||||||
|
|
||||||
|
**Working-tree status:** implemented via `errorMessage(e)`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## `wzp_oboe_start` Return Codes Reference
|
||||||
|
|
||||||
|
| Code | Meaning |
|
||||||
|
|------|---------|
|
||||||
|
| 0 | Success |
|
||||||
|
| -1 | Already running (`g_running == true` at entry) |
|
||||||
|
| -2 | `captureBuilder.openStream` failed |
|
||||||
|
| -3 | `playoutBuilder.openStream` failed |
|
||||||
|
| -4 | `g_capture_stream->requestStart()` failed |
|
||||||
|
| -5 | `g_playout_stream->requestStart()` failed |
|
||||||
|
| -6 | streams failed to reach `Started` before poll timeout |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Reproduction Steps
|
||||||
|
|
||||||
|
1. Fresh install (or clear app data) on Nothing Phone A059
|
||||||
|
2. Grant microphone permission when prompted
|
||||||
|
3. Configure relay `193.180.213.68:4433`, room `general`
|
||||||
|
4. Tap "Join Voice"
|
||||||
|
5. Observe: button shows "Connecting…" indefinitely
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Diagnostic Steps
|
||||||
|
|
||||||
|
We have never captured `adb logcat` from a failing connect. This is the single highest-value diagnostic:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
adb logcat -s "wzp-native" "wzp-desktop" "RustStd" | grep -E "audio|oboe|start|handshake|connect"
|
||||||
|
```
|
||||||
|
|
||||||
|
Key log lines to look for:
|
||||||
|
|
||||||
|
| Log line | Diagnosis |
|
||||||
|
|----------|-----------|
|
||||||
|
| `connect:reuse_endpoint` | Whether media is sharing the existing signal endpoint |
|
||||||
|
| `connect:handshake_start` followed by 10s timeout | Relay media handshake is stuck before Android audio starts |
|
||||||
|
| `connect:handshake_done` | Network/relay handshake succeeded; continue to audio diagnostics |
|
||||||
|
| `connect:android_audio_preflight` | Shows `wzp-native` load state and RECORD_AUDIO permission |
|
||||||
|
| `connect:audio_start_start` with no done/failed | Native Oboe call is hanging |
|
||||||
|
| `wzp_oboe_start: already running` | Issue A — g_running not cleared |
|
||||||
|
| `Failed to open capture stream: ErrorPermissionDenied` | Issue B — mic permission delay |
|
||||||
|
| `Failed to start capture` / `Failed to start playout` | Oboe HAL error, code -4 or -5 |
|
||||||
|
| `both streams Started after N polls` | audio_start succeeded |
|
||||||
|
| `audio_start task panic` | spawn_blocking panic (shouldn't happen) |
|
||||||
|
| `wzp_native_audio_start failed: code X` | Rust caught it, toast should be visible |
|
||||||
|
|
||||||
|
Alternatively: enable **Call debug logs** in Settings, reproduce, use the share button to extract logs without USB.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Proposed Fixes (Prioritized)
|
||||||
|
|
||||||
|
1. **Validate `-6` from `wzp_oboe_start` on poll timeout** on Android builder/device — eliminates silent false-success
|
||||||
|
2. **Add mic permission pre-check** in Kotlin before calling into Rust — surface a cleaner error if permission is not yet effective
|
||||||
|
3. **If `-6` reproduces on Nothing A059, test startup sequencing:** request/start capture before `MODE_IN_COMMUNICATION`, add a short post-permission delay, or retry once after a full `wzp_oboe_stop`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Related Files
|
||||||
|
|
||||||
|
- `crates/wzp-native/cpp/oboe_bridge.cpp` — `wzp_oboe_start` implementation
|
||||||
|
- `crates/wzp-native/src/lib.rs:238` — `audio_start_inner` (Rust FFI wrapper)
|
||||||
|
- `desktop/src-tauri/src/engine.rs:576–635` — `CallEngine::start` audio section
|
||||||
|
- `desktop/src/main.ts:328–360` — `joinVoiceBtn` click handler
|
||||||
|
- `crates/wzp-client/src/handshake.rs:105` — handshake timeout
|
||||||
165
docs/bugs/002-macos-vpio-playout-silent.md
Normal file
165
docs/bugs/002-macos-vpio-playout-silent.md
Normal file
@@ -0,0 +1,165 @@
|
|||||||
|
# BUG-002: macOS VPIO Playout Silent — Audio Decoded But Not Heard
|
||||||
|
|
||||||
|
**Severity:** P0 — outgoing audio (Mac mic → peer) works, but the user hears nothing on the Mac side
|
||||||
|
**Status:** Instrumented on 2026-05-25; awaiting next VPIO vs CPAL repro
|
||||||
|
**Branch:** `experimental-ui`
|
||||||
|
**Build observed:** `01f55ca` (Mac desktop), same-day Android `01f55ca`
|
||||||
|
**Last investigated:** 2026-05-25
|
||||||
|
**Platforms confirmed affected:** macOS desktop (VPIO path)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Symptom
|
||||||
|
|
||||||
|
In a relay-forwarded group call between macOS and Android in the same room (`General`, `count:2`):
|
||||||
|
|
||||||
|
- The Mac user can be **heard** on Android (Mac→Android leg works).
|
||||||
|
- The Mac user **hears nothing** when the Android peer speaks (Android→Mac playout silent).
|
||||||
|
- Muting the Android peer's mic results in total silence on both ends — confirming the only audio the user perceived during the call was the Mac→Android leg playing through the Android speaker.
|
||||||
|
|
||||||
|
This was initially misreported as "I hear myself on Android" — the user was hearing their own Mac mic looped through Android playout, not an actual echo bug.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Evidence
|
||||||
|
|
||||||
|
### Mac log excerpt (`01f55ca`, fingerprint `63ba…`, 10:31:22)
|
||||||
|
|
||||||
|
```
|
||||||
|
10:31:23 media:room_update {"count":2, participants:[Akbar fa06…, Manwe 63ba…]}
|
||||||
|
10:31:23 media:first_recv {"codec":"Opus24k","payload_bytes":27,"t_ms":933}
|
||||||
|
10:31:25 media:recv_heartbeat {"codec":"Opus24k","decode_errs":0,"decoded_frames":140,"last_written":960,"written_samples":134400}
|
||||||
|
10:31:29 media:recv_heartbeat {"codec":"Opus32k","decoded_frames":338,"last_written":960,"written_samples":324480}
|
||||||
|
10:31:35 media:recv_heartbeat {"codec":"Opus6k","decoded_frames":595,"last_written":1920,"written_samples":618240}
|
||||||
|
…
|
||||||
|
10:31:57 media:recv_heartbeat {"codec":"Opus6k","decoded_frames":1086,"last_written":1920,"written_samples":1560960}
|
||||||
|
```
|
||||||
|
|
||||||
|
Recv path is healthy:
|
||||||
|
- `decode_errs:0` throughout
|
||||||
|
- `decoded_frames` climbs monotonically 140 → 1086
|
||||||
|
- `written_samples` reaches 1.56 M (≈32 s of 48 kHz mono)
|
||||||
|
- `last_written` correctly flips 960 (Opus24k/32k, 20 ms) ↔ 1920 (Opus6k, 40 ms)
|
||||||
|
|
||||||
|
**Conclusion:** packets arrive, decode succeeds, samples are written into `playout_ring`. The breakage is **downstream of the ring write**, i.e. in the macOS playout consumer (the VPIO `set_render_callback`).
|
||||||
|
|
||||||
|
### Mac send path also works
|
||||||
|
`media:send_heartbeat` shows `last_rms` spiking to 168, 477, 867, 1458 in response to speech. Android's recv log for the same window decoded those frames successfully.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Suspected Root Cause
|
||||||
|
|
||||||
|
`crates/wzp-client/src/audio_vpio.rs:128–147` — the render (output) callback reads from `playout_ring` in `FRAME_SAMPLES` (960) chunks. Three plausible failure modes:
|
||||||
|
|
||||||
|
### Hypothesis A: Codec-change frame-size mismatch
|
||||||
|
Mid-call codec switches (`Opus24k` → `Opus32k` → `Opus6k`) change the frame size written into the ring (960 ↔ 1920 samples per frame). The render callback reads in fixed 960-sample chunks. The ring is FIFO and should absorb this, but if `AudioRing` semantics drop partial frames or stall on alignment, the consumer side could starve while `written_samples` continues to climb on the producer side.
|
||||||
|
|
||||||
|
`engine.rs:1852` and `engine.rs:1895` write into `playout_ring` directly with the decoder's output length (variable). Worth confirming `AudioRing::read` handles arbitrary chunk sizes vs producer.
|
||||||
|
|
||||||
|
### Hypothesis B: VPIO output element never actually started
|
||||||
|
`audio_vpio.rs:151` calls `au.start()` once on the combined VPIO unit. VPIO is supposed to start both input and output elements together, but if AEC initialization fails silently, output rendering may be suppressed while input still produces callbacks. The `[vpio] capture callback: N f32 samples` log line proves input callbacks fire — but there is **no equivalent log line for the render callback**. We do not know whether the render callback is being invoked at all.
|
||||||
|
|
||||||
|
### Hypothesis C: Output device routing
|
||||||
|
VPIO may have grabbed an unexpected default output (e.g. the previous Bluetooth headset, an HDMI sink, or a virtual device). The render callback runs and pulls samples, but they're sent to a device the user can't hear.
|
||||||
|
|
||||||
|
### Hypothesis D: AEC over-suppression
|
||||||
|
VPIO's AEC uses the render callback as the far-end reference. If the unit decides the far-end and near-end are too correlated (it shouldn't here — different speakers in different rooms), it could attenuate playout. Unlikely to produce 100 % silence but listed for completeness.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Instrumentation Added
|
||||||
|
|
||||||
|
As of the current workspace, the desktop client emits VPIO render/capture counters into the normal call debug log when OS AEC is enabled:
|
||||||
|
|
||||||
|
```
|
||||||
|
vpio:render_heartbeat {
|
||||||
|
"capture_callbacks": ...,
|
||||||
|
"capture_samples": ...,
|
||||||
|
"render_callbacks": ...,
|
||||||
|
"render_requested_samples": ...,
|
||||||
|
"render_read_samples": ...,
|
||||||
|
"render_underrun_callbacks": ...,
|
||||||
|
"render_nonzero_callbacks": ...,
|
||||||
|
"render_last_requested": ...,
|
||||||
|
"render_last_read": ...,
|
||||||
|
"render_last_rms": ...,
|
||||||
|
"render_last_ring_available": ...
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Interpretation:
|
||||||
|
|
||||||
|
- `render_callbacks == 0`: VPIO output callback is not running. Focus on VPIO initialization / output element start.
|
||||||
|
- `render_callbacks > 0` and `render_read_samples == 0` while `media:recv_heartbeat.written_samples` climbs: VPIO callback runs but the ring it reads is not receiving the same samples the recv task writes, or the callback is draining before writes arrive.
|
||||||
|
- `render_read_samples > 0` and `render_last_rms > 0` while the user hears silence: VPIO is feeding non-zero speaker samples to CoreAudio; focus on output device routing or VoiceProcessingIO suppression.
|
||||||
|
- CPAL fallback test: disable OS AEC in settings. If CPAL playback is audible with the same call, the failure is VPIO-specific.
|
||||||
|
|
||||||
|
## Proposed Diagnostic Steps (Prioritized)
|
||||||
|
|
||||||
|
1. **Reproduce with current instrumentation** and compare `media:recv_heartbeat` to `vpio:render_heartbeat`.
|
||||||
|
|
||||||
|
2. **One-shot render callback stderr log is now present** (`audio_vpio.rs`) mirroring the existing capture-side `eprintln!`:
|
||||||
|
```rust
|
||||||
|
let logged_render = Arc::new(AtomicBool::new(false));
|
||||||
|
…
|
||||||
|
if !logged_render.swap(true, Ordering::Relaxed) {
|
||||||
|
eprintln!("[vpio] render callback: {} f32 samples, ring_read={}", ch.len(), read);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
This will immediately distinguish Hypothesis B (callback never fires) from A/C/D (callback fires but output is silent or misrouted).
|
||||||
|
|
||||||
|
3. **Periodically log render-callback stats** — total samples pulled from ring, samples requested per callback, non-zero render callback count, and last render RMS. Compare against producer-side `written_samples` to confirm consumer is keeping up.
|
||||||
|
|
||||||
|
4. **Verify output device** via `AudioUnitGetProperty(kAudioOutputUnitProperty_CurrentDevice, Output)` immediately after `au.start()`. Log device name. If it doesn't match the user's intended speaker, force-set the default output device.
|
||||||
|
|
||||||
|
5. **Test with codec pinned** — set `WZP_FORCE_CODEC=Opus24k` (or wire a temporary CLI flag) so codec doesn't change mid-call. If audio works with a pinned codec, Hypothesis A is confirmed and `AudioRing` chunk handling needs review.
|
||||||
|
|
||||||
|
6. **Compare CPAL fallback path** — disable OS AEC in settings and reproduce. If CPAL playback works, the bug is VPIO-specific.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Open Questions
|
||||||
|
|
||||||
|
- Does the macOS render callback have permission to write to the user's selected output device? Apple changed CoreAudio output-device permission semantics in macOS 14+.
|
||||||
|
- Is `_audio_unit: AudioUnit` being dropped early? It's stored in `VpioAudio` and that struct is boxed into `audio_handle: Box<dyn Any + Send>` in `engine.rs:1573`, which is held by `CallEngine`. Should be alive for the call duration — confirm no early-drop path.
|
||||||
|
- Are there any `os_log` / Console.app warnings from `AudioToolbox` / `CoreAudio` / `AVAudioSession` during the call?
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Reproduction Steps
|
||||||
|
|
||||||
|
1. Start macOS desktop client (build `01f55ca` or later), join relay `193.180.213.68:4433`, room `General`.
|
||||||
|
2. Start Android client (same build), join same relay + room.
|
||||||
|
3. Confirm `media:room_update count:2` on both ends.
|
||||||
|
4. Speak into Android mic.
|
||||||
|
5. Observe: Mac log shows `decoded_frames` climbing, `decode_errs:0`, `written_samples` increasing. User hears nothing on Mac speakers.
|
||||||
|
6. Speak into Mac mic — Android user hears Mac audio fine, confirming Mac→Android works.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Related Files
|
||||||
|
|
||||||
|
- `crates/wzp-client/src/audio_vpio.rs:128–147` — render callback (primary suspect)
|
||||||
|
- `crates/wzp-client/src/audio_vpio.rs:35–161` — full VPIO start sequence
|
||||||
|
- `crates/wzp-client/src/audio_ring.rs` — ring buffer used by both producer and consumer
|
||||||
|
- `desktop/src-tauri/src/engine.rs:1562–1600` — VPIO vs CPAL selection
|
||||||
|
- `desktop/src-tauri/src/engine.rs:1760–1900` — recv task writing into `playout_ring`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Fix Plan (Once Diagnosed)
|
||||||
|
|
||||||
|
| Diagnosis | Fix |
|
||||||
|
|-----------|-----|
|
||||||
|
| A — frame-size mismatch | Make `AudioRing` consumer drain variable chunks, or buffer to fixed 960 in recv task before ring write |
|
||||||
|
| B — render callback not firing | Investigate VPIO initialization order; consider separate input + output `AudioUnit` instances |
|
||||||
|
| C — wrong output device | Set `kAudioOutputUnitProperty_CurrentDevice` explicitly to `kAudioObjectSystemObject` default output at start |
|
||||||
|
| D — AEC suppression | Test with VPIO bypass mode (`kAUVoiceIOProperty_BypassVoiceProcessing`) on; if audio returns, file CoreAudio quirk and tune AEC config |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Cross-References
|
||||||
|
|
||||||
|
- BUG-001 (Android join-voice hang) — separate issue, already mitigated; current Android build joins room successfully and recv works.
|
||||||
|
- Memory: `project_desktop_client.md` notes the desktop rewrite uses CPAL + VoiceProcessingIO with "direct playout, OS-level AEC" — this bug is the first failure of that path under real call conditions.
|
||||||
415
docs/bugs/003-android-to-macos-video-banding.md
Normal file
415
docs/bugs/003-android-to-macos-video-banding.md
Normal file
@@ -0,0 +1,415 @@
|
|||||||
|
# BUG-003: Android to macOS Video Banding / Horizontal Lines
|
||||||
|
|
||||||
|
**Severity:** P0/P1 - Android camera video is visibly corrupted on macOS at common resolutions.
|
||||||
|
**Status:** Root cause identified 2026-05-26; candidate fix in `crates/wzp-video/src/videotoolbox.rs`. Awaiting on-device verification.
|
||||||
|
**Branch:** `main`.
|
||||||
|
**Latest build observed:** `3ea25a0` (`fix(android): use MediaCodec input layout for video encode`).
|
||||||
|
**Direction affected:** Android camera -> macOS desktop display.
|
||||||
|
**Direction mostly OK:** macOS camera -> Android display.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Root Cause (2026-05-26)
|
||||||
|
|
||||||
|
The Android H.264 bitstream is **valid**: the locally-encoded `.h264` files and
|
||||||
|
the macOS-reassembled `.h264` files both decode cleanly with software ffmpeg.
|
||||||
|
SPS reports the expected `960x540`, `coded_height=544`, `yuv420p`, High profile,
|
||||||
|
level 3.1.
|
||||||
|
|
||||||
|
The corruption appears purely on the macOS receive side. The shiguredo
|
||||||
|
`I420Frame` wrapper around `CVPixelBuffer` exposes each plane as
|
||||||
|
`bytes_per_row * height` bytes — i.e. the raw plane buffer including the
|
||||||
|
per-row stride padding that CoreVideo adds for alignment. `VideoToolboxDecoder`
|
||||||
|
was concatenating those slices verbatim, then handing the buffer downstream
|
||||||
|
tagged as tight I420 of `width x height`. The JPEG-encoding consumer
|
||||||
|
(`i420_to_jpeg_bytes` in `desktop/src-tauri/src/lib.rs`) indexes the buffer
|
||||||
|
with tight strides `width` and `width/2`, so any plane where
|
||||||
|
`bytes_per_row > tight_stride` produces per-row drift in the consumer's reads.
|
||||||
|
|
||||||
|
Numerical confirmation from the corrupted dump
|
||||||
|
`000002_desktop_remote_decoded_f000001_960x540.jpg`:
|
||||||
|
|
||||||
|
- Banding period along the diagonal: exactly **32 luma rows** = 16 chroma rows.
|
||||||
|
- Per-column-slice peak offsets shift by ~5 rows per 230-column step, i.e. the
|
||||||
|
bands are a tilted diagonal, not horizontal — consistent with one chroma row
|
||||||
|
of drift accumulating per 16 chroma rows of consumer read.
|
||||||
|
- Solving `u_stride / (u_stride - chroma_width) = 16` with `chroma_width = 480`
|
||||||
|
yields `u_stride = 512`. That is exactly the 64-byte aligned chroma stride
|
||||||
|
CoreVideo emits for a 480-wide plane.
|
||||||
|
- Luma at 960 wide is already 64-aligned, so `y_stride = 960` and the luma
|
||||||
|
plane is unaffected. This matches the bug doc note that 640x360 looks fine
|
||||||
|
(chroma_width 320 is also 64-aligned, no padding needed).
|
||||||
|
|
||||||
|
## Fix
|
||||||
|
|
||||||
|
`crates/wzp-video/src/videotoolbox.rs` now has an `i420_frame_to_tight` helper
|
||||||
|
that copies each plane row-by-row using its own `bytes_per_row`, producing a
|
||||||
|
genuine tight I420 buffer of `width * height + 2 * (width/2) * (height/2)`
|
||||||
|
bytes. All three decoders (H.264, HEVC, AV1) call the helper instead of
|
||||||
|
concatenating raw plane slices. On the first successful decode each decoder
|
||||||
|
logs the actual plane dimensions and strides (`tracing::info!` at target
|
||||||
|
`wzp_video::videotoolbox`) so future similar bugs are easier to diagnose
|
||||||
|
without re-deriving from band spacing.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Symptom
|
||||||
|
|
||||||
|
When Android sends camera video to macOS, the macOS view shows repeated horizontal green/magenta line bands over the decoded picture. The lines cover the whole decoded frame, including black side bars added by the Android portrait-camera contain/crop fix.
|
||||||
|
|
||||||
|
The Android camera crop/zoom problem is fixed now: the Android front camera is no longer cover-cropped into an extreme zoom. The remaining bug is the line/banding corruption.
|
||||||
|
|
||||||
|
The issue is easy to see at H.264 960x540. At 640x360 it has been reported as visually good or much better. HEVC behaves differently: minimum resolution can look good, but 960x540 and 1280x720 tend to pause or deliver only bursts of frames.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Current State
|
||||||
|
|
||||||
|
Recent commits relevant to this bug:
|
||||||
|
|
||||||
|
```text
|
||||||
|
3ea25a0 fix(android): use MediaCodec input layout for video encode
|
||||||
|
1124726 fix(video): add frame metadata and Android encode diagnostics
|
||||||
|
9a77459 feat(video): add codec and resolution controls
|
||||||
|
f85efb9 fix(video): improve android stream smoothness
|
||||||
|
31b2caa fix(video): request keyframes after packet loss
|
||||||
|
079e21e fix(video): resync decoder after packet gaps
|
||||||
|
e676641 fix(android): suppress debuggable lint for diagnostic builds
|
||||||
|
9713efc chore(android): add release debuggable build
|
||||||
|
```
|
||||||
|
|
||||||
|
Important behavior:
|
||||||
|
|
||||||
|
- Android source dumps are clean.
|
||||||
|
- Android I420 roundtrip dumps are clean.
|
||||||
|
- macOS decoded remote Android frames are corrupted.
|
||||||
|
- Android receiving macOS video is generally clean.
|
||||||
|
- Transport/reassembly is probably not the primary issue: early Android local encoded `.h264` files match the corresponding macOS remote reassembled `.h264` prefix/length.
|
||||||
|
- The bug is likely in Android MediaCodec encoder input layout/color handling, H.264 non-macroblock-aligned dimensions/cropping, or macOS VideoToolbox interpretation of Android-encoded H.264.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Reproduction Build
|
||||||
|
|
||||||
|
Use the Tauri Android pipeline, not the legacy native Android Gradle app.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /Users/manwe/CascadeProjects/warzonePhone
|
||||||
|
git status --short
|
||||||
|
git log -1 --oneline
|
||||||
|
./scripts/android-build-async.sh --release-debuggable --wait
|
||||||
|
```
|
||||||
|
|
||||||
|
The APK lands here:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
/Users/manwe/CascadeProjects/warzonePhone/target/tauri-android-apk/wzp-tauri-arm64.apk
|
||||||
|
```
|
||||||
|
|
||||||
|
Install it:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
adb install -r /Users/manwe/CascadeProjects/warzonePhone/target/tauri-android-apk/wzp-tauri-arm64.apk
|
||||||
|
```
|
||||||
|
|
||||||
|
Use `--release-debuggable` for this bug. Plain debug builds can mask the issue because they run at much lower frame rate and look like a slideshow. Plain release builds are not usable for `run-as` frame-dump retrieval.
|
||||||
|
|
||||||
|
Critical build trap: `scripts/android-build-async.sh` runs `scripts/build-tauri-android.sh`, which SSHes to `SepehrHomeserverdk` and resets the remote source to `origin/$BRANCH`. Uncommitted local changes are ignored by the Android build. Commit and push before building, or the phone may run old code.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## macOS Build / Run
|
||||||
|
|
||||||
|
For local desktop repro:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /Users/manwe/CascadeProjects/warzonePhone/desktop
|
||||||
|
npm install
|
||||||
|
npm run tauri dev
|
||||||
|
```
|
||||||
|
|
||||||
|
Enable call debug logs in the app settings before starting the call. The in-app call log only keeps the last 200 entries; use the copy/share buttons if preserving textual logs matters.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Repro Steps
|
||||||
|
|
||||||
|
1. Start the macOS desktop client.
|
||||||
|
2. Start the Android `--release-debuggable` APK.
|
||||||
|
3. Join the same room, usually `general`.
|
||||||
|
4. Use the same relay as the current manual tests, e.g. `172.16.81.135:4433`, unless testing relay-specific behavior.
|
||||||
|
5. Turn camera on for both clients.
|
||||||
|
6. Set both sides to H.264.
|
||||||
|
7. Set Android send resolution to 960x540. Mac can be 960x540 or higher.
|
||||||
|
8. Observe Android camera video on macOS.
|
||||||
|
|
||||||
|
Expected failure: macOS shows Android video with repeated horizontal green/magenta lines. Android camera source preview and Android frame dumps are clean.
|
||||||
|
|
||||||
|
Useful comparison tests:
|
||||||
|
|
||||||
|
| Codec / resolution | Observed result |
|
||||||
|
|---|---|
|
||||||
|
| H.264 960x540 | Lines/banding on macOS for Android video |
|
||||||
|
| H.264 640x360 | Reported good or much better; smoother |
|
||||||
|
| H.264 1280x720 | Lines/banding and/or worse smoothness |
|
||||||
|
| HEVC 1280x720 | Mac video smooth on Android; Android video on Mac pauses and can look zoomed/corrupt |
|
||||||
|
| HEVC 960x540 | Same pause pattern, shorter pauses |
|
||||||
|
| HEVC minimum resolution | Reported good on both devices |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Artifact Collection
|
||||||
|
|
||||||
|
### Clear old dumps before a fresh run
|
||||||
|
|
||||||
|
macOS:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
rm -rf "$HOME/Library/Application Support/com.wzp.desktop/.wzp/frame-dumps"
|
||||||
|
```
|
||||||
|
|
||||||
|
Android:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
adb shell run-as com.wzp.desktop rm -rf .wzp/frame-dumps
|
||||||
|
```
|
||||||
|
|
||||||
|
The Android clear command requires a debuggable build. If `run-as` fails, rebuild with `--release-debuggable`.
|
||||||
|
|
||||||
|
### Pull Android dumps
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /Users/manwe/CascadeProjects/warzonePhone
|
||||||
|
./scripts/pull-android-frame-dumps.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
Output directory:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/Users/manwe/CascadeProjects/warzonePhone/android-frame-dumps/frame-dumps
|
||||||
|
```
|
||||||
|
|
||||||
|
The pull script packages files using:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
adb exec-out "run-as com.wzp.desktop tar -C .wzp -cf - frame-dumps"
|
||||||
|
```
|
||||||
|
|
||||||
|
### macOS dump directory
|
||||||
|
|
||||||
|
```text
|
||||||
|
/Users/manwe/Library/Application Support/com.wzp.desktop/.wzp/frame-dumps
|
||||||
|
```
|
||||||
|
|
||||||
|
### Important dump names
|
||||||
|
|
||||||
|
| Dump suffix | Meaning |
|
||||||
|
|---|---|
|
||||||
|
| `android_camera_jpeg_in_fXXXXXX_<WxH>.jpg` | Raw browser/camera JPEG entering Rust from Android WebView |
|
||||||
|
| `android_camera_i420_roundtrip_fXXXXXX_<WxH>.jpg` | Android camera frame after JS/canvas -> Rust I420 conversion, converted back to JPEG |
|
||||||
|
| `android_local_encoded_fXXXXXX.h264` / `.h265` | Encoded Android camera bitstream before packetization |
|
||||||
|
| `desktop_remote_encoded_reassembled_fXXXXXX.h264` / `.h265` | macOS reassembled encoded bitstream received from Android |
|
||||||
|
| `desktop_remote_decoded_fXXXXXX_<WxH>.jpg` | macOS decoded Android video frame, where the lines show |
|
||||||
|
| `android_remote_decoded_fXXXXXX_<WxH>.jpg` | Android decoded macOS video frame |
|
||||||
|
|
||||||
|
Known useful local examples from the latest sessions:
|
||||||
|
|
||||||
|
```text
|
||||||
|
Clean Android source:
|
||||||
|
/Users/manwe/CascadeProjects/warzonePhone/android-frame-dumps/frame-dumps/000407_android_camera_jpeg_in_f000150_960x540.jpg
|
||||||
|
/Users/manwe/CascadeProjects/warzonePhone/android-frame-dumps/frame-dumps/000408_android_camera_i420_roundtrip_f000150_960x540.jpg
|
||||||
|
|
||||||
|
Corrupt macOS decode:
|
||||||
|
/Users/manwe/Library/Application Support/com.wzp.desktop/.wzp/frame-dumps/000236_desktop_remote_decoded_f000030_960x540.jpg
|
||||||
|
/Users/manwe/Library/Application Support/com.wzp.desktop/.wzp/frame-dumps/000241_desktop_remote_decoded_f000060_960x540.jpg
|
||||||
|
/Users/manwe/Library/Application Support/com.wzp.desktop/.wzp/frame-dumps/000244_desktop_remote_decoded_f000090_960x540.jpg
|
||||||
|
|
||||||
|
Encoded bitstream comparison:
|
||||||
|
/Users/manwe/CascadeProjects/warzonePhone/android-frame-dumps/frame-dumps/000005_android_local_encoded_f000001.h264
|
||||||
|
/Users/manwe/Library/Application Support/com.wzp.desktop/.wzp/frame-dumps/000064_desktop_remote_encoded_reassembled_f000001.h264
|
||||||
|
```
|
||||||
|
|
||||||
|
These files are local artifacts, not committed test fixtures.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Text Logs
|
||||||
|
|
||||||
|
### In-app call debug log
|
||||||
|
|
||||||
|
Enable `Call debug logs` in settings before joining. The UI buffer is limited to 200 entries. Use the in-app copy/share buttons immediately after the repro.
|
||||||
|
|
||||||
|
Useful events:
|
||||||
|
|
||||||
|
```text
|
||||||
|
camera:get_user_media_ok
|
||||||
|
camera:capture_clock
|
||||||
|
camera:capture_frame
|
||||||
|
video:first_camera_frame
|
||||||
|
video:camera_frame_sample
|
||||||
|
video:encoded_frame
|
||||||
|
video:first_send
|
||||||
|
video:first_recv
|
||||||
|
video:first_reassembled
|
||||||
|
video:reassembled_frame
|
||||||
|
video:decoder_init_start
|
||||||
|
video:first_decoded_frame
|
||||||
|
video:decoded_frame_sample
|
||||||
|
video:frame_dump
|
||||||
|
video:byte_dump
|
||||||
|
```
|
||||||
|
|
||||||
|
The crop fix is active when Android `camera:capture_frame` includes portrait source dimensions with a landscape send frame, for example:
|
||||||
|
|
||||||
|
```text
|
||||||
|
camera:capture_frame {"frame_no":150,"width":960,"height":540,"source_width":540,"source_height":960,...}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Android logcat
|
||||||
|
|
||||||
|
Logcat can be noisy and may not always retain the in-app call debug entries. Still useful commands:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
adb logcat -c
|
||||||
|
adb logcat -v time | rg 'camera:capture_frame|video:frame_dump|video:byte_dump|video:first_camera_frame|video:camera_frame_sample|video:encoded_frame|h264_encoder_input|hevc_encoder_input|MediaCodec input format|decoder_debug'
|
||||||
|
```
|
||||||
|
|
||||||
|
For post-run collection:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
adb logcat -d -v time > /tmp/wzp-android-logcat.txt
|
||||||
|
rg 'camera:|video:|h264_encoder_input|hevc_encoder_input|MediaCodec|decoder_debug' /tmp/wzp-android-logcat.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
If no `h264_encoder_input` / `hevc_encoder_input` entries appear, the current `tracing::info!` path in `crates/wzp-video/src/mediacodec.rs` may not be making it into Android logcat. Convert that diagnostic to `emit_call_debug` from the caller if the next step needs guaranteed visibility.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## What We Know
|
||||||
|
|
||||||
|
### The Android camera/canvas path is probably clean
|
||||||
|
|
||||||
|
The Android dumps for `android_camera_jpeg_in` and `android_camera_i420_roundtrip` at 960x540 are clean. They show the portrait front camera contained inside a landscape frame with black side bars. This means the former zoom/crop bug is fixed and the current bands are not introduced by CSS, canvas sizing, or the browser camera preview.
|
||||||
|
|
||||||
|
### The corruption appears after encode/decode
|
||||||
|
|
||||||
|
The corrupt lines are present in `desktop_remote_decoded_*`. They cover black bars as well as image content, which points to frame buffer / codec layout corruption rather than a real scene artifact.
|
||||||
|
|
||||||
|
### Transport is not the leading suspect
|
||||||
|
|
||||||
|
`android_local_encoded_f000001.h264` and `desktop_remote_encoded_reassembled_f000001.h264` have matching sizes/prefixes in the latest diagnostic run. That does not fully prove every later packet is perfect, but it makes relay/datagram/reassembly much less likely as the root cause.
|
||||||
|
|
||||||
|
Relays should not need changes for this bug unless the wire format changes. The relay forwards datagrams and does not inspect video frame internals.
|
||||||
|
|
||||||
|
### Resolution alignment is suspicious
|
||||||
|
|
||||||
|
960x540 has a height that is not divisible by 16. H.264 macroblock encoders commonly encode 960x544 and signal cropping to 960x540. The horizontal line bands may be a crop/padding/chroma-plane issue. Testing 960x544 and/or 960x528 is a high-value next step.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Code Areas
|
||||||
|
|
||||||
|
Primary suspects:
|
||||||
|
|
||||||
|
- `crates/wzp-video/src/mediacodec.rs` - Android MediaCodec H.264/HEVC encoder and decoder, color format, stride, slice height handling.
|
||||||
|
- `desktop/src-tauri/src/engine.rs` - packet send/receive, decode lifecycle, frame/byte dump calls.
|
||||||
|
- `desktop/src-tauri/src/lib.rs` - `maybe_dump_video_jpeg`, `maybe_dump_video_bytes`, app-data paths, call-debug event plumbing.
|
||||||
|
- `desktop/src/main.ts` - browser camera capture, canvas scaling, codec/resolution settings, UI debug log buffer.
|
||||||
|
- `crates/wzp-video/src/transport.rs` - video packetization/reassembly and `WZV1` metadata header.
|
||||||
|
|
||||||
|
The latest attempted fix in `mediacodec.rs` uses `codec.input_format()` on Android API 28+ to derive encoder input stride/slice/color layout. Since the lines persist, either those fields are not reliable for this encoder, the chosen color format conversion is wrong, or macOS decode/crop interpretation is involved.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Recommended Next Debug Steps
|
||||||
|
|
||||||
|
1. Verify whether Android logs the encoder input format on the failing build.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
adb logcat -d -v time | rg 'h264_encoder_input|hevc_encoder_input|input_color_format|effective_stride|effective_slice'
|
||||||
|
```
|
||||||
|
|
||||||
|
If absent, make this an app call-debug event instead of plain tracing so it appears in the copied call log.
|
||||||
|
|
||||||
|
2. Add Android loopback decode of `android_local_encoded_*` before network.
|
||||||
|
|
||||||
|
Dump a new `android_local_decoded_fXXXXXX_<WxH>.jpg` immediately after encoding. If this local Android decode already has bands, the encoder output is bad. If Android local decode is clean but macOS decode is bad, focus on H.264 SPS cropping / VideoToolbox decode assumptions.
|
||||||
|
|
||||||
|
3. Test macroblock-aligned debug resolutions.
|
||||||
|
|
||||||
|
Add or force:
|
||||||
|
|
||||||
|
```text
|
||||||
|
960x544
|
||||||
|
960x528
|
||||||
|
640x368
|
||||||
|
640x352
|
||||||
|
```
|
||||||
|
|
||||||
|
If 960x544 fixes the lines, the bug is almost certainly H.264 crop/padding handling. If 960x528 fixes it but 960x544 does not, inspect bottom padding and crop signaling.
|
||||||
|
|
||||||
|
4. Offline-decode `android_local_encoded_*.h264` with a known-good decoder.
|
||||||
|
|
||||||
|
Example on a machine with working ffmpeg:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ffmpeg -f h264 -i android-frame-dumps/frame-dumps/000005_android_local_encoded_f000001.h264 -frames:v 1 /tmp/android-local-f1.png
|
||||||
|
ffmpeg -f h264 -i "$HOME/Library/Application Support/com.wzp.desktop/.wzp/frame-dumps/000064_desktop_remote_encoded_reassembled_f000001.h264" -frames:v 1 /tmp/macos-remote-f1.png
|
||||||
|
```
|
||||||
|
|
||||||
|
Note: Homebrew ffmpeg on this Mac was broken during debugging with a missing `libvpx.11.dylib`, so do not assume `/opt/homebrew/bin/ffmpeg` works until fixed.
|
||||||
|
|
||||||
|
5. Try explicit Android encoder input variants.
|
||||||
|
|
||||||
|
Test one variable at a time:
|
||||||
|
|
||||||
|
- Force planar color format `COLOR_FormatYUV420Planar` / value `19` and feed I420.
|
||||||
|
- Force semiplanar and try NV12 vs NV21/VU order.
|
||||||
|
- Use `COLOR_FormatYUV420Flexible` if accepted by this device.
|
||||||
|
- Use `stride = width`, `slice_height = align_up(height, 16)` only.
|
||||||
|
- Use `stride = align_up(width, 16)`, `slice_height = align_up(height, 16)`.
|
||||||
|
|
||||||
|
6. Parse SPS from Android H.264 output.
|
||||||
|
|
||||||
|
Confirm encoded dimensions and frame cropping offsets for 960x540. Compare Android output against macOS output. If SPS says 960x544 with crop to 540, test whether VideoToolbox applies the crop correctly.
|
||||||
|
|
||||||
|
7. Keep relay out of the first debugging loop.
|
||||||
|
|
||||||
|
The relay is unlikely to affect deterministic decoded line bands when local encoded and remote reassembled payloads match. Only redeploy relay if packet framing changes.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Verification Criteria For A Fix
|
||||||
|
|
||||||
|
A candidate fix is good when:
|
||||||
|
|
||||||
|
- Android `android_camera_jpeg_in` and `android_camera_i420_roundtrip` remain clean.
|
||||||
|
- Android `android_local_decoded`, if added, is clean.
|
||||||
|
- macOS `desktop_remote_decoded` is clean at H.264 960x540.
|
||||||
|
- 960x540 is smooth enough for normal calls, not a debug-build slideshow.
|
||||||
|
- H.264 1280x720 either works or fails in an understood performance-only way.
|
||||||
|
- HEVC behavior is not regressed from current minimum-resolution success.
|
||||||
|
|
||||||
|
Run at least:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo check -p wzp-video --target aarch64-linux-android
|
||||||
|
cargo check -p wzp-video -p wzp-client -p wzp-desktop
|
||||||
|
```
|
||||||
|
|
||||||
|
Then build Android with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/android-build-async.sh --release-debuggable --wait
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Open Questions
|
||||||
|
|
||||||
|
- Does the failing Android device actually report encoder input `stride`, `slice-height`, and `color-format` after `start()`? The code asks for this, but recent logcat sampling did not show the `h264_encoder_input` tracing lines.
|
||||||
|
- Does Android local decode of its own encoded H.264 reproduce the same lines?
|
||||||
|
- Is 960x540 failing because H.264 encodes a 544-high macroblock frame and macOS crops or interprets chroma padding incorrectly?
|
||||||
|
- Are the green/magenta bands chroma-plane corruption, luma padding leakage, or debug overlay from an encoder surface path? Current pipeline uses byte-buffer input, not surface input.
|
||||||
|
- Is HEVC's pause behavior a separate decoder buffering/keyframe issue or the same layout problem expressed differently?
|
||||||
|
|
||||||
113
scripts/android-build-async.sh
Executable file
113
scripts/android-build-async.sh
Executable file
@@ -0,0 +1,113 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Fire-and-forget Android APK builder.
|
||||||
|
#
|
||||||
|
# Runs ./scripts/build-tauri-android.sh inside a LOCAL tmux session so the
|
||||||
|
# build survives terminal disconnects. The wrapped script SSHes to
|
||||||
|
# SepehrHomeserverdk on its own — we don't try to upload+run anything on
|
||||||
|
# the remote (that would re-SSH from the remote to itself, which fails).
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# ./scripts/android-build-async.sh # build current branch, arm64
|
||||||
|
# ./scripts/android-build-async.sh --init # also run cargo tauri android init
|
||||||
|
# ./scripts/android-build-async.sh --rust # force-clean Rust target cache
|
||||||
|
# ./scripts/android-build-async.sh --no-pull # skip git fetch on remote
|
||||||
|
# ./scripts/android-build-async.sh --debug # debug APK
|
||||||
|
# ./scripts/android-build-async.sh --release-debuggable # release APK with run-as dumps
|
||||||
|
# ./scripts/android-build-async.sh --wait # block until done, then tail status
|
||||||
|
#
|
||||||
|
# Progress / completion: ntfy.sh/wzp (handled by build-tauri-android.sh).
|
||||||
|
# Monitor locally: tmux attach -t wzp-android-local
|
||||||
|
# tail -f /tmp/wzp-tauri-build-local.log
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
TMUX_SESSION="wzp-android-local"
|
||||||
|
LOCAL_LOG="/tmp/wzp-tauri-build-local.log"
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||||
|
REPO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||||
|
BUILD_SCRIPT="$SCRIPT_DIR/build-tauri-android.sh"
|
||||||
|
|
||||||
|
if ! command -v tmux >/dev/null 2>&1; then
|
||||||
|
echo "ERROR: tmux is not installed locally. Install with: brew install tmux"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ ! -x "$BUILD_SCRIPT" ]; then
|
||||||
|
echo "ERROR: $BUILD_SCRIPT not found or not executable"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
BRANCH="${WZP_BRANCH:-$(git -C "$REPO_DIR" branch --show-current 2>/dev/null || echo "")}"
|
||||||
|
if [ -z "$BRANCH" ]; then
|
||||||
|
echo "ERROR: could not determine branch (detached HEAD?). Set WZP_BRANCH=name."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
DO_WAIT=0
|
||||||
|
PASS_ARGS=()
|
||||||
|
for arg in "$@"; do
|
||||||
|
case "$arg" in
|
||||||
|
--wait) DO_WAIT=1 ;;
|
||||||
|
*) PASS_ARGS+=("$arg") ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
log() { echo -e "\033[1;36m>>> $*\033[0m"; }
|
||||||
|
|
||||||
|
# Kill any prior session that might still be hanging around.
|
||||||
|
tmux kill-session -t "$TMUX_SESSION" 2>/dev/null || true
|
||||||
|
|
||||||
|
# Write a launcher script — avoids fragile quoting inside `tmux new-session`.
|
||||||
|
LAUNCHER="$(mktemp -t wzp-android-launcher.XXXXXX)"
|
||||||
|
chmod +x "$LAUNCHER"
|
||||||
|
{
|
||||||
|
echo "#!/usr/bin/env bash"
|
||||||
|
echo "set -o pipefail"
|
||||||
|
echo "cd $(printf %q "$REPO_DIR")"
|
||||||
|
echo "export WZP_BRANCH=$(printf %q "$BRANCH")"
|
||||||
|
printf 'bash %q' "$BUILD_SCRIPT"
|
||||||
|
for a in "${PASS_ARGS[@]:-}"; do
|
||||||
|
[ -z "$a" ] && continue
|
||||||
|
printf ' %q' "$a"
|
||||||
|
done
|
||||||
|
echo " 2>&1 | tee $(printf %q "$LOCAL_LOG")"
|
||||||
|
echo "echo DONE_EXIT_CODE=\$? >> $(printf %q "$LOCAL_LOG")"
|
||||||
|
} > "$LAUNCHER"
|
||||||
|
|
||||||
|
# Create the log file up front so `tail -f` works immediately.
|
||||||
|
: > "$LOCAL_LOG"
|
||||||
|
|
||||||
|
log "Starting local tmux session '$TMUX_SESSION' (branch: $BRANCH)..."
|
||||||
|
log "Build script: $BUILD_SCRIPT ${PASS_ARGS[*]:-}"
|
||||||
|
log "Launcher: $LAUNCHER"
|
||||||
|
log "Local log: $LOCAL_LOG"
|
||||||
|
|
||||||
|
tmux new-session -d -s "$TMUX_SESSION" -c "$REPO_DIR" "bash $LAUNCHER; exec bash"
|
||||||
|
|
||||||
|
# Verify the session actually started.
|
||||||
|
sleep 1
|
||||||
|
if ! tmux has-session -t "$TMUX_SESSION" 2>/dev/null; then
|
||||||
|
echo "ERROR: tmux session '$TMUX_SESSION' failed to start. Launcher contents:"
|
||||||
|
cat "$LAUNCHER"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "Build dispatched! ntfy.sh/wzp will notify on completion."
|
||||||
|
echo ""
|
||||||
|
echo " Monitor : tail -f $LOCAL_LOG"
|
||||||
|
echo " Status : tail -5 $LOCAL_LOG"
|
||||||
|
echo " Attach : tmux attach -t $TMUX_SESSION"
|
||||||
|
echo " Kill : tmux kill-session -t $TMUX_SESSION"
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
if [ "$DO_WAIT" = "0" ]; then
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "Waiting for build to finish (watching $LOCAL_LOG)..."
|
||||||
|
until grep -qE 'DONE_EXIT_CODE|APK_REMOTE_PATH=|FAILED' "$LOCAL_LOG" 2>/dev/null; do
|
||||||
|
sleep 20
|
||||||
|
done
|
||||||
|
|
||||||
|
log "Build session ended. Last 20 lines:"
|
||||||
|
tail -20 "$LOCAL_LOG"
|
||||||
@@ -10,6 +10,7 @@ set -euo pipefail
|
|||||||
# ./scripts/build-linux-docker.sh --pull Git pull before building
|
# ./scripts/build-linux-docker.sh --pull Git pull before building
|
||||||
# ./scripts/build-linux-docker.sh --clean Clean Rust target cache
|
# ./scripts/build-linux-docker.sh --clean Clean Rust target cache
|
||||||
# ./scripts/build-linux-docker.sh --install Download binaries locally after build
|
# ./scripts/build-linux-docker.sh --install Download binaries locally after build
|
||||||
|
# ./scripts/build-linux-docker.sh --deploy Download + deploy wzp-relay to relay servers
|
||||||
|
|
||||||
REMOTE_HOST="SepehrHomeserverdk"
|
REMOTE_HOST="SepehrHomeserverdk"
|
||||||
BASE_DIR="/mnt/storage/manBuilder"
|
BASE_DIR="/mnt/storage/manBuilder"
|
||||||
@@ -21,17 +22,26 @@ SSH_OPTS="-o ConnectTimeout=15 -o ServerAliveInterval=15 -o ServerAliveCountMax=
|
|||||||
# (opus-DRED-v2 as of 2026-04-11). Override with `WZP_BRANCH=<name> ./build-linux-docker.sh`
|
# (opus-DRED-v2 as of 2026-04-11). Override with `WZP_BRANCH=<name> ./build-linux-docker.sh`
|
||||||
# if you need a different one — e.g. to rebuild the relay from a feature
|
# if you need a different one — e.g. to rebuild the relay from a feature
|
||||||
# branch for A/B testing.
|
# branch for A/B testing.
|
||||||
WZP_BRANCH="${WZP_BRANCH:-opus-DRED-v2}"
|
WZP_BRANCH="${WZP_BRANCH:-$(git -C "$(dirname "$0")/.." branch --show-current 2>/dev/null || echo "experimental-ui")}"
|
||||||
|
|
||||||
|
# Relay servers to deploy to when --deploy is passed.
|
||||||
|
# Format: "user@host:binary_dir:tmux_session"
|
||||||
|
RELAY_SERVERS=(
|
||||||
|
"manwe@manwehs:/home/manwe/wzp:5"
|
||||||
|
"manwe@pangolin.manko.yoga:/home/manwe/wzp-linux:0"
|
||||||
|
)
|
||||||
|
|
||||||
DO_PULL=1
|
DO_PULL=1
|
||||||
DO_CLEAN=0
|
DO_CLEAN=0
|
||||||
DO_INSTALL=0
|
DO_INSTALL=0
|
||||||
|
DO_DEPLOY=0
|
||||||
for arg in "$@"; do
|
for arg in "$@"; do
|
||||||
case "$arg" in
|
case "$arg" in
|
||||||
--pull) DO_PULL=1 ;;
|
--pull) DO_PULL=1 ;;
|
||||||
--no-pull) DO_PULL=0 ;;
|
--no-pull) DO_PULL=0 ;;
|
||||||
--clean) DO_CLEAN=1 ;;
|
--clean) DO_CLEAN=1 ;;
|
||||||
--install) DO_INSTALL=1 ;;
|
--install) DO_INSTALL=1 ;;
|
||||||
|
--deploy) DO_DEPLOY=1; DO_INSTALL=1 ;;
|
||||||
esac
|
esac
|
||||||
done
|
done
|
||||||
|
|
||||||
@@ -95,20 +105,15 @@ docker run --rm --user 1000:1000 \
|
|||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
cd /build/source
|
cd /build/source
|
||||||
|
|
||||||
echo ">>> Building relay + client + web + bench..."
|
echo ">>> Building relay + web..."
|
||||||
cargo build --release --bin wzp-relay --bin wzp-client --bin wzp-web --bin wzp-bench 2>&1 | tail -5
|
cargo build --release --bin wzp-relay --bin wzp-web 2>&1 | tail -5
|
||||||
|
|
||||||
echo ">>> Building audio client..."
|
|
||||||
cargo build --release --bin wzp-client --features audio 2>&1 | tail -3
|
|
||||||
cp target/release/wzp-client target/release/wzp-client-audio
|
|
||||||
cargo build --release --bin wzp-client 2>&1 | tail -3
|
|
||||||
|
|
||||||
echo ">>> Binaries:"
|
echo ">>> Binaries:"
|
||||||
ls -lh target/release/wzp-relay target/release/wzp-client target/release/wzp-client-audio target/release/wzp-web target/release/wzp-bench
|
ls -lh target/release/wzp-relay target/release/wzp-web
|
||||||
|
|
||||||
echo ">>> Packaging..."
|
echo ">>> Packaging..."
|
||||||
tar czf /tmp/wzp-linux-x86_64.tar.gz \
|
tar czf /tmp/wzp-linux-x86_64.tar.gz \
|
||||||
-C target/release wzp-relay wzp-client wzp-client-audio wzp-web wzp-bench
|
-C target/release wzp-relay wzp-web
|
||||||
|
|
||||||
echo "BINARIES_BUILT"
|
echo "BINARIES_BUILT"
|
||||||
'
|
'
|
||||||
@@ -121,7 +126,7 @@ TARBALL="$BASE_DIR/data/cache-linux/target/release/../../../wzp-linux-x86_64.tar
|
|||||||
docker run --rm \
|
docker run --rm \
|
||||||
-v "$BASE_DIR/data/cache-linux/target:/build/target" \
|
-v "$BASE_DIR/data/cache-linux/target:/build/target" \
|
||||||
wzp-android-builder bash -c \
|
wzp-android-builder bash -c \
|
||||||
"cp /build/target/release/wzp-relay /build/target/release/wzp-client /build/target/release/wzp-client-audio /build/target/release/wzp-web /build/target/release/wzp-bench /tmp/ && tar czf /tmp/wzp-linux-x86_64.tar.gz -C /tmp wzp-relay wzp-client wzp-client-audio wzp-web wzp-bench && cat /tmp/wzp-linux-x86_64.tar.gz" \
|
"cp /build/target/release/wzp-relay /build/target/release/wzp-web /tmp/ && tar czf /tmp/wzp-linux-x86_64.tar.gz -C /tmp wzp-relay wzp-web && cat /tmp/wzp-linux-x86_64.tar.gz" \
|
||||||
> /tmp/wzp-linux-x86_64.tar.gz
|
> /tmp/wzp-linux-x86_64.tar.gz
|
||||||
|
|
||||||
URL=$(curl -s -F "file=@/tmp/wzp-linux-x86_64.tar.gz" -H "Authorization: $rusty_auth_token" "$rusty_address")
|
URL=$(curl -s -F "file=@/tmp/wzp-linux-x86_64.tar.gz" -H "Authorization: $rusty_auth_token" "$rusty_address")
|
||||||
@@ -149,6 +154,46 @@ echo " Monitor: ssh $REMOTE_HOST 'tail -f /tmp/wzp-linux-build.log'"
|
|||||||
echo " Status: ssh $REMOTE_HOST 'tail -5 /tmp/wzp-linux-build.log'"
|
echo " Status: ssh $REMOTE_HOST 'tail -5 /tmp/wzp-linux-build.log'"
|
||||||
echo ""
|
echo ""
|
||||||
|
|
||||||
|
# Deploy wzp-relay to a single relay server.
|
||||||
|
# $1 = "user@host" $2 = binary_dir $3 = tmux_session
|
||||||
|
deploy_relay() {
|
||||||
|
local TARGET="$1"
|
||||||
|
local BINARY_DIR="$2"
|
||||||
|
local TMUX_SESSION="$3"
|
||||||
|
local DEPLOY_OPTS="-o ConnectTimeout=15 -o StrictHostKeyChecking=accept-new -o LogLevel=ERROR"
|
||||||
|
|
||||||
|
log "Deploying wzp-relay to $TARGET ($BINARY_DIR) ..."
|
||||||
|
|
||||||
|
# Copy new binary atomically
|
||||||
|
scp $DEPLOY_OPTS "$LOCAL_OUTPUT/wzp-relay" "$TARGET:$BINARY_DIR/wzp-relay.new"
|
||||||
|
ssh $DEPLOY_OPTS "$TARGET" "chmod +x $BINARY_DIR/wzp-relay.new && mv $BINARY_DIR/wzp-relay.new $BINARY_DIR/wzp-relay"
|
||||||
|
|
||||||
|
# Capture current args, stop, restart in same tmux session
|
||||||
|
ssh $DEPLOY_OPTS "$TARGET" bash <<DEPLOY
|
||||||
|
set -euo pipefail
|
||||||
|
RELAY_PID=\$(pgrep -f './wzp-relay' | head -1 || true)
|
||||||
|
if [ -z "\$RELAY_PID" ]; then
|
||||||
|
echo "WARNING: no running wzp-relay found on $TARGET — binary replaced, start it manually"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
# Capture args from /proc (everything after the binary name)
|
||||||
|
RELAY_ARGS=\$(tr '\\0' ' ' < /proc/\$RELAY_PID/cmdline | sed 's|^[^ ]* ||; s| *\$||')
|
||||||
|
echo "Stopping relay PID \$RELAY_PID (args: \$RELAY_ARGS)"
|
||||||
|
tmux send-keys -t $TMUX_SESSION C-c 2>/dev/null || kill -TERM \$RELAY_PID 2>/dev/null || true
|
||||||
|
sleep 2
|
||||||
|
echo "Starting new relay..."
|
||||||
|
tmux send-keys -t $TMUX_SESSION "cd $BINARY_DIR && ./wzp-relay \$RELAY_ARGS" Enter 2>/dev/null || true
|
||||||
|
echo "Deploy done on $TARGET"
|
||||||
|
DEPLOY
|
||||||
|
|
||||||
|
# Get the running version and notify
|
||||||
|
local DEPLOYED_VER
|
||||||
|
DEPLOYED_VER=$(ssh $DEPLOY_OPTS "$TARGET" "$BINARY_DIR/wzp-relay --version 2>/dev/null | awk '{print \$2}'" || echo "unknown")
|
||||||
|
curl -s -d "wzp-relay deployed to ${TARGET%%:*} — version $DEPLOYED_VER" "$NTFY_TOPIC" > /dev/null 2>&1 || true
|
||||||
|
|
||||||
|
log "Deployed to $TARGET"
|
||||||
|
}
|
||||||
|
|
||||||
# Optionally wait and download
|
# Optionally wait and download
|
||||||
if [ "$DO_INSTALL" = "1" ]; then
|
if [ "$DO_INSTALL" = "1" ]; then
|
||||||
log "Waiting for build..."
|
log "Waiting for build..."
|
||||||
@@ -170,5 +215,19 @@ if [ "$DO_INSTALL" = "1" ]; then
|
|||||||
log "Done! Binaries in $LOCAL_OUTPUT/"
|
log "Done! Binaries in $LOCAL_OUTPUT/"
|
||||||
else
|
else
|
||||||
err "Build failed"
|
err "Build failed"
|
||||||
|
exit 1
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Deploy to relay servers
|
||||||
|
if [ "$DO_DEPLOY" = "1" ]; then
|
||||||
|
if [ ! -f "$LOCAL_OUTPUT/wzp-relay" ]; then
|
||||||
|
err "wzp-relay binary not found in $LOCAL_OUTPUT — install step may have failed"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
for SERVER in "${RELAY_SERVERS[@]}"; do
|
||||||
|
IFS=: read -r TARGET BINARY_DIR TMUX_SESSION <<< "$SERVER"
|
||||||
|
deploy_relay "$TARGET" "$BINARY_DIR" "$TMUX_SESSION"
|
||||||
|
done
|
||||||
|
log "All relay servers updated!"
|
||||||
|
fi
|
||||||
|
|||||||
@@ -15,8 +15,9 @@ set -euo pipefail
|
|||||||
# - Output: desktop/src-tauri/gen/android/.../*.apk
|
# - Output: desktop/src-tauri/gen/android/.../*.apk
|
||||||
#
|
#
|
||||||
# Usage:
|
# Usage:
|
||||||
# ./scripts/build-tauri-android.sh # full pipeline (debug, arm64 only)
|
# ./scripts/build-tauri-android.sh # full pipeline (release, arm64 only)
|
||||||
# ./scripts/build-tauri-android.sh --release # release APK
|
# ./scripts/build-tauri-android.sh --debug # debug APK (faster, no optimisation)
|
||||||
|
# ./scripts/build-tauri-android.sh --release-debuggable # release APK with android:debuggable=true
|
||||||
# ./scripts/build-tauri-android.sh --no-pull # skip git fetch
|
# ./scripts/build-tauri-android.sh --no-pull # skip git fetch
|
||||||
# ./scripts/build-tauri-android.sh --rust # force-clean rust target
|
# ./scripts/build-tauri-android.sh --rust # force-clean rust target
|
||||||
# ./scripts/build-tauri-android.sh --init # also run `cargo tauri android init`
|
# ./scripts/build-tauri-android.sh --init # also run `cargo tauri android init`
|
||||||
@@ -38,7 +39,8 @@ SSH_OPTS="-o ConnectTimeout=15 -o ServerAliveInterval=15 -o ServerAliveCountMax=
|
|||||||
REBUILD_RUST=0
|
REBUILD_RUST=0
|
||||||
DO_PULL=1
|
DO_PULL=1
|
||||||
DO_INIT=0
|
DO_INIT=0
|
||||||
BUILD_RELEASE=0
|
BUILD_RELEASE=1
|
||||||
|
RELEASE_DEBUGGABLE=0
|
||||||
BUILD_ARCH="arm64"
|
BUILD_ARCH="arm64"
|
||||||
NEXT_IS_ARCH=0
|
NEXT_IS_ARCH=0
|
||||||
for arg in "$@"; do
|
for arg in "$@"; do
|
||||||
@@ -52,7 +54,8 @@ for arg in "$@"; do
|
|||||||
--pull) DO_PULL=1 ;;
|
--pull) DO_PULL=1 ;;
|
||||||
--no-pull) DO_PULL=0 ;;
|
--no-pull) DO_PULL=0 ;;
|
||||||
--init) DO_INIT=1 ;;
|
--init) DO_INIT=1 ;;
|
||||||
--release) BUILD_RELEASE=1 ;;
|
--debug) BUILD_RELEASE=0 ;;
|
||||||
|
--release-debuggable) RELEASE_DEBUGGABLE=1 ;;
|
||||||
--arch) NEXT_IS_ARCH=1 ;;
|
--arch) NEXT_IS_ARCH=1 ;;
|
||||||
-h|--help)
|
-h|--help)
|
||||||
sed -n '3,32p' "$0"
|
sed -n '3,32p' "$0"
|
||||||
@@ -93,6 +96,7 @@ REBUILD_RUST="${3:-0}"
|
|||||||
DO_INIT="${4:-0}"
|
DO_INIT="${4:-0}"
|
||||||
BUILD_RELEASE="${5:-0}"
|
BUILD_RELEASE="${5:-0}"
|
||||||
BUILD_ARCH="${6:-arm64}"
|
BUILD_ARCH="${6:-arm64}"
|
||||||
|
RELEASE_DEBUGGABLE="${7:-0}"
|
||||||
|
|
||||||
LOG_FILE=/tmp/wzp-tauri-build.log
|
LOG_FILE=/tmp/wzp-tauri-build.log
|
||||||
GIT_HASH="unknown" # populated after fetch
|
GIT_HASH="unknown" # populated after fetch
|
||||||
@@ -192,6 +196,7 @@ docker run --rm \
|
|||||||
-e DO_INIT="$DO_INIT" \
|
-e DO_INIT="$DO_INIT" \
|
||||||
-e PROFILE_FLAG="$PROFILE_FLAG" \
|
-e PROFILE_FLAG="$PROFILE_FLAG" \
|
||||||
-e BUILD_ARCH="$BUILD_ARCH" \
|
-e BUILD_ARCH="$BUILD_ARCH" \
|
||||||
|
-e RELEASE_DEBUGGABLE="$RELEASE_DEBUGGABLE" \
|
||||||
-v "$BASE_DIR/data/source:/build/source" \
|
-v "$BASE_DIR/data/source:/build/source" \
|
||||||
-v "$BASE_DIR/data/cache/cargo-registry:/home/builder/.cargo/registry" \
|
-v "$BASE_DIR/data/cache/cargo-registry:/home/builder/.cargo/registry" \
|
||||||
-v "$BASE_DIR/data/cache/cargo-git:/home/builder/.cargo/git" \
|
-v "$BASE_DIR/data/cache/cargo-git:/home/builder/.cargo/git" \
|
||||||
@@ -218,6 +223,29 @@ if [ "${DO_INIT}" = "1" ] || [ ! -x gen/android/gradlew ]; then
|
|||||||
cargo tauri android init 2>&1 | tail -20
|
cargo tauri android init 2>&1 | tail -20
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if [ "${RELEASE_DEBUGGABLE}" = "1" ]; then
|
||||||
|
MANIFEST="gen/android/app/src/main/AndroidManifest.xml"
|
||||||
|
if [ -f "$MANIFEST" ]; then
|
||||||
|
echo ">>> Marking release APK debuggable for frame-dump run-as access"
|
||||||
|
if ! grep -q "xmlns:tools=" "$MANIFEST"; then
|
||||||
|
perl -0pi -e "s/<manifest\\b/<manifest xmlns:tools=\"http:\\/\\/schemas.android.com\\/tools\"/s" "$MANIFEST"
|
||||||
|
fi
|
||||||
|
if grep -q "android:debuggable=" "$MANIFEST"; then
|
||||||
|
sed -i "s/android:debuggable=\"[^\"]*\"/android:debuggable=\"true\"/" "$MANIFEST"
|
||||||
|
else
|
||||||
|
perl -0pi -e "s/(<application\\b[^>]*)(>)/\$1\\n android:debuggable=\"true\"\$2/s" "$MANIFEST"
|
||||||
|
fi
|
||||||
|
if grep -q "tools:ignore=" "$MANIFEST"; then
|
||||||
|
sed -i "s/tools:ignore=\"[^\"]*\"/tools:ignore=\"HardcodedDebugMode\"/" "$MANIFEST"
|
||||||
|
else
|
||||||
|
perl -0pi -e "s/(<application\\b[^>]*)(>)/\$1\\n tools:ignore=\"HardcodedDebugMode\"\$2/s" "$MANIFEST"
|
||||||
|
fi
|
||||||
|
grep -n "debuggable\\|<application" "$MANIFEST"
|
||||||
|
else
|
||||||
|
echo ">>> WARNING: AndroidManifest.xml not found; release APK will not be debuggable"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
# ─── Arch list from BUILD_ARCH env var ───────────────────────────────────
|
# ─── Arch list from BUILD_ARCH env var ───────────────────────────────────
|
||||||
case "${BUILD_ARCH}" in
|
case "${BUILD_ARCH}" in
|
||||||
arm64) ARCHS="arm64" ;;
|
arm64) ARCHS="arm64" ;;
|
||||||
@@ -302,6 +330,7 @@ done
|
|||||||
|
|
||||||
APK_OUTPUT_DIR="/build/source/target/apk-output"
|
APK_OUTPUT_DIR="/build/source/target/apk-output"
|
||||||
mkdir -p "$APK_OUTPUT_DIR"
|
mkdir -p "$APK_OUTPUT_DIR"
|
||||||
|
rm -f "$APK_OUTPUT_DIR"/wzp-tauri-*.apk
|
||||||
|
|
||||||
for ARCH in $ARCHS; do
|
for ARCH in $ARCHS; do
|
||||||
TARGET=$(tauri_target "$ARCH")
|
TARGET=$(tauri_target "$ARCH")
|
||||||
@@ -321,8 +350,35 @@ for ARCH in $ARCHS; do
|
|||||||
echo ">>> cargo tauri android build ${PROFILE_FLAG} --target $TARGET --apk"
|
echo ">>> cargo tauri android build ${PROFILE_FLAG} --target $TARGET --apk"
|
||||||
cargo tauri android build ${PROFILE_FLAG} --target "$TARGET" --apk
|
cargo tauri android build ${PROFILE_FLAG} --target "$TARGET" --apk
|
||||||
|
|
||||||
|
# ─── Workaround: Tauri CLI 2.10.x does not copy frontendDist to the
|
||||||
|
# Android assets folder. The Rust build step writes tauri.conf.json
|
||||||
|
# there correctly, but index.html and the JS/CSS assets are never
|
||||||
|
# transferred, causing the WebView to fail with "Asset not found:
|
||||||
|
# index.html" at runtime.
|
||||||
|
#
|
||||||
|
# Fix: inject the missing files directly into the unsigned APK (which
|
||||||
|
# is just a ZIP file). The existing zipalign + apksigner step below
|
||||||
|
# handles realignment and signing, so this produces a valid APK.
|
||||||
|
# Re-running Gradle is NOT used here because the Gradle Rust build
|
||||||
|
# task (BuildTask.kt) calls `cargo tauri android android-studio-script`
|
||||||
|
# which requires the full Tauri CLI environment and fails standalone.
|
||||||
|
BUILD_VARIANT="debug"
|
||||||
|
[ -z "${PROFILE_FLAG}" ] && BUILD_VARIANT="release"
|
||||||
|
UNSIGNED_APK_PATH="gen/android/app/build/outputs/apk/universal/${BUILD_VARIANT}/app-universal-${BUILD_VARIANT}-unsigned.apk"
|
||||||
|
if [ -f "$UNSIGNED_APK_PATH" ] && ! unzip -l "$UNSIGNED_APK_PATH" 2>/dev/null | grep -q "assets/index.html"; then
|
||||||
|
echo ">>> frontend assets missing from APK — patching unsigned APK directly"
|
||||||
|
PATCH_DIR="/tmp/apk-frontend-patch-$$"
|
||||||
|
rm -rf "$PATCH_DIR"
|
||||||
|
mkdir -p "$PATCH_DIR/assets"
|
||||||
|
cp -r /build/source/desktop/dist/. "$PATCH_DIR/assets/"
|
||||||
|
(cd "$PATCH_DIR" && zip -r /build/source/desktop/src-tauri/"$UNSIGNED_APK_PATH" assets/)
|
||||||
|
rm -rf "$PATCH_DIR"
|
||||||
|
echo ">>> APK patched: $(ls -lh "$UNSIGNED_APK_PATH" | awk "{print \$5}")"
|
||||||
|
echo ">>> assets in APK: $(unzip -l "$UNSIGNED_APK_PATH" | grep "assets/" | wc -l) entries"
|
||||||
|
fi
|
||||||
|
|
||||||
# Copy produced APK with arch suffix
|
# Copy produced APK with arch suffix
|
||||||
BUILT_APK=$(find gen/android -name "*.apk" -newer "$APK_OUTPUT_DIR" -type f 2>/dev/null | head -1)
|
BUILT_APK=$(find "gen/android/app/build/outputs/apk" -path "*/${BUILD_VARIANT}/*.apk" -type f 2>/dev/null | sort | head -1)
|
||||||
if [ -z "$BUILT_APK" ]; then
|
if [ -z "$BUILT_APK" ]; then
|
||||||
BUILT_APK=$(find gen/android -name "*.apk" -type f 2>/dev/null | sort -t/ -k1 | tail -1)
|
BUILT_APK=$(find gen/android -name "*.apk" -type f 2>/dev/null | sort -t/ -k1 | tail -1)
|
||||||
fi
|
fi
|
||||||
@@ -334,6 +390,12 @@ for ARCH in $ARCHS; do
|
|||||||
# Release builds are unsigned by default. Sign with the release
|
# Release builds are unsigned by default. Sign with the release
|
||||||
# keystore (checked into the repo at android/keystore/) so the
|
# keystore (checked into the repo at android/keystore/) so the
|
||||||
# APK can be installed on real devices.
|
# APK can be installed on real devices.
|
||||||
|
if [ "${BUILD_VARIANT}" = "debug" ]; then
|
||||||
|
echo ">>> Debug APK selected; preserving Gradle debug signing and android:debuggable=true"
|
||||||
|
echo ">>> $ARCH APK: $(ls -lh "$OUT_APK" | awk "{print \$5}")"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
# Pick keystore + credentials (release preferred, debug fallback)
|
# Pick keystore + credentials (release preferred, debug fallback)
|
||||||
KS_RELEASE="/build/source/android/keystore/wzp-release.jks"
|
KS_RELEASE="/build/source/android/keystore/wzp-release.jks"
|
||||||
KS_DEBUG="/build/source/android/keystore/wzp-debug.jks"
|
KS_DEBUG="/build/source/android/keystore/wzp-debug.jks"
|
||||||
@@ -427,11 +489,11 @@ REMOTE_SCRIPT
|
|||||||
|
|
||||||
ssh_cmd "chmod +x /tmp/wzp-tauri-build.sh"
|
ssh_cmd "chmod +x /tmp/wzp-tauri-build.sh"
|
||||||
|
|
||||||
notify_local "WZP Tauri Android build dispatched (branch=$BRANCH, arch=$BUILD_ARCH, release=$BUILD_RELEASE)"
|
notify_local "WZP Tauri Android build dispatched (branch=$BRANCH, arch=$BUILD_ARCH, release=$BUILD_RELEASE, release-debuggable=$RELEASE_DEBUGGABLE)"
|
||||||
log "Triggering remote build (branch=$BRANCH, arch=$BUILD_ARCH)..."
|
log "Triggering remote build (branch=$BRANCH, arch=$BUILD_ARCH)..."
|
||||||
|
|
||||||
# Run; last lines are APK_REMOTE_PATH=... (one per arch)
|
# Run; last lines are APK_REMOTE_PATH=... (one per arch)
|
||||||
REMOTE_OUTPUT=$(ssh_cmd "/tmp/wzp-tauri-build.sh '$BRANCH' '$DO_PULL' '$REBUILD_RUST' '$DO_INIT' '$BUILD_RELEASE' '$BUILD_ARCH'" || true)
|
REMOTE_OUTPUT=$(ssh_cmd "/tmp/wzp-tauri-build.sh '$BRANCH' '$DO_PULL' '$REBUILD_RUST' '$DO_INIT' '$BUILD_RELEASE' '$BUILD_ARCH' '$RELEASE_DEBUGGABLE'" || true)
|
||||||
echo "$REMOTE_OUTPUT" | tail -60
|
echo "$REMOTE_OUTPUT" | tail -60
|
||||||
|
|
||||||
# Download all produced APKs
|
# Download all produced APKs
|
||||||
|
|||||||
25
scripts/pull-android-frame-dumps.sh
Executable file
25
scripts/pull-android-frame-dumps.sh
Executable file
@@ -0,0 +1,25 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
PACKAGE="${1:-com.wzp.desktop}"
|
||||||
|
OUT_DIR="${2:-android-frame-dumps}"
|
||||||
|
LOCAL_TAR="wzp-frame-dumps.tar"
|
||||||
|
APP_DUMP_DIR="${WZP_ANDROID_DUMP_ROOT:-.wzp}"
|
||||||
|
trap 'rm -f "$LOCAL_TAR"' EXIT
|
||||||
|
|
||||||
|
if [ "${1:-}" = "-h" ] || [ "${1:-}" = "--help" ]; then
|
||||||
|
echo "Usage: $0 [package] [out-dir]"
|
||||||
|
echo "Default package: com.wzp.desktop"
|
||||||
|
echo "Default out-dir: android-frame-dumps"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo ">>> Packaging frame dumps from $PACKAGE..."
|
||||||
|
adb exec-out "run-as $PACKAGE tar -C $APP_DUMP_DIR -cf - frame-dumps" > "$LOCAL_TAR"
|
||||||
|
|
||||||
|
rm -rf "$OUT_DIR"
|
||||||
|
mkdir -p "$OUT_DIR"
|
||||||
|
tar -xf "$LOCAL_TAR" -C "$OUT_DIR"
|
||||||
|
|
||||||
|
echo ">>> Pulled dumps:"
|
||||||
|
find "$OUT_DIR" -type f | sort | sed 's#^# #'
|
||||||
6
vault/.obsidian/app.json
vendored
Normal file
6
vault/.obsidian/app.json
vendored
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
{
|
||||||
|
"legacyEditor": false,
|
||||||
|
"livePreview": true,
|
||||||
|
"defaultViewMode": "source",
|
||||||
|
"promptDelete": false
|
||||||
|
}
|
||||||
1
vault/.obsidian/workspace.json
vendored
Normal file
1
vault/.obsidian/workspace.json
vendored
Normal file
@@ -0,0 +1 @@
|
|||||||
|
{}
|
||||||
128
vault/00 - Home.md
Normal file
128
vault/00 - Home.md
Normal file
@@ -0,0 +1,128 @@
|
|||||||
|
---
|
||||||
|
tags: [home, wzp]
|
||||||
|
type: index
|
||||||
|
---
|
||||||
|
|
||||||
|
# WarzonePhone Vault
|
||||||
|
|
||||||
|
WarzonePhone (WZP) is a custom lossy VoIP protocol and application stack built in Rust. It features a 7-crate workspace, Opus + Codec2 audio codecs, RaptorQ FEC, QUIC transport, and a Tauri-based Android client. The project spans relay infrastructure, P2P direct calling, AV1 video, and federated relay gossip.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
- [[Architecture/Architecture|Architecture Overview]]
|
||||||
|
- [[Architecture/WZP-Spec|WZP Protocol Spec]]
|
||||||
|
- [[Architecture/Protocol-Audit|Protocol Audit]]
|
||||||
|
- [[Architecture/Design|Design Doc]]
|
||||||
|
- [[Architecture/WS-Relay-Spec|WebSocket Relay Spec]]
|
||||||
|
- [[Architecture/Extensibility|Extensibility]]
|
||||||
|
- [[Architecture/Road-To-Video|Road to Video]]
|
||||||
|
- [[Architecture/Attack-Surface-Relay-Abuse|Attack Surface: Relay Abuse]]
|
||||||
|
- [[Architecture/Refactor-Codebase-Audit|Refactor: Codebase Audit]]
|
||||||
|
- [[Architecture/Refactor-Relay-Concurrency|Refactor: Relay Concurrency]]
|
||||||
|
- [[Architecture/Branch-Desktop-Audio-Rewrite|Branch: Desktop Audio Rewrite]]
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Active Work
|
||||||
|
|
||||||
|
- [[Reference/Handoff-2026-05-12|Handoff 2026-05-12]] — current state handoff doc
|
||||||
|
- [[PRDs/TASKS|TASKS — Status Board]]
|
||||||
|
- [[Audit/Audit-2026-05-25|Audit 2026-05-25]]
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## PRDs
|
||||||
|
|
||||||
|
### Audio & Codec
|
||||||
|
- [[PRDs/PRD-adaptive-quality|Adaptive Quality]]
|
||||||
|
- [[PRDs/PRD-bluetooth-audio|Bluetooth Audio]]
|
||||||
|
- [[PRDs/PRD-coordinated-codec|Coordinated Codec]]
|
||||||
|
- [[PRDs/PRD-dred-integration|DRED Integration]]
|
||||||
|
- [[PRDs/PRD-studio-quality|Studio Quality]]
|
||||||
|
|
||||||
|
### Networking & P2P
|
||||||
|
- [[PRDs/PRD-p2p-direct|P2P Direct Calling]]
|
||||||
|
- [[PRDs/PRD-hard-nat|Hard NAT Traversal]]
|
||||||
|
- [[PRDs/PRD-ice-regather|ICE Regather]]
|
||||||
|
- [[PRDs/PRD-mtu-discovery|MTU Discovery]]
|
||||||
|
- [[PRDs/PRD-netcheck|Network Check]]
|
||||||
|
- [[PRDs/PRD-network-awareness|Network Awareness]]
|
||||||
|
- [[PRDs/PRD-portmap|Port Mapping]]
|
||||||
|
- [[PRDs/PRD-public-stun|Public STUN]]
|
||||||
|
- [[PRDs/PRD-transport-feedback-bwe|Transport Feedback BWE]]
|
||||||
|
|
||||||
|
### Relay
|
||||||
|
- [[PRDs/PRD-relay-concurrency|Relay Concurrency]]
|
||||||
|
- [[PRDs/PRD-relay-conformance|Relay Conformance]]
|
||||||
|
- [[PRDs/PRD-relay-federation|Relay Federation]]
|
||||||
|
- [[PRDs/PRD-relay-federation-gossip|Relay Federation Gossip]]
|
||||||
|
- [[PRDs/PRD-relay-selection|Relay Selection]]
|
||||||
|
|
||||||
|
### Video
|
||||||
|
- [[PRDs/PRD-video-v1|Video V1]]
|
||||||
|
- [[PRDs/PRD-video-multicodec|Video Multicodec]]
|
||||||
|
- [[PRDs/PRD-video-quality-priority|Video Quality Priority]]
|
||||||
|
- [[PRDs/PRD-video-simulcast|Video Simulcast]]
|
||||||
|
|
||||||
|
### Protocol & Security
|
||||||
|
- [[PRDs/PRD-protocol-hardening|Protocol Hardening]]
|
||||||
|
- [[PRDs/PRD-protocol-analyzer|Protocol Analyzer]]
|
||||||
|
- [[PRDs/PRD-wire-format-v2|Wire Format V2]]
|
||||||
|
- [[PRDs/PRD-delegated-trust|Delegated Trust]]
|
||||||
|
|
||||||
|
### Other
|
||||||
|
- [[PRDs/PRD-engine-dedup|Engine Dedup]]
|
||||||
|
- [[PRDs/PRD-local-recording|Local Recording]]
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Android
|
||||||
|
|
||||||
|
- [[Android/Architecture|Android Architecture]]
|
||||||
|
- [[Android/Build-Guide|Build Guide]]
|
||||||
|
- [[Android/Roadmap|Android Roadmap]]
|
||||||
|
- [[Android/Debugging|Debugging]]
|
||||||
|
- [[Android/Maintenance|Maintenance]]
|
||||||
|
- [[Android/Fix-Audio-Ring-Desync|Fix: Audio Ring Desync]]
|
||||||
|
- [[Android/Fix-Capture-Thread-Crash|Fix: Capture Thread Crash]]
|
||||||
|
- [[Android/README|Android README]]
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Reference
|
||||||
|
|
||||||
|
- [[Reference/API|API Reference]]
|
||||||
|
- [[Reference/Usage|Usage]]
|
||||||
|
- [[Reference/User-Guide|User Guide]]
|
||||||
|
- [[Reference/Administration|Administration]]
|
||||||
|
- [[Reference/Telemetry|Telemetry]]
|
||||||
|
- [[Reference/Progress|Progress]]
|
||||||
|
- [[Reference/Featherchat-Integration|FeatherChat Integration]]
|
||||||
|
- [[Reference/Featherchat|FeatherChat]]
|
||||||
|
- [[Reference/WZP-FC-Shared-Crates|WZP-FC Shared Crates]]
|
||||||
|
- [[Reference/Integration-Tasks|Integration Tasks]]
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Reports
|
||||||
|
|
||||||
|
### Approved
|
||||||
|
- [[Reports/T1.1-report|T1.1]] · [[Reports/T1.1.1-report|T1.1.1]] · [[Reports/T1.1.2-report|T1.1.2]]
|
||||||
|
- [[Reports/T1.2-report|T1.2]] · [[Reports/T1.2.1-report|T1.2.1]]
|
||||||
|
- [[Reports/T1.3-report|T1.3]] · [[Reports/T1.4-report|T1.4]] · [[Reports/T1.4.1-report|T1.4.1]]
|
||||||
|
- [[Reports/T1.5-report|T1.5]] · [[Reports/T1.5.1-report|T1.5.1]] · [[Reports/T1.5.2-report|T1.5.2]]
|
||||||
|
- [[Reports/T1.6-report|T1.6]] · [[Reports/T1.7-report|T1.7]] · [[Reports/T1.8-report|T1.8]]
|
||||||
|
- [[Reports/T2.1-report|T2.1]] · [[Reports/T2.2-report|T2.2]]
|
||||||
|
- [[Reports/T4.2-report|T4.2]] · [[Reports/T4.2.1-report|T4.2.1]] · [[Reports/T4.3-report|T4.3]] · [[Reports/T4.3.1-report|T4.3.1]]
|
||||||
|
- [[Reports/T4.4-report|T4.4]] · [[Reports/T4.5-report|T4.5]] · [[Reports/T4.6-report|T4.6]] · [[Reports/T4.7-report|T4.7]]
|
||||||
|
- [[Reports/T5.1-report|T5.1]] · [[Reports/T5.2-report|T5.2]] · [[Reports/T5.3-report|T5.3]]
|
||||||
|
|
||||||
|
### Pending Review
|
||||||
|
- [[Reports/T2.3-report|T2.3]] · [[Reports/T2.4-report|T2.4]] · [[Reports/T2.5-report|T2.5]] · [[Reports/T2.6-report|T2.6]]
|
||||||
|
- [[Reports/T3.1-report|T3.1]] · [[Reports/T3.2-report|T3.2]] · [[Reports/T3.3-report|T3.3]] · [[Reports/T3.4-report|T3.4]] · [[Reports/T3.5-report|T3.5]]
|
||||||
|
- [[Reports/T4.1-report|T4.1]]
|
||||||
|
- [[Reports/T5.1.1-report|T5.1.1]] · [[Reports/T5.4-report|T5.4]] · [[Reports/T5.5-report|T5.5]] · [[Reports/T5.6-report|T5.6]]
|
||||||
|
- [[Reports/T5.7-report|T5.7]] · [[Reports/T5.7.1-report|T5.7.1]] · [[Reports/T5.8-report|T5.8]]
|
||||||
|
- [[Reports/T6.1-report|T6.1]] · [[Reports/T6.1.2-report|T6.1.2]] · [[Reports/T6.2-report|T6.2]]
|
||||||
405
vault/Android/Architecture.md
Normal file
405
vault/Android/Architecture.md
Normal file
@@ -0,0 +1,405 @@
|
|||||||
|
---
|
||||||
|
tags: [android, wzp]
|
||||||
|
type: reference
|
||||||
|
---
|
||||||
|
|
||||||
|
# Architecture
|
||||||
|
|
||||||
|
## System Overview
|
||||||
|
|
||||||
|
The Android client is a four-layer stack: Kotlin UI, JNI bridge, Rust engine, and C++ audio I/O. Each layer communicates through well-defined interfaces with minimal coupling.
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
graph TB
|
||||||
|
subgraph "Kotlin (Main Thread)"
|
||||||
|
CA[CallActivity]
|
||||||
|
VM[CallViewModel]
|
||||||
|
UI[InCallScreen<br/>Compose UI]
|
||||||
|
CA --> VM
|
||||||
|
VM --> UI
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph "JNI Bridge"
|
||||||
|
JB[jni_bridge.rs<br/>panic-safe FFI]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph "Rust Engine"
|
||||||
|
ENG[WzpEngine<br/>Orchestrator]
|
||||||
|
CT[Codec Thread<br/>20ms real-time loop]
|
||||||
|
NET[Tokio Runtime<br/>2 async workers]
|
||||||
|
PIPE[Pipeline<br/>Encode/Decode/FEC/Jitter]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph "C++ Audio"
|
||||||
|
OBOE[Oboe Bridge<br/>Capture + Playout callbacks]
|
||||||
|
RB[Ring Buffers<br/>Lock-free SPSC]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph "Network"
|
||||||
|
QUIC[QUIC Connection<br/>quinn]
|
||||||
|
RELAY[WZP Relay<br/>SFU Room]
|
||||||
|
end
|
||||||
|
|
||||||
|
VM <-->|"JNI calls<br/>+ JSON stats"| JB
|
||||||
|
JB <--> ENG
|
||||||
|
ENG --> CT
|
||||||
|
ENG --> NET
|
||||||
|
CT <--> PIPE
|
||||||
|
CT <-->|"Atomic R/W"| RB
|
||||||
|
OBOE <-->|"Atomic R/W"| RB
|
||||||
|
CT <-->|"mpsc channels"| NET
|
||||||
|
NET <-->|"QUIC datagrams<br/>+ streams"| QUIC
|
||||||
|
QUIC <--> RELAY
|
||||||
|
```
|
||||||
|
|
||||||
|
## Thread Model
|
||||||
|
|
||||||
|
The engine uses four distinct thread contexts, each with specific responsibilities and real-time constraints.
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
graph LR
|
||||||
|
subgraph "Android Main Thread"
|
||||||
|
UI_T["UI + JNI calls<br/>startCall / stopCall / getStats"]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph "Oboe Audio Thread (system)"
|
||||||
|
AUD["Capture callback: mic → ring buf<br/>Playout callback: ring buf → speaker<br/>⚡ Highest priority, no allocations"]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph "Codec Thread (wzp-codec)"
|
||||||
|
COD["20ms loop:<br/>1. Read capture ring buf<br/>2. AEC → AGC → Encode<br/>3. Send to network channel<br/>4. Recv from network channel<br/>5. FEC → Jitter → Decode<br/>6. Write playout ring buf<br/>⚡ Pinned to big core, RT priority"]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph "Tokio Runtime (2 workers)"
|
||||||
|
NET_S["Send task:<br/>Channel → MediaPacket → QUIC datagram"]
|
||||||
|
NET_R["Recv task:<br/>QUIC datagram → MediaPacket → Channel"]
|
||||||
|
HS["Handshake:<br/>CallOffer → CallAnswer"]
|
||||||
|
end
|
||||||
|
|
||||||
|
UI_T -->|"mpsc command channel"| COD
|
||||||
|
COD -->|"tokio::mpsc send_tx"| NET_S
|
||||||
|
NET_R -->|"tokio::mpsc recv_tx"| COD
|
||||||
|
AUD <-->|"Atomic ring buffers"| COD
|
||||||
|
```
|
||||||
|
|
||||||
|
### Thread Priorities and Constraints
|
||||||
|
|
||||||
|
| Thread | Priority | Allocations | Blocking | Lock-free |
|
||||||
|
|--------|----------|-------------|----------|-----------|
|
||||||
|
| Oboe audio | SCHED_FIFO (system) | None | Never | Yes |
|
||||||
|
| Codec | RT priority, big core | Pre-allocated buffers | sleep(remainder of 20ms) | Ring buf: yes, Stats: Mutex |
|
||||||
|
| Tokio workers | Normal | Allowed | Async only | N/A |
|
||||||
|
| Main/JNI | Normal | Allowed | Allowed | N/A |
|
||||||
|
|
||||||
|
## Call Lifecycle
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
sequenceDiagram
|
||||||
|
participant User
|
||||||
|
participant UI as InCallScreen
|
||||||
|
participant VM as CallViewModel
|
||||||
|
participant ENG as WzpEngine (JNI)
|
||||||
|
participant NET as Tokio Network
|
||||||
|
participant RELAY as WZP Relay
|
||||||
|
|
||||||
|
User->>UI: Tap CALL
|
||||||
|
UI->>VM: startCall()
|
||||||
|
VM->>ENG: init() + startCall(relay, room)
|
||||||
|
ENG->>ENG: Create tokio runtime
|
||||||
|
ENG->>NET: Spawn network task
|
||||||
|
|
||||||
|
NET->>RELAY: QUIC connect (SNI = room name)
|
||||||
|
RELAY-->>NET: Connection established
|
||||||
|
|
||||||
|
Note over NET,RELAY: Crypto Handshake
|
||||||
|
NET->>RELAY: CallOffer {identity_pub, ephemeral_pub, signature, profiles}
|
||||||
|
RELAY-->>NET: CallAnswer {ephemeral_pub, chosen_profile, signature}
|
||||||
|
NET->>NET: Derive ChaCha20-Poly1305 session
|
||||||
|
|
||||||
|
ENG->>ENG: Spawn codec thread
|
||||||
|
Note over ENG: State → Active
|
||||||
|
|
||||||
|
loop Every 20ms
|
||||||
|
ENG->>ENG: Read mic → AEC → AGC → Encode
|
||||||
|
ENG->>NET: Encoded frame via channel
|
||||||
|
NET->>RELAY: MediaPacket via QUIC DATAGRAM
|
||||||
|
RELAY->>NET: MediaPacket from other peer
|
||||||
|
NET->>ENG: MediaPacket via channel
|
||||||
|
ENG->>ENG: FEC → Jitter → Decode → Speaker
|
||||||
|
end
|
||||||
|
|
||||||
|
User->>UI: Tap END
|
||||||
|
UI->>VM: stopCall()
|
||||||
|
VM->>ENG: stopCall()
|
||||||
|
ENG->>ENG: Set running=false, send Stop command
|
||||||
|
ENG->>ENG: Join codec thread
|
||||||
|
ENG->>NET: Drop tokio runtime
|
||||||
|
NET->>RELAY: Connection close
|
||||||
|
```
|
||||||
|
|
||||||
|
## Audio Pipeline Detail
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
graph LR
|
||||||
|
subgraph "Capture Path"
|
||||||
|
MIC[Microphone] -->|"48kHz i16"| OBOE_C[Oboe Capture<br/>Callback]
|
||||||
|
OBOE_C -->|"ring_write()"| RB_C[Capture<br/>Ring Buffer]
|
||||||
|
RB_C -->|"read_capture()"| AEC[Echo<br/>Canceller]
|
||||||
|
AEC --> AGC[Auto Gain<br/>Control]
|
||||||
|
AGC --> ENC[AdaptiveEncoder<br/>Opus 24k]
|
||||||
|
ENC -->|"Vec u8"| FEC_E[RaptorQ<br/>FEC Encoder]
|
||||||
|
FEC_E -->|"send_tx"| CHAN_S[Send Channel]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph "Network"
|
||||||
|
CHAN_S --> PKT_S[MediaPacket<br/>Header + Payload]
|
||||||
|
PKT_S -->|"QUIC DATAGRAM"| RELAY[Relay SFU]
|
||||||
|
RELAY -->|"QUIC DATAGRAM"| PKT_R[MediaPacket<br/>Deserialize]
|
||||||
|
PKT_R -->|"recv_tx"| CHAN_R[Recv Channel]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph "Playout Path"
|
||||||
|
CHAN_R --> FEC_D[RaptorQ<br/>FEC Decoder]
|
||||||
|
FEC_D --> JB[Jitter Buffer<br/>10-250 pkts]
|
||||||
|
JB --> DEC[AdaptiveDecoder<br/>Opus 24k]
|
||||||
|
DEC -->|"48kHz i16"| AEC_REF[AEC Far-End<br/>Reference]
|
||||||
|
DEC -->|"write_playout()"| RB_P[Playout<br/>Ring Buffer]
|
||||||
|
RB_P -->|"ring_read()"| OBOE_P[Oboe Playout<br/>Callback]
|
||||||
|
OBOE_P --> SPK[Speaker]
|
||||||
|
end
|
||||||
|
```
|
||||||
|
|
||||||
|
### Audio Parameters
|
||||||
|
|
||||||
|
| Parameter | Value | Notes |
|
||||||
|
|-----------|-------|-------|
|
||||||
|
| Sample rate | 48,000 Hz | Opus native rate |
|
||||||
|
| Channels | 1 (mono) | VoIP only |
|
||||||
|
| Frame size | 960 samples | 20ms at 48kHz |
|
||||||
|
| Ring buffer | 7,680 samples | 160ms (8 frames) |
|
||||||
|
| Bit depth | 16-bit signed int | PCM format |
|
||||||
|
| AEC tail | 100ms | Echo canceller filter length |
|
||||||
|
|
||||||
|
## Crypto Handshake
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
sequenceDiagram
|
||||||
|
participant Client as Android Client
|
||||||
|
participant Relay as WZP Relay
|
||||||
|
|
||||||
|
Note over Client: Identity seed (32 bytes, random per launch)
|
||||||
|
Note over Client: HKDF → Ed25519 signing key + X25519 static key
|
||||||
|
|
||||||
|
Client->>Client: Generate ephemeral X25519 keypair
|
||||||
|
Client->>Client: Sign(ephemeral_pub || "call-offer") with Ed25519
|
||||||
|
|
||||||
|
Client->>Relay: SignalMessage::CallOffer<br/>{identity_pub, ephemeral_pub, signature, [GOOD, DEGRADED, CATASTROPHIC]}
|
||||||
|
|
||||||
|
Relay->>Relay: Verify Ed25519 signature
|
||||||
|
Relay->>Relay: Generate own ephemeral X25519
|
||||||
|
Relay->>Relay: Sign(ephemeral_pub || "call-answer")
|
||||||
|
Relay->>Relay: DH(relay_ephemeral, client_ephemeral) → shared secret
|
||||||
|
Relay->>Relay: HKDF(shared_secret) → ChaCha20-Poly1305 key
|
||||||
|
|
||||||
|
Relay->>Client: SignalMessage::CallAnswer<br/>{identity_pub, ephemeral_pub, signature, chosen_profile=GOOD}
|
||||||
|
|
||||||
|
Client->>Client: Verify relay signature
|
||||||
|
Client->>Client: DH(client_ephemeral, relay_ephemeral) → same shared secret
|
||||||
|
Client->>Client: HKDF(shared_secret) → same ChaCha20-Poly1305 key
|
||||||
|
|
||||||
|
Note over Client,Relay: Both sides now have identical session key
|
||||||
|
Note over Client,Relay: Media packets can be encrypted (not yet applied)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Key Derivation Chain
|
||||||
|
|
||||||
|
```
|
||||||
|
Identity Seed (32 bytes, random)
|
||||||
|
│
|
||||||
|
├── HKDF(seed, info="warzone-ed25519") → Ed25519 signing key
|
||||||
|
│ └── Public key = identity_pub (32 bytes)
|
||||||
|
│ └── SHA-256(identity_pub)[:16] = fingerprint (16 bytes)
|
||||||
|
│
|
||||||
|
└── HKDF(seed, info="warzone-x25519") → X25519 static key (unused currently)
|
||||||
|
|
||||||
|
Per-Call Ephemeral:
|
||||||
|
Random X25519 keypair → ephemeral_pub (sent in CallOffer)
|
||||||
|
|
||||||
|
Session Key:
|
||||||
|
DH(our_ephemeral_secret, peer_ephemeral_pub) → shared_secret
|
||||||
|
HKDF(shared_secret, info="warzone-session-key") → ChaCha20-Poly1305 key (32 bytes)
|
||||||
|
```
|
||||||
|
|
||||||
|
## QUIC Transport
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
graph TB
|
||||||
|
subgraph "QUIC Connection"
|
||||||
|
EP[Client Endpoint<br/>0.0.0.0:0 UDP]
|
||||||
|
CONN[Connection to Relay<br/>SNI = room name]
|
||||||
|
|
||||||
|
subgraph "Unreliable Channel"
|
||||||
|
DG_S[Send DATAGRAM<br/>MediaPacket serialized]
|
||||||
|
DG_R[Recv DATAGRAM<br/>MediaPacket deserialized]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph "Reliable Channel"
|
||||||
|
ST_S[Open bidi stream<br/>JSON length-prefixed<br/>SignalMessage]
|
||||||
|
ST_R[Accept bidi stream<br/>JSON length-prefixed<br/>SignalMessage]
|
||||||
|
end
|
||||||
|
|
||||||
|
EP --> CONN
|
||||||
|
CONN --> DG_S
|
||||||
|
CONN --> DG_R
|
||||||
|
CONN --> ST_S
|
||||||
|
CONN --> ST_R
|
||||||
|
end
|
||||||
|
```
|
||||||
|
|
||||||
|
### QUIC Configuration (VoIP-tuned)
|
||||||
|
|
||||||
|
| Setting | Value | Rationale |
|
||||||
|
|---------|-------|-----------|
|
||||||
|
| ALPN | `wzp` | Protocol identification |
|
||||||
|
| Idle timeout | 30s | Keep connection alive during silence |
|
||||||
|
| Keep-alive | 5s | Prevent NAT timeout |
|
||||||
|
| Datagram receive buffer | 65 KB | Buffer for burst arrivals |
|
||||||
|
| Flow control (recv) | 256 KB | Conservative for VoIP |
|
||||||
|
| Flow control (send) | 128 KB | Prevent bufferbloat |
|
||||||
|
| TLS | Self-signed certs | Development mode |
|
||||||
|
| Certificate verification | Disabled | Client accepts any cert |
|
||||||
|
|
||||||
|
## MediaPacket Wire Format
|
||||||
|
|
||||||
|
```
|
||||||
|
12-byte header:
|
||||||
|
┌─────────────────────────────────────────────────┐
|
||||||
|
│ Byte 0: V(1) T(1) CodecID(4) Q(1) FecHi(1) │
|
||||||
|
│ Byte 1: FecLo(6) unused(2) │
|
||||||
|
│ Byte 2-3: Sequence number (u16 BE) │
|
||||||
|
│ Byte 4-7: Timestamp ms (u32 BE) │
|
||||||
|
│ Byte 8: FEC block ID │
|
||||||
|
│ Byte 9: FEC symbol index │
|
||||||
|
│ Byte 10: Reserved │
|
||||||
|
│ Byte 11: CSRC count │
|
||||||
|
├─────────────────────────────────────────────────┤
|
||||||
|
│ Payload: Opus-encoded audio frame │
|
||||||
|
├─────────────────────────────────────────────────┤
|
||||||
|
│ Optional: QualityReport (4 bytes, if Q=1) │
|
||||||
|
│ loss_pct(u8) rtt_4ms(u8) jitter_ms(u8) │
|
||||||
|
│ bitrate_cap_kbps(u8) │
|
||||||
|
└─────────────────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
## Relay Room Mode (SFU)
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
graph LR
|
||||||
|
subgraph "Room: android"
|
||||||
|
P1[Phone A<br/>QUIC conn] -->|MediaPacket| RELAY[Relay SFU]
|
||||||
|
RELAY -->|MediaPacket| P2[Phone B<br/>QUIC conn]
|
||||||
|
P2 -->|MediaPacket| RELAY
|
||||||
|
RELAY -->|MediaPacket| P1
|
||||||
|
end
|
||||||
|
|
||||||
|
Note1["Room name from QUIC TLS SNI<br/>No auth required<br/>Packets forwarded to all others"]
|
||||||
|
```
|
||||||
|
|
||||||
|
The relay operates as a Selective Forwarding Unit:
|
||||||
|
1. Client connects via QUIC, room name extracted from TLS SNI
|
||||||
|
2. Crypto handshake completes (relay has its own ephemeral identity)
|
||||||
|
3. Client joins named room
|
||||||
|
4. All received media packets are forwarded to every other participant in the room
|
||||||
|
5. Signaling messages are not forwarded (point-to-point with relay)
|
||||||
|
|
||||||
|
## Adaptive Quality System
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
graph TD
|
||||||
|
QR[QualityReport<br/>loss%, RTT, jitter] --> AQC[AdaptiveQualityController]
|
||||||
|
|
||||||
|
AQC -->|"loss<10%, RTT<400ms"| GOOD[GOOD<br/>Opus 24kbps<br/>FEC 20%<br/>20ms frames]
|
||||||
|
AQC -->|"loss 10-40%<br/>RTT 400-600ms"| DEG[DEGRADED<br/>Opus 6kbps<br/>FEC 50%<br/>40ms frames]
|
||||||
|
AQC -->|"loss>40%<br/>RTT>600ms"| CAT[CATASTROPHIC<br/>Codec2 1.2kbps<br/>FEC 100%<br/>40ms frames]
|
||||||
|
|
||||||
|
GOOD -->|"Hysteresis:<br/>sustained degradation"| DEG
|
||||||
|
DEG -->|"Sustained improvement"| GOOD
|
||||||
|
DEG -->|"Further degradation"| CAT
|
||||||
|
CAT -->|"Improvement"| DEG
|
||||||
|
```
|
||||||
|
|
||||||
|
| Profile | Codec | Bitrate | FEC Ratio | Frame Size | FEC Block |
|
||||||
|
|---------|-------|---------|-----------|------------|-----------|
|
||||||
|
| GOOD | Opus 24k | 24 kbps | 20% | 20ms | 5 frames |
|
||||||
|
| DEGRADED | Opus 6k | 6 kbps | 50% | 40ms | 10 frames |
|
||||||
|
| CATASTROPHIC | Codec2 1.2k | 1.2 kbps | 100% | 40ms | 8 frames |
|
||||||
|
|
||||||
|
## Module Dependency Graph
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
graph BT
|
||||||
|
PROTO[wzp-proto<br/>Types, traits, jitter,<br/>quality, session]
|
||||||
|
CODEC[wzp-codec<br/>Opus, Codec2, AEC,<br/>AGC, resampling]
|
||||||
|
FEC[wzp-fec<br/>RaptorQ fountain codes]
|
||||||
|
CRYPTO[wzp-crypto<br/>Ed25519, X25519,<br/>ChaCha20-Poly1305]
|
||||||
|
TRANSPORT[wzp-transport<br/>QUIC, datagrams,<br/>signaling streams]
|
||||||
|
ANDROID[wzp-android<br/>Engine, JNI bridge,<br/>Oboe audio, pipeline]
|
||||||
|
RELAY[wzp-relay<br/>SFU, rooms, auth,<br/>metrics, probes]
|
||||||
|
|
||||||
|
CODEC --> PROTO
|
||||||
|
FEC --> PROTO
|
||||||
|
CRYPTO --> PROTO
|
||||||
|
TRANSPORT --> PROTO
|
||||||
|
ANDROID --> PROTO
|
||||||
|
ANDROID --> CODEC
|
||||||
|
ANDROID --> FEC
|
||||||
|
ANDROID --> CRYPTO
|
||||||
|
ANDROID --> TRANSPORT
|
||||||
|
RELAY --> PROTO
|
||||||
|
RELAY --> CRYPTO
|
||||||
|
RELAY --> TRANSPORT
|
||||||
|
```
|
||||||
|
|
||||||
|
## File Map
|
||||||
|
|
||||||
|
### Kotlin (`android/app/src/main/java/com/wzp/`)
|
||||||
|
|
||||||
|
| File | Purpose |
|
||||||
|
|------|---------|
|
||||||
|
| `WzpApplication.kt` | App entry, notification channel creation |
|
||||||
|
| `engine/WzpEngine.kt` | JNI wrapper for native engine |
|
||||||
|
| `engine/WzpCallback.kt` | Callback interface for engine events |
|
||||||
|
| `engine/CallStats.kt` | Stats data class with JSON deserialization |
|
||||||
|
| `ui/call/CallActivity.kt` | Activity host, permissions, theme |
|
||||||
|
| `ui/call/CallViewModel.kt` | MVVM state holder, stats polling |
|
||||||
|
| `ui/call/InCallScreen.kt` | Compose UI (idle + in-call states) |
|
||||||
|
| `service/CallService.kt` | Foreground service, wake/wifi locks |
|
||||||
|
| `audio/AudioRouteManager.kt` | Speaker/earpiece/Bluetooth routing |
|
||||||
|
|
||||||
|
### Rust (`crates/wzp-android/src/`)
|
||||||
|
|
||||||
|
| File | Purpose |
|
||||||
|
|------|---------|
|
||||||
|
| `lib.rs` | Module declarations |
|
||||||
|
| `jni_bridge.rs` | JNI FFI (panic-safe, proper jni crate) |
|
||||||
|
| `engine.rs` | Call orchestrator (threads, channels, lifecycle) |
|
||||||
|
| `pipeline.rs` | Codec pipeline (AEC, AGC, encode, FEC, jitter, decode) |
|
||||||
|
| `audio_android.rs` | Oboe backend, SPSC ring buffers, RT scheduling |
|
||||||
|
| `commands.rs` | Engine command enum |
|
||||||
|
| `stats.rs` | CallState/CallStats types (serde) |
|
||||||
|
|
||||||
|
### C++ (`crates/wzp-android/cpp/`)
|
||||||
|
|
||||||
|
| File | Purpose |
|
||||||
|
|------|---------|
|
||||||
|
| `oboe_bridge.h` | FFI header for Rust-C++ audio interface |
|
||||||
|
| `oboe_bridge.cpp` | Oboe capture/playout callbacks, ring buffer I/O |
|
||||||
|
| `oboe_stub.cpp` | No-op stub for non-Android builds |
|
||||||
|
|
||||||
|
### Build
|
||||||
|
|
||||||
|
| File | Purpose |
|
||||||
|
|------|---------|
|
||||||
|
| `android/app/build.gradle.kts` | Android build config, cargo-ndk task |
|
||||||
|
| `crates/wzp-android/Cargo.toml` | Rust dependencies (cdylib output) |
|
||||||
|
| `crates/wzp-android/build.rs` | C++ compilation, Oboe fetch |
|
||||||
160
vault/Android/Build-Guide.md
Normal file
160
vault/Android/Build-Guide.md
Normal file
@@ -0,0 +1,160 @@
|
|||||||
|
---
|
||||||
|
tags: [android, wzp]
|
||||||
|
type: reference
|
||||||
|
---
|
||||||
|
|
||||||
|
# Build Guide
|
||||||
|
|
||||||
|
## Prerequisites
|
||||||
|
|
||||||
|
| Tool | Version | Purpose |
|
||||||
|
|------|---------|---------|
|
||||||
|
| JDK | 17 | Android Gradle builds |
|
||||||
|
| Android SDK | 34 | Compile SDK |
|
||||||
|
| Android NDK | 26.1.10909125 | Native C++/Rust compilation |
|
||||||
|
| Rust | 1.85+ | Native engine (edition 2024) |
|
||||||
|
| cargo-ndk | latest | Cross-compile Rust → Android |
|
||||||
|
| `aarch64-linux-android` target | - | Rust target for ARM64 |
|
||||||
|
|
||||||
|
### Install Rust Android target
|
||||||
|
|
||||||
|
```bash
|
||||||
|
rustup target add aarch64-linux-android
|
||||||
|
cargo install cargo-ndk
|
||||||
|
```
|
||||||
|
|
||||||
|
### Environment Variables
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export JAVA_HOME="/usr/lib/jvm/java-17-openjdk-amd64"
|
||||||
|
export ANDROID_HOME="$HOME/android-sdk"
|
||||||
|
export ANDROID_NDK_HOME="$ANDROID_HOME/ndk/26.1.10909125"
|
||||||
|
|
||||||
|
# For manual cargo-ndk builds (Gradle sets these automatically):
|
||||||
|
export CC_aarch64_linux_android="$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android21-clang"
|
||||||
|
export CXX_aarch64_linux_android="$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android21-clang++"
|
||||||
|
export AR_aarch64_linux_android="$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar"
|
||||||
|
```
|
||||||
|
|
||||||
|
## Build Commands
|
||||||
|
|
||||||
|
### Full Build (Gradle drives everything)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd android
|
||||||
|
./gradlew assembleRelease
|
||||||
|
```
|
||||||
|
|
||||||
|
This runs:
|
||||||
|
1. `cargoNdkBuild` task: invokes `cargo ndk -t arm64-v8a -o app/src/main/jniLibs build --release -p wzp-android`
|
||||||
|
2. Compiles Kotlin/Compose code
|
||||||
|
3. Packages APK with signing
|
||||||
|
|
||||||
|
### Native Library Only
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo ndk -t arm64-v8a -o android/app/src/main/jniLibs build --release -p wzp-android
|
||||||
|
```
|
||||||
|
|
||||||
|
Output: `android/app/src/main/jniLibs/arm64-v8a/libwzp_android.so`
|
||||||
|
|
||||||
|
### Skip Native Rebuild
|
||||||
|
|
||||||
|
If the `.so` hasn't changed:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd android
|
||||||
|
./gradlew assembleRelease -x cargoNdkBuild
|
||||||
|
```
|
||||||
|
|
||||||
|
### Debug Build
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd android
|
||||||
|
./gradlew assembleDebug
|
||||||
|
```
|
||||||
|
|
||||||
|
Debug APK is ~8.9 MB (unstripped `.so`), release is ~6.9 MB.
|
||||||
|
|
||||||
|
## Signing
|
||||||
|
|
||||||
|
### Debug
|
||||||
|
|
||||||
|
```
|
||||||
|
Keystore: android/keystore/wzp-debug.jks
|
||||||
|
Password: android
|
||||||
|
Key alias: wzp-debug
|
||||||
|
```
|
||||||
|
|
||||||
|
### Release
|
||||||
|
|
||||||
|
```
|
||||||
|
Keystore: android/keystore/wzp-release.jks
|
||||||
|
Password: wzphone2024
|
||||||
|
Key alias: wzp-release
|
||||||
|
```
|
||||||
|
|
||||||
|
Both keystores are checked into the repo for development convenience. For production, replace with proper key management.
|
||||||
|
|
||||||
|
## Build Artifacts
|
||||||
|
|
||||||
|
| Artifact | Path | Size |
|
||||||
|
|----------|------|------|
|
||||||
|
| Debug APK | `android/app/build/outputs/apk/debug/app-debug.apk` | ~8.9 MB |
|
||||||
|
| Release APK | `android/app/build/outputs/apk/release/app-release.apk` | ~6.9 MB |
|
||||||
|
| Native lib | `android/app/src/main/jniLibs/arm64-v8a/libwzp_android.so` | ~5 MB |
|
||||||
|
|
||||||
|
## ABI Support
|
||||||
|
|
||||||
|
Currently only `arm64-v8a` (ARM64) is built. This covers 95%+ of modern Android devices.
|
||||||
|
|
||||||
|
To add more ABIs, edit `build.gradle.kts`:
|
||||||
|
|
||||||
|
```kotlin
|
||||||
|
ndk { abiFilters += listOf("arm64-v8a", "armeabi-v7a") }
|
||||||
|
```
|
||||||
|
|
||||||
|
And update the cargo-ndk command in `cargoNdkBuild` task:
|
||||||
|
|
||||||
|
```kotlin
|
||||||
|
commandLine("cargo", "ndk", "-t", "arm64-v8a", "-t", "armeabi-v7a", ...)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Oboe Dependency
|
||||||
|
|
||||||
|
The Oboe C++ audio library is fetched at build time by `build.rs`:
|
||||||
|
|
||||||
|
1. Attempts `git clone` of Oboe 1.8.1 into `$OUT_DIR/oboe`
|
||||||
|
2. If successful, compiles `oboe_bridge.cpp` with Oboe headers
|
||||||
|
3. If clone fails (no network), falls back to `oboe_stub.cpp` (no-op audio)
|
||||||
|
|
||||||
|
This means **first build requires internet** to fetch Oboe. Subsequent builds use the cached checkout.
|
||||||
|
|
||||||
|
## Common Build Issues
|
||||||
|
|
||||||
|
### `cargo ndk` not found
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo install cargo-ndk
|
||||||
|
```
|
||||||
|
|
||||||
|
### Missing Android target
|
||||||
|
|
||||||
|
```bash
|
||||||
|
rustup target add aarch64-linux-android
|
||||||
|
```
|
||||||
|
|
||||||
|
### NDK not found
|
||||||
|
|
||||||
|
Ensure `ANDROID_NDK_HOME` points to the NDK directory containing `toolchains/llvm/`.
|
||||||
|
|
||||||
|
### C++ compilation errors
|
||||||
|
|
||||||
|
Check that `CXX_aarch64_linux_android` points to a valid clang++ from the NDK.
|
||||||
|
|
||||||
|
### Gradle daemon issues
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./gradlew --stop
|
||||||
|
./gradlew assembleRelease --no-daemon
|
||||||
|
```
|
||||||
219
vault/Android/Debugging.md
Normal file
219
vault/Android/Debugging.md
Normal file
@@ -0,0 +1,219 @@
|
|||||||
|
---
|
||||||
|
tags: [android, wzp]
|
||||||
|
type: reference
|
||||||
|
---
|
||||||
|
|
||||||
|
# Debugging Guide
|
||||||
|
|
||||||
|
## Crash on Launch
|
||||||
|
|
||||||
|
### Symptom: App crashes immediately after opening
|
||||||
|
|
||||||
|
**Most likely cause: Namespace mismatch in AndroidManifest.xml**
|
||||||
|
|
||||||
|
The Gradle namespace is `com.wzp.phone` but all Kotlin classes are in package `com.wzp.*`. If the manifest uses shorthand names (`.WzpApplication`, `.ui.call.CallActivity`), Android resolves them as `com.wzp.phone.WzpApplication` which doesn't exist.
|
||||||
|
|
||||||
|
**Fix**: Always use fully-qualified class names in the manifest:
|
||||||
|
|
||||||
|
```xml
|
||||||
|
<!-- WRONG -->
|
||||||
|
<application android:name=".WzpApplication">
|
||||||
|
<activity android:name=".ui.call.CallActivity">
|
||||||
|
|
||||||
|
<!-- CORRECT -->
|
||||||
|
<application android:name="com.wzp.WzpApplication">
|
||||||
|
<activity android:name="com.wzp.ui.call.CallActivity">
|
||||||
|
```
|
||||||
|
|
||||||
|
### Symptom: Crash in `System.loadLibrary("wzp_android")`
|
||||||
|
|
||||||
|
The native `.so` is missing or incompatible. Check:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Verify the .so exists in the APK
|
||||||
|
unzip -l app-release.apk | grep libwzp
|
||||||
|
# Should show: lib/arm64-v8a/libwzp_android.so
|
||||||
|
|
||||||
|
# Verify ABI matches device
|
||||||
|
adb shell getprop ro.product.cpu.abi
|
||||||
|
# Should return: arm64-v8a
|
||||||
|
```
|
||||||
|
|
||||||
|
### Symptom: Crash when calling `nativeGetStats()` (returns null jstring)
|
||||||
|
|
||||||
|
The JNI bridge must return a valid `jstring`, not a null pointer. The Kotlin side declares the return as `String?` (nullable) and wraps in try/catch:
|
||||||
|
|
||||||
|
```kotlin
|
||||||
|
fun getStats(): String {
|
||||||
|
if (nativeHandle == 0L) return "{}"
|
||||||
|
return try {
|
||||||
|
nativeGetStats(nativeHandle) ?: "{}"
|
||||||
|
} catch (_: Exception) {
|
||||||
|
"{}"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Symptom: Tracing subscriber panic
|
||||||
|
|
||||||
|
`tracing_subscriber::fmt()` writes to stdout, which doesn't exist on Android. The init was removed. If you need logging, use `android_logger` crate instead.
|
||||||
|
|
||||||
|
## Logcat Filters
|
||||||
|
|
||||||
|
### View all WZP logs
|
||||||
|
|
||||||
|
```bash
|
||||||
|
adb logcat -s wzp-android:V wzp-codec:V wzp-net:V
|
||||||
|
```
|
||||||
|
|
||||||
|
### View Rust tracing output (if android_logger is added)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
adb logcat | grep -E "(wzp|WzpEngine|CallActivity)"
|
||||||
|
```
|
||||||
|
|
||||||
|
### View Oboe audio logs
|
||||||
|
|
||||||
|
```bash
|
||||||
|
adb logcat -s AAudio:V oboe:V
|
||||||
|
```
|
||||||
|
|
||||||
|
### View native crashes
|
||||||
|
|
||||||
|
```bash
|
||||||
|
adb logcat -s DEBUG:V libc:V
|
||||||
|
```
|
||||||
|
|
||||||
|
Look for `signal 11 (SIGSEGV)` or `signal 6 (SIGABRT)` with a backtrace in `libwzp_android.so`.
|
||||||
|
|
||||||
|
### Symbolicate native crash
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Find the .so with debug symbols (before stripping)
|
||||||
|
SO_PATH="target/aarch64-linux-android/release/libwzp_android.so"
|
||||||
|
|
||||||
|
# Use addr2line from NDK
|
||||||
|
$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-addr2line \
|
||||||
|
-e $SO_PATH -f 0x<address_from_crash>
|
||||||
|
```
|
||||||
|
|
||||||
|
## Network Issues
|
||||||
|
|
||||||
|
### Call stuck on "Connecting..."
|
||||||
|
|
||||||
|
The QUIC handshake to the relay is failing. Common causes:
|
||||||
|
|
||||||
|
1. **Relay not running**: Verify the relay is listening:
|
||||||
|
```bash
|
||||||
|
nc -zvu 172.16.81.125 4433
|
||||||
|
```
|
||||||
|
|
||||||
|
2. **Wrong relay address**: Hardcoded in `CallViewModel.kt`:
|
||||||
|
```kotlin
|
||||||
|
const val DEFAULT_RELAY = "172.16.81.125:4433"
|
||||||
|
```
|
||||||
|
|
||||||
|
3. **QUIC blocked by firewall**: QUIC uses UDP. Many networks block UDP traffic. Ensure UDP port 4433 is open.
|
||||||
|
|
||||||
|
4. **TLS handshake failure**: The client uses `client_config()` which disables certificate verification. If the relay's QUIC config changed, this may fail.
|
||||||
|
|
||||||
|
### Connected but no audio
|
||||||
|
|
||||||
|
1. **Microphone permission denied**: Check Android settings. The app requests `RECORD_AUDIO` on first launch.
|
||||||
|
|
||||||
|
2. **Oboe failed to start**: The codec thread logs this. Check logcat for "failed to start audio".
|
||||||
|
|
||||||
|
3. **Ring buffer underrun**: The stats overlay shows "Under" count. High underruns mean the codec thread isn't keeping up.
|
||||||
|
|
||||||
|
4. **Network not forwarding**: If both phones show "Active" but frame counters aren't increasing, the relay may not be forwarding. Check relay logs.
|
||||||
|
|
||||||
|
### High packet loss
|
||||||
|
|
||||||
|
The stats overlay shows loss percentage. Common causes:
|
||||||
|
|
||||||
|
- Wi-Fi congestion (try cellular or move closer to AP)
|
||||||
|
- UDP throttling by carrier/ISP
|
||||||
|
- Relay overloaded (check relay metrics)
|
||||||
|
|
||||||
|
## Audio Issues
|
||||||
|
|
||||||
|
### Echo
|
||||||
|
|
||||||
|
AEC (Acoustic Echo Cancellation) is enabled by default with a 100ms tail. If echo persists:
|
||||||
|
|
||||||
|
- The AEC may need a longer tail for the specific acoustic environment
|
||||||
|
- Speaker volume too high overwhelms the canceller
|
||||||
|
- Check that `last_decoded_farend` is being set (playout path working)
|
||||||
|
|
||||||
|
### Robot voice / glitching
|
||||||
|
|
||||||
|
Usually caused by jitter buffer underruns. The jitter buffer adapts between 10-250 packets. Check:
|
||||||
|
|
||||||
|
- `jitter_buffer_depth` in stats (should be > 0 during active call)
|
||||||
|
- `underruns` counter (should not climb rapidly)
|
||||||
|
- Network jitter (high jitter_ms causes adaptation)
|
||||||
|
|
||||||
|
### No sound from speaker
|
||||||
|
|
||||||
|
1. Check `isSpeaker` state in the UI
|
||||||
|
2. Oboe playout stream may have failed — check logcat for Oboe errors
|
||||||
|
3. Ring buffer might be empty — check `framesDecoded` counter
|
||||||
|
|
||||||
|
## JNI Issues
|
||||||
|
|
||||||
|
### `UnsatisfiedLinkError: No implementation found for...`
|
||||||
|
|
||||||
|
The JNI function name doesn't match. JNI names must follow the pattern:
|
||||||
|
```
|
||||||
|
Java_com_wzp_engine_WzpEngine_<methodName>
|
||||||
|
```
|
||||||
|
|
||||||
|
If the package structure changes, all JNI function names must be updated in `jni_bridge.rs`.
|
||||||
|
|
||||||
|
### Panic across FFI boundary
|
||||||
|
|
||||||
|
All JNI functions wrap their body in `panic::catch_unwind()`. If a Rust panic escapes to Java, it causes a `SIGABRT`. The catch_unwind returns safe defaults:
|
||||||
|
|
||||||
|
| Function | Panic return |
|
||||||
|
|----------|--------------|
|
||||||
|
| `nativeInit` | 0 (null handle) |
|
||||||
|
| `nativeStartCall` | -1 (error) |
|
||||||
|
| `nativeGetStats` | `JObject::null()` |
|
||||||
|
| Others | void (silently swallowed) |
|
||||||
|
|
||||||
|
### Thread safety
|
||||||
|
|
||||||
|
All JNI methods must be called from the same thread (Android main thread). The `EngineHandle` is a raw pointer — concurrent access is undefined behavior.
|
||||||
|
|
||||||
|
## Stats JSON Format
|
||||||
|
|
||||||
|
The `nativeGetStats()` returns JSON matching this Rust struct:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"state": "Active",
|
||||||
|
"duration_secs": 42.5,
|
||||||
|
"quality_tier": 0,
|
||||||
|
"loss_pct": 0.5,
|
||||||
|
"rtt_ms": 45,
|
||||||
|
"jitter_ms": 12,
|
||||||
|
"jitter_buffer_depth": 3,
|
||||||
|
"frames_encoded": 2125,
|
||||||
|
"frames_decoded": 2100,
|
||||||
|
"underruns": 5
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Kotlin deserializes this via `CallStats.fromJson()` using `org.json.JSONObject` (Android built-in, no library needed).
|
||||||
|
|
||||||
|
## Diagnostic Checklist
|
||||||
|
|
||||||
|
When something doesn't work, check in this order:
|
||||||
|
|
||||||
|
1. **APK installed for correct ABI?** (`arm64-v8a` only)
|
||||||
|
2. **Manifest class names fully qualified?** (no dots prefix)
|
||||||
|
3. **Relay running and reachable?** (`nc -zvu <host> <port>`)
|
||||||
|
4. **Microphone permission granted?**
|
||||||
|
5. **Stats polling working?** (check if frame counters increment)
|
||||||
|
6. **Logcat for native crashes?** (`adb logcat -s DEBUG:V`)
|
||||||
|
7. **Network connectivity?** (UDP port open, no firewall)
|
||||||
399
vault/Android/Fix-Audio-Ring-Desync.md
Normal file
399
vault/Android/Fix-Audio-Ring-Desync.md
Normal file
@@ -0,0 +1,399 @@
|
|||||||
|
---
|
||||||
|
tags: [android, wzp]
|
||||||
|
type: reference
|
||||||
|
---
|
||||||
|
|
||||||
|
# Fix: AudioRing SPSC Buffer Cursor Desync
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
A critical bug causes 10-16 seconds of bidirectional audio silence mid-call (~25-30s in). Both participants go silent at the exact same moment. The QUIC transport, relay, Opus codec, and FEC are all healthy — the bug is in the lock-free ring buffer that transfers decoded PCM from the Rust recv task to the Kotlin AudioTrack playout thread.
|
||||||
|
|
||||||
|
**Root cause:** `AudioRing::write()` modifies `read_pos` from the producer thread during overflow handling (lines 68-72 of `audio_ring.rs`). This violates the SPSC invariant — only the consumer should own `read_pos`. When both threads write to `read_pos`, a race corrupts the cursor state, causing the reader to see an empty or stale buffer for 12-16 seconds.
|
||||||
|
|
||||||
|
**Full forensics:** `debug/INCIDENT-2026-04-06-playout-ring-desync.md`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Solution: Reader-Detects-Lap Architecture
|
||||||
|
|
||||||
|
The writer NEVER touches `read_pos`. On overflow, the writer simply overwrites old buffer data and advances `write_pos`. The reader detects it was lapped and self-corrects by snapping its own `read_pos` forward.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Implementation Steps
|
||||||
|
|
||||||
|
### Step 1: Rewrite `AudioRing`
|
||||||
|
|
||||||
|
**File:** `crates/wzp-android/src/audio_ring.rs`
|
||||||
|
|
||||||
|
Replace the entire implementation with:
|
||||||
|
|
||||||
|
**Constants:**
|
||||||
|
```rust
|
||||||
|
/// Ring buffer capacity — must be a power of 2 for bitmask indexing.
|
||||||
|
/// 16384 samples = 341.3ms at 48kHz mono. Provides 70% more headroom
|
||||||
|
/// than the previous 9600 (200ms) for surviving Android GC pauses.
|
||||||
|
const RING_CAPACITY: usize = 16384; // 2^14
|
||||||
|
const RING_MASK: usize = RING_CAPACITY - 1;
|
||||||
|
```
|
||||||
|
|
||||||
|
**Struct:**
|
||||||
|
```rust
|
||||||
|
pub struct AudioRing {
|
||||||
|
buf: Box<[i16; RING_CAPACITY]>,
|
||||||
|
write_pos: AtomicUsize, // monotonically increasing, ONLY written by producer
|
||||||
|
read_pos: AtomicUsize, // monotonically increasing, ONLY written by consumer
|
||||||
|
overflow_count: AtomicU64, // incremented by reader when it detects a lap
|
||||||
|
underrun_count: AtomicU64, // incremented by reader when ring is empty
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**`write()` — producer. Does NOT touch `read_pos`:**
|
||||||
|
```rust
|
||||||
|
pub fn write(&self, samples: &[i16]) -> usize {
|
||||||
|
let count = samples.len().min(RING_CAPACITY);
|
||||||
|
let w = self.write_pos.load(Ordering::Relaxed);
|
||||||
|
|
||||||
|
for i in 0..count {
|
||||||
|
unsafe {
|
||||||
|
let ptr = self.buf.as_ptr() as *mut i16;
|
||||||
|
*ptr.add((w + i) & RING_MASK) = samples[i];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
self.write_pos.store(w.wrapping_add(count), Ordering::Release);
|
||||||
|
count
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**`read()` — consumer. Detects lap, self-corrects:**
|
||||||
|
```rust
|
||||||
|
pub fn read(&self, out: &mut [i16]) -> usize {
|
||||||
|
let w = self.write_pos.load(Ordering::Acquire);
|
||||||
|
let mut r = self.read_pos.load(Ordering::Relaxed);
|
||||||
|
|
||||||
|
let mut avail = w.wrapping_sub(r);
|
||||||
|
|
||||||
|
// Lap detection: writer has overwritten our unread data.
|
||||||
|
// Snap read_pos forward to oldest valid data in the buffer.
|
||||||
|
// Safe because we (the reader) are the sole owner of read_pos.
|
||||||
|
if avail > RING_CAPACITY {
|
||||||
|
r = w.wrapping_sub(RING_CAPACITY);
|
||||||
|
avail = RING_CAPACITY;
|
||||||
|
self.overflow_count.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
|
let count = out.len().min(avail);
|
||||||
|
if count == 0 {
|
||||||
|
if w == r {
|
||||||
|
self.underrun_count.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
for i in 0..count {
|
||||||
|
out[i] = unsafe { *self.buf.as_ptr().add((r + i) & RING_MASK) };
|
||||||
|
}
|
||||||
|
|
||||||
|
self.read_pos.store(r.wrapping_add(count), Ordering::Release);
|
||||||
|
count
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**`available()` — clamped for external callers:**
|
||||||
|
```rust
|
||||||
|
pub fn available(&self) -> usize {
|
||||||
|
let w = self.write_pos.load(Ordering::Acquire);
|
||||||
|
let r = self.read_pos.load(Ordering::Relaxed);
|
||||||
|
w.wrapping_sub(r).min(RING_CAPACITY)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**`free_space()` — keep for API compat:**
|
||||||
|
```rust
|
||||||
|
pub fn free_space(&self) -> usize {
|
||||||
|
RING_CAPACITY.saturating_sub(self.available())
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Diagnostic accessors:**
|
||||||
|
```rust
|
||||||
|
pub fn overflow_count(&self) -> u64 {
|
||||||
|
self.overflow_count.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn underrun_count(&self) -> u64 {
|
||||||
|
self.underrun_count.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Constructor:**
|
||||||
|
```rust
|
||||||
|
pub fn new() -> Self {
|
||||||
|
debug_assert!(RING_CAPACITY.is_power_of_two());
|
||||||
|
Self {
|
||||||
|
buf: Box::new([0i16; RING_CAPACITY]),
|
||||||
|
write_pos: AtomicUsize::new(0),
|
||||||
|
read_pos: AtomicUsize::new(0),
|
||||||
|
overflow_count: AtomicU64::new(0),
|
||||||
|
underrun_count: AtomicU64::new(0),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Imports to add:** `use std::sync::atomic::AtomicU64;`
|
||||||
|
|
||||||
|
**Safety comment update:**
|
||||||
|
```rust
|
||||||
|
// SAFETY: AudioRing is SPSC — one thread writes (producer), one reads (consumer).
|
||||||
|
// The producer only writes write_pos. The consumer only writes read_pos.
|
||||||
|
// Neither thread writes the other's cursor. Buffer indices are derived from
|
||||||
|
// the owning thread's cursor, ensuring no concurrent access to the same index.
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Step 2: Add counter fields to `CallStats`
|
||||||
|
|
||||||
|
**File:** `crates/wzp-android/src/stats.rs`
|
||||||
|
|
||||||
|
Add three fields to the `CallStats` struct (after `fec_recovered`):
|
||||||
|
|
||||||
|
```rust
|
||||||
|
/// Playout ring overflow count (reader was lapped by writer).
|
||||||
|
pub playout_overflows: u64,
|
||||||
|
/// Playout ring underrun count (reader found empty buffer).
|
||||||
|
pub playout_underruns: u64,
|
||||||
|
/// Capture ring overflow count.
|
||||||
|
pub capture_overflows: u64,
|
||||||
|
```
|
||||||
|
|
||||||
|
These derive `Default` (= 0) automatically via the existing `#[derive(Default)]`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Step 3: Wire ring diagnostics into engine stats + logging
|
||||||
|
|
||||||
|
**File:** `crates/wzp-android/src/engine.rs`
|
||||||
|
|
||||||
|
**3a.** In `get_stats()` (~line 181), populate the new fields:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
stats.playout_overflows = self.state.playout_ring.overflow_count();
|
||||||
|
stats.playout_underruns = self.state.playout_ring.underrun_count();
|
||||||
|
stats.capture_overflows = self.state.capture_ring.overflow_count();
|
||||||
|
```
|
||||||
|
|
||||||
|
**3b.** In the recv task periodic stats log, add ring health:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
info!(
|
||||||
|
frames_decoded,
|
||||||
|
fec_recovered,
|
||||||
|
recv_errors,
|
||||||
|
max_recv_gap_ms,
|
||||||
|
playout_avail = state.playout_ring.available(),
|
||||||
|
playout_overflows = state.playout_ring.overflow_count(),
|
||||||
|
playout_underruns = state.playout_ring.underrun_count(),
|
||||||
|
"recv stats"
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
|
**3c.** In the send task periodic stats log, add capture ring health:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
info!(
|
||||||
|
seq = s,
|
||||||
|
block_id,
|
||||||
|
frames_sent,
|
||||||
|
frames_dropped,
|
||||||
|
send_errors,
|
||||||
|
ring_avail = state.capture_ring.available(),
|
||||||
|
capture_overflows = state.capture_ring.overflow_count(),
|
||||||
|
"send stats"
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Step 4: Parse new stats in Kotlin
|
||||||
|
|
||||||
|
**File:** `android/app/src/main/java/com/wzp/engine/CallStats.kt`
|
||||||
|
|
||||||
|
Add fields to the data class:
|
||||||
|
|
||||||
|
```kotlin
|
||||||
|
val playoutOverflows: Long = 0,
|
||||||
|
val playoutUnderruns: Long = 0,
|
||||||
|
val captureOverflows: Long = 0,
|
||||||
|
```
|
||||||
|
|
||||||
|
Add parsing in `fromJson()`:
|
||||||
|
|
||||||
|
```kotlin
|
||||||
|
playoutOverflows = obj.optLong("playout_overflows", 0),
|
||||||
|
playoutUnderruns = obj.optLong("playout_underruns", 0),
|
||||||
|
captureOverflows = obj.optLong("capture_overflows", 0),
|
||||||
|
```
|
||||||
|
|
||||||
|
No UI changes needed — these fields will appear in debug report JSON automatically.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Step 5: Unit tests
|
||||||
|
|
||||||
|
**File:** `crates/wzp-android/src/audio_ring.rs` — add `#[cfg(test)] mod tests`
|
||||||
|
|
||||||
|
```rust
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn capacity_is_power_of_two() {
|
||||||
|
assert!(RING_CAPACITY.is_power_of_two());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn basic_write_read() {
|
||||||
|
let ring = AudioRing::new();
|
||||||
|
let input: Vec<i16> = (0..960).map(|i| i as i16).collect();
|
||||||
|
ring.write(&input);
|
||||||
|
assert_eq!(ring.available(), 960);
|
||||||
|
|
||||||
|
let mut output = vec![0i16; 960];
|
||||||
|
let read = ring.read(&mut output);
|
||||||
|
assert_eq!(read, 960);
|
||||||
|
assert_eq!(output, input);
|
||||||
|
assert_eq!(ring.available(), 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn wraparound() {
|
||||||
|
let ring = AudioRing::new();
|
||||||
|
let frame = vec![42i16; 960];
|
||||||
|
// Write enough to wrap the buffer multiple times
|
||||||
|
for _ in 0..20 {
|
||||||
|
ring.write(&frame);
|
||||||
|
let mut out = vec![0i16; 960];
|
||||||
|
ring.read(&mut out);
|
||||||
|
assert!(out.iter().all(|&s| s == 42));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn overflow_detected_by_reader() {
|
||||||
|
let ring = AudioRing::new();
|
||||||
|
// Write more than RING_CAPACITY without reading
|
||||||
|
let big = vec![7i16; RING_CAPACITY + 960];
|
||||||
|
ring.write(&big[..RING_CAPACITY]);
|
||||||
|
ring.write(&big[RING_CAPACITY..]);
|
||||||
|
|
||||||
|
// Reader should detect lap
|
||||||
|
let mut out = vec![0i16; 960];
|
||||||
|
let read = ring.read(&mut out);
|
||||||
|
assert!(read > 0);
|
||||||
|
assert_eq!(ring.overflow_count(), 1);
|
||||||
|
// Data should be from the most recent writes
|
||||||
|
assert!(out.iter().all(|&s| s == 7));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn writer_never_modifies_read_pos() {
|
||||||
|
let ring = AudioRing::new();
|
||||||
|
// Read pos should stay at 0 until read() is called
|
||||||
|
let data = vec![1i16; RING_CAPACITY + 960];
|
||||||
|
ring.write(&data);
|
||||||
|
// read_pos is private, but we can check available() > CAPACITY
|
||||||
|
// which proves write() didn't advance read_pos
|
||||||
|
let w = ring.write_pos.load(std::sync::atomic::Ordering::Relaxed);
|
||||||
|
let r = ring.read_pos.load(std::sync::atomic::Ordering::Relaxed);
|
||||||
|
assert_eq!(r, 0, "write() must not modify read_pos");
|
||||||
|
assert!(w.wrapping_sub(r) > RING_CAPACITY);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn underrun_counted() {
|
||||||
|
let ring = AudioRing::new();
|
||||||
|
let mut out = vec![0i16; 960];
|
||||||
|
let read = ring.read(&mut out);
|
||||||
|
assert_eq!(read, 0);
|
||||||
|
assert_eq!(ring.underrun_count(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn overflow_recovery_reads_recent_data() {
|
||||||
|
let ring = AudioRing::new();
|
||||||
|
// Fill with old data
|
||||||
|
let old = vec![1i16; RING_CAPACITY];
|
||||||
|
ring.write(&old);
|
||||||
|
// Overwrite with new data (lapping the reader)
|
||||||
|
let new_data = vec![99i16; 960];
|
||||||
|
ring.write(&new_data);
|
||||||
|
|
||||||
|
// Reader should snap forward and get recent data
|
||||||
|
let mut out = vec![0i16; RING_CAPACITY];
|
||||||
|
let read = ring.read(&mut out);
|
||||||
|
assert_eq!(read, RING_CAPACITY);
|
||||||
|
// The last 960 samples should be 99
|
||||||
|
assert!(out[RING_CAPACITY - 960..].iter().all(|&s| s == 99));
|
||||||
|
assert_eq!(ring.overflow_count(), 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Memory Ordering Reference
|
||||||
|
|
||||||
|
| Operation | Ordering | Rationale |
|
||||||
|
|-----------|----------|-----------|
|
||||||
|
| `write_pos.store` in `write()` | Release | Buffer writes visible before cursor advances |
|
||||||
|
| `write_pos.load` in `read()` | Acquire | Pairs with Release above — sees all buffer writes |
|
||||||
|
| `write_pos.load` in `write()` | Relaxed | Writer is sole owner of write_pos |
|
||||||
|
| `read_pos.load` in `read()` | Relaxed | Reader is sole owner of read_pos |
|
||||||
|
| `read_pos.store` in `read()` | Release | Makes available() consistent from any thread |
|
||||||
|
| `read_pos.load` in `available()` | Relaxed | Informational only, slight staleness OK |
|
||||||
|
| All counters | Relaxed | Diagnostic only |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Capacity Tradeoff
|
||||||
|
|
||||||
|
| Capacity | Duration | Memory | Verdict |
|
||||||
|
|----------|----------|--------|---------|
|
||||||
|
| 8192 (2^13) | 170ms | 16KB | Less than current 200ms — risky |
|
||||||
|
| **16384 (2^14)** | **341ms** | **32KB** | **70% more headroom, bitmask indexing** |
|
||||||
|
| 32768 (2^15) | 682ms | 64KB | Excessive latency on overflow recovery |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
1. `cargo test -p wzp-android` — new unit tests pass
|
||||||
|
2. `cargo ndk -t arm64-v8a build --release -p wzp-android` — ARM cross-compile succeeds
|
||||||
|
3. Build APK, install on both test devices (Nothing A059 + Pixel 6)
|
||||||
|
4. 2+ minute call — verify no audio gaps
|
||||||
|
5. Check debug report JSON: `playout_overflows` should be 0 or very small
|
||||||
|
6. Check logcat `wzp_android` tag: send/recv stats show healthy ring state
|
||||||
|
7. Stress test: play music through one device speaker while on call — forces high ring throughput
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Files to Modify
|
||||||
|
|
||||||
|
| File | What changes |
|
||||||
|
|------|-------------|
|
||||||
|
| `crates/wzp-android/src/audio_ring.rs` | Complete rewrite — the core fix |
|
||||||
|
| `crates/wzp-android/src/stats.rs` | Add 3 counter fields |
|
||||||
|
| `crates/wzp-android/src/engine.rs` | Wire counters into get_stats() + periodic logs |
|
||||||
|
| `android/app/src/main/java/com/wzp/engine/CallStats.kt` | Parse 3 new JSON fields |
|
||||||
|
|
||||||
|
## What Does NOT Change
|
||||||
|
|
||||||
|
- `AudioPipeline.kt` — calls `readAudio()`/`writeAudio()` unchanged; ring fix is transparent
|
||||||
|
- `jni_bridge.rs` — JNI bridge passes through unchanged
|
||||||
|
- `audio_android.rs` — separate Oboe-based ring, currently unused, different design
|
||||||
|
- Relay code — relay is confirmed healthy
|
||||||
|
- Desktop client — uses `Mutex + mpsc`, not `AudioRing`
|
||||||
154
vault/Android/Fix-Capture-Thread-Crash.md
Normal file
154
vault/Android/Fix-Capture-Thread-Crash.md
Normal file
@@ -0,0 +1,154 @@
|
|||||||
|
---
|
||||||
|
tags: [android, wzp]
|
||||||
|
type: reference
|
||||||
|
---
|
||||||
|
|
||||||
|
# Fix: Capture/Playout Thread Use-After-Free on Hangup
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
App crashes (SIGSEGV) when hanging up a call. The capture thread (`wzp-capture`) calls `engine.writeAudio()` via JNI after `teardown()` has freed the native engine handle. Same race exists for the playout thread's `readAudio()`.
|
||||||
|
|
||||||
|
**Root cause:** TOCTOU race between the `nativeHandle == 0L` check in `WzpEngine.writeAudio()`/`readAudio()` and `destroy()` freeing the native memory on the ViewModel thread. Audio threads can't be joined (libcrypto TLS destructor crash), so there's no synchronization between `stopAudio()` and `destroy()`.
|
||||||
|
|
||||||
|
**Full forensics:** `debug/INCIDENT-2026-04-06-capture-thread-use-after-free.md`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Solution: Destroy Latch
|
||||||
|
|
||||||
|
Add a `CountDownLatch(2)` that both audio threads count down after exiting their loops. `teardown()` awaits the latch (with timeout) before calling `destroy()`, guaranteeing no in-flight JNI calls.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Implementation Steps
|
||||||
|
|
||||||
|
### Step 1: Add a drain latch to `AudioPipeline`
|
||||||
|
|
||||||
|
**File:** `android/app/src/main/java/com/wzp/audio/AudioPipeline.kt`
|
||||||
|
|
||||||
|
Add a `CountDownLatch` field:
|
||||||
|
|
||||||
|
```kotlin
|
||||||
|
import java.util.concurrent.CountDownLatch
|
||||||
|
import java.util.concurrent.TimeUnit
|
||||||
|
|
||||||
|
class AudioPipeline(private val context: Context) {
|
||||||
|
// ... existing fields ...
|
||||||
|
|
||||||
|
/** Latch counted down by each audio thread after exiting its loop.
|
||||||
|
* stop() does NOT wait on this — teardown waits via awaitDrain(). */
|
||||||
|
private var drainLatch: CountDownLatch? = null
|
||||||
|
```
|
||||||
|
|
||||||
|
In `start()`, create the latch before spawning threads:
|
||||||
|
|
||||||
|
```kotlin
|
||||||
|
fun start(engine: WzpEngine) {
|
||||||
|
if (running) return
|
||||||
|
running = true
|
||||||
|
drainLatch = CountDownLatch(2) // one for capture, one for playout
|
||||||
|
|
||||||
|
captureThread = Thread({
|
||||||
|
runCapture(engine)
|
||||||
|
drainLatch?.countDown() // signal: capture loop exited
|
||||||
|
parkThread()
|
||||||
|
}, "wzp-capture").apply { ... }
|
||||||
|
|
||||||
|
playoutThread = Thread({
|
||||||
|
runPlayout(engine)
|
||||||
|
drainLatch?.countDown() // signal: playout loop exited
|
||||||
|
parkThread()
|
||||||
|
}, "wzp-playout").apply { ... }
|
||||||
|
// ...
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Add `awaitDrain()` — called by ViewModel before `destroy()`:
|
||||||
|
|
||||||
|
```kotlin
|
||||||
|
/** Block until both audio threads have exited their loops (max 200ms).
|
||||||
|
* After this returns, no more JNI calls to the engine will be made. */
|
||||||
|
fun awaitDrain(): Boolean {
|
||||||
|
return drainLatch?.await(200, TimeUnit.MILLISECONDS) ?: true
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`stop()` remains unchanged (non-blocking, sets `running = false`).
|
||||||
|
|
||||||
|
### Step 2: Update `CallViewModel.teardown()` to await drain
|
||||||
|
|
||||||
|
**File:** `android/app/src/main/java/com/wzp/ui/call/CallViewModel.kt`
|
||||||
|
|
||||||
|
Change teardown to wait for audio threads before destroying:
|
||||||
|
|
||||||
|
```kotlin
|
||||||
|
private fun teardown(stopService: Boolean = true) {
|
||||||
|
Log.i(TAG, "teardown: stopping audio, stopService=$stopService")
|
||||||
|
val hadCall = audioStarted
|
||||||
|
CallService.onStopFromNotification = null
|
||||||
|
stopAudio() // sets running=false (non-blocking)
|
||||||
|
stopStatsPolling()
|
||||||
|
|
||||||
|
// Wait for audio threads to exit their loops before destroying the engine.
|
||||||
|
// This guarantees no in-flight JNI calls to writeAudio/readAudio.
|
||||||
|
val drained = audioPipeline?.awaitDrain() ?: true
|
||||||
|
if (!drained) {
|
||||||
|
Log.w(TAG, "teardown: audio threads did not drain in time")
|
||||||
|
}
|
||||||
|
audioPipeline = null
|
||||||
|
|
||||||
|
Log.i(TAG, "teardown: stopping engine")
|
||||||
|
try { engine?.stopCall() } catch (e: Exception) { Log.w(TAG, "stopCall err: $e") }
|
||||||
|
try { engine?.destroy() } catch (e: Exception) { Log.w(TAG, "destroy err: $e") }
|
||||||
|
engine = null
|
||||||
|
engineInitialized = false
|
||||||
|
// ... rest unchanged
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Key change:** `awaitDrain()` is called AFTER `stopAudio()` (which sets `running=false`) but BEFORE `engine?.destroy()`. The latch guarantees both threads have exited their `while(running)` loops and will never call `writeAudio`/`readAudio` again.
|
||||||
|
|
||||||
|
Also move `audioPipeline = null` to after `awaitDrain()` to keep the reference alive for the latch call.
|
||||||
|
|
||||||
|
### Step 3: Move `stopAudio()` pipeline nulling
|
||||||
|
|
||||||
|
**File:** `android/app/src/main/java/com/wzp/ui/call/CallViewModel.kt`
|
||||||
|
|
||||||
|
In `stopAudio()`, do NOT null out the pipeline — let `teardown()` handle it after drain:
|
||||||
|
|
||||||
|
```kotlin
|
||||||
|
private fun stopAudio() {
|
||||||
|
if (!audioStarted) return
|
||||||
|
audioPipeline?.stop() // sets running=false
|
||||||
|
// DON'T null audioPipeline here — teardown() needs it for awaitDrain()
|
||||||
|
audioRouteManager?.unregister()
|
||||||
|
audioRouteManager?.setSpeaker(false)
|
||||||
|
_isSpeaker.value = false
|
||||||
|
audioStarted = false
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Files to Modify
|
||||||
|
|
||||||
|
| File | What changes |
|
||||||
|
|------|-------------|
|
||||||
|
| `android/.../audio/AudioPipeline.kt` | Add `CountDownLatch`, `countDown()` in threads, `awaitDrain()` method |
|
||||||
|
| `android/.../ui/call/CallViewModel.kt` | `teardown()` calls `awaitDrain()` before `destroy()`; `stopAudio()` doesn't null pipeline |
|
||||||
|
|
||||||
|
## What Does NOT Change
|
||||||
|
|
||||||
|
- `WzpEngine.kt` — the `nativeHandle == 0L` guard stays as defense-in-depth
|
||||||
|
- `jni_bridge.rs` — `panic::catch_unwind` stays as last resort
|
||||||
|
- `AudioPipeline.stop()` — remains non-blocking
|
||||||
|
- Thread parking — still needed to avoid libcrypto TLS crash
|
||||||
|
|
||||||
|
## Verification
|
||||||
|
|
||||||
|
1. Build APK, install on test device
|
||||||
|
2. Make a call, hang up — verify no crash in logcat (`adb logcat -s AndroidRuntime:E DEBUG:F`)
|
||||||
|
3. Rapid call/hangup/call/hangup cycles — stress the teardown path
|
||||||
|
4. Check logcat for `teardown: audio threads did not drain in time` — should never appear under normal conditions
|
||||||
|
5. Verify debug report still works after hangup (latch doesn't interfere with report collection)
|
||||||
195
vault/Android/Maintenance.md
Normal file
195
vault/Android/Maintenance.md
Normal file
@@ -0,0 +1,195 @@
|
|||||||
|
---
|
||||||
|
tags: [android, wzp]
|
||||||
|
type: reference
|
||||||
|
---
|
||||||
|
|
||||||
|
# Maintenance Guide
|
||||||
|
|
||||||
|
## Code Map — Where to Change Things
|
||||||
|
|
||||||
|
### Changing the relay address or room
|
||||||
|
|
||||||
|
Edit `CallViewModel.kt`:
|
||||||
|
```kotlin
|
||||||
|
companion object {
|
||||||
|
const val DEFAULT_RELAY = "172.16.81.125:4433"
|
||||||
|
const val DEFAULT_ROOM = "android"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
For a proper settings screen, add a new Composable in `ui/` that persists to `SharedPreferences` and passes values to `viewModel.startCall(relay, room)`.
|
||||||
|
|
||||||
|
### Adding authentication
|
||||||
|
|
||||||
|
1. In `CallViewModel.startCall()`, pass a token parameter
|
||||||
|
2. In `engine.rs`, after QUIC connect but before CallOffer, send:
|
||||||
|
```rust
|
||||||
|
transport.send_signal(&SignalMessage::AuthToken { token: auth_token }).await?;
|
||||||
|
```
|
||||||
|
3. Wait for the relay to accept before proceeding to handshake
|
||||||
|
4. Start relay with `--auth-url <featherchat-endpoint>`
|
||||||
|
|
||||||
|
### Enabling media encryption
|
||||||
|
|
||||||
|
The crypto session is already derived in `engine.rs` but not applied to packets. To enable:
|
||||||
|
|
||||||
|
1. Pass `_session` (currently unused) to the send/recv tasks
|
||||||
|
2. Before `transport.send_media()`, encrypt the payload:
|
||||||
|
```rust
|
||||||
|
let mut ciphertext = Vec::new();
|
||||||
|
session.encrypt(&header_bytes, &payload, &mut ciphertext)?;
|
||||||
|
packet.payload = Bytes::from(ciphertext);
|
||||||
|
```
|
||||||
|
3. After `transport.recv_media()`, decrypt:
|
||||||
|
```rust
|
||||||
|
let mut plaintext = Vec::new();
|
||||||
|
session.decrypt(&header_bytes, &pkt.payload, &mut plaintext)?;
|
||||||
|
pkt.payload = Bytes::from(plaintext);
|
||||||
|
```
|
||||||
|
|
||||||
|
### Adding a new codec / quality profile
|
||||||
|
|
||||||
|
1. Define the profile in `wzp-proto/src/codec_id.rs`
|
||||||
|
2. Implement `AudioEncoder`/`AudioDecoder` traits in `wzp-codec`
|
||||||
|
3. Register in `AdaptiveEncoder`/`AdaptiveDecoder` switch logic
|
||||||
|
4. Add to `supported_profiles` in the CallOffer (engine.rs)
|
||||||
|
|
||||||
|
### Changing audio parameters
|
||||||
|
|
||||||
|
- **Sample rate**: Change `FRAME_SAMPLES` in `audio_android.rs` and `WzpOboeConfig.sample_rate` in `oboe_bridge.cpp`. Must match the codec's expected rate.
|
||||||
|
- **Frame duration**: Change `FRAME_SAMPLES` (960 = 20ms at 48kHz, 1920 = 40ms)
|
||||||
|
- **Ring buffer size**: Change `RING_CAPACITY` in `audio_android.rs`
|
||||||
|
- **AEC tail length**: Change the `100` in `Pipeline::new()` → `EchoCanceller::new(48000, 100)`
|
||||||
|
|
||||||
|
### Adding x86_64 support (emulator)
|
||||||
|
|
||||||
|
1. `build.gradle.kts`: add `"x86_64"` to `abiFilters`
|
||||||
|
2. `cargoNdkBuild` task: add `-t x86_64`
|
||||||
|
3. `build.rs`: handle `x86_64-linux-android` target for Oboe
|
||||||
|
4. Note: Oboe in the emulator uses a different audio HAL — audio quality will differ
|
||||||
|
|
||||||
|
## Dependency Overview
|
||||||
|
|
||||||
|
### Rust Crate Dependencies (wzp-android)
|
||||||
|
|
||||||
|
| Crate | Version | Purpose | Upgrade risk |
|
||||||
|
|-------|---------|---------|--------------|
|
||||||
|
| `jni` | 0.21 | Java FFI | Low — stable API |
|
||||||
|
| `tokio` | 1.x | Async runtime | Low |
|
||||||
|
| `quinn` | 0.11 | QUIC transport | Medium — breaking changes between 0.x |
|
||||||
|
| `rustls` | 0.23 | TLS for QUIC | Medium — tied to quinn version |
|
||||||
|
| `serde_json` | 1.x | Stats serialization | Low |
|
||||||
|
| `anyhow` | 1.x | Error handling | Low |
|
||||||
|
| `tracing` | 0.1 | Logging | Low |
|
||||||
|
| `rand` | 0.8 | Random seed generation | Low |
|
||||||
|
|
||||||
|
### Workspace Crate Dependencies
|
||||||
|
|
||||||
|
| Crate | Purpose | Key trait |
|
||||||
|
|-------|---------|-----------|
|
||||||
|
| `wzp-proto` | Shared types and traits | `MediaTransport`, `AudioEncoder`, `KeyExchange` |
|
||||||
|
| `wzp-codec` | Opus + Codec2 + signal processing | `AdaptiveEncoder`, `EchoCanceller` |
|
||||||
|
| `wzp-fec` | RaptorQ FEC | `RaptorQFecEncoder` |
|
||||||
|
| `wzp-crypto` | Key exchange + encryption | `WarzoneKeyExchange`, `ChaChaSession` |
|
||||||
|
| `wzp-transport` | QUIC connection management | `QuinnTransport`, `connect()` |
|
||||||
|
|
||||||
|
### Android/Kotlin Dependencies
|
||||||
|
|
||||||
|
| Library | Version | Purpose |
|
||||||
|
|---------|---------|---------|
|
||||||
|
| `compose-bom` | 2024.01.00 | Compose version alignment |
|
||||||
|
| `material3` | (from BOM) | UI components |
|
||||||
|
| `activity-compose` | 1.8.2 | Activity integration |
|
||||||
|
| `lifecycle-runtime-ktx` | 2.7.0 | ViewModel + coroutines |
|
||||||
|
| `core-ktx` | 1.12.0 | Kotlin extensions |
|
||||||
|
|
||||||
|
## Updating Dependencies
|
||||||
|
|
||||||
|
### Rust
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo update -p wzp-android
|
||||||
|
cargo ndk -t arm64-v8a build --release -p wzp-android
|
||||||
|
```
|
||||||
|
|
||||||
|
Watch for `quinn`/`rustls` version coupling. They must be compatible:
|
||||||
|
- quinn 0.11 requires rustls 0.23
|
||||||
|
|
||||||
|
### Android/Kotlin
|
||||||
|
|
||||||
|
Update versions in `android/app/build.gradle.kts`. Key compatibility:
|
||||||
|
- `kotlinCompilerExtensionVersion` must match the Kotlin version
|
||||||
|
- `compose-bom` version determines all Compose library versions
|
||||||
|
- `compileSdk` and `targetSdk` should stay in sync
|
||||||
|
|
||||||
|
### NDK
|
||||||
|
|
||||||
|
If upgrading the NDK:
|
||||||
|
1. Update `ndkVersion` in `build.gradle.kts`
|
||||||
|
2. Update `ANDROID_NDK_HOME` environment variable
|
||||||
|
3. Update `CC_aarch64_linux_android` and friends
|
||||||
|
4. Verify Oboe still builds with the new toolchain
|
||||||
|
|
||||||
|
## Key Invariants to Preserve
|
||||||
|
|
||||||
|
1. **JNI function names must match package structure**: If the Kotlin package changes, all `Java_com_wzp_engine_WzpEngine_*` functions in `jni_bridge.rs` must be renamed.
|
||||||
|
|
||||||
|
2. **Manifest uses fully-qualified class names**: Never use `.ClassName` shorthand because the Gradle namespace (`com.wzp.phone`) differs from the Kotlin package (`com.wzp`).
|
||||||
|
|
||||||
|
3. **Stats JSON field names are snake_case**: Rust serializes with serde defaults (snake_case). Kotlin's `CallStats.fromJson()` expects `duration_secs`, `loss_pct`, etc.
|
||||||
|
|
||||||
|
4. **Ring buffer ordering**: Producer uses Release store on write index, consumer uses Acquire load. Breaking this causes torn reads.
|
||||||
|
|
||||||
|
5. **Codec thread owns Pipeline**: Pipeline is `!Send` (Opus encoder state). It must never be accessed from another thread.
|
||||||
|
|
||||||
|
6. **panic::catch_unwind on all JNI functions**: Rust panics unwinding across the FFI boundary is UB. Every JNI-exposed function must catch panics.
|
||||||
|
|
||||||
|
7. **Channel capacity (64)**: Both `send_tx` and `recv_tx` are bounded at 64 packets. If the network is slow, packets are dropped (`try_send` best-effort).
|
||||||
|
|
||||||
|
## Testing
|
||||||
|
|
||||||
|
### Unit Tests (Rust)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Run all workspace tests (host, not Android)
|
||||||
|
cargo test
|
||||||
|
|
||||||
|
# Run only wzp-android tests (uses oboe_stub.cpp on host)
|
||||||
|
cargo test -p wzp-android
|
||||||
|
```
|
||||||
|
|
||||||
|
Note: Pipeline, codec, FEC, crypto tests run on the host. Audio tests use stubs.
|
||||||
|
|
||||||
|
### On-Device Testing
|
||||||
|
|
||||||
|
1. Build and install debug APK
|
||||||
|
2. Open app, tap CALL
|
||||||
|
3. Verify in logcat:
|
||||||
|
- `WzpEngine created via JNI`
|
||||||
|
- `connecting to relay...`
|
||||||
|
- `QUIC connected to relay`
|
||||||
|
- `CallOffer sent`
|
||||||
|
- `handshake complete, call active`
|
||||||
|
- `codec thread started`
|
||||||
|
4. Check stats overlay: frame counters should increment
|
||||||
|
5. Speak into mic — other connected device should hear audio
|
||||||
|
|
||||||
|
### Stress Testing
|
||||||
|
|
||||||
|
- Run a call for 30+ minutes — check for memory leaks (stats should be stable)
|
||||||
|
- Kill and restart the relay — client should eventually get a connection error
|
||||||
|
- Toggle mute rapidly — verify no crashes
|
||||||
|
- Switch speaker on/off — verify audio route changes
|
||||||
|
|
||||||
|
## Performance Monitoring
|
||||||
|
|
||||||
|
Key metrics to watch during a call:
|
||||||
|
|
||||||
|
| Metric | Healthy Range | Warning | Critical |
|
||||||
|
|--------|--------------|---------|----------|
|
||||||
|
| frames_encoded | Increasing ~50/sec | Stalled | 0 |
|
||||||
|
| frames_decoded | Increasing ~50/sec | Stalled | 0 |
|
||||||
|
| underruns | < 5/min | > 20/min | > 100/min |
|
||||||
|
| jitter_buffer_depth | 2-5 | 0 or >10 | N/A |
|
||||||
|
| loss_pct | < 5% | 5-20% | > 20% |
|
||||||
|
| rtt_ms | < 100ms | 100-300ms | > 500ms |
|
||||||
46
vault/Android/README.md
Normal file
46
vault/Android/README.md
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
---
|
||||||
|
tags: [android, wzp]
|
||||||
|
type: reference
|
||||||
|
---
|
||||||
|
|
||||||
|
# WarzonePhone Android Client
|
||||||
|
|
||||||
|
The WZP Android client is a native VoIP application built with Kotlin/Jetpack Compose on top of a Rust audio engine. It connects to WZP relay servers over QUIC, providing encrypted voice calls with adaptive quality, forward error correction, and acoustic echo cancellation.
|
||||||
|
|
||||||
|
## Quick Start
|
||||||
|
|
||||||
|
1. **Build**: `cd android && ./gradlew assembleRelease` (requires NDK 26.1, cargo-ndk)
|
||||||
|
2. **Install**: `adb install app/build/outputs/apk/release/app-release.apk`
|
||||||
|
3. **Run**: Open "WZ Phone", tap **CALL** to connect to the hardcoded relay
|
||||||
|
4. **Relay**: Must be running at the configured address (default `172.16.81.125:4433`)
|
||||||
|
|
||||||
|
## Current State (April 2025)
|
||||||
|
|
||||||
|
| Feature | Status |
|
||||||
|
|---------|--------|
|
||||||
|
| QUIC transport to relay | Working |
|
||||||
|
| Crypto handshake (X25519 + Ed25519) | Working |
|
||||||
|
| Opus 24k encoding/decoding | Working |
|
||||||
|
| Oboe audio I/O (48kHz mono) | Working |
|
||||||
|
| AEC / AGC signal processing | Working |
|
||||||
|
| RaptorQ FEC | Wired (repair symbols not sent yet) |
|
||||||
|
| Jitter buffer | Working |
|
||||||
|
| Adaptive quality switching | Codec-ready, not network-driven yet |
|
||||||
|
| Authentication (featherChat) | Skipped (relay has no --auth-url) |
|
||||||
|
| Media encryption (ChaCha20-Poly1305) | Session derived but not applied to packets |
|
||||||
|
| Foreground service / wake locks | Implemented, not started from UI |
|
||||||
|
|
||||||
|
## Documentation Index
|
||||||
|
|
||||||
|
- [Architecture](architecture.md) - System design, data flow diagrams, thread model
|
||||||
|
- [Build Guide](build-guide.md) - Build environment setup, dependencies, signing
|
||||||
|
- [Debugging](debugging.md) - Crash diagnosis, logcat filters, common issues
|
||||||
|
- [Maintenance](maintenance.md) - Code map, dependency management, upgrade paths
|
||||||
|
- [Roadmap](roadmap.md) - Planned work and known gaps
|
||||||
|
|
||||||
|
## Key Design Decisions
|
||||||
|
|
||||||
|
- **Rust native engine**: All audio processing, codecs, FEC, crypto, and networking run in Rust. Kotlin is UI-only.
|
||||||
|
- **Lock-free audio**: SPSC ring buffers with atomic ordering between Oboe C++ callbacks and the Rust codec thread. No mutexes in the audio path.
|
||||||
|
- **cargo-ndk**: The native library (`libwzp_android.so`) is cross-compiled for `arm64-v8a` using cargo-ndk, invoked automatically by Gradle's `cargoNdkBuild` task.
|
||||||
|
- **Single-activity Compose**: One `CallActivity` hosts all UI via Jetpack Compose with `CallViewModel` as the state holder.
|
||||||
117
vault/Android/Roadmap.md
Normal file
117
vault/Android/Roadmap.md
Normal file
@@ -0,0 +1,117 @@
|
|||||||
|
---
|
||||||
|
tags: [android, wzp]
|
||||||
|
type: reference
|
||||||
|
---
|
||||||
|
|
||||||
|
# Roadmap & Known Gaps
|
||||||
|
|
||||||
|
## Current State Summary
|
||||||
|
|
||||||
|
The Android client can connect to a WZP relay, complete the crypto handshake, and exchange audio in real-time. Two phones on the same network can talk to each other through the relay.
|
||||||
|
|
||||||
|
## What Works (April 2025)
|
||||||
|
|
||||||
|
- QUIC transport to relay with room-based SFU
|
||||||
|
- Full crypto handshake (X25519 ephemeral + Ed25519 signatures)
|
||||||
|
- Opus 24kbps encoding/decoding at 48kHz
|
||||||
|
- Lock-free audio I/O via Oboe (capture + playout)
|
||||||
|
- AEC (acoustic echo cancellation) with 100ms tail
|
||||||
|
- AGC (automatic gain control)
|
||||||
|
- RaptorQ FEC encoder/decoder (wired to pipeline)
|
||||||
|
- Adaptive jitter buffer (10-250 packets)
|
||||||
|
- UI with connect/disconnect, mute, speaker, live stats
|
||||||
|
- Random identity seed per app launch
|
||||||
|
|
||||||
|
## Known Gaps
|
||||||
|
|
||||||
|
### P0 — Must fix for usable calls
|
||||||
|
|
||||||
|
| Gap | Impact | Where to fix |
|
||||||
|
|-----|--------|--------------|
|
||||||
|
| **Media encryption not applied** | Audio sent in cleartext over QUIC | `engine.rs` — pass `_session` to send/recv, encrypt/decrypt payloads |
|
||||||
|
| **FEC repair symbols not sent** | No loss recovery — audio gaps on packet loss | `engine.rs` send task — call `fec_encoder.generate_repair()` and send repair packets |
|
||||||
|
| **Quality reports not sent** | Relay can't monitor quality, no adaptive switching | `engine.rs` — periodically attach `QualityReport` to MediaPacket header |
|
||||||
|
| **CallService not started** | Call dies when app is backgrounded | `CallViewModel.startCall()` — call `CallService.start(context)` |
|
||||||
|
|
||||||
|
### P1 — Important for production
|
||||||
|
|
||||||
|
| Gap | Impact | Where to fix |
|
||||||
|
|-----|--------|--------------|
|
||||||
|
| **Hardcoded relay address** | Can't change server without rebuild | Add settings screen with `SharedPreferences` |
|
||||||
|
| **No reconnection logic** | Connection drop = call over | `engine.rs` network task — detect disconnect, retry with backoff |
|
||||||
|
| **No adaptive quality switching** | Stays on GOOD profile even in bad conditions | Wire `AdaptiveQualityController` to network path quality from `QuinnTransport` |
|
||||||
|
| **Identity seed not persisted** | New identity every launch | Save seed to Android Keystore or SharedPreferences |
|
||||||
|
| **No Bluetooth audio routing** | `AudioRouteManager` exists but not wired to UI | Add Bluetooth button to InCallScreen, call `AudioRouteManager` methods |
|
||||||
|
| **No ringtone/notification for incoming** | Only outgoing calls supported | Need signaling for call setup (currently both sides initiate independently) |
|
||||||
|
|
||||||
|
### P2 — Nice to have
|
||||||
|
|
||||||
|
| Gap | Impact | Where to fix |
|
||||||
|
|-----|--------|--------------|
|
||||||
|
| **No android_logger** | Rust tracing output lost on Android | Add `android_logger` crate, init in `nativeInit()` |
|
||||||
|
| **Stats don't include network metrics** | Loss/RTT/jitter always 0 | Feed `QuinnTransport.path_quality()` back to stats |
|
||||||
|
| **No ProGuard/R8 minification** | Release APK larger than necessary | Enable `isMinifyEnabled = true` in build.gradle.kts |
|
||||||
|
| **Single ABI (arm64-v8a)** | No support for older 32-bit devices or emulators | Add `armeabi-v7a` and `x86_64` to cargo-ndk build |
|
||||||
|
| **No call history** | Can't see past calls | Add Room database for call log |
|
||||||
|
| **No contact integration** | Manual relay/room entry | Add contacts with fingerprint-based identity |
|
||||||
|
|
||||||
|
## Architecture Evolution Plan
|
||||||
|
|
||||||
|
### Phase 1: Make Calls Reliable (current → next)
|
||||||
|
|
||||||
|
```
|
||||||
|
[x] QUIC connection to relay
|
||||||
|
[x] Crypto handshake
|
||||||
|
[x] Audio encode/decode pipeline
|
||||||
|
[ ] Media encryption (ChaCha20-Poly1305)
|
||||||
|
[ ] FEC repair packet transmission
|
||||||
|
[ ] Foreground service for background calls
|
||||||
|
[ ] Reconnection on network change
|
||||||
|
```
|
||||||
|
|
||||||
|
### Phase 2: Quality & Polish
|
||||||
|
|
||||||
|
```
|
||||||
|
[ ] Adaptive quality (GOOD → DEGRADED → CATASTROPHIC switching)
|
||||||
|
[ ] Quality reports in MediaPacket headers
|
||||||
|
[ ] Network path quality display (real RTT, loss, jitter)
|
||||||
|
[ ] Settings screen (relay, room, seed persistence)
|
||||||
|
[ ] Bluetooth/wired headset audio routing
|
||||||
|
[ ] Rust android_logger for debugging
|
||||||
|
```
|
||||||
|
|
||||||
|
### Phase 3: Production Features
|
||||||
|
|
||||||
|
```
|
||||||
|
[ ] featherChat authentication
|
||||||
|
[ ] Persistent identity (Android Keystore)
|
||||||
|
[ ] Push notifications for incoming calls
|
||||||
|
[ ] Multi-party rooms (already supported by relay)
|
||||||
|
[ ] Call transfer
|
||||||
|
[ ] End-to-end encryption (bypass relay decryption)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Dependency Upgrade Path
|
||||||
|
|
||||||
|
### quinn 0.11 → 0.12 (when released)
|
||||||
|
|
||||||
|
Quinn 0.12 will likely require rustls 0.24. Update both together:
|
||||||
|
1. `Cargo.toml`: bump quinn and rustls versions
|
||||||
|
2. Check `client_config()` and `server_config()` in wzp-transport for API changes
|
||||||
|
3. DATAGRAM API may change — check `send_datagram()` / `read_datagram()`
|
||||||
|
|
||||||
|
### Compose BOM 2024.01 → 2025.x
|
||||||
|
|
||||||
|
The `LinearProgressIndicator` `progress` parameter changed from `Float` to `() -> Float` in Material3 1.2+. If upgrading the BOM:
|
||||||
|
|
||||||
|
```kotlin
|
||||||
|
// Old (current):
|
||||||
|
LinearProgressIndicator(progress = level, ...)
|
||||||
|
|
||||||
|
// New (Material3 1.2+):
|
||||||
|
LinearProgressIndicator(progress = { level }, ...)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Kotlin 1.9 → 2.x
|
||||||
|
|
||||||
|
Kotlin 2.0 changed the Compose compiler plugin. Update `kotlinCompilerExtensionVersion` in `composeOptions` and the Kotlin Gradle plugin version together.
|
||||||
1245
vault/Architecture/Architecture.md
Normal file
1245
vault/Architecture/Architecture.md
Normal file
File diff suppressed because it is too large
Load Diff
233
vault/Architecture/Attack-Surface-Relay-Abuse.md
Normal file
233
vault/Architecture/Attack-Surface-Relay-Abuse.md
Normal file
@@ -0,0 +1,233 @@
|
|||||||
|
---
|
||||||
|
tags: [architecture, wzp]
|
||||||
|
type: architecture
|
||||||
|
---
|
||||||
|
|
||||||
|
# Relay Abuse: Attack Surface & Mitigations
|
||||||
|
|
||||||
|
> WZP is end-to-end encrypted. The relay forwards ciphertext and cannot inspect payload content. This document enumerates the abuse vectors that survive E2E and the mitigations available without breaking it.
|
||||||
|
>
|
||||||
|
> Motivating threat: a PoC on another project (LiveKit) showed that an E2E SFU with no conformance enforcement can be repurposed as a free arbitrary-data tunnel. WZP must not be that.
|
||||||
|
|
||||||
|
## Threat model
|
||||||
|
|
||||||
|
### In scope
|
||||||
|
|
||||||
|
- **Bulk data tunneling.** Attacker uses a legitimate handshake, then pushes arbitrary bytes (file transfer, piracy, scraped traffic) through media datagrams.
|
||||||
|
- **Bandwidth parasitism.** Attacker uses the relay as a cheap forwarder for unrelated traffic at scale.
|
||||||
|
- **Quota / billing evasion.** Attacker disguises high-bandwidth use as low-bandwidth audio.
|
||||||
|
- **DoS via amplification.** Attacker sends one packet → SFU fans out to N peers, multiplying egress cost N×.
|
||||||
|
|
||||||
|
### Out of scope (cannot be solved without breaking E2E)
|
||||||
|
|
||||||
|
- **Steganography inside real audio.** Modulating Opus-encoded waveforms to encode a covert channel. Information-theoretic limit; ~tens to hundreds of bps achievable; economically uninteresting.
|
||||||
|
- **Modem-over-call.** Real audio whose semantic content is data. Same limit.
|
||||||
|
- **Slow exfiltration under all rate caps.** Attacker who stays within audio's natural bandwidth envelope, indefinitely.
|
||||||
|
|
||||||
|
### Threat actor profile
|
||||||
|
|
||||||
|
We are defending against **economically motivated abuse at scale**, not against a determined nation-state covert channel. The former needs bandwidth and is loud; the latter is impossible to stop and not worth the engineering cost.
|
||||||
|
|
||||||
|
## What the relay can observe
|
||||||
|
|
||||||
|
Despite E2E, the relay sees a lot. None of this is encrypted to the relay:
|
||||||
|
|
||||||
|
| Observable | Source | Bits available |
|
||||||
|
|---|---|---|
|
||||||
|
| `CodecID` (declared codec) | `MediaHeader`, AAD | 4 (today) / 6 (v2) |
|
||||||
|
| `MediaType` (audio / video / data / control) | `MediaHeader` v2 | 2 |
|
||||||
|
| `sequence`, `timestamp_ms` | `MediaHeader` | 32 + 32 |
|
||||||
|
| `fec_block_id`, `fec_symbol_idx`, `FecRatio`, `T` (repair) | `MediaHeader` | varies |
|
||||||
|
| `KeyFrame` bit | `MediaHeader` v2 | 1 |
|
||||||
|
| `Q` flag (QualityReport trailer present) | `MediaHeader` | 1 |
|
||||||
|
| Packet size | QUIC layer | — |
|
||||||
|
| Packet inter-arrival timing | QUIC layer | — |
|
||||||
|
| Aggregate bytes/sec per session | RelayMetrics | — |
|
||||||
|
| Source fingerprint, src IP | Session state | — |
|
||||||
|
|
||||||
|
This is enough surface for strong conformance enforcement without ever touching encrypted payload.
|
||||||
|
|
||||||
|
## Mitigation tiers
|
||||||
|
|
||||||
|
Listed in order of cost-to-implement vs. decisiveness. Tier A alone kills the gross-abuse threat. Higher tiers add defense in depth.
|
||||||
|
|
||||||
|
### Tier A — Codec-conformance bitrate caps
|
||||||
|
|
||||||
|
For each declared `CodecID`, the wire bitrate has a math-derivable hard ceiling:
|
||||||
|
|
||||||
|
```
|
||||||
|
ceiling_bps[CodecID] = nominal_bitrate * (1 + max_FEC_ratio) * (1 + overhead_pct)
|
||||||
|
= nominal * 3.0 * 1.15 // FEC max 2.0 → factor 3.0
|
||||||
|
```
|
||||||
|
|
||||||
|
| Codec | Nominal | Hard ceiling |
|
||||||
|
|---|---|---|
|
||||||
|
| Opus 64k | 64 kbps | ~221 kbps |
|
||||||
|
| Opus 24k | 24 kbps | ~83 kbps |
|
||||||
|
| Opus 6k | 6 kbps | ~21 kbps |
|
||||||
|
| Codec2 1200 | 1.2 kbps | ~4 kbps |
|
||||||
|
| ComfortNoise | 0 | ~2 kbps |
|
||||||
|
|
||||||
|
Sliding 1 s window per session. Sustained excess → hard violation, close session.
|
||||||
|
|
||||||
|
Decisive against bulk tunneling. False-positive rate negligible if ceilings set at math-derived max × 1.5.
|
||||||
|
|
||||||
|
### Tier B — Packet-rate conformance
|
||||||
|
|
||||||
|
Each codec has a fixed frame interval (20 ms or 40 ms), so legal `pps` is 25 or 50, plus FEC repair packets (max ~150 pps total at FEC ratio 2.0). Anything sustaining > 200 pps for an audio codec is not audio.
|
||||||
|
|
||||||
|
### Tier C — Timestamp-rate consistency
|
||||||
|
|
||||||
|
`timestamp_ms` advances at the declared frame interval. `Δtimestamp / Δseq` over a rolling window should match the codec's frame duration ±2×. Divergence catches abusers who send audio-rate small packets but burn fields for payload.
|
||||||
|
|
||||||
|
### Tier D — Per-codec packet-size sanity
|
||||||
|
|
||||||
|
EWMA of packet size per session, compared to per-codec typical:
|
||||||
|
|
||||||
|
| Codec | Typical | Reject above |
|
||||||
|
|---|---|---|
|
||||||
|
| Opus 24k 20 ms | 60–80 B | 160 B |
|
||||||
|
| Opus 6k 40 ms | 30–40 B | 90 B |
|
||||||
|
| Codec2 1200 40 ms | 6 B | 30 B |
|
||||||
|
| ComfortNoise | 0–4 B | 16 B |
|
||||||
|
|
||||||
|
### Tier E — Per-fingerprint / per-IP token bucket
|
||||||
|
|
||||||
|
Aggregate quota regardless of declared codec:
|
||||||
|
|
||||||
|
```
|
||||||
|
For each (fingerprint, src_ip):
|
||||||
|
monthly_bytes_quota authenticated = 50 GB (tune)
|
||||||
|
anonymous = 1 GB
|
||||||
|
per-session cap audio = 256 kbps
|
||||||
|
video = 5 Mbps
|
||||||
|
burst = 30 s at 2× cap
|
||||||
|
```
|
||||||
|
|
||||||
|
Won't stop a single rogue session under cap; bounds aggregate blast radius and makes relay economics predictable.
|
||||||
|
|
||||||
|
### Tier F — Behavioral entropy / statistical fingerprinting
|
||||||
|
|
||||||
|
The deeper layer. Computed continuously per session over 10–30 s windows. Combined score flags streams that pass declared-codec checks but do not statistically look like real media.
|
||||||
|
|
||||||
|
**Why this works:** real audio and real video have very specific statistical signatures that tunneled data does not naturally produce, and that an attacker would have to deliberately and expensively mimic. The signatures differ wildly between audio and video — which is exactly why we separate them (see next section).
|
||||||
|
|
||||||
|
#### Audio fingerprint features
|
||||||
|
|
||||||
|
| Feature | Real Opus speech | Tunneled data |
|
||||||
|
|---|---|---|
|
||||||
|
| **IAT coefficient of variation** | 0.1–0.4 (clocked) | > 1.0 (bursty) |
|
||||||
|
| **Payload-size distribution** | Bimodal: speech 60–80 B + silence/CN 0–10 B | Unimodal, large, MTU-skewed |
|
||||||
|
| **Silence fraction** | 10–40 % (real conversation pauses) | < 2 % |
|
||||||
|
| **Bitrate over 30 s** | Tracks nominal codec ±20 % | Often saturates ceiling |
|
||||||
|
| **`Q` flag cadence** | Periodic, regular | Absent or random |
|
||||||
|
| **DRED / FEC ratio response** | Tracks `QualityReport` trend | Static or noise |
|
||||||
|
|
||||||
|
Single derived score: `audio_legitimacy ∈ [0, 1]`. Below threshold (e.g. 0.3) for 60 s → flag.
|
||||||
|
|
||||||
|
#### Video fingerprint features (post-V1)
|
||||||
|
|
||||||
|
| Feature | Real H.264 / AV1 video | Tunneled data |
|
||||||
|
|---|---|---|
|
||||||
|
| **Keyframe periodicity** | Regular (every 1–4 s, or on PLI) | Absent or uniform `KeyFrame=1` |
|
||||||
|
| **Frame-size ratio (I / P)** | 5–20× | ≈ 1× |
|
||||||
|
| **Burst structure** | One I-frame = N packets in < 5 ms, then quiet | Uniform spacing |
|
||||||
|
| **Bitrate response to BWE feedback** | Tracks `TransportFeedback::remb_bps` | Ignores it |
|
||||||
|
| **Resolution / FPS implied by bitrate** | Coherent (240 p ≠ 8 Mbps) | Incoherent |
|
||||||
|
| **NACK / PLI responsiveness** | Sender produces keyframe within 200 ms | No response |
|
||||||
|
|
||||||
|
Single derived score: `video_legitimacy ∈ [0, 1]`.
|
||||||
|
|
||||||
|
#### Implementation shape
|
||||||
|
|
||||||
|
```rust
|
||||||
|
pub struct LegitimacyScorer {
|
||||||
|
media_type: MediaType,
|
||||||
|
iat_ewma: ExponentialMovingAverage,
|
||||||
|
iat_variance: ExponentialMovingVariance,
|
||||||
|
size_histogram: SizeBuckets<8>,
|
||||||
|
silence_count: u32,
|
||||||
|
speech_count: u32,
|
||||||
|
quality_reports_seen: u32,
|
||||||
|
keyframe_intervals: RingBuffer<u32, 16>,
|
||||||
|
window_start: Instant,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LegitimacyScorer {
|
||||||
|
pub fn observe(&mut self, header: &MediaHeader, payload_len: usize, now: Instant);
|
||||||
|
pub fn score(&self) -> f32; // [0, 1]
|
||||||
|
pub fn verdict(&self) -> Verdict; // Legitimate | Suspect | Abusive
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Cheap: a few floats and counters per session. Update on every packet, score every 1 s, escalate over 30+ s.
|
||||||
|
|
||||||
|
### Tier G — Reactive response
|
||||||
|
|
||||||
|
A scoring system needs a response policy:
|
||||||
|
|
||||||
|
| Verdict | Action |
|
||||||
|
|---|---|
|
||||||
|
| Legitimate | None |
|
||||||
|
| Suspect | Apply tighter Tier-E quota; emit `relay_conformance_suspect_total` |
|
||||||
|
| Abusive | Close session with `Hangup::PolicyViolation`; log to audit; cool-down fingerprint |
|
||||||
|
| Repeat-abusive | Lower-tier quota across the federation (gossip via federation channel) |
|
||||||
|
|
||||||
|
Never silent-drop. Always close with a typed reason so legitimate users hitting a bug get a clear error.
|
||||||
|
|
||||||
|
## Separating audio and video
|
||||||
|
|
||||||
|
**Yes — this is one of the strongest arguments for the v2 `MediaType` bit and should be a hard design rule.**
|
||||||
|
|
||||||
|
Audio and video have nothing in common statistically:
|
||||||
|
|
||||||
|
| Property | Audio | Video |
|
||||||
|
|---|---|---|
|
||||||
|
| Bitrate | 6–64 kbps | 100 kbps – 5 Mbps |
|
||||||
|
| Packet rate | 25–50 pps | 500–2000 pps |
|
||||||
|
| Packet size | 6–160 B | 200–1450 B |
|
||||||
|
| Burst structure | Clocked, near-CBR | Bursty (I-frames) |
|
||||||
|
| Silence | Common (10–40 %) | Meaningless |
|
||||||
|
| Loss tolerance | High (PLC, DRED) | Variable (keyframes critical) |
|
||||||
|
| Recovery primitive | FEC + DRED | NACK + PLI + keyframe cache |
|
||||||
|
|
||||||
|
A single scoring model trying to cover both would have to be so permissive at the union of envelopes that it would let tunnels through. **Separation is mandatory for Tier F to work.**
|
||||||
|
|
||||||
|
### What separation requires
|
||||||
|
|
||||||
|
1. **`MediaType:2` in `MediaHeader` v2** (already in `ROAD-TO-VIDEO.md` Phase V1). Without this, the relay must keep a `CodecID → MediaType` table and update it every time a codec is added — fragile.
|
||||||
|
2. **Per-`MediaType` conformance rules.** A and B and D have separate tables per type. Tier F has separate scorers.
|
||||||
|
3. **Per-`MediaType` quotas.** Tier E uses two buckets: `audio_bps_cap`, `video_bps_cap`. A session in audio-only mode never gets to spend the video budget. A video session has both, audio-priority.
|
||||||
|
4. **Per-`MediaType` keyframe/silence semantics.** `KeyFrame` bit is meaningless for audio; silence fraction is meaningless for video. The scorer needs to know which features apply.
|
||||||
|
|
||||||
|
### Bonus: separation also helps the SFU
|
||||||
|
|
||||||
|
Beyond abuse detection, the same separation makes graceful degradation cleaner: under congestion the relay can drop video packets first while preserving audio, because it knows which is which without parsing the codec table.
|
||||||
|
|
||||||
|
## Open questions for later decision
|
||||||
|
|
||||||
|
1. **Hard-close on first hard violation, or three-strikes?** Three-strikes is friendlier but lets twice the abuse through. Recommend hard-close + clear typed reason; legitimate users will reconnect, abusers won't try again at the same fingerprint.
|
||||||
|
2. **Where do verdicts persist?** In-memory per relay is simplest. Federated gossip is more powerful but a new attack surface (poisoning).
|
||||||
|
3. **Threshold tuning.** All thresholds in this doc are first-pass math. Real numbers come from a few weeks of Prometheus data on legitimate traffic before any enforcement turns on.
|
||||||
|
4. **Anonymous vs. authenticated split.** featherChat-authed users get generous quotas; anonymous users get tight ones. This makes the economics of mass abuse hostile (need many real identities) without locking out small legitimate use.
|
||||||
|
5. **What to log.** Conformance hits should be Prometheus counters + ringbuffer of recent violations; never log raw payload content (even encrypted) for privacy.
|
||||||
|
|
||||||
|
## Suggested implementation order (whenever this is picked up)
|
||||||
|
|
||||||
|
| Step | What | Why first |
|
||||||
|
|---|---|---|
|
||||||
|
| 1 | Land v2 wire format with `MediaType:2` | Prereq for separation; already on the road-to-video plan |
|
||||||
|
| 2 | Tier A + B + C as `wzp-relay/src/conformance.rs` | Kills bulk tunneling; cheap; no false positives if math is right |
|
||||||
|
| 3 | Prometheus metrics for violations + raw observables (IAT, size, silence frac) | Gather baseline of legitimate traffic before tightening |
|
||||||
|
| 4 | Tier D + E (size sanity + token bucket) | Defense in depth |
|
||||||
|
| 5 | Tier F scorer, audio-only first; tuned against the baseline from step 3 | Adds covert-tunnel pressure |
|
||||||
|
| 6 | Tier F video scorer once video is in production | Same shape, different features |
|
||||||
|
| 7 | Tier G response policy + audit log | Operationalize |
|
||||||
|
|
||||||
|
Steps 1–2 are decisive against the LiveKit-style PoC. The rest is steady tightening as real traffic accumulates.
|
||||||
|
|
||||||
|
## What this does NOT promise
|
||||||
|
|
||||||
|
- It does not stop a patient adversary running a slow covert channel inside real audio. Nothing E2E-preserving can.
|
||||||
|
- It does not detect content (no CSAM scan, no copyright fingerprint). Those would require breaking E2E and are out of scope by design.
|
||||||
|
- It does not eliminate abuse — it makes abuse loud, expensive, and detectable, which is the realistic goal for any E2E system.
|
||||||
169
vault/Architecture/Branch-Desktop-Audio-Rewrite.md
Normal file
169
vault/Architecture/Branch-Desktop-Audio-Rewrite.md
Normal file
@@ -0,0 +1,169 @@
|
|||||||
|
---
|
||||||
|
tags: [architecture, wzp]
|
||||||
|
type: architecture
|
||||||
|
---
|
||||||
|
|
||||||
|
# Branch: `feat/desktop-audio-rewrite`
|
||||||
|
|
||||||
|
Home of the Tauri desktop client for macOS, Windows, and Linux. Named "audio-rewrite" because the original driver was replacing a CPAL-only audio pipeline with platform-native backends that support OS-level echo cancellation (VoiceProcessingIO on macOS, WASAPI Communications on Windows), but the branch has grown into the full desktop story — Windows cross-compilation, vendored dependencies, history UI, direct calling, the whole thing.
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
The desktop client shares 100% of its frontend (`desktop/src/`) and Tauri command layer (`desktop/src-tauri/src/lib.rs`, `engine.rs`, `history.rs`) with the Android build on `android-rewrite`. Differences are limited to:
|
||||||
|
|
||||||
|
- **Audio backends**, which are platform-gated via Cargo target-dep sections in `desktop/src-tauri/Cargo.toml` and feature flags in `crates/wzp-client/Cargo.toml`.
|
||||||
|
- **Identity storage paths**, which resolve via Tauri's `app_data_dir()` (`~/Library/Application Support/…` on macOS, `%APPDATA%\…` on Windows, `~/.local/share/…` on Linux).
|
||||||
|
- **Build toolchains**: native `cargo build` on macOS/Linux, `cargo xwin` cross-compile from Linux for Windows via Docker on SepehrHomeserverdk.
|
||||||
|
|
||||||
|
## Audio backend matrix
|
||||||
|
|
||||||
|
| Target | Capture | Playback | AEC |
|
||||||
|
|---|---|---|---|
|
||||||
|
| macOS | CPAL (WASAPI/CoreAudio via cpal crate) OR VoiceProcessingIO (native Core Audio) | CPAL | VoiceProcessingIO native AEC (when `vpio` feature enabled) |
|
||||||
|
| Windows (default) | CPAL → WASAPI shared mode | CPAL → WASAPI shared mode | None |
|
||||||
|
| Windows (AEC build) | Direct WASAPI with `IAudioClient2::SetClientProperties(AudioCategory_Communications)` | CPAL → WASAPI shared mode | **OS-level**: Windows routes the capture stream through the driver's communications APO chain (AEC + NS + AGC) |
|
||||||
|
| Linux | CPAL → ALSA/PulseAudio | CPAL → ALSA/PulseAudio | None |
|
||||||
|
|
||||||
|
The macOS VPIO path is gated behind the `vpio` feature in `wzp-client` and the `coreaudio-rs` dep is itself `cfg(target_os = "macos")`, so enabling the feature on Windows or Linux is a no-op.
|
||||||
|
|
||||||
|
The Windows AEC path is gated behind the `windows-aec` feature, also target-gated (the `windows` crate dep is only pulled in on Windows), and re-exports `WasapiAudioCapture as AudioCapture` when enabled so downstream code doesn't need to know which backend is active. The current Windows build at `target/windows-exe/wzp-desktop.exe` has `windows-aec` on; a baseline noAEC build is preserved at `target/windows-exe/wzp-desktop-noAEC.exe` for A/B comparison on real hardware.
|
||||||
|
|
||||||
|
See [`BRANCH-android-rewrite.md`](BRANCH-android-rewrite.md) for Oboe audio on Android, which is its own story.
|
||||||
|
|
||||||
|
## Recent major work
|
||||||
|
|
||||||
|
### 1. Desktop direct calling feature (commit `2fd9465` and neighbors)
|
||||||
|
|
||||||
|
Brought direct 1:1 calls to macOS with full parity to the Android client:
|
||||||
|
|
||||||
|
- **Identity path fix**: the desktop `CallEngine::start` was loading seed from `$HOME/.wzp/identity` while `register_signal` used Tauri's `app_data_dir()`, producing two different fingerprints per run. Both now route through `load_or_create_seed()` which uses `app_data_dir()` everywhere.
|
||||||
|
- **Call history with dedup**: `history.rs` stores a `Vec<CallHistoryEntry>` with a `CallDirection` enum (`Placed | Received | Missed`). The `log` function dedupes by `call_id` so an outgoing call isn't logged twice as "missed" (when the signal loop's `DirectCallOffer` handler fires) and then again as "placed" (when `place_call` returns). Instead the entry is updated in place.
|
||||||
|
- **Recent contacts row**: a horizontal chip UI in the direct-call panel showing the last N peers with friendly aliases, clickable to re-dial.
|
||||||
|
- **Deregister button**: lets a user drop their signal registration without quitting the app, useful when switching identities.
|
||||||
|
- **Random alias derivation**: a new client sees a human-friendly alias like "silent-forest-41" derived deterministically from its seed, so it's identifiable in the UI before manual naming.
|
||||||
|
- **Default room "general"** instead of "android", since the desktop client is not Android.
|
||||||
|
|
||||||
|
### 2. macOS VoiceProcessingIO integration
|
||||||
|
|
||||||
|
`crates/wzp-client/src/audio_vpio.rs` — a native Core Audio implementation using `AUGraph` + `AudioComponentInstance` with the VPIO audio unit. Gives you hardware-accelerated AEC (same AEC Apple ships in FaceTime / iMessage audio / voice memos) at the cost of tight coupling to Apple frameworks. Lock-free ring pattern matches the CPAL path so the upper layers don't notice the difference.
|
||||||
|
|
||||||
|
Enabled by `features = ["audio", "vpio"]` in the macOS target section of `desktop/src-tauri/Cargo.toml`.
|
||||||
|
|
||||||
|
### 3. Windows cross-compilation via cargo-xwin
|
||||||
|
|
||||||
|
Cross-compiling Rust + Tauri to `x86_64-pc-windows-msvc` from Linux using `cargo-xwin`, which downloads the Microsoft CRT + Windows SDK on demand and drives `clang-cl` as the compiler. No Windows machine is needed for the build itself — only for runtime testing.
|
||||||
|
|
||||||
|
**Build infrastructure**:
|
||||||
|
|
||||||
|
- `scripts/Dockerfile.windows-builder` — Debian bookworm + Rust + cargo-xwin + Node 20 + cmake + ninja + llvm + clang + lld + nasm. Pre-warms the xwin MSVC CRT cache at image build time (saves ~4 minutes per cold build).
|
||||||
|
- `scripts/build-windows-docker.sh` — fire-and-forget remote build via Docker on SepehrHomeserverdk. Same pattern as `build-tauri-android.sh`. Uploads the `.exe` to rustypaste and fires an `ntfy.sh/wzp` notification on start and on completion.
|
||||||
|
- `scripts/build-windows-cloud.sh` — alternative pipeline using a temporary Hetzner Cloud VPS. Slower (full VM spin-up), more expensive, but useful when Docker image rebuilds would be disruptive.
|
||||||
|
|
||||||
|
**Two critical blockers resolved** on the way to a working `.exe`:
|
||||||
|
|
||||||
|
1. **libopus SSE4.1 / SSSE3 intrinsic compile failure**. `audiopus_sys` vendors libopus 1.3.1, whose `CMakeLists.txt` gates the per-file `-msse4.1` `COMPILE_FLAGS` behind `if(NOT MSVC)`. Under `clang-cl`, CMake sets `MSVC=1` (because `CMAKE_C_COMPILER_FRONTEND_VARIANT=MSVC` triggers `Platform/Windows-MSVC.cmake` which unconditionally sets the variable), so the per-file flag is never set and the SSE4.1 source files compile without the target feature — then fail with 20+ "always_inline function '_mm_cvtepi16_epi32' requires target feature 'sse4.1'" errors.
|
||||||
|
|
||||||
|
Fixed by **vendoring audiopus_sys into `vendor/audiopus_sys/`** and patching its bundled libopus to introduce an `MSVC_CL` variable that is true only for real `cl.exe` (distinguished via `CMAKE_C_COMPILER_ID STREQUAL "MSVC"`). The eight `if(NOT MSVC)` SIMD guards are flipped to `if(NOT MSVC_CL)` and the global `/arch` block at line 445 becomes `if(MSVC_CL)`, so clang-cl gets the GCC-style per-file flags while real cl.exe keeps the `/arch:AVX` / `/arch:SSE2` globals.
|
||||||
|
|
||||||
|
Wired in via `[patch.crates-io] audiopus_sys = { path = "vendor/audiopus_sys" }` at the workspace root.
|
||||||
|
|
||||||
|
Upstream tracking: [xiph/opus#256](https://github.com/xiph/opus/issues/256), [xiph/opus PR #257](https://github.com/xiph/opus/pull/257) (both stale).
|
||||||
|
|
||||||
|
2. **tauri-build needs `icons/icon.ico` for the Windows PE resource**. The desktop only had `icon.png`. Generated a multi-size ICO (16/24/32/48/64/128/256) from the existing placeholder via Pillow and committed it. Placeholder quality — real branded icons can replace it later.
|
||||||
|
|
||||||
|
### 4. Windows `AudioCategory_Communications` capture path (task #24)
|
||||||
|
|
||||||
|
`crates/wzp-client/src/audio_wasapi.rs` — direct WASAPI capture via `IMMDeviceEnumerator → IAudioClient2 → SetClientProperties` with `AudioCategory_Communications`. This tells Windows "this is a VoIP call" and Windows routes the capture stream through the driver's registered communications APO chain, which on most Win10/11 consumer hardware includes AEC, NS, and AGC.
|
||||||
|
|
||||||
|
**Caveat**: quality is driver-dependent. On a machine with a good communications APO (Intel Smart Sound, Dolby, modern Realtek on Win11 24H2+, anything with Voice Clarity enabled) it's excellent. On generic class-compliant drivers with no communications APO registered, it's a no-op. For a guaranteed AEC regardless of driver, see task #26 which tracks implementing the classic Voice Capture DSP (`CLSID_CWMAudioAEC`) as a fallback.
|
||||||
|
|
||||||
|
Gated behind the `windows-aec` feature in `wzp-client`. Enabled by default in the Windows target section of `desktop/src-tauri/Cargo.toml`.
|
||||||
|
|
||||||
|
## Build pipelines
|
||||||
|
|
||||||
|
### Native macOS / Linux
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd desktop
|
||||||
|
npm install
|
||||||
|
npm run build
|
||||||
|
cd src-tauri
|
||||||
|
cargo build --release --bin wzp-desktop
|
||||||
|
```
|
||||||
|
|
||||||
|
### Windows x86_64 via Docker on SepehrHomeserverdk
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/build-windows-docker.sh # Full: pull + build + download
|
||||||
|
./scripts/build-windows-docker.sh --no-pull # Skip git fetch
|
||||||
|
./scripts/build-windows-docker.sh --rust # Force-clean Rust target
|
||||||
|
./scripts/build-windows-docker.sh --image-build # (Re)build the Docker image (fire-and-forget)
|
||||||
|
```
|
||||||
|
|
||||||
|
Output lands at `target/windows-exe/wzp-desktop.exe`. Both `wzp-desktop.exe` and `wzp-desktop-noAEC.exe` can coexist in that directory; the script writes `wzp-desktop.exe` so renaming the prior build to `-noAEC.exe` (or any other name) before rebuilding preserves it.
|
||||||
|
|
||||||
|
### Windows x86_64 via Hetzner Cloud (alternative)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/build-windows-cloud.sh # Full: create VM → build → download → destroy
|
||||||
|
./scripts/build-windows-cloud.sh --prepare # Create VM and install deps only
|
||||||
|
./scripts/build-windows-cloud.sh --build # Build on existing VM
|
||||||
|
./scripts/build-windows-cloud.sh --destroy # Delete the VM
|
||||||
|
WZP_KEEP_VM=1 ./scripts/build-windows-cloud.sh # Keep VM alive after build for debug
|
||||||
|
```
|
||||||
|
|
||||||
|
Remember to destroy the VM at end of day with `--destroy`.
|
||||||
|
|
||||||
|
### Linux x86_64 (relay + CLI + bench)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./scripts/build-linux-docker.sh # Fire-and-forget remote Docker build
|
||||||
|
./scripts/build-linux-docker.sh --install # Wait for completion and download
|
||||||
|
```
|
||||||
|
|
||||||
|
Uses the same `wzp-android-builder` Docker image as Android (not a separate image), since the deps (Rust + cmake + ring prereqs) are the same.
|
||||||
|
|
||||||
|
## Testing
|
||||||
|
|
||||||
|
### Direct calling parity
|
||||||
|
|
||||||
|
1. Build on two machines (macOS + Windows, or two macOS, or any combination).
|
||||||
|
2. Both machines register on the same relay.
|
||||||
|
3. Copy one machine's fingerprint into the other's direct-call panel.
|
||||||
|
4. Place the call. Confirm ringing UI on the callee and "calling…" UI on the caller.
|
||||||
|
5. Answer. Confirm audio flows both ways.
|
||||||
|
6. Hang up from either side. Confirm call-history entries are labeled correctly (`Outgoing` on caller, `Incoming` on callee, never `Missed` on a successful call).
|
||||||
|
|
||||||
|
### Windows AEC A/B
|
||||||
|
|
||||||
|
1. Install `wzp-desktop-noAEC.exe` and `wzp-desktop.exe` on the same Windows box.
|
||||||
|
2. Join a call from each (separately) while a second machine plays known audio through the first machine's speakers.
|
||||||
|
3. On the remote (listening) side: the `noAEC` call should have clear audible echo; the AEC call should have minimal or no echo after a 1–2 s convergence period.
|
||||||
|
4. If both builds sound identical (with echo) → the `AudioCategory_Communications` switch isn't triggering the driver's APO chain. Investigate via task #26 (Voice Capture DSP fallback).
|
||||||
|
|
||||||
|
## Known quirks
|
||||||
|
|
||||||
|
1. **libopus vendor path is workspace-relative**. `[patch.crates-io] audiopus_sys = { path = "vendor/audiopus_sys" }` works from any crate in the workspace because Cargo resolves it against the root `Cargo.toml`'s directory. If the workspace is moved or vendored into another workspace, update the path.
|
||||||
|
|
||||||
|
2. **`cargo xwin` overwrites `override.cmake` on every invocation**. Any attempt to patch `~/.cache/cargo-xwin/cmake/clang-cl/override.cmake` at Docker image build time is inert because `src/compiler/clang_cl.rs` line ~444 writes the bundled file fresh on every run. All real fixes must land in the source tree (via the vendored audiopus_sys, as done here), not in the cargo-xwin cache.
|
||||||
|
|
||||||
|
3. **WebView2 runtime is a prerequisite on Windows 10**. Windows 11 ships with it. If the `.exe` launches and immediately exits with no error on a Win10 machine, that's the missing runtime — install it from [Microsoft's Evergreen bootstrapper](https://developer.microsoft.com/en-us/microsoft-edge/webview2/).
|
||||||
|
|
||||||
|
4. **Rust 2024 edition `unsafe_op_in_unsafe_fn` lint**. The WASAPI backend in `audio_wasapi.rs` emits ~18 of these warnings because Rust 2024 requires explicit `unsafe { ... }` blocks inside `unsafe fn` bodies. The warnings don't block the build and don't affect runtime behavior; cleaning them up is tracked informally as tech debt.
|
||||||
|
|
||||||
|
## Files of interest
|
||||||
|
|
||||||
|
| Path | Purpose |
|
||||||
|
|---|---|
|
||||||
|
| `desktop/src/` | Shared frontend (TypeScript + HTML + CSS) |
|
||||||
|
| `desktop/src-tauri/src/lib.rs` | Tauri commands shared with Android |
|
||||||
|
| `desktop/src-tauri/src/engine.rs` | `CallEngine` wrapper |
|
||||||
|
| `desktop/src-tauri/src/history.rs` | Persistent call history store with dedup |
|
||||||
|
| `crates/wzp-client/src/audio_io.rs` | CPAL capture + playback (baseline) |
|
||||||
|
| `crates/wzp-client/src/audio_vpio.rs` | macOS VoiceProcessingIO capture (AEC) |
|
||||||
|
| `crates/wzp-client/src/audio_wasapi.rs` | Windows WASAPI communications capture (AEC) |
|
||||||
|
| `vendor/audiopus_sys/opus/CMakeLists.txt` | Patched libopus for clang-cl SIMD |
|
||||||
|
| `scripts/Dockerfile.windows-builder` | Windows cross-compile Docker image |
|
||||||
|
| `scripts/build-windows-docker.sh` | Remote Docker build pipeline |
|
||||||
|
| `scripts/build-windows-cloud.sh` | Hetzner VPS alternative pipeline |
|
||||||
|
| `scripts/build-linux-docker.sh` | Linux x86_64 relay/CLI build pipeline |
|
||||||
666
vault/Architecture/Design.md
Normal file
666
vault/Architecture/Design.md
Normal file
@@ -0,0 +1,666 @@
|
|||||||
|
---
|
||||||
|
tags: [architecture, wzp]
|
||||||
|
type: architecture
|
||||||
|
---
|
||||||
|
|
||||||
|
# WarzonePhone Design Document
|
||||||
|
|
||||||
|
> Custom encrypted VoIP protocol built in Rust. Designed for hostile network conditions: 5-70% packet loss, 100-500 kbps throughput, 300-800 ms RTT. Multi-platform: Desktop (Tauri), Android, CLI, Web.
|
||||||
|
|
||||||
|
## System Overview
|
||||||
|
|
||||||
|
WarzonePhone is a voice-over-IP system built from scratch in Rust, targeting reliable encrypted voice communication over severely degraded networks. The protocol uses adaptive codecs (Opus + Codec2), fountain-code FEC (RaptorQ), and end-to-end ChaCha20-Poly1305 encryption over a QUIC transport layer.
|
||||||
|
|
||||||
|
The system comprises three categories of components:
|
||||||
|
|
||||||
|
1. **Protocol crates** -- a Rust workspace of 7 library crates with a star dependency graph enabling parallel development
|
||||||
|
2. **Client applications** -- Desktop (Tauri), Android (Kotlin + JNI), CLI, and Web (browser bridge)
|
||||||
|
3. **Relay infrastructure** -- SFU relay daemons with federation, health probing, and Prometheus metrics
|
||||||
|
|
||||||
|
### Design Principles
|
||||||
|
|
||||||
|
- **User sovereignty** -- client-driven route selection, BIP39 identity backup, no central authority
|
||||||
|
- **End-to-end encryption** -- relays never see plaintext audio; SFU forwarding preserves E2E encryption
|
||||||
|
- **Adaptive resilience** -- automatic codec and FEC switching based on observed network quality
|
||||||
|
- **Parallel development** -- star dependency graph allows 5 agents/developers to work simultaneously with zero merge conflicts
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
### Crate Overview
|
||||||
|
|
||||||
|
The workspace contains 7 core crates plus integration binaries:
|
||||||
|
|
||||||
|
| Crate | Purpose | Key Dependencies |
|
||||||
|
|-------|---------|-----------------|
|
||||||
|
| `wzp-proto` | Protocol types, traits, wire format | serde, bytes |
|
||||||
|
| `wzp-codec` | Audio codecs (Opus, Codec2, RNNoise) | audiopus, codec2, nnnoiseless |
|
||||||
|
| `wzp-fec` | Forward error correction | raptorq |
|
||||||
|
| `wzp-crypto` | Cryptography and identity | ed25519-dalek, x25519-dalek, chacha20poly1305, bip39 |
|
||||||
|
| `wzp-transport` | QUIC transport layer | quinn, rustls |
|
||||||
|
| `wzp-relay` | Relay daemon (SFU, federation, metrics) | tokio, prometheus |
|
||||||
|
| `wzp-client` | Call engine and CLI | All above |
|
||||||
|
|
||||||
|
Additional integration targets: `wzp-web` (browser bridge via WebSocket), Android native library (JNI), Desktop (Tauri).
|
||||||
|
|
||||||
|
### Dependency Graph
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
graph TD
|
||||||
|
PROTO["wzp-proto<br/>(Types, Traits, Wire Format)"]
|
||||||
|
|
||||||
|
CODEC["wzp-codec<br/>(Opus + Codec2 + RNNoise)"]
|
||||||
|
FEC["wzp-fec<br/>(RaptorQ FEC)"]
|
||||||
|
CRYPTO["wzp-crypto<br/>(ChaCha20 + Identity)"]
|
||||||
|
TRANSPORT["wzp-transport<br/>(QUIC / Quinn)"]
|
||||||
|
|
||||||
|
RELAY["wzp-relay<br/>(Relay Daemon)"]
|
||||||
|
CLIENT["wzp-client<br/>(CLI + Call Engine)"]
|
||||||
|
WEB["wzp-web<br/>(Browser Bridge)"]
|
||||||
|
DESKTOP["Desktop<br/>(Tauri + CPAL)"]
|
||||||
|
ANDROID["Android<br/>(Kotlin + JNI)"]
|
||||||
|
|
||||||
|
PROTO --> CODEC
|
||||||
|
PROTO --> FEC
|
||||||
|
PROTO --> CRYPTO
|
||||||
|
PROTO --> TRANSPORT
|
||||||
|
|
||||||
|
CODEC --> CLIENT
|
||||||
|
FEC --> CLIENT
|
||||||
|
CRYPTO --> CLIENT
|
||||||
|
TRANSPORT --> CLIENT
|
||||||
|
|
||||||
|
CODEC --> RELAY
|
||||||
|
FEC --> RELAY
|
||||||
|
CRYPTO --> RELAY
|
||||||
|
TRANSPORT --> RELAY
|
||||||
|
|
||||||
|
CLIENT --> WEB
|
||||||
|
CLIENT --> DESKTOP
|
||||||
|
CLIENT --> ANDROID
|
||||||
|
TRANSPORT --> WEB
|
||||||
|
|
||||||
|
FC["warzone-protocol<br/>(featherChat Identity)"] -.->|path dep| CRYPTO
|
||||||
|
|
||||||
|
style PROTO fill:#6c5ce7,color:#fff
|
||||||
|
style RELAY fill:#ff9f43,color:#fff
|
||||||
|
style CLIENT fill:#00b894,color:#fff
|
||||||
|
style WEB fill:#0984e3,color:#fff
|
||||||
|
style DESKTOP fill:#0984e3,color:#fff
|
||||||
|
style ANDROID fill:#0984e3,color:#fff
|
||||||
|
style FC fill:#fd79a8,color:#fff
|
||||||
|
```
|
||||||
|
|
||||||
|
The star pattern ensures each leaf crate (`wzp-codec`, `wzp-fec`, `wzp-crypto`, `wzp-transport`) depends only on `wzp-proto` and never on each other. This enables:
|
||||||
|
|
||||||
|
- **Parallel development** -- 5 agents work on 5 crates with no merge conflicts
|
||||||
|
- **Independent testing** -- each crate has self-contained tests
|
||||||
|
- **Pluggability** -- any implementation can be swapped by implementing the same trait
|
||||||
|
- **Fast compilation** -- changing one leaf only recompiles that leaf and integration crates
|
||||||
|
|
||||||
|
## Audio Pipeline
|
||||||
|
|
||||||
|
### Encode Pipeline (Mic to Network)
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
sequenceDiagram
|
||||||
|
participant Mic as Microphone
|
||||||
|
participant RNN as RNNoise Denoise
|
||||||
|
participant VAD as Silence Detector
|
||||||
|
participant ENC as Opus/Codec2 Encode
|
||||||
|
participant FEC as RaptorQ FEC Encode
|
||||||
|
participant INT as Interleaver
|
||||||
|
participant HDR as Header Assembly
|
||||||
|
participant CRYPT as ChaCha20-Poly1305
|
||||||
|
participant QUIC as QUIC Datagram
|
||||||
|
|
||||||
|
Mic->>RNN: PCM i16 x 960 (20ms @ 48kHz)
|
||||||
|
RNN->>VAD: Denoised samples (2 x 480)
|
||||||
|
alt Silence detected (>100ms)
|
||||||
|
VAD->>ENC: ComfortNoise packet (every 200ms)
|
||||||
|
else Active speech or hangover
|
||||||
|
VAD->>ENC: Active audio frame
|
||||||
|
end
|
||||||
|
ENC->>FEC: Compressed frame (padded to 256 bytes)
|
||||||
|
FEC->>FEC: Accumulate block (5-10 frames)
|
||||||
|
FEC->>INT: Source + repair symbols
|
||||||
|
INT->>HDR: Interleaved packets (depth=3)
|
||||||
|
HDR->>CRYPT: MediaHeader (12B) or MiniHeader (4B)
|
||||||
|
CRYPT->>QUIC: Header=AAD, Payload=encrypted
|
||||||
|
```
|
||||||
|
|
||||||
|
### Decode Pipeline (Network to Speaker)
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
sequenceDiagram
|
||||||
|
participant QUIC as QUIC Datagram
|
||||||
|
participant CRYPT as ChaCha20-Poly1305
|
||||||
|
participant HDR as Header Parse
|
||||||
|
participant DEINT as De-interleaver
|
||||||
|
participant FEC as RaptorQ FEC Decode
|
||||||
|
participant JIT as Jitter Buffer
|
||||||
|
participant DEC as Opus/Codec2 Decode
|
||||||
|
participant SPK as Speaker
|
||||||
|
|
||||||
|
QUIC->>CRYPT: Encrypted packet
|
||||||
|
CRYPT->>HDR: Decrypt (header=AAD)
|
||||||
|
HDR->>DEINT: Parsed MediaHeader + payload
|
||||||
|
DEINT->>FEC: Reordered symbols
|
||||||
|
FEC->>FEC: Reconstruct from any K of K+R symbols
|
||||||
|
FEC->>JIT: Recovered audio frames
|
||||||
|
JIT->>JIT: Sequence-ordered BTreeMap
|
||||||
|
JIT->>DEC: Pop when depth >= target
|
||||||
|
DEC->>SPK: PCM i16 x 960
|
||||||
|
```
|
||||||
|
|
||||||
|
## Codec System
|
||||||
|
|
||||||
|
WarzonePhone uses a dual-codec architecture to cover the full range of network conditions:
|
||||||
|
|
||||||
|
### Opus (Primary)
|
||||||
|
|
||||||
|
Opus is the primary codec for normal to degraded conditions. It operates at 48 kHz natively with built-in inband FEC and DTX (discontinuous transmission). The `audiopus` crate provides mature Rust bindings to libopus.
|
||||||
|
|
||||||
|
| Profile | Bitrate | Frame Duration | FEC Ratio | Total Bandwidth | Use Case |
|
||||||
|
|---------|---------|---------------|-----------|----------------|----------|
|
||||||
|
| Studio 64k | 64 kbps | 20ms | 10% | 70.4 kbps | LAN, excellent WiFi |
|
||||||
|
| Studio 48k | 48 kbps | 20ms | 10% | 52.8 kbps | Good WiFi, wired |
|
||||||
|
| Studio 32k | 32 kbps | 20ms | 10% | 35.2 kbps | WiFi, LTE |
|
||||||
|
| Good (24k) | 24 kbps | 20ms | 20% | 28.8 kbps | WiFi, LTE, decent links |
|
||||||
|
| Opus 16k | 16 kbps | 20ms | 20% | 19.2 kbps | 3G, moderate congestion |
|
||||||
|
| Degraded (6k) | 6 kbps | 40ms | 50% | 9.0 kbps | 3G, congested WiFi |
|
||||||
|
|
||||||
|
### Codec2 (Fallback)
|
||||||
|
|
||||||
|
Codec2 is a narrowband vocoder designed for HF radio links with extreme bandwidth constraints. It operates at 8 kHz, and the adaptive layer handles 48 kHz <-> 8 kHz resampling transparently. The pure-Rust `codec2` crate means no C dependencies.
|
||||||
|
|
||||||
|
| Profile | Bitrate | Frame Duration | FEC Ratio | Total Bandwidth | Use Case |
|
||||||
|
|---------|---------|---------------|-----------|----------------|----------|
|
||||||
|
| Codec2 3200 | 3.2 kbps | 20ms | 50% | 4.8 kbps | Poor conditions |
|
||||||
|
| Catastrophic (1200) | 1.2 kbps | 40ms | 100% | 2.4 kbps | Satellite, extreme loss |
|
||||||
|
|
||||||
|
### ComfortNoise
|
||||||
|
|
||||||
|
When the silence detector identifies no speech activity for over 100ms, the encoder switches to emitting a ComfortNoise packet every 200ms instead of encoding silence. This provides approximately 50% bandwidth savings in typical conversations.
|
||||||
|
|
||||||
|
### Adaptive Switching
|
||||||
|
|
||||||
|
The `AdaptiveEncoder`/`AdaptiveDecoder` in `wzp-codec` hold both codec instances and switch between them based on the active `QualityProfile`. This avoids codec re-initialization latency during tier transitions. The `AdaptiveQualityController` in `wzp-proto` manages tier transitions with hysteresis:
|
||||||
|
|
||||||
|
- **Downgrade**: 3 consecutive bad reports (2 on cellular networks)
|
||||||
|
- **Upgrade**: 10 consecutive good reports (one tier at a time)
|
||||||
|
- **Network handoff**: WiFi-to-cellular switch triggers preemptive one-tier downgrade plus a temporary 10-second FEC boost (+20%)
|
||||||
|
|
||||||
|
Quality tier classification thresholds:
|
||||||
|
|
||||||
|
| Tier | WiFi/Unknown | Cellular |
|
||||||
|
|------|-------------|----------|
|
||||||
|
| Good | loss < 10%, RTT < 400ms | loss < 8%, RTT < 300ms |
|
||||||
|
| Degraded | loss 10-40%, RTT 400-600ms | loss 8-25%, RTT 300-500ms |
|
||||||
|
| Catastrophic | loss > 40%, RTT > 600ms | loss > 25%, RTT > 500ms |
|
||||||
|
|
||||||
|
## Forward Error Correction (FEC)
|
||||||
|
|
||||||
|
### Why RaptorQ Over Reed-Solomon
|
||||||
|
|
||||||
|
WarzonePhone uses RaptorQ (RFC 6330) fountain codes via the `raptorq` crate:
|
||||||
|
|
||||||
|
1. **Rateless** -- generate arbitrary repair symbols on the fly; if conditions worsen mid-block, generate additional repair without re-encoding
|
||||||
|
2. **Efficient decoding** -- decode from any K symbols with high probability (typically K + 1 or K + 2 suffice)
|
||||||
|
3. **Lower complexity** -- O(K) encoding/decoding time vs O(K^2) for Reed-Solomon
|
||||||
|
4. **Variable block sizes** -- 1-56,403 source symbols per block (WZP uses 5-10)
|
||||||
|
|
||||||
|
### FEC Block Structure
|
||||||
|
|
||||||
|
Each FEC block consists of 5-10 audio frames padded to 256-byte symbols with a 2-byte LE length prefix:
|
||||||
|
|
||||||
|
```
|
||||||
|
[len:u16 LE][audio_frame][zero_padding_to_256_bytes]
|
||||||
|
```
|
||||||
|
|
||||||
|
### Loss Survival by FEC Ratio
|
||||||
|
|
||||||
|
With 5 source frames per block:
|
||||||
|
|
||||||
|
| FEC Ratio | Repair Symbols | Survives Loss | Profile |
|
||||||
|
|-----------|---------------|---------------|---------|
|
||||||
|
| 10% | 1 | 1 of 6 (16.7%) | Studio |
|
||||||
|
| 20% | 1 | 1 of 6 (16.7%) | Good |
|
||||||
|
| 50% | 3 | 3 of 8 (37.5%) | Degraded |
|
||||||
|
| 100% | 5 | 5 of 10 (50.0%) | Catastrophic |
|
||||||
|
|
||||||
|
### Interleaving
|
||||||
|
|
||||||
|
Burst loss protection via depth-3 interleaving: packets from 3 consecutive FEC blocks are interleaved before transmission. A burst of 3 consecutive lost packets affects 3 different blocks (1 loss each) rather than destroying 1 block entirely.
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
graph LR
|
||||||
|
subgraph "FEC Encoder"
|
||||||
|
F1[Frame 1] --> BLK[Source Block<br/>5-10 frames]
|
||||||
|
F2[Frame 2] --> BLK
|
||||||
|
F3[Frame 3] --> BLK
|
||||||
|
F4[Frame 4] --> BLK
|
||||||
|
F5[Frame 5] --> BLK
|
||||||
|
BLK --> SRC[Source Symbols]
|
||||||
|
BLK --> REP[Repair Symbols<br/>ratio-dependent]
|
||||||
|
SRC --> INT[Interleaver<br/>depth=3]
|
||||||
|
REP --> INT
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph "Network"
|
||||||
|
INT --> LOSS{Packet Loss}
|
||||||
|
LOSS -->|some lost| RCV[Received Symbols]
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph "FEC Decoder"
|
||||||
|
RCV --> DEINT[De-interleaver]
|
||||||
|
DEINT --> RAPTORQ[RaptorQ Decode<br/>Any K of K+R]
|
||||||
|
RAPTORQ --> OUT[Original Frames]
|
||||||
|
end
|
||||||
|
|
||||||
|
style LOSS fill:#e17055,color:#fff
|
||||||
|
style RAPTORQ fill:#00b894,color:#fff
|
||||||
|
```
|
||||||
|
|
||||||
|
## Transport Layer
|
||||||
|
|
||||||
|
### Why QUIC Over Raw UDP
|
||||||
|
|
||||||
|
WarzonePhone uses QUIC (via the `quinn` crate) rather than raw UDP for several reasons:
|
||||||
|
|
||||||
|
| Feature | Benefit |
|
||||||
|
|---------|---------|
|
||||||
|
| DATAGRAM frames (RFC 9221) | Unreliable delivery without head-of-line blocking -- behaves like UDP for media |
|
||||||
|
| Reliable streams | Multiplexed signaling (CallOffer, Hangup, Rekey) without a separate TCP connection |
|
||||||
|
| Congestion control | Prevents overwhelming degraded links, important when chaining relays |
|
||||||
|
| Connection migration | Connections survive IP address changes (WiFi to cellular handoff) |
|
||||||
|
| TLS 1.3 built-in | Transport-level encryption protects headers and signaling |
|
||||||
|
| NAT keepalive | 5-second interval maintains NAT bindings without application-level pings |
|
||||||
|
| Firewall traversal | Runs on UDP port 443 with `wzp` ALPN identifier |
|
||||||
|
|
||||||
|
The tradeoff is approximately 20-40 bytes of additional per-packet overhead compared to raw UDP.
|
||||||
|
|
||||||
|
### Wire Formats
|
||||||
|
|
||||||
|
#### MediaHeader (12 bytes)
|
||||||
|
|
||||||
|
```
|
||||||
|
Byte 0: [V:1][T:1][CodecID:4][Q:1][FecRatioHi:1]
|
||||||
|
Byte 1: [FecRatioLo:6][unused:2]
|
||||||
|
Bytes 2-3: sequence (u16 BE)
|
||||||
|
Bytes 4-7: timestamp_ms (u32 BE)
|
||||||
|
Byte 8: fec_block_id (u8)
|
||||||
|
Byte 9: fec_symbol_idx (u8)
|
||||||
|
Byte 10: reserved
|
||||||
|
Byte 11: csrc_count
|
||||||
|
|
||||||
|
V = version (0), T = is_repair, CodecID = codec, Q = quality_report appended
|
||||||
|
```
|
||||||
|
|
||||||
|
#### MiniHeader (4 bytes, compressed)
|
||||||
|
|
||||||
|
```
|
||||||
|
Bytes 0-1: timestamp_delta_ms (u16 BE)
|
||||||
|
Bytes 2-3: payload_len (u16 BE)
|
||||||
|
|
||||||
|
Preceded by FRAME_TYPE_MINI (0x01). Full header every 50 frames (~1s).
|
||||||
|
Saves 8 bytes/packet (67% header reduction).
|
||||||
|
```
|
||||||
|
|
||||||
|
#### TrunkFrame (batched datagrams)
|
||||||
|
|
||||||
|
```
|
||||||
|
[count:u16]
|
||||||
|
[session_id:2][len:u16][payload:len] x count
|
||||||
|
|
||||||
|
Packs multiple session packets into one QUIC datagram.
|
||||||
|
Max 10 entries or 1200 bytes, flushed every 5ms.
|
||||||
|
```
|
||||||
|
|
||||||
|
#### QualityReport (4 bytes, optional trailer)
|
||||||
|
|
||||||
|
```
|
||||||
|
Byte 0: loss_pct (0-255 maps to 0-100%)
|
||||||
|
Byte 1: rtt_4ms (0-255 maps to 0-1020ms)
|
||||||
|
Byte 2: jitter_ms
|
||||||
|
Byte 3: bitrate_cap_kbps
|
||||||
|
```
|
||||||
|
|
||||||
|
### Bandwidth Summary
|
||||||
|
|
||||||
|
| Profile | Audio | FEC Overhead | Total | Silence Savings |
|
||||||
|
|---------|-------|-------------|-------|----------------|
|
||||||
|
| Studio 64k | 64 kbps | 10% = 6.4 kbps | **70.4 kbps** | ~50% with DTX |
|
||||||
|
| Studio 48k | 48 kbps | 10% = 4.8 kbps | **52.8 kbps** | ~50% with DTX |
|
||||||
|
| Studio 32k | 32 kbps | 10% = 3.2 kbps | **35.2 kbps** | ~50% with DTX |
|
||||||
|
| Good (24k) | 24 kbps | 20% = 4.8 kbps | **28.8 kbps** | ~50% with DTX |
|
||||||
|
| Degraded (6k) | 6 kbps | 50% = 3.0 kbps | **9.0 kbps** | ~50% with DTX |
|
||||||
|
| Catastrophic (1.2k) | 1.2 kbps | 100% = 1.2 kbps | **2.4 kbps** | ~50% with DTX |
|
||||||
|
|
||||||
|
Additional savings: MiniHeaders save 8 bytes/packet (67% header reduction). Trunking shares QUIC overhead across multiplexed sessions.
|
||||||
|
|
||||||
|
## Security
|
||||||
|
|
||||||
|
### Identity Model
|
||||||
|
|
||||||
|
Every user has a persistent identity derived from a 32-byte seed:
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
graph TD
|
||||||
|
SEED["32-byte Seed<br/>(BIP39 Mnemonic: 24 words)"] --> HKDF1["HKDF<br/>info='warzone-ed25519'"]
|
||||||
|
SEED --> HKDF2["HKDF<br/>info='warzone-x25519'"]
|
||||||
|
|
||||||
|
HKDF1 --> ED["Ed25519 SigningKey<br/>(Digital Signatures)"]
|
||||||
|
HKDF2 --> X25519["X25519 StaticSecret<br/>(Key Agreement)"]
|
||||||
|
|
||||||
|
ED --> VKEY["Ed25519 VerifyingKey<br/>(Public)"]
|
||||||
|
X25519 --> XPUB["X25519 PublicKey<br/>(Public)"]
|
||||||
|
|
||||||
|
VKEY --> FP["Fingerprint<br/>SHA-256(pubkey), truncated 16 bytes<br/>xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:xxxx"]
|
||||||
|
|
||||||
|
style SEED fill:#6c5ce7,color:#fff
|
||||||
|
style FP fill:#fd79a8,color:#fff
|
||||||
|
style ED fill:#ee5a24,color:#fff
|
||||||
|
style X25519 fill:#00b894,color:#fff
|
||||||
|
```
|
||||||
|
|
||||||
|
**BIP39 Mnemonic Backup**: The 32-byte seed can be encoded as a 24-word BIP39 mnemonic for human-readable backup. The same seed produces the same identity on any platform.
|
||||||
|
|
||||||
|
**featherChat Compatibility**: The identity derivation is compatible with the Warzone messenger (featherChat), allowing a shared identity across messaging and calling.
|
||||||
|
|
||||||
|
### Cryptographic Handshake
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
sequenceDiagram
|
||||||
|
participant C as Caller
|
||||||
|
participant R as Relay / Callee
|
||||||
|
|
||||||
|
Note over C: Derive identity from seed<br/>Ed25519 + X25519 via HKDF
|
||||||
|
|
||||||
|
C->>C: Generate ephemeral X25519 keypair
|
||||||
|
C->>C: Sign(ephemeral_pub || "call-offer")
|
||||||
|
C->>R: CallOffer { identity_pub, ephemeral_pub, signature, profiles }
|
||||||
|
|
||||||
|
R->>R: Verify Ed25519 signature
|
||||||
|
R->>R: Generate ephemeral X25519 keypair
|
||||||
|
R->>R: shared_secret = DH(eph_b, eph_a)
|
||||||
|
R->>R: session_key = HKDF(shared_secret, "warzone-session-key")
|
||||||
|
R->>R: Sign(ephemeral_pub || "call-answer")
|
||||||
|
R->>C: CallAnswer { identity_pub, ephemeral_pub, signature, profile }
|
||||||
|
|
||||||
|
C->>C: Verify signature
|
||||||
|
C->>C: shared_secret = DH(eph_a, eph_b)
|
||||||
|
C->>C: session_key = HKDF(shared_secret)
|
||||||
|
|
||||||
|
Note over C,R: Both have identical ChaCha20-Poly1305 session key
|
||||||
|
C->>R: Encrypted media (QUIC datagrams)
|
||||||
|
R->>C: Encrypted media (QUIC datagrams)
|
||||||
|
|
||||||
|
Note over C,R: Rekey every 65,536 packets<br/>New ephemeral DH + HKDF mix
|
||||||
|
```
|
||||||
|
|
||||||
|
### Encryption Details
|
||||||
|
|
||||||
|
| Component | Algorithm | Purpose |
|
||||||
|
|-----------|-----------|---------|
|
||||||
|
| Identity signing | Ed25519 | Authenticate handshake messages |
|
||||||
|
| Key agreement | X25519 (ephemeral) | Derive shared secret |
|
||||||
|
| Key derivation | HKDF-SHA256 | Derive session key from shared secret |
|
||||||
|
| Media encryption | ChaCha20-Poly1305 | Encrypt audio payloads (16-byte tag) |
|
||||||
|
| Nonce construction | Deterministic from sequence number | No nonce reuse, no state sync needed |
|
||||||
|
| Anti-replay | Sliding window (64-packet) | Reject duplicate/old packets |
|
||||||
|
| Forward secrecy | Rekey every 65,536 packets | New ephemeral DH + HKDF mix |
|
||||||
|
|
||||||
|
**Why ChaCha20-Poly1305 over AES-GCM**:
|
||||||
|
- Faster on hardware without AES-NI (ARM phones, Raspberry Pi relays)
|
||||||
|
- Inherently constant-time (add-rotate-XOR only)
|
||||||
|
- Compatible with Warzone messenger (featherChat)
|
||||||
|
- Same 16-byte authentication tag overhead as AES-GCM
|
||||||
|
|
||||||
|
**AEAD with AAD**: The MediaHeader is used as Associated Authenticated Data. The header is authenticated but not encrypted, allowing relays to read routing information (block ID, sequence number) without decrypting the payload.
|
||||||
|
|
||||||
|
### Trust on First Use (TOFU)
|
||||||
|
|
||||||
|
Clients remember the relay's TLS certificate fingerprint after first connection. If the fingerprint changes on a subsequent connection, the desktop client shows a "Server Key Changed" warning dialog. The relay derives its TLS certificate deterministically from its persisted identity seed, so the fingerprint is stable across restarts.
|
||||||
|
|
||||||
|
## Relay Architecture
|
||||||
|
|
||||||
|
### Room Mode (Default SFU)
|
||||||
|
|
||||||
|
In room mode, the relay acts as a Selective Forwarding Unit. Clients join named rooms via the QUIC SNI (Server Name Indication) field. The relay forwards each participant's encrypted packets to all other participants in the room without decoding or re-encoding.
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
graph TB
|
||||||
|
subgraph "Room Mode (SFU)"
|
||||||
|
C1[Client 1] -->|"QUIC SNI=room-hash"| RM[Room Manager]
|
||||||
|
C2[Client 2] -->|"QUIC SNI=room-hash"| RM
|
||||||
|
C3[Client 3] -->|"QUIC SNI=room-hash"| RM
|
||||||
|
RM --> R1[Room 'podcast']
|
||||||
|
R1 -->|fan-out| C1
|
||||||
|
R1 -->|fan-out| C2
|
||||||
|
R1 -->|fan-out| C3
|
||||||
|
end
|
||||||
|
|
||||||
|
style RM fill:#ff9f43,color:#fff
|
||||||
|
style R1 fill:#fdcb6e
|
||||||
|
```
|
||||||
|
|
||||||
|
**SFU vs MCU trade-off**: SFU was chosen because it preserves end-to-end encryption (the relay never sees plaintext audio). An MCU would need to decode, mix, and re-encode, breaking E2E encryption. The trade-off is O(N) bandwidth at the relay for N participants.
|
||||||
|
|
||||||
|
### Forward Mode
|
||||||
|
|
||||||
|
With `--remote`, the relay forwards all traffic to a remote relay. Used for chaining relays across lossy or censored links:
|
||||||
|
|
||||||
|
```
|
||||||
|
Client --> Relay A (--remote B) --> Relay B --> Destination Client
|
||||||
|
```
|
||||||
|
|
||||||
|
The relay pipeline in forward mode: FEC decode, jitter buffer, then FEC re-encode for the next hop.
|
||||||
|
|
||||||
|
## Federation
|
||||||
|
|
||||||
|
### Overview
|
||||||
|
|
||||||
|
Two or more relays form a federation mesh. Each relay is an independent SFU. When configured to trust each other, they bridge **global rooms** -- participants on relay A in a global room hear participants on relay B in the same room.
|
||||||
|
|
||||||
|
### Configuration
|
||||||
|
|
||||||
|
Federation uses three TOML configuration sections:
|
||||||
|
|
||||||
|
- `[[peers]]` -- outbound connections to peer relays (url + TLS fingerprint)
|
||||||
|
- `[[trusted]]` -- inbound connections accepted from relays (TLS fingerprint only)
|
||||||
|
- `[[global_rooms]]` -- room names to bridge across all federated peers
|
||||||
|
|
||||||
|
### Federation Topology
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
graph TB
|
||||||
|
subgraph "Relay A (EU)"
|
||||||
|
A_RM[Room Manager]
|
||||||
|
A_FM[Federation Manager]
|
||||||
|
A1[Alice - local]
|
||||||
|
A2[Bob - local]
|
||||||
|
A_RM --> A_FM
|
||||||
|
end
|
||||||
|
|
||||||
|
subgraph "Relay B (US)"
|
||||||
|
B_RM[Room Manager]
|
||||||
|
B_FM[Federation Manager]
|
||||||
|
B1[Charlie - local]
|
||||||
|
B_RM --> B_FM
|
||||||
|
end
|
||||||
|
|
||||||
|
A_FM <-->|"QUIC SNI='_federation'<br/>GlobalRoomActive/Inactive<br/>Media forwarding"| B_FM
|
||||||
|
|
||||||
|
A1 -->|media| A_RM
|
||||||
|
A2 -->|media| A_RM
|
||||||
|
B1 -->|media| B_RM
|
||||||
|
|
||||||
|
A_RM -->|"federated fan-out"| A1
|
||||||
|
A_RM -->|"federated fan-out"| A2
|
||||||
|
B_RM -->|"federated fan-out"| B1
|
||||||
|
|
||||||
|
style A_FM fill:#6c5ce7,color:#fff
|
||||||
|
style B_FM fill:#6c5ce7,color:#fff
|
||||||
|
style A_RM fill:#ff9f43,color:#fff
|
||||||
|
style B_RM fill:#ff9f43,color:#fff
|
||||||
|
```
|
||||||
|
|
||||||
|
### Protocol
|
||||||
|
|
||||||
|
1. On startup, each relay connects to all configured `[[peers]]` via QUIC with SNI `"_federation"`
|
||||||
|
2. After QUIC handshake, sends `FederationHello { tls_fingerprint }` for identity verification
|
||||||
|
3. Peer verifies the fingerprint against its `[[trusted]]` or `[[peers]]` list
|
||||||
|
4. When a local participant joins a global room, sends `GlobalRoomActive { room }` to all peers
|
||||||
|
5. When the last local participant leaves, sends `GlobalRoomInactive { room }`
|
||||||
|
6. Media is forwarded as `[room_hash:8][original_media_packet]` -- the relay does not decrypt
|
||||||
|
|
||||||
|
### What Relays Do NOT Do
|
||||||
|
|
||||||
|
- **No transcoding** -- media passes through as-is
|
||||||
|
- **No re-encryption** -- packets are already encrypted E2E
|
||||||
|
- **No central coordinator** -- each relay independently connects to configured peers
|
||||||
|
- **No automatic peer discovery** -- peers must be explicitly configured
|
||||||
|
|
||||||
|
### Failure Handling
|
||||||
|
|
||||||
|
- If a peer goes down, local rooms continue working; federated participants disappear from presence
|
||||||
|
- Reconnection: every 30 seconds with exponential backoff up to 5 minutes
|
||||||
|
- If a peer restarts with a different identity, the fingerprint check fails with a clear log message
|
||||||
|
|
||||||
|
## Jitter Buffer
|
||||||
|
|
||||||
|
The jitter buffer balances latency vs quality:
|
||||||
|
|
||||||
|
| Setting | Client | Relay |
|
||||||
|
|---------|--------|-------|
|
||||||
|
| Target depth | 10 packets (200ms) | 50 packets (1s) |
|
||||||
|
| Minimum before playout | 3 packets (60ms) | 25 packets (500ms) |
|
||||||
|
| Maximum cap | 250 packets (5s) | 250 packets (5s) |
|
||||||
|
|
||||||
|
The relay uses a deeper buffer to absorb jitter from lossy inter-relay links. The client uses a shallower buffer for lower latency.
|
||||||
|
|
||||||
|
The adaptive playout delay tracks jitter via exponential moving average and adjusts the target depth:
|
||||||
|
|
||||||
|
```
|
||||||
|
target_delay = ceil(jitter_ema / 20ms) + 2
|
||||||
|
```
|
||||||
|
|
||||||
|
**Known limitation**: The current jitter buffer does not use timestamp-based playout scheduling. It relies on sequence-number ordering only, which can lead to drift during long calls.
|
||||||
|
|
||||||
|
## Signal Messages
|
||||||
|
|
||||||
|
Signal messages are sent over reliable QUIC streams as length-prefixed JSON:
|
||||||
|
|
||||||
|
```
|
||||||
|
[4-byte length prefix][serde_json payload]
|
||||||
|
```
|
||||||
|
|
||||||
|
| Message | Purpose |
|
||||||
|
|---------|---------|
|
||||||
|
| `CallOffer` | Identity, ephemeral key, signature, supported profiles |
|
||||||
|
| `CallAnswer` | Identity, ephemeral key, signature, chosen profile |
|
||||||
|
| `AuthToken` | featherChat bearer token for relay authentication |
|
||||||
|
| `Hangup` | Reason: Normal, Busy, Declined, Timeout, Error |
|
||||||
|
| `Hold` / `Unhold` | Call hold state |
|
||||||
|
| `Mute` / `Unmute` | Mic mute state |
|
||||||
|
| `Transfer` | Call transfer to another relay/fingerprint |
|
||||||
|
| `Rekey` | New ephemeral key for forward secrecy |
|
||||||
|
| `QualityUpdate` | Quality report + recommended profile |
|
||||||
|
| `Ping` / `Pong` | Latency measurement (timestamp_ms) |
|
||||||
|
| `RoomUpdate` | Participant list changes |
|
||||||
|
| `PresenceUpdate` | Federation presence gossip |
|
||||||
|
| `RouteQuery` / `RouteResponse` | Presence discovery for routing |
|
||||||
|
| `FederationHello` | Relay identity during federation setup |
|
||||||
|
| `GlobalRoomActive` / `GlobalRoomInactive` | Federation room bridging |
|
||||||
|
|
||||||
|
## Test Coverage
|
||||||
|
|
||||||
|
571 tests across all crates, 0 failures:
|
||||||
|
|
||||||
|
| Crate | Tests | Key Coverage |
|
||||||
|
|-------|-------|-------------|
|
||||||
|
| wzp-proto | 41 | Wire format, jitter buffer, quality tiers, mini-frames, trunking |
|
||||||
|
| wzp-codec | 31 | Opus/Codec2 roundtrip, silence detection, noise suppression |
|
||||||
|
| wzp-fec | 22 | RaptorQ encode/decode, loss recovery, interleaving |
|
||||||
|
| wzp-crypto | 34 + 28 compat | Encrypt/decrypt, handshake, anti-replay, featherChat identity |
|
||||||
|
| wzp-transport | 2 | QUIC connection setup |
|
||||||
|
| wzp-relay | 40 + 4 integration | Room ACL, session mgmt, metrics, probes, mesh, trunking |
|
||||||
|
| wzp-client | 30 + 2 integration | Encoder/decoder, quality adapter, silence, drift, sweep |
|
||||||
|
| wzp-web | 2 | Metrics |
|
||||||
|
|
||||||
|
## Audio Routing (Android)
|
||||||
|
|
||||||
|
WarzonePhone supports three audio output routes on Android: **Earpiece**, **Speaker**, and **Bluetooth SCO**. The user cycles through available routes with a single button.
|
||||||
|
|
||||||
|
### Audio mode lifecycle
|
||||||
|
|
||||||
|
`MODE_IN_COMMUNICATION` is set **when the call engine starts** (right before Oboe `audio_start()`), not at app launch. This is critical — setting it early hijacks system audio routing (e.g. music drops from BT A2DP to earpiece). `MODE_NORMAL` is restored when the call engine stops.
|
||||||
|
|
||||||
|
```
|
||||||
|
App launch → MODE_NORMAL (other apps' audio unaffected)
|
||||||
|
Call start → set_audio_mode_communication() → MODE_IN_COMMUNICATION
|
||||||
|
Call end → audio_stop() → set_audio_mode_normal() → MODE_NORMAL
|
||||||
|
```
|
||||||
|
|
||||||
|
### Route lifecycle
|
||||||
|
|
||||||
|
1. Call starts → Earpiece (default).
|
||||||
|
2. User taps route button → cycles to next available route.
|
||||||
|
3. Route change requires Oboe stream restart (~60-400ms) because AAudio silently tears down streams on some OEMs when the routing target changes mid-stream.
|
||||||
|
4. Bluetooth disconnect mid-call → `AudioDeviceCallback.onAudioDevicesRemoved` fires → auto-fallback to Earpiece or Speaker.
|
||||||
|
|
||||||
|
### Bluetooth SCO
|
||||||
|
|
||||||
|
SCO (Synchronous Connection Oriented) is the correct Bluetooth profile for VoIP — it provides bidirectional mono audio at 8/16 kHz with ~30ms latency. A2DP (stereo, high-quality) is unidirectional and adds 100-200ms of buffering, making it unsuitable for real-time voice.
|
||||||
|
|
||||||
|
On API 31+ (Android 12), we use the modern `setCommunicationDevice(AudioDeviceInfo)` API to route audio to the BT SCO device. The deprecated `startBluetoothSco()` + `setBluetoothScoOn()` path is used as fallback on older APIs. `setBluetoothScoOn()` is silently rejected on Android 12+ for non-system apps.
|
||||||
|
|
||||||
|
BT SCO devices only support 8/16kHz sample rates, but our pipeline runs at 48kHz. When BT is active, Oboe opens in **BT mode** (`bt_active=1`): capture skips `setSampleRate(48000)` and `setInputPreset(VoiceCommunication)`, letting the system open at the device's native rate. Oboe's `SampleRateConversionQuality::Best` resamples to/from 48kHz for our ring buffers.
|
||||||
|
|
||||||
|
### Two app variants
|
||||||
|
|
||||||
|
Both the native Kotlin app (`AudioRouteManager.kt`) and the Tauri app (`android_audio.rs` JNI bridge) support BT SCO routing. The native app uses `AudioDeviceCallback` for automatic device detection; the Tauri app uses `getAvailableCommunicationDevices()` (API 31+) or `getDevices()` on demand.
|
||||||
|
|
||||||
|
## Network Change Response
|
||||||
|
|
||||||
|
The `AdaptiveQualityController` in `wzp-proto` reacts to network transport changes signaled via `signal_network_change(NetworkContext)`:
|
||||||
|
|
||||||
|
| Transition | Response |
|
||||||
|
|-----------|----------|
|
||||||
|
| WiFi → Cellular | Preemptive 1-tier quality downgrade + 10s FEC boost |
|
||||||
|
| Cellular → WiFi | FEC boost only (quality recovers via normal adaptive logic) |
|
||||||
|
| Any change | Reset hysteresis counters to avoid stale state |
|
||||||
|
|
||||||
|
On Android, `NetworkMonitor.kt` wraps `ConnectivityManager.NetworkCallback` and classifies the transport type using bandwidth heuristics (no `READ_PHONE_STATE` needed). The classification is delivered to the Rust engine via JNI → `AtomicU8` → recv task polling — the same lock-free cross-task signaling pattern used for adaptive profile switches.
|
||||||
|
|
||||||
|
### Cellular generation heuristics
|
||||||
|
|
||||||
|
| Downstream bandwidth | Classification |
|
||||||
|
|---------------------|---------------|
|
||||||
|
| >= 100 Mbps | 5G NR |
|
||||||
|
| >= 10 Mbps | LTE |
|
||||||
|
| < 10 Mbps | 3G or worse |
|
||||||
|
|
||||||
|
These thresholds are conservative. Carriers over-report bandwidth, but for VoIP quality decisions the exact generation matters less than the rough category.
|
||||||
|
|
||||||
|
## Build Requirements
|
||||||
|
|
||||||
|
- **Rust** 1.85+ (2024 edition)
|
||||||
|
- **Linux**: cmake, pkg-config, libasound2-dev (for audio feature)
|
||||||
|
- **macOS**: Xcode command line tools (CoreAudio included)
|
||||||
|
- **Android**: NDK 26.1 (r26b), cmake 3.25-3.28 (system package)
|
||||||
|
|
||||||
|
### Android APK Builds
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# arm64 only (default, 25MB release APK)
|
||||||
|
./scripts/build-tauri-android.sh --init --release --arch arm64
|
||||||
|
|
||||||
|
# armv7 only (smaller devices)
|
||||||
|
./scripts/build-tauri-android.sh --init --release --arch armv7
|
||||||
|
|
||||||
|
# both architectures as separate APKs
|
||||||
|
./scripts/build-tauri-android.sh --init --release --arch all
|
||||||
|
```
|
||||||
|
|
||||||
|
Release APKs are signed with `android/keystore/wzp-release.jks` via `apksigner`. Per-arch builds produce separate APKs (~25MB each vs ~50MB universal) for easier sharing with testers.
|
||||||
209
vault/Architecture/Extensibility.md
Normal file
209
vault/Architecture/Extensibility.md
Normal file
@@ -0,0 +1,209 @@
|
|||||||
|
---
|
||||||
|
tags: [architecture, wzp]
|
||||||
|
type: architecture
|
||||||
|
---
|
||||||
|
|
||||||
|
# WarzonePhone Extension Points & Future Features
|
||||||
|
|
||||||
|
## Trait-Based Architecture
|
||||||
|
|
||||||
|
The protocol is designed around trait interfaces defined in `crates/wzp-proto/src/traits.rs`. Any implementation that satisfies the trait contract can be plugged in without modifying other crates.
|
||||||
|
|
||||||
|
### Adding a New Audio Codec
|
||||||
|
|
||||||
|
Implement `AudioEncoder` and `AudioDecoder` from `wzp_proto::traits`:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
pub trait AudioEncoder: Send + Sync {
|
||||||
|
fn encode(&mut self, pcm: &[i16], out: &mut [u8]) -> Result<usize, CodecError>;
|
||||||
|
fn codec_id(&self) -> CodecId;
|
||||||
|
fn set_profile(&mut self, profile: QualityProfile) -> Result<(), CodecError>;
|
||||||
|
fn max_frame_bytes(&self) -> usize;
|
||||||
|
fn set_inband_fec(&mut self, _enabled: bool) {}
|
||||||
|
fn set_dtx(&mut self, _enabled: bool) {}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub trait AudioDecoder: Send + Sync {
|
||||||
|
fn decode(&mut self, encoded: &[u8], pcm: &mut [i16]) -> Result<usize, CodecError>;
|
||||||
|
fn decode_lost(&mut self, pcm: &mut [i16]) -> Result<usize, CodecError>;
|
||||||
|
fn codec_id(&self) -> CodecId;
|
||||||
|
fn set_profile(&mut self, profile: QualityProfile) -> Result<(), CodecError>;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Steps:
|
||||||
|
1. Add a new variant to `CodecId` in `crates/wzp-proto/src/codec_id.rs` (uses 4-bit wire encoding, currently 5 of 16 values used)
|
||||||
|
2. Implement `AudioEncoder` and `AudioDecoder` for your codec
|
||||||
|
3. Register the codec in `AdaptiveEncoder`/`AdaptiveDecoder` in `crates/wzp-codec/src/adaptive.rs`
|
||||||
|
4. Add a `QualityProfile` constant for the new codec
|
||||||
|
|
||||||
|
### Adding a New FEC Scheme
|
||||||
|
|
||||||
|
Implement `FecEncoder` and `FecDecoder` from `wzp_proto::traits`:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
pub trait FecEncoder: Send + Sync {
|
||||||
|
fn add_source_symbol(&mut self, data: &[u8]) -> Result<(), FecError>;
|
||||||
|
fn generate_repair(&mut self, ratio: f32) -> Result<Vec<(u8, Vec<u8>)>, FecError>;
|
||||||
|
fn finalize_block(&mut self) -> Result<u8, FecError>;
|
||||||
|
fn current_block_id(&self) -> u8;
|
||||||
|
fn current_block_size(&self) -> usize;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub trait FecDecoder: Send + Sync {
|
||||||
|
fn add_symbol(&mut self, block_id: u8, symbol_index: u8, is_repair: bool, data: &[u8]) -> Result<(), FecError>;
|
||||||
|
fn try_decode(&mut self, block_id: u8) -> Result<Option<Vec<Vec<u8>>>, FecError>;
|
||||||
|
fn expire_before(&mut self, block_id: u8);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
For example, a Reed-Solomon implementation would maintain the same block/symbol structure but use a different coding algorithm internally. The FEC block ID and symbol index fields in `MediaHeader` support any scheme that fits the block/symbol model.
|
||||||
|
|
||||||
|
### Adding a New Transport
|
||||||
|
|
||||||
|
Implement `MediaTransport` from `wzp_proto::traits`:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
#[async_trait]
|
||||||
|
pub trait MediaTransport: Send + Sync {
|
||||||
|
async fn send_media(&self, packet: &MediaPacket) -> Result<(), TransportError>;
|
||||||
|
async fn recv_media(&self) -> Result<Option<MediaPacket>, TransportError>;
|
||||||
|
async fn send_signal(&self, msg: &SignalMessage) -> Result<(), TransportError>;
|
||||||
|
async fn recv_signal(&self) -> Result<Option<SignalMessage>, TransportError>;
|
||||||
|
fn path_quality(&self) -> PathQuality;
|
||||||
|
async fn close(&self) -> Result<(), TransportError>;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
A raw UDP transport, a WebRTC data channel transport, or a TCP tunnel transport could all implement this trait.
|
||||||
|
|
||||||
|
## Obfuscation Layer (Phase 2)
|
||||||
|
|
||||||
|
The `ObfuscationLayer` trait is defined in `crates/wzp-proto/src/traits.rs` but not yet implemented:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
pub trait ObfuscationLayer: Send + Sync {
|
||||||
|
fn obfuscate(&mut self, data: &[u8], out: &mut Vec<u8>) -> Result<(), ObfuscationError>;
|
||||||
|
fn deobfuscate(&mut self, data: &[u8], out: &mut Vec<u8>) -> Result<(), ObfuscationError>;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Planned implementations:
|
||||||
|
- **TLS-in-TLS**: Wrap QUIC traffic inside a TLS connection to port 443, making it look like ordinary HTTPS
|
||||||
|
- **HTTP/2 mimicry**: Frame QUIC packets as HTTP/2 data frames
|
||||||
|
- **Random padding**: Add random-length padding to defeat traffic analysis
|
||||||
|
- **Domain fronting**: Use CDN infrastructure to hide the true destination
|
||||||
|
|
||||||
|
The obfuscation layer sits between the crypto layer and the transport layer in the protocol stack, wrapping encrypted packets before transmission.
|
||||||
|
|
||||||
|
## FeatherChat / Warzone Messenger Integration
|
||||||
|
|
||||||
|
As described in `docs/featherchat.md`, WarzonePhone is designed to integrate with the existing Warzone messenger.
|
||||||
|
|
||||||
|
### Shared Identity Model
|
||||||
|
|
||||||
|
Both WarzonePhone and Warzone use the same identity derivation:
|
||||||
|
- 32-byte seed (BIP39 mnemonic backup)
|
||||||
|
- HKDF with context strings: `"warzone-ed25519-identity"` and `"warzone-x25519-identity"`
|
||||||
|
- Ed25519 for signing, X25519 for encryption
|
||||||
|
- Fingerprint: `SHA-256(Ed25519_pub)[:16]`
|
||||||
|
|
||||||
|
This is implemented in `crates/wzp-crypto/src/handshake.rs` as `WarzoneKeyExchange::from_identity_seed()`.
|
||||||
|
|
||||||
|
### Signaling via Existing WebSocket
|
||||||
|
|
||||||
|
Call initiation flows through the Warzone messenger's existing WebSocket connection:
|
||||||
|
1. Caller looks up callee via `@alias`, federated address, or raw fingerprint
|
||||||
|
2. Caller sends `WireMessage::CallOffer` through the existing message channel
|
||||||
|
3. Callee receives the offer and responds with `WireMessage::CallAnswer`
|
||||||
|
4. Both sides establish a direct QUIC connection to the relay using ephemeral keys from the signaling exchange
|
||||||
|
|
||||||
|
The `SignalMessage::CallOffer` and `SignalMessage::CallAnswer` variants in `crates/wzp-proto/src/packet.rs` carry the same fields needed for this flow.
|
||||||
|
|
||||||
|
### Key Derivation from Existing Shared Secret
|
||||||
|
|
||||||
|
When two Warzone users already have an X3DH shared secret from their messaging session, call keys can be derived from it:
|
||||||
|
- `HKDF(x3dh_shared_secret, "warzone-call-session")` -> 32-byte session key
|
||||||
|
- Or: fresh ephemeral exchange per call (current implementation) for independent forward secrecy
|
||||||
|
|
||||||
|
### Unified Addressing
|
||||||
|
|
||||||
|
The Warzone addressing system resolves user identities across multiple namespaces:
|
||||||
|
|
||||||
|
| Method | Format | Resolution |
|
||||||
|
|--------|--------|------------|
|
||||||
|
| Local alias | `@manwe` | Server resolves to fingerprint |
|
||||||
|
| Federated | `@manwe.b1.example.com` | DNS TXT record -> fingerprint + endpoint |
|
||||||
|
| ENS | `@manwe.eth` | Ethereum address -> fingerprint (planned) |
|
||||||
|
| Raw fingerprint | `xxxx:xxxx:...` | Direct lookup |
|
||||||
|
|
||||||
|
A user calls `@manwe` the same way they message `@manwe`.
|
||||||
|
|
||||||
|
## Authentication: Caller Verification Before Bridging
|
||||||
|
|
||||||
|
Currently, relays forward packets without verifying caller identity. To add authentication:
|
||||||
|
|
||||||
|
1. **Relay-side handshake**: The relay receives the `CallOffer`, verifies the Ed25519 signature, and checks the caller's identity against an allowlist before accepting the connection.
|
||||||
|
|
||||||
|
2. **Implementation point**: `crates/wzp-relay/src/handshake.rs` already implements `accept_handshake()` which performs signature verification. To gate admission, add an authorization check after signature verification.
|
||||||
|
|
||||||
|
3. **Token-based auth**: Add a `token: Vec<u8>` field to `CallOffer` containing a relay-issued authentication token (e.g., signed by the relay operator's key).
|
||||||
|
|
||||||
|
## Multi-Relay Mesh
|
||||||
|
|
||||||
|
The current two-relay chain (`--remote` flag) can be extended to a multi-hop mesh:
|
||||||
|
|
||||||
|
```
|
||||||
|
Client -> Relay A -> Relay B -> Relay C -> Destination
|
||||||
|
```
|
||||||
|
|
||||||
|
Each hop uses the relay pipeline (FEC decode -> jitter buffer -> FEC re-encode) to absorb loss on each link independently. This requires:
|
||||||
|
|
||||||
|
1. Relay discovery and route selection (not yet implemented)
|
||||||
|
2. Per-hop FEC parameters (each link may have different loss characteristics)
|
||||||
|
3. Cumulative latency management (each hop adds jitter buffer delay)
|
||||||
|
|
||||||
|
## Video Support
|
||||||
|
|
||||||
|
The trait architecture supports video by adding:
|
||||||
|
|
||||||
|
1. **Video codec trait**: Similar to `AudioEncoder`/`AudioDecoder` but for video frames
|
||||||
|
2. **Codec choices**: AV1 (best compression, higher CPU), VP9 SVC (scalable, moderate CPU)
|
||||||
|
3. **Separate FEC strategy**: Video frames are larger and more critical (I-frames vs P-frames need different protection levels)
|
||||||
|
4. **SVC (Scalable Video Coding)**: With VP9 SVC, the relay can drop enhancement layers without transcoding, adapting video quality to each receiver's bandwidth
|
||||||
|
|
||||||
|
Video would add new `CodecId` variants and a separate `QualityProfile` for video parameters.
|
||||||
|
|
||||||
|
## Android Native Client
|
||||||
|
|
||||||
|
The workspace is designed with Android in mind (`wzp-client` description mentions "for Android (JNI) and Windows desktop"):
|
||||||
|
|
||||||
|
1. **JNI bindings**: Use `jni` crate or `uniffi` to expose `CallEncoder`, `CallDecoder`, and `MediaTransport` to Kotlin/Java
|
||||||
|
2. **Audio I/O**: Android uses AAudio or OpenSL ES instead of cpal
|
||||||
|
3. **Build**: Cross-compile with `cargo ndk` targeting `aarch64-linux-android` and `armv7-linux-androideabi`
|
||||||
|
4. **Permissions**: `RECORD_AUDIO`, `INTERNET`, `WAKE_LOCK`
|
||||||
|
|
||||||
|
## STUN/TURN NAT Traversal Integration
|
||||||
|
|
||||||
|
The `SignalMessage::IceCandidate` variant is already defined for NAT traversal:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
IceCandidate { candidate: String }
|
||||||
|
```
|
||||||
|
|
||||||
|
Integration would involve:
|
||||||
|
1. STUN server queries to discover the client's public IP/port
|
||||||
|
2. ICE candidate exchange via the signaling channel
|
||||||
|
3. TURN relay fallback when direct UDP is blocked
|
||||||
|
4. Integration with the existing QUIC transport (QUIC can traverse NATs via its connection migration)
|
||||||
|
|
||||||
|
## Bandwidth Estimation and Adaptive Bitrate
|
||||||
|
|
||||||
|
The `PathMonitor` in `crates/wzp-transport/src/path_monitor.rs` already estimates bandwidth from observed packet rates. To close the loop:
|
||||||
|
|
||||||
|
1. Feed `PathMonitor::quality()` into `AdaptiveQualityController::observe()` as `QualityReport` values
|
||||||
|
2. The controller will trigger tier transitions when conditions change
|
||||||
|
3. Propagate the new `QualityProfile` to both encoder (codec switch) and FEC (ratio change)
|
||||||
|
4. Signal the peer via `SignalMessage::QualityUpdate` so both sides switch simultaneously
|
||||||
|
|
||||||
|
The framework is in place; the missing piece is the integration wiring in the client's main loop to periodically generate quality reports from path metrics.
|
||||||
113
vault/Architecture/Protocol-Audit.md
Normal file
113
vault/Architecture/Protocol-Audit.md
Normal file
@@ -0,0 +1,113 @@
|
|||||||
|
---
|
||||||
|
tags: [architecture, wzp]
|
||||||
|
type: architecture
|
||||||
|
---
|
||||||
|
|
||||||
|
# WZP Protocol Audit
|
||||||
|
|
||||||
|
> Protocol-level review of WZP as of 2026-05-11. See `WZP-SPEC.md` for the spec being audited.
|
||||||
|
|
||||||
|
## Strengths
|
||||||
|
|
||||||
|
- **QUIC datagrams instead of raw UDP + SRTP** — buys TLS 1.3, PLPMTUD, path migration, and ACK-based loss/RTT estimation. Quinn's `PathSnapshot` feeding `DredTuner` is something WebRTC stacks build from scratch.
|
||||||
|
- **Continuous DRED tuning.** Mapping RTT / loss / jitter to a continuous Opus DRED lookback window is genuinely better than discrete tiers — most stacks treat DRED as on/off.
|
||||||
|
- **MiniHeader (49/50).** At 50 pps that is ~400 B/s saved per stream; meaningful at scale.
|
||||||
|
- **SFU never decodes.** Preserves E2E. Most SFUs (LiveKit, Janus) terminate SRTP at the SFU.
|
||||||
|
- **RaptorQ for low-bitrate Codec2 + DRED for Opus.** Correct split — DRED is cheaper than FEC at high bitrate; RaptorQ shines when you can afford many small symbols.
|
||||||
|
|
||||||
|
## Weaknesses
|
||||||
|
|
||||||
|
### W1. `u16` sequence wraps every ~21 minutes at 50 pps
|
||||||
|
Anti-replay window is 64 packets so wrap is safe for replay. **But** the jitter buffer's `BTreeMap<u16, _>` will misorder across the wrap boundary if a packet is delayed more than ~32 k frames. Widen to `u32` (or version the field).
|
||||||
|
|
||||||
|
### W2. `fec_block_id: u8` wraps every 256 blocks (~25 s at 5-frame blocks)
|
||||||
|
A late-joining peer or a slow reconstructor can collide block IDs. Widen to `u16` or carry an epoch counter.
|
||||||
|
|
||||||
|
### W3. `timestamp_ms` rebase behavior at rekey is unspecified
|
||||||
|
Rekey every 65,536 packets (~22 min). If `timestamp_ms` resets, downstream sync glitches. If it does not, document explicitly.
|
||||||
|
|
||||||
|
### W4. `MiniHeader` has no `seq`
|
||||||
|
Receiver infers absolute seq from the most recent full header + frame count. One missed full header (every 50 frames = 1 s) leaves 49 packets with unknown absolute seq. Acceptable for audio with short jitter buffers — **fatal for video** where one missed full header can desync an entire GOP. **Add `seq_delta: u8` to MiniHeader before video lands.**
|
||||||
|
|
||||||
|
### W5. `QualityReport` placement vs. AEAD
|
||||||
|
A 4-byte trailer on encrypted media is fine **iff it sits inside the AEAD payload**. If it is outside, anything stripping the last 4 bytes corrupts decryption and creates a downgrade vector. Verify in `packet.rs`; if outside, move it inside or AAD-bind it.
|
||||||
|
|
||||||
|
### W6. Adaptive controller is loss / RTT-only — no bandwidth estimator
|
||||||
|
Quinn exposes `cwnd` and `bytes_in_flight`, but `AdaptiveQualityController` does not consume them. Under low utilization you cannot detect that you *could* upgrade to Opus 64 k. **For video this is mandatory** — without BWE you will either oscillate or never use available capacity.
|
||||||
|
|
||||||
|
### W7. No NACK / explicit retransmit path
|
||||||
|
For audio with DRED + FEC this is fine. For video keyframes it is wasteful — an I-frame is 50–200 packets, protecting at 50 % FEC doubles bitrate. A NACK path is cheap and far cheaper than blanket FEC for I-frames.
|
||||||
|
|
||||||
|
### W8. TrunkFrame batching multiplies AEAD cost
|
||||||
|
Each inner payload is its own AEAD operation. At 10 entries that is 10× ChaCha calls per recv. Fine on x86 / ARM with AES-NI / NEON; profile on weak Android (Nothing A059 baseline).
|
||||||
|
|
||||||
|
### W9. `CodecID` is 4 bits → max 16 codecs; 9 already used
|
||||||
|
Adding H.264, H.265, AV1, VP9 takes you to 13. Land the widening **before** deployment — either steal from `reserved` / `csrc_count` to make CodecID 8-bit, or split into `MediaType:2 / CodecID:6`. Doing this post-deployment is painful.
|
||||||
|
|
||||||
|
### W10. No `MediaType` field
|
||||||
|
Audio vs. video vs. data is implicit in CodecID. A 2-bit `MediaType` lets the SFU apply per-type policy (drop video first under congestion, prioritize audio fan-out) without a codec lookup.
|
||||||
|
|
||||||
|
### W11. Anti-replay window 64 packets is tight for video
|
||||||
|
One keyframe burst can be 100+ packets; a single reordered earlier packet stalls the window. Bump to 256 or 1024 for video streams, or maintain a per-stream window.
|
||||||
|
|
||||||
|
### W12. `SignalMessage` has no version byte
|
||||||
|
Bincode + `#[serde(default, skip_serializing_if)]` covers field additions but not variant removal or semantic change. Lead every variant with `version: u8`.
|
||||||
|
|
||||||
|
### W13. RoomManager Mutex per-packet — **RESOLVED**
|
||||||
|
Already flagged in `ARCHITECTURE.md`. At ~1500 pps/sender for video this becomes a real ceiling.
|
||||||
|
|
||||||
|
**Resolution (T3.1):** `RoomManager` now stores `DashMap<String, Arc<RwLock<Room>>>` instead of `DashMap<String, Room>`. The DashMap guard is held only long enough to clone the `Arc`; all per-room operations (fan-out `others()`, quality `observe_quality()`, join/leave) then acquire the room-level `std::sync::RwLock`. This lets concurrent `others()` calls share a read lock while writers hold the write lock, eliminating the per-packet DashMap contention that was the original concern.
|
||||||
|
|
||||||
|
### W14. No receiver → sender congestion feedback beyond inline QualityReport
|
||||||
|
For video you need REMB-style or transport-CC-style explicit BWE feedback at ~50 ms cadence, independent of media packets.
|
||||||
|
|
||||||
|
## Priorities
|
||||||
|
|
||||||
|
| Priority | Issue | Why |
|
||||||
|
|---|---|---|
|
||||||
|
| P0 | W9 (CodecID width), W10 (MediaType), W4 (MiniHeader seq_delta) | Wire-format changes — must land before video, painful to change post-deploy |
|
||||||
|
| P0 | W1 (seq u16 → u32) | Same window; audio benefits too |
|
||||||
|
| P1 | W6 (BWE), W14 (transport feedback) | Blocking for usable video; improves audio adaptation |
|
||||||
|
| P1 | W5 (QualityReport in AEAD) | Security correctness |
|
||||||
|
| P2 | W2 (fec_block_id width), W11 (anti-replay window), W12 (signal version byte) | Long-tail correctness |
|
||||||
|
| P2 | W7 (NACK path), W13 (RoomManager lock) | Video performance, not correctness |
|
||||||
|
| P3 | W3 (timestamp rebase doc), W8 (AEAD profiling) | Documentation / measurement |
|
||||||
|
|
||||||
|
## Resolution status (2026-05-11)
|
||||||
|
|
||||||
|
The v2 wire format specified in `ROAD-TO-VIDEO.md` Phase V1 addresses:
|
||||||
|
|
||||||
|
| Issue | Resolved by |
|
||||||
|
|---|---|
|
||||||
|
| W1 (seq u16 → u32) | `sequence: u32` in MediaHeader v2 |
|
||||||
|
| W4 (MiniHeader seq) | `seq_delta: u8` added; MiniHeader v2 is 5 B |
|
||||||
|
| W9 (CodecID width) | Widened to 8-bit (room for 256) |
|
||||||
|
| W10 (MediaType) | Explicit `media_type: u8` byte |
|
||||||
|
|
||||||
|
W6 / W14 (BWE + TransportFeedback) addressed in Phase V2. W7 (NACK) addressed in Phase V2 / V4. Others remain open.
|
||||||
|
|
||||||
|
## Known pre-existing clippy debt (as of T1.5.2)
|
||||||
|
|
||||||
|
Measured at commit `c93d302` on `experimental-ui` (2026-05-11).
|
||||||
|
|
||||||
|
`cargo clippy --workspace --all-targets -- -D warnings` fails in two crates with **pre-existing** errors (verified against `HEAD~1`). These are not introduced by any Wave 1 task; they should be cleaned up in a dedicated hygiene sprint or accepted as known debt.
|
||||||
|
|
||||||
|
### `wzp-codec` — 9 errors
|
||||||
|
|
||||||
|
| Category | Count | Lint | Files |
|
||||||
|
|---|---|---|---|
|
||||||
|
| Manual saturating sub | 1 | `clippy::implicit_saturating_sub` | `aec.rs:117` |
|
||||||
|
| Needless range loop | 2 | `clippy::needless_range_loop` | `aec.rs:164`, `resample.rs:51` |
|
||||||
|
| Manual `div_ceil` | 2 | `clippy::manual_div_ceil` | `codec2_dec.rs:48`, `codec2_enc.rs:48` |
|
||||||
|
| Manual `clamp` | 2 | `clippy::manual_clamp` | `denoise.rs:59`, `opus_enc.rs:250` |
|
||||||
|
| Manual ASCII case-cmp | 1 | `clippy::manual_ascii_check` | `opus_enc.rs:99` |
|
||||||
|
| Same-item push in loop | 1 | `clippy::same_item_push` | `resample.rs:184` |
|
||||||
|
|
||||||
|
### `warzone-protocol` (submodule `deps/featherchat`) — 3 errors
|
||||||
|
|
||||||
|
| Category | Count | Lint | Files |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `clone` on `Copy` type | 1 | `clippy::clone_on_copy` | `ratchet.rs:202` |
|
||||||
|
| Missing `Default` impl | 2 | `clippy::new_without_default` | `types.rs:59`, `types.rs:69` |
|
||||||
|
|
||||||
|
**Policy:** New tasks must not add *new* clippy errors in crates they touch. The 12 errors above are grandfathered; a follow-up cleanup task should be scheduled to fix them (especially the `wzp-codec` ones, which are straightforward mechanical replacements).
|
||||||
276
vault/Architecture/Refactor-Codebase-Audit.md
Normal file
276
vault/Architecture/Refactor-Codebase-Audit.md
Normal file
@@ -0,0 +1,276 @@
|
|||||||
|
---
|
||||||
|
tags: [architecture, wzp]
|
||||||
|
type: architecture
|
||||||
|
---
|
||||||
|
|
||||||
|
# Codebase Refactoring Audit (2026-04-13)
|
||||||
|
|
||||||
|
> Full analysis of the WarzonePhone codebase after the DashMap relay refactor, DRED continuous tuning, and adaptive quality wiring. The codebase is ~15K lines of Rust across 8 crates plus a 1.7K-line Tauri engine. This document identifies every refactoring opportunity ranked by impact.
|
||||||
|
|
||||||
|
## Critical: engine.rs is 1,705 Lines With ~35% Duplication
|
||||||
|
|
||||||
|
`desktop/src-tauri/src/engine.rs` has two nearly-identical `CallEngine::start()` implementations:
|
||||||
|
- **Android path:** 880 lines (lines 321–1200)
|
||||||
|
- **Desktop path:** 430 lines (lines 1203–1633)
|
||||||
|
|
||||||
|
### What's Duplicated (350+ lines)
|
||||||
|
|
||||||
|
| Block | Android Lines | Desktop Lines | Size | Identical? |
|
||||||
|
|-------|--------------|---------------|------|-----------|
|
||||||
|
| CallConfig initialization | 529–539 | 1353–1363 | 23 lines | Yes |
|
||||||
|
| DRED tuner + frame_samples setup | 541–555 | 1360–1375 | 15 lines | Yes |
|
||||||
|
| Adaptive quality profile switch | 651–665 | 1414–1428 | 15 lines | Yes |
|
||||||
|
| Codec-to-QualityProfile match | 852–864 | 1488–1500 | 19 lines | Yes |
|
||||||
|
| DRED ingest + gap fill | 886–902 | 1511–1528 | 17 lines | Yes |
|
||||||
|
| Quality report ingestion | 905–912 | 1531–1538 | 8 lines | Yes |
|
||||||
|
| Signal task (entire thing) | 1133–1180 | 1569–1616 | 48 lines | Yes |
|
||||||
|
|
||||||
|
### Suggested Fix: Extract Shared Helpers
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// Top of engine.rs — shared between both platforms
|
||||||
|
|
||||||
|
fn build_call_config(quality: &str) -> CallConfig { ... }
|
||||||
|
|
||||||
|
fn codec_to_profile(codec: CodecId) -> QualityProfile { ... }
|
||||||
|
|
||||||
|
fn check_adaptive_switch(
|
||||||
|
pending: &AtomicU8,
|
||||||
|
encoder: &mut CallEncoder,
|
||||||
|
tuner: &mut DredTuner,
|
||||||
|
frame_samples: &mut usize,
|
||||||
|
tx_codec: &Mutex<String>,
|
||||||
|
) { ... }
|
||||||
|
|
||||||
|
async fn run_signal_task(
|
||||||
|
transport: Arc<QuinnTransport>,
|
||||||
|
running: Arc<AtomicBool>,
|
||||||
|
pending_profile: Arc<AtomicU8>,
|
||||||
|
participants: Arc<Mutex<Vec<ParticipantInfo>>>,
|
||||||
|
) { ... }
|
||||||
|
```
|
||||||
|
|
||||||
|
This would reduce engine.rs by ~200 lines and make the Android/desktop paths only differ in their audio I/O (Oboe vs CPAL).
|
||||||
|
|
||||||
|
**Effort:** 2-3 hours. **Impact:** High — every future change to the send/recv pipeline currently requires editing two places.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## High: SignalMessage Enum Has 36 Variants
|
||||||
|
|
||||||
|
`crates/wzp-proto/src/packet.rs` (1,727 lines) has a `SignalMessage` enum with 36 variants mixing orthogonal concerns:
|
||||||
|
|
||||||
|
- Legacy call signaling (CallOffer, CallAnswer, IceCandidate, Rekey...)
|
||||||
|
- Direct calling (RegisterPresence, DirectCallOffer, DirectCallAnswer, CallSetup...)
|
||||||
|
- Federation (FederationHello, GlobalRoomActive/Inactive, FederatedSignalForward)
|
||||||
|
- Relay control (SessionForward, PresenceUpdate, RouteQuery, RoomUpdate)
|
||||||
|
- NAT traversal (Reflect, ReflectResponse, MediaPathReport)
|
||||||
|
- Quality (QualityUpdate, QualityDirective)
|
||||||
|
- Call control (Ping/Pong, Hold/Unhold, Mute/Unmute, Transfer)
|
||||||
|
|
||||||
|
Every new feature adds variants here, and every match on `SignalMessage` must handle all 36 arms (or use `_` wildcard).
|
||||||
|
|
||||||
|
### Suggested Fix: Sub-Enum Grouping
|
||||||
|
|
||||||
|
```rust
|
||||||
|
enum SignalMessage {
|
||||||
|
Call(CallSignal), // CallOffer, CallAnswer, IceCandidate, Rekey, Hangup...
|
||||||
|
Direct(DirectCallSignal), // RegisterPresence, DirectCallOffer, CallSetup, MediaPathReport...
|
||||||
|
Federation(FedSignal), // FederationHello, GlobalRoomActive, FederatedSignalForward...
|
||||||
|
Control(ControlSignal), // Ping/Pong, Hold/Unhold, Mute/Unmute, QualityDirective...
|
||||||
|
Relay(RelaySignal), // SessionForward, PresenceUpdate, RouteQuery, RoomUpdate...
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Caution:** This is a wire-format change. Serde serialization must remain backward-compatible with already-deployed relays. Use `#[serde(untagged)]` or versioned deserialization. Consider doing this as a v2 protocol bump.
|
||||||
|
|
||||||
|
**Effort:** 1 day. **Impact:** High for maintainability, but risky for wire compatibility.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## High: Federation Has Zero Tests
|
||||||
|
|
||||||
|
`crates/wzp-relay/src/federation.rs` (1,132 lines) has **no unit tests and no integration tests**. This is the most complex file in the relay crate, handling:
|
||||||
|
|
||||||
|
- Peer link management (connect, reconnect, stale sweep)
|
||||||
|
- Federation media egress (forward_to_peers)
|
||||||
|
- Federation media ingress (handle_datagram: dedup, rate limit, local delivery, multi-hop)
|
||||||
|
- Cross-relay signal forwarding
|
||||||
|
- Room event subscription and GlobalRoomActive/Inactive broadcasting
|
||||||
|
|
||||||
|
The relay crate has 91 tests, but none cover federation. Any refactoring of federation (like the DashMap migration or clone-before-send) is flying blind.
|
||||||
|
|
||||||
|
### Suggested Fix
|
||||||
|
|
||||||
|
Priority test cases:
|
||||||
|
1. `forward_to_peers` with 0, 1, 3 peers — verify datagram construction and label tracking
|
||||||
|
2. `handle_datagram` — dedup (same packet twice → second dropped), rate limit (exceed → dropped)
|
||||||
|
3. Stale presence sweeper — verify cleanup after timeout
|
||||||
|
4. `broadcast_signal` — verify signal reaches all peers
|
||||||
|
5. Multi-hop forward — verify source peer excluded from re-forward
|
||||||
|
|
||||||
|
**Effort:** 1 day. **Impact:** Critical for safe refactoring.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Medium: Federation `peer_links` Lock-During-Send
|
||||||
|
|
||||||
|
`broadcast_signal()` (line 216) holds `peer_links` Mutex **across async `send_signal()` calls**. A slow peer blocks all signal delivery. `forward_to_peers()` (line 406) holds it during sync sends (less severe but still serializes).
|
||||||
|
|
||||||
|
### Fix (30 minutes)
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// Before:
|
||||||
|
let links = self.peer_links.lock().await;
|
||||||
|
for (fp, link) in links.iter() {
|
||||||
|
link.transport.send_signal(msg).await; // lock held across await!
|
||||||
|
}
|
||||||
|
|
||||||
|
// After:
|
||||||
|
let peers: Vec<_> = {
|
||||||
|
let links = self.peer_links.lock().await;
|
||||||
|
links.values().map(|l| (l.label.clone(), l.transport.clone())).collect()
|
||||||
|
};
|
||||||
|
for (label, transport) in &peers {
|
||||||
|
transport.send_signal(msg).await; // no lock held
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Apply to `forward_to_peers()`, `broadcast_signal()`, and `send_signal_to_peer()`.
|
||||||
|
|
||||||
|
**Effort:** 30 minutes. **Impact:** Medium — eliminates last lock-during-I/O pattern.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Medium: Magic Numbers Scattered Through engine.rs
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// These appear as literals in multiple places:
|
||||||
|
tokio::time::sleep(Duration::from_millis(5)) // 6 occurrences
|
||||||
|
tokio::time::sleep(Duration::from_millis(100)) // 2 occurrences
|
||||||
|
Duration::from_millis(200) // 2 occurrences (signal timeout)
|
||||||
|
Duration::from_secs(10) // 1 occurrence (QUIC connect timeout)
|
||||||
|
Duration::from_secs(2) // 2 occurrences (heartbeat interval)
|
||||||
|
const DRED_POLL_INTERVAL: u32 = 25; // defined twice (Android + desktop)
|
||||||
|
vec![0i16; 1920] // 2 occurrences (should use FRAME_SAMPLES_40MS)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Fix
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// Top of engine.rs
|
||||||
|
const CAPTURE_POLL_MS: u64 = 5;
|
||||||
|
const RECV_TIMEOUT_MS: u64 = 100;
|
||||||
|
const SIGNAL_TIMEOUT_MS: u64 = 200;
|
||||||
|
const CONNECT_TIMEOUT_SECS: u64 = 10;
|
||||||
|
const HEARTBEAT_INTERVAL_SECS: u64 = 2;
|
||||||
|
const DRED_POLL_INTERVAL: u32 = 25;
|
||||||
|
// Already exists: const FRAME_SAMPLES_40MS: usize = 1920;
|
||||||
|
```
|
||||||
|
|
||||||
|
**Effort:** 15 minutes. **Impact:** Low but prevents bugs from inconsistent values.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Medium: CLI Arg Parsing in Relay main.rs
|
||||||
|
|
||||||
|
`parse_args()` in main.rs is 154 lines of manual `while i < args.len()` parsing with `match args[i].as_str()`. Every new flag adds 5-10 lines of boilerplate.
|
||||||
|
|
||||||
|
### Suggested Fix
|
||||||
|
|
||||||
|
Replace with `clap` derive macro:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
#[derive(clap::Parser)]
|
||||||
|
struct RelayArgs {
|
||||||
|
#[arg(long, default_value = "0.0.0.0:4433")]
|
||||||
|
listen: SocketAddr,
|
||||||
|
#[arg(long)]
|
||||||
|
remote: Option<String>,
|
||||||
|
#[arg(long)]
|
||||||
|
auth_url: Option<String>,
|
||||||
|
// ...
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Effort:** 1 hour. **Impact:** Medium — cleaner, auto-generates `--help`, validates types at parse time.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Medium: Error Handling Inconsistency
|
||||||
|
|
||||||
|
13 instances of `.ok()` silently swallowing errors on `transport.close()` across the relay. Federation signal forwarding has inconsistent error handling — some paths log, some don't.
|
||||||
|
|
||||||
|
### Fix
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// Helper at top of main.rs/federation.rs:
|
||||||
|
async fn close_transport(t: &impl MediaTransport, context: &str) {
|
||||||
|
if let Err(e) = t.close().await {
|
||||||
|
tracing::debug!(context, error = %e, "transport close error (non-fatal)");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Effort:** 30 minutes. **Impact:** Better observability when debugging connection issues.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Low: Unused Crypto Fields
|
||||||
|
|
||||||
|
`crates/wzp-crypto/src/handshake.rs` has `x25519_static_secret` and `x25519_static_public` fields marked `#[allow(dead_code)]`. These are derived from the identity seed but never used in any handshake flow.
|
||||||
|
|
||||||
|
**Decision needed:** Are these intended for a future feature (static key federation auth)? If not, remove. If yes, document the intended use.
|
||||||
|
|
||||||
|
**Effort:** 5 minutes to remove, or 10 minutes to document.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Low: 20 Unsafe Functions Missing Safety Docs
|
||||||
|
|
||||||
|
`crates/wzp-native/src/lib.rs` has 20 `unsafe` functions (extern "C" FFI bridge to Oboe) without `/// # Safety` documentation. Clippy flags all of them.
|
||||||
|
|
||||||
|
**Effort:** 30 minutes. **Impact:** Clippy clean, better documentation for contributors.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Low: quality.rs vs dred_tuner.rs Overlap
|
||||||
|
|
||||||
|
Both files deal with network quality → codec decisions, but they're complementary:
|
||||||
|
- `quality.rs`: discrete tier classification (Good/Degraded/Catastrophic) → codec profile
|
||||||
|
- `dred_tuner.rs`: continuous DRED frame mapping from loss/RTT/jitter
|
||||||
|
|
||||||
|
No consolidation needed, but add cross-references:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// In dred_tuner.rs:
|
||||||
|
//! See also: `quality.rs` for discrete tier classification that drives
|
||||||
|
//! codec switching. DredTuner operates within a tier, adjusting DRED
|
||||||
|
//! parameters continuously.
|
||||||
|
|
||||||
|
// In quality.rs:
|
||||||
|
//! See also: `dred_tuner.rs` for continuous DRED tuning within a tier.
|
||||||
|
```
|
||||||
|
|
||||||
|
**Effort:** 5 minutes.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Summary: Priority Matrix
|
||||||
|
|
||||||
|
| # | Refactor | Effort | Impact | Risk |
|
||||||
|
|---|----------|--------|--------|------|
|
||||||
|
| 1 | Extract shared engine.rs helpers | 2-3h | High | Low |
|
||||||
|
| 2 | Federation tests | 1 day | Critical | None |
|
||||||
|
| 3 | Federation clone-before-send | 30 min | Medium | Low |
|
||||||
|
| 4 | Extract magic numbers to constants | 15 min | Low | None |
|
||||||
|
| 5 | Error handling helpers | 30 min | Medium | None |
|
||||||
|
| 6 | CLI parser → clap | 1h | Medium | Low |
|
||||||
|
| 7 | SignalMessage sub-enums | 1 day | High | High (wire compat) |
|
||||||
|
| 8 | Safety docs on unsafe fns | 30 min | Low | None |
|
||||||
|
| 9 | Remove/document dead crypto fields | 5 min | Low | None |
|
||||||
|
| 10 | Cross-reference quality.rs ↔ dred_tuner.rs | 5 min | Low | None |
|
||||||
|
|
||||||
|
**Recommended order:** 4 → 3 → 5 → 1 → 2 → 6 → 8 → 9 → 10 → 7
|
||||||
|
|
||||||
|
Items 4, 3, 5 are quick wins (under 1 hour total). Item 1 is the biggest maintainability win. Item 2 is the most important for safety. Item 7 should wait for a protocol version bump.
|
||||||
261
vault/Architecture/Refactor-Relay-Concurrency.md
Normal file
261
vault/Architecture/Refactor-Relay-Concurrency.md
Normal file
@@ -0,0 +1,261 @@
|
|||||||
|
---
|
||||||
|
tags: [architecture, wzp]
|
||||||
|
type: architecture
|
||||||
|
---
|
||||||
|
|
||||||
|
# Relay Concurrency Refactor Guide
|
||||||
|
|
||||||
|
> Post-DashMap analysis: what was done, what remains, and what to do next.
|
||||||
|
|
||||||
|
## What Was Done (2026-04-13)
|
||||||
|
|
||||||
|
Replaced the global `Arc<Mutex<RoomManager>>` with `DashMap<String, Room>` inside `RoomManager`. The relay's media forwarding hot path no longer serializes through a single lock.
|
||||||
|
|
||||||
|
### Before
|
||||||
|
|
||||||
|
```
|
||||||
|
Participant A recv_media()
|
||||||
|
→ room_mgr.lock().await ← ALL participants, ALL rooms compete here
|
||||||
|
→ mgr.observe_quality(...) ← O(N) quality computation inside lock
|
||||||
|
→ mgr.others(...) ← clone Vec<ParticipantSender>
|
||||||
|
→ drop(lock)
|
||||||
|
→ fan-out sends
|
||||||
|
```
|
||||||
|
|
||||||
|
One `tokio::sync::Mutex` guarding all rooms, all participants, all quality state. A 100-room relay was effectively single-threaded for media forwarding.
|
||||||
|
|
||||||
|
### After
|
||||||
|
|
||||||
|
```
|
||||||
|
Participant A recv_media()
|
||||||
|
→ room_mgr.observe_quality(...) ← DashMap::get_mut(), per-room shard lock
|
||||||
|
→ room_mgr.others(...) ← DashMap::get(), shared shard lock
|
||||||
|
→ fan-out sends ← no lock held
|
||||||
|
```
|
||||||
|
|
||||||
|
64 internal shards. Rooms on different shards are fully parallel. Rooms on the same shard use RwLock semantics — reads (`others()`) are concurrent, writes (`observe_quality()`, `join()`, `leave()`) are exclusive per-shard only.
|
||||||
|
|
||||||
|
### Files Changed
|
||||||
|
|
||||||
|
| File | Change |
|
||||||
|
|------|--------|
|
||||||
|
| `crates/wzp-relay/Cargo.toml` | Added `dashmap = "6"` |
|
||||||
|
| `crates/wzp-relay/src/room.rs` | `HashMap<String, Room>` → `DashMap<String, Room>`, per-room quality/tier, all methods `&self` |
|
||||||
|
| `crates/wzp-relay/src/main.rs` | `Arc<Mutex<RoomManager>>` → `Arc<RoomManager>`, 3 lock sites removed |
|
||||||
|
| `crates/wzp-relay/src/federation.rs` | 11 lock sites removed, `room_mgr` field type changed |
|
||||||
|
| `crates/wzp-relay/src/ws.rs` | 3 lock sites removed, `room_mgr` field type changed |
|
||||||
|
|
||||||
|
### Measured Improvement
|
||||||
|
|
||||||
|
| Metric | Before | After |
|
||||||
|
|--------|--------|-------|
|
||||||
|
| Lock type (rooms) | 1 global `tokio::sync::Mutex` | 64-shard `DashMap` with per-shard RwLock |
|
||||||
|
| Cross-room blocking | Yes (all rooms share 1 lock) | No (rooms are independent) |
|
||||||
|
| Read concurrency within room | None (Mutex is exclusive) | Yes (`get()` is shared) |
|
||||||
|
| `.lock().await` sites | 20 across 4 files | 0 for room operations |
|
||||||
|
| Test count | 314 passing | 314 passing (0 regressions) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Current Lock Inventory
|
||||||
|
|
||||||
|
### Tier 0: Eliminated (Room Hot Path)
|
||||||
|
|
||||||
|
These are gone — DashMap handles them internally:
|
||||||
|
|
||||||
|
- ~~`room_mgr.lock().await` in media forwarding~~ → `room_mgr.others()` (DashMap shard)
|
||||||
|
- ~~`room_mgr.lock().await` in quality tracking~~ → `room_mgr.observe_quality()` (DashMap shard)
|
||||||
|
- ~~`room_mgr.lock().await` in join/leave~~ → `room_mgr.join()` / `.leave()` (DashMap entry)
|
||||||
|
|
||||||
|
### Tier 1: Federation `peer_links` (Medium Priority)
|
||||||
|
|
||||||
|
**Location:** `crates/wzp-relay/src/federation.rs:142`
|
||||||
|
```rust
|
||||||
|
peer_links: Arc<Mutex<HashMap<String, PeerLink>>>
|
||||||
|
```
|
||||||
|
|
||||||
|
**22 lock sites** across federation.rs. The most important:
|
||||||
|
|
||||||
|
| Method | Line | Hold Duration | I/O While Locked | Frequency |
|
||||||
|
|--------|------|---------------|-------------------|-----------|
|
||||||
|
| `forward_to_peers()` | 406 | 1-5ms (iterate + sync send) | Sync only | Per-packet batch |
|
||||||
|
| `broadcast_signal()` | 216 | N × send_signal latency | **YES (async)** | Per-signal |
|
||||||
|
| `handle_datagram()` multi-hop | 1123 | 1-2ms (iterate + sync send) | Sync only | Per-federation-packet |
|
||||||
|
| `send_signal_to_peer()` | 246 | send_signal latency | **YES (async)** | Per-signal |
|
||||||
|
| Stale sweeper | 523 | 1-5ms | No | Every 5s |
|
||||||
|
|
||||||
|
**Impact:** Only matters with 5+ federation peers or high federation datagram rates (>1000 pps). For 1-3 peers, contention is negligible.
|
||||||
|
|
||||||
|
### Tier 2: Control Plane (Low Priority)
|
||||||
|
|
||||||
|
These are on the connection setup / signal path, not the media hot path:
|
||||||
|
|
||||||
|
| Lock | Location | Frequency |
|
||||||
|
|------|----------|-----------|
|
||||||
|
| `session_mgr` | main.rs:450 | Per-connection setup |
|
||||||
|
| `signal_hub` | main.rs:453 | Per-signal lookup |
|
||||||
|
| `call_registry` | main.rs:454 | Per-call setup |
|
||||||
|
| `presence` | main.rs:283 | Per-presence change |
|
||||||
|
| `ACL` | room.rs:357 | Per-room join |
|
||||||
|
|
||||||
|
**Impact:** None. These handle rare events (connection setup, call signaling) and hold locks for <5ms with no I/O inside.
|
||||||
|
|
||||||
|
### Tier 3: Forward Mode Pipeline (Niche)
|
||||||
|
|
||||||
|
| Lock | Location | Notes |
|
||||||
|
|------|----------|-------|
|
||||||
|
| `RelayPipeline` | main.rs:198, 228 | Only used in `--remote` forward mode (relay-to-relay), not SFU room mode |
|
||||||
|
|
||||||
|
**Impact:** None for normal operation. Forward mode is a niche deployment.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Suggested Next Refactors (Priority Order)
|
||||||
|
|
||||||
|
### 1. Federation `peer_links` Clone-Before-Send
|
||||||
|
|
||||||
|
**Effort:** 30 minutes
|
||||||
|
**Impact:** Eliminates the lock-held-during-iteration pattern in `forward_to_peers()` and `broadcast_signal()`
|
||||||
|
|
||||||
|
**Current:**
|
||||||
|
```rust
|
||||||
|
pub async fn forward_to_peers(&self, ...) {
|
||||||
|
let links = self.peer_links.lock().await; // held for entire loop
|
||||||
|
for (_fp, link) in links.iter() {
|
||||||
|
link.transport.send_raw_datagram(&tagged); // sync, but lock still held
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Fix:**
|
||||||
|
```rust
|
||||||
|
pub async fn forward_to_peers(&self, ...) {
|
||||||
|
let peers: Vec<(String, Arc<QuinnTransport>)> = {
|
||||||
|
let links = self.peer_links.lock().await;
|
||||||
|
links.values().map(|l| (l.label.clone(), l.transport.clone())).collect()
|
||||||
|
}; // lock released — hold time: ~1μs for Arc clones
|
||||||
|
|
||||||
|
for (label, transport) in &peers {
|
||||||
|
transport.send_raw_datagram(&tagged); // no lock held
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Same treatment for `broadcast_signal()` (line 216) which currently holds the lock across **async** `send_signal()` calls — this is the worst offender since a slow peer blocks all signal delivery.
|
||||||
|
|
||||||
|
### 2. Federation `peer_links` → DashMap
|
||||||
|
|
||||||
|
**Effort:** 2 hours
|
||||||
|
**Impact:** Per-peer sharding, eliminates all cross-peer contention
|
||||||
|
|
||||||
|
Only worth doing if:
|
||||||
|
- Running 10+ federation peers
|
||||||
|
- `forward_to_peers()` shows up in profiling
|
||||||
|
- The clone-before-send fix from suggestion 1 is insufficient
|
||||||
|
|
||||||
|
```rust
|
||||||
|
peer_links: DashMap<String, PeerLink>
|
||||||
|
```
|
||||||
|
|
||||||
|
Most lock sites become `self.peer_links.get(&fp)` or `.get_mut(&fp)`. The multi-hop forward loop would use `.iter()` which takes temporary shared locks per shard.
|
||||||
|
|
||||||
|
### 3. Quality Tracking Out of Hot Path
|
||||||
|
|
||||||
|
**Effort:** 1 day
|
||||||
|
**Impact:** Reduces per-packet DashMap shard lock from exclusive (`get_mut`) to shared (`get`)
|
||||||
|
|
||||||
|
Currently, every packet with a `QualityReport` calls `observe_quality()` which uses `rooms.get_mut()` (exclusive shard lock). This serializes quality-carrying packets within the same DashMap shard.
|
||||||
|
|
||||||
|
**Fix:** Use per-participant `AtomicU8` for latest loss/RTT (written lock-free from hot path). A background task (every 1s) reads the atomics, computes tiers via `rooms.get_mut()`, and broadcasts `QualityDirective`. The per-packet hot path becomes purely read-only: `rooms.get()` → `others()`.
|
||||||
|
|
||||||
|
```rust
|
||||||
|
struct ParticipantQualityAtomic {
|
||||||
|
latest_loss: AtomicU8, // written per-packet (lock-free)
|
||||||
|
latest_rtt: AtomicU8, // written per-packet (lock-free)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hot path (per-packet):
|
||||||
|
if let Some(ref qr) = pkt.quality_report {
|
||||||
|
participant_quality.latest_loss.store(qr.loss_pct, Ordering::Relaxed);
|
||||||
|
participant_quality.latest_rtt.store(qr.rtt_4ms, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
let others = room_mgr.others(&room_name, participant_id); // DashMap::get() — shared lock
|
||||||
|
|
||||||
|
// Background task (every 1 second):
|
||||||
|
for room in room_mgr.rooms.iter_mut() { // DashMap::iter_mut() — exclusive per-shard
|
||||||
|
room.recompute_tiers_from_atomics();
|
||||||
|
if tier_changed { broadcast QualityDirective }
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4. Lock-Free Participant Snapshot (Future)
|
||||||
|
|
||||||
|
**Effort:** 0.5 day
|
||||||
|
**Impact:** Zero-lock media hot path
|
||||||
|
|
||||||
|
Replace `Vec<Participant>` in `Room` with an `arc-swap` snapshot:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
struct Room {
|
||||||
|
participants: Vec<Participant>,
|
||||||
|
sender_snapshot: arc_swap::ArcSwap<Vec<ParticipantSender>>,
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The snapshot is rebuilt on join/leave (rare). The hot path does `sender_snapshot.load()` — an atomic pointer read with zero locking. DashMap wouldn't even be involved in the per-packet path.
|
||||||
|
|
||||||
|
Only worth doing if DashMap shard contention becomes measurable in profiling (unlikely for rooms <100 people).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Decision Matrix
|
||||||
|
|
||||||
|
| Scenario | Current (DashMap) | + Clone-Before-Send | + Quality Atomics | + arc-swap |
|
||||||
|
|----------|-------------------|---------------------|-------------------|-----------|
|
||||||
|
| 10 rooms × 5 people | Saturates all cores | Same | Same | Same |
|
||||||
|
| 1 room × 100 people | Good (shared read) | Same | Better (no exclusive) | Best |
|
||||||
|
| 5 federation peers | 1-5ms contention | <1μs contention | Same | Same |
|
||||||
|
| 20 federation peers | 10-20ms contention | <1μs contention | Same | Same |
|
||||||
|
| 1000 rooms × 3 people | Excellent | Same | Same | Same |
|
||||||
|
|
||||||
|
**Recommendation:** Do suggestion 1 (clone-before-send, 30 min) now. Everything else is future optimization that current workloads don't need.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Concurrency Diagram (Current State)
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────────────────────────────┐
|
||||||
|
│ tokio multi-threaded │
|
||||||
|
│ work-stealing runtime │
|
||||||
|
└───────────────┬─────────────────┘
|
||||||
|
│
|
||||||
|
┌────────────────────────────┼────────────────────────────┐
|
||||||
|
│ │ │
|
||||||
|
┌──────▼──────┐ ┌───────▼───────┐ ┌───────▼───────┐
|
||||||
|
│ QUIC Accept │ │ Federation │ │ Signal Hub │
|
||||||
|
│ (per-conn │ │ (per-peer │ │ (per-client │
|
||||||
|
│ task) │ │ task) │ │ task) │
|
||||||
|
└──────┬──────┘ └───────┬───────┘ └───────┬───────┘
|
||||||
|
│ │ │
|
||||||
|
┌──────▼──────┐ ┌───────▼───────┐ ┌───────▼───────┐
|
||||||
|
│ Per-Room │ │ peer_links │ │ signal_hub │
|
||||||
|
│ DashMap │◄──64 shards│ Mutex │◄──1 lock │ Mutex │
|
||||||
|
│ (media hot │ │ (federation │ │ (signal │
|
||||||
|
│ path) │ │ hot path) │ │ plane) │
|
||||||
|
└─────────────┘ └───────────────┘ └───────────────┘
|
||||||
|
│ │
|
||||||
|
No cross-room Low frequency
|
||||||
|
blocking (<1 call/sec)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Files Reference
|
||||||
|
|
||||||
|
| File | Lines | Role |
|
||||||
|
|------|-------|------|
|
||||||
|
| `crates/wzp-relay/src/room.rs` | ~1275 | DashMap room storage, participant management, quality tracking, media forwarding loops |
|
||||||
|
| `crates/wzp-relay/src/federation.rs` | ~1152 | Peer link management, federation media egress/ingress, signal forwarding |
|
||||||
|
| `crates/wzp-relay/src/main.rs` | ~1746 | Connection accept, handshake dispatch, signal handling, room/federation wiring |
|
||||||
|
| `crates/wzp-relay/src/ws.rs` | ~250 | WebSocket bridge, room integration |
|
||||||
|
| `crates/wzp-relay/src/metrics.rs` | ~200 | Prometheus counters (lock-free atomics) |
|
||||||
|
| `crates/wzp-relay/src/trunk.rs` | ~150 | TrunkBatcher (per-instance, no shared state) |
|
||||||
290
vault/Architecture/Road-To-Video.md
Normal file
290
vault/Architecture/Road-To-Video.md
Normal file
@@ -0,0 +1,290 @@
|
|||||||
|
---
|
||||||
|
tags: [architecture, wzp]
|
||||||
|
type: architecture
|
||||||
|
---
|
||||||
|
|
||||||
|
# Road to Video
|
||||||
|
|
||||||
|
> Plan for adding video to WZP. Audio remains unchanged through Phase V1; video is additive. See `PROTOCOL-AUDIT.md` for the issues this plan addresses.
|
||||||
|
|
||||||
|
## Premise
|
||||||
|
|
||||||
|
The transport, crypto, session, federation, and SFU layers are codec-agnostic. The work is concentrated in:
|
||||||
|
|
||||||
|
1. Wire format (CodecID width, MediaType, MiniHeader seq, simulcast hooks)
|
||||||
|
2. Framer / depacketizer (NAL fragmentation, access-unit reassembly)
|
||||||
|
3. Bandwidth estimator (Quinn cwnd + transport feedback)
|
||||||
|
4. Keyframe semantics (PLI, NACK, keyframe cache at SFU)
|
||||||
|
5. Capture / encode pipeline (VideoToolbox / MediaCodec / NVENC)
|
||||||
|
|
||||||
|
## Implementation Status (as of 2026-05-25)
|
||||||
|
|
||||||
|
| Phase | Description | Status |
|
||||||
|
|---|---|---|
|
||||||
|
| V1 — Wire format | 16B MediaHeader v2, 5B MiniHeader v2, MediaType, u32 seq, 8-bit CodecID | ✅ Complete (T1.x) |
|
||||||
|
| V2 — Transport additions | BWE, NACK loop, TransportFeedback, dynamic FEC boost on I-frames | 🔲 Not started |
|
||||||
|
| V3 — `wzp-video` crate | H.264 baseline framer/depacketizer, VideoToolbox/MediaCodec/dav1d encoders | ✅ Substantially complete (T4.x, T5.x, T6.x) |
|
||||||
|
| V3 — H.264 Baseline | Single-layer H.264 | ✅ Complete |
|
||||||
|
| V3 — H.265 | VideoToolbox + MediaCodec H.265 | ✅ Complete (T5.x) |
|
||||||
|
| V3 — AV1 | dav1d + SVT-AV1 (non-Android), VideoToolbox AV1 (macOS M3+) | ✅ Complete; Android MediaCodec AV1 compile errors pending (T4.3.1.1) |
|
||||||
|
| V3 — Android MediaCodec | NDK 0.9 API migration for `mediacodec.rs` | 🔴 Blocked (31 compile errors) |
|
||||||
|
| V3 — Call engine wiring | `create_video_encoder()` integrated into active call negotiation | 🔴 Not started (T6.1.2 follow-up) |
|
||||||
|
| V4 — Keyframe & loss policy | NACK path, PLI, keyframe cache at SFU | 🟡 Framework present (`nack.rs`); not wired |
|
||||||
|
| V5 — Video adaptive controller | `VideoQualityController` + `PriorityMode` | 🟡 Controller built (`controller.rs`); not wired into call |
|
||||||
|
| V5 — Simulcast | Simulcast layer management | 🟡 `simulcast.rs` present; not wired |
|
||||||
|
| V6 — SFU changes | Keyframe cache, per-receiver layer selection, PLI suppression | 🟡 PLI suppression wired; keyframe cache + layer selection not started |
|
||||||
|
| V6 — Video scorer | `VideoScorer` legitimacy detection | 🟡 Built (`video_scorer.rs`); `observe()` not wired into room forwarding |
|
||||||
|
| V7 — Capture pipeline | Camera capture (AVCaptureSession, Camera2, NVENC) | 🔲 Not started |
|
||||||
|
|
||||||
|
**Legend:** ✅ Complete · 🟡 Partial/Framework only · 🔴 Blocked · 🔲 Not started
|
||||||
|
|
||||||
|
### Critical path to first video call
|
||||||
|
|
||||||
|
1. Fix Android MediaCodec compile errors (T4.3.1.1) — ~2h
|
||||||
|
2. Wire `create_video_encoder()` into call engine codec negotiation (T6.1.2) — ~2h
|
||||||
|
3. Fix crypto nonce bug (`decrypt()` must use `MediaHeader.seq`) — see `AUDIT-2026-05-25.md` C1 — ~1h
|
||||||
|
4. Wire `VideoScorer::observe()` into relay room forwarding (T6.2 follow-up) — ~2h
|
||||||
|
5. Implement Phase V2 BWE (mandatory for usable video) — ~3–4 days
|
||||||
|
6. Implement capture pipeline for at least one platform (V7) — ~1 week
|
||||||
|
|
||||||
|
## Phase V1 — Wire format & negotiation (no new code paths yet)
|
||||||
|
|
||||||
|
Bump protocol version. Land all wire changes together so compat breaks exactly once.
|
||||||
|
|
||||||
|
### Sizing decision (2026-05-11)
|
||||||
|
|
||||||
|
Hypothetical benchmarks on 12 B packed vs 16 B byte-aligned showed the overhead delta is invisible across every realistic scenario:
|
||||||
|
|
||||||
|
| Scenario | Δ overhead (12 B → 16 B) | Δ % of stream |
|
||||||
|
|---|---|---|
|
||||||
|
| Opus 24k audio (MiniHeader 49/50) | 4 B/s | 0.013 % |
|
||||||
|
| Codec2 1200 audio | 2 B/s | 0.13 % |
|
||||||
|
| H.264 SD 500 kbps video | 1.6 kbps | 0.32 % |
|
||||||
|
| H.264 HD 2.5 Mbps video | 7.1 kbps | 0.28 % |
|
||||||
|
| H.264 FHD 5 Mbps video | 14.1 kbps | 0.28 % |
|
||||||
|
|
||||||
|
Trunking cap (10) binds before MTU for audio, so TrunkFrame layout is unaffected. ChaCha20-Poly1305 cost is dominated by AEAD setup, not byte count — 4 extra bytes per packet is < 0.1 % of AEAD CPU on Cortex-A55.
|
||||||
|
|
||||||
|
**Decision: 16 B byte-aligned.** Bit-packing saves nothing material and costs recurring debug / fuzzer / evolution complexity. Reserves headroom for the next decade.
|
||||||
|
|
||||||
|
### `MediaHeader` v2 (16 B byte-aligned)
|
||||||
|
|
||||||
|
```
|
||||||
|
Byte 0: version (u8) currently 0x02
|
||||||
|
Byte 1: flags (u8) [T:1][Q:1][KeyFrame:1][FrameEnd:1][reserved:4]
|
||||||
|
T = FEC repair
|
||||||
|
Q = QualityReport trailer present
|
||||||
|
KeyFrame = packet belongs to an I-frame (video)
|
||||||
|
FrameEnd = last packet of an access unit (video)
|
||||||
|
Byte 2: media_type (u8) 0=audio, 1=video, 2=data, 3=control
|
||||||
|
Byte 3: codec_id (u8) widened from 4-bit (room for 256)
|
||||||
|
Byte 4: stream_id (u8) simulcast layer; 0=base
|
||||||
|
Byte 5: fec_ratio (u8) 0..200 → 0.0..2.0
|
||||||
|
Bytes 6-9: sequence (u32 BE)
|
||||||
|
Bytes 10-13: timestamp_ms (u32 BE)
|
||||||
|
Bytes 14-15: fec_block_id (u16 BE)
|
||||||
|
audio: low 8 bits block_id, high 8 bits symbol_idx
|
||||||
|
video: full u16 block_id (large blocks for I-frames)
|
||||||
|
```
|
||||||
|
|
||||||
|
- `version=2` is a hard switch — old clients receive a typed `Hangup::ProtocolVersionMismatch`.
|
||||||
|
- `media_type` (W10) lets the SFU drop video first under load without a codec lookup.
|
||||||
|
- `KeyFrame` lets a joining peer fast-forward to the next I-frame; SFU keyframe cache keys on it.
|
||||||
|
- `FrameEnd` lets the depacketizer fire an access unit without counting packets.
|
||||||
|
- `stream_id` is forward-compatible for simulcast (Phase V5).
|
||||||
|
- `sequence` widened to u32 (W1) — also benefits audio.
|
||||||
|
|
||||||
|
### `MiniHeader` v2 (5 B)
|
||||||
|
|
||||||
|
```
|
||||||
|
[FRAME_TYPE_MINI = 0x01]
|
||||||
|
Byte 0: seq_delta (u8) ← new (W4)
|
||||||
|
Bytes 1-2: timestamp_delta_ms (u16 BE)
|
||||||
|
Bytes 3-4: payload_len (u16 BE)
|
||||||
|
```
|
||||||
|
|
||||||
|
Audio-only in V1. Video pays the full 16 B header per packet (every frame is a new access unit; no clean periodic structure to compress).
|
||||||
|
|
||||||
|
### New codec IDs
|
||||||
|
|
||||||
|
| ID | Codec | Notes |
|
||||||
|
|---|---|---|
|
||||||
|
| 9 | H.264 baseline | Universal HW encode coverage; ship first |
|
||||||
|
| 10 | H.264 main | Slight quality win over baseline; same HW |
|
||||||
|
| 11 | H.265 main | Apple A10+ universal, Snapdragon since ~2017, NVENC GTX 9xx+; ~30 % win vs H.264 |
|
||||||
|
| 12 | AV1 | Apple M3/A17+, Snapdragon 8 Gen 3+, RTX 40+, Arc, RX 7000+; best efficiency, narrow HW |
|
||||||
|
| 13 | VP9 | Reserved; may not implement |
|
||||||
|
|
||||||
|
Negotiation: `CallOffer.supported_codecs: Vec<CodecId>`. Both sides pick the highest mutually supported codec from preference cascade `[AV1, H.265, H.264 main, H.264 baseline]`.
|
||||||
|
|
||||||
|
### `QualityProfile` extension
|
||||||
|
|
||||||
|
Add:
|
||||||
|
- `video_bitrate_kbps: Option<u32>`
|
||||||
|
- `video_resolution: Option<(u16, u16)>`
|
||||||
|
- `video_fps: Option<u8>`
|
||||||
|
- `priority_mode: PriorityMode` (see Phase V5)
|
||||||
|
|
||||||
|
`CallOffer` / `CallAnswer` already negotiate profiles — slot video into the same path.
|
||||||
|
|
||||||
|
### Acceptance
|
||||||
|
- All 571 audio tests pass with `V=2` headers.
|
||||||
|
- Old v1 clients refused gracefully (clear error in `CallAnswer`).
|
||||||
|
|
||||||
|
## Phase V2 — Transport additions
|
||||||
|
|
||||||
|
**Decision (2026-05-11): all media on QUIC datagrams; no separate "reliable media" stream.**
|
||||||
|
|
||||||
|
A QUIC stream for I-frames was considered and rejected. A 200 KB I-frame on a 1 Mbps mobile link takes ~1.6 s to transit a stream, and the next I-frame queues behind it (HoL blocking by design). Datagrams + NACK + dynamic per-keyframe FEC degrade more gracefully on the lossy links we care about.
|
||||||
|
|
||||||
|
1. **All media on datagrams.** Uniform wire format; no HoL.
|
||||||
|
2. **NACK loop for video P-frames.** When `RTT < 2 × frame_interval`, receiver NACKs missing P-frame packets via `SignalMessage::Nack { stream_id, seqs }`. Otherwise (high RTT) skip NACK and request a keyframe via `PictureLossIndication`.
|
||||||
|
3. **Dynamic FEC boost on I-frames.** Encoder bumps `fec_ratio` to ~0.5 for keyframe packets (k=20 source → r=10 repair). Recovers most I-frame loss without a round trip.
|
||||||
|
4. **SPS/PPS / parameter sets on the existing signal stream.** Reliable, ordered, one-time at session start. Re-sent on codec switch. No new stream needed.
|
||||||
|
5. **`SignalMessage::TransportFeedback`** — `{ acked_seqs: Vec<u32>, nacked_seqs: Vec<u32>, remb_bps: u32, recv_time_us: u64 }`. Sent every 50 ms or every N packets, whichever first. Feeds BWE.
|
||||||
|
6. **`BandwidthEstimator` in `wzp-proto`** — consumes Quinn `cwnd`, `bytes_in_flight`, plus `TransportFeedback`. Output: `target_send_bps = min(cwnd_bps * 0.9, remb_bps)`.
|
||||||
|
|
||||||
|
### Acceptance
|
||||||
|
- Audio adapts to bandwidth (not just loss/RTT); fewer oscillations between 24 k and 32 k Opus on stable links.
|
||||||
|
- BWE output is on Prometheus.
|
||||||
|
- NACK round-trip recovery verified under 1–5 % packet loss at RTT ≤ 100 ms.
|
||||||
|
|
||||||
|
## Phase V3 — `wzp-video` crate
|
||||||
|
|
||||||
|
New crate parallel to `wzp-codec`:
|
||||||
|
|
||||||
|
```
|
||||||
|
wzp-video/
|
||||||
|
src/
|
||||||
|
encoder.rs # trait VideoEncoder; VideoToolboxEncoder, MediaCodecEncoder,
|
||||||
|
# OpenH264Encoder fallback
|
||||||
|
decoder.rs # trait VideoDecoder
|
||||||
|
framer.rs # NAL unit fragmentation to MTU-sized chunks
|
||||||
|
# (simpler than RFC 6184 FU-A — we own both ends)
|
||||||
|
depacketizer.rs # Reassemble NALs, emit access units
|
||||||
|
keyframe.rs # Keyframe request handling
|
||||||
|
```
|
||||||
|
|
||||||
|
Framing rules:
|
||||||
|
- One access unit → N packets, each ≤ MTU − 12 (MediaHeader) − 16 (AEAD tag).
|
||||||
|
- `sequence` global per stream; `timestamp_ms` is presentation time.
|
||||||
|
- `KeyFrame` bit set on every packet of an I-frame.
|
||||||
|
- Last packet of frame: "frame end" bit (steal from `StreamId` or repurpose `reserved`).
|
||||||
|
|
||||||
|
Platform encoders:
|
||||||
|
- macOS / iOS: VideoToolbox
|
||||||
|
- Android: MediaCodec (surface texture path, no CPU copy)
|
||||||
|
- Windows: MediaFoundation → NVENC / QSV / AMF
|
||||||
|
- Linux: VAAPI / NVENC; OpenH264 software fallback
|
||||||
|
|
||||||
|
### Acceptance
|
||||||
|
- Unidirectional H.264 call working between two desktop clients.
|
||||||
|
- CPU usage on M1 < 5 % at 720p30; on Android mid-tier < 15 %.
|
||||||
|
|
||||||
|
## Phase V4 — Keyframe & loss policy
|
||||||
|
|
||||||
|
- On packet loss inside a P-frame: NACK if RTT < 2× frame interval, otherwise request keyframe via `SignalMessage::PictureLossIndication { stream_id }`.
|
||||||
|
- Joining peer: relay sends most recent keyframe from its cache.
|
||||||
|
- Tier downgrade: drop to lower simulcast layer, request keyframe for the new layer.
|
||||||
|
|
||||||
|
### Acceptance
|
||||||
|
- Black-screen-on-join < 200 ms when keyframe cache is warm.
|
||||||
|
- < 1 keyframe / 2 s on stable links; bursty on lossy links.
|
||||||
|
|
||||||
|
## Phase V5 — Video adaptive controller + PriorityMode
|
||||||
|
|
||||||
|
### `PriorityMode` on `QualityProfile`
|
||||||
|
|
||||||
|
```rust
|
||||||
|
pub enum PriorityMode {
|
||||||
|
AudioFirst, // default for calls: audio absolute priority, video elastic
|
||||||
|
VideoFirst, // user override: video priority, audio degrades second
|
||||||
|
ScreenShare, // video + slide-fallback; audio = intelligible speech only
|
||||||
|
Balanced, // proportional split, no absolute priority
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Selected at call setup. Mutable mid-call via `SignalMessage::SetPriorityMode { mode }`. Defaults to `AudioFirst` for voice/video calls; presentation apps set `ScreenShare`; users can override to `VideoFirst` from settings.
|
||||||
|
|
||||||
|
### `VideoQualityController`
|
||||||
|
|
||||||
|
```
|
||||||
|
inputs: bwe_bps, loss_pct, rtt_ms, encoder_queue_ms, priority_mode
|
||||||
|
outputs: target_bitrate, target_fps, target_resolution, simulcast_layer
|
||||||
|
|
||||||
|
allocation gate (per PriorityMode):
|
||||||
|
|
||||||
|
AudioFirst:
|
||||||
|
audio_budget = max(24 kbps, audio_tier_min)
|
||||||
|
video_budget = bwe_bps - audio_budget
|
||||||
|
Under congestion: video → 0 before audio degrades.
|
||||||
|
|
||||||
|
VideoFirst:
|
||||||
|
video_budget = max(video_floor, target_video_kbps)
|
||||||
|
audio_budget = bwe_bps - video_budget
|
||||||
|
Audio degrades first to Opus 16 k; video held at floor.
|
||||||
|
|
||||||
|
ScreenShare:
|
||||||
|
video_budget = bwe_bps - 16 kbps // audio gets just Opus 16 k floor
|
||||||
|
If video_budget < SD floor: switch encoder to slide mode
|
||||||
|
(single high-quality I-frame every 2-5s instead of continuous video).
|
||||||
|
Audio floor in this mode is Opus 16 k (speech only, no music).
|
||||||
|
|
||||||
|
Balanced:
|
||||||
|
audio_budget = bwe_bps * 0.15
|
||||||
|
video_budget = bwe_bps * 0.85
|
||||||
|
Both degrade proportionally.
|
||||||
|
```
|
||||||
|
|
||||||
|
Slide mode in `ScreenShare` is an encoder policy on the existing `wzp-video` framer (lower fps, higher per-frame quality, prefer HEVC/AV1 for text). No wire format change.
|
||||||
|
|
||||||
|
### Acceptance
|
||||||
|
- On a 100 kbps link in `AudioFirst`, audio stays at Opus 24 k and video drops to 0.
|
||||||
|
- On a 100 kbps link in `ScreenShare`, slide mode emits one I-frame every 3 s and audio holds Opus 16 k.
|
||||||
|
- On a 5 Mbps link, video ramps to top simulcast layer within 10 s.
|
||||||
|
- `SetPriorityMode` mid-call is honored within 1 s.
|
||||||
|
|
||||||
|
## Phase V6 — SFU changes
|
||||||
|
|
||||||
|
- **Per-room keyframe cache.** Latest I-frame per `(sender, stream_id)`. Sent to new joiners immediately. Eliminates "black screen for 2 seconds" on join.
|
||||||
|
- **Per-receiver layer selection.** Sender uploads ~3 simulcast layers; relay decides which to forward to each receiver based on their last `QualityReport`. Critical for N > 3 rooms.
|
||||||
|
- **PLI suppression.** If 10 receivers PLI within 200 ms, send one `KeyframeRequest` upstream, not 10.
|
||||||
|
|
||||||
|
### Acceptance
|
||||||
|
- 8-peer room with mixed link quality; high-quality peers see HD, low-quality peers see SD, no peer holds the room back.
|
||||||
|
- PLI traffic at SFU upstream < 1 / s under simulated mass packet loss.
|
||||||
|
|
||||||
|
## Phase V7 — Capture pipeline (platform-specific)
|
||||||
|
|
||||||
|
- macOS: `AVCaptureSession` → VideoToolbox → `wzp-video`. Wire into Tauri backend.
|
||||||
|
- Android: Camera2 → MediaCodec → JNI bridge into `wzp-native` or sibling cdylib. Surface texture path.
|
||||||
|
- Desktop Tauri (Windows): MediaFoundation → NVENC.
|
||||||
|
|
||||||
|
### Acceptance
|
||||||
|
- Camera permission flows on all platforms.
|
||||||
|
- < 50 ms end-to-end capture-to-encode latency on M1.
|
||||||
|
|
||||||
|
## Deferred
|
||||||
|
|
||||||
|
- **SVC** (per-layer temporal scalability in one bitstream). Simulcast (separate streams per layer) is enough for v1; wire format already supports it via `StreamId`.
|
||||||
|
- **Screen sharing.** Same codec path with a different capture source.
|
||||||
|
- **Group video keys.** Existing X25519 session key works; no protocol change needed.
|
||||||
|
|
||||||
|
## Suggested order of work
|
||||||
|
|
||||||
|
| Step | Effort | Output |
|
||||||
|
|---|---|---|
|
||||||
|
| 1. Wire format v2: 16 B MediaHeader, 5 B MiniHeader, MediaType, KeyFrame, FrameEnd, u32 seq, 8-bit CodecID | ~1 day | Audio still works under new header layout |
|
||||||
|
| 2. TransportFeedback + BandwidthEstimator (Quinn cwnd + remb) | 3–4 days | Audio adaptation improves; BWE on Prom |
|
||||||
|
| 3. `wzp-video` crate, H.264 baseline single-layer | 1–2 weeks | Unidirectional video call works |
|
||||||
|
| 4. NACK path + dynamic FEC boost on I-frames | 4–5 days | Loss recovery for video |
|
||||||
|
| 5. Keyframe cache at SFU + PLI suppression | 1 week | Fast join, low PLI traffic |
|
||||||
|
| 6. H.265 codec support (reuse framer) | 3 days | ~30 % quality win on Apple HW |
|
||||||
|
| 7. Simulcast + per-receiver layer selection | 1 week | Mixed-quality rooms work |
|
||||||
|
| 8. `VideoQualityController` + PriorityMode (incl. ScreenShare slide mode) | 1 week | Graceful degradation under congestion, user choice |
|
||||||
|
| 9. AV1 codec (gated on HW telemetry) | 4–5 days | Top-tier efficiency on capable devices |
|
||||||
|
| 10. Native capture pipelines (VideoToolbox / MediaCodec / NVENC) | 2 weeks | Production camera support per OS |
|
||||||
|
|
||||||
|
Step 1 is the lowest-regret, highest-leverage change and unlocks everything else.
|
||||||
|
|
||||||
|
Steps 3 + 6 + 9 form the codec rollout: ship H.264 first (works everywhere → unblocks integration testing on every device), add H.265 once framer is stable (low-effort, big Apple win), gate AV1 on real device telemetry. By 2028 we should be in a position to deprecate H.264 if telemetry says < 5 % of sessions still need it.
|
||||||
262
vault/Architecture/WS-Relay-Spec.md
Normal file
262
vault/Architecture/WS-Relay-Spec.md
Normal file
@@ -0,0 +1,262 @@
|
|||||||
|
---
|
||||||
|
tags: [architecture, wzp]
|
||||||
|
type: architecture
|
||||||
|
---
|
||||||
|
|
||||||
|
# WS Support in wzp-relay — Implementation Spec
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
Add WebSocket listener to `wzp-relay` so browsers connect directly, eliminating `wzp-web` bridge.
|
||||||
|
|
||||||
|
```
|
||||||
|
Before: Browser → WS → wzp-web → QUIC → wzp-relay
|
||||||
|
After: Browser → WS → wzp-relay (handles both WS + QUIC)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
wzp-relay
|
||||||
|
├── QUIC listener (:4433) — native clients, inter-relay
|
||||||
|
├── WS listener (:8080) — browsers via Caddy
|
||||||
|
│ ├── GET /ws/{room} — WebSocket upgrade
|
||||||
|
│ └── Auth: first msg = {"type":"auth","token":"..."}
|
||||||
|
└── Shared RoomManager — both transports in same rooms
|
||||||
|
```
|
||||||
|
|
||||||
|
## Key Changes
|
||||||
|
|
||||||
|
### 1. Abstract `Participant` over transport type
|
||||||
|
|
||||||
|
**File: `room.rs`**
|
||||||
|
|
||||||
|
Currently:
|
||||||
|
```rust
|
||||||
|
struct Participant {
|
||||||
|
id: ParticipantId,
|
||||||
|
_addr: std::net::SocketAddr,
|
||||||
|
transport: Arc<wzp_transport::QuinnTransport>,
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Change to:
|
||||||
|
```rust
|
||||||
|
struct Participant {
|
||||||
|
id: ParticipantId,
|
||||||
|
_addr: std::net::SocketAddr,
|
||||||
|
sender: ParticipantSender,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// How to send a media packet to a participant.
|
||||||
|
enum ParticipantSender {
|
||||||
|
Quic(Arc<wzp_transport::QuinnTransport>),
|
||||||
|
WebSocket(tokio::sync::mpsc::Sender<bytes::Bytes>),
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The `others()` method returns `Vec<ParticipantSender>` instead of `Vec<Arc<QuinnTransport>>`.
|
||||||
|
|
||||||
|
`ParticipantSender` implements a `send_pcm(&self, data: &[u8])` method:
|
||||||
|
- **Quic**: wraps in `MediaPacket`, calls `transport.send_media()`
|
||||||
|
- **WebSocket**: sends raw binary frame via the mpsc channel
|
||||||
|
|
||||||
|
### 2. Add `join_ws()` to RoomManager
|
||||||
|
|
||||||
|
```rust
|
||||||
|
pub fn join_ws(
|
||||||
|
&mut self,
|
||||||
|
room_name: &str,
|
||||||
|
addr: std::net::SocketAddr,
|
||||||
|
sender: tokio::sync::mpsc::Sender<bytes::Bytes>,
|
||||||
|
fingerprint: Option<&str>,
|
||||||
|
) -> Result<ParticipantId, String>
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3. Add WS listener in `main.rs`
|
||||||
|
|
||||||
|
New flag: `--ws-port 8080`
|
||||||
|
|
||||||
|
```rust
|
||||||
|
if let Some(ws_port) = config.ws_port {
|
||||||
|
let room_mgr = room_mgr.clone();
|
||||||
|
let auth_url = config.auth_url.clone();
|
||||||
|
let metrics = metrics.clone();
|
||||||
|
tokio::spawn(run_ws_server(ws_port, room_mgr, auth_url, metrics));
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4. WebSocket handler (`ws.rs` — new file)
|
||||||
|
|
||||||
|
```rust
|
||||||
|
use axum::{
|
||||||
|
extract::{ws::{Message, WebSocket}, Path, WebSocketUpgrade},
|
||||||
|
routing::get,
|
||||||
|
Router,
|
||||||
|
};
|
||||||
|
|
||||||
|
async fn ws_handler(
|
||||||
|
Path(room): Path<String>,
|
||||||
|
ws: WebSocketUpgrade,
|
||||||
|
/* state */
|
||||||
|
) -> impl IntoResponse {
|
||||||
|
ws.on_upgrade(move |socket| handle_ws(socket, room, state))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_ws(mut socket: WebSocket, room: String, state: WsState) {
|
||||||
|
let addr = /* peer addr */;
|
||||||
|
|
||||||
|
// 1. Auth: first message must be {"type":"auth","token":"..."}
|
||||||
|
let fingerprint = if let Some(ref auth_url) = state.auth_url {
|
||||||
|
match socket.recv().await {
|
||||||
|
Some(Ok(Message::Text(text))) => {
|
||||||
|
let parsed: serde_json::Value = serde_json::from_str(&text)?;
|
||||||
|
if parsed["type"] == "auth" {
|
||||||
|
let token = parsed["token"].as_str().unwrap();
|
||||||
|
let client = auth::validate_token(auth_url, token).await?;
|
||||||
|
Some(client.fingerprint)
|
||||||
|
} else { return; }
|
||||||
|
}
|
||||||
|
_ => return,
|
||||||
|
}
|
||||||
|
} else { None };
|
||||||
|
|
||||||
|
// 2. Create mpsc channel for outbound frames
|
||||||
|
let (tx, mut rx) = tokio::sync::mpsc::channel::<bytes::Bytes>(64);
|
||||||
|
|
||||||
|
// 3. Join room
|
||||||
|
let participant_id = {
|
||||||
|
let mut mgr = state.room_mgr.lock().await;
|
||||||
|
mgr.join_ws(&room, addr, tx, fingerprint.as_deref())?
|
||||||
|
};
|
||||||
|
|
||||||
|
// 4. Run send/recv loops
|
||||||
|
let (mut ws_tx, mut ws_rx) = socket.split();
|
||||||
|
|
||||||
|
// Outbound: mpsc rx → WS send
|
||||||
|
let send_task = tokio::spawn(async move {
|
||||||
|
while let Some(data) = rx.recv().await {
|
||||||
|
if ws_tx.send(Message::Binary(data.to_vec())).await.is_err() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Inbound: WS recv → fan-out to room
|
||||||
|
loop {
|
||||||
|
match ws_rx.next().await {
|
||||||
|
Some(Ok(Message::Binary(data))) => {
|
||||||
|
// Raw PCM Int16 from browser — fan-out to all others
|
||||||
|
let others = {
|
||||||
|
let mgr = state.room_mgr.lock().await;
|
||||||
|
mgr.others(&room, participant_id)
|
||||||
|
};
|
||||||
|
for other in &others {
|
||||||
|
other.send_raw(&data);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Some(Ok(Message::Close(_))) | None => break,
|
||||||
|
_ => continue,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 5. Cleanup
|
||||||
|
send_task.abort();
|
||||||
|
let mut mgr = state.room_mgr.lock().await;
|
||||||
|
mgr.leave(&room, participant_id);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5. Cross-transport fan-out
|
||||||
|
|
||||||
|
When a QUIC participant sends audio → WS participants receive raw PCM bytes.
|
||||||
|
When a WS participant sends audio → QUIC participants receive a `MediaPacket`.
|
||||||
|
|
||||||
|
The `ParticipantSender::send_raw()` method:
|
||||||
|
```rust
|
||||||
|
impl ParticipantSender {
|
||||||
|
async fn send_raw(&self, pcm_bytes: &[u8]) {
|
||||||
|
match self {
|
||||||
|
ParticipantSender::WebSocket(tx) => {
|
||||||
|
let _ = tx.try_send(bytes::Bytes::copy_from_slice(pcm_bytes));
|
||||||
|
}
|
||||||
|
ParticipantSender::Quic(transport) => {
|
||||||
|
// Wrap raw PCM in a MediaPacket
|
||||||
|
let pkt = MediaPacket {
|
||||||
|
header: MediaHeader::default_pcm(),
|
||||||
|
payload: bytes::Bytes::copy_from_slice(pcm_bytes),
|
||||||
|
quality_report: None,
|
||||||
|
};
|
||||||
|
let _ = transport.send_media(&pkt).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
For QUIC→WS direction, `run_participant` extracts `pkt.payload` bytes and sends to WS channels.
|
||||||
|
|
||||||
|
### 6. Dependencies to add
|
||||||
|
|
||||||
|
```toml
|
||||||
|
# wzp-relay/Cargo.toml
|
||||||
|
axum = { version = "0.8", features = ["ws"] }
|
||||||
|
tokio = { version = "1", features = ["full"] } # already present
|
||||||
|
```
|
||||||
|
|
||||||
|
### 7. Config change
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// config.rs
|
||||||
|
pub struct RelayConfig {
|
||||||
|
// ... existing fields ...
|
||||||
|
pub ws_port: Option<u16>,
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 8. Docker compose change (featherChat side)
|
||||||
|
|
||||||
|
Remove `wzp-web` service entirely. Update Caddy to proxy `/audio/*` to relay's WS port:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# Before:
|
||||||
|
wzp-web:
|
||||||
|
entrypoint: ["wzp-web"]
|
||||||
|
command: ["--port", "8080", "--relay", "172.28.0.10:4433"]
|
||||||
|
|
||||||
|
# After: REMOVED. Relay handles WS directly.
|
||||||
|
|
||||||
|
wzp-relay:
|
||||||
|
command:
|
||||||
|
- "--listen"
|
||||||
|
- "0.0.0.0:4433"
|
||||||
|
- "--ws-port"
|
||||||
|
- "8080"
|
||||||
|
- "--auth-url"
|
||||||
|
- "http://warzone-server:7700/v1/auth/validate"
|
||||||
|
```
|
||||||
|
|
||||||
|
## What Stays the Same
|
||||||
|
|
||||||
|
- Browser's `startAudio()` — unchanged, still connects WS to `/audio/ws/ROOM`
|
||||||
|
- Caddy proxies `/audio/*` → relay:8080 (same path, different backend)
|
||||||
|
- Auth flow — same JSON token as first message
|
||||||
|
- PCM format — same Int16 binary frames
|
||||||
|
- QUIC clients — unchanged, still connect to :4433
|
||||||
|
- Room naming, ACL, session management — all unchanged
|
||||||
|
|
||||||
|
## Testing
|
||||||
|
|
||||||
|
1. Start relay with `--ws-port 8080 --listen 0.0.0.0:4433`
|
||||||
|
2. Open browser, initiate call via featherChat
|
||||||
|
3. Verify audio flows (both directions)
|
||||||
|
4. Verify QUIC + WS clients can be in same room (mixed mode)
|
||||||
|
5. Verify auth works
|
||||||
|
6. Verify room cleanup on disconnect
|
||||||
|
|
||||||
|
## Migration Path
|
||||||
|
|
||||||
|
1. Implement WS in relay
|
||||||
|
2. Test with featherChat (no featherChat changes needed)
|
||||||
|
3. Remove wzp-web from Docker stack
|
||||||
|
4. Later: add WebTransport alongside WS
|
||||||
152
vault/Architecture/WZP-Spec.md
Normal file
152
vault/Architecture/WZP-Spec.md
Normal file
@@ -0,0 +1,152 @@
|
|||||||
|
---
|
||||||
|
tags: [architecture, wzp]
|
||||||
|
type: architecture
|
||||||
|
---
|
||||||
|
|
||||||
|
# WZP Protocol Specification (one-page reference)
|
||||||
|
|
||||||
|
> Distilled from `docs/ARCHITECTURE.md` and the `wzp-proto` crate. Authoritative wire details live in `crates/wzp-proto/src/packet.rs`.
|
||||||
|
>
|
||||||
|
> **Status:** v2 is the deployed protocol (audio + video, 16 B header, MediaType, u32 seq). v1 clients are rejected with `Hangup::ProtocolVersionMismatch`.
|
||||||
|
|
||||||
|
## Layer summary
|
||||||
|
|
||||||
|
| Layer | WZP | FaceTime equivalent |
|
||||||
|
|---|---|---|
|
||||||
|
| Transport | **QUIC datagrams** (Quinn), PLPMTUD 1200 → 1452 | RTP/SRTP over UDP, ICE |
|
||||||
|
| Signaling | `SignalMessage` (bincode) over a QUIC stream, SNI = hashed room name | APNs-tunneled binary plist |
|
||||||
|
| Identity | Ed25519 + X25519 from BIP39 seed; fingerprint = SHA-256(pubkey)[..16] | IDS RSA + ECDSA per device |
|
||||||
|
| Key agreement | X25519 DH + HKDF, Ed25519 signatures, rekey every 65,536 packets | Per-call DH signed by IDS keys |
|
||||||
|
| Bulk crypto | ChaCha20-Poly1305, 64-packet sliding anti-replay | SRTP (AES-CTR + HMAC) |
|
||||||
|
| Loss recovery | **RaptorQ FEC + Opus DRED + classical PLC** | NACK / PLI + reference-picture selection |
|
||||||
|
| Adaptive | 3-tier hysteresis (Good / Degraded / Catastrophic) + continuous DRED tuner | Per-frame bitrate ladder |
|
||||||
|
| Topology | SFU rooms + inter-relay federation + P2P via ICE | Mesh ≤ ~3, SFU above, Apple relays |
|
||||||
|
| Header | 16 B `MediaHeader` v2 / 5 B `MiniHeader` (49 of 50), 4 B `QualityReport` trailer | RTP 12 B + extensions |
|
||||||
|
|
||||||
|
## Distinctive choices
|
||||||
|
|
||||||
|
- **QUIC datagrams instead of raw UDP + SRTP.** Brings TLS 1.3, PLPMTUD, path migration, and ACK-based RTT/loss estimation for free.
|
||||||
|
- **Continuous DRED tuning.** Maps live `(loss%, RTT, jitter)` to a continuous Opus DRED lookback window. Most stacks treat DRED as discrete tiers.
|
||||||
|
- **MiniHeader (5 B for 49/50 packets).** Saves ~11 B/packet ≈ 550 B/s/stream at 50 pps vs. the full 16 B header.
|
||||||
|
- **E2E-preserving SFU.** The relay forwards encrypted datagrams; it never decrypts media. Room membership uses SNI = `hash(room_name)`.
|
||||||
|
- **Codec coordination via `QualityReport` trailer.** Receivers attach 4-byte loss/RTT/jitter/cap to media packets; the SFU broadcasts `QualityDirective` so all senders in a room converge on the same tier.
|
||||||
|
|
||||||
|
## Wire format (current — v2)
|
||||||
|
|
||||||
|
### `MediaHeader` v2 (16 bytes, byte-aligned)
|
||||||
|
|
||||||
|
```
|
||||||
|
Byte 0: version (u8) 0x02
|
||||||
|
Byte 1: flags (u8) [T:1][Q:1][KeyFrame:1][FrameEnd:1][reserved:4]
|
||||||
|
Byte 2: media_type (u8) 0=audio, 1=video, 2=data, 3=control
|
||||||
|
Byte 3: codec_id (u8) 0-255 (see codec table)
|
||||||
|
Byte 4: stream_id (u8) simulcast layer; 0=base
|
||||||
|
Byte 5: fec_ratio (u8) 0..200 → 0.0..2.0
|
||||||
|
Bytes 6-9: sequence (u32 BE)
|
||||||
|
Bytes 10-13: timestamp_ms (u32 BE)
|
||||||
|
Bytes 14-15: fec_block_id (u16 BE)
|
||||||
|
```
|
||||||
|
|
||||||
|
| Field | Bits | Meaning |
|
||||||
|
|---|---|---|
|
||||||
|
| version | 8 | Must be `0x02`; v1 clients receive `Hangup::ProtocolVersionMismatch` |
|
||||||
|
| T (bit 7 of flags) | 1 | 1 = FEC repair packet |
|
||||||
|
| Q (bit 6 of flags) | 1 | QualityReport trailer present |
|
||||||
|
| KeyFrame (bit 5 of flags) | 1 | Packet belongs to a video I-frame |
|
||||||
|
| FrameEnd (bit 4 of flags) | 1 | Last packet of an access unit |
|
||||||
|
| reserved (bits 3-0 of flags) | 4 | Must be zero |
|
||||||
|
| media_type | 8 | 0=audio, 1=video, 2=data, 3=control |
|
||||||
|
| codec_id | 8 | See codec table (widened from v1's 4-bit field) |
|
||||||
|
| stream_id | 8 | Simulcast layer; 0=base layer |
|
||||||
|
| fec_ratio | 8 | 0..200 → 0.0..2.0 |
|
||||||
|
| sequence | 32 | Monotonically increasing packet seq (not reset by rekey) |
|
||||||
|
| timestamp_ms | 32 | ms since session start. Monotonic across the full session; **not reset by rekey** |
|
||||||
|
| fec_block_id | 16 | FEC source block ID |
|
||||||
|
|
||||||
|
### Codec table
|
||||||
|
|
||||||
|
| ID | Codec | Bitrate | Sample | Frame |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| 0 | Opus 24k | 24 kbps | 48 kHz | 20 ms |
|
||||||
|
| 1 | Opus 16k | 16 kbps | 48 kHz | 20 ms |
|
||||||
|
| 2 | Opus 6k | 6 kbps | 48 kHz | 40 ms |
|
||||||
|
| 3 | Codec2 3200 | 3.2 kbps | 8 kHz | 20 ms |
|
||||||
|
| 4 | Codec2 1200 | 1.2 kbps | 8 kHz | 40 ms |
|
||||||
|
| 5 | ComfortNoise | 0 | 48 kHz | 20 ms |
|
||||||
|
| 6 | Opus 32k | 32 kbps | 48 kHz | 20 ms |
|
||||||
|
| 7 | Opus 48k | 48 kbps | 48 kHz | 20 ms |
|
||||||
|
| 8 | Opus 64k | 64 kbps | 48 kHz | 20 ms |
|
||||||
|
| 9 | H.264 Baseline | — | — | — |
|
||||||
|
| 10 | H.264 Main | — | — | — |
|
||||||
|
| 11 | H.265 Main | — | — | — |
|
||||||
|
| 12 | AV1 Main | — | — | — |
|
||||||
|
|
||||||
|
### `MiniHeader` v2 (5 bytes, compressed — 49 of every 50 packets)
|
||||||
|
|
||||||
|
```
|
||||||
|
[FRAME_TYPE_MINI = 0x01]
|
||||||
|
Byte 0: seq_delta (u8)
|
||||||
|
Bytes 1-2: timestamp_delta_ms (u16 BE)
|
||||||
|
Bytes 3-4: payload_len (u16 BE)
|
||||||
|
```
|
||||||
|
|
||||||
|
Full header sent every 50th packet to resync.
|
||||||
|
|
||||||
|
### `TrunkFrame` (batched, relay-internal)
|
||||||
|
|
||||||
|
```
|
||||||
|
[count: u16]
|
||||||
|
[session_id: 2][len: u16][payload: len] × count
|
||||||
|
```
|
||||||
|
|
||||||
|
Up to 10 entries or PMTUD-discovered MTU; flushed every 5 ms.
|
||||||
|
|
||||||
|
### `QualityReport` (4 bytes, optional inline trailer)
|
||||||
|
|
||||||
|
```
|
||||||
|
Byte 0: loss_pct (0-255 → 0-100%)
|
||||||
|
Byte 1: rtt_4ms (0-255 → 0-1020 ms)
|
||||||
|
Byte 2: jitter_ms (0-255 ms)
|
||||||
|
Byte 3: bitrate_cap_kbps (0-255 kbps)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Version negotiation
|
||||||
|
|
||||||
|
- `version=0x02` in `MediaHeader` is a hard switch — there is no fallback negotiation.
|
||||||
|
- Both endpoints must speak v2. A v1 peer receives `Hangup::ProtocolVersionMismatch` immediately.
|
||||||
|
- Relays inspect only `version` and `media_type`; they never downgrade or translate between versions.
|
||||||
|
|
||||||
|
## Session lifecycle
|
||||||
|
|
||||||
|
```
|
||||||
|
Idle → Connecting → Handshaking → Active ⇄ Rekeying → Closed
|
||||||
|
```
|
||||||
|
|
||||||
|
- `CallOffer { identity_pub, ephemeral_pub, signature, profiles }`
|
||||||
|
- `CallAnswer { identity_pub, ephemeral_pub, signature, chosen_profile }`
|
||||||
|
- `session_key = HKDF(X25519_DH(eph_a, eph_b), "warzone-session-key")`
|
||||||
|
- Rekey every 65,536 packets via fresh ephemeral DH.
|
||||||
|
|
||||||
|
## SFU forwarding rules
|
||||||
|
|
||||||
|
1. Fan-out to all room participants except the sender.
|
||||||
|
2. Failed sends are skipped; forwarding is best-effort.
|
||||||
|
3. The relay never decrypts media.
|
||||||
|
4. With trunking on, packets to the same receiver are batched (flush 5 ms).
|
||||||
|
5. `QualityDirective` is broadcast when the room-wide tier degrades.
|
||||||
|
|
||||||
|
## Adaptive quality (audio, today)
|
||||||
|
|
||||||
|
| Tier | Codec | FEC | Frame |
|
||||||
|
|---|---|---|---|
|
||||||
|
| Good | Opus 24 k | 20 % | 20 ms |
|
||||||
|
| Degraded | Opus 6 k | 50 % | 40 ms |
|
||||||
|
| Catastrophic | Codec2 1200 | 100 % | 40 ms |
|
||||||
|
|
||||||
|
Hysteresis: 3 reports to downgrade (2 on cellular), 10 to upgrade.
|
||||||
|
|
||||||
|
## NAT traversal (Phase 8)
|
||||||
|
|
||||||
|
- Candidate types: Host, Port-mapped (NAT-PMP / PCP / UPnP), Server-reflexive (STUN), Relay.
|
||||||
|
- Hard-NAT port prediction with `classify_port_allocation()` → `predict_ports()` → `HardNatProbe` signal.
|
||||||
|
- Mid-call re-gather: `CandidateUpdate { generation }`.
|
||||||
237
vault/Audit/Audit-2026-05-25.md
Normal file
237
vault/Audit/Audit-2026-05-25.md
Normal file
@@ -0,0 +1,237 @@
|
|||||||
|
---
|
||||||
|
tags: [audit, wzp]
|
||||||
|
type: audit
|
||||||
|
created: 2026-05-25
|
||||||
|
---
|
||||||
|
|
||||||
|
# WarzonePhone Protocol Audit — 2026-05-25
|
||||||
|
|
||||||
|
**Auditor:** Claude Sonnet 4.6 (assisted)
|
||||||
|
**Branch:** `experimental-ui` @ `f3e3ee5`
|
||||||
|
**Scope:** All workspace crates (`wzp-proto`, `wzp-codec`, `wzp-fec`, `wzp-crypto`, `wzp-transport`, `wzp-relay`, `wzp-client`, `wzp-android`, `wzp-native`, `wzp-video`)
|
||||||
|
**Test baseline:** 702 passing (excludes `wzp-android`)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Executive Summary
|
||||||
|
|
||||||
|
The audio call path is functionally correct and cryptographically sound on clean network paths. **There is a session-breaking bug in the crypto nonce derivation (C1) that will cause a permanent decryption failure on any out-of-order UDP delivery.** This is the single highest-priority fix — it will manifest as periodic session crashes under normal internet conditions. Video has a solid architectural foundation but three hard blockers remain before shipping: the AEAD coverage gap (C2), dead video scorer (C3), and Android MediaCodec compile failure (C4).
|
||||||
|
|
||||||
|
The project is in good shape overall. The crypto design (X25519, HKDF, ChaCha20-Poly1305, Ed25519 identity, SAS verification) is sound. The SFU-never-decrypts architecture is rare and valuable. The codec adaptation (Opus DRED + Codec2 RaptorQ split) is genuinely innovative. The eight issues below are fixable in ~12 engineer-hours.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Critical
|
||||||
|
|
||||||
|
### C1 — Nonce derives from `recv_seq` counter, not `MediaHeader.seq`
|
||||||
|
|
||||||
|
**File:** `crates/wzp-crypto/src/session.rs:132`
|
||||||
|
**Severity:** Critical — session-breaking on any packet reorder
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// decrypt()
|
||||||
|
let nonce_bytes = nonce::build_nonce(&self.session_id, self.recv_seq, Direction::Send);
|
||||||
|
// ...
|
||||||
|
self.recv_seq = self.recv_seq.wrapping_add(1); // line 148
|
||||||
|
```
|
||||||
|
|
||||||
|
`recv_seq` increments once per successful `decrypt()` call. The sender's `send_seq` also increments once per `encrypt()` call (line 120). In perfect in-order delivery they stay synchronized. With any reorder or mid-stream packet loss they permanently diverge. Once diverged, every subsequent packet uses the wrong nonce → AEAD tag mismatch → every packet fails for the rest of the session.
|
||||||
|
|
||||||
|
This isn't a low-probability edge case. UDP over any internet path reorders packets routinely. The `multiple_packets_roundtrip` test (line 254) only exercises in-order delivery. HANDOFF-2026-05-12.md acknowledges this as a known latent item: *"AEAD nonce derivation: switch to `MediaHeader::seq`"*.
|
||||||
|
|
||||||
|
The anti-replay check at lines 152–161 already parses `MediaHeader` and has `header.seq` available. The fix is one line in `decrypt()`:
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// Use sender's wire-level seq as nonce input, not a local counter.
|
||||||
|
// This survives reordering because both sides derive the same nonce from
|
||||||
|
// the same field. recv_seq was wrong: it diverged from send_seq on any
|
||||||
|
// reorder, breaking all subsequent decryptions for the session.
|
||||||
|
let header = parse_header(header_bytes)
|
||||||
|
.ok_or_else(|| CryptoError::Internal("header parse failed".into()))?;
|
||||||
|
let nonce_bytes = nonce::build_nonce(&self.session_id, header.seq, Direction::Send);
|
||||||
|
```
|
||||||
|
|
||||||
|
Remove `recv_seq` field from `ChaChaSession` (it's now redundant — anti-replay uses `header.seq` directly). On the encrypt side, verify that `self.send_seq` equals the `seq` written into the `MediaHeader` at the call site.
|
||||||
|
|
||||||
|
**Estimated effort:** ~1 hour including test coverage for out-of-order delivery.
|
||||||
|
|
||||||
|
> **Note on rekey seq reset:** The agent initially flagged `send_seq/recv_seq = 0` in `complete_rekey()` as a separate critical issue. This is a false positive — `install_key()` rotates `session_id` (hash of new key), so pre-/post-rekey nonces live in distinct namespaces. The reset is intentional and cryptographically safe.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### C2 — AEAD not wired to every QUIC datagram send path
|
||||||
|
|
||||||
|
**File:** `crates/wzp-client/src/analyzer.rs:363` (only confirmed decrypt call site)
|
||||||
|
**Severity:** Critical — potential plaintext media leakage
|
||||||
|
|
||||||
|
The HANDOFF document explicitly flags this: *"Encryption is implemented in `wzp-crypto` but not yet on every QUIC datagram path."* The `analyzer.rs` path decrypts inbound packets. What needs verification: every outbound `send_datagram()` / `write_datagram()` call across `wzp-client` and `wzp-transport` must pass through `ChaChaSession::encrypt()`.
|
||||||
|
|
||||||
|
**Required action:** Grep every `send_datagram` call site. Confirm each path encrypts before transmit. Add a CI-level test or `#[forbid(dead_code)]`-style assertion that makes a plaintext send path impossible to merge. Until this is verified, the E2E security claim cannot be made.
|
||||||
|
|
||||||
|
**Estimated effort:** ~1 hour audit + test.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### C3 — `VideoScorer::observe()` never called — scorer is dead code
|
||||||
|
|
||||||
|
**File:** `crates/wzp-relay/src/room.rs:1263–1266`
|
||||||
|
**Severity:** Critical — relay abuse control for video is completely absent
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// T6.2-follow-up: feed video packets to VideoScorer here.
|
||||||
|
// video_scorer.observe(&pkt.header, pkt.payload.len(), now, bwe_kbps);
|
||||||
|
```
|
||||||
|
|
||||||
|
`video_scorer.rs` was delivered in T6.2 with legitimacy scoring, keyframe regularity checks, I/P ratio analysis, and a verdict enum. The observe call was never wired into the packet forwarding loop. The scorer compiles but accumulates no data. Any participant can flood the room with malformed video or synthetic keyframe bursts and the relay will forward everything without challenge.
|
||||||
|
|
||||||
|
**Fix:** Wire `video_scorer.observe(...)` at the TODO marker and integrate `legitimacy_score()` into the forwarding decision (drop or rate-limit streams with `Verdict::Malicious`). Add an integration test: synthetic high-frequency keyframe bursts should trigger a `Malicious` verdict within 2 seconds.
|
||||||
|
|
||||||
|
**Estimated effort:** ~2 hours.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### C4 — `wzp-video` Android target fails to compile (31 errors)
|
||||||
|
|
||||||
|
**File:** `crates/wzp-video/src/mediacodec.rs`
|
||||||
|
**Severity:** Critical — Android video is completely blocked
|
||||||
|
|
||||||
|
Five error categories from the NDK 0.9 API migration, all documented in HANDOFF-2026-05-12.md. `dav1d`/`svt-av1` were cfg-gated off Android in `f3e3ee5`; these 31 errors are the remaining MediaCodec API mismatch.
|
||||||
|
|
||||||
|
| Error | Count | Root cause | Fix |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `E0277` `NonNull<AMediaCodec>` not `Send` | ~3 | Raw pointer held across `tokio::spawn` boundary | `struct SendMediaCodec(NonNull<…>); unsafe impl Send for SendMediaCodec {}` — or use `ndk::media::MediaCodec` owned type (already `Send`) |
|
||||||
|
| `E0308` `&[MaybeUninit<u8>]` vs `&[u8]` | many | NDK 0.9 returns uninit slices | `MaybeUninit::write_slice` or transmute pattern |
|
||||||
|
| `E0425` missing `BITRATE_MODE_CBR` | 1+ | Constant renamed in NDK 0.9 | Check `ndk` crate docs for current name |
|
||||||
|
| `E0433` `ndk_sys` not a dep | several | Direct `ndk_sys` import; only `ndk = "0.9"` declared | Add `ndk-sys` as explicit dep or use safe `ndk` wrappers |
|
||||||
|
| `E0599` `InputBuffer::index()` / `OutputBuffer::index()` private | 2 | API changed in NDK 0.9 | Use buffer through safe queue/dequeue API |
|
||||||
|
|
||||||
|
Nothing live is blocked today — `wzp-video` is not yet consumed by Tauri Android. But video on Android cannot progress until this compiles.
|
||||||
|
|
||||||
|
**Reproduce:**
|
||||||
|
```bash
|
||||||
|
ssh -i ~/CascadeProjects/wzp manwe@manwehs \
|
||||||
|
'cd ~/wzp-builder/data/source && \
|
||||||
|
docker run --rm \
|
||||||
|
-v ~/wzp-builder/data/source:/build/source \
|
||||||
|
-v ~/wzp-builder/data/cache/cargo-registry:/home/builder/.cargo/registry \
|
||||||
|
-v ~/wzp-builder/data/cache/cargo-git:/home/builder/.cargo/git \
|
||||||
|
-v ~/wzp-builder/data/cache/target:/build/source/target \
|
||||||
|
wzp-android-builder:latest \
|
||||||
|
bash -c "cd /build/source && cargo build --target aarch64-linux-android -p wzp-video 2>&1 | tail -60"'
|
||||||
|
```
|
||||||
|
|
||||||
|
**Estimated effort:** ~2 hours (one commit per error category).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## High
|
||||||
|
|
||||||
|
### H1 — AV1 call engine wiring missing
|
||||||
|
|
||||||
|
**Source:** HANDOFF-2026-05-12.md (T6.1.2 open item)
|
||||||
|
**File:** `crates/wzp-video/src/factory.rs`
|
||||||
|
|
||||||
|
`factory.rs` and step tables landed in commit `086d0a4`. No caller yet invokes `create_video_encoder(Av1Main, ...)`. The entire AV1 path is reachable only from tests. Video on macOS/Linux desktop requires wiring `create_video_encoder` into the call engine's media negotiation path.
|
||||||
|
|
||||||
|
**Estimated effort:** ~1–2 hours.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### H2 — `fec_block_id: u8` wraps every ~25 seconds
|
||||||
|
|
||||||
|
**File:** `crates/wzp-fec/src/encoder.rs` (`block_id.wrapping_add(1)` on u8)
|
||||||
|
**Reference:** PROTOCOL-AUDIT.md W2 (deferred P2)
|
||||||
|
|
||||||
|
At 5 frames/block (Codec2), u8 ID wraps at block 256 ≈ 25 seconds. A slow reconstructor or late-joining peer will collide block IDs with in-flight blocks. The window distance check in `block_manager.rs` partially mitigates this but can't prevent all collisions. Widen to `u16` in the next wire-format revision.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Medium
|
||||||
|
|
||||||
|
### M1 — `SignalMessage` has no version byte
|
||||||
|
|
||||||
|
**File:** `crates/wzp-proto/src/session.rs` (SignalMessage enum)
|
||||||
|
**Reference:** PROTOCOL-AUDIT.md W12
|
||||||
|
|
||||||
|
`bincode + serde(default)` handles field additions but not variant removal or semantic changes. Any variant deprecation is silent at the wire level. This becomes a correctness risk when federation routes `SignalMessage`s across relay versions. Add `version: u8` as a leading field to all variants before federation ships.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### M2 — BWE not consumed by `AdaptiveQualityController`
|
||||||
|
|
||||||
|
**Reference:** PROTOCOL-AUDIT.md W6, deferred to Phase V2
|
||||||
|
|
||||||
|
Quinn exposes `cwnd` and `bytes_in_flight`, but `AdaptiveQualityController` does not consume them. Loss + RTT adaptation works for audio. For video, without bandwidth estimation the encoder cannot detect available uplink capacity and will either oscillate or permanently under-utilize bandwidth. Mandatory before video production.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### M3 — PLI suppression window hardcoded at 200ms
|
||||||
|
|
||||||
|
**File:** `crates/wzp-relay/src/room.rs:1060`
|
||||||
|
|
||||||
|
Not adaptive to link speed. On slow links 200ms may allow multiple keyframe requests. Accept for Phase 1; make configurable in Phase 2.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### M4 — Repair packet index wrapping in FEC encoder
|
||||||
|
|
||||||
|
**File:** `crates/wzp-fec/src/encoder.rs:140`
|
||||||
|
|
||||||
|
```rust
|
||||||
|
let idx = (num_source as u8).wrapping_add(i as u8);
|
||||||
|
```
|
||||||
|
|
||||||
|
If `num_source + repair_count > 255`, indices wrap silently. In practice bounded by `frames_per_block` (5–10), so max sum is ~20. Low risk today; widen to u16 when `fec_block_id` is widened (H2).
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### M5 — `timestamp_ms` monotonicity after rekey not enforced
|
||||||
|
|
||||||
|
**Reference:** PROTOCOL-AUDIT.md W3
|
||||||
|
|
||||||
|
Spec: `timestamp_ms` must not reset on rekey. The code correctly does not reset it, but there is no assertion to prevent regression. Add a debug assert in `complete_rekey()` that `new_session.next_timestamp >= old_session.last_timestamp`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Low / Accepted Debt
|
||||||
|
|
||||||
|
| ID | Description | File | Accepted in |
|
||||||
|
|---|---|---|---|
|
||||||
|
| L1 | 9 pre-existing clippy lints in `wzp-codec` | `aec.rs`, `denoise.rs`, `opus_enc.rs`, `codec2_{enc,dec}.rs`, `resample.rs` | PROTOCOL-AUDIT.md |
|
||||||
|
| L2 | 3 clippy errors in `deps/featherchat` submodule | `ratchet.rs`, `types.rs` | PROTOCOL-AUDIT.md |
|
||||||
|
| L3 | Audio anti-replay window 64 packets | `wzp-crypto/src/session.rs:89` | Accepted — jitter buffer + PLC masks loss |
|
||||||
|
| L4 | Debug tap logs at INFO with no rate limiting | `wzp-relay/src/room.rs:46–59` | Safe in dev; add 1:100 sampling for prod |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## What Was Not Found
|
||||||
|
|
||||||
|
These are explicitly confirmed sound after code-level verification:
|
||||||
|
|
||||||
|
- **Anti-replay bitmap** — correct u32 wrapping, per-stream isolation, window sizing by `MediaType`
|
||||||
|
- **HKDF + X25519 + Ed25519 key agreement** — standard construction, no gaps
|
||||||
|
- **SAS code derivation** — SHA-256(shared_secret)[:4] as 4-digit voice verification code
|
||||||
|
- **Rekey forward secrecy** — `session_id` rotation on rekey isolates nonce namespaces; seq counter reset is intentional and safe
|
||||||
|
- **MiniHeader v2 `seq_delta`** — fully implemented at `wzp-proto/src/packet.rs:469–526` with tests; PROTOCOL-AUDIT resolution table is accurate
|
||||||
|
- **SFU E2E preservation** — relay ciphertext passthrough, no plaintext access
|
||||||
|
- **RaptorQ for Codec2** — correct tool for the bitrate regime
|
||||||
|
- **DRED continuous tuning** — better than discrete tiers; 15% loss floor is empirically grounded
|
||||||
|
- **Jitter buffer** — BTreeMap with wrapping-aware comparisons, EWMA adaptive playout delay, solid
|
||||||
|
- **Quinn QUIC datagram transport** — correct primitives for unreliable media
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Fix Priority Table
|
||||||
|
|
||||||
|
| # | Issue | Category | Effort | Blocks |
|
||||||
|
|---|---|---|---|---|
|
||||||
|
| 1 | C1: nonce → `MediaHeader.seq` | Crypto | 1h | All sessions on lossy paths |
|
||||||
|
| 2 | C2: verify AEAD on all datagram send paths | Crypto | 1h | E2E security claim |
|
||||||
|
| 3 | C3: wire `VideoScorer::observe()` into room | Relay | 2h | Relay abuse control for video |
|
||||||
|
| 4 | C4: NDK 0.9 `mediacodec.rs` migration (5 categories) | Android | 2h | Android video |
|
||||||
|
| 5 | H1: wire AV1 factory into call engine | Video | 2h | Desktop video |
|
||||||
|
| 6 | H2: widen `fec_block_id` to `u16` | FEC/Wire | 30min | Next protocol release |
|
||||||
|
| 7 | M1: `SignalMessage` version byte | Proto | 1h | Federation correctness |
|
||||||
|
| 8 | M2: BWE into `AdaptiveQualityController` | Transport | 2–3 days | Video production quality |
|
||||||
|
|
||||||
|
**Total for C1–H1 (items 1–5):** ~8 hours focused engineering.
|
||||||
219
vault/PRDs/PRD-adaptive-quality.md
Normal file
219
vault/PRDs/PRD-adaptive-quality.md
Normal file
@@ -0,0 +1,219 @@
|
|||||||
|
---
|
||||||
|
tags: [prd, wzp]
|
||||||
|
type: prd
|
||||||
|
---
|
||||||
|
|
||||||
|
# PRD: Adaptive Quality Control (Auto Codec)
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
When a user selects "Auto" quality, the system currently just starts at Opus 24k (GOOD) and never changes. There is no runtime adaptation — if the network degrades mid-call, audio breaks up instead of gracefully stepping down to a lower bitrate codec. Conversely, if the network is excellent, the user stays on 24k when they could have studio-quality 64k.
|
||||||
|
|
||||||
|
The relay already sends `QualityReport` messages with loss % and RTT, and a `QualityAdapter` exists in `call.rs` that classifies network conditions into GOOD/DEGRADED/CATASTROPHIC — but none of this is wired into the Android or desktop engines.
|
||||||
|
|
||||||
|
## Solution
|
||||||
|
|
||||||
|
Wire the existing `QualityAdapter` into both engines so that "Auto" mode continuously monitors network quality and switches codecs mid-call. The full quality range should be used:
|
||||||
|
|
||||||
|
```
|
||||||
|
Excellent network → Studio 64k (best quality)
|
||||||
|
Good network → Opus 24k (default)
|
||||||
|
Degraded network → Opus 6k (lower bitrate, more FEC)
|
||||||
|
Poor network → Codec2 3.2k (vocoder, heavy FEC)
|
||||||
|
Catastrophic → Codec2 1.2k (minimum viable voice)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────────────────┐
|
||||||
|
Relay ──────────► │ QualityReport │ loss %, RTT, jitter
|
||||||
|
│ (every ~1s) │
|
||||||
|
└────────┬────────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌─────────────────────┐
|
||||||
|
│ QualityAdapter │ classify + hysteresis
|
||||||
|
│ (3-report window) │
|
||||||
|
└────────┬────────────┘
|
||||||
|
│ recommend new profile
|
||||||
|
▼
|
||||||
|
┌──────────────┴──────────────┐
|
||||||
|
│ │
|
||||||
|
▼ ▼
|
||||||
|
┌────────────────┐ ┌────────────────┐
|
||||||
|
│ Encoder │ │ Decoder │
|
||||||
|
│ set_profile() │ │ (auto-switch │
|
||||||
|
│ + FEC update │ │ already works)│
|
||||||
|
└────────────────┘ └────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
## Existing Infrastructure
|
||||||
|
|
||||||
|
### What already exists (in `crates/wzp-client/src/call.rs`)
|
||||||
|
|
||||||
|
1. **`QualityAdapter`** (lines 97-196):
|
||||||
|
- Sliding window of `QualityReport` messages
|
||||||
|
- `classify()`: loss > 15% or RTT > 200ms → CATASTROPHIC, loss > 5% or RTT > 100ms → DEGRADED, else → GOOD
|
||||||
|
- `should_switch()`: hysteresis — requires 3 consecutive reports recommending the same profile before switching
|
||||||
|
- Prevents oscillation between profiles
|
||||||
|
|
||||||
|
2. **`QualityReport`** (in `wzp-proto/src/packet.rs`):
|
||||||
|
- Sent by relay piggy-backed on media packets
|
||||||
|
- Fields: `loss_pct` (u8, 0-255 scaled), `rtt_4ms` (u8, RTT in 4ms units), `jitter_ms`, `bitrate_cap_kbps`
|
||||||
|
|
||||||
|
3. **`CallEncoder::set_profile()`** / **`CallDecoder` auto-switch**:
|
||||||
|
- Encoder can switch codec mid-stream
|
||||||
|
- Decoder already auto-detects incoming codec from packet headers
|
||||||
|
|
||||||
|
### What's been implemented since PRD was written
|
||||||
|
|
||||||
|
1. **QualityReport ingestion** — ~~neither Android engine nor desktop engine reads quality reports from the relay~~ **Done**: both Android (`crates/wzp-android/src/engine.rs`) and desktop (`desktop/src-tauri/src/engine.rs`) recv tasks ingest quality reports and feed `AdaptiveQualityController`
|
||||||
|
2. **Profile switch loop** — ~~no periodic check~~ **Done**: `pending_profile` AtomicU8 bridges recv→send task in both engines; send task applies profile switch at frame boundary
|
||||||
|
3. **Notification to UI** — ~~when quality changes, the UI should show the current active codec~~ **Done**: `tx_codec`/`rx_codec` in desktop `EngineStatus`; `currentCodec`/`peerCodec` in Android `CallStats`
|
||||||
|
|
||||||
|
### What's still missing
|
||||||
|
|
||||||
|
1. **Upward adaptation** — `QualityAdapter` only classifies into 3 tiers (GOOD/DEGRADED/CATASTROPHIC). Needs extension to recommend studio tiers when conditions are excellent (loss < 1%, RTT < 50ms). See Phase 2 below.
|
||||||
|
2. **Relay QualityDirective handling** — relay broadcasts coordinated quality directives but neither engine processes them (signals are silently discarded). See PRD-coordinated-codec.md for details.
|
||||||
|
|
||||||
|
## Requirements
|
||||||
|
|
||||||
|
### Phase 1: Basic Adaptive (3-tier)
|
||||||
|
|
||||||
|
**Both Android and Desktop:**
|
||||||
|
|
||||||
|
1. **Ingest QualityReports**: In the recv loop, extract `quality_report` from incoming `MediaPacket`s when present. Feed to `QualityAdapter`.
|
||||||
|
|
||||||
|
2. **Periodic quality check**: Every 1 second (or on each QualityReport), call `adapter.should_switch(¤t_profile)`. If it returns `Some(new_profile)`:
|
||||||
|
- Switch the encoder: `encoder.set_profile(new_profile)`
|
||||||
|
- Update FEC encoder: `fec_enc = create_encoder(&new_profile)`
|
||||||
|
- Update frame size if changed (e.g., 20ms → 40ms)
|
||||||
|
- Log the switch
|
||||||
|
|
||||||
|
3. **Frame size adaptation on switch**: When switching from 20ms to 40ms frames (or vice versa):
|
||||||
|
- Android: update `frame_samples` variable, resize `capture_buf`
|
||||||
|
- Desktop: same — the send loop reads `frame_samples` dynamically
|
||||||
|
|
||||||
|
4. **UI indicator**: Show current active codec in the call screen stats line.
|
||||||
|
- Android: add to `CallStats` and display in stats text
|
||||||
|
- Desktop: add to `get_status` response and display in stats div
|
||||||
|
|
||||||
|
5. **Only in Auto mode**: Adaptive switching should only happen when the user selected "Auto". If they manually selected a profile, respect their choice.
|
||||||
|
|
||||||
|
### Phase 2: Extended Range (5-tier)
|
||||||
|
|
||||||
|
Extend `QualityAdapter::classify()` to use the full codec range:
|
||||||
|
|
||||||
|
| Condition | Profile | Codec |
|
||||||
|
|-----------|---------|-------|
|
||||||
|
| loss < 1% AND RTT < 30ms | STUDIO_64K | Opus 64k |
|
||||||
|
| loss < 1% AND RTT < 50ms | STUDIO_48K | Opus 48k |
|
||||||
|
| loss < 2% AND RTT < 80ms | STUDIO_32K | Opus 32k |
|
||||||
|
| loss < 5% AND RTT < 100ms | GOOD | Opus 24k |
|
||||||
|
| loss < 15% AND RTT < 200ms | DEGRADED | Opus 6k |
|
||||||
|
| loss >= 15% OR RTT >= 200ms | CATASTROPHIC | Codec2 1.2k |
|
||||||
|
|
||||||
|
With hysteresis:
|
||||||
|
- **Downgrade**: 3 consecutive reports (fast reaction to degradation)
|
||||||
|
- **Upgrade**: 5 consecutive reports (slow, cautious improvement)
|
||||||
|
- **Studio upgrade**: 10 consecutive reports (very conservative — avoid bouncing to 64k on brief good patches)
|
||||||
|
|
||||||
|
### Phase 3: Bandwidth Probing
|
||||||
|
|
||||||
|
Rather than relying solely on loss/RTT:
|
||||||
|
1. Start at GOOD
|
||||||
|
2. After 10 seconds of stable call, probe upward by switching to STUDIO_32K
|
||||||
|
3. If no quality degradation after 5 seconds, probe to STUDIO_48K
|
||||||
|
4. If degradation detected, immediately fall back
|
||||||
|
5. This discovers the true available bandwidth rather than guessing from loss stats
|
||||||
|
|
||||||
|
## Implementation Plan
|
||||||
|
|
||||||
|
### Android (`crates/wzp-android/src/engine.rs`)
|
||||||
|
|
||||||
|
```rust
|
||||||
|
// In the recv loop, after decoding:
|
||||||
|
if let Some(ref qr) = pkt.quality_report {
|
||||||
|
quality_adapter.ingest(qr);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Periodic check (every 50 frames ≈ 1 second):
|
||||||
|
if auto_profile && frames_decoded % 50 == 0 {
|
||||||
|
if let Some(new_profile) = quality_adapter.should_switch(¤t_profile) {
|
||||||
|
info!(from = ?current_profile.codec, to = ?new_profile.codec, "auto: switching quality");
|
||||||
|
let _ = encoder_ref.lock().set_profile(new_profile);
|
||||||
|
fec_enc_ref.lock() = create_encoder(&new_profile);
|
||||||
|
current_profile = new_profile;
|
||||||
|
frame_samples = frame_samples_for(&new_profile);
|
||||||
|
// Resize capture buffer if needed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Challenge**: The encoder is in the send task and the quality reports arrive in the recv task. Need shared state (AtomicU8 for profile index, or a channel).
|
||||||
|
|
||||||
|
**Recommended approach**: Use an `AtomicU8` that the recv task writes and the send task reads:
|
||||||
|
```rust
|
||||||
|
let pending_profile = Arc::new(AtomicU8::new(0xFF)); // 0xFF = no change
|
||||||
|
|
||||||
|
// Recv task: when adapter recommends switch
|
||||||
|
pending_profile.store(new_profile_index, Ordering::Release);
|
||||||
|
|
||||||
|
// Send task: check at frame boundary
|
||||||
|
let p = pending_profile.swap(0xFF, Ordering::Acquire);
|
||||||
|
if p != 0xFF { /* apply switch */ }
|
||||||
|
```
|
||||||
|
|
||||||
|
### Desktop (`desktop/src-tauri/src/engine.rs`)
|
||||||
|
|
||||||
|
Same pattern. The desktop engine already has separate send/recv tasks with shared atomics for mic_muted, etc. Add a `pending_profile: Arc<AtomicU8>` following the same pattern.
|
||||||
|
|
||||||
|
### Desktop CLI (`crates/wzp-client/src/call.rs`)
|
||||||
|
|
||||||
|
The `CallEncoder` already has `set_profile()`. The `CallDecoder` already auto-switches. Just need to:
|
||||||
|
1. Add `QualityAdapter` to `CallDecoder`
|
||||||
|
2. Feed quality reports in `ingest()`
|
||||||
|
3. Check `should_switch()` in `decode_next()`
|
||||||
|
4. Emit the recommendation via a callback or return value
|
||||||
|
|
||||||
|
## Testing
|
||||||
|
|
||||||
|
1. **Local test with tc/netem**: Use Linux traffic control to simulate loss/latency:
|
||||||
|
```bash
|
||||||
|
# Simulate 10% loss, 150ms RTT
|
||||||
|
tc qdisc add dev lo root netem loss 10% delay 75ms
|
||||||
|
# Run 2 clients in auto mode, verify they switch to DEGRADED
|
||||||
|
```
|
||||||
|
|
||||||
|
2. **CLI test**: Run `wzp-client --profile auto` between two instances with simulated network conditions
|
||||||
|
|
||||||
|
3. **Relay quality reports**: Verify the relay actually sends QualityReport messages. If it doesn't yet, that needs to be implemented first (check relay code).
|
||||||
|
|
||||||
|
## Open Questions
|
||||||
|
|
||||||
|
1. **Does the relay currently send QualityReports?** If not, Phase 1 is blocked until the relay implements per-client loss/RTT tracking and report generation. The relay sees all packets and can compute loss % per sender.
|
||||||
|
|
||||||
|
2. **Codec2 3.2k placement**: Should auto mode use Codec2 3.2k between DEGRADED and CATASTROPHIC? It's 20ms frames (lower latency than Opus 6k's 40ms) but speech-only quality.
|
||||||
|
|
||||||
|
3. **Cross-client adaptation**: If client A is on GOOD and client B auto-adapts to CATASTROPHIC, client A still sends Opus 24k. Client B can decode it fine (auto-switch on recv). But should A also be told to lower quality to save B's bandwidth? This requires signaling between clients.
|
||||||
|
|
||||||
|
## Milestones
|
||||||
|
|
||||||
|
| Phase | Scope | Effort | Status |
|
||||||
|
|-------|-------|--------|--------|
|
||||||
|
| 0 | Verify relay sends QualityReports | 0.5 day | Done |
|
||||||
|
| 1a | Wire QualityAdapter in Android engine | 1 day | Done |
|
||||||
|
| 1b | Wire QualityAdapter in desktop engine | 1 day | Done |
|
||||||
|
| 1c | UI indicator (current codec) | 0.5 day | Done |
|
||||||
|
| 2 | Extended 5-tier classification (Studio64k→Catastrophic) | 0.5 day | Done (2026-04-13) |
|
||||||
|
| 3 | Bandwidth probing | 2 days | Pending (task #10) |
|
||||||
|
|
||||||
|
## Implementation Status Update (2026-04-13)
|
||||||
|
|
||||||
|
All phases implemented:
|
||||||
|
- Phase 1: QualityAdapter with 3-tier classification — DONE
|
||||||
|
- Phase 2: Extended 5-tier (Studio 64k/48k/32k + GOOD + DEGRADED + CATASTROPHIC) — DONE
|
||||||
|
- Phase 3: Bandwidth probing — NOT DONE (see remaining tasks)
|
||||||
|
- P2P adaptive quality: QualityReport::from_path_stats() + self-observation from quinn stats — DONE
|
||||||
|
- Both relay and P2P calls now have full adaptive quality switching
|
||||||
110
vault/PRDs/PRD-bluetooth-audio.md
Normal file
110
vault/PRDs/PRD-bluetooth-audio.md
Normal file
@@ -0,0 +1,110 @@
|
|||||||
|
---
|
||||||
|
tags: [prd, wzp]
|
||||||
|
type: prd
|
||||||
|
---
|
||||||
|
|
||||||
|
# PRD: Bluetooth Audio Routing
|
||||||
|
|
||||||
|
> Phase: Implemented
|
||||||
|
> Status: Ready for testing
|
||||||
|
> Platforms: Android (native Kotlin app + Tauri desktop app)
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
WarzonePhone had `AudioRouteManager.kt` with complete Bluetooth SCO support, but it was disconnected from both UIs. Users with Bluetooth headsets had no way to route call audio to them.
|
||||||
|
|
||||||
|
## Solution
|
||||||
|
|
||||||
|
Wire Bluetooth SCO routing end-to-end through both app variants, replacing the binary speaker toggle with a 3-way audio route cycle: **Earpiece → Speaker → Bluetooth**.
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────────────────────────────────────────────────┐
|
||||||
|
│ Native Kotlin App (com.wzp) │
|
||||||
|
│ │
|
||||||
|
│ InCallScreen ──► CallViewModel ──► AudioRouteManager
|
||||||
|
│ (Compose UI) cycleAudioRoute() setSpeaker() │
|
||||||
|
│ "Ear/Spk/BT" audioRoute Flow setBluetoothSco()
|
||||||
|
│ isBluetoothAvailable()
|
||||||
|
└─────────────────────────────────────────────────────┘
|
||||||
|
|
||||||
|
┌─────────────────────────────────────────────────────┐
|
||||||
|
│ Tauri Desktop App (com.wzp.desktop) │
|
||||||
|
│ │
|
||||||
|
│ main.ts ──► Tauri Commands ──► android_audio.rs │
|
||||||
|
│ cycleAudioRoute() set_bluetooth_sco() JNI calls │
|
||||||
|
│ "Ear/Spk/BT" is_bluetooth_available() │
|
||||||
|
│ get_audio_route() │
|
||||||
|
│ │
|
||||||
|
│ After each route change: Oboe stop + start │
|
||||||
|
│ (spawn_blocking to avoid stalling tokio) │
|
||||||
|
└─────────────────────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
## Components Modified
|
||||||
|
|
||||||
|
### Native Kotlin App
|
||||||
|
|
||||||
|
| File | Change |
|
||||||
|
|------|--------|
|
||||||
|
| `CallViewModel.kt` | Added `audioRoute: StateFlow<AudioRoute>`, `cycleAudioRoute()`, wired `onRouteChanged` callback |
|
||||||
|
| `InCallScreen.kt` | `ControlRow` now takes `audioRoute: AudioRoute` + `onCycleRoute`, displays Ear/Spk/BT with distinct colors |
|
||||||
|
|
||||||
|
### Tauri App
|
||||||
|
|
||||||
|
| File | Change |
|
||||||
|
|------|--------|
|
||||||
|
| `android_audio.rs` | `setCommunicationDevice()` (API 31+) with `startBluetoothSco()` fallback; `set_audio_mode_communication/normal()` for call lifecycle |
|
||||||
|
| `lib.rs` | `set_bluetooth_sco`, `is_bluetooth_available`, `get_audio_route` Tauri commands; SCO polling + 500ms route delay |
|
||||||
|
| `wzp_native.rs` | Added `audio_start_bt()` for BT-mode Oboe (skips 48kHz + VoiceCommunication preset) |
|
||||||
|
| `oboe_bridge.cpp` | `bt_active` flag: capture skips sample rate + input preset; playout uses `Usage::Media`; both use `Shared` mode + `SampleRateConversionQuality::Best` |
|
||||||
|
| `engine.rs` | `set_audio_mode_communication()` before `audio_start()`; `set_audio_mode_normal()` after `audio_stop()` |
|
||||||
|
| `MainActivity.kt` | Removed `MODE_IN_COMMUNICATION` from app launch — deferred to call start |
|
||||||
|
| `main.ts` | Replaced `speakerphoneOn` toggle with `currentAudioRoute` cycling logic |
|
||||||
|
| `style.css` | Added `.bt-on` CSS class (blue-400 highlight) |
|
||||||
|
|
||||||
|
## Audio Route Lifecycle
|
||||||
|
|
||||||
|
1. **App launch** → `MODE_NORMAL` (other apps' audio unaffected — BT A2DP music keeps playing)
|
||||||
|
2. **Call starts** → `MODE_IN_COMMUNICATION` set via JNI, Oboe opens with earpiece routing
|
||||||
|
3. **User taps route button** → cycles to next available route
|
||||||
|
4. **Route changes** → `setCommunicationDevice()` (API 31+) + Oboe restart in BT mode or normal mode
|
||||||
|
5. **BT device disconnects mid-call** → `AudioDeviceCallback.onAudioDevicesRemoved` fires → auto-fallback to Earpiece/Speaker
|
||||||
|
6. **Call ends** → route reset, `MODE_NORMAL` restored
|
||||||
|
|
||||||
|
## Route Cycling Logic
|
||||||
|
|
||||||
|
```
|
||||||
|
Available routes = [Earpiece, Speaker] + [Bluetooth] if SCO device connected
|
||||||
|
|
||||||
|
Tap cycle:
|
||||||
|
Earpiece → Speaker → Bluetooth (if available) → Earpiece → ...
|
||||||
|
|
||||||
|
If BT not available:
|
||||||
|
Earpiece → Speaker → Earpiece → ...
|
||||||
|
```
|
||||||
|
|
||||||
|
## Permissions
|
||||||
|
|
||||||
|
- `BLUETOOTH_CONNECT` (Android 12+) — already in `AndroidManifest.xml`
|
||||||
|
- `MODIFY_AUDIO_SETTINGS` — already in manifest
|
||||||
|
|
||||||
|
## Known Limitations
|
||||||
|
|
||||||
|
- **SCO only** — no A2DP (stereo music profile). SCO is correct for VoIP (bidirectional mono).
|
||||||
|
- **API 31+ required for modern path** — `setCommunicationDevice()` is the primary BT routing API. Fallback to deprecated `startBluetoothSco()` on API < 31 (untested).
|
||||||
|
- **BT SCO capture at 8/16kHz** — Oboe resamples to 48kHz via `SampleRateConversionQuality::Best`. Quality is inherently limited by the SCO codec (CVSD at 8kHz or mSBC at 16kHz).
|
||||||
|
- **No auto-switch on BT connect** — when a BT device connects mid-call, user must tap the route button.
|
||||||
|
- **500ms route switch delay** — after `setCommunicationDevice()` returns, the audio policy needs time to apply the bt-sco route. We wait 500ms before restarting Oboe.
|
||||||
|
|
||||||
|
## Testing
|
||||||
|
|
||||||
|
1. Pair a Bluetooth SCO headset with Android device
|
||||||
|
2. Start call → verify Earpiece is default
|
||||||
|
3. Tap route → Speaker (audio moves to loudspeaker, button shows "Spk")
|
||||||
|
4. Tap route → BT (audio moves to headset, button shows "BT", blue highlight)
|
||||||
|
5. Tap route → Earpiece (audio back to earpiece, button shows "Ear")
|
||||||
|
6. Disconnect BT mid-call → verify auto-fallback
|
||||||
|
7. Verify both app variants work identically
|
||||||
|
8. Verify no audio glitches during route transitions
|
||||||
226
vault/PRDs/PRD-coordinated-codec.md
Normal file
226
vault/PRDs/PRD-coordinated-codec.md
Normal file
@@ -0,0 +1,226 @@
|
|||||||
|
---
|
||||||
|
tags: [prd, wzp]
|
||||||
|
type: prd
|
||||||
|
---
|
||||||
|
|
||||||
|
# PRD: Coordinated Codec Switching (Relay-Judged Quality)
|
||||||
|
|
||||||
|
## Problem
|
||||||
|
|
||||||
|
The current adaptive quality system (`QualityAdapter` in call.rs) exists but isn't wired into either engine. Clients encode at a fixed quality chosen at call start. When network conditions change mid-call, audio degrades instead of gracefully stepping down. When conditions improve, clients stay on low quality unnecessarily.
|
||||||
|
|
||||||
|
Additionally, in SFU mode with multiple participants, uncoordinated codec switching creates asymmetry: if client A upgrades to 64k while B stays on 24k, bandwidth is wasted. Participants should switch together.
|
||||||
|
|
||||||
|
## Solution
|
||||||
|
|
||||||
|
The **relay acts as the quality judge** since it sees both sides of every connection. It monitors packet loss, jitter, and RTT per participant, then signals quality recommendations. Clients react to these signals with coordinated codec switches.
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────┐ ┌─────────┐ ┌─────────┐
|
||||||
|
│ Client A │◄──────►│ Relay │◄──────►│ Client B │
|
||||||
|
│ │ │ (judge) │ │ │
|
||||||
|
│ Encoder │ │ │ │ Encoder │
|
||||||
|
│ Decoder │ │ Monitor │ │ Decoder │
|
||||||
|
└─────────┘ │ per-peer│ └─────────┘
|
||||||
|
│ quality │
|
||||||
|
└────┬────┘
|
||||||
|
│
|
||||||
|
Quality Signals:
|
||||||
|
- StableSignal (conditions good)
|
||||||
|
- DegradeSignal (conditions bad)
|
||||||
|
- UpgradeProposal (try higher quality?)
|
||||||
|
- UpgradeConfirm (all agreed, switch at T)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Quality Classification (Relay-Side)
|
||||||
|
|
||||||
|
The relay monitors each participant's connection quality:
|
||||||
|
|
||||||
|
| Condition | Classification | Action |
|
||||||
|
|-----------|---------------|--------|
|
||||||
|
| loss >= 15% OR RTT >= 200ms | Critical | Immediate downgrade signal |
|
||||||
|
| loss >= 5% OR RTT >= 100ms | Degraded | Downgrade signal after 3 reports |
|
||||||
|
| loss < 2% AND RTT < 80ms | Good | Stable signal |
|
||||||
|
| loss < 1% AND RTT < 50ms for 30s | Excellent | Upgrade proposal |
|
||||||
|
| loss < 0.5% AND RTT < 30ms for 60s | Studio | Studio upgrade proposal |
|
||||||
|
|
||||||
|
## Coordinated Switching Protocol
|
||||||
|
|
||||||
|
### Downgrade (fast, safety-first)
|
||||||
|
|
||||||
|
1. Relay detects degradation for ANY participant
|
||||||
|
2. Relay sends `QualityUpdate { recommended_profile: DEGRADED }` to ALL participants
|
||||||
|
3. ALL participants immediately switch encoder to the recommended profile
|
||||||
|
4. No negotiation — downgrade is mandatory and instant
|
||||||
|
|
||||||
|
### Upgrade (slow, consensual)
|
||||||
|
|
||||||
|
1. Relay detects sustained good conditions for ALL participants (threshold: 30s stable)
|
||||||
|
2. Relay sends `UpgradeProposal { target_profile, switch_timestamp }` to all
|
||||||
|
3. Each client responds: `UpgradeAccept` or `UpgradeReject`
|
||||||
|
4. If ALL accept within 5s → Relay sends `UpgradeConfirm { profile, switch_at_ms }`
|
||||||
|
5. All clients switch encoder at the agreed timestamp (relative to session clock)
|
||||||
|
6. If ANY rejects or times out → upgrade cancelled, stay on current profile
|
||||||
|
|
||||||
|
### Asymmetric Encoding (SFU optimization)
|
||||||
|
|
||||||
|
In SFU mode, each client encodes independently. The relay could allow:
|
||||||
|
- Client A (strong connection): encode at 64k
|
||||||
|
- Client B (weak connection): encode at 6k
|
||||||
|
- Relay forwards A's 64k to B's decoder (auto-switch handles it)
|
||||||
|
- B benefits from A's quality without needing to send at 64k
|
||||||
|
|
||||||
|
This requires NO protocol changes — just each client independently following the relay's recommendation for their own encoding quality. The decoder already handles any codec.
|
||||||
|
|
||||||
|
### Split Network Consideration
|
||||||
|
|
||||||
|
If participant A has great quality but participant C has terrible quality:
|
||||||
|
- Option 1: **Match weakest link** — everyone encodes at C's level (current approach, simple)
|
||||||
|
- Option 2: **Per-participant recommendations** — A encodes at 64k, C encodes at 6k. B (good connection) receives and decodes both. Works because decoders auto-switch per packet.
|
||||||
|
- Option 3: **Relay transcoding** — relay re-encodes A's 64k as 6k for C. Adds CPU on relay, but saves bandwidth for C. Future feature.
|
||||||
|
|
||||||
|
Recommended: start with Option 1 (match weakest), add Option 2 later.
|
||||||
|
|
||||||
|
## Signal Messages (New/Modified)
|
||||||
|
|
||||||
|
```rust
|
||||||
|
/// Quality signal from relay to client
|
||||||
|
QualityDirective {
|
||||||
|
/// Recommended profile to use for encoding
|
||||||
|
recommended_profile: QualityProfile,
|
||||||
|
/// Reason for the recommendation
|
||||||
|
reason: QualityReason,
|
||||||
|
}
|
||||||
|
|
||||||
|
enum QualityReason {
|
||||||
|
/// Network conditions require this quality level
|
||||||
|
NetworkCondition,
|
||||||
|
/// Coordinated upgrade — all participants agreed
|
||||||
|
CoordinatedUpgrade,
|
||||||
|
/// Coordinated downgrade — weakest link determines level
|
||||||
|
CoordinatedDowngrade,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Upgrade proposal from relay
|
||||||
|
UpgradeProposal {
|
||||||
|
target_profile: QualityProfile,
|
||||||
|
/// Milliseconds from now when the switch would happen
|
||||||
|
switch_delay_ms: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Client response to upgrade proposal
|
||||||
|
UpgradeResponse {
|
||||||
|
accepted: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Confirmed upgrade — all clients switch at this time
|
||||||
|
UpgradeConfirm {
|
||||||
|
profile: QualityProfile,
|
||||||
|
/// Session-relative timestamp to switch (ms since call start)
|
||||||
|
switch_at_session_ms: u64,
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Relay-Side Implementation
|
||||||
|
|
||||||
|
### Per-Participant Quality Tracking
|
||||||
|
|
||||||
|
```rust
|
||||||
|
struct ParticipantQuality {
|
||||||
|
/// Sliding window of recent observations
|
||||||
|
loss_samples: VecDeque<f32>, // last 30 seconds
|
||||||
|
rtt_samples: VecDeque<u32>, // last 30 seconds
|
||||||
|
jitter_samples: VecDeque<u32>,
|
||||||
|
/// Current classification
|
||||||
|
classification: QualityClass,
|
||||||
|
/// How long current classification has been stable
|
||||||
|
stable_since: Instant,
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Quality Monitor Task (on relay)
|
||||||
|
|
||||||
|
Runs alongside the SFU forwarding loop:
|
||||||
|
1. Every 1 second, compute per-participant quality from QUIC connection stats
|
||||||
|
2. Classify each participant
|
||||||
|
3. If ANY participant degrades → send downgrade to ALL
|
||||||
|
4. If ALL participants stable for threshold → propose upgrade
|
||||||
|
5. Track upgrade negotiation state
|
||||||
|
|
||||||
|
### Integration with Existing Code
|
||||||
|
|
||||||
|
The relay already has access to:
|
||||||
|
- `QuinnTransport::path_quality()` → loss, RTT, jitter, bandwidth estimates
|
||||||
|
- `QualityReport` embedded in media packet headers
|
||||||
|
- Per-session metrics in `RelayMetrics`
|
||||||
|
|
||||||
|
The quality monitor just needs to read these existing metrics and produce signals.
|
||||||
|
|
||||||
|
## Client-Side Implementation
|
||||||
|
|
||||||
|
### Handling Quality Signals
|
||||||
|
|
||||||
|
In the recv loop (both Android engine and desktop engine):
|
||||||
|
```rust
|
||||||
|
SignalMessage::QualityDirective { recommended_profile, .. } => {
|
||||||
|
// Immediate: switch encoder to recommended profile
|
||||||
|
encoder.set_profile(recommended_profile)?;
|
||||||
|
fec_enc = create_encoder(&recommended_profile);
|
||||||
|
frame_samples = frame_samples_for(&recommended_profile);
|
||||||
|
info!(codec = ?recommended_profile.codec, "quality directive: switched");
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### P2P Quality (simpler case)
|
||||||
|
|
||||||
|
For P2P calls (no relay), both clients directly observe quality:
|
||||||
|
1. Each client runs its own `QualityAdapter` on the direct connection
|
||||||
|
2. When quality changes, client proposes to peer via signal
|
||||||
|
3. Simpler negotiation: only 2 parties, no relay middleman
|
||||||
|
4. Same coordinated switching logic, just peer-to-peer signals
|
||||||
|
|
||||||
|
## Backporting P2P → Relay
|
||||||
|
|
||||||
|
The quality monitoring and codec switching logic is identical:
|
||||||
|
- **P2P**: client observes quality directly → proposes switch to peer
|
||||||
|
- **Relay**: relay observes quality → proposes switch to all clients
|
||||||
|
|
||||||
|
The only difference is WHO makes the decision (client vs relay) and HOW many participants need to agree (2 vs N).
|
||||||
|
|
||||||
|
Implementation strategy: build for P2P first (simpler, 2 parties), then wrap the same logic with relay-mediated signals for SFU mode.
|
||||||
|
|
||||||
|
## Milestones
|
||||||
|
|
||||||
|
| Phase | Scope | Effort |
|
||||||
|
|-------|-------|--------|
|
||||||
|
| 1 | Relay-side quality monitor (per-participant tracking) | 1 day |
|
||||||
|
| 2 | Downgrade signal (immediate, match weakest) | 1 day |
|
||||||
|
| 3 | Client handling of QualityDirective | 1 day (both engines) |
|
||||||
|
| 4 | Upgrade proposal + negotiation protocol | 2 days |
|
||||||
|
| 5 | P2P quality adaptation (direct observation) | 1 day |
|
||||||
|
| 6 | Per-participant asymmetric encoding (Option 2) | 1 day |
|
||||||
|
|
||||||
|
## Implementation Status (2026-04-13)
|
||||||
|
|
||||||
|
Phases 1-2 are implemented. Phase 3 has a critical gap.
|
||||||
|
|
||||||
|
### What was built
|
||||||
|
|
||||||
|
- **`QualityDirective` signal** (`crates/wzp-proto/src/packet.rs`): New `SignalMessage` variant with `recommended_profile` and optional `reason`
|
||||||
|
- **`ParticipantQuality`** (`crates/wzp-relay/src/room.rs`): Per-participant quality tracking using `AdaptiveQualityController`, created on join, removed on leave
|
||||||
|
- **Weakest-link broadcast**: `observe_quality()` method computes room-wide worst tier, broadcasts `QualityDirective` to all participants when tier changes
|
||||||
|
- **Desktop engine handling** (`desktop/src-tauri/src/engine.rs`): `AdaptiveQualityController` in recv task, `pending_profile` AtomicU8 bridge to send task, auto-mode profile switching based on **inbound quality reports**
|
||||||
|
|
||||||
|
### Phase 3 completed (2026-04-13)
|
||||||
|
|
||||||
|
Both engines now handle `QualityDirective` signals from the relay:
|
||||||
|
- **Desktop** (`engine.rs`): both P2P and relay signal tasks match `QualityDirective`, extract `recommended_profile`, store index via `sig_pending_profile.store(idx, Release)`. Send task picks it up at the next frame boundary.
|
||||||
|
- **Android** (`engine.rs`): signal task matches `QualityDirective`, stores via `pending_profile_recv.store(idx, Release)`.
|
||||||
|
|
||||||
|
Relay-coordinated codec switching is now end-to-end: relay monitors → broadcasts directive → clients switch.
|
||||||
|
|
||||||
|
### Phase remaining
|
||||||
|
|
||||||
|
- Phase 4: Upgrade proposal/negotiation protocol for quality recovery (task #28)
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user