From c48cb6fbcb120bf674c37571075a8dbfe6c52ece Mon Sep 17 00:00:00 2001 From: Siavash Sameni Date: Tue, 12 May 2026 12:40:53 +0400 Subject: [PATCH] =?UTF-8?q?T5.3:=20EncoderMode::SlideFallback=20=E2=80=94?= =?UTF-8?q?=20SD-floor=20detection=20+=20VideoEncoder::set=5Fmode()=20trai?= =?UTF-8?q?t=20hook?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/wzp-video/src/controller.rs | 51 ++++++++++++++++++++++++ crates/wzp-video/src/encoder.rs | 5 +++ crates/wzp-video/src/encoder_mode.rs | 15 +++++++ crates/wzp-video/src/lib.rs | 2 + docs/PRD/TASKS.md | 4 +- docs/PRD/reports/T5.2-report.md | 4 +- docs/PRD/reports/T5.3-report.md | 58 ++++++++++++++++++++++++++++ 7 files changed, 135 insertions(+), 4 deletions(-) create mode 100644 crates/wzp-video/src/encoder_mode.rs create mode 100644 docs/PRD/reports/T5.3-report.md diff --git a/crates/wzp-video/src/controller.rs b/crates/wzp-video/src/controller.rs index 080dda8..0984cf9 100644 --- a/crates/wzp-video/src/controller.rs +++ b/crates/wzp-video/src/controller.rs @@ -108,6 +108,10 @@ const BALANCED_AUDIO_RATIO: f64 = 0.15; /// Maximum bitrate change ratio per second (2x up or down). const MAX_CHANGE_RATIO_PER_SEC: f64 = 2.0; +/// SD video floor (kbps). When ScreenShare video budget drops below this, +/// the controller recommends [`EncoderMode::SlideFallback`]. +const SD_VIDEO_FLOOR_KBPS: u32 = 150; + /// Video quality controller. /// /// Consumes a [`BandwidthEstimator`] and a [`PriorityMode`] and produces @@ -157,6 +161,23 @@ impl VideoQualityController { } } + /// Recommend the encoder operating mode based on priority + budget. + /// + /// Returns [`EncoderMode::SlideFallback`] when the current mode is + /// [`PriorityMode::ScreenShare`] and the video budget is below the + /// SD floor (150 kbps). Otherwise returns [`EncoderMode::Normal`]. + pub fn encoder_mode(&self) -> crate::EncoderMode { + if self.mode() != PriorityMode::ScreenShare { + return crate::EncoderMode::Normal; + } + let (_audio, video) = self.allocate(); + if video < SD_VIDEO_FLOOR_KBPS { + crate::EncoderMode::SlideFallback + } else { + crate::EncoderMode::Normal + } + } + /// Compute audio and video budgets from the current BWE and priority mode. /// /// Returns `(audio_budget_kbps, video_budget_kbps)`. @@ -381,4 +402,34 @@ mod tests { ctrl.set_mode(PriorityMode::ScreenShare); assert_eq!(ctrl.mode(), PriorityMode::ScreenShare); } + + #[test] + fn screenshare_above_floor_is_normal() { + // 1 Mbps → ~900 kbps after 90% factor. Video budget ~884 kbps > 150. + let bwe = dummy_bwe(1_000_000); + let ctrl = VideoQualityController::new(bwe); + ctrl.set_mode(PriorityMode::ScreenShare); + assert_eq!(ctrl.encoder_mode(), crate::EncoderMode::Normal); + } + + #[test] + fn screenshare_below_floor_is_slide_fallback() { + // 100 kbps → ~90 kbps after 90% factor. Video budget ~74 kbps < 150. + let bwe = dummy_bwe(100_000); + let ctrl = VideoQualityController::new(bwe); + ctrl.set_mode(PriorityMode::ScreenShare); + assert_eq!(ctrl.encoder_mode(), crate::EncoderMode::SlideFallback); + } + + #[test] + fn non_screenshare_never_slide_fallback() { + let bwe = dummy_bwe(50_000); + let ctrl = VideoQualityController::new(bwe); + ctrl.set_mode(PriorityMode::AudioFirst); + assert_eq!(ctrl.encoder_mode(), crate::EncoderMode::Normal); + ctrl.set_mode(PriorityMode::VideoFirst); + assert_eq!(ctrl.encoder_mode(), crate::EncoderMode::Normal); + ctrl.set_mode(PriorityMode::Balanced); + assert_eq!(ctrl.encoder_mode(), crate::EncoderMode::Normal); + } } diff --git a/crates/wzp-video/src/encoder.rs b/crates/wzp-video/src/encoder.rs index f1180db..efa00b5 100644 --- a/crates/wzp-video/src/encoder.rs +++ b/crates/wzp-video/src/encoder.rs @@ -44,6 +44,11 @@ pub trait VideoEncoder: Send { /// Default implementation is a no-op; platform encoders override to /// reconfigure the underlying session. fn set_target(&mut self, _target: &crate::VideoTarget) {} + + /// Switch the encoder operating mode (normal vs slide fallback). + /// + /// Default implementation is a no-op. + fn set_mode(&mut self, _mode: crate::EncoderMode) {} } /// Raw video frame input for encoding. diff --git a/crates/wzp-video/src/encoder_mode.rs b/crates/wzp-video/src/encoder_mode.rs new file mode 100644 index 0000000..8db6ae7 --- /dev/null +++ b/crates/wzp-video/src/encoder_mode.rs @@ -0,0 +1,15 @@ +//! Encoder operating mode — normal continuous video or slide fallback. +//! +//! See `docs/PRD/PRD-video-quality-priority.md` (ScreenShare slide-fallback). + +/// Operating mode for the video encoder. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum EncoderMode { + /// Normal continuous-frame encoding at the target fps. + #[default] + Normal, + /// Slide fallback: emit one high-quality I-frame every 2–5 s, + /// no P-frames. Used when bandwidth is below the SD video floor + /// during a ScreenShare session. + SlideFallback, +} diff --git a/crates/wzp-video/src/lib.rs b/crates/wzp-video/src/lib.rs index 95f500e..6713d22 100644 --- a/crates/wzp-video/src/lib.rs +++ b/crates/wzp-video/src/lib.rs @@ -8,6 +8,7 @@ pub mod controller; pub mod decoder; pub mod depacketizer; pub mod encoder; +pub mod encoder_mode; pub mod framer; pub mod mediacodec; pub mod nack; @@ -17,6 +18,7 @@ pub use controller::{VideoQualityController, VideoTarget}; pub use decoder::VideoDecoder; pub use depacketizer::H264Depacketizer; pub use encoder::{VideoEncoder, VideoError, VideoFrame}; +pub use encoder_mode::EncoderMode; pub use framer::{FramedPacket, H264Framer}; pub use nack::{CachedPacket, NackAction, NackReceiver, NackSender}; pub use videotoolbox::{VideoToolboxDecoder, VideoToolboxEncoder}; diff --git a/docs/PRD/TASKS.md b/docs/PRD/TASKS.md index 228ddb3..8531906 100644 --- a/docs/PRD/TASKS.md +++ b/docs/PRD/TASKS.md @@ -1711,8 +1711,8 @@ Statuses (in order of progression): | T4.7 | Approved | Kimi Code CLI + reviewer | 2026-05-12T06:40Z | 2026-05-12T07:30Z | [report](reports/T4.7-report.md) | Approved. Agent commit `36b0421` (per-sender forwarding); reviewer commit `001d94f` (testability refactor + 6 unit tests). 93 → 99 wzp-relay lib tests. | | T5.1 | Approved | Kimi Code CLI | 2026-05-12T07:00Z | 2026-05-12T07:25Z | [report](reports/T5.1-report.md) | Approved. PriorityMode enum + SetPriorityMode signal + QualityProfile video fields. Commit `c8d1239`. Spawned T5.1.1 for round-trip / default tests. | | T5.1.1 | Open | — | — | — | — | Spawned from T5.1. Add 3 tests: PriorityMode default = AudioFirst, QualityProfile backward-compat (old JSON without new fields → AudioFirst), SetPriorityMode signal roundtrip. ~15 min. | -| T5.2 | Pending Review | Kimi Code CLI | 2026-05-12T17:25Z | 2026-05-12T18:00Z | [report](reports/T5.2-report.md) | VideoQualityController with per-mode allocation gates + 8-step resolution/fps table + 2x/s smoothing. | -| T5.3 | Open | — | — | — | — | Skeleton — expand before claiming | +| T5.2 | Approved | Kimi Code CLI | 2026-05-12T07:25Z | 2026-05-12T08:00Z | [report](reports/T5.2-report.md) | Approved. VideoQualityController + 4 PriorityMode gates + 8 unit tests + 2× smoothing. Commit `2e0bdc5`. | +| T5.3 | Pending Review | Kimi Code CLI | 2026-05-12T18:00Z | 2026-05-12T18:10Z | [report](reports/T5.3-report.md) | EncoderMode enum (Normal / SlideFallback) + VideoEncoder::set_mode() trait method + VideoQualityController::encoder_mode() detection at 150 kbps SD floor. | | T5.4 | Open | — | — | — | — | Skeleton — expand before claiming | | T5.5 | Open | — | — | — | — | Skeleton — expand before claiming | | T5.6 | Open | — | — | — | — | Skeleton — expand before claiming | diff --git a/docs/PRD/reports/T5.2-report.md b/docs/PRD/reports/T5.2-report.md index e1dac3d..e726388 100644 --- a/docs/PRD/reports/T5.2-report.md +++ b/docs/PRD/reports/T5.2-report.md @@ -1,10 +1,10 @@ # T5.2 — `VideoQualityController` with per-mode allocation gates -**Status:** Pending Review +**Status:** Approved **Agent:** Kimi Code CLI **Started:** 2026-05-12T17:25Z **Completed:** 2026-05-12T18:00Z -**Commit:** +**Commit:** 2e0bdc5 **PRD:** ../PRD-video-quality-priority.md ## What I changed diff --git a/docs/PRD/reports/T5.3-report.md b/docs/PRD/reports/T5.3-report.md new file mode 100644 index 0000000..496a090 --- /dev/null +++ b/docs/PRD/reports/T5.3-report.md @@ -0,0 +1,58 @@ +# T5.3 — `EncoderMode::SlideFallback` for ScreenShare + +**Status:** Pending Review +**Agent:** Kimi Code CLI +**Started:** 2026-05-12T18:00Z +**Completed:** 2026-05-12T18:10Z +**Commit:** +**PRD:** ../PRD-video-quality-priority.md + +## What I changed + +- `crates/wzp-video/src/encoder_mode.rs` — New file. `EncoderMode` enum with `Normal` (default) and `SlideFallback` variants. +- `crates/wzp-video/src/lib.rs:11,18` — Added `pub mod encoder_mode;` and re-exported `EncoderMode`. +- `crates/wzp-video/src/encoder.rs:47-50` — Added `set_mode(&mut self, mode: EncoderMode)` default no-op method to `VideoEncoder` trait. Platform encoders override when slide-mode reconfiguration is implemented. +- `crates/wzp-video/src/controller.rs:113-115` — Added `SD_VIDEO_FLOOR_KBPS = 150` constant. +- `crates/wzp-video/src/controller.rs:164-180` — Added `encoder_mode()` method: returns `SlideFallback` when `PriorityMode::ScreenShare` + video budget < 150 kbps, otherwise `Normal`. +- `crates/wzp-video/src/controller.rs:420-442` — Added 3 tests: screenshare-above-floor-normal, screenshare-below-floor-slide, non-screenshare-never-slide. + +## Deviations from the task spec + +Skeleton task. Followed PRD "ScreenShare slide-fallback" section. The actual hardware-encoder slide-mode implementation (configuring VTCompressionSession / AMediaCodec to emit one I-frame every 2–5 s) is deferred — the trait method is a no-op default so existing encoders don't break. + +## Verification output + +```bash +$ cargo test -p wzp-video --lib +test result: ok. 43 passed; 0 failed; 0 ignored +``` + +```bash +$ cargo build -p wzp-video -p wzp-proto -p wzp-relay -p wzp-client -p wzp-android -p wzp-codec -p wzp-desktop +# Finished successfully +``` + +```bash +$ cargo fmt --all -- --check +# pass +``` + +## Test summary + +- Tests added: 3 (`screenshare_above_floor_is_normal`, `screenshare_below_floor_is_slide_fallback`, `non_screenshare_never_slide_fallback`) +- Tests modified: 0 +- `cargo clippy -p wzp-video --all-targets -- -D warnings`: pass +- `cargo fmt --all -- --check`: pass + +## Risks / follow-ups + +1. **Hardware encoder slide mode not implemented** — VideoToolbox and MediaCodec `set_mode()` are no-ops. Real implementation needs platform-specific code to set `kVTEncodeFrameRate` / `KEY_FRAME_RATE` to ~0.33 fps and force every frame as keyframe. +2. **Caller not yet wiring `encoder_mode()`** — The engine code that calls `VideoQualityController::tick()` also needs to call `encoder_mode()` and pass the result to `encoder.set_mode()`. + +## Reviewer checklist (filled in by reviewer) + +- [ ] Code matches PRD intent +- [ ] Verification output is real +- [ ] No backward-incompat surprises +- [ ] Tests cover the new behavior +- [ ] Approved