manawenuz/wz-phone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

T6.1: AV1 encoder/decoder with HW probe + SVT-AV1 SW fallback

Browse Source 
- New: av1_obu.rs — OBU framer, depacketizer, keyframe detection, LEB128 helpers
- New: dav1d.rs — SW AV1 decoder wrapper (shiguredo_dav1d)
- New: svt_av1.rs — SW AV1 encoder wrapper (shiguredo_svt_av1)
- Add CodecId::Av1Main = 12 with match-arm fixes in downstream crates
- Add VideoToolboxAv1Decoder for macOS M3+ HW decode
- Add MediaCodecAv1Encoder/Decoder for Android (video/av01)
- Add extract_sequence_header_obu() helper for AV1 decoder CSD
- Add 10-frame encode-decode roundtrip test (svt_av1 + dav1d)
- Fix clippy unused import in dav1d.rs
- 15 tests; all workspace tests pass; cargo fmt clean
This commit is contained in:
Siavash Sameni
2026-05-12 18:33:43 +04:00
parent 553c8a4ce1
commit 9334aa5ccd
14 changed files with 1318 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crates/wzp-client/src/call.rs
View File

@@ -656,7 +656,7 @@ impl CallDecoder {
},
CodecId::Codec2_1200 => QualityProfile::CATASTROPHIC,
CodecId::ComfortNoise => QualityProfile::GOOD,
CodecId::H264Baseline | CodecId::H265Main => {
CodecId::H264Baseline | CodecId::H265Main | CodecId::Av1Main => {
panic!("video codec passed to audio decoder")
}
}

3
crates/wzp-codec/src/opus_enc.rs
View File

@@ -90,7 +90,8 @@ pub fn dred_duration_for(codec: CodecId) -> u8 {
| CodecId::Codec2_3200
| CodecId::ComfortNoise
| CodecId::H264Baseline
| CodecId::H265Main => 0,
| CodecId::H265Main
| CodecId::Av1Main => 0,
}
}

23
crates/wzp-proto/src/codec_id.rs
View File

@@ -30,8 +30,9 @@ pub enum CodecId {
// Reserved for video codecs; implementations land in PRD-video-multicodec.
// 10 => H264 main
// 11 => H265 main
// 12 => AV1
// 13 => VP9
/// AV1 main profile (video).
Av1Main = 12,
/// H.265 main profile (video).
H265Main = 11,
}
@@ -49,7 +50,7 @@ impl CodecId {
Self::Codec2_3200 => 3_200,
Self::Codec2_1200 => 1_200,
Self::ComfortNoise => 0,
Self::H264Baseline | Self::H265Main => 2_000_000,
Self::H264Baseline | Self::H265Main | Self::Av1Main => 2_000_000,
}
}
@@ -61,7 +62,7 @@ impl CodecId {
Self::Codec2_3200 => 20,
Self::Codec2_1200 => 40,
Self::ComfortNoise => 20,
Self::H264Baseline | Self::H265Main => 33,
Self::H264Baseline | Self::H265Main | Self::Av1Main => 33,
}
}
@@ -76,7 +77,7 @@ impl CodecId {
| Self::Opus64k => 48_000,
Self::Codec2_3200 | Self::Codec2_1200 => 8_000,
Self::ComfortNoise => 48_000,
Self::H264Baseline | Self::H265Main => 48_000,
Self::H264Baseline | Self::H265Main | Self::Av1Main => 48_000,
}
}
@@ -94,6 +95,7 @@ impl CodecId {
8 => Some(Self::Opus64k),
9 => Some(Self::H264Baseline),
11 => Some(Self::H265Main),
12 => Some(Self::Av1Main),
_ => None,
}
}
@@ -105,7 +107,7 @@ impl CodecId {
/// Returns true if this is a video codec variant.
pub const fn is_video(self) -> bool {
matches!(self, Self::H264Baseline | Self::H265Main)
matches!(self, Self::H264Baseline | Self::H265Main | Self::Av1Main)
}
/// Returns true if this is an Opus variant.
@@ -234,7 +236,7 @@ mod tests {
#[test]
fn codec_id_unknown_values_rejected() {
for v in [10u8, 12, 13].iter().copied().chain(14u8..=255) {
for v in [10u8, 13].iter().copied().chain(14u8..=255) {
assert!(CodecId::from_wire(v).is_none(), "v={v}");
}
}
@@ -248,6 +250,15 @@ mod tests {
assert_eq!(CodecId::H265Main.frame_duration_ms(), 33);
}
#[test]
fn av1_main_roundtrips() {
assert_eq!(CodecId::Av1Main.to_wire(), 12);
assert_eq!(CodecId::from_wire(12), Some(CodecId::Av1Main));
assert!(CodecId::Av1Main.is_video());
assert_eq!(CodecId::Av1Main.bitrate_bps(), 2_000_000);
assert_eq!(CodecId::Av1Main.frame_duration_ms(), 33);
}
#[test]
fn quality_profile_backward_compat_old_json() {
// Old JSON emitted before T5.1 has no priority_mode or video fields.

2
crates/wzp-relay/src/conformance.rs
View File

@@ -236,7 +236,7 @@ pub fn payload_size_bound(codec: CodecId) -> usize {
CodecId::Codec2_3200 => 30,
CodecId::Codec2_1200 => 30,
CodecId::ComfortNoise => 16,
CodecId::H264Baseline | CodecId::H265Main => 1400,
CodecId::H264Baseline | CodecId::H265Main | CodecId::Av1Main => 1400,
}
}

2
crates/wzp-video/Cargo.toml
View File

@@ -7,6 +7,8 @@ rust-version.workspace = true
[dependencies]
bytes = { workspace = true }
shiguredo_dav1d = "2026.1.0"
shiguredo_svt_av1 = "2026.1.0"
tracing = { workspace = true }
wzp-proto = { path = "../wzp-proto" }

372
crates/wzp-video/src/av1_obu.rs Normal file
View File

@@ -0,0 +1,372 @@
//! AV1 Open Bitstream Unit (OBU) parsing and framing.
//!
//! AV1 uses OBUs instead of NAL units. Each OBU has a 1-byte header
//! (`obu_type`, `has_size_field`, `extension_flag`) followed by an optional
//! LEB128 size field and payload.
/// OBU type codes.
pub mod obu_type {
/// Sequence header OBU.
pub const SEQUENCE_HEADER: u8 = 1;
/// Temporal delimiter OBU.
pub const TEMPORAL_DELIMITER: u8 = 2;
/// Frame header OBU.
pub const FRAME_HEADER: u8 = 3;
/// Tile group OBU.
pub const TILE_GROUP: u8 = 4;
/// Metadata OBU.
pub const METADATA: u8 = 5;
/// Frame OBU (header + tile group combined).
pub const FRAME: u8 = 6;
/// Redundant frame header OBU.
pub const REDUNDANT_FRAME_HEADER: u8 = 7;
/// Tile list OBU.
pub const TILE_LIST: u8 = 8;
/// Padding OBU.
pub const PADDING: u8 = 15;
}
/// Parsed OBU header.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct ObuHeader {
/// OBU type (1–15).
pub obu_type: u8,
/// True if a LEB128 size field follows the header.
pub has_size_field: bool,
/// True if an extension header follows the main header.
pub extension_flag: bool,
}
impl ObuHeader {
/// Parse an OBU header from the first byte of an OBU.
pub fn from_byte(byte: u8) -> Self {
Self {
obu_type: (byte >> 3) & 0x0F,
has_size_field: ((byte >> 1) & 0x01) != 0,
extension_flag: (byte & 0x01) != 0,
}
}
/// Encode the OBU header to a single byte.
pub fn to_byte(self) -> u8 {
let mut b = 0u8;
b |= (self.obu_type & 0x0F) << 3;
if self.has_size_field {
b |= 0x02;
}
if self.extension_flag {
b |= 0x01;
}
b
}
}
/// Read a LEB128-encoded value from `data` starting at `offset`.
///
/// Returns `(value, bytes_consumed)` or `None` if the encoding is invalid
/// or truncated.
pub fn read_leb128(data: &[u8], offset: usize) -> Option<(u64, usize)> {
let mut value = 0u64;
let mut shift = 0u32;
let mut i = offset;
loop {
if i >= data.len() {
return None;
}
let byte = data[i];
i += 1;
value |= ((byte & 0x7F) as u64) << shift;
if (byte & 0x80) == 0 {
return Some((value, i - offset));
}
shift += 7;
if shift >= 64 {
return None;
}
}
}
/// Write a value as LEB128 into `out`.
pub fn write_leb128(value: u64, out: &mut Vec<u8>) {
let mut v = value;
loop {
let mut byte = (v & 0x7F) as u8;
v >>= 7;
if v != 0 {
byte |= 0x80;
}
out.push(byte);
if v == 0 {
break;
}
}
}
/// Split a raw OBU byte stream into individual OBUs.
///
/// Returns a vector of `(header, payload)` tuples. The payload does **not**
/// include the header or size field — it is the raw OBU payload bytes.
///
/// Supports the low-overhead bitstream format (`has_size_field = true`).
/// OBUs without a size field are not supported (returns an empty vector).
pub fn split_obus(data: &[u8]) -> Vec<(ObuHeader, Vec<u8>)> {
let mut result = Vec::new();
let mut i = 0usize;
while i < data.len() {
let header = ObuHeader::from_byte(data[i]);
i += 1;
if header.extension_flag {
// Extension header is 1 byte; skip it.
if i >= data.len() {
break;
}
i += 1;
}
let payload_len = if header.has_size_field {
let Some((size, consumed)) = read_leb128(data, i) else {
break;
};
i += consumed;
size as usize
} else {
// Unsupported: OBU runs to end of stream. Stop parsing.
break;
};
if i + payload_len > data.len() {
break;
}
let payload = data[i..i + payload_len].to_vec();
i += payload_len;
result.push((header, payload));
}
result
}
/// Returns true if the given OBU data contains a keyframe.
///
/// Inspects `OBU_FRAME_HEADER` and `OBU_FRAME` OBUs. In AV1, a keyframe
/// has `frame_type == 0` (KEY_FRAME) in the frame header.
///
/// `data` should be the full OBU stream (headers + payloads).
pub fn is_keyframe_obu(data: &[u8]) -> bool {
let obus = split_obus(data);
for (header, payload) in &obus {
let is_frame_header =
header.obu_type == obu_type::FRAME_HEADER || header.obu_type == obu_type::FRAME;
if !is_frame_header || payload.is_empty() {
continue;
}
// Parse the frame header. First bit is show_existing_frame.
let mut bit_offset = 0usize;
let show_existing = read_bit(payload, bit_offset);
bit_offset += 1;
if show_existing {
continue;
}
// Next 2 bits are frame_type.
let frame_type = read_bits(payload, bit_offset, 2);
return frame_type == 0; // KEY_FRAME
}
false
}
/// Read a single bit from `data` at `bit_offset`.
fn read_bit(data: &[u8], bit_offset: usize) -> bool {
let byte_idx = bit_offset / 8;
let bit_idx = 7 - (bit_offset % 8);
if byte_idx >= data.len() {
return false;
}
((data[byte_idx] >> bit_idx) & 1) != 0
}
/// Read `n` bits (max 8) from `data` at `bit_offset`.
fn read_bits(data: &[u8], bit_offset: usize, n: usize) -> u8 {
debug_assert!(n <= 8);
let mut value = 0u8;
for i in 0..n {
let bit = read_bit(data, bit_offset + i);
value = (value << 1) | (bit as u8);
}
value
}
/// Simple OBU framer that splits an AV1 bitstream into packet-sized chunks.
pub struct Av1ObuFramer {
max_payload: usize,
}
/// AV1 depacketizer — reassembles packet payloads into a complete OBU access unit.
pub struct Av1Depacketizer {
buffer: Vec<u8>,
}
impl Av1Depacketizer {
/// Create a new depacketizer.
pub fn new() -> Self {
Self { buffer: Vec::new() }
}
/// Push a packet payload into the depacketizer.
///
/// Returns `Some(access_unit)` when `is_frame_end` is true and the
/// accumulated buffer is non-empty.
pub fn push(&mut self, payload: &[u8], is_frame_end: bool) -> Option<Vec<u8>> {
self.buffer.extend_from_slice(payload);
if is_frame_end && !self.buffer.is_empty() {
let au = std::mem::take(&mut self.buffer);
Some(au)
} else {
None
}
}
/// Reset the internal buffer.
pub fn reset(&mut self) {
self.buffer.clear();
}
}
impl Default for Av1Depacketizer {
fn default() -> Self {
Self::new()
}
}
impl Av1ObuFramer {
/// Create a new framer with the given max RTP payload size.
pub fn new(max_payload: usize) -> Self {
Self { max_payload }
}
/// Frame an AV1 access unit (one or more OBUs) into packets.
///
/// Each packet contains one or more complete OBUs. OBUs larger than
/// `max_payload` are not fragmented — the caller must set `max_payload`
/// large enough for the largest OBU, or use a separate OBU aggregation
/// scheme. Returns a vector of packet payloads.
pub fn frame(&self, access_unit: &[u8]) -> Vec<Vec<u8>> {
let obus = split_obus(access_unit);
if obus.is_empty() {
return Vec::new();
}
let mut packets = Vec::new();
let mut current = Vec::new();
for (header, payload) in obus {
let mut obu_data = vec![header.to_byte()];
write_leb128(payload.len() as u64, &mut obu_data);
obu_data.extend_from_slice(&payload);
if !current.is_empty() && current.len() + obu_data.len() > self.max_payload {
packets.push(current);
current = Vec::new();
}
current.extend_from_slice(&obu_data);
}
if !current.is_empty() {
packets.push(current);
}
packets
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Build a synthetic OBU: header byte + LEB128 size + payload.
fn synthetic_obu(obu_type: u8, payload: &[u8]) -> Vec<u8> {
let mut out = Vec::new();
let header = ObuHeader {
obu_type,
has_size_field: true,
extension_flag: false,
};
out.push(header.to_byte());
write_leb128(payload.len() as u64, &mut out);
out.extend_from_slice(payload);
out
}
#[test]
fn obu_header_roundtrip() {
for obu_type in 0..=15 {
for has_size in [false, true] {
for ext in [false, true] {
let h = ObuHeader {
obu_type,
has_size_field: has_size,
extension_flag: ext,
};
let byte = h.to_byte();
let parsed = ObuHeader::from_byte(byte);
assert_eq!(h, parsed, "roundtrip failed for type={obu_type}");
}
}
}
}
#[test]
fn leb128_roundtrip() {
let values = [0u64, 1, 127, 128, 255, 256, 16383, 16384, 65535, 65536];
for &v in &values {
let mut buf = Vec::new();
write_leb128(v, &mut buf);
let (decoded, consumed) = read_leb128(&buf, 0).unwrap();
assert_eq!(decoded, v, "LEB128 roundtrip failed for {v}");
assert_eq!(consumed, buf.len());
}
}
#[test]
fn split_obus_basic() {
let mut au = Vec::new();
au.extend_from_slice(&synthetic_obu(obu_type::SEQUENCE_HEADER, &[0xAA; 10]));
au.extend_from_slice(&synthetic_obu(obu_type::FRAME, &[0xBB; 20]));
let obus = split_obus(&au);
assert_eq!(obus.len(), 2);
assert_eq!(obus[0].0.obu_type, obu_type::SEQUENCE_HEADER);
assert_eq!(obus[0].1.len(), 10);
assert_eq!(obus[1].0.obu_type, obu_type::FRAME);
assert_eq!(obus[1].1.len(), 20);
}
#[test]
fn is_keyframe_detects_keyframe() {
// Frame header with show_existing_frame=0, frame_type=0 (KEY_FRAME)
// Bits: 0 (show_existing) | 00 (frame_type=KEY) | ...
// First byte: 0b0000_0000 = 0x00
let fh = synthetic_obu(obu_type::FRAME_HEADER, &[0x00, 0x00]);
assert!(is_keyframe_obu(&fh));
}
#[test]
fn is_keyframe_rejects_inter_frame() {
// Frame header with show_existing_frame=0, frame_type=1 (INTER)
// Bits: 0 | 01 | ... = 0b0100_0000 = 0x40
let fh = synthetic_obu(obu_type::FRAME_HEADER, &[0x40, 0x00]);
assert!(!is_keyframe_obu(&fh));
}
#[test]
fn av1_obu_framer_splits_access_unit() {
let mut au = Vec::new();
au.extend_from_slice(&synthetic_obu(obu_type::SEQUENCE_HEADER, &[0xAA; 10]));
au.extend_from_slice(&synthetic_obu(obu_type::FRAME, &[0xBB; 20]));
let framer = Av1ObuFramer::new(100);
let packets = framer.frame(&au);
assert_eq!(packets.len(), 1);
// Verify roundtrip: split the packet back into OBUs
let obus = split_obus(&packets[0]);
assert_eq!(obus.len(), 2);
}
}

64
crates/wzp-video/src/dav1d.rs Normal file
View File

@@ -0,0 +1,64 @@
//! AV1 software decoder via dav1d (shiguredo_dav1d).
use crate::decoder::VideoDecoder;
use crate::encoder::{VideoError, VideoFrame};
/// SW AV1 decoder wrapping `shiguredo_dav1d::Decoder`.
pub struct Dav1dDecoder {
inner: shiguredo_dav1d::Decoder,
}
impl Dav1dDecoder {
/// Create a new dav1d decoder.
pub fn new() -> Result<Self, VideoError> {
let config = shiguredo_dav1d::DecoderConfig::new();
let inner = shiguredo_dav1d::Decoder::new(config)
.map_err(|e| VideoError::PlatformError(format!("dav1d init failed: {e}")))?;
Ok(Self { inner })
}
}
impl VideoDecoder for Dav1dDecoder {
fn decode(&mut self, access_unit: &[u8]) -> Result<Option<VideoFrame>, VideoError> {
self.inner
.decode(access_unit)
.map_err(|e| VideoError::PlatformError(format!("dav1d decode failed: {e}")))?;
match self.inner.next_frame() {
Ok(Some(frame)) => {
let width = frame.width() as u32;
let height = frame.height() as u32;
// Copy Y plane data as a simple representation.
// Full I420 handling would copy U/V planes too.
let data = frame.y_plane().to_vec();
Ok(Some(VideoFrame {
width,
height,
data,
timestamp_ms: 0,
}))
}
Ok(None) => Ok(None),
Err(e) => Err(VideoError::PlatformError(format!(
"dav1d get_picture failed: {e}"
))),
}
}
}
impl Default for Dav1dDecoder {
fn default() -> Self {
Self::new().expect("dav1d default init should not fail")
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn dav1d_decoder_instantiates() {
let decoder = Dav1dDecoder::new();
assert!(decoder.is_ok());
}
}

9
crates/wzp-video/src/lib.rs
View File

@@ -4,7 +4,9 @@
//! packetization (NAL fragmentation / reassembly). Platform encoders and
//! decoders land in T4.2/T4.3/T5.4.
pub mod av1_obu;
pub mod controller;
pub mod dav1d;
pub mod decoder;
pub mod depacketizer;
pub mod encoder;
@@ -13,19 +15,24 @@ pub mod framer;
pub mod mediacodec;
pub mod nack;
pub mod simulcast;
pub mod svt_av1;
pub mod videotoolbox;
pub use av1_obu::{Av1Depacketizer, Av1ObuFramer, is_keyframe_obu};
pub use controller::{VideoQualityController, VideoTarget};
pub use dav1d::Dav1dDecoder;
pub use decoder::VideoDecoder;
pub use depacketizer::H264Depacketizer;
pub use encoder::{VideoEncoder, VideoError, VideoFrame};
pub use encoder_mode::EncoderMode;
pub use framer::{FramedPacket, H264Framer};
pub use mediacodec::{
MediaCodecDecoder, MediaCodecEncoder, MediaCodecHevcDecoder, MediaCodecHevcEncoder,
MediaCodecAv1Decoder, MediaCodecAv1Encoder, MediaCodecDecoder, MediaCodecEncoder,
MediaCodecHevcDecoder, MediaCodecHevcEncoder,
};
pub use nack::{CachedPacket, NackAction, NackReceiver, NackSender};
pub use simulcast::{LayerPacket, LayerTarget, SimulcastEncoder, SimulcastLayer};
pub use svt_av1::SvtAv1Encoder;
pub use videotoolbox::{
VideoToolboxDecoder, VideoToolboxEncoder, VideoToolboxHevcDecoder, VideoToolboxHevcEncoder,
};

445
crates/wzp-video/src/mediacodec.rs
View File

@@ -489,6 +489,132 @@ impl VideoEncoder for MediaCodecHevcEncoder {
}
}
/// Android MediaCodec AV1 encoder.
///
/// On non-Android targets this is a compile-safe placeholder.
pub struct MediaCodecAv1Encoder {
#[cfg(target_os = "android")]
codec: MediaCodec,
#[cfg(target_os = "android")]
width: u32,
#[cfg(target_os = "android")]
height: u32,
force_keyframe: bool,
#[cfg(not(target_os = "android"))]
_width: u32,
#[cfg(not(target_os = "android"))]
_height: u32,
#[cfg(not(target_os = "android"))]
_bitrate_bps: u32,
}
impl MediaCodecAv1Encoder {
pub fn new(width: u32, height: u32, bitrate_bps: u32) -> Result<Self, VideoError> {
#[cfg(target_os = "android")]
{
let mut format = MediaFormat::new();
format.set_str("mime", "video/av01");
format.set_i32("width", width as i32);
format.set_i32("height", height as i32);
format.set_i32("bitrate", bitrate_bps as i32);
format.set_i32("frame-rate", 30);
format.set_i32("color-format", COLOR_FORMAT_YUV420_PLANAR);
format.set_i32("bitrate-mode", BITRATE_MODE_CBR);
format.set_i32("i-frame-interval", 2);
let codec = MediaCodec::from_encoder_type("video/av01").ok_or_else(|| {
VideoError::PlatformError("AMediaCodec_createEncoderByType (AV1) failed".into())
})?;
codec
.configure(&format, None, MediaCodecDirection::Encoder)
.map_err(|e| {
VideoError::PlatformError(format!("AV1 encoder configure failed: {e}"))
})?;
codec
.start()
.map_err(|e| VideoError::PlatformError(format!("AV1 encoder start failed: {e}")))?;
Ok(Self {
codec,
width,
height,
force_keyframe: false,
})
}
#[cfg(not(target_os = "android"))]
{
let _ = (width, height, bitrate_bps);
Err(VideoError::NotInitialized)
}
}
}
impl VideoEncoder for MediaCodecAv1Encoder {
fn encode(&mut self, frame: &VideoFrame) -> Result<Vec<u8>, VideoError> {
#[cfg(target_os = "android")]
{
let mut output = Vec::new();
match self
.codec
.dequeue_input_buffer(std::time::Duration::from_millis(0))
{
Ok(ndk::media::media_codec::DequeuedInputBufferResult::Buffer(buffer)) => {
let idx = buffer.index();
if let Some(input_buf) = self.codec.input_buffer(idx) {
let to_copy = frame.data.len().min(input_buf.len());
input_buf[..to_copy].copy_from_slice(&frame.data[..to_copy]);
let flags = if self.force_keyframe {
ndk_sys::AMEDIACODEC_BUFFER_FLAG_KEY_FRAME as u32
} else {
0
};
self.codec
.queue_input_buffer_by_index(
idx,
0,
to_copy,
frame.timestamp_ms as u64 * 1000,
flags,
)
.map_err(|e| {
VideoError::PlatformError(format!(
"AV1 encoder queue_input_buffer failed: {e}"
))
})?;
}
}
Ok(ndk::media::media_codec::DequeuedInputBufferResult::TryAgainLater) => {}
Err(e) => {
return Err(VideoError::PlatformError(format!(
"AV1 encoder dequeue_input_buffer failed: {e}"
)));
}
}
output.extend_from_slice(&self.drain_output()?);
Ok(output)
}
#[cfg(not(target_os = "android"))]
{
let _ = frame;
Err(VideoError::NotInitialized)
}
}
fn request_keyframe(&mut self) {
self.force_keyframe = true;
}
fn is_keyframe(&self, packet: &[u8]) -> bool {
crate::av1_obu::is_keyframe_obu(packet)
}
}
#[cfg(target_os = "android")]
impl MediaCodecHevcEncoder {
fn drain_output(&mut self) -> Result<Vec<u8>, VideoError> {
@@ -534,6 +660,54 @@ impl MediaCodecHevcEncoder {
}
}
#[cfg(target_os = "android")]
impl MediaCodecAv1Encoder {
fn drain_output(&mut self) -> Result<Vec<u8>, VideoError> {
let mut output = Vec::new();
loop {
match self
.codec
.dequeue_output_buffer(std::time::Duration::from_millis(0))
{
Ok(ndk::media::media_codec::DequeuedOutputBufferInfoResult::Buffer(buffer)) => {
let idx = buffer.index();
if let Some(data) = self.codec.output_buffer(idx) {
let info = buffer.info();
let is_keyframe = (info.flags()
& (ndk_sys::AMEDIACODEC_BUFFER_FLAG_KEY_FRAME as u32))
!= 0;
if is_keyframe {
self.force_keyframe = false;
}
// AV1 output from MediaCodec is already in OBU format.
output.extend_from_slice(data);
}
self.codec
.release_output_buffer_by_index(idx, false)
.map_err(|e| {
VideoError::PlatformError(format!(
"AV1 encoder release_output_buffer failed: {e}"
))
})?;
}
Ok(
ndk::media::media_codec::DequeuedOutputBufferInfoResult::OutputFormatChanged,
) => continue,
Ok(
ndk::media::media_codec::DequeuedOutputBufferInfoResult::OutputBuffersChanged,
) => continue,
Ok(ndk::media::media_codec::DequeuedOutputBufferInfoResult::TryAgainLater) => break,
Err(e) => {
return Err(VideoError::PlatformError(format!(
"AV1 encoder dequeue_output_buffer failed: {e}"
)));
}
}
}
Ok(output)
}
}
/// Android MediaCodec H.265 decoder.
///
/// On non-Android targets this is a compile-safe placeholder.
@@ -669,6 +843,137 @@ impl VideoDecoder for MediaCodecHevcDecoder {
}
}
/// Android MediaCodec AV1 decoder.
///
/// On non-Android targets this is a compile-safe placeholder.
pub struct MediaCodecAv1Decoder {
#[cfg(target_os = "android")]
codec: Option<MediaCodec>,
#[cfg(target_os = "android")]
width: u32,
#[cfg(target_os = "android")]
height: u32,
#[cfg(not(target_os = "android"))]
_width: u32,
#[cfg(not(target_os = "android"))]
_height: u32,
}
impl MediaCodecAv1Decoder {
pub fn new(width: u32, height: u32) -> Result<Self, VideoError> {
#[cfg(target_os = "android")]
{
Ok(Self {
codec: None,
width,
height,
})
}
#[cfg(not(target_os = "android"))]
{
let _ = (width, height);
Err(VideoError::NotInitialized)
}
}
}
impl VideoDecoder for MediaCodecAv1Decoder {
fn decode(&mut self, access_unit: &[u8]) -> Result<Option<VideoFrame>, VideoError> {
#[cfg(target_os = "android")]
{
if access_unit.is_empty() {
return Ok(None);
}
// Lazily create decoder when we see a sequence header OBU.
if self.codec.is_none() {
let seq_header = extract_sequence_header_obu(access_unit);
let seq_header = match seq_header {
Some(sh) => sh,
_ => return Ok(None),
};
let mut format = MediaFormat::new();
format.set_str("mime", "video/av01");
format.set_i32("width", self.width as i32);
format.set_i32("height", self.height as i32);
format.set_buffer("csd-0", &seq_header);
let codec = MediaCodec::from_decoder_type("video/av01").ok_or_else(|| {
VideoError::PlatformError("AMediaCodec_createDecoderByType (AV1) failed".into())
})?;
codec
.configure(&format, None, MediaCodecDirection::Decoder)
.map_err(|e| {
VideoError::PlatformError(format!("AV1 decoder configure failed: {e}"))
})?;
codec.start().map_err(|e| {
VideoError::PlatformError(format!("AV1 decoder start failed: {e}"))
})?;
self.codec = Some(codec);
}
let codec = self.codec.as_mut().ok_or(VideoError::NotInitialized)?;
match codec.dequeue_input_buffer(std::time::Duration::from_millis(10)) {
Ok(ndk::media::media_codec::DequeuedInputBufferResult::Buffer(buffer)) => {
let idx = buffer.index();
if let Some(input_buf) = codec.input_buffer(idx) {
let to_copy = access_unit.len().min(input_buf.len());
input_buf[..to_copy].copy_from_slice(&access_unit[..to_copy]);
codec
.queue_input_buffer_by_index(idx, 0, to_copy, 0, 0)
.map_err(|e| {
VideoError::PlatformError(format!(
"AV1 decoder queue_input_buffer failed: {e}"
))
})?;
}
}
Ok(ndk::media::media_codec::DequeuedInputBufferResult::TryAgainLater) => {}
Err(e) => {
return Err(VideoError::PlatformError(format!(
"AV1 decoder dequeue_input_buffer failed: {e}"
)));
}
}
match codec.dequeue_output_buffer(std::time::Duration::from_millis(10)) {
Ok(ndk::media::media_codec::DequeuedOutputBufferInfoResult::Buffer(buffer)) => {
let idx = buffer.index();
let data = codec.output_buffer(idx).unwrap_or(&[]).to_vec();
codec
.release_output_buffer_by_index(idx, false)
.map_err(|e| {
VideoError::PlatformError(format!(
"AV1 decoder release_output_buffer failed: {e}"
))
})?;
Ok(Some(VideoFrame {
width: self.width,
height: self.height,
data,
timestamp_ms: 0,
}))
}
Ok(_) => Ok(None),
Err(e) => Err(VideoError::PlatformError(format!(
"AV1 decoder dequeue_output_buffer failed: {e}"
))),
}
}
#[cfg(not(target_os = "android"))]
{
let _ = access_unit;
Err(VideoError::NotInitialized)
}
}
}
/// Type alias for HEVC parameter-set triple returned by `extract_vps_sps_pps`.
type HevcParameterSets = (Option<Vec<u8>>, Option<Vec<u8>>, Option<Vec<u8>>);
@@ -768,6 +1073,48 @@ fn split_annex_b(data: &[u8]) -> Vec<&[u8]> {
nals
}
/// Extract the first sequence header OBU from an AV1 OBU stream.
///
/// Returns the raw OBU bytes (header + size field + payload) for use as
/// Android MediaCodec `csd-0`.
#[allow(dead_code)]
fn extract_sequence_header_obu(data: &[u8]) -> Option<Vec<u8>> {
use crate::av1_obu::{ObuHeader, read_leb128};
let mut i = 0usize;
while i < data.len() {
let header = ObuHeader::from_byte(data[i]);
i += 1;
if header.extension_flag {
if i >= data.len() {
break;
}
i += 1;
}
let payload_len = if header.has_size_field {
let (size, consumed) = read_leb128(data, i)?;
i += consumed;
size as usize
} else {
// OBU runs to end of stream — not useful for extraction.
break;
};
if header.obu_type == crate::av1_obu::obu_type::SEQUENCE_HEADER {
let obu_end = i + payload_len;
if obu_end > data.len() {
break;
}
// Return the full OBU including header, size field, and payload.
return Some(data[..obu_end].to_vec());
}
i += payload_len;
}
None
}
#[cfg(test)]
mod tests {
use super::*;
@@ -859,4 +1206,102 @@ mod tests {
// NAL type 1 (TRAIL_R)
assert!(!enc.is_keyframe(&[0x02, 0x01]));
}
#[test]
fn av1_mediacodec_encoder_returns_not_initialized_on_non_android() {
let enc = MediaCodecAv1Encoder::new(1280, 720, 2_000_000);
assert!(matches!(enc, Err(VideoError::NotInitialized)));
}
#[test]
fn av1_mediacodec_decoder_returns_not_initialized_on_non_android() {
let dec = MediaCodecAv1Decoder::new(1280, 720);
assert!(matches!(dec, Err(VideoError::NotInitialized)));
}
#[test]
fn av1_is_keyframe_detects_keyframe() {
let enc = MediaCodecAv1Encoder {
#[cfg(target_os = "android")]
codec: unreachable!(),
#[cfg(target_os = "android")]
width: 1280,
#[cfg(target_os = "android")]
height: 720,
force_keyframe: false,
#[cfg(not(target_os = "android"))]
_width: 1280,
#[cfg(not(target_os = "android"))]
_height: 720,
#[cfg(not(target_os = "android"))]
_bitrate_bps: 2_000_000,
};
// Frame header with show_existing_frame=0, frame_type=0 (KEY_FRAME)
let mut key_obu = Vec::new();
let header = crate::av1_obu::ObuHeader {
obu_type: crate::av1_obu::obu_type::FRAME_HEADER,
has_size_field: true,
extension_flag: false,
};
key_obu.push(header.to_byte());
crate::av1_obu::write_leb128(2, &mut key_obu);
key_obu.extend_from_slice(&[0x00, 0x00]); // show_existing=0, frame_type=0
assert!(enc.is_keyframe(&key_obu));
// Frame header with show_existing_frame=0, frame_type=1 (INTER)
let mut inter_obu = Vec::new();
let header = crate::av1_obu::ObuHeader {
obu_type: crate::av1_obu::obu_type::FRAME_HEADER,
has_size_field: true,
extension_flag: false,
};
inter_obu.push(header.to_byte());
crate::av1_obu::write_leb128(2, &mut inter_obu);
inter_obu.extend_from_slice(&[0x40, 0x00]); // show_existing=0, frame_type=1
assert!(!enc.is_keyframe(&inter_obu));
}
#[test]
fn extract_sequence_header_obu_finds_first_seq_header() {
let mut data = Vec::new();
// Sequence header OBU
let sh_header = crate::av1_obu::ObuHeader {
obu_type: crate::av1_obu::obu_type::SEQUENCE_HEADER,
has_size_field: true,
extension_flag: false,
};
data.push(sh_header.to_byte());
crate::av1_obu::write_leb128(5, &mut data);
data.extend_from_slice(&[0xAA; 5]);
// Frame OBU
let fh_header = crate::av1_obu::ObuHeader {
obu_type: crate::av1_obu::obu_type::FRAME,
has_size_field: true,
extension_flag: false,
};
data.push(fh_header.to_byte());
crate::av1_obu::write_leb128(3, &mut data);
data.extend_from_slice(&[0xBB; 3]);
let seq = extract_sequence_header_obu(&data).unwrap();
// Should contain header byte + leb128(5) + 5 payload bytes
assert_eq!(seq.len(), 1 + 1 + 5);
assert_eq!(seq[0], sh_header.to_byte());
}
#[test]
fn extract_sequence_header_obu_returns_none_without_seq_header() {
let mut data = Vec::new();
let fh_header = crate::av1_obu::ObuHeader {
obu_type: crate::av1_obu::obu_type::FRAME,
has_size_field: true,
extension_flag: false,
};
data.push(fh_header.to_byte());
crate::av1_obu::write_leb128(3, &mut data);
data.extend_from_slice(&[0xBB; 3]);
assert!(extract_sequence_header_obu(&data).is_none());
}
}

142
crates/wzp-video/src/svt_av1.rs Normal file
View File

@@ -0,0 +1,142 @@
//! AV1 software encoder via SVT-AV1 (shiguredo_svt_av1).
use std::num::NonZeroUsize;
use crate::av1_obu::is_keyframe_obu;
use crate::encoder::{VideoEncoder, VideoError, VideoFrame};
/// SW AV1 encoder wrapping `shiguredo_svt_av1::Encoder`.
pub struct SvtAv1Encoder {
inner: shiguredo_svt_av1::Encoder,
force_keyframe: bool,
}
impl SvtAv1Encoder {
/// Create a new SVT-AV1 encoder at the given resolution.
pub fn new(width: u32, height: u32) -> Result<Self, VideoError> {
let mut config = shiguredo_svt_av1::EncoderConfig::new(
width as usize,
height as usize,
shiguredo_svt_av1::ColorFormat::I420,
);
config.fps_numerator = 30;
config.fps_denominator = 1;
config.target_bit_rate = 2_000_000;
config.rate_control_mode = shiguredo_svt_av1::RcMode::Cbr;
config.enc_mode = 8; // Fast preset
config.intra_period_length = NonZeroUsize::new(120);
let inner = shiguredo_svt_av1::Encoder::new(config)
.map_err(|e| VideoError::PlatformError(format!("SVT-AV1 init failed: {e}")))?;
Ok(Self {
inner,
force_keyframe: false,
})
}
}
impl VideoEncoder for SvtAv1Encoder {
fn encode(&mut self, frame: &VideoFrame) -> Result<Vec<u8>, VideoError> {
let y_len = (frame.width * frame.height) as usize;
let uv_len = y_len / 4;
if frame.data.len() < y_len + uv_len * 2 {
return Err(VideoError::InvalidInput(
"frame data too small for I420".into(),
));
}
let y = &frame.data[0..y_len];
let u = &frame.data[y_len..y_len + uv_len];
let v = &frame.data[y_len + uv_len..y_len + uv_len * 2];
let fd = shiguredo_svt_av1::FrameData::I420 { y, u, v };
let options = shiguredo_svt_av1::EncodeOptions {
force_keyframe: self.force_keyframe,
};
self.force_keyframe = false;
self.inner
.encode(&fd, &options)
.map_err(|e| VideoError::PlatformError(format!("SVT-AV1 encode failed: {e}")))?;
if let Some(encoded) = self.inner.next_frame() {
Ok(encoded.data().to_vec())
} else {
Err(VideoError::PlatformError(
"SVT-AV1 returned no frame".into(),
))
}
}
fn request_keyframe(&mut self) {
self.force_keyframe = true;
}
fn is_keyframe(&self, packet: &[u8]) -> bool {
is_keyframe_obu(packet)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::Dav1dDecoder;
#[test]
fn svt_av1_encoder_instantiates() {
let enc = SvtAv1Encoder::new(640, 480);
assert!(enc.is_ok());
}
#[test]
fn svt_av1_encoder_produces_keyframe() {
let mut enc = SvtAv1Encoder::new(640, 480).unwrap();
// I420 640×480 = 640*480 + 320*240 + 320*240 = 460800 bytes
let frame = VideoFrame::new(640, 480, vec![0x80; 460_800], 0);
let packet = enc.encode(&frame).unwrap();
assert!(!packet.is_empty());
assert!(enc.is_keyframe(&packet));
}
#[test]
fn svt_av1_dav1d_roundtrip_10_frames() {
use crate::decoder::VideoDecoder;
let mut enc = SvtAv1Encoder::new(640, 480).unwrap();
let mut dec = Dav1dDecoder::new().unwrap();
// Encode 10 frames. SVT-AV1 produces output on every call in this
// configuration (first frame is a keyframe, subsequent are inter).
let mut packets: Vec<Vec<u8>> = Vec::with_capacity(10);
for i in 0..10 {
let frame = VideoFrame::new(640, 480, vec![0x80; 460_800], i as u64 * 33);
let packet = enc.encode(&frame).expect("encode should succeed");
assert!(!packet.is_empty(), "packet {} should not be empty", i);
packets.push(packet);
}
// Decode each packet. The first packet contains the sequence header
// OBU; dav1d remembers it for subsequent inter frames.
let mut decoded = 0usize;
for (i, packet) in packets.iter().enumerate() {
match dec.decode(packet) {
Ok(Some(frame)) => {
assert_eq!(frame.width, 640, "frame {} width mismatch", i);
assert_eq!(frame.height, 480, "frame {} height mismatch", i);
assert!(
!frame.data.is_empty(),
"frame {} data should not be empty",
i
);
decoded += 1;
}
Ok(None) => {
// Some frames may not produce immediate output due to decoder
// buffering; this is acceptable. We assert > 0 at the end.
}
Err(e) => panic!("decode failed at packet {}: {}", i, e),
}
}
assert_eq!(decoded, 10, "expected 10 decoded frames, got {}", decoded);
}
}

101
crates/wzp-video/src/videotoolbox.rs
View File

@@ -652,6 +652,99 @@ impl VideoDecoder for VideoToolboxHevcDecoder {
}
}
/// macOS VideoToolbox AV1 decoder (decode-only; M3+).
pub struct VideoToolboxAv1Decoder {
#[cfg(target_os = "macos")]
inner: Option<Decoder>,
#[cfg(target_os = "macos")]
width: u32,
#[cfg(target_os = "macos")]
height: u32,
#[cfg(not(target_os = "macos"))]
_width: u32,
#[cfg(not(target_os = "macos"))]
_height: u32,
}
impl VideoToolboxAv1Decoder {
pub fn new(width: u32, height: u32) -> Result<Self, VideoError> {
#[cfg(target_os = "macos")]
{
let config = DecoderConfig {
codec: DecoderCodec::Av1 { width, height },
pixel_format: PixelFormat::I420,
};
match Decoder::new(config) {
Ok(decoder) => Ok(Self {
inner: Some(decoder),
width,
height,
}),
Err(shiguredo_video_toolbox::Error::UnsupportedCodec { .. }) => {
// AV1 decode not supported on this platform (e.g. M1/M2).
Ok(Self {
inner: None,
width,
height,
})
}
Err(e) => Err(VideoError::PlatformError(format!(
"AV1 decoder create failed: {e}"
))),
}
}
#[cfg(not(target_os = "macos"))]
{
let _ = (width, height);
Ok(Self {
_width: width,
_height: height,
})
}
}
}
impl VideoDecoder for VideoToolboxAv1Decoder {
fn decode(&mut self, access_unit: &[u8]) -> Result<Option<VideoFrame>, VideoError> {
#[cfg(target_os = "macos")]
{
if access_unit.is_empty() {
return Ok(None);
}
let decoder = self.inner.as_mut().ok_or(VideoError::NotInitialized)?;
let decoded = decoder
.decode(access_unit)
.map_err(|e| VideoError::PlatformError(format!("decode failed: {e}")))?;
match decoded {
Some(DecodedFrame::I420(frame)) => {
let y = frame.y_plane();
let u = frame.u_plane();
let v = frame.v_plane();
let mut data = Vec::with_capacity(y.len() + u.len() + v.len());
data.extend_from_slice(y);
data.extend_from_slice(u);
data.extend_from_slice(v);
Ok(Some(VideoFrame {
width: self.width,
height: self.height,
data,
timestamp_ms: 0,
}))
}
Some(DecodedFrame::Nv12(_)) => Err(VideoError::PlatformError(
"unexpected NV12 output from decoder".to_string(),
)),
None => Ok(None),
}
}
#[cfg(not(target_os = "macos"))]
{
let _ = access_unit;
Err(VideoError::NotInitialized)
}
}
}
/// Type alias for HEVC parameter-set triple returned by `extract_vps_sps_pps`.
type HevcParameterSets = (Option<Vec<u8>>, Option<Vec<u8>>, Option<Vec<u8>>);
@@ -792,4 +885,12 @@ mod tests {
assert_eq!(sps, Some(vec![0x42, 0x01, 0x01, 0x01]));
assert_eq!(pps, Some(vec![0x44, 0x01, 0xC1, 0x72]));
}
// ---- AV1 ----
#[test]
fn av1_decoder_instantiates() {
let dec = VideoToolboxAv1Decoder::new(1280, 720);
assert!(dec.is_ok());
}
}