manawenuz/wz-phone
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

T5.4: H.265 encoder/decoder wrappers — VideoToolbox + MediaCodec, CodecId::H265Main

Browse Source
This commit is contained in:
Siavash Sameni
2026-05-12 14:50:20 +04:00
parent 9c41d1acdd
commit b197651557
7 changed files with 759 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
crates/wzp-client/src/call.rs
View File

@@ -656,8 +656,8 @@ impl CallDecoder {
},
CodecId::Codec2_1200 => QualityProfile::CATASTROPHIC,
CodecId::ComfortNoise => QualityProfile::GOOD,
CodecId::H264Baseline => {
panic!("H264Baseline is a video codec; audio decoder called with video profile")
CodecId::H264Baseline | CodecId::H265Main => {
panic!("video codec passed to audio decoder")
}
}
}

3
crates/wzp-codec/src/opus_enc.rs
View File

@@ -89,7 +89,8 @@ pub fn dred_duration_for(codec: CodecId) -> u8 {
CodecId::Codec2_1200
| CodecId::Codec2_3200
| CodecId::ComfortNoise
| CodecId::H264Baseline => 0,
| CodecId::H264Baseline
| CodecId::H265Main => 0,
}
}

36
crates/wzp-proto/src/codec_id.rs
View File

@@ -32,6 +32,8 @@ pub enum CodecId {
// 11 => H265 main
// 12 => AV1
// 13 => VP9
/// H.265 main profile (video).
H265Main = 11,
}
impl CodecId {
@@ -47,7 +49,7 @@ impl CodecId {
Self::Codec2_3200 => 3_200,
Self::Codec2_1200 => 1_200,
Self::ComfortNoise => 0,
Self::H264Baseline => 2_000_000,
Self::H264Baseline | Self::H265Main => 2_000_000,
}
}
@@ -59,7 +61,7 @@ impl CodecId {
Self::Codec2_3200 => 20,
Self::Codec2_1200 => 40,
Self::ComfortNoise => 20,
Self::H264Baseline => 33,
Self::H264Baseline | Self::H265Main => 33,
}
}
@@ -74,7 +76,7 @@ impl CodecId {
| Self::Opus64k => 48_000,
Self::Codec2_3200 | Self::Codec2_1200 => 8_000,
Self::ComfortNoise => 48_000,
Self::H264Baseline => 48_000,
Self::H264Baseline | Self::H265Main => 48_000,
}
}
@@ -91,6 +93,7 @@ impl CodecId {
7 => Some(Self::Opus48k),
8 => Some(Self::Opus64k),
9 => Some(Self::H264Baseline),
11 => Some(Self::H265Main),
_ => None,
}
}
@@ -102,7 +105,7 @@ impl CodecId {
/// Returns true if this is a video codec variant.
pub const fn is_video(self) -> bool {
matches!(self, Self::H264Baseline)
matches!(self, Self::H264Baseline | Self::H265Main)
}
/// Returns true if this is an Opus variant.
@@ -226,12 +229,33 @@ impl QualityProfile {
#[cfg(test)]
mod tests {
use super::CodecId;
use super::{CodecId, QualityProfile};
use crate::PriorityMode;
#[test]
fn codec_id_unknown_values_rejected() {
for v in 10u8..=255 {
for v in [10u8, 12, 13].iter().copied().chain(14u8..=255) {
assert!(CodecId::from_wire(v).is_none(), "v={v}");
}
}
#[test]
fn h265_main_roundtrips() {
assert_eq!(CodecId::H265Main.to_wire(), 11);
assert_eq!(CodecId::from_wire(11), Some(CodecId::H265Main));
assert!(CodecId::H265Main.is_video());
assert_eq!(CodecId::H265Main.bitrate_bps(), 2_000_000);
assert_eq!(CodecId::H265Main.frame_duration_ms(), 33);
}
#[test]
fn quality_profile_backward_compat_old_json() {
// Old JSON emitted before T5.1 has no priority_mode or video fields.
let old_json = r#"{"codec":"Opus24k","fec_ratio":0.2,"frame_duration_ms":20,"frames_per_block":5}"#;
let parsed: QualityProfile = serde_json::from_str(old_json).unwrap();
assert_eq!(parsed.priority_mode, PriorityMode::AudioFirst);
assert_eq!(parsed.video_bitrate_kbps, None);
assert_eq!(parsed.video_resolution, None);
assert_eq!(parsed.video_fps, None);
}
}

2
crates/wzp-relay/src/conformance.rs
View File

@@ -232,7 +232,7 @@ pub fn payload_size_bound(codec: CodecId) -> usize {
CodecId::Codec2_3200 => 30,
CodecId::Codec2_1200 => 30,
CodecId::ComfortNoise => 16,
CodecId::H264Baseline => 1400,
CodecId::H264Baseline | CodecId::H265Main => 1400,
}
}

7
crates/wzp-video/src/lib.rs
View File

@@ -1,8 +1,8 @@
//! WZP video pipeline — H.264 baseline framer and depacketizer.
//! WZP video pipeline — H.264 / H.265 framer and depacketizer.
//!
//! This crate lives alongside `wzp-codec` and handles video-specific
//! packetization (NAL fragmentation / reassembly). Platform encoders and
//! decoders land in T4.2/T4.3.
//! decoders land in T4.2/T4.3/T5.4.
pub mod controller;
pub mod decoder;
@@ -20,8 +20,9 @@ pub use depacketizer::H264Depacketizer;
pub use encoder::{VideoEncoder, VideoError, VideoFrame};
pub use encoder_mode::EncoderMode;
pub use framer::{FramedPacket, H264Framer};
pub use mediacodec::{MediaCodecDecoder, MediaCodecEncoder, MediaCodecHevcDecoder, MediaCodecHevcEncoder};
pub use nack::{CachedPacket, NackAction, NackReceiver, NackSender};
pub use videotoolbox::{VideoToolboxDecoder, VideoToolboxEncoder};
pub use videotoolbox::{VideoToolboxDecoder, VideoToolboxEncoder, VideoToolboxHevcDecoder, VideoToolboxHevcEncoder};
#[cfg(test)]
mod tests {

379
crates/wzp-video/src/mediacodec.rs
View File

@@ -1,4 +1,4 @@
//! Android MediaCodec H.264 encoder / decoder (Android only).
//! Android MediaCodec H.264 / H.265 encoder / decoder (Android only).
//!
//! On Android targets this uses the `ndk` crate's safe bindings around
//! `AMediaCodec`. On non-Android targets all methods return
@@ -350,6 +350,346 @@ impl VideoDecoder for MediaCodecDecoder {
}
}
// ============================================================================
// H.265 / HEVC
// ============================================================================
/// Android MediaCodec H.265 encoder.
///
/// On non-Android targets this is a compile-safe placeholder.
pub struct MediaCodecHevcEncoder {
#[cfg(target_os = "android")]
codec: MediaCodec,
#[cfg(target_os = "android")]
width: u32,
#[cfg(target_os = "android")]
height: u32,
force_keyframe: bool,
#[cfg(not(target_os = "android"))]
_width: u32,
#[cfg(not(target_os = "android"))]
_height: u32,
#[cfg(not(target_os = "android"))]
_bitrate_bps: u32,
}
impl MediaCodecHevcEncoder {
pub fn new(width: u32, height: u32, bitrate_bps: u32) -> Result<Self, VideoError> {
#[cfg(target_os = "android")]
{
let mut format = MediaFormat::new();
format.set_str("mime", "video/hevc");
format.set_i32("width", width as i32);
format.set_i32("height", height as i32);
format.set_i32("bitrate", bitrate_bps as i32);
format.set_i32("frame-rate", 30);
format.set_i32("i-frame-interval", 1);
format.set_i32("color-format", COLOR_FORMAT_YUV420_PLANAR);
let codec = MediaCodec::from_encoder_type("video/hevc").ok_or_else(|| {
VideoError::PlatformError("AMediaCodec_createEncoderByType (HEVC) failed".into())
})?;
codec
.configure(&format, None, MediaCodecDirection::Encoder)
.map_err(|e| VideoError::PlatformError(format!("configure failed: {e}")))?;
codec
.start()
.map_err(|e| VideoError::PlatformError(format!("start failed: {e}")))?;
Ok(Self {
codec,
width,
height,
force_keyframe: false,
})
}
#[cfg(not(target_os = "android"))]
{
let _ = (width, height, bitrate_bps);
Err(VideoError::NotInitialized)
}
}
}
impl VideoEncoder for MediaCodecHevcEncoder {
fn encode(&mut self, frame: &VideoFrame) -> Result<Vec<u8>, VideoError> {
#[cfg(target_os = "android")]
{
let y_size = (self.width * self.height) as usize;
let uv_size = y_size / 4;
let expected = y_size + uv_size * 2;
if frame.data.len() < expected {
return Err(VideoError::InvalidInput(format!(
"I420 frame too small: {} bytes, expected {expected}",
frame.data.len()
)));
}
let mut annex_b = self.drain_output()?;
match self
.codec
.dequeue_input_buffer(std::time::Duration::from_millis(10))
{
Ok(ndk::media::media_codec::DequeuedInputBufferResult::Buffer(buffer)) => {
let idx = buffer.index();
if let Some(input_buf) = self.codec.input_buffer(idx) {
let to_copy = frame.data.len().min(input_buf.len());
input_buf[..to_copy].copy_from_slice(&frame.data[..to_copy]);
let flags = if self.force_keyframe {
ndk_sys::AMEDIACODEC_BUFFER_FLAG_KEY_FRAME as u32
} else {
0
};
self.codec
.queue_input_buffer_by_index(
idx,
0,
to_copy,
frame.timestamp_ms as u64 * 1000,
flags,
)
.map_err(|e| {
VideoError::PlatformError(format!("queue_input_buffer failed: {e}"))
})?;
}
}
Ok(ndk::media::media_codec::DequeuedInputBufferResult::TryAgainLater) => {}
Err(e) => {
return Err(VideoError::PlatformError(format!(
"dequeue_input_buffer failed: {e}"
)));
}
}
annex_b.extend_from_slice(&self.drain_output()?);
Ok(annex_b)
}
#[cfg(not(target_os = "android"))]
{
let _ = frame;
Err(VideoError::NotInitialized)
}
}
fn request_keyframe(&mut self) {
self.force_keyframe = true;
}
fn is_keyframe(&self, packet: &[u8]) -> bool {
if packet.len() < 2 {
return false;
}
let nal_type = (packet[0] >> 1) & 0x3F;
nal_type == 19 || nal_type == 20
}
}
#[cfg(target_os = "android")]
impl MediaCodecHevcEncoder {
fn drain_output(&mut self) -> Result<Vec<u8>, VideoError> {
let mut output = Vec::new();
loop {
match self
.codec
.dequeue_output_buffer(std::time::Duration::from_millis(0))
{
Ok(ndk::media::media_codec::DequeuedOutputBufferInfoResult::Buffer(buffer)) => {
let idx = buffer.index();
if let Some(data) = self.codec.output_buffer(idx) {
let info = buffer.info();
let is_keyframe = (info.flags()
& (ndk_sys::AMEDIACODEC_BUFFER_FLAG_KEY_FRAME as u32))
!= 0;
if is_keyframe {
self.force_keyframe = false;
}
output.extend_from_slice(&avcc_to_annexb(data));
}
self.codec
.release_output_buffer_by_index(idx, false)
.map_err(|e| {
VideoError::PlatformError(format!("release_output_buffer failed: {e}"))
})?;
}
Ok(
ndk::media::media_codec::DequeuedOutputBufferInfoResult::OutputFormatChanged,
) => continue,
Ok(
ndk::media::media_codec::DequeuedOutputBufferInfoResult::OutputBuffersChanged,
) => continue,
Ok(ndk::media::media_codec::DequeuedOutputBufferInfoResult::TryAgainLater) => break,
Err(e) => {
return Err(VideoError::PlatformError(format!(
"dequeue_output_buffer failed: {e}"
)));
}
}
}
Ok(output)
}
}
/// Android MediaCodec H.265 decoder.
///
/// On non-Android targets this is a compile-safe placeholder.
pub struct MediaCodecHevcDecoder {
#[cfg(target_os = "android")]
codec: Option<MediaCodec>,
#[cfg(target_os = "android")]
width: u32,
#[cfg(target_os = "android")]
height: u32,
#[cfg(not(target_os = "android"))]
_width: u32,
#[cfg(not(target_os = "android"))]
_height: u32,
}
impl MediaCodecHevcDecoder {
pub fn new(width: u32, height: u32) -> Result<Self, VideoError> {
#[cfg(target_os = "android")]
{
Ok(Self {
codec: None,
width,
height,
})
}
#[cfg(not(target_os = "android"))]
{
let _ = (width, height);
Err(VideoError::NotInitialized)
}
}
}
impl VideoDecoder for MediaCodecHevcDecoder {
fn decode(&mut self, access_unit: &[u8]) -> Result<Option<VideoFrame>, VideoError> {
#[cfg(target_os = "android")]
{
if access_unit.is_empty() {
return Ok(None);
}
// Lazily create decoder when we see VPS/SPS/PPS.
if self.codec.is_none() {
let (vps, sps, pps) = extract_vps_sps_pps(access_unit);
let (vps, sps, pps) = match (vps, sps, pps) {
(Some(v), Some(s), Some(p)) => (v, s, p),
_ => return Ok(None),
};
let mut format = MediaFormat::new();
format.set_str("mime", "video/hevc");
format.set_i32("width", self.width as i32);
format.set_i32("height", self.height as i32);
format.set_buffer("csd-0", &vps);
format.set_buffer("csd-1", &sps);
format.set_buffer("csd-2", &pps);
let codec = MediaCodec::from_decoder_type("video/hevc").ok_or_else(|| {
VideoError::PlatformError("AMediaCodec_createDecoderByType (HEVC) failed".into())
})?;
codec
.configure(&format, None, MediaCodecDirection::Decoder)
.map_err(|e| {
VideoError::PlatformError(format!("decoder configure failed: {e}"))
})?;
codec
.start()
.map_err(|e| VideoError::PlatformError(format!("decoder start failed: {e}")))?;
self.codec = Some(codec);
}
let codec = self.codec.as_mut().ok_or(VideoError::NotInitialized)?;
match codec.dequeue_input_buffer(std::time::Duration::from_millis(10)) {
Ok(ndk::media::media_codec::DequeuedInputBufferResult::Buffer(buffer)) => {
let idx = buffer.index();
if let Some(input_buf) = codec.input_buffer(idx) {
let to_copy = access_unit.len().min(input_buf.len());
input_buf[..to_copy].copy_from_slice(&access_unit[..to_copy]);
codec
.queue_input_buffer_by_index(idx, 0, to_copy, 0, 0)
.map_err(|e| {
VideoError::PlatformError(format!(
"decoder queue_input_buffer failed: {e}"
))
})?;
}
}
Ok(ndk::media::media_codec::DequeuedInputBufferResult::TryAgainLater) => {}
Err(e) => {
return Err(VideoError::PlatformError(format!(
"decoder dequeue_input_buffer failed: {e}"
)));
}
}
match codec.dequeue_output_buffer(std::time::Duration::from_millis(10)) {
Ok(ndk::media::media_codec::DequeuedOutputBufferInfoResult::Buffer(buffer)) => {
let idx = buffer.index();
let data = codec.output_buffer(idx).unwrap_or(&[]).to_vec();
codec
.release_output_buffer_by_index(idx, false)
.map_err(|e| {
VideoError::PlatformError(format!(
"decoder release_output_buffer failed: {e}"
))
})?;
Ok(Some(VideoFrame {
width: self.width,
height: self.height,
data,
timestamp_ms: 0,
}))
}
Ok(_) => Ok(None),
Err(e) => Err(VideoError::PlatformError(format!(
"decoder dequeue_output_buffer failed: {e}"
))),
}
}
#[cfg(not(target_os = "android"))]
{
let _ = access_unit;
Err(VideoError::NotInitialized)
}
}
}
/// Parse an Annex-B access unit and return the first VPS, SPS and PPS found (HEVC).
#[allow(dead_code)]
fn extract_vps_sps_pps(annex_b: &[u8]) -> (Option<Vec<u8>>, Option<Vec<u8>>, Option<Vec<u8>>) {
let nals = split_annex_b(annex_b);
let mut vps = None;
let mut sps = None;
let mut pps = None;
for nal in nals {
if nal.len() < 2 {
continue;
}
let nal_type = (nal[0] >> 1) & 0x3F;
if nal_type == 32 && vps.is_none() {
vps = Some(nal.to_vec());
} else if nal_type == 33 && sps.is_none() {
sps = Some(nal.to_vec());
} else if nal_type == 34 && pps.is_none() {
pps = Some(nal.to_vec());
}
}
(vps, sps, pps)
}
/// Convert an AVCC blob (4-byte big-endian length prefixes) to Annex-B
/// (4-byte start codes `0x00 0x00 0x00 0x01`).
#[allow(dead_code)]
@@ -477,4 +817,41 @@ mod tests {
];
assert_eq!(annex_b, expected);
}
#[test]
fn hevc_mediacodec_encoder_returns_not_initialized_on_non_android() {
let enc = MediaCodecHevcEncoder::new(1280, 720, 2_000_000);
assert!(matches!(enc, Err(VideoError::NotInitialized)));
}
#[test]
fn hevc_mediacodec_decoder_returns_not_initialized_on_non_android() {
let dec = MediaCodecHevcDecoder::new(1280, 720);
assert!(matches!(dec, Err(VideoError::NotInitialized)));
}
#[test]
fn hevc_is_keyframe_detects_idr() {
let enc = MediaCodecHevcEncoder {
#[cfg(target_os = "android")]
codec: unreachable!(),
#[cfg(target_os = "android")]
width: 1280,
#[cfg(target_os = "android")]
height: 720,
force_keyframe: false,
#[cfg(not(target_os = "android"))]
_width: 1280,
#[cfg(not(target_os = "android"))]
_height: 720,
#[cfg(not(target_os = "android"))]
_bitrate_bps: 2_000_000,
};
// NAL type 19 (IDR_W_RADL): first byte = 0b0_010011_0 = 0x26
assert!(enc.is_keyframe(&[0x26, 0x01]));
// NAL type 20 (IDR_N_LP): first byte = 0b0_010100_0 = 0x28
assert!(enc.is_keyframe(&[0x28, 0x01]));
// NAL type 1 (TRAIL_R)
assert!(!enc.is_keyframe(&[0x02, 0x01]));
}
}

344
crates/wzp-video/src/videotoolbox.rs
View File

@@ -1,4 +1,4 @@
//! Apple VideoToolbox H.264 encoder / decoder (macOS only).
//! Apple VideoToolbox H.264 / H.265 encoder / decoder (macOS only).
use crate::decoder::VideoDecoder;
use crate::encoder::{VideoEncoder, VideoError, VideoFrame};
@@ -7,7 +7,8 @@ use crate::encoder::{VideoEncoder, VideoError, VideoFrame};
mod imp {
pub use shiguredo_video_toolbox::{
CodecConfig, DecodedFrame, Decoder, DecoderCodec, DecoderConfig, EncodeOptions, Encoder,
EncoderConfig, FrameData, H264EncoderConfig, H264EntropyMode, H264Profile, PixelFormat,
EncoderConfig, FrameData, H264EncoderConfig, H264EntropyMode, H264Profile, HevcEncoderConfig,
HevcProfile, PixelFormat,
};
}
@@ -384,6 +385,295 @@ impl VideoDecoder for VideoToolboxDecoder {
}
}
// ============================================================================
// H.265 / HEVC
// ============================================================================
/// macOS VideoToolbox H.265 encoder.
pub struct VideoToolboxHevcEncoder {
#[cfg(target_os = "macos")]
inner: Encoder,
force_keyframe: bool,
#[cfg(not(target_os = "macos"))]
_width: u32,
#[cfg(not(target_os = "macos"))]
_height: u32,
#[cfg(not(target_os = "macos"))]
_bitrate_bps: u32,
}
impl VideoToolboxHevcEncoder {
pub fn new(width: u32, height: u32, bitrate_bps: u32) -> Result<Self, VideoError> {
#[cfg(target_os = "macos")]
{
let config = EncoderConfig {
width,
height,
codec: CodecConfig::Hevc(HevcEncoderConfig {
profile: HevcProfile::Main,
allow_open_gop: false,
}),
pixel_format: PixelFormat::I420,
average_bitrate: Some(bitrate_bps as u64),
fps_numerator: 30,
fps_denominator: 1,
prioritize_encoding_speed_over_quality: true,
real_time: true,
maximize_power_efficiency: false,
allow_frame_reordering: false,
allow_temporal_compression: false,
max_key_frame_interval: std::num::NonZeroU32::new(30),
max_key_frame_interval_duration: None,
max_frame_delay_count: std::num::NonZeroU32::new(1),
};
let inner = Encoder::new(config).map_err(|e| {
VideoError::PlatformError(format!("VTCompressionSessionCreate (HEVC) failed: {e}"))
})?;
Ok(Self {
inner,
force_keyframe: false,
})
}
#[cfg(not(target_os = "macos"))]
{
let _ = (width, height, bitrate_bps);
Ok(Self {
_width: width,
_height: height,
_bitrate_bps: bitrate_bps,
force_keyframe: false,
})
}
}
}
impl VideoEncoder for VideoToolboxHevcEncoder {
fn encode(&mut self, frame: &VideoFrame) -> Result<Vec<u8>, VideoError> {
#[cfg(target_os = "macos")]
{
let width = frame.width as usize;
let height = frame.height as usize;
let y_size = width * height;
let uv_size = y_size / 4;
let expected = y_size + uv_size * 2;
if frame.data.len() < expected {
return Err(VideoError::InvalidInput(format!(
"I420 frame too small: {} bytes, expected {expected}",
frame.data.len()
)));
}
let y = &frame.data[0..y_size];
let u = &frame.data[y_size..y_size + uv_size];
let v = &frame.data[y_size + uv_size..y_size + uv_size * 2];
let frame_data = FrameData::I420 { y, u, v };
let options = EncodeOptions {
force_key_frame: self.force_keyframe,
};
self.inner
.encode(&frame_data, &options)
.map_err(|e| VideoError::PlatformError(format!("encode failed: {e}")))?;
let mut annex_b = Vec::new();
let mut emitted_keyframe = false;
while let Some(encoded) = self
.inner
.next_frame()
.map_err(|e| VideoError::PlatformError(format!("next_frame failed: {e}")))?
{
if encoded.keyframe {
emitted_keyframe = true;
}
for vps in &encoded.vps_list {
annex_b.extend_from_slice(&[0x00, 0x00, 0x00, 0x01]);
annex_b.extend_from_slice(vps);
}
for sps in &encoded.sps_list {
annex_b.extend_from_slice(&[0x00, 0x00, 0x00, 0x01]);
annex_b.extend_from_slice(sps);
}
for pps in &encoded.pps_list {
annex_b.extend_from_slice(&[0x00, 0x00, 0x00, 0x01]);
annex_b.extend_from_slice(pps);
}
annex_b.extend_from_slice(&avcc_to_annexb(&encoded.data));
}
if emitted_keyframe {
self.force_keyframe = false;
}
Ok(annex_b)
}
#[cfg(not(target_os = "macos"))]
{
let _ = frame;
Err(VideoError::NotInitialized)
}
}
fn request_keyframe(&mut self) {
self.force_keyframe = true;
}
fn is_keyframe(&self, packet: &[u8]) -> bool {
if packet.len() < 2 {
return false;
}
let nal_type = (packet[0] >> 1) & 0x3F;
// NAL type 19 = IDR_W_RADL, 20 = IDR_N_LP.
nal_type == 19 || nal_type == 20
}
}
/// macOS VideoToolbox H.265 decoder.
pub struct VideoToolboxHevcDecoder {
#[cfg(target_os = "macos")]
inner: Option<Decoder>,
#[cfg(target_os = "macos")]
width: u32,
#[cfg(target_os = "macos")]
height: u32,
#[cfg(not(target_os = "macos"))]
_width: u32,
#[cfg(not(target_os = "macos"))]
_height: u32,
}
impl VideoToolboxHevcDecoder {
pub fn new(width: u32, height: u32) -> Result<Self, VideoError> {
#[cfg(target_os = "macos")]
{
Ok(Self {
inner: None,
width,
height,
})
}
#[cfg(not(target_os = "macos"))]
{
let _ = (width, height);
Ok(Self {
_width: width,
_height: height,
})
}
}
#[cfg(target_os = "macos")]
fn ensure_decoder(&mut self, vps: &[u8], sps: &[u8], pps: &[u8]) -> Result<(), VideoError> {
let needs_create = self.inner.is_none();
let needs_update = if let Some(dec) = &mut self.inner {
let codec = DecoderCodec::Hevc {
vps,
sps,
pps,
nalu_len_bytes: 4,
};
dec.update_format(codec).is_err()
} else {
false
};
if needs_create || needs_update {
let config = DecoderConfig {
codec: DecoderCodec::Hevc {
vps,
sps,
pps,
nalu_len_bytes: 4,
},
pixel_format: PixelFormat::I420,
};
self.inner = Some(
Decoder::new(config)
.map_err(|e| VideoError::PlatformError(format!("decoder create: {e}")))?,
);
}
Ok(())
}
}
impl VideoDecoder for VideoToolboxHevcDecoder {
fn decode(&mut self, access_unit: &[u8]) -> Result<Option<VideoFrame>, VideoError> {
#[cfg(target_os = "macos")]
{
if access_unit.is_empty() {
return Ok(None);
}
let (vps, sps, pps) = extract_vps_sps_pps(access_unit);
if let (Some(v), Some(s), Some(p)) = (&vps, &sps, &pps) {
self.ensure_decoder(v, s, p)?;
}
let decoder = self.inner.as_mut().ok_or(VideoError::NotInitialized)?;
let avcc = annexb_to_avcc(access_unit);
if avcc.is_empty() {
return Ok(None);
}
let decoded = decoder
.decode(&avcc)
.map_err(|e| VideoError::PlatformError(format!("decode failed: {e}")))?;
match decoded {
Some(DecodedFrame::I420(frame)) => {
let y = frame.y_plane();
let u = frame.u_plane();
let v = frame.v_plane();
let mut data = Vec::with_capacity(y.len() + u.len() + v.len());
data.extend_from_slice(y);
data.extend_from_slice(u);
data.extend_from_slice(v);
Ok(Some(VideoFrame {
width: self.width,
height: self.height,
data,
timestamp_ms: 0,
}))
}
Some(DecodedFrame::Nv12(_)) => Err(VideoError::PlatformError(
"unexpected NV12 output from decoder".to_string(),
)),
None => Ok(None),
}
}
#[cfg(not(target_os = "macos"))]
{
let _ = access_unit;
Err(VideoError::NotInitialized)
}
}
}
/// Parse an Annex-B access unit and return the first VPS, SPS and PPS found (HEVC).
fn extract_vps_sps_pps(annex_b: &[u8]) -> (Option<Vec<u8>>, Option<Vec<u8>>, Option<Vec<u8>>) {
let nals = split_annex_b(annex_b);
let mut vps = None;
let mut sps = None;
let mut pps = None;
for nal in nals {
if nal.len() < 2 {
continue;
}
let nal_type = (nal[0] >> 1) & 0x3F;
if nal_type == 32 && vps.is_none() {
vps = Some(nal.to_vec());
} else if nal_type == 33 && sps.is_none() {
sps = Some(nal.to_vec());
} else if nal_type == 34 && pps.is_none() {
pps = Some(nal.to_vec());
}
}
(vps, sps, pps)
}
#[cfg(test)]
mod tests {
use super::*;
@@ -449,4 +739,54 @@ mod tests {
assert_eq!(sps, Some(vec![0x67, 0x42, 0xC0, 0x1E]));
assert_eq!(pps, Some(vec![0x68, 0xCE, 0x3C, 0x80]));
}
// ---- H.265 / HEVC ----
#[test]
fn hevc_encoder_instantiates() {
let enc = VideoToolboxHevcEncoder::new(1280, 720, 2_000_000);
assert!(enc.is_ok());
}
#[test]
fn hevc_decoder_instantiates() {
let dec = VideoToolboxHevcDecoder::new(1280, 720);
assert!(dec.is_ok());
}
#[test]
fn hevc_is_keyframe_detects_idr() {
let enc = VideoToolboxHevcEncoder::new(1280, 720, 2_000_000).unwrap();
// NAL type 19 (IDR_W_RADL): first byte = 0b0_010011_0 = 0x26
assert!(enc.is_keyframe(&[0x26, 0x01]));
// NAL type 20 (IDR_N_LP): first byte = 0b0_010100_0 = 0x28
assert!(enc.is_keyframe(&[0x28, 0x01]));
// NAL type 1 (TRAIL_R): first byte = 0b0_000001_0 = 0x02
assert!(!enc.is_keyframe(&[0x02, 0x01]));
}
#[test]
fn hevc_request_keyframe_sets_flag() {
let mut enc = VideoToolboxHevcEncoder::new(1280, 720, 2_000_000).unwrap();
assert!(!enc.force_keyframe);
enc.request_keyframe();
assert!(enc.force_keyframe);
}
#[test]
fn extract_vps_sps_pps_finds_hevc_params() {
// VPS (type 32): first byte = 0b0_100000_0 = 0x40
// SPS (type 33): first byte = 0b0_100001_0 = 0x42
// PPS (type 34): first byte = 0b0_100010_0 = 0x44
let au = vec![
0x00, 0x00, 0x00, 0x01, 0x40, 0x01, 0x0C, 0x01, // VPS
0x00, 0x00, 0x00, 0x01, 0x42, 0x01, 0x01, 0x01, // SPS
0x00, 0x00, 0x00, 0x01, 0x44, 0x01, 0xC1, 0x72, // PPS
0x00, 0x00, 0x00, 0x01, 0x26, 0x01, 0xAF, 0x09, // IDR
];
let (vps, sps, pps) = extract_vps_sps_pps(&au);
assert_eq!(vps, Some(vec![0x40, 0x01, 0x0C, 0x01]));
assert_eq!(sps, Some(vec![0x42, 0x01, 0x01, 0x01]));
assert_eq!(pps, Some(vec![0x44, 0x01, 0xC1, 0x72]));
}
}