From b197651557a36d9fef81ad3d5fe11a77325d18ba Mon Sep 17 00:00:00 2001 From: Siavash Sameni Date: Tue, 12 May 2026 14:50:20 +0400 Subject: [PATCH] =?UTF-8?q?T5.4:=20H.265=20encoder/decoder=20wrappers=20?= =?UTF-8?q?=E2=80=94=20VideoToolbox=20+=20MediaCodec,=20CodecId::H265Main?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/wzp-client/src/call.rs | 4 +- crates/wzp-codec/src/opus_enc.rs | 3 +- crates/wzp-proto/src/codec_id.rs | 36 ++- crates/wzp-relay/src/conformance.rs | 2 +- crates/wzp-video/src/lib.rs | 7 +- crates/wzp-video/src/mediacodec.rs | 379 ++++++++++++++++++++++++++- crates/wzp-video/src/videotoolbox.rs | 344 +++++++++++++++++++++++- 7 files changed, 759 insertions(+), 16 deletions(-) diff --git a/crates/wzp-client/src/call.rs b/crates/wzp-client/src/call.rs index 0472cc3..1245be5 100644 --- a/crates/wzp-client/src/call.rs +++ b/crates/wzp-client/src/call.rs @@ -656,8 +656,8 @@ impl CallDecoder { }, CodecId::Codec2_1200 => QualityProfile::CATASTROPHIC, CodecId::ComfortNoise => QualityProfile::GOOD, - CodecId::H264Baseline => { - panic!("H264Baseline is a video codec; audio decoder called with video profile") + CodecId::H264Baseline | CodecId::H265Main => { + panic!("video codec passed to audio decoder") } } } diff --git a/crates/wzp-codec/src/opus_enc.rs b/crates/wzp-codec/src/opus_enc.rs index 6cdfdbc..5c6e7db 100644 --- a/crates/wzp-codec/src/opus_enc.rs +++ b/crates/wzp-codec/src/opus_enc.rs @@ -89,7 +89,8 @@ pub fn dred_duration_for(codec: CodecId) -> u8 { CodecId::Codec2_1200 | CodecId::Codec2_3200 | CodecId::ComfortNoise - | CodecId::H264Baseline => 0, + | CodecId::H264Baseline + | CodecId::H265Main => 0, } } diff --git a/crates/wzp-proto/src/codec_id.rs b/crates/wzp-proto/src/codec_id.rs index 979a5c7..99e4a26 100644 --- a/crates/wzp-proto/src/codec_id.rs +++ b/crates/wzp-proto/src/codec_id.rs @@ -32,6 +32,8 @@ pub enum CodecId { // 11 => H265 main // 12 => AV1 // 13 => VP9 + /// H.265 main profile (video). + H265Main = 11, } impl CodecId { @@ -47,7 +49,7 @@ impl CodecId { Self::Codec2_3200 => 3_200, Self::Codec2_1200 => 1_200, Self::ComfortNoise => 0, - Self::H264Baseline => 2_000_000, + Self::H264Baseline | Self::H265Main => 2_000_000, } } @@ -59,7 +61,7 @@ impl CodecId { Self::Codec2_3200 => 20, Self::Codec2_1200 => 40, Self::ComfortNoise => 20, - Self::H264Baseline => 33, + Self::H264Baseline | Self::H265Main => 33, } } @@ -74,7 +76,7 @@ impl CodecId { | Self::Opus64k => 48_000, Self::Codec2_3200 | Self::Codec2_1200 => 8_000, Self::ComfortNoise => 48_000, - Self::H264Baseline => 48_000, + Self::H264Baseline | Self::H265Main => 48_000, } } @@ -91,6 +93,7 @@ impl CodecId { 7 => Some(Self::Opus48k), 8 => Some(Self::Opus64k), 9 => Some(Self::H264Baseline), + 11 => Some(Self::H265Main), _ => None, } } @@ -102,7 +105,7 @@ impl CodecId { /// Returns true if this is a video codec variant. pub const fn is_video(self) -> bool { - matches!(self, Self::H264Baseline) + matches!(self, Self::H264Baseline | Self::H265Main) } /// Returns true if this is an Opus variant. @@ -226,12 +229,33 @@ impl QualityProfile { #[cfg(test)] mod tests { - use super::CodecId; + use super::{CodecId, QualityProfile}; + use crate::PriorityMode; #[test] fn codec_id_unknown_values_rejected() { - for v in 10u8..=255 { + for v in [10u8, 12, 13].iter().copied().chain(14u8..=255) { assert!(CodecId::from_wire(v).is_none(), "v={v}"); } } + + #[test] + fn h265_main_roundtrips() { + assert_eq!(CodecId::H265Main.to_wire(), 11); + assert_eq!(CodecId::from_wire(11), Some(CodecId::H265Main)); + assert!(CodecId::H265Main.is_video()); + assert_eq!(CodecId::H265Main.bitrate_bps(), 2_000_000); + assert_eq!(CodecId::H265Main.frame_duration_ms(), 33); + } + + #[test] + fn quality_profile_backward_compat_old_json() { + // Old JSON emitted before T5.1 has no priority_mode or video fields. + let old_json = r#"{"codec":"Opus24k","fec_ratio":0.2,"frame_duration_ms":20,"frames_per_block":5}"#; + let parsed: QualityProfile = serde_json::from_str(old_json).unwrap(); + assert_eq!(parsed.priority_mode, PriorityMode::AudioFirst); + assert_eq!(parsed.video_bitrate_kbps, None); + assert_eq!(parsed.video_resolution, None); + assert_eq!(parsed.video_fps, None); + } } diff --git a/crates/wzp-relay/src/conformance.rs b/crates/wzp-relay/src/conformance.rs index 7ac3102..5741486 100644 --- a/crates/wzp-relay/src/conformance.rs +++ b/crates/wzp-relay/src/conformance.rs @@ -232,7 +232,7 @@ pub fn payload_size_bound(codec: CodecId) -> usize { CodecId::Codec2_3200 => 30, CodecId::Codec2_1200 => 30, CodecId::ComfortNoise => 16, - CodecId::H264Baseline => 1400, + CodecId::H264Baseline | CodecId::H265Main => 1400, } } diff --git a/crates/wzp-video/src/lib.rs b/crates/wzp-video/src/lib.rs index 6713d22..5f0f8d0 100644 --- a/crates/wzp-video/src/lib.rs +++ b/crates/wzp-video/src/lib.rs @@ -1,8 +1,8 @@ -//! WZP video pipeline — H.264 baseline framer and depacketizer. +//! WZP video pipeline — H.264 / H.265 framer and depacketizer. //! //! This crate lives alongside `wzp-codec` and handles video-specific //! packetization (NAL fragmentation / reassembly). Platform encoders and -//! decoders land in T4.2/T4.3. +//! decoders land in T4.2/T4.3/T5.4. pub mod controller; pub mod decoder; @@ -20,8 +20,9 @@ pub use depacketizer::H264Depacketizer; pub use encoder::{VideoEncoder, VideoError, VideoFrame}; pub use encoder_mode::EncoderMode; pub use framer::{FramedPacket, H264Framer}; +pub use mediacodec::{MediaCodecDecoder, MediaCodecEncoder, MediaCodecHevcDecoder, MediaCodecHevcEncoder}; pub use nack::{CachedPacket, NackAction, NackReceiver, NackSender}; -pub use videotoolbox::{VideoToolboxDecoder, VideoToolboxEncoder}; +pub use videotoolbox::{VideoToolboxDecoder, VideoToolboxEncoder, VideoToolboxHevcDecoder, VideoToolboxHevcEncoder}; #[cfg(test)] mod tests { diff --git a/crates/wzp-video/src/mediacodec.rs b/crates/wzp-video/src/mediacodec.rs index b294239..bd771b4 100644 --- a/crates/wzp-video/src/mediacodec.rs +++ b/crates/wzp-video/src/mediacodec.rs @@ -1,4 +1,4 @@ -//! Android MediaCodec H.264 encoder / decoder (Android only). +//! Android MediaCodec H.264 / H.265 encoder / decoder (Android only). //! //! On Android targets this uses the `ndk` crate's safe bindings around //! `AMediaCodec`. On non-Android targets all methods return @@ -350,6 +350,346 @@ impl VideoDecoder for MediaCodecDecoder { } } +// ============================================================================ +// H.265 / HEVC +// ============================================================================ + +/// Android MediaCodec H.265 encoder. +/// +/// On non-Android targets this is a compile-safe placeholder. +pub struct MediaCodecHevcEncoder { + #[cfg(target_os = "android")] + codec: MediaCodec, + #[cfg(target_os = "android")] + width: u32, + #[cfg(target_os = "android")] + height: u32, + force_keyframe: bool, + #[cfg(not(target_os = "android"))] + _width: u32, + #[cfg(not(target_os = "android"))] + _height: u32, + #[cfg(not(target_os = "android"))] + _bitrate_bps: u32, +} + +impl MediaCodecHevcEncoder { + pub fn new(width: u32, height: u32, bitrate_bps: u32) -> Result { + #[cfg(target_os = "android")] + { + let mut format = MediaFormat::new(); + format.set_str("mime", "video/hevc"); + format.set_i32("width", width as i32); + format.set_i32("height", height as i32); + format.set_i32("bitrate", bitrate_bps as i32); + format.set_i32("frame-rate", 30); + format.set_i32("i-frame-interval", 1); + format.set_i32("color-format", COLOR_FORMAT_YUV420_PLANAR); + + let codec = MediaCodec::from_encoder_type("video/hevc").ok_or_else(|| { + VideoError::PlatformError("AMediaCodec_createEncoderByType (HEVC) failed".into()) + })?; + + codec + .configure(&format, None, MediaCodecDirection::Encoder) + .map_err(|e| VideoError::PlatformError(format!("configure failed: {e}")))?; + + codec + .start() + .map_err(|e| VideoError::PlatformError(format!("start failed: {e}")))?; + + Ok(Self { + codec, + width, + height, + force_keyframe: false, + }) + } + #[cfg(not(target_os = "android"))] + { + let _ = (width, height, bitrate_bps); + Err(VideoError::NotInitialized) + } + } +} + +impl VideoEncoder for MediaCodecHevcEncoder { + fn encode(&mut self, frame: &VideoFrame) -> Result, VideoError> { + #[cfg(target_os = "android")] + { + let y_size = (self.width * self.height) as usize; + let uv_size = y_size / 4; + let expected = y_size + uv_size * 2; + if frame.data.len() < expected { + return Err(VideoError::InvalidInput(format!( + "I420 frame too small: {} bytes, expected {expected}", + frame.data.len() + ))); + } + + let mut annex_b = self.drain_output()?; + + match self + .codec + .dequeue_input_buffer(std::time::Duration::from_millis(10)) + { + Ok(ndk::media::media_codec::DequeuedInputBufferResult::Buffer(buffer)) => { + let idx = buffer.index(); + if let Some(input_buf) = self.codec.input_buffer(idx) { + let to_copy = frame.data.len().min(input_buf.len()); + input_buf[..to_copy].copy_from_slice(&frame.data[..to_copy]); + + let flags = if self.force_keyframe { + ndk_sys::AMEDIACODEC_BUFFER_FLAG_KEY_FRAME as u32 + } else { + 0 + }; + + self.codec + .queue_input_buffer_by_index( + idx, + 0, + to_copy, + frame.timestamp_ms as u64 * 1000, + flags, + ) + .map_err(|e| { + VideoError::PlatformError(format!("queue_input_buffer failed: {e}")) + })?; + } + } + Ok(ndk::media::media_codec::DequeuedInputBufferResult::TryAgainLater) => {} + Err(e) => { + return Err(VideoError::PlatformError(format!( + "dequeue_input_buffer failed: {e}" + ))); + } + } + + annex_b.extend_from_slice(&self.drain_output()?); + Ok(annex_b) + } + #[cfg(not(target_os = "android"))] + { + let _ = frame; + Err(VideoError::NotInitialized) + } + } + + fn request_keyframe(&mut self) { + self.force_keyframe = true; + } + + fn is_keyframe(&self, packet: &[u8]) -> bool { + if packet.len() < 2 { + return false; + } + let nal_type = (packet[0] >> 1) & 0x3F; + nal_type == 19 || nal_type == 20 + } +} + +#[cfg(target_os = "android")] +impl MediaCodecHevcEncoder { + fn drain_output(&mut self) -> Result, VideoError> { + let mut output = Vec::new(); + loop { + match self + .codec + .dequeue_output_buffer(std::time::Duration::from_millis(0)) + { + Ok(ndk::media::media_codec::DequeuedOutputBufferInfoResult::Buffer(buffer)) => { + let idx = buffer.index(); + if let Some(data) = self.codec.output_buffer(idx) { + let info = buffer.info(); + let is_keyframe = (info.flags() + & (ndk_sys::AMEDIACODEC_BUFFER_FLAG_KEY_FRAME as u32)) + != 0; + if is_keyframe { + self.force_keyframe = false; + } + output.extend_from_slice(&avcc_to_annexb(data)); + } + self.codec + .release_output_buffer_by_index(idx, false) + .map_err(|e| { + VideoError::PlatformError(format!("release_output_buffer failed: {e}")) + })?; + } + Ok( + ndk::media::media_codec::DequeuedOutputBufferInfoResult::OutputFormatChanged, + ) => continue, + Ok( + ndk::media::media_codec::DequeuedOutputBufferInfoResult::OutputBuffersChanged, + ) => continue, + Ok(ndk::media::media_codec::DequeuedOutputBufferInfoResult::TryAgainLater) => break, + Err(e) => { + return Err(VideoError::PlatformError(format!( + "dequeue_output_buffer failed: {e}" + ))); + } + } + } + Ok(output) + } +} + +/// Android MediaCodec H.265 decoder. +/// +/// On non-Android targets this is a compile-safe placeholder. +pub struct MediaCodecHevcDecoder { + #[cfg(target_os = "android")] + codec: Option, + #[cfg(target_os = "android")] + width: u32, + #[cfg(target_os = "android")] + height: u32, + #[cfg(not(target_os = "android"))] + _width: u32, + #[cfg(not(target_os = "android"))] + _height: u32, +} + +impl MediaCodecHevcDecoder { + pub fn new(width: u32, height: u32) -> Result { + #[cfg(target_os = "android")] + { + Ok(Self { + codec: None, + width, + height, + }) + } + #[cfg(not(target_os = "android"))] + { + let _ = (width, height); + Err(VideoError::NotInitialized) + } + } +} + +impl VideoDecoder for MediaCodecHevcDecoder { + fn decode(&mut self, access_unit: &[u8]) -> Result, VideoError> { + #[cfg(target_os = "android")] + { + if access_unit.is_empty() { + return Ok(None); + } + + // Lazily create decoder when we see VPS/SPS/PPS. + if self.codec.is_none() { + let (vps, sps, pps) = extract_vps_sps_pps(access_unit); + let (vps, sps, pps) = match (vps, sps, pps) { + (Some(v), Some(s), Some(p)) => (v, s, p), + _ => return Ok(None), + }; + + let mut format = MediaFormat::new(); + format.set_str("mime", "video/hevc"); + format.set_i32("width", self.width as i32); + format.set_i32("height", self.height as i32); + format.set_buffer("csd-0", &vps); + format.set_buffer("csd-1", &sps); + format.set_buffer("csd-2", &pps); + + let codec = MediaCodec::from_decoder_type("video/hevc").ok_or_else(|| { + VideoError::PlatformError("AMediaCodec_createDecoderByType (HEVC) failed".into()) + })?; + + codec + .configure(&format, None, MediaCodecDirection::Decoder) + .map_err(|e| { + VideoError::PlatformError(format!("decoder configure failed: {e}")) + })?; + + codec + .start() + .map_err(|e| VideoError::PlatformError(format!("decoder start failed: {e}")))?; + + self.codec = Some(codec); + } + + let codec = self.codec.as_mut().ok_or(VideoError::NotInitialized)?; + + match codec.dequeue_input_buffer(std::time::Duration::from_millis(10)) { + Ok(ndk::media::media_codec::DequeuedInputBufferResult::Buffer(buffer)) => { + let idx = buffer.index(); + if let Some(input_buf) = codec.input_buffer(idx) { + let to_copy = access_unit.len().min(input_buf.len()); + input_buf[..to_copy].copy_from_slice(&access_unit[..to_copy]); + codec + .queue_input_buffer_by_index(idx, 0, to_copy, 0, 0) + .map_err(|e| { + VideoError::PlatformError(format!( + "decoder queue_input_buffer failed: {e}" + )) + })?; + } + } + Ok(ndk::media::media_codec::DequeuedInputBufferResult::TryAgainLater) => {} + Err(e) => { + return Err(VideoError::PlatformError(format!( + "decoder dequeue_input_buffer failed: {e}" + ))); + } + } + + match codec.dequeue_output_buffer(std::time::Duration::from_millis(10)) { + Ok(ndk::media::media_codec::DequeuedOutputBufferInfoResult::Buffer(buffer)) => { + let idx = buffer.index(); + let data = codec.output_buffer(idx).unwrap_or(&[]).to_vec(); + codec + .release_output_buffer_by_index(idx, false) + .map_err(|e| { + VideoError::PlatformError(format!( + "decoder release_output_buffer failed: {e}" + )) + })?; + + Ok(Some(VideoFrame { + width: self.width, + height: self.height, + data, + timestamp_ms: 0, + })) + } + Ok(_) => Ok(None), + Err(e) => Err(VideoError::PlatformError(format!( + "decoder dequeue_output_buffer failed: {e}" + ))), + } + } + #[cfg(not(target_os = "android"))] + { + let _ = access_unit; + Err(VideoError::NotInitialized) + } + } +} + +/// Parse an Annex-B access unit and return the first VPS, SPS and PPS found (HEVC). +#[allow(dead_code)] +fn extract_vps_sps_pps(annex_b: &[u8]) -> (Option>, Option>, Option>) { + let nals = split_annex_b(annex_b); + let mut vps = None; + let mut sps = None; + let mut pps = None; + for nal in nals { + if nal.len() < 2 { + continue; + } + let nal_type = (nal[0] >> 1) & 0x3F; + if nal_type == 32 && vps.is_none() { + vps = Some(nal.to_vec()); + } else if nal_type == 33 && sps.is_none() { + sps = Some(nal.to_vec()); + } else if nal_type == 34 && pps.is_none() { + pps = Some(nal.to_vec()); + } + } + (vps, sps, pps) +} + /// Convert an AVCC blob (4-byte big-endian length prefixes) to Annex-B /// (4-byte start codes `0x00 0x00 0x00 0x01`). #[allow(dead_code)] @@ -477,4 +817,41 @@ mod tests { ]; assert_eq!(annex_b, expected); } + + #[test] + fn hevc_mediacodec_encoder_returns_not_initialized_on_non_android() { + let enc = MediaCodecHevcEncoder::new(1280, 720, 2_000_000); + assert!(matches!(enc, Err(VideoError::NotInitialized))); + } + + #[test] + fn hevc_mediacodec_decoder_returns_not_initialized_on_non_android() { + let dec = MediaCodecHevcDecoder::new(1280, 720); + assert!(matches!(dec, Err(VideoError::NotInitialized))); + } + + #[test] + fn hevc_is_keyframe_detects_idr() { + let enc = MediaCodecHevcEncoder { + #[cfg(target_os = "android")] + codec: unreachable!(), + #[cfg(target_os = "android")] + width: 1280, + #[cfg(target_os = "android")] + height: 720, + force_keyframe: false, + #[cfg(not(target_os = "android"))] + _width: 1280, + #[cfg(not(target_os = "android"))] + _height: 720, + #[cfg(not(target_os = "android"))] + _bitrate_bps: 2_000_000, + }; + // NAL type 19 (IDR_W_RADL): first byte = 0b0_010011_0 = 0x26 + assert!(enc.is_keyframe(&[0x26, 0x01])); + // NAL type 20 (IDR_N_LP): first byte = 0b0_010100_0 = 0x28 + assert!(enc.is_keyframe(&[0x28, 0x01])); + // NAL type 1 (TRAIL_R) + assert!(!enc.is_keyframe(&[0x02, 0x01])); + } } diff --git a/crates/wzp-video/src/videotoolbox.rs b/crates/wzp-video/src/videotoolbox.rs index d057dd5..7c0f572 100644 --- a/crates/wzp-video/src/videotoolbox.rs +++ b/crates/wzp-video/src/videotoolbox.rs @@ -1,4 +1,4 @@ -//! Apple VideoToolbox H.264 encoder / decoder (macOS only). +//! Apple VideoToolbox H.264 / H.265 encoder / decoder (macOS only). use crate::decoder::VideoDecoder; use crate::encoder::{VideoEncoder, VideoError, VideoFrame}; @@ -7,7 +7,8 @@ use crate::encoder::{VideoEncoder, VideoError, VideoFrame}; mod imp { pub use shiguredo_video_toolbox::{ CodecConfig, DecodedFrame, Decoder, DecoderCodec, DecoderConfig, EncodeOptions, Encoder, - EncoderConfig, FrameData, H264EncoderConfig, H264EntropyMode, H264Profile, PixelFormat, + EncoderConfig, FrameData, H264EncoderConfig, H264EntropyMode, H264Profile, HevcEncoderConfig, + HevcProfile, PixelFormat, }; } @@ -384,6 +385,295 @@ impl VideoDecoder for VideoToolboxDecoder { } } +// ============================================================================ +// H.265 / HEVC +// ============================================================================ + +/// macOS VideoToolbox H.265 encoder. +pub struct VideoToolboxHevcEncoder { + #[cfg(target_os = "macos")] + inner: Encoder, + force_keyframe: bool, + #[cfg(not(target_os = "macos"))] + _width: u32, + #[cfg(not(target_os = "macos"))] + _height: u32, + #[cfg(not(target_os = "macos"))] + _bitrate_bps: u32, +} + +impl VideoToolboxHevcEncoder { + pub fn new(width: u32, height: u32, bitrate_bps: u32) -> Result { + #[cfg(target_os = "macos")] + { + let config = EncoderConfig { + width, + height, + codec: CodecConfig::Hevc(HevcEncoderConfig { + profile: HevcProfile::Main, + allow_open_gop: false, + }), + pixel_format: PixelFormat::I420, + average_bitrate: Some(bitrate_bps as u64), + fps_numerator: 30, + fps_denominator: 1, + prioritize_encoding_speed_over_quality: true, + real_time: true, + maximize_power_efficiency: false, + allow_frame_reordering: false, + allow_temporal_compression: false, + max_key_frame_interval: std::num::NonZeroU32::new(30), + max_key_frame_interval_duration: None, + max_frame_delay_count: std::num::NonZeroU32::new(1), + }; + let inner = Encoder::new(config).map_err(|e| { + VideoError::PlatformError(format!("VTCompressionSessionCreate (HEVC) failed: {e}")) + })?; + Ok(Self { + inner, + force_keyframe: false, + }) + } + #[cfg(not(target_os = "macos"))] + { + let _ = (width, height, bitrate_bps); + Ok(Self { + _width: width, + _height: height, + _bitrate_bps: bitrate_bps, + force_keyframe: false, + }) + } + } +} + +impl VideoEncoder for VideoToolboxHevcEncoder { + fn encode(&mut self, frame: &VideoFrame) -> Result, VideoError> { + #[cfg(target_os = "macos")] + { + let width = frame.width as usize; + let height = frame.height as usize; + let y_size = width * height; + let uv_size = y_size / 4; + let expected = y_size + uv_size * 2; + + if frame.data.len() < expected { + return Err(VideoError::InvalidInput(format!( + "I420 frame too small: {} bytes, expected {expected}", + frame.data.len() + ))); + } + + let y = &frame.data[0..y_size]; + let u = &frame.data[y_size..y_size + uv_size]; + let v = &frame.data[y_size + uv_size..y_size + uv_size * 2]; + + let frame_data = FrameData::I420 { y, u, v }; + let options = EncodeOptions { + force_key_frame: self.force_keyframe, + }; + + self.inner + .encode(&frame_data, &options) + .map_err(|e| VideoError::PlatformError(format!("encode failed: {e}")))?; + + let mut annex_b = Vec::new(); + let mut emitted_keyframe = false; + while let Some(encoded) = self + .inner + .next_frame() + .map_err(|e| VideoError::PlatformError(format!("next_frame failed: {e}")))? + { + if encoded.keyframe { + emitted_keyframe = true; + } + for vps in &encoded.vps_list { + annex_b.extend_from_slice(&[0x00, 0x00, 0x00, 0x01]); + annex_b.extend_from_slice(vps); + } + for sps in &encoded.sps_list { + annex_b.extend_from_slice(&[0x00, 0x00, 0x00, 0x01]); + annex_b.extend_from_slice(sps); + } + for pps in &encoded.pps_list { + annex_b.extend_from_slice(&[0x00, 0x00, 0x00, 0x01]); + annex_b.extend_from_slice(pps); + } + annex_b.extend_from_slice(&avcc_to_annexb(&encoded.data)); + } + + if emitted_keyframe { + self.force_keyframe = false; + } + + Ok(annex_b) + } + #[cfg(not(target_os = "macos"))] + { + let _ = frame; + Err(VideoError::NotInitialized) + } + } + + fn request_keyframe(&mut self) { + self.force_keyframe = true; + } + + fn is_keyframe(&self, packet: &[u8]) -> bool { + if packet.len() < 2 { + return false; + } + let nal_type = (packet[0] >> 1) & 0x3F; + // NAL type 19 = IDR_W_RADL, 20 = IDR_N_LP. + nal_type == 19 || nal_type == 20 + } +} + +/// macOS VideoToolbox H.265 decoder. +pub struct VideoToolboxHevcDecoder { + #[cfg(target_os = "macos")] + inner: Option, + #[cfg(target_os = "macos")] + width: u32, + #[cfg(target_os = "macos")] + height: u32, + #[cfg(not(target_os = "macos"))] + _width: u32, + #[cfg(not(target_os = "macos"))] + _height: u32, +} + +impl VideoToolboxHevcDecoder { + pub fn new(width: u32, height: u32) -> Result { + #[cfg(target_os = "macos")] + { + Ok(Self { + inner: None, + width, + height, + }) + } + #[cfg(not(target_os = "macos"))] + { + let _ = (width, height); + Ok(Self { + _width: width, + _height: height, + }) + } + } + + #[cfg(target_os = "macos")] + fn ensure_decoder(&mut self, vps: &[u8], sps: &[u8], pps: &[u8]) -> Result<(), VideoError> { + let needs_create = self.inner.is_none(); + let needs_update = if let Some(dec) = &mut self.inner { + let codec = DecoderCodec::Hevc { + vps, + sps, + pps, + nalu_len_bytes: 4, + }; + dec.update_format(codec).is_err() + } else { + false + }; + + if needs_create || needs_update { + let config = DecoderConfig { + codec: DecoderCodec::Hevc { + vps, + sps, + pps, + nalu_len_bytes: 4, + }, + pixel_format: PixelFormat::I420, + }; + self.inner = Some( + Decoder::new(config) + .map_err(|e| VideoError::PlatformError(format!("decoder create: {e}")))?, + ); + } + Ok(()) + } +} + +impl VideoDecoder for VideoToolboxHevcDecoder { + fn decode(&mut self, access_unit: &[u8]) -> Result, VideoError> { + #[cfg(target_os = "macos")] + { + if access_unit.is_empty() { + return Ok(None); + } + + let (vps, sps, pps) = extract_vps_sps_pps(access_unit); + + if let (Some(v), Some(s), Some(p)) = (&vps, &sps, &pps) { + self.ensure_decoder(v, s, p)?; + } + + let decoder = self.inner.as_mut().ok_or(VideoError::NotInitialized)?; + + let avcc = annexb_to_avcc(access_unit); + if avcc.is_empty() { + return Ok(None); + } + + let decoded = decoder + .decode(&avcc) + .map_err(|e| VideoError::PlatformError(format!("decode failed: {e}")))?; + + match decoded { + Some(DecodedFrame::I420(frame)) => { + let y = frame.y_plane(); + let u = frame.u_plane(); + let v = frame.v_plane(); + let mut data = Vec::with_capacity(y.len() + u.len() + v.len()); + data.extend_from_slice(y); + data.extend_from_slice(u); + data.extend_from_slice(v); + Ok(Some(VideoFrame { + width: self.width, + height: self.height, + data, + timestamp_ms: 0, + })) + } + Some(DecodedFrame::Nv12(_)) => Err(VideoError::PlatformError( + "unexpected NV12 output from decoder".to_string(), + )), + None => Ok(None), + } + } + #[cfg(not(target_os = "macos"))] + { + let _ = access_unit; + Err(VideoError::NotInitialized) + } + } +} + +/// Parse an Annex-B access unit and return the first VPS, SPS and PPS found (HEVC). +fn extract_vps_sps_pps(annex_b: &[u8]) -> (Option>, Option>, Option>) { + let nals = split_annex_b(annex_b); + let mut vps = None; + let mut sps = None; + let mut pps = None; + for nal in nals { + if nal.len() < 2 { + continue; + } + let nal_type = (nal[0] >> 1) & 0x3F; + if nal_type == 32 && vps.is_none() { + vps = Some(nal.to_vec()); + } else if nal_type == 33 && sps.is_none() { + sps = Some(nal.to_vec()); + } else if nal_type == 34 && pps.is_none() { + pps = Some(nal.to_vec()); + } + } + (vps, sps, pps) +} + #[cfg(test)] mod tests { use super::*; @@ -449,4 +739,54 @@ mod tests { assert_eq!(sps, Some(vec![0x67, 0x42, 0xC0, 0x1E])); assert_eq!(pps, Some(vec![0x68, 0xCE, 0x3C, 0x80])); } + + // ---- H.265 / HEVC ---- + + #[test] + fn hevc_encoder_instantiates() { + let enc = VideoToolboxHevcEncoder::new(1280, 720, 2_000_000); + assert!(enc.is_ok()); + } + + #[test] + fn hevc_decoder_instantiates() { + let dec = VideoToolboxHevcDecoder::new(1280, 720); + assert!(dec.is_ok()); + } + + #[test] + fn hevc_is_keyframe_detects_idr() { + let enc = VideoToolboxHevcEncoder::new(1280, 720, 2_000_000).unwrap(); + // NAL type 19 (IDR_W_RADL): first byte = 0b0_010011_0 = 0x26 + assert!(enc.is_keyframe(&[0x26, 0x01])); + // NAL type 20 (IDR_N_LP): first byte = 0b0_010100_0 = 0x28 + assert!(enc.is_keyframe(&[0x28, 0x01])); + // NAL type 1 (TRAIL_R): first byte = 0b0_000001_0 = 0x02 + assert!(!enc.is_keyframe(&[0x02, 0x01])); + } + + #[test] + fn hevc_request_keyframe_sets_flag() { + let mut enc = VideoToolboxHevcEncoder::new(1280, 720, 2_000_000).unwrap(); + assert!(!enc.force_keyframe); + enc.request_keyframe(); + assert!(enc.force_keyframe); + } + + #[test] + fn extract_vps_sps_pps_finds_hevc_params() { + // VPS (type 32): first byte = 0b0_100000_0 = 0x40 + // SPS (type 33): first byte = 0b0_100001_0 = 0x42 + // PPS (type 34): first byte = 0b0_100010_0 = 0x44 + let au = vec![ + 0x00, 0x00, 0x00, 0x01, 0x40, 0x01, 0x0C, 0x01, // VPS + 0x00, 0x00, 0x00, 0x01, 0x42, 0x01, 0x01, 0x01, // SPS + 0x00, 0x00, 0x00, 0x01, 0x44, 0x01, 0xC1, 0x72, // PPS + 0x00, 0x00, 0x00, 0x01, 0x26, 0x01, 0xAF, 0x09, // IDR + ]; + let (vps, sps, pps) = extract_vps_sps_pps(&au); + assert_eq!(vps, Some(vec![0x40, 0x01, 0x0C, 0x01])); + assert_eq!(sps, Some(vec![0x42, 0x01, 0x01, 0x01])); + assert_eq!(pps, Some(vec![0x44, 0x01, 0xC1, 0x72])); + } }