From 58b5718cea3528d417b0f7471e9b98da65c8a8e6 Mon Sep 17 00:00:00 2001 From: sscobici Date: Mon, 14 Oct 2024 23:05:08 +0300 Subject: [PATCH] core, isomp4: collect many extra_data in VideoCodecParameters --- symphonia-core/src/codecs/video.rs | 61 +++++++--- symphonia-format-isomp4/src/atoms/avcc.rs | 27 +++-- symphonia-format-isomp4/src/atoms/esds.rs | 17 ++- symphonia-format-isomp4/src/atoms/hvcc.rs | 27 +++-- symphonia-format-isomp4/src/atoms/stsd.rs | 136 +++++++--------------- symphonia-format-mkv/src/codecs.rs | 28 +++-- 6 files changed, 149 insertions(+), 147 deletions(-) diff --git a/symphonia-core/src/codecs/video.rs b/symphonia-core/src/codecs/video.rs index ac869c52..578f2eb1 100644 --- a/symphonia-core/src/codecs/video.rs +++ b/symphonia-core/src/codecs/video.rs @@ -53,6 +53,29 @@ impl fmt::Display for VideoCodecId { } } +/// An `VideoExtraDataId` is a unique identifier used to identify a specific video extra data. +#[repr(transparent)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct VideoExtraDataId(u32); + +/// Null video extra data ID. +pub const VIDEO_EXTRA_DATA_ID_NULL: VideoExtraDataId = VideoExtraDataId(0x0); + +impl Default for VideoExtraDataId { + fn default() -> Self { + VIDEO_EXTRA_DATA_ID_NULL + } +} + +/// Extra data for a video codec. +#[derive(Clone, Debug, Default)] +pub struct VideoExtraData { + /// The extra data ID. + pub id: VideoExtraDataId, + /// Extra data (defined by codec) + pub data: Box<[u8]>, +} + /// Codec parameters for video codecs. #[derive(Clone, Debug, Default)] pub struct VideoCodecParameters { @@ -67,21 +90,10 @@ pub struct VideoCodecParameters { /// Video height. pub height: Option, /// Extra data (defined by the codec). - pub extra_data: Option>, + pub extra_data: Vec, } impl VideoCodecParameters { - pub fn new() -> VideoCodecParameters { - VideoCodecParameters { - codec: CODEC_ID_NULL_VIDEO, - profile: None, - level: None, - width: None, - height: None, - extra_data: None, - } - } - /// Provide the `VideoCodecId`. pub fn for_codec(&mut self, codec: VideoCodecId) -> &mut Self { self.codec = codec; @@ -112,9 +124,9 @@ impl VideoCodecParameters { self } - /// Provide codec extra data. - pub fn with_extra_data(&mut self, data: Box<[u8]>) -> &mut Self { - self.extra_data = Some(data); + /// Adds codec's extra data. + pub fn add_extra_data(&mut self, data: VideoExtraData) -> &mut Self { + self.extra_data.push(data); self } } @@ -379,4 +391,23 @@ pub mod well_known { /// VC-1 Advanced Profile pub const CODEC_PROFILE_VC1_ADVANCED: CodecProfile = CodecProfile(2); } + + pub mod extra_data { + use super::super::VideoExtraDataId; + + /// AVCDecoderConfigurationRecord + pub const VIDEO_EXTRA_DATA_ID_AVC_DECODER_CONFIG: VideoExtraDataId = VideoExtraDataId(1); + + /// HEVCDecoderConfigurationRecord + pub const VIDEO_EXTRA_DATA_ID_HEVC_DECODER_CONFIG: VideoExtraDataId = VideoExtraDataId(2); + + /// VP9DecoderConfiguration + pub const VIDEO_EXTRA_DATA_ID_VP9_DECODER_CONFIG: VideoExtraDataId = VideoExtraDataId(3); + + /// AV1DecoderConfiguration + pub const VIDEO_EXTRA_DATA_ID_AV1_DECODER_CONFIG: VideoExtraDataId = VideoExtraDataId(4); + + /// DolbyVisionConfiguration + pub const VIDEO_EXTRA_DATA_ID_DOLBY_VISION_CONFIG: VideoExtraDataId = VideoExtraDataId(5); + } } diff --git a/symphonia-format-isomp4/src/atoms/avcc.rs b/symphonia-format-isomp4/src/atoms/avcc.rs index 00e39070..d49aa901 100644 --- a/symphonia-format-isomp4/src/atoms/avcc.rs +++ b/symphonia-format-isomp4/src/atoms/avcc.rs @@ -6,19 +6,22 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use symphonia_common::mpeg::video::AVCDecoderConfigurationRecord; +use symphonia_core::codecs::video::well_known::extra_data::VIDEO_EXTRA_DATA_ID_AVC_DECODER_CONFIG; use symphonia_core::codecs::video::well_known::CODEC_ID_H264; -use symphonia_core::codecs::video::VideoCodecParameters; +use symphonia_core::codecs::video::VideoExtraData; use symphonia_core::codecs::CodecProfile; use symphonia_core::errors::{Error, Result}; use symphonia_core::io::ReadBytes; use crate::atoms::{Atom, AtomHeader}; +use super::stsd::VisualSampleEntry; + #[allow(dead_code)] #[derive(Debug)] pub struct AvcCAtom { /// AVC extra data (AVCDecoderConfigurationRecord). - extra_data: Box<[u8]>, + extra_data: VideoExtraData, profile: CodecProfile, level: u32, } @@ -31,20 +34,22 @@ impl Atom for AvcCAtom { .data_len() .ok_or_else(|| Error::DecodeError("isomp4 (avcC): expected atom size to be known"))?; - let extra_data = reader.read_boxed_slice_exact(len as usize)?; + let avc_data = VideoExtraData { + id: VIDEO_EXTRA_DATA_ID_AVC_DECODER_CONFIG, + data: reader.read_boxed_slice_exact(len as usize)?, + }; - let avc_config = AVCDecoderConfigurationRecord::read(&extra_data)?; + let avc_config = AVCDecoderConfigurationRecord::read(&avc_data.data)?; - Ok(Self { extra_data, profile: avc_config.profile, level: avc_config.level }) + Ok(Self { extra_data: avc_data, profile: avc_config.profile, level: avc_config.level }) } } impl AvcCAtom { - pub fn fill_codec_params(&self, codec_params: &mut VideoCodecParameters) { - codec_params - .for_codec(CODEC_ID_H264) - .with_profile(self.profile) - .with_level(self.level) - .with_extra_data(self.extra_data.clone()); + pub fn fill_video_sample_entry(&self, entry: &mut VisualSampleEntry) { + entry.codec_id = CODEC_ID_H264; + entry.profile = Some(self.profile); + entry.level = Some(self.level); + entry.extra_data.push(self.extra_data.clone()); } } diff --git a/symphonia-format-isomp4/src/atoms/esds.rs b/symphonia-format-isomp4/src/atoms/esds.rs index 13722221..4a9630ca 100644 --- a/symphonia-format-isomp4/src/atoms/esds.rs +++ b/symphonia-format-isomp4/src/atoms/esds.rs @@ -6,7 +6,7 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use symphonia_core::codecs::audio::AudioCodecParameters; -use symphonia_core::codecs::video::VideoCodecParameters; +use symphonia_core::codecs::video::{VideoExtraData, VIDEO_EXTRA_DATA_ID_NULL}; use symphonia_core::codecs::CodecId; use symphonia_core::errors::{decode_error, unsupported_error, Error, Result}; use symphonia_core::io::{FiniteStream, ReadBytes, ScopedStream}; @@ -15,6 +15,8 @@ use crate::atoms::{Atom, AtomHeader}; use log::{debug, warn}; +use super::stsd::VisualSampleEntry; + const ES_DESCRIPTOR: u8 = 0x03; const DECODER_CONFIG_DESCRIPTOR: u8 = 0x04; const DECODER_SPECIFIC_DESCRIPTOR: u8 = 0x05; @@ -109,14 +111,14 @@ impl EsdsAtom { } /// If the elementary stream descriptor describes an video stream, populate the provided - /// video codec parameters. - pub fn fill_video_codec_params(&self, codec_params: &mut VideoCodecParameters) -> Result<()> { + /// video sample entry. + pub fn fill_video_sample_entry(&self, entry: &mut VisualSampleEntry) -> Result<()> { use symphonia_core::codecs::video::CODEC_ID_NULL_VIDEO; match get_codec_id_from_object_type(self.descriptor.dec_config.object_type_indication) { Some(CodecId::Video(id)) => { // Object type indication identified an video codec. - codec_params.for_codec(id); + entry.codec_id = id; } Some(_) => { // Object type indication identified a non-video codec. This is unexpected. @@ -124,12 +126,15 @@ impl EsdsAtom { } None => { // Unknown object type indication. - codec_params.for_codec(CODEC_ID_NULL_VIDEO); + entry.codec_id = CODEC_ID_NULL_VIDEO; } } if let Some(ds_config) = &self.descriptor.dec_config.dec_specific_info { - codec_params.with_extra_data(ds_config.extra_data.clone()); + entry.extra_data.push(VideoExtraData { + id: VIDEO_EXTRA_DATA_ID_NULL, + data: ds_config.extra_data.clone(), + }); } Ok(()) diff --git a/symphonia-format-isomp4/src/atoms/hvcc.rs b/symphonia-format-isomp4/src/atoms/hvcc.rs index 048e01ac..c19a1a24 100644 --- a/symphonia-format-isomp4/src/atoms/hvcc.rs +++ b/symphonia-format-isomp4/src/atoms/hvcc.rs @@ -6,19 +6,22 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use symphonia_common::mpeg::video::HEVCDecoderConfigurationRecord; +use symphonia_core::codecs::video::well_known::extra_data::VIDEO_EXTRA_DATA_ID_HEVC_DECODER_CONFIG; use symphonia_core::codecs::video::well_known::CODEC_ID_HEVC; -use symphonia_core::codecs::video::VideoCodecParameters; +use symphonia_core::codecs::video::VideoExtraData; use symphonia_core::codecs::CodecProfile; use symphonia_core::errors::{Error, Result}; use symphonia_core::io::ReadBytes; use crate::atoms::{Atom, AtomHeader}; +use super::stsd::VisualSampleEntry; + #[allow(dead_code)] #[derive(Debug)] pub struct HvcCAtom { /// HEVC extra data (HEVCDecoderConfigurationRecord). - extra_data: Box<[u8]>, + extra_data: VideoExtraData, profile: CodecProfile, level: u32, } @@ -31,20 +34,22 @@ impl Atom for HvcCAtom { .data_len() .ok_or_else(|| Error::DecodeError("isomp4 (hvcC): expected atom size to be known"))?; - let extra_data = reader.read_boxed_slice_exact(len as usize)?; + let hevc_data = VideoExtraData { + id: VIDEO_EXTRA_DATA_ID_HEVC_DECODER_CONFIG, + data: reader.read_boxed_slice_exact(len as usize)?, + }; - let hevc_config = HEVCDecoderConfigurationRecord::read(&extra_data)?; + let hevc_config = HEVCDecoderConfigurationRecord::read(&hevc_data.data)?; - Ok(Self { extra_data, profile: hevc_config.profile, level: hevc_config.level }) + Ok(Self { extra_data: hevc_data, profile: hevc_config.profile, level: hevc_config.level }) } } impl HvcCAtom { - pub fn fill_codec_params(&self, codec_params: &mut VideoCodecParameters) { - codec_params - .for_codec(CODEC_ID_HEVC) - .with_profile(self.profile) - .with_level(self.level) - .with_extra_data(self.extra_data.clone()); + pub fn fill_video_sample_entry(&self, entry: &mut VisualSampleEntry) { + entry.codec_id = CODEC_ID_HEVC; + entry.profile = Some(self.profile); + entry.level = Some(self.level); + entry.extra_data.push(self.extra_data.clone()); } } diff --git a/symphonia-format-isomp4/src/atoms/stsd.rs b/symphonia-format-isomp4/src/atoms/stsd.rs index 1bb04d49..d52f458c 100644 --- a/symphonia-format-isomp4/src/atoms/stsd.rs +++ b/symphonia-format-isomp4/src/atoms/stsd.rs @@ -22,8 +22,8 @@ use symphonia_core::codecs::audio::well_known::{CODEC_ID_PCM_U32BE, CODEC_ID_PCM use symphonia_core::codecs::audio::{AudioCodecId, AudioCodecParameters, CODEC_ID_NULL_AUDIO}; use symphonia_core::codecs::subtitle::well_known::CODEC_ID_MOV_TEXT; use symphonia_core::codecs::subtitle::SubtitleCodecParameters; -use symphonia_core::codecs::video::VideoCodecParameters; -use symphonia_core::codecs::CodecParameters; +use symphonia_core::codecs::video::{VideoCodecId, VideoCodecParameters, VideoExtraData}; +use symphonia_core::codecs::{CodecParameters, CodecProfile}; use symphonia_core::errors::{decode_error, unsupported_error, Result}; use symphonia_core::io::ReadBytes; @@ -157,26 +157,19 @@ impl StsdAtom { Some(CodecParameters::Audio(codec_params)) } SampleEntry::Visual(entry) => { - let mut codec_params = VideoCodecParameters::new(); - - codec_params.with_width(entry.width).with_height(entry.height); - - match entry.codec_specific { - Some(VisualCodecSpecific::Esds(ref esds)) => { - // ESDS is not a video specific atom. Returns an error if not an video - // elementary stream. - esds.fill_video_codec_params(&mut codec_params).ok()?; - } - Some(VisualCodecSpecific::Avc1(ref avc)) => { - avc.fill_codec_params(&mut codec_params); - } - Some(VisualCodecSpecific::Dvh1(ref hevc)) - | Some(VisualCodecSpecific::Dvhe(ref hevc)) - | Some(VisualCodecSpecific::Hev1(ref hevc)) - | Some(VisualCodecSpecific::Hvc1(ref hevc)) => { - hevc.fill_codec_params(&mut codec_params); - } - _ => (), + let mut codec_params = VideoCodecParameters { + width: Some(entry.width), + height: Some(entry.height), + codec: entry.codec_id, + extra_data: entry.extra_data.clone(), + ..Default::default() + }; + + if let Some(profile) = entry.profile { + codec_params.with_profile(profile); + } + if let Some(level) = entry.level { + codec_params.with_level(level); } Some(CodecParameters::Video(codec_params)) @@ -591,7 +584,7 @@ fn read_audio_sample_entry( /// Visual sample entry. #[allow(dead_code)] -#[derive(Debug)] +#[derive(Debug, Default)] pub struct VisualSampleEntry { pub width: u16, pub height: u16, @@ -600,22 +593,10 @@ pub struct VisualSampleEntry { /// Frame count per sample. pub frame_count: u16, pub compressor: Option, - pub codec_specific: Option, -} - -#[allow(dead_code)] -#[derive(Debug)] -pub enum VisualCodecSpecific { - Esds(EsdsAtom), - Av1, - Avc1(AvcCAtom), - Dvhe(HvcCAtom), - Dvh1(HvcCAtom), - Hev1(HvcCAtom), - Hvc1(HvcCAtom), - Mp4v, - Vp8, - Vp9, + pub codec_id: VideoCodecId, + pub profile: Option, + pub level: Option, + pub extra_data: Vec, } fn read_visual_sample_entry( @@ -635,17 +616,20 @@ fn read_visual_sample_entry( // Reserved. reader.ignore_bytes(16)?; - let width = reader.read_be_u16()?; - let height = reader.read_be_u16()?; - let horiz_res = f64::from(FpU16::parse_raw(reader.read_be_u32()?)); - let vert_res = f64::from(FpU16::parse_raw(reader.read_be_u32()?)); + let mut entry = VisualSampleEntry { + width: reader.read_be_u16()?, + height: reader.read_be_u16()?, + horiz_res: f64::from(FpU16::parse_raw(reader.read_be_u32()?)), + vert_res: f64::from(FpU16::parse_raw(reader.read_be_u32()?)), + ..Default::default() + }; // Reserved. let _ = reader.read_be_u32()?; - let frame_count = reader.read_be_u16()?; + entry.frame_count = reader.read_be_u16()?; - let compressor = { + entry.compressor = { let len = usize::from(reader.read_u8()?); let mut name = [0u8; 31]; @@ -658,57 +642,23 @@ fn read_visual_sample_entry( }; let _depth = reader.read_be_u16()?; - - // Reserved. - reader.read_be_u16()?; + let _color_table_id = reader.read_be_u16()?; let mut iter = AtomIterator::new(reader, header); - let mut codec_specific = None; - while let Some(entry_header) = iter.next()? { match entry_header.atom_type { + AtomType::Esds => { + let atom = iter.read_atom::()?; + atom.fill_video_sample_entry(&mut entry)?; + } AtomType::AvcConfiguration => { - // AVC - if header.atom_type != AtomType::VisualSampleEntryAvc1 || codec_specific.is_some() { - return decode_error("isomp4: invalid avc configuration sample entry"); - } - - codec_specific = Some(VisualCodecSpecific::Avc1(iter.read_atom::()?)); + let atom = iter.read_atom::()?; + atom.fill_video_sample_entry(&mut entry); } AtomType::HevcConfiguration => { - // HEVC - match header.atom_type { - AtomType::VisualSampleEntryDvhe => { - if codec_specific.is_some() { - return decode_error("isomp4: invalid dvhe configuration sample entry"); - } - codec_specific = - Some(VisualCodecSpecific::Dvhe(iter.read_atom::()?)); - } - AtomType::VisualSampleEntryDvh1 => { - if codec_specific.is_some() { - return decode_error("isomp4: invalid dvh1 configuration sample entry"); - } - codec_specific = - Some(VisualCodecSpecific::Dvh1(iter.read_atom::()?)); - } - AtomType::VisualSampleEntryHev1 => { - if codec_specific.is_some() { - return decode_error("isomp4: invalid hev1 configuration sample entry"); - } - codec_specific = - Some(VisualCodecSpecific::Hev1(iter.read_atom::()?)); - } - AtomType::VisualSampleEntryHvc1 => { - if codec_specific.is_some() { - return decode_error("isomp4: invalid hvc1 configuration sample entry"); - } - codec_specific = - Some(VisualCodecSpecific::Hvc1(iter.read_atom::()?)); - } - _ => {} - } + let atom = iter.read_atom::()?; + atom.fill_video_sample_entry(&mut entry); } _ => { debug!("unknown visual sample entry sub-atom: {:?}.", entry_header.atom_type()); @@ -716,15 +666,7 @@ fn read_visual_sample_entry( } } - Ok(SampleEntry::Visual(VisualSampleEntry { - width, - height, - horiz_res, - vert_res, - frame_count, - compressor, - codec_specific, - })) + Ok(SampleEntry::Visual(entry)) } #[derive(Debug)] diff --git a/symphonia-format-mkv/src/codecs.rs b/symphonia-format-mkv/src/codecs.rs index 67241c57..ee9a4144 100644 --- a/symphonia-format-mkv/src/codecs.rs +++ b/symphonia-format-mkv/src/codecs.rs @@ -5,6 +5,10 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. +use extra_data::{ + VIDEO_EXTRA_DATA_ID_AV1_DECODER_CONFIG, VIDEO_EXTRA_DATA_ID_AVC_DECODER_CONFIG, + VIDEO_EXTRA_DATA_ID_HEVC_DECODER_CONFIG, VIDEO_EXTRA_DATA_ID_VP9_DECODER_CONFIG, +}; use log::warn; use symphonia_common::mpeg::video::{ @@ -15,7 +19,9 @@ use symphonia_core::codecs::audio::well_known::{CODEC_ID_FLAC, CODEC_ID_VORBIS}; use symphonia_core::codecs::audio::AudioCodecParameters; use symphonia_core::codecs::audio::{well_known::*, AudioCodecId}; use symphonia_core::codecs::subtitle::{well_known::*, SubtitleCodecId, SubtitleCodecParameters}; -use symphonia_core::codecs::video::{well_known::*, VideoCodecId, VideoCodecParameters}; +use symphonia_core::codecs::video::{ + well_known::*, VideoCodecId, VideoCodecParameters, VideoExtraData, VIDEO_EXTRA_DATA_ID_NULL, +}; use symphonia_core::codecs::{CodecId, CodecParameters, CodecProfile}; use symphonia_core::errors::{decode_error, Error, Result}; @@ -106,9 +112,12 @@ fn make_video_codec_params( } }; - let mut codec_params = VideoCodecParameters::new(); - - codec_params.for_codec(id); + let mut codec_params = VideoCodecParameters { + codec: id, + width: Some(video.pixel_width), + height: Some(video.pixel_height), + ..Default::default() + }; if let Some(profile) = profile { codec_params.with_profile(profile); @@ -118,10 +127,15 @@ fn make_video_codec_params( codec_params.with_level(level); } - codec_params.with_width(video.pixel_width).with_height(video.pixel_height); - if let Some(codec_private) = track.codec_private { - codec_params.with_extra_data(codec_private); + let extra_data_id = match id { + CODEC_ID_H264 => VIDEO_EXTRA_DATA_ID_AVC_DECODER_CONFIG, + CODEC_ID_HEVC => VIDEO_EXTRA_DATA_ID_HEVC_DECODER_CONFIG, + CODEC_ID_VP9 => VIDEO_EXTRA_DATA_ID_VP9_DECODER_CONFIG, + CODEC_ID_AV1 => VIDEO_EXTRA_DATA_ID_AV1_DECODER_CONFIG, + _ => VIDEO_EXTRA_DATA_ID_NULL, + }; + codec_params.add_extra_data(VideoExtraData { id: extra_data_id, data: codec_private }); } Ok(Some(CodecParameters::Video(codec_params)))