| // Copyright 2023 The ChromiumOS Authors | |
| // Use of this source code is governed by a BSD-style license that can be | |
| // found in the LICENSE file. | |
| use std::os::fd::AsFd; | |
| use std::os::fd::BorrowedFd; | |
| use std::rc::Rc; | |
| use anyhow::anyhow; | |
| use crate::codec::av1::parser::FrameHeaderObu; | |
| use crate::codec::av1::parser::FrameObu; | |
| use crate::codec::av1::parser::FrameType; | |
| use crate::codec::av1::parser::ObuAction; | |
| use crate::codec::av1::parser::ObuType; | |
| use crate::codec::av1::parser::ParsedObu; | |
| use crate::codec::av1::parser::Parser; | |
| use crate::codec::av1::parser::SequenceHeaderObu; | |
| use crate::codec::av1::parser::TileGroupObu; | |
| use crate::codec::av1::parser::NUM_REF_FRAMES; | |
| use crate::decoder::stateless::DecodeError; | |
| use crate::decoder::stateless::DecodingState; | |
| use crate::decoder::stateless::NewPictureResult; | |
| use crate::decoder::stateless::StatelessBackendResult; | |
| use crate::decoder::stateless::StatelessCodec; | |
| use crate::decoder::stateless::StatelessDecoder; | |
| use crate::decoder::stateless::StatelessDecoderBackend; | |
| use crate::decoder::stateless::StatelessDecoderBackendPicture; | |
| use crate::decoder::stateless::StatelessVideoDecoder; | |
| use crate::decoder::stateless::TryFormat; | |
| use crate::decoder::BlockingMode; | |
| use crate::decoder::DecodedHandle; | |
| use crate::decoder::PoolLayer; | |
| use crate::Resolution; | |
| #[cfg(test)] | |
| mod dummy; | |
| #[cfg(feature = "vaapi")] | |
| mod vaapi; | |
| /// Stateless backend methods specific to AV1. | |
| pub trait StatelessAV1DecoderBackend: | |
| StatelessDecoderBackend + StatelessDecoderBackendPicture<Av1> | |
| { | |
| /// Called when a new Sequence Header OBU is parsed. The | |
| /// `highest_spatial_layer` argument refers to the maximum layer selected by | |
| /// the client through `set_operating_point()` and the scalability | |
| /// information present in the stream, if any. | |
| fn new_sequence( | |
| &mut self, | |
| sequence: &Rc<SequenceHeaderObu>, | |
| highest_spatial_layer: Option<u32>, | |
| ) -> StatelessBackendResult<()>; | |
| /// Called when the decoder determines that a new picture was found. The backend allocates all | |
| /// the resources it needs to process that picture. | |
| fn new_picture( | |
| &mut self, | |
| hdr: &FrameHeaderObu, | |
| timestamp: u64, | |
| highest_spatial_layer: Option<u32>, | |
| ) -> NewPictureResult<Self::Picture>; | |
| /// Called to set the global parameters of a picture. | |
| fn begin_picture( | |
| &mut self, | |
| picture: &mut Self::Picture, | |
| sequence: &SequenceHeaderObu, | |
| hdr: &FrameHeaderObu, | |
| reference_frames: &[Option<Self::Handle>; NUM_REF_FRAMES], | |
| ) -> StatelessBackendResult<()>; | |
| /// Called to dispatch a decode operation to the backend. | |
| #[allow(clippy::too_many_arguments)] | |
| fn decode_tile_group( | |
| &mut self, | |
| picture: &mut Self::Picture, | |
| tile_group: TileGroupObu, | |
| ) -> StatelessBackendResult<()>; | |
| /// Called when the decoder wants the backend to finish the decoding | |
| /// operations for `picture`. At this point, `decode_tile` has been called | |
| /// for all tiles. | |
| fn submit_picture(&mut self, picture: Self::Picture) -> StatelessBackendResult<Self::Handle>; | |
| } | |
| /// State of the picture being currently decoded. | |
| /// | |
| /// Stored between calls to [`StatelessDecoder::decode_tile_group`] that belong to the same | |
| /// picture. | |
| enum CurrentPicState<H: DecodedHandle, P> { | |
| /// A regular frame | |
| RegularFrame { | |
| /// Data for the current picture as extracted from the stream. | |
| header: FrameHeaderObu, | |
| /// Backend-specific data for that picture. | |
| backend_picture: P, | |
| }, | |
| /// A frame that has 'show_existing_frame' set. | |
| ShowExistingFrame { | |
| /// Data for the current picture as extracted from the stream. | |
| header: FrameHeaderObu, | |
| /// The handle of the reference frame that this frame points to. | |
| handle: H, | |
| }, | |
| } | |
| pub struct AV1DecoderState<H: DecodedHandle, P> { | |
| /// AV1 bitstream parser. | |
| parser: Parser, | |
| /// The reference frames in use. | |
| reference_frames: [Option<H>; NUM_REF_FRAMES], | |
| /// Keeps track of the last values seen for negotiation purposes. | |
| sequence: Option<Rc<SequenceHeaderObu>>, | |
| /// The picture currently being decoded. We need to preserve it between | |
| /// calls to `decode` because multiple tiles will be processed in different | |
| /// calls to `decode`. | |
| current_pic: Option<CurrentPicState<H, P>>, | |
| /// Keep track of the number of frames we've processed for logging purposes. | |
| frame_count: u32, | |
| /// For SVC streams, we only want to output the highest layer possible given | |
| /// the choice of operating point. | |
| highest_spatial_layer: Option<u32>, | |
| } | |
| impl<H, P> Default for AV1DecoderState<H, P> | |
| where | |
| H: DecodedHandle, | |
| { | |
| fn default() -> Self { | |
| Self { | |
| parser: Default::default(), | |
| reference_frames: Default::default(), | |
| sequence: Default::default(), | |
| current_pic: Default::default(), | |
| frame_count: Default::default(), | |
| highest_spatial_layer: Default::default(), | |
| } | |
| } | |
| } | |
| /// [`StatelessCodec`] structure to use in order to create a AV1 stateless decoder. | |
| /// | |
| /// # Accepted input | |
| /// | |
| /// the VP9 specification requires the last byte of the chunk to contain the superframe marker. | |
| /// Thus, a decoder using this codec processes exactly one encoded chunk per call to | |
| /// [`StatelessDecoder::decode`], and always returns the size of the passed input if successful. | |
| pub struct Av1; | |
| impl StatelessCodec for Av1 { | |
| type FormatInfo = Rc<SequenceHeaderObu>; | |
| type DecoderState<H: DecodedHandle, P> = AV1DecoderState<H, P>; | |
| } | |
| impl<B> StatelessDecoder<Av1, B> | |
| where | |
| B: StatelessAV1DecoderBackend, | |
| B::Handle: Clone, | |
| { | |
| fn decode_frame_header( | |
| &mut self, | |
| frame_header: FrameHeaderObu, | |
| timestamp: u64, | |
| ) -> Result<(), DecodeError> { | |
| log::debug!( | |
| "Processing frame {} with timestamp {}", | |
| self.codec.frame_count, | |
| timestamp | |
| ); | |
| if frame_header.show_existing_frame { | |
| let ref_frame = self.codec.reference_frames | |
| [frame_header.frame_to_show_map_idx as usize] | |
| .as_ref() | |
| .ok_or(anyhow!("Broken stream: no reference picture to display"))?; | |
| self.codec.current_pic = Some(CurrentPicState::ShowExistingFrame { | |
| header: frame_header, | |
| handle: ref_frame.clone(), | |
| }); | |
| } else if let Some(sequence) = &self.codec.sequence { | |
| let mut backend_picture = self.backend.new_picture( | |
| &frame_header, | |
| timestamp, | |
| self.codec.highest_spatial_layer, | |
| )?; | |
| self.backend.begin_picture( | |
| &mut backend_picture, | |
| sequence, | |
| &frame_header, | |
| &self.codec.reference_frames, | |
| )?; | |
| self.codec.current_pic = Some(CurrentPicState::RegularFrame { | |
| header: frame_header.clone(), | |
| backend_picture, | |
| }); | |
| } else { | |
| log::warn!("invalid stream: frame header received while no valid sequence ongoing"); | |
| } | |
| Ok(()) | |
| } | |
| fn decode_tile_group(&mut self, tile_group: TileGroupObu) -> anyhow::Result<()> { | |
| let picture = match self.codec.current_pic.as_mut() { | |
| Some(CurrentPicState::RegularFrame { | |
| backend_picture, .. | |
| }) => backend_picture, | |
| Some(CurrentPicState::ShowExistingFrame { .. }) => { | |
| return Err(anyhow!("Broken stream: cannot decode a tile group for a frame with show_existing_frame set")); | |
| } | |
| None => { | |
| return Err(anyhow!( | |
| "Broken stream: cannot decode a tile group without first decoding a frame header" | |
| )) | |
| } | |
| }; | |
| self.backend.decode_tile_group(picture, tile_group)?; | |
| Ok(()) | |
| } | |
| fn decode_frame(&mut self, frame: FrameObu, timestamp: u64) -> Result<(), DecodeError> { | |
| let FrameObu { header, tile_group } = frame; | |
| self.decode_frame_header(header, timestamp)?; | |
| self.decode_tile_group(tile_group)?; | |
| Ok(()) | |
| } | |
| fn submit_frame(&mut self, timestamp: u64) -> anyhow::Result<()> { | |
| log::debug!( | |
| "Finishing frame {} with timestamp: {}", | |
| self.codec.frame_count, | |
| timestamp | |
| ); | |
| let picture = self.codec.current_pic.take(); | |
| let (handle, header) = match picture { | |
| Some(CurrentPicState::RegularFrame { | |
| header, | |
| backend_picture, | |
| }) => { | |
| let handle = self.backend.submit_picture(backend_picture)?; | |
| if self.blocking_mode == BlockingMode::Blocking { | |
| handle.sync()?; | |
| } | |
| (handle, header) | |
| } | |
| Some(CurrentPicState::ShowExistingFrame { header, handle }) => (handle, header), | |
| None => return Err(anyhow!("Broken stream: no picture to submit")), | |
| }; | |
| let update_refs = if header.show_existing_frame { | |
| header.frame_type == FrameType::KeyFrame | |
| } else { | |
| true | |
| }; | |
| if update_refs { | |
| let mut refresh_frame_flags = header.refresh_frame_flags; | |
| #[allow(clippy::needless_range_loop)] | |
| for i in 0..NUM_REF_FRAMES { | |
| if (refresh_frame_flags & 1) == 1 { | |
| log::debug!( | |
| "Replacing reference frame {} to new timestamp {} on frame count: {}", | |
| i, | |
| timestamp, | |
| self.codec.frame_count | |
| ); | |
| self.codec.reference_frames[i] = Some(handle.clone()); | |
| } | |
| refresh_frame_flags >>= 1; | |
| } | |
| } | |
| let show_existing_frame = header.show_existing_frame; | |
| if header.show_frame || show_existing_frame { | |
| match self.codec.highest_spatial_layer { | |
| None => self.ready_queue.push(handle), | |
| Some(highest_spatial_layer) => { | |
| if header.obu_header.spatial_id >= highest_spatial_layer { | |
| self.ready_queue.push(handle); | |
| } else { | |
| log::debug!( | |
| "Dropping frame with spatial_id {}", | |
| header.obu_header.spatial_id | |
| ); | |
| } | |
| } | |
| } | |
| } | |
| self.codec | |
| .parser | |
| .ref_frame_update(&header) | |
| .map_err(|err| anyhow!(err))?; | |
| self.codec.frame_count += 1; | |
| Ok(()) | |
| } | |
| } | |
| impl<B> StatelessVideoDecoder for StatelessDecoder<Av1, B> | |
| where | |
| B: StatelessAV1DecoderBackend + TryFormat<Av1>, | |
| B::Handle: Clone + 'static, | |
| { | |
| type Handle = B::Handle; | |
| type FramePool = B::FramePool; | |
| /// Decode an AV1 stream. | |
| /// | |
| /// `bitstream` should initially be submitted as a whole temporal unit, however a call to this | |
| /// method will only consume a single OBU. The caller must be careful to check the return value | |
| /// and resubmit the remainder if the whole bitstream has not been consumed. | |
| fn decode(&mut self, timestamp: u64, bitstream: &[u8]) -> Result<usize, DecodeError> { | |
| let obu = match self | |
| .codec | |
| .parser | |
| .read_obu(bitstream) | |
| .map_err(|err| DecodeError::ParseFrameError(err))? | |
| { | |
| ObuAction::Process(obu) => obu, | |
| // This OBU should be dropped. | |
| ObuAction::Drop(length) => return Ok(length as usize), | |
| }; | |
| let obu_length = obu.bytes_used; | |
| let is_decode_op = matches!( | |
| obu.header.obu_type, | |
| ObuType::Frame | ObuType::FrameHeader | ObuType::TileGroup | |
| ); | |
| if is_decode_op { | |
| match self.decoding_state { | |
| /* we want to be here */ | |
| DecodingState::Decoding => (), | |
| /* otherwise... */ | |
| DecodingState::AwaitingStreamInfo => { | |
| /* Skip input until we get information from the stream. */ | |
| return Ok(obu_length); | |
| } | |
| /* Ask the client to confirm the format before we can process this. */ | |
| DecodingState::AwaitingFormat(_) => return Err(DecodeError::CheckEvents), | |
| DecodingState::Reset => { | |
| let mut parser = self.codec.parser.clone(); | |
| let is_key_frame = match obu.header.obu_type { | |
| ObuType::Frame | ObuType::FrameHeader => { | |
| let fh = parser | |
| .parse_frame_header_obu(&obu) | |
| .map_err(|err| DecodeError::ParseFrameError(err))?; | |
| fh.frame_type == FrameType::KeyFrame | |
| } | |
| _ => false, | |
| }; | |
| /* we can only resume from key frames */ | |
| if !is_key_frame { | |
| return Ok(obu_length); | |
| } else { | |
| self.decoding_state = DecodingState::Decoding; | |
| } | |
| } | |
| } | |
| } | |
| /* We are in `Decoding` state if we reached here */ | |
| match self | |
| .codec | |
| .parser | |
| .parse_obu(obu) | |
| .map_err(|err| DecodeError::ParseFrameError(err))? | |
| { | |
| ParsedObu::SequenceHeader(sequence) => { | |
| let sequence_differs = match &self.codec.sequence { | |
| Some(old_sequence) => **old_sequence != *sequence, | |
| None => true, | |
| }; | |
| if matches!(self.decoding_state, DecodingState::AwaitingStreamInfo) | |
| || sequence_differs | |
| { | |
| if self.codec.current_pic.is_some() { | |
| return Err(DecodeError::DecoderError(anyhow!( | |
| "broken stream: a picture is being decoded while a new sequence header is encountered" | |
| ))); | |
| } | |
| /* make sure we sync *before* we clear any state in the backend */ | |
| for f in &mut self.ready_queue.queue { | |
| /* TODO: this fixes av1-1-b8-03-sizeup on Intel | |
| * gen12, but we apparently do not do the same in | |
| * VP9. How is it that we do not get similar crashes there? | |
| * | |
| * TODO: syncing before calling new_sequence() in VP9 may fix some tests | |
| */ | |
| f.sync()?; | |
| } | |
| log::debug!( | |
| "found new sequence, resolution: {:?}, profile: {:?}, bit depth: {:?}", | |
| Resolution::from(( | |
| sequence.max_frame_width_minus_1 as u32 + 1, | |
| sequence.max_frame_height_minus_1 as u32 + 1 | |
| )), | |
| sequence.seq_profile, | |
| sequence.bit_depth | |
| ); | |
| /* there is nothing to drain, much like vp8 and vp9 */ | |
| self.codec.highest_spatial_layer = self.codec.parser.highest_operating_point(); | |
| self.backend | |
| .new_sequence(&sequence, self.codec.highest_spatial_layer)?; | |
| self.await_format_change(sequence); | |
| } | |
| } | |
| ParsedObu::FrameHeader(frame_header) => { | |
| if self.codec.current_pic.is_some() { | |
| /* submit this frame immediately, as we need to update the | |
| * DPB and the reference info state *before* processing the | |
| * next frame */ | |
| self.submit_frame(timestamp)?; | |
| } | |
| self.decode_frame_header(frame_header, timestamp)?; | |
| } | |
| ParsedObu::TileGroup(tile_group) => { | |
| self.decode_tile_group(tile_group)?; | |
| } | |
| ParsedObu::Frame(frame) => { | |
| if self.codec.current_pic.is_some() { | |
| /* submit this frame immediately, as we need to update the | |
| * DPB and the reference info state *before* processing the | |
| * next frame */ | |
| self.submit_frame(timestamp)?; | |
| } | |
| self.decode_frame(frame, timestamp)?; | |
| /* submit this frame immediately, as we need to update the | |
| * DPB and the reference info state *before* processing the | |
| * next frame */ | |
| self.submit_frame(timestamp)?; | |
| } | |
| ParsedObu::TileList => { | |
| return Err(DecodeError::DecoderError(anyhow!( | |
| "large tile scale mode is not supported" | |
| ))); | |
| } | |
| other => { | |
| log::debug!("skipping OBU of type {:?}", other.obu_type()); | |
| } | |
| } | |
| /* Submit the last frame if we have reached the end of the temporal unit. */ | |
| if bitstream.len() == obu_length && self.codec.current_pic.is_some() { | |
| self.submit_frame(timestamp)?; | |
| } | |
| Ok(obu_length) | |
| } | |
| fn flush(&mut self) -> Result<(), super::DecodeError> { | |
| // Note: all the submitted frames are already in the ready queue. | |
| self.codec.reference_frames = Default::default(); | |
| self.decoding_state = DecodingState::Reset; | |
| Ok(()) | |
| } | |
| fn frame_pool(&mut self, layer: PoolLayer) -> Vec<&mut B::FramePool> { | |
| self.backend.frame_pool(layer) | |
| } | |
| fn stream_info(&self) -> Option<&crate::decoder::StreamInfo> { | |
| self.backend.stream_info() | |
| } | |
| fn next_event(&mut self) -> Option<crate::decoder::DecoderEvent<B::Handle>> { | |
| self.query_next_event(|decoder, sequence| { | |
| decoder.codec.sequence = Some(Rc::clone(sequence)); | |
| }) | |
| } | |
| fn poll_fd(&self) -> BorrowedFd { | |
| self.epoll_fd.0.as_fd() | |
| } | |
| } | |
| #[cfg(test)] | |
| pub mod tests { | |
| use crate::bitstream_utils::IvfIterator; | |
| use crate::decoder::stateless::av1::Av1; | |
| use crate::decoder::stateless::tests::test_decode_stream; | |
| use crate::decoder::stateless::tests::TestStream; | |
| use crate::decoder::stateless::StatelessDecoder; | |
| use crate::decoder::BlockingMode; | |
| use crate::utils::simple_playback_loop; | |
| use crate::utils::simple_playback_loop_owned_frames; | |
| use crate::DecodedFormat; | |
| /// Run `test` using the dummy decoder, in both blocking and non-blocking modes. | |
| fn test_decoder_dummy(test: &TestStream, blocking_mode: BlockingMode) { | |
| let decoder = StatelessDecoder::<Av1, _>::new_dummy(blocking_mode).unwrap(); | |
| test_decode_stream( | |
| |d, s, f| { | |
| simple_playback_loop( | |
| d, | |
| IvfIterator::new(s), | |
| f, | |
| &mut simple_playback_loop_owned_frames, | |
| DecodedFormat::NV12, | |
| blocking_mode, | |
| ) | |
| }, | |
| decoder, | |
| test, | |
| false, | |
| false, | |
| ); | |
| } | |
| /// Same as Chromium's test-25fps.av1.ivf | |
| pub const DECODE_TEST_25FPS: TestStream = TestStream { | |
| stream: include_bytes!("../../codec/av1/test_data/test-25fps.ivf.av1"), | |
| crcs: include_str!("../../codec/av1/test_data/test-25fps.ivf.av1.crc"), | |
| }; | |
| #[test] | |
| fn test_25fps_block() { | |
| test_decoder_dummy(&DECODE_TEST_25FPS, BlockingMode::Blocking); | |
| } | |
| #[test] | |
| fn test_25fps_nonblock() { | |
| test_decoder_dummy(&DECODE_TEST_25FPS, BlockingMode::NonBlocking); | |
| } | |
| } |