//! Module for creating and controlling V4L2 decoders. //! //! Decoders are created using [`v4l2r_decoder_new`] and remain //! active until being given to [`v4l2r_decoder_destroy`]. They expect //! to be fed encoded buffers in the format specified at creation time using //! [`v4l2r_decoder_decode`]. //! //! Decoders communicate with the client using an event callback that is invoked //! on a dedicated thread. This callback signals events of interest, like a //! frame being decoded, or a change in the output format (due to e.g. a dynamic //! resolution change). The output format is initially undefined and a format //! change event will be produced before any frame can be decoded. #![allow(non_camel_case_types)] use log::{debug, error, info, warn}; use nix::sys::time::{TimeVal, TimeValLike}; use std::{ ffi::CStr, mem::MaybeUninit, os::raw::{c_char, c_int, c_uint, c_void}, path::Path, sync::Arc, }; use v4l2r::{ bindings, decoder::{ stateful::{Decoder, Decoding, DrainError}, CompletedInputBuffer, DecoderEvent, DecoderEventCallback, FormatChangedCallback, FormatChangedReply, InputDoneCallback, }, device::queue::{direction::Capture, dqbuf::DqBuffer, qbuf::OutputQueueable, FormatBuilder}, memory::DmaBufHandle, PixelFormat, PlaneLayout, Rect, }; use crate::memory::{ v4l2r_video_frame, v4l2r_video_frame_provider, v4l2r_video_frame_provider_queue_frame, DmaBufFd, VideoFrameMemoryType, }; type DynCbDecoder = Decoder< Decoding< Vec>, Arc, Box>>>, Box>>, Box>>, >, >; /// A V4L2 decoder instance. pub struct v4l2r_decoder { decoder: DynCbDecoder, // Reference to the video frame provider for our callbacks. provider: Option>, // Keep the size of input buffers at hand. input_buf_size: u64, } /// Callback called when the decoder is done with a buffer submitted using /// [`v4l2r_decoder_decode`]. /// /// The first argument is the `cb_data` pointer given /// to [`v4l2r_decoder_new`]. The second argument is the dequeued V4L2 buffer. /// The client can use the `timestamp.tv_sec` member of `buffer` to match this /// buffer with the `bitstream_id` parameter of [`v4l2r_decoder_decode`] and /// understand which buffer has just completed. /// /// This callback is only called during calls to [`v4l2r_decoder_decode`] and /// [`v4l2r_decoder_kick`]. pub type v4l2r_decoder_input_done_cb = extern "C" fn(*mut c_void, *const bindings::v4l2_buffer); #[repr(C)] pub struct v4l2r_decoder_frame_decoded_event { /// Dequeued V4L2 buffer that has produced the frame. Useful to check for /// flags and errors. buffer: *const bindings::v4l2_buffer, /// One of the frames previously made available to the decoder using /// [`v4l2r_video_frame_provider_queue_frame`]. /// /// [`v4l2r_video_frame_provider_queue_frame`]: /// crate::memory::v4l2r_video_frame_provider_queue_frame frame: v4l2r_video_frame, } /// Event produced every time the output format of the stream changes. /// This includes when the initial format is determined by the decoder, and any /// subsequent dynamic resolution change in the stream. #[repr(C)] pub struct v4l2r_decoder_format_changed_event { /// New format for decoded frames produced after this event. new_format: *mut bindings::v4l2_format, /// Visible rectangle for decoded frames produced after this event. visible_rect: bindings::v4l2_rect, /// Pointer to the video frame provider the client must use to provide /// frames to decode into. /// /// When the client receives this event, it must stop using the previous /// video frame provider (if any) as soon as possible and destroy it using /// `v4l2r_video_frame_provider_drop`. Any video frame still queued to an /// old provider and that has not been seen in a previous `FrameDecoded` /// event can be considered as returned to the client. Upon receiving this /// event, the client is guaranteed to not receive any frame in the previous /// format, or from the previous provider. /// /// The client is responsible for allocating video frames in the new format /// and start giving them to the new provider using /// [`v4l2r_video_frame_provider_queue_frame`]. /// /// [`v4l2r_video_frame_provider_queue_frame`]: /// crate::memory::v4l2r_video_frame_provider_queue_frame new_provider: *const v4l2r_video_frame_provider, /// Minimum number of output buffers required by the decoder to operate /// properly. /// /// The client must allocate at least `min_num_frames` (but no more than /// 32), otherwise the decoder might starve. min_num_frames: c_uint, } /// Decoding-related events. These events can be produced at any time between /// calls to [`v4l2r_decoder_new`] and [`v4l2r_decoder_destroy`] and /// are passed to the events callback. #[repr(C)] pub enum v4l2r_decoder_event { // TODO for frames that have a zero-size, just recycle the handles // on-the-spot and pass the relevant event instead! FrameDecoded(v4l2r_decoder_frame_decoded_event), FormatChanged(v4l2r_decoder_format_changed_event), EndOfStream, } /// Events callback. This callback is guaranteed to always be called from the /// same thread, i.e. events are completely sequential. pub type v4l2r_decoder_event_cb = extern "C" fn(*mut c_void, *mut v4l2r_decoder_event); fn set_capture_format_cb( f: FormatBuilder, desired_pixel_format: Option, visible_rect: Rect, min_num_buffers: usize, decoder: *mut v4l2r_decoder, event_cb: v4l2r_decoder_event_cb, cb_data: *mut c_void, ) -> anyhow::Result>> { // Safe unless the C part did something funny with the decoder returned by // `v4l2r_decoder_new`. let decoder = unsafe { decoder.as_mut().unwrap() }; let mut v4l2_format: bindings::v4l2_format = match desired_pixel_format { Some(format) => f.set_pixelformat(format).apply()?, None => f.apply()?, }; // Create new memory provider on the heap and update our internal pointer. let new_provider = Arc::new(v4l2r_video_frame_provider::new()); // Reference for our own callbacks. decoder.provider = Some(Arc::clone(&new_provider)); // Reference owned by the client. Will be dropped when it calls // `v4l2r_video_frame_provider_drop`. let provider_client_ref = Arc::clone(&new_provider); // TODO check return value. event_cb( cb_data, &mut v4l2r_decoder_event::FormatChanged(v4l2r_decoder_format_changed_event { new_format: &mut v4l2_format, visible_rect: visible_rect.into(), new_provider: Arc::into_raw(provider_client_ref), min_num_frames: min_num_buffers as c_uint, }), ); Ok(FormatChangedReply { provider: new_provider, // TODO: can't the provider report the memory type that it is // actually serving itself? mem_type: VideoFrameMemoryType, // Since we are using DMABUF, always allocate the maximum number of // V4L2 buffers (32) since they are virtually free. This gives more // flexibility for the client as to how many frames it can allocate. num_buffers: bindings::VIDEO_MAX_FRAME as usize, }) } fn frame_decoded_cb( decoder: &mut v4l2r_decoder, mut dqbuf: DqBuffer, event_cb: v4l2r_decoder_event_cb, cb_data: *mut c_void, ) { let frame = dqbuf.take_handles().unwrap(); debug!( "Video frame {} ({}) decoded from V4L2 buffer {} (flags: {:?})", frame.id, dqbuf.data.timestamp().tv_sec, dqbuf.data.index(), dqbuf.data.flags(), ); let mut v4l2_data = dqbuf.data.clone(); // Drop the DQBuffer early so the C callback can reuse the V4L2 // buffer if it needs to. drop(dqbuf); // Immediately recycle empty frames. We will pass the corresponding // event to the client. if *v4l2_data.get_first_plane().bytesused == 0 { debug!( "Immediately recycling zero-sized frame {} {}", frame.id, v4l2_data.is_last() ); // Should be safe as `provider` is initialized in the format // change callback and is thus valid, as well as `frame`. match &decoder.provider { Some(provider) => unsafe { v4l2r_video_frame_provider_queue_frame(provider.as_ref(), frame); }, None => { error!("Frame decoded callback called while no provider set!"); } } } else { // TODO check return value? event_cb( cb_data, &mut v4l2r_decoder_event::FrameDecoded(v4l2r_decoder_frame_decoded_event { buffer: v4l2_data.as_mut_ptr() as *const _, frame, }), ); } } // A void pointer that can be sent across threads. This is usually not allowed // by Rust, but is necessary for us to call back into the V4L2RustDecoder. struct SendablePtr(*mut T); impl Clone for SendablePtr { fn clone(&self) -> Self { *self } } impl Copy for SendablePtr {} unsafe impl Send for SendablePtr {} unsafe impl Sync for SendablePtr {} #[allow(clippy::too_many_arguments)] fn v4l2r_decoder_new_safe( path: &Path, input_format_fourcc: u32, num_input_buffers: usize, input_buffer_size: usize, output_format_fourcc: u32, input_done_cb: v4l2r_decoder_input_done_cb, event_cb: v4l2r_decoder_event_cb, cb_data: *mut c_void, ) -> *mut v4l2r_decoder { let decoder = match Decoder::open(path) { Ok(decoder) => decoder, Err(e) => { error!("failed to open decoder {}: {:#?}", path.display(), e); return std::ptr::null_mut(); } }; info!( "Opened decoder {} with format {}, {} input buffers of size {}", path.display(), v4l2r::PixelFormat::from(input_format_fourcc), num_input_buffers, input_buffer_size ); let format_builder = |f: FormatBuilder| { let pixel_format = input_format_fourcc.into(); let format = match f .set_pixelformat(pixel_format) .set_planes_layout(vec![PlaneLayout { sizeimage: input_buffer_size as u32, ..Default::default() }]) .apply::() { Ok(format) if format.pixelformat == pixel_format => format, Ok(_) => { return Err(anyhow::anyhow!( "Unrecognized OUTPUT format {:?}", pixel_format )) } Err(e) => return Err(e.into()), }; debug!( "Decoder requires input buffer size of: {}", format.plane_fmt[0].sizeimage ); Ok(()) }; let decoder = match decoder.set_output_format(format_builder) { Ok(decoder) => decoder, Err(e) => { error!("Error while setting output format: {}", e); return std::ptr::null_mut(); } }; let output_format = match output_format_fourcc { 0 => None, fourcc => Some(PixelFormat::from(fourcc)), }; let cb_data = SendablePtr(cb_data); let decoder = match decoder.allocate_output_buffers::>>(num_input_buffers) { Ok(decoder) => decoder, Err(e) => { error!("Error while allocating OUTPUT buffers: {}", e); return std::ptr::null_mut(); } }; // Reserve memory on the heap for our decoder and take a pointer that we // can use in our callbacks. let mut decoder_box = Box::new(MaybeUninit::::uninit()); let decoder_ptr = SendablePtr(decoder_box.as_mut_ptr()); let event_handler = move |event: DecoderEvent>| { // Make Rust 2021 happy. let decoder_ptr = decoder_ptr; let cb_data = cb_data; let decoder = unsafe { decoder_ptr.0.as_mut().unwrap() }; match event { DecoderEvent::FrameDecoded(dqbuf) => { frame_decoded_cb(decoder, dqbuf, event_cb, cb_data.0) } DecoderEvent::EndOfStream => event_cb(cb_data.0, &mut v4l2r_decoder_event::EndOfStream), }; }; let res = decoder.start( Box::new( move |buf: CompletedInputBuffer>>| { match buf { CompletedInputBuffer::Dequeued(mut dqbuf) => { debug!("Input buffer {} done", dqbuf.data.index()); // TODO check return value? input_done_cb(cb_data.0, dqbuf.data.as_mut_ptr() as *const _); } // Just drop canceled buffers for now - the client will remove // them on its side as well. // TODO add a status parameter to the callback and invoke it? // that way the client does not need to clear its own list... CompletedInputBuffer::Canceled(_) => (), } }, ) as Box>>>, Box::new(event_handler) as Box>>, Box::new( move |f: FormatBuilder, visible_rect: Rect, min_num_buffers: usize| -> anyhow::Result>> { // Make Rust 2021 happy. let decoder_ptr = decoder_ptr; let cb_data = cb_data; set_capture_format_cb( f, output_format, visible_rect, min_num_buffers, decoder_ptr.0, event_cb, cb_data.0, ) }, ) as Box>>, ); let decoder = match res { Ok(decoder) => decoder, Err(e) => { error!("Cannot start decoder: {}", e); return std::ptr::null_mut(); } }; let input_format: v4l2r::Format = decoder.get_output_format().unwrap(); let decoder = v4l2r_decoder { decoder, provider: None, input_buf_size: input_format.plane_fmt[0].sizeimage as u64, }; let decoder_box = unsafe { // Replace our uninitialized heap memory with our valid decoder. decoder_box.as_mut_ptr().write(decoder); // Convert the Box> into Box // now that we know the decoder is properly initialized. It would be // better to use Box::assume_init but as of rustc 1.50 this method is // still in nightly only. Box::from_raw(Box::into_raw(decoder_box) as *mut v4l2r_decoder) }; info!("Decoder {:p}: successfully started", decoder_box.as_ref()); Box::into_raw(decoder_box) } fn v4l2r_decoder_decode_safe( decoder: &mut v4l2r_decoder, bitstream_id: i32, fd: c_int, bytes_used: usize, ) -> c_int { let v4l2_buffer = match decoder.decoder.get_buffer() { Ok(buffer) => buffer, Err(e) => { error!("Error obtaining V4L2 buffer: {}", e); return -1; } }; let v4l2_buffer_id = v4l2_buffer.index(); match v4l2_buffer .set_timestamp(TimeVal::seconds(bitstream_id as i64)) .queue_with_handles( vec![DmaBufHandle::from(DmaBufFd::new( fd, decoder.input_buf_size, ))], &[bytes_used], ) { Ok(()) => (), Err(e) => { error!("Error while queueing buffer: {}", e); return -1; } }; v4l2_buffer_id as c_int } /// Create a new decoder for a given encoded format. /// /// * `path` is the path to the V4L2 device that will be used for decoding. /// * `input_format_fourcc` is the FOURCC code of the encoded format we will /// decode, e.g. "H264" or "VP80". /// * `num_input_buffers` is the number of input buffers we wish to use. It /// should correspond to the number of different buffers containing input data /// that will be given to this decoder. /// * `input_buffer_size` is the desired size of input buffers. The decoder may /// adjust this value, so the client should call /// [`v4l2r_decoder_get_input_format`] to confirm the actual expected value. /// * `output_format_fourcc` is the FOURCC code of the desired pixel format for /// output frames (e.g. "NV12"). It can also be 0, in which case the decoder /// will use whichever pixel format is active by default. /// * `input_done_cb` is a pointer to a callback function to be called whenever /// an encoded input buffer is done being processed. This callback is /// guaranteed to be invoked during calls to [`v4l2r_decoder_decode`] or /// [`v4l2r_decoder_kick`], i.e. it will always be called in the current /// thread. /// * `event_cb` is a pointer to a function to be called for handling the /// various events produced by the decoder. See [`v4l2r_decoder_event`] for /// more details on events. This callback is guaranteed to be called from a /// separate, unique thread, therefore the events can be assumed to be /// sequential (i.e. two events cannot be produced at the same time from two /// different threads). /// * `cb_data` is a pointer that will always be passed as the first parameter /// of the `input_done_cb` and `events_cb`. /// /// # Safety /// The passed `path` must be a valid, zero-terminated C string containining the /// path to the device. Expect a crash if passing an invalid string. #[no_mangle] pub unsafe extern "C" fn v4l2r_decoder_new( path: *const c_char, input_format_fourcc: u32, num_input_buffers: usize, input_buffer_size: usize, output_format_fourcc: u32, input_done_cb: v4l2r_decoder_input_done_cb, event_cb: v4l2r_decoder_event_cb, cb_data: *mut c_void, ) -> *mut v4l2r_decoder { let cstr = CStr::from_ptr(path); let rstr = cstr.to_str().unwrap(); let path = Path::new(&rstr); v4l2r_decoder_new_safe( path, input_format_fourcc, num_input_buffers, input_buffer_size, output_format_fourcc, input_done_cb, event_cb, cb_data, ) } /// Stop and destroy a decoder. /// /// Stop `decoder` and destroy it. This function DOES take ownership of /// `decoder`, which must absolutely not be used after this call. /// /// It is guaranteed that none of the callbacks passed to [`v4l2r_decoder_new`] /// will be called after this function has returned. /// /// # Safety /// /// `decoder` must be a valid pointer to a decoder returned by /// `v4l2r_decoder_new`. Passing a NULL or invalid pointer will cause a crash. /// `decoder` must not be used again after this function is called. #[no_mangle] pub unsafe extern "C" fn v4l2r_decoder_destroy(decoder: *mut v4l2r_decoder) { info!("Decoder {:p}: destroying", decoder); if decoder.is_null() { warn!("Trying to destroy a NULL decoder"); return; } let decoder = Box::from_raw(decoder); match decoder.decoder.stop() { Ok(_) => (), Err(e) => error!("Error while stopping decoder: {}", e), } } /// Obtain the current input format (i.e. the format set on the *OUTPUT* queue). /// /// Obtain the current input format for `decoder` and write it into `format`. /// This function can be called at any time since a decoder always have a valid /// input format. /// /// Returns 0 in case of success, -1 if an error occured, in which case `format` /// is not overwritten. /// /// # Safety /// /// `decoder` must be a valid pointer to a decoder instance. `format` must point /// to valid memory that can receive a `v4l2_format` #[no_mangle] pub unsafe extern "C" fn v4l2r_decoder_get_input_format( decoder: *const v4l2r_decoder, format: *mut bindings::v4l2_format, ) -> c_int { assert!(!decoder.is_null()); assert!(!format.is_null()); let decoder = &*decoder; let format = &mut *format; *format = match decoder.decoder.get_output_format() { Ok(format) => format, Err(e) => { error!("Error while getting output format: {}", e); return -1; } }; 0 } /// Decode the encoded data referenced by `fd`. /// /// The decoder does NOT take ownership of `fd` and won't close it. /// /// `bitstream_id` is the identifier of this input buffer. The produced frames /// will carry this identifier in they timestamp. /// /// `bytes_used` is amount of encoded data within that buffer. /// /// The value returned is the index of the V4L2 buffer `fd` has been queued with. /// It can be used to know when `fd` is done being decoded as a `v4l2_buffer` of /// the same index will be passed as argument to the *input done callback* when /// this is the case. /// /// In case of error, -1 is returned. /// /// # Safety /// /// `decoder` must be a valid pointer to a decoder returned by /// [`v4l2r_decoder_new`]. Passing a NULL or invalid pointer will cause a crash. /// `fd` is expected to be a valid DMABUF FD backed by enough memory for the /// expected input buffer size. Failure to provide a valid FD will return in an /// ioctl error (but no crash). #[no_mangle] pub unsafe extern "C" fn v4l2r_decoder_decode( decoder: *mut v4l2r_decoder, bitstream_id: i32, fd: c_int, bytes_used: usize, ) -> c_int { debug!( "Decoder {:p}: decoding bitstream id {}", decoder, bitstream_id ); assert!(!decoder.is_null()); let decoder = &mut *decoder; v4l2r_decoder_decode_safe(decoder, bitstream_id, fd, bytes_used) } /// Kick the decoder and see if some input buffers fall as a result. /// /// No, really. Completed input buffers are typically checked when calling /// [`v4l2r_decoder_decode`] (which is also the time when the input done /// callback is invoked), but this mechanism is not foolproof: if the client /// works with a limited set of input buffers and queues them all before an /// output frame can be produced, then the client has no more material to call /// [`v4l2r_decoder_decode`] with and thus no input buffer will ever be /// dequeued, resulting in the decoder being blocked. /// /// This method mitigates this problem by adding a way to check for completed /// input buffers and calling the input done callback without the need for new /// encoded content. It is suggested to call it from the same thread that /// invokes [`v4l2r_decoder_decode`] every time we get a /// [`v4l2r_decoder_frame_decoded_event`]. That way the client can recycle its /// input buffers and the decoding process does not get stuck. /// /// # Safety /// /// `decoder` must be a valid pointer to a decoder returned by /// [`v4l2r_decoder_new`]. Passing a NULL or invalid pointer will cause a crash. #[no_mangle] pub unsafe extern "C" fn v4l2r_decoder_kick(decoder: *const v4l2r_decoder) { assert!(!decoder.is_null()); let decoder = &*decoder; match decoder.decoder.kick() { Ok(()) => (), Err(e) => { error!("Error while kicking decoder: {}", e); } } } /// Possible responses for the [`v4l2r_decoder_drain`] commmand. #[repr(C)] #[allow(clippy::upper_case_acronyms)] pub enum v4l2r_decoder_drain_response { /// The drain has already completed as [`v4l2r_decoder_drain`] returned. DRAIN_COMPLETED, /// The drain has started but will be completed when we receive a /// [`v4l2r_decoder_event::EndOfStream`] event. DRAIN_STARTED, /// Drain cannot be done at the moment because not enough input buffers /// have been processed to know the output format. TRY_AGAIN, /// An error has occurred. ERROR, } /// # Safety /// /// `decoder` must be a valid pointer to a decoder returned by /// [`v4l2r_decoder_new`]. Passing a NULL or invalid pointer will cause a crash. #[no_mangle] pub unsafe extern "C" fn v4l2r_decoder_drain( decoder: *const v4l2r_decoder, blocking: bool, ) -> v4l2r_decoder_drain_response { assert!(!decoder.is_null()); let decoder = &*decoder; match decoder.decoder.drain(blocking) { Ok(true) => v4l2r_decoder_drain_response::DRAIN_COMPLETED, Ok(false) => v4l2r_decoder_drain_response::DRAIN_STARTED, Err(DrainError::TryAgain) => v4l2r_decoder_drain_response::TRY_AGAIN, Err(e) => { error!("Error while draining decoder: {}", e); v4l2r_decoder_drain_response::ERROR } } } /// # Safety /// /// `decoder` must be a valid pointer to a decoder returned by /// [`v4l2r_decoder_new`]. Passing a NULL or invalid pointer will cause a crash. #[no_mangle] pub unsafe extern "C" fn v4l2r_decoder_flush(decoder: *const v4l2r_decoder) { assert!(!decoder.is_null()); let decoder = &*decoder; match decoder.decoder.flush() { Ok(()) => (), Err(e) => { error!("Error while flushing decoder: {:#?}", e); } } }