@@ -0,0 +1,1023 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2025-2026 Red Hat Inc.
+ * Author: Philipp Stanner <pstanner@redhat.com>
+ */
+
+//! DMA Fence support.
+//!
+//! Reference: <https://docs.kernel.org/driver-api/dma-buf.html#c.dma_fence>
+//!
+//! header: [`include/linux/dma-fence.h`](srctree/include/linux/dma-fence.h)
+
+use crate::{
+ alloc::AllocError,
+ bindings,
+ container_of,
+ error::to_result,
+ prelude::*,
+ types::ForeignOwnable,
+ types::Opaque, //
+};
+
+use core::{
+ marker::PhantomData,
+ mem::ManuallyDrop,
+ ops::Deref,
+ ptr,
+ ptr::{
+ drop_in_place,
+ NonNull, //
+ }, //
+};
+
+use kernel::{
+ str::CString,
+ sync::{
+ aref::{
+ ARef,
+ AlwaysRefCounted, //
+ },
+ atomic::{
+ Atomic,
+ Relaxed, //
+ },
+ rcu::rcu_barrier, //
+ }, //
+};
+
+/// VTable for dma_fence backend_ops callbacks.
+//
+// Mandatory dma_fence backend_ops are implemented implicitly through
+// [`FenceContext`]. Additional ones shall get implemented on this trait.
+pub trait FenceContextOps {
+ /// The generic payload data for [`DriverFence`]s created on this fctx.
+ type FenceDataType: Send + Sync;
+}
+
+/// A dma-fence context. A fence context takes care of associating related fences
+/// with each other, providing each with raising sequence numbers and a common
+/// identifier.
+#[pin_data(PinnedDrop)]
+pub struct FenceContext<T: FenceContextOps + Send + Sync> {
+ /// The fence context number.
+ nr: u64,
+ /// The sequence number for the next fence created.
+ seqno: Atomic<u64>,
+ // The name parameters can be accessed by the dma_fence backend_ops. UAF
+ // errors are prevented by the `call_rcu()` in `drop_driver_fence_data()`.
+ /// The name of the driver this FenceContext's fences belong to.
+ driver_name: CString,
+ /// The name of the timeline this FenceContext's fences belong to.
+ timeline_name: CString,
+ /// The number of all unsignaled fences on this context.
+ // Used to prevent bugs due to forgotten fences.
+ //
+ // The lifetime on `DriverFence`s should typically prevent this from
+ // happening.
+ //
+ // However, we cannot fully guarantee in Rust that `DriverFence`s will not
+ // be forgotten, e.g., through `core::mem::forget()`. This could circumvent
+ // the lifetime which intends to enforce that all fences disappear before
+ // their context.
+ nr_of_unsignaled_fences: Atomic<usize>,
+ /// The user's data.
+ #[pin]
+ data: T,
+}
+
+impl<'a, T: Send + Sync + FenceContextOps> FenceContext<T> {
+ // This can later be extended as a vtable in case other parties need support
+ // for the more "exotic" callbacks.
+ const OPS: bindings::dma_fence_ops = bindings::dma_fence_ops {
+ get_driver_name: Some(Self::get_driver_name),
+ get_timeline_name: Some(Self::get_timeline_name),
+ enable_signaling: None,
+ signaled: None,
+ // Deprecated.
+ wait: None,
+ // Must never be implemented for these abstractions.
+ release: None,
+ set_deadline: None,
+ };
+
+ /// Create a new `FenceContext`.
+ pub fn new<E>(
+ initial_seqno: u64,
+ driver_name: &CStr,
+ timeline_name: &CStr,
+ data: impl PinInit<T, E>,
+ ) -> impl PinInit<Self, Error>
+ where
+ Error: From<E>,
+ {
+ let driver_name = CString::try_from(driver_name);
+ let timeline_name = CString::try_from(timeline_name);
+ try_pin_init!(Self {
+ // SAFETY: `dma_fence_context_alloc()` merely works on a global
+ // atomic. Parameter `1` is the number of contexts we want to
+ // allocate.
+ nr: unsafe { bindings::dma_fence_context_alloc(1) },
+ seqno: Atomic::new(initial_seqno),
+ driver_name: driver_name?,
+ timeline_name: timeline_name?,
+ nr_of_unsignaled_fences: Atomic::new(0),
+ data <- data,
+ })
+ }
+
+ fn next_seqno(&self) -> u64 {
+ self.seqno.fetch_add(1, Relaxed)
+ }
+
+ /// Allocate the memory for a [`DriverFence`] and already store `data` inside.
+ ///
+ /// This is needed because many times, creation of a [`DriverFence`] must not
+ /// fail, and allocating might deadlock in some situations.
+ ///
+ /// The `data` you pass here must not perform any operations that are illegal
+ /// in atomic context in its [`Drop`] implementation.
+ pub fn new_fence_allocation(
+ &self,
+ data: T::FenceDataType,
+ ) -> Result<DriverFenceAllocation<'_, T>> {
+ let fence_data = DriverFenceData {
+ rcu_head: Default::default(),
+ // `inner` remains uninitialized until a `DriverFence` takes over.
+ inner: Fence {
+ inner: Opaque::uninit(),
+ },
+ fctx: self,
+ data,
+ };
+
+ // In order to support the C dma_fence callbacks, it is necessary for
+ // a `Fence` and a `DriverFence` to live in the same allocation,
+ // because the C backend passes a dma_fence, from which the driver most
+ // likely wants to be able to access its `data` in `DriverFence`.
+ //
+ // Hence, we need the manage the memory manually. It will be freed by the
+ // C backend automatically once the refcount within `Fence` drops to 0.
+ let data = KBox::new(fence_data, GFP_KERNEL | __GFP_ZERO)?;
+
+ Ok(DriverFenceAllocation {
+ data,
+ ops: &Self::OPS,
+ })
+ }
+
+ extern "C" fn get_driver_name(ptr: *mut bindings::dma_fence) -> *const c_char {
+ // SAFETY: The C backend only invokes this callback with `ptr` pointing
+ // to a valid, unsignaled `bindings::dma_fence`. All fences created in
+ // this module always reside within `Fence` which always resides in a
+ // `DriverFenceData`, thus satisfying the function's safety
+ // requirements.
+ let fctx = unsafe { Self::from_raw_fence(ptr) };
+
+ fctx.driver_name.as_char_ptr()
+ }
+
+ extern "C" fn get_timeline_name(ptr: *mut bindings::dma_fence) -> *const c_char {
+ // SAFETY: The C backend only invokes this callback with `ptr` pointing
+ // to a valid, unsignaled `bindings::dma_fence`. All fences created in
+ // this module always reside within `Fence` which always resides in a
+ // `DriverFenceData`, thus satisfying the function's safety
+ // requirements.
+ let fctx = unsafe { Self::from_raw_fence(ptr) };
+
+ fctx.timeline_name.as_char_ptr()
+ }
+
+ /// Create a [`FenceContext`] from an associated [`bindings::dma_fence`].
+ ///
+ /// # Safety
+ ///
+ /// `ptr` must be a valid pointer to a [`bindings::dma_fence`] which resides
+ /// within a [`Fence`], which in turn resides in a [`DriverFenceData`].
+ unsafe fn from_raw_fence(ptr: *mut bindings::dma_fence) -> &'a Self {
+ let opaque_fence = Opaque::cast_from(ptr);
+
+ // SAFETY: Safe due to the function's overall safety requirements.
+ let fence_ptr = unsafe { container_of!(opaque_fence, Fence, inner) };
+
+ // CAST: `DriverFenceData` is `repr(C)` and a `Fence` is its first member.
+ let fence_data_ptr: *const DriverFenceData<'a, T> = fence_ptr.cast();
+
+ // SAFETY: Safe because of the comments directly above.
+ let fence_data = unsafe { &*fence_data_ptr };
+
+ fence_data.fctx
+ }
+}
+
+#[pinned_drop]
+impl<T: FenceContextOps + Send + Sync> PinnedDrop for FenceContext<T> {
+ fn drop(self: Pin<&mut Self>) {
+ // Fence ops callbacks can be called on unsignaled fences. Since these
+ // callbacks can access the fence context and its data, it needs to be
+ // guaranteed that a context only drops after all associated
+ // `DriverFence`s have been dropped. This is unlikely to occur, but
+ // would result in silent UAF. Throw a panic to prevent that.
+ //
+ // TODO:
+ // It would be better if the fence context signals all forgotten fences
+ // itself. To do so, it would keep a list of unsignaled fences. That
+ // list's members would have to be pre-allocated (see
+ // `FenceCallback::new_fence_allocation()`).
+ if self.nr_of_unsignaled_fences.load(Relaxed) != 0 {
+ panic!("Forgotten fences in FenceContext.");
+ }
+
+ // Ensure that the driver cannot unload while there are still dma_fence
+ // callbacks running. At the same time, the RCU barrier addresses the
+ // problem inherited by the C backend, in which backend ops callbacks
+ // might be accessing the fence while it is being signaled (or shortly
+ // after). This could cause UAF access on the fence context's
+ // `fctx.driver_name` and `fctx.timeline_name`.
+ //
+ // Wait for the RCU callbacks in `DriverFence::drop`.
+ rcu_barrier();
+ }
+}
+
+/// Error type for fence callback registration.
+///
+/// Generic over `T` so that `AlreadySignaled` can return the callback to the
+/// caller, allowing it to reclaim any resources owned by the callback (e.g.,
+/// a fence handle that needs to be signaled).
+#[derive(Debug)]
+pub enum CallbackError<T> {
+ /// The fence was already signaled. The callback is returned so the caller
+ /// can extract owned resources without losing them.
+ AlreadySignaled(T),
+ /// Some other error occurred during registration.
+ Other(Error),
+}
+
+impl<T> From<CallbackError<T>> for Error {
+ #[inline]
+ fn from(err: CallbackError<T>) -> Self {
+ match err {
+ CallbackError::AlreadySignaled(_) => ENOENT,
+ CallbackError::Other(e) => e,
+ }
+ }
+}
+
+impl<T> From<AllocError> for CallbackError<T> {
+ #[inline]
+ fn from(e: AllocError) -> Self {
+ CallbackError::Other(Error::from(e))
+ }
+}
+
+/// Trait for callbacks that can be registered on fences.
+///
+/// When the fence signals, the callback will be invoked.
+///
+/// # Example
+///
+/// ```rust
+/// use kernel::dma_buf::FenceCallback;
+///
+/// struct MyCallback {
+/// // Your callback state here
+/// }
+///
+/// impl FenceCallback for MyCallback {
+/// fn on_signal(&mut self) {
+/// pr_info!("Fence signaled!");
+/// // Handle fence completion
+/// }
+/// }
+/// ```
+pub trait FenceCallback: Send + 'static {
+ /// Called when the fence is signaled.
+ ///
+ /// This is called from the fence signaling path, which may be in interrupt
+ /// context or with locks held, which is why `self` is only borrowed, so that
+ /// it cannot drop. Implementations must not sleep or perform
+ /// long-running operations.
+ ///
+ /// An implementation likely wants to inform itself (e.g., through a work item)
+ /// within this callback that the associated [`FenceCallbackRegistration`]
+ /// can now be dropped.
+ fn on_signal(&mut self);
+}
+
+/// A callback registration on a fence.
+///
+/// When this object is dropped, the callback is automatically removed if it
+/// hasn't been called yet.
+#[pin_data(PinnedDrop)]
+pub struct FenceCallbackRegistration<T: FenceCallback + 'static> {
+ #[pin]
+ callback_foreign: Opaque<bindings::dma_fence_cb>,
+ callback: ManuallyDrop<T>,
+ fence: ARef<Fence>,
+}
+
+impl<T: FenceCallback> FenceCallbackRegistration<T> {
+ /// Create a [`PinInit`] closure for registering a callback on a fence.
+ ///
+ /// The actual attempt at registering the callback will take place once you
+ /// call an allocator's `pin_init()` function.
+ ///
+ /// On success the callback is pinned in place and will fire when the fence
+ /// signals. On `AlreadySignaled` the callback is returned to the caller so
+ /// that owned resources can be reclaimed.
+ pub fn new<'a>(fence: &'a Fence, callback: T) -> impl PinInit<Self, CallbackError<T>> + 'a
+ where
+ T: 'a,
+ {
+ try_pin_init!(Self {
+ // We need to fully initialize the fence because after
+ // `dma_fence_add_callback()` ran, the callback might immediately
+ // get invoked.
+ callback: ManuallyDrop::new(callback),
+ fence: ARef::from(fence),
+ callback_foreign <- Opaque::try_ffi_init(|ptr| {
+ // SAFETY: `fence.inner.get()` is a valid, initialized `struct
+ // dma_fence`. `ptr` points to the `struct dma_fence_cb` field
+ // within the pinned allocation, so it remains valid until
+ // `dma_fence_remove_callback()` in `PinnedDrop` or until the
+ // callback fires.
+ let ret = unsafe {
+ to_result(bindings::dma_fence_add_callback(
+ fence.inner.get(),
+ ptr,
+ Some(Self::dma_fence_callback),
+ ))
+ };
+ match ret {
+ Ok(()) => Ok(()),
+ Err(e) => {
+ // SAFETY: We could not register the callback. Thus,
+ // C will not use it. So we can just take it back
+ // and pass it to the user again.
+ let cb_back = unsafe { ManuallyDrop::take(callback) };
+ if e == ENOENT {
+ Err(CallbackError::AlreadySignaled(cb_back))
+ } else {
+ Err(CallbackError::Other(e))
+ }
+ },
+ }
+ }),
+ }? CallbackError<T>)
+ }
+
+ /// Raw dma fence callback that is called by the C code.
+ ///
+ /// # Safety
+ ///
+ /// This is only called by the dma_fence subsystem with valid pointers.
+ unsafe extern "C" fn dma_fence_callback(
+ _fence: *mut bindings::dma_fence,
+ callback_foreign: *mut bindings::dma_fence_cb,
+ ) {
+ let ptr = Opaque::cast_from(callback_foreign).cast_mut();
+
+ // SAFETY: All `cb` we can receive here have been created in such a way
+ // that they are embedded into a `FenceCallbackRegistration`. The
+ // backend ensures synchronisation so whoever holds the registration
+ // object cannot drop it while this code is running. See
+ // `FenceCallbackRegistration::drop`.
+ unsafe {
+ let reg: *mut Self = container_of!(ptr, Self, callback_foreign);
+
+ (*reg).callback.on_signal();
+ }
+ }
+
+ /// Returns a reference to the fence this callback is registered on.
+ #[inline]
+ pub fn fence(&self) -> &Fence {
+ &self.fence
+ }
+}
+
+#[pinned_drop]
+impl<T: FenceCallback> PinnedDrop for FenceCallbackRegistration<T> {
+ fn drop(self: Pin<&mut Self>) {
+ // Always call `dma_fence_remove_callback()`, even if the callback
+ // already ran. This is necessary for synchronization:
+ // `dma_fence_remove_callback()` acquires `fence->lock`, which ensures
+ // that any in-flight `dma_fence_signal()` (which calls our callback
+ // while holding the same lock) has completed before we free the struct.
+ //
+ // Without this, Drop can race with a concurrent signal:
+ // CPU0 (signal, lock held): take() -> on_signal(fence_ref) (in progress)
+ // CPU1 (drop): skips lock -> frees struct
+ // CPU0: accesses fence_ref -> use-after-free
+ //
+ // When the callback has already fired, the signal path detached the
+ // list node via `INIT_LIST_HEAD()`, so dma_fence_remove_callback just
+ // sees an empty node and returns false — the lock acquisition is the
+ // only thing that matters.
+ //
+ // SAFETY: The fence pointer is valid and the cb was initialized by
+ // `dma_fence_add_callback()` during construction.
+ unsafe {
+ bindings::dma_fence_remove_callback(self.fence.as_raw(), self.callback_foreign.get());
+ }
+
+ // SAFETY: This is literally the drop implementation, so no one has
+ // dropped this so far; so we can do it now.
+ unsafe { ManuallyDrop::<T>::drop(self.project().callback) };
+ }
+}
+
+// SAFETY: FenceCallbackRegistration can be sent between threads.
+unsafe impl<T: FenceCallback> Send for FenceCallbackRegistration<T> {}
+
+// SAFETY: &FenceCallbackRegistration can be shared between threads if &T can.
+unsafe impl<T: FenceCallback> Sync for FenceCallbackRegistration<T> where T: Sync {}
+
+/// The receiving counterpart of a [`DriverFence`].
+///
+/// The Rust DMA fence implementation has a dualistic design: [`DriverFence`]s
+/// are the producer-side, intended to be always owned by only one party. That
+/// party has the monopoly on signaling the fence.
+///
+/// A [`Fence`] is the counterpart for consumers. Thus, [`Fence`]s are always
+/// refcounted and can shared with an arbitrary number of parties, including
+/// userspace. A [`Fence`] can only be used for actions such as checking the
+/// fence's status or for registering callbacks on it.
+///
+/// Once the associated [`DriverFence`] signals, all
+/// [`FenceCallbackRegistration`]s registered on the [`Fence`] will be executed.
+///
+/// A [`Fence`] can arbitrarily outlive its [`DriverFence`] and the
+/// [`FenceContext`]. Signaling a [`DriverFence`] decouples it from its
+/// [`Fence`]s.
+#[repr(transparent)]
+pub struct Fence {
+ /// The actual dma_fence passed to C.
+ inner: Opaque<bindings::dma_fence>,
+}
+
+/// Guard helper for locking within this module.
+///
+/// Its only purpose for now is to avoid a number of unsafe lock-unlock cycles.
+/// It is never used outside of this module.
+// TODO: This should be made more canonical, probably by basing it on a
+// SpinLockIrqGuard once available.
+struct FenceGuard<'a> {
+ inner: &'a Fence,
+ flags: usize,
+}
+
+impl<'a> Deref for FenceGuard<'a> {
+ type Target = &'a Fence;
+
+ fn deref(&self) -> &Self::Target {
+ &self.inner
+ }
+}
+
+impl Drop for FenceGuard<'_> {
+ fn drop(&mut self) {
+ // SAFETY: `fence` is valid because `self` is valid. `flag_ptr` is
+ // merely a pointer to an integer, which lives as long as this function.
+ // When a `FenceGuard` exists, the lock has been taken by definition.
+ unsafe { bindings::dma_fence_unlock_irqrestore(self.as_raw(), &raw mut self.flags) };
+ }
+}
+
+// SAFETY: Fences are literally designed to be shared between threads.
+unsafe impl Send for Fence {}
+// SAFETY: Fences are literally designed to be shared between threads.
+unsafe impl Sync for Fence {}
+
+impl Fence {
+ /// Check whether the fence was signaled at the moment of the function call.
+ ///
+ /// Note that this can return `true` for a [`Fence`] whose [`DriverFence`]
+ /// has not yet been dropped. The reason is that the fence ops callbacks can
+ /// cause the fence to get signaled by the C backend.
+ #[inline]
+ pub fn is_signaled(&self) -> bool {
+ // We should not use `dma_fence_is_signaled_locked()` here, because
+ // according to the C backend's recommendations, that function is
+ // problematic and we should avoid calling that function with a lock
+ // held.
+
+ // SAFETY: Inner `fence` is valid because `self` is valid.
+ let ret = unsafe { bindings::dma_fence_is_signaled(self.as_raw()) };
+
+ // To be as robust as possible for the future we guarantee that an API
+ // caller can 100% rely on the signaling being completed (i.e., all
+ // fence callbacks ran), so we have to take the lock.
+ //
+ // The reason is that the C dma_fence backend currently does not
+ // carefully synchronize the `dma_fence_is_signaled()` function with the
+ // proper spinlock. This can lead to the function returning `true` while
+ // fence callbacks are still being executed. This can be mitigated by
+ // guarding the entire function with the spinlock.
+ //
+ // The fundamental reason is that the C backend currently does guard
+ // setting of the fence's signaled-bit with the fence's spinlock, but
+ // reading is done locklessly.
+ //
+ // See commit c8a5d5ea3ba6a.
+ let _ = self.lock();
+
+ ret
+ }
+
+ /// Lock the fence. A helper only to be used internally in this module.
+ fn lock(&self) -> FenceGuard<'_> {
+ let mut guard = FenceGuard {
+ inner: self,
+ flags: 0,
+ };
+
+ // SAFETY: `fence` is valid because `self` is valid. `flag_ptr` is
+ // merely a pointer to an integer, whose lifetime is tied to the guard
+ // object.
+ unsafe { bindings::dma_fence_lock_irqsave(self.as_raw(), &raw mut guard.flags) };
+
+ guard
+ }
+
+ /// Get the fence's sequence number.
+ #[inline]
+ pub fn seqno(&self) -> u64 {
+ // SAFETY: Valid because `self` is valid.
+ unsafe { (*self.as_raw()).seqno }
+ }
+
+ fn as_raw(&self) -> *mut bindings::dma_fence {
+ self.inner.get()
+ }
+
+ /// Create a [`Fence`] from a raw C [`bindings::dma_fence`].
+ ///
+ /// # Safety
+ ///
+ /// `ptr` must point to an initialized fence that is embedded into a [`Fence`].
+ #[inline]
+ pub unsafe fn from_raw<'a>(ptr: *mut bindings::dma_fence) -> &'a Self {
+ // SAFETY: Safe as per the function's overall safety requirements.
+ unsafe { &*ptr.cast() }
+ }
+}
+
+// SAFETY: These implement the C backends refcounting methods which are proven
+// to work correctly.
+unsafe impl AlwaysRefCounted for Fence {
+ fn inc_ref(&self) {
+ // SAFETY: `self.as_raw()` is a pointer to a valid `struct dma_fence`.
+ unsafe { bindings::dma_fence_get(self.as_raw()) }
+ }
+
+ unsafe fn dec_ref(ptr: NonNull<Self>) {
+ // SAFETY: `ptr` is never a NULL pointer; and when `dec_ref()` is called
+ // the fence is by definition still valid.
+ let fence = unsafe { (*ptr.as_ptr()).inner.get() };
+
+ // SAFETY: `fence` was created validly above. When `dec_ref()` is called,
+ // there is by definition still a reference alive that can be put.
+ unsafe { bindings::dma_fence_put(fence) }
+ }
+}
+
+// Necessary to guarantee that `inner` always comes first and can be freed by C.
+// Also useful for using casts instead of container_of().
+#[repr(C)]
+#[pin_data]
+struct DriverFenceData<'a, T: Send + Sync + FenceContextOps> {
+ #[pin]
+ /// The inner fence.
+ // Must always be the first member so that unsafe casting works; but also
+ // necessary so that the C backend can free the allocation (coming from our
+ // Rust code) with kfree_rcu().
+ inner: Fence,
+ /// Callback head for dropping this in a deferred manner through RCU.
+ rcu_head: bindings::callback_head,
+ /// Reference to access the FenceContext.
+ fctx: &'a FenceContext<T>,
+ /// The API user's data. It is essential that the data only performs
+ /// operations legal in atomic context in its [`Drop`] implementation.
+ #[pin]
+ data: T::FenceDataType,
+}
+
+/// A synchronization primitive mainly for GPU drivers.
+///
+/// The Rust DMA fence implementation has a dualistic design: [`DriverFence`]s
+/// are the producer-side, intended to be always owned by only one party. That
+/// party has the monopoly on signaling the fence.
+///
+/// A [`Fence`] is the counterpart for consumers. Thus, [`Fence`]s are always
+/// refcounted and can be shared with an arbitrary number of parties, including
+/// userspace. A [`Fence`] can only be used for actions such as checking the
+/// fence's status or for registering callbacks on it.
+///
+/// Once the associated [`DriverFence`] signals, all
+/// [`FenceCallbackRegistration`]s registered on a [`Fence`] will be executed.
+///
+/// A [`Fence`] can arbitrarily outlive its [`DriverFence`] and the
+/// [`FenceContext`]. Signaling a [`DriverFence`] decouples it from its
+/// [`Fence`]s.
+///
+/// It is crucial that a [`DriverFence`] always correctly represents the state
+/// of the associated job on the hardware. Especially, it is strictly necessary
+/// that the owner ensures that all [`DriverFence`]s eventually get signaled.
+/// As a last resort, a [`DriverFence`] will signal itself if it drops
+/// unsignaled and print a warning.
+///
+/// This design intends to implement the [`bindings::dma_fence_ops`] in such a
+/// way that the driver-data necessary to implement the callback's functionality
+/// resides in the [`FenceContext`]. Thus, a [`DriverFence`] contains a
+/// reference to the context, which can be accessed in the callbacks. The
+/// implementation, therefore, ensures that a [`DriverFence`] cannot outlive its
+/// [`FenceContext`]. Unfortunately, this can be circumvented under certain
+/// circumstances in Rust (e.g., usage of [`core::mem::forget`]).
+///
+/// In the unlikely case of such violations, a panic is thrown.
+///
+/// # Examples
+///
+/// ```
+/// use kernel::{
+/// dma_buf::{
+/// DriverFence,
+/// FenceContext,
+/// FenceContextOps,
+/// FenceCallback,
+/// FenceCallbackRegistration,
+/// },
+/// str::CString,
+/// sync::aref::ARef, //
+/// };
+/// use core::fmt::Display;
+///
+/// struct CallbackData { }
+///
+/// impl FenceCallback for CallbackData {
+/// fn on_signal(&mut self) {
+/// pr_info!("DmaFence callback executed.\n");
+/// }
+/// }
+///
+/// #[pin_data]
+/// struct FenceContextData {}
+///
+/// impl FenceContextData {
+/// fn new() -> impl PinInit<Self> {
+/// pin_init!(Self {})
+/// }
+/// }
+///
+/// impl FenceContextOps for FenceContextData {
+/// type FenceDataType = FenceData;
+/// }
+///
+/// let fctx_data = FenceContextData::new();
+///
+///
+/// let mut fctx = KBox::pin_init(
+/// FenceContext::new(0, c"dummy_driver", c"dummy_timeline", fctx_data),
+/// GFP_KERNEL
+/// )?;
+///
+/// struct FenceData {
+/// data: CString,
+/// }
+///
+/// let fence_data = FenceData { data: c"dummy_data".try_into()? };
+///
+/// let fence_alloc = fctx.new_fence_allocation(fence_data)?;
+/// let mut fence = fence_alloc.new_fence();
+///
+/// let cb_data = CallbackData { };
+/// let waiting_fence = ARef::from(fence.as_fence());
+/// let cb_reg = FenceCallbackRegistration::new(&waiting_fence, cb_data);
+/// let cb_reg = KBox::pin_init(cb_reg, GFP_KERNEL)?;
+///
+/// // TODO signalling guards
+/// assert_eq!(waiting_fence.is_signaled(), false);
+/// fence.signal(Ok(()));
+/// assert_eq!(waiting_fence.is_signaled(), true);
+///
+/// Ok::<(), Error>(())
+/// ```
+pub struct DriverFence<'a, T: Send + Sync + FenceContextOps> {
+ /// The actual content of the fence. Lives in a [`NonNull`] so that its
+ /// memory can be managed independently. Valid until both the [`DriverFence`]
+ /// and all associated [`Fence`]s have disappeared.
+ data: NonNull<DriverFenceData<'a, T>>,
+}
+
+/// A pre-prepared DMA fence, carrying the user's data and the memory it and the
+/// fence reside in. Only useful for creating a [`DriverFence`]. Splitting
+/// allocation and full initialization is necessary because fences cannot be
+/// allocated dynamically in some circumstances (deadlock).
+pub struct DriverFenceAllocation<'a, T: Send + Sync + FenceContextOps> {
+ /// The memory for the actual content of the fence.
+ /// Handed over to a [`DriverFence`], or deallocated once the
+ /// [`DriverFenceAllocation`] drops.
+ data: KBox<DriverFenceData<'a, T>>,
+ /// Reference for the ops for the associated [`FenceContext`]
+ ops: &'static bindings::dma_fence_ops,
+}
+
+impl<'a, T: Send + Sync + FenceContextOps> DriverFenceAllocation<'a, T> {
+ /// Create a new [`DriverFence`], the signalable counterpart of a [`Fence`].
+ ///
+ /// This increments the sequence number in the associated [`FenceContext`].
+ pub fn new_fence(self) -> DriverFence<'a, T> {
+ // We feed the C dma_fence backend a NULL for the spinlock so that it
+ // uses per-fence locks automatically.
+ let null_ptr: *mut bindings::spinlock = ptr::null_mut();
+ let seqno = self.data.fctx.next_seqno();
+ let fence_ptr = self.as_raw();
+ // SAFETY: `fence_ptr` has been created directly above. It will live
+ // at least as long as `Self`. The same applies to `&Self::OPS`.
+ unsafe {
+ bindings::dma_fence_init(fence_ptr, self.ops, null_ptr, self.data.fctx.nr, seqno)
+ };
+
+ self.data.fctx.nr_of_unsignaled_fences.fetch_add(1, Relaxed);
+
+ // A `DriverFenceAllocation`'s purpose is to carry allocated memory, so
+ // that `DriverFence`s can always be created without allocating. In this
+ // method, ownership over that memory is transferred to the new
+ // `DriverFence` and managed through refcounting. The C dma_fence
+ // backend will ultimately free the memory once the refcount reaches 0.
+ let ptr = KBox::into_raw(self.data);
+ // SAFETY: `ptr` was just created validly directly above.
+ let ptr = unsafe { NonNull::new_unchecked(ptr) };
+
+ DriverFence { data: ptr }
+ }
+
+ fn as_raw(&self) -> *mut bindings::dma_fence {
+ self.data.inner.inner.get()
+ }
+}
+
+impl<'a, T: Send + Sync + FenceContextOps> DriverFence<'a, T> {
+ fn as_raw(&self) -> *mut bindings::dma_fence {
+ // SAFETY: Valid because `self` is valid.
+ let fence_data = unsafe { &*self.data.as_ptr() };
+
+ fence_data.inner.inner.get()
+ }
+
+ /// Create a [`DriverFence`] from a raw pointer to a [`bindings::dma_fence`].
+ ///
+ /// # Safety
+ ///
+ /// `ptr` must be a valid pointer to a `dma_fence` that was obtained through
+ /// a [`DriverFence`] with matching generic data for both fence and associated
+ /// [`FenceContext`].
+ unsafe fn from_raw(ptr: *mut bindings::dma_fence) -> Self {
+ let opaque_fence = Opaque::cast_from(ptr);
+
+ // SAFETY: Safe due to the function's overall safety requirements.
+ let fence_ptr = unsafe { container_of!(opaque_fence, Fence, inner) };
+
+ // DriverFenceData is `repr(C)` and a Fence is its first member.
+ let fence_data_ptr = fence_ptr as *mut DriverFenceData<'a, T>;
+
+ // SAFETY: `fence_data_ptr` was created validly above.
+ let data = unsafe { NonNull::new_unchecked(fence_data_ptr) };
+
+ Self { data }
+ }
+
+ /// Return the underlying [`Fence`].
+ #[inline]
+ pub fn as_fence(&self) -> &Fence {
+ // SAFETY: `self` is by definition still valid, and it cannot drop until
+ // this new reference is gone.
+ unsafe { Fence::from_raw(self.as_raw()) }
+ }
+
+ /// Signal the fence. This will invoke all registered callbacks.
+ pub fn signal(self, res: Result) {
+ let fence = self.as_fence().lock();
+
+ // SAFETY: `fence` is valid because `self` is valid. The lock must be
+ // held, which we acquired directly above.
+ if !unsafe { bindings::dma_fence_test_signaled_flag(fence.as_raw()) } {
+ if let Err(err) = res {
+ // SAFETY: `fence` is valid because `self` is valid. The fence
+ // must not have been signaled yet, which we check directly above.
+ unsafe { bindings::dma_fence_set_error(fence.as_raw(), err.to_errno()) };
+ }
+ // SAFETY: `fence` is valid because `self` is valid. The lock must
+ // be held, which we acquired above.
+ unsafe { bindings::dma_fence_signal_locked(fence.as_raw()) };
+ }
+
+ // SAFETY: `self.data` is valid because `self` is valid.
+ let fctx = unsafe { self.data.as_ref().fctx };
+ let _ = fctx.nr_of_unsignaled_fences.fetch_sub(1, Relaxed);
+ }
+}
+
+// SAFETY: Fences are literally designed to be shared between threads.
+unsafe impl<'a, T: Send + Sync + FenceContextOps> Send for DriverFence<'a, T> {}
+// SAFETY: Fences are literally designed to be shared between threads.
+unsafe impl<'a, T: Send + Sync + FenceContextOps> Sync for DriverFence<'a, T> {}
+
+impl<'a, T: Send + Sync + FenceContextOps> Deref for DriverFence<'a, T> {
+ type Target = T::FenceDataType;
+
+ fn deref(&self) -> &Self::Target {
+ // SAFETY: Thanks to refcounting, `data` is always valid as long as `self` is.
+ let data = unsafe { &*self.data.as_ptr() };
+
+ &data.data
+ }
+}
+
+/// A borrow wrapper for [`DriverFence`]. Implements [`Deref`].
+pub struct DriverFenceBorrow<'a, T: Send + Sync + FenceContextOps> {
+ driver_fence: ManuallyDrop<DriverFence<'a, T>>,
+ _lifetime: PhantomData<&'a T>,
+}
+
+impl<'a, T: Send + Sync + FenceContextOps> Deref for DriverFenceBorrow<'a, T> {
+ type Target = DriverFence<'a, T>;
+
+ fn deref(&self) -> &Self::Target {
+ self.driver_fence.deref()
+ }
+}
+
+// SAFETY: The Rust dma_fence abstractions are already designed around the inner
+// C `dma_fence`, which can serve safely as the identification point when being
+// owned by C. Moreover, safety is ensured by not dropping `DriverFence` and by
+// only allowing operations without side effects on the Borrowed type.
+unsafe impl<T: Send + Sync + FenceContextOps> ForeignOwnable for DriverFence<'_, T> {
+ type Borrowed<'a>
+ = DriverFenceBorrow<'a, T>
+ where
+ Self: 'a;
+ type BorrowedMut<'a>
+ = DriverFenceBorrow<'a, T>
+ where
+ Self: 'a;
+
+ const FOREIGN_ALIGN: usize = core::mem::align_of::<bindings::dma_fence>();
+
+ fn into_foreign(self) -> *mut c_void {
+ let fence = self;
+
+ let ptr = fence.as_raw();
+
+ // DriverFence must not drop.
+ let _ = ManuallyDrop::new(fence);
+
+ ptr.cast()
+ }
+
+ unsafe fn from_foreign(ptr: *mut c_void) -> Self {
+ // SAFETY: Safe because the trait implementation only invokes this with
+ // a valid `ptr`, associated to a `DriverFence` with matching generic data.
+ unsafe { Self::from_raw(ptr.cast()) }
+ }
+
+ unsafe fn borrow<'a>(ptr: *mut c_void) -> Self::Borrowed<'a>
+ where
+ Self: 'a,
+ {
+ // SAFETY: The trait implementation ensures that `ptr` always resides
+ // within a [`Fence`] within a [`DriverFenceData`].
+ let driver_fence = unsafe { Self::from_raw(ptr.cast()) };
+
+ let driver_fence = ManuallyDrop::new(driver_fence);
+
+ DriverFenceBorrow {
+ driver_fence,
+ _lifetime: PhantomData,
+ }
+ }
+
+ unsafe fn borrow_mut<'a>(ptr: *mut c_void) -> Self::BorrowedMut<'a>
+ // FIXME: The bound below and the one above in `borrow` should actually be
+ // unnecessary since the compiler should be able to completely derive all
+ // necessary information automatically. There is currently a compiler bug
+ // preventing that, though:
+ //
+ // https://github.com/rust-lang/rust/issues/155430.
+ //
+ // (Help to) fix the compiler bug and remove the bounds afterwards.
+ where
+ Self: 'a,
+ {
+ // SAFETY: The trait implementation ensures that `ptr` always resides
+ // within a [`Fence`] within a [`DriverFenceData`].
+ let driver_fence = unsafe { Self::from_raw(ptr.cast()) };
+
+ let driver_fence = ManuallyDrop::new(driver_fence);
+
+ DriverFenceBorrow {
+ driver_fence,
+ _lifetime: PhantomData,
+ }
+ }
+}
+
+impl<'a, T: Send + Sync + FenceContextOps> Drop for DriverFence<'a, T> {
+ fn drop(&mut self) {
+ let guard = self.as_fence().lock();
+
+ // Use dma_fence_test_signaled_flag() instead of
+ // dma_fence_is_signaled_locked() because the C backend wants to get rid
+ // of the latter.
+
+ // SAFETY: `guard` is valid until the `call_rcu()` below.
+ let signaled: bool = unsafe { bindings::dma_fence_test_signaled_flag(guard.as_raw()) };
+ if !signaled {
+ pr_err!("DriverFence drops unsignaled. Danger of memory corruption!\n");
+ // SAFETY: `guard` is valid until the `call_rcu()` below. The fence
+ // must not have been signaled yet, which we check directly above.
+ unsafe { bindings::dma_fence_set_error(guard.as_raw(), ECANCELED.to_errno()) };
+ // SAFETY: `guard` is valid until the `call_rcu()` below. The lock
+ // must be held, which we acquired above.
+ unsafe { bindings::dma_fence_signal_locked(guard.as_raw()) };
+
+ // SAFETY: `self.data` is valid because `self` is valid.
+ let fctx = unsafe { self.data.as_ref().fctx };
+ let _ = fctx.nr_of_unsignaled_fences.fetch_sub(1, Relaxed);
+ }
+ drop(guard);
+
+ // `DriverFenceData` could be accessed through some dma_fence
+ // callbacks right now. Access is being revoked in principle above by
+ // signaling the fence, but since the C backend does not guarantee
+ // perfect full synchronization, we have to wait for one grace period to
+ // ensure that all accessors of `DriverFenceData` (through the
+ // dma_fence_ops accessible through a `Fence`) are gone.
+
+ if !core::mem::needs_drop::<T::FenceDataType>() {
+ // SAFETY: Once a `DriverFence` is initialized, the inner `fence` is
+ // valid and initialized. It is valid until the refcount drops to 0,
+ // which can earliest happen once we drop the `DriverFence`'s
+ // reference here.
+ unsafe { bindings::dma_fence_put(self.as_raw()) };
+ return;
+ }
+
+ // SAFETY: Valid because `self` is valid.
+ let rcu_head_ptr = unsafe { &raw mut (*self.data.as_ptr()).rcu_head };
+
+ // SAFETY: `call_rcu()` is always safe to be called. `rcu_head_ptr` was
+ // created validly above. The module must perform a `synchronize_rcu()`
+ // or `rcu_barrier()` call to guard against module unload.
+ unsafe { bindings::call_rcu(rcu_head_ptr, Some(drop_driver_fence_data::<T>)) };
+ }
+}
+
+// TODO:
+// The entire call_rcu() mechanism in the drop above and the code below would be
+// unnecessary if C's dma_fence_signal() could be reworked in a way that after it
+// ran, the caller knows that no fence_ops callbacks can be running anymore.
+// In other words, if the dma_fence backend would use its spinlock for full
+// synchronization.
+//
+// Then we could move the drop_in_place() and dma_fence_put() upwards into the
+// drop() implementation and call it a day.
+
+/// Finally really drop this `DriverFence<T>`
+///
+/// # Safety
+///
+/// `head` references the `rcu_head` field of an `DriverFenceData<T>`. All
+/// accessors to that `DriverFenceData<T>` must be gone by now. This must be
+/// ensured by signalling the associated `DriverFence<T>` and then waiting
+/// for a grace period until calling this function here.
+unsafe extern "C" fn drop_driver_fence_data<T: Send + Sync + FenceContextOps>(
+ head: *mut bindings::callback_head,
+) {
+ // SAFETY: Caller provides a pointer to the `rcu_head` field of a `DriverFenceData<C>`.
+ let fence_data = unsafe { container_of!(head, DriverFenceData<'_, T>, rcu_head) };
+
+ // SAFETY: `fence_data` was created validly above. All the fence's data will
+ // only drop below, but the raw pointer to the raw C `dma_fence` remains
+ // valid because the reference count is only decremented at the end of the
+ // function.
+ let fence = unsafe { (*fence_data).inner.inner.get() };
+
+ // SAFETY: `fence_data` was created validly above. The user has already
+ // dropped the only conventional accessor to the user data, the `DriverFence`,
+ // one grace period ago. All accessors are gone now.
+ unsafe { drop_in_place(&raw mut (*fence_data).data) };
+
+ // The inner `Fence` explicitly does not get dropped because there may be
+ // many more users / consumers, each holding their own reference.
+
+ // SAFETY: Once a `DriverFence` is initialized, the inner `fence` is valid
+ // and initialized. It is valid until the refcount drops to 0, which can
+ // earliest happen once we drop the `DriverFence`'s reference here.
+ unsafe { bindings::dma_fence_put(fence) };
+
+ // The actual memory the data associated with a `DriverFence` lives in
+ // gets freed by the C dma_fence backend once the fence's refcount reaches 0.
+}