// SPDX-License-Identifier: GPL-2.0
// Copyright (C) 2025 Google LLC.
use kernel::{
list::{AtomicTracker, List, ListArc, ListLinks, TryNewListArc},
prelude::*,
seq_file::SeqFile,
seq_print,
sync::lock::{spinlock::SpinLockBackend, Guard},
sync::{Arc, LockedBy, SpinLock},
uapi,
};
use crate::{
defs::*,
error::BinderError,
process::{NodeRefInfo, Process, ProcessInner},
thread::Thread,
transaction::Transaction,
BinderReturnWriter, DArc, DLArc, DTRWrap, DeliverToRead,
};
use core::mem;
use core::ptr;
mod wrapper;
pub(crate) use self::wrapper::CritIncrWrapper;
#[derive(Debug)]
pub(crate) struct CouldNotDeliverCriticalIncrement;
/// Keeps track of how this node is scheduled.
///
/// There are two ways to schedule a node to a work list. Just schedule the node itself, or
/// allocate a wrapper that references the node and schedule the wrapper. These wrappers exists to
/// make it possible to "move" a node from one list to another - when `do_work` is called directly
/// on the `Node`, then it's a no-op if there's also a pending wrapper.
///
/// Wrappers are generally only needed for zero-to-one refcount increments, and there are two cases
/// of this: weak increments and strong increments. We call such increments "critical" because it
/// is critical that they are delivered to the thread doing the increment. Some examples:
///
/// * One thread makes a zero-to-one strong increment, and another thread makes a zero-to-one weak
/// increment. Delivering the node to the thread doing the weak increment is wrong, since the
/// thread doing the strong increment may have ended a long time ago when the command is actually
/// processed by userspace.
///
/// * We have a weak reference and are about to drop it on one thread. But then another thread does
/// a zero-to-one strong increment. If the strong increment gets sent to the thread that was
/// about to drop the weak reference, then the strong increment could be processed after the
/// other thread has already exited, which would be too late.
///
/// Note that trying to create a `ListArc` to the node can succeed even if `has_normal_push` is
/// set. This is because another thread might just have popped the node from a todo list, but not
/// yet called `do_work`. However, if `has_normal_push` is false, then creating a `ListArc` should
/// always succeed.
///
/// Like the other fields in `NodeInner`, the delivery state is protected by the process lock.
struct DeliveryState {
/// Is the `Node` currently scheduled?
has_pushed_node: bool,
/// Is a wrapper currently scheduled?
///
/// The wrapper is used only for strong zero2one increments.
has_pushed_wrapper: bool,
/// Is the currently scheduled `Node` scheduled due to a weak zero2one increment?
///
/// Weak zero2one operations are always scheduled using the `Node`.
has_weak_zero2one: bool,
/// Is the currently scheduled wrapper/`Node` scheduled due to a strong zero2one increment?
///
/// If `has_pushed_wrapper` is set, then the strong zero2one increment was scheduled using the
/// wrapper. Otherwise, `has_pushed_node` must be set and it was scheduled using the `Node`.
has_strong_zero2one: bool,
}
impl DeliveryState {
fn should_normal_push(&self) -> bool {
!self.has_pushed_node && !self.has_pushed_wrapper
}
fn did_normal_push(&mut self) {
assert!(self.should_normal_push());
self.has_pushed_node = true;
}
fn should_push_weak_zero2one(&self) -> bool {
!self.has_weak_zero2one && !self.has_strong_zero2one
}
fn can_push_weak_zero2one_normally(&self) -> bool {
!self.has_pushed_node
}
fn did_push_weak_zero2one(&mut self) {
assert!(self.should_push_weak_zero2one());
assert!(self.can_push_weak_zero2one_normally());
self.has_pushed_node = true;
self.has_weak_zero2one = true;
}
fn should_push_strong_zero2one(&self) -> bool {
!self.has_strong_zero2one
}
fn can_push_strong_zero2one_normally(&self) -> bool {
!self.has_pushed_node
}
fn did_push_strong_zero2one(&mut self) {
assert!(self.should_push_strong_zero2one());
assert!(self.can_push_strong_zero2one_normally());
self.has_pushed_node = true;
self.has_strong_zero2one = true;
}
fn did_push_strong_zero2one_wrapper(&mut self) {
assert!(self.should_push_strong_zero2one());
assert!(!self.can_push_strong_zero2one_normally());
self.has_pushed_wrapper = true;
self.has_strong_zero2one = true;
}
}
struct CountState {
/// The reference count.
count: usize,
/// Whether the process that owns this node thinks that we hold a refcount on it. (Note that
/// even if count is greater than one, we only increment it once in the owning process.)
has_count: bool,
}
impl CountState {
fn new() -> Self {
Self {
count: 0,
has_count: false,
}
}
}
struct NodeInner {
/// Strong refcounts held on this node by `NodeRef` objects.
strong: CountState,
/// Weak refcounts held on this node by `NodeRef` objects.
weak: CountState,
delivery_state: DeliveryState,
/// The binder driver guarantees that oneway transactions sent to the same node are serialized,
/// that is, userspace will not be given the next one until it has finished processing the
/// previous oneway transaction. This is done to avoid the case where two oneway transactions
/// arrive in opposite order from the order in which they were sent. (E.g., they could be
/// delivered to two different threads, which could appear as-if they were sent in opposite
/// order.)
///
/// To fix that, we store pending oneway transactions in a separate list in the node, and don't
/// deliver the next oneway transaction until userspace signals that it has finished processing
/// the previous oneway transaction by calling the `BC_FREE_BUFFER` ioctl.
oneway_todo: List<DTRWrap<Transaction>>,
/// Keeps track of whether this node has a pending oneway transaction.
///
/// When this is true, incoming oneway transactions are stored in `oneway_todo`, instead of
/// being delivered directly to the process.
has_oneway_transaction: bool,
/// List of processes to deliver a notification to when this node is destroyed (usually due to
/// the process dying).
death_list: List<DTRWrap<NodeDeath>, 1>,
/// List of processes to deliver freeze notifications to.
freeze_list: KVVec<Arc<Process>>,
/// The number of active BR_INCREFS or BR_ACQUIRE operations. (should be maximum two)
///
/// If this is non-zero, then we postpone any BR_RELEASE or BR_DECREFS notifications until the
/// active operations have ended. This avoids the situation an increment and decrement get
/// reordered from userspace's perspective.
active_inc_refs: u8,
/// List of `NodeRefInfo` objects that reference this node.
refs: List<NodeRefInfo, { NodeRefInfo::LIST_NODE }>,
}
use kernel::bindings::rb_node_layout;
use mem::offset_of;
pub(crate) const NODE_LAYOUT: rb_node_layout = rb_node_layout {
arc_offset: Arc::<Node>::DATA_OFFSET + offset_of!(DTRWrap<Node>, wrapped),
debug_id: offset_of!(Node, debug_id),
ptr: offset_of!(Node, ptr),
};
#[pin_data]
pub(crate) struct Node {
pub(crate) debug_id: usize,
ptr: u64,
pub(crate) cookie: u64,
pub(crate) flags: u32,
pub(crate) owner: Arc<Process>,
inner: LockedBy<NodeInner, ProcessInner>,
#[pin]
links_track: AtomicTracker,
}
kernel::list::impl_list_arc_safe! {
impl ListArcSafe<0> for Node {
tracked_by links_track: AtomicTracker;
}
}
// Make `oneway_todo` work.
kernel::list::impl_list_item! {
impl ListItem<0> for DTRWrap<Transaction> {
using ListLinks { self.links.inner };
}
}
impl Node {
pub(crate) fn new(
ptr: u64,
cookie: u64,
flags: u32,
owner: Arc<Process>,
) -> impl PinInit<Self> {
pin_init!(Self {
inner: LockedBy::new(
&owner.inner,
NodeInner {
strong: CountState::new(),
weak: CountState::new(),
delivery_state: DeliveryState {
has_pushed_node: false,
has_pushed_wrapper: false,
has_weak_zero2one: false,
has_strong_zero2one: false,
},
death_list: List::new(),
oneway_todo: List::new(),
freeze_list: KVVec::new(),
has_oneway_transaction: false,
active_inc_refs: 0,
refs: List::new(),
},
),
debug_id: super::next_debug_id(),
ptr,
cookie,
flags,
owner,
links_track <- AtomicTracker::new(),
})
}
pub(crate) fn has_oneway_transaction(&self, owner_inner: &mut ProcessInner) -> bool {
let inner = self.inner.access_mut(owner_inner);
inner.has_oneway_transaction
}
#[inline(never)]
pub(crate) fn full_debug_print(
&self,
m: &SeqFile,
owner_inner: &mut ProcessInner,
) -> Result<()> {
let inner = self.inner.access_mut(owner_inner);
seq_print!(
m,
" node {}: u{:016x} c{:016x} hs {} hw {} cs {} cw {}",
self.debug_id,
self.ptr,
self.cookie,
inner.strong.has_count,
inner.weak.has_count,
inner.strong.count,
inner.weak.count,
);
if !inner.refs.is_empty() {
seq_print!(m, " proc");
for node_ref in &inner.refs {
seq_print!(m, " {}", node_ref.process.task.pid());
}
}
seq_print!(m, "\n");
for t in &inner.oneway_todo {
t.debug_print_inner(m, " pending async transaction ");
}
Ok(())
}
/// Insert the `NodeRef` into this `refs` list.
///
/// # Safety
///
/// It must be the case that `info.node_ref.node` is this node.
pub(crate) unsafe fn insert_node_info(
&self,
info: ListArc<NodeRefInfo, { NodeRefInfo::LIST_NODE }>,
) {
self.inner
.access_mut(&mut self.owner.inner.lock())
.refs
.push_front(info);
}
/// Insert the `NodeRef` into this `refs` list.
///
/// # Safety
///
/// It must be the case that `info.node_ref.node` is this node.
pub(crate) unsafe fn remove_node_info(
&self,
info: &NodeRefInfo,
) -> Option<ListArc<NodeRefInfo, { NodeRefInfo::LIST_NODE }>> {
// SAFETY: We always insert `NodeRefInfo` objects into the `refs` list of the node that it
// references in `info.node_ref.node`. That is this node, so `info` cannot possibly be in
// the `refs` list of another node.
unsafe {
self.inner
.access_mut(&mut self.owner.inner.lock())
.refs
.remove(info)
}
}
/// An id that is unique across all binder nodes on the system. Used as the key in the
/// `by_node` map.
pub(crate) fn global_id(&self) -> usize {
ptr::from_ref(self).addr()
}
pub(crate) fn get_id(&self) -> (u64, u64) {
(self.ptr, self.cookie)
}
pub(crate) fn add_death(
&self,
death: ListArc<DTRWrap<NodeDeath>, 1>,
guard: &mut Guard<'_, ProcessInner, SpinLockBackend>,
) {
assert!(
core::ptr::eq(self, &**death.node),
"attempt to add NodeDeath to the wrong death list"
);
self.inner.access_mut(guard).death_list.push_back(death);
}
pub(crate) fn inc_ref_done_locked(
self: &DArc<Node>,
_strong: bool,
owner_inner: &mut ProcessInner,
) -> Option<DLArc<Node>> {
let inner = self.inner.access_mut(owner_inner);
if inner.active_inc_refs == 0 {
binder_debug!(UserError, "inc_ref_done called when no active inc_refs");
return None;
}
inner.active_inc_refs -= 1;
if inner.active_inc_refs == 0 {
// Having active inc_refs can inhibit dropping of ref-counts. Calculate whether we
// would send a refcount decrement, and if so, tell the caller to schedule us.
let strong = inner.strong.count > 0;
let has_strong = inner.strong.has_count;
let weak = strong || inner.weak.count > 0;
let has_weak = inner.weak.has_count;
let should_drop_weak = !weak && has_weak;
let should_drop_strong = !strong && has_strong;
// If we want to drop the ref-count again, tell the caller to schedule a work node for
// that.
let need_push = should_drop_weak || should_drop_strong;
if need_push && inner.delivery_state.should_normal_push() {
let list_arc = ListArc::try_from_arc(self.clone()).ok().unwrap();
inner.delivery_state.did_normal_push();
Some(list_arc)
} else {
None
}
} else {
None
}
}
pub(crate) fn update_refcount_locked(
self: &DArc<Node>,
inc: bool,
strong: bool,
count: usize,
owner_inner: &mut ProcessInner,
) -> Option<DLArc<Node>> {
let is_dead = owner_inner.is_dead;
let inner = self.inner.access_mut(owner_inner);
// Get a reference to the state we'll update.
let state = if strong {
&mut inner.strong
} else {
&mut inner.weak
};
// Update the count and determine whether we need to push work.
let need_push = if inc {
state.count += count;
// TODO: This method shouldn't be used for zero-to-one increments.
!is_dead && !state.has_count
} else {
if state.count < count {
pr_err!("Failure: refcount underflow!");
return None;
}
state.count -= count;
!is_dead && state.count == 0 && state.has_count
};
if need_push && inner.delivery_state.should_normal_push() {
let list_arc = ListArc::try_from_arc(self.clone()).ok().unwrap();
inner.delivery_state.did_normal_push();
Some(list_arc)
} else {
None
}
}
pub(crate) fn incr_refcount_allow_zero2one(
self: &DArc<Self>,
strong: bool,
owner_inner: &mut ProcessInner,
) -> Result<Option<DLArc<Node>>, CouldNotDeliverCriticalIncrement> {
let is_dead = owner_inner.is_dead;
let inner = self.inner.access_mut(owner_inner);
// Get a reference to the state we'll update.
let state = if strong {
&mut inner.strong
} else {
&mut inner.weak
};
// Update the count and determine whether we need to push work.
state.count += 1;
if is_dead || state.has_count {
return Ok(None);
}
// Userspace needs to be notified of this.
if !strong && inner.delivery_state.should_push_weak_zero2one() {
assert!(inner.delivery_state.can_push_weak_zero2one_normally());
let list_arc = ListArc::try_from_arc(self.clone()).ok().unwrap();
inner.delivery_state.did_push_weak_zero2one();
Ok(Some(list_arc))
} else if strong && inner.delivery_state.should_push_strong_zero2one() {
if inner.delivery_state.can_push_strong_zero2one_normally() {
let list_arc = ListArc::try_from_arc(self.clone()).ok().unwrap();
inner.delivery_state.did_push_strong_zero2one();
Ok(Some(list_arc))
} else {
state.count -= 1;
Err(CouldNotDeliverCriticalIncrement)
}
} else {
// Work is already pushed, and we don't need to push again.
Ok(None)
}
}
pub(crate) fn incr_refcount_allow_zero2one_with_wrapper(
self: &DArc<Self>,
strong: bool,
wrapper: CritIncrWrapper,
owner_inner: &mut ProcessInner,
) -> Option<DLArc<dyn DeliverToRead>> {
match self.incr_refcount_allow_zero2one(strong, owner_inner) {
Ok(Some(node)) => Some(node as DLArc<dyn DeliverToRead>),
Ok(None) => None,
Err(CouldNotDeliverCriticalIncrement) => {
assert!(strong);
let inner = self.inner.access_mut(owner_inner);
inner.strong.count += 1;
inner.delivery_state.did_push_strong_zero2one_wrapper();
Some(wrapper.init(self.clone()))
}
}
}
pub(crate) fn update_refcount(self: &DArc<Self>, inc: bool, count: usize, strong: bool) {
self.owner
.inner
.lock()
.update_node_refcount(self, inc, strong, count, None);
}
pub(crate) fn populate_counts(
&self,
out: &mut BinderNodeInfoForRef,
guard: &Guard<'_, ProcessInner, SpinLockBackend>,
) {
let inner = self.inner.access(guard);
out.strong_count = inner.strong.count as u32;
out.weak_count = inner.weak.count as u32;
}
pub(crate) fn populate_debug_info(
&self,
out: &mut BinderNodeDebugInfo,
guard: &Guard<'_, ProcessInner, SpinLockBackend>,
) {
out.ptr = self.ptr as uapi::binder_uintptr_t;
out.cookie = self.cookie as uapi::binder_uintptr_t;
let inner = self.inner.access(guard);
if inner.strong.has_count {
out.has_strong_ref = 1;
}
if inner.weak.has_count {
out.has_weak_ref = 1;
}
}
pub(crate) fn force_has_count(&self, guard: &mut Guard<'_, ProcessInner, SpinLockBackend>) {
let inner = self.inner.access_mut(guard);
inner.strong.has_count = true;
inner.weak.has_count = true;
}
fn write(&self, writer: &mut BinderReturnWriter<'_>, code: u32) -> Result {
writer.write_code(code)?;
writer.write_payload(&self.ptr)?;
writer.write_payload(&self.cookie)?;
Ok(())
}
pub(crate) fn submit_oneway(
&self,
transaction: DLArc<Transaction>,
guard: &mut Guard<'_, ProcessInner, SpinLockBackend>,
) -> Result<(), (BinderError, DLArc<dyn DeliverToRead>)> {
if guard.is_dead {
return Err((BinderError::new_dead(), transaction));
}
let inner = self.inner.access_mut(guard);
if inner.has_oneway_transaction {
inner.oneway_todo.push_back(transaction);
} else {
inner.has_oneway_transaction = true;
guard.push_work(&self.owner, transaction)?;
}
Ok(())
}
pub(crate) fn release(&self) {
let mut guard = self.owner.inner.lock();
while let Some(work) = self.inner.access_mut(&mut guard).oneway_todo.pop_front() {
drop(guard);
work.into_arc().cancel();
guard = self.owner.inner.lock();
}
while let Some(death) = self.inner.access_mut(&mut guard).death_list.pop_front() {
drop(guard);
death.into_arc().set_dead();
guard = self.owner.inner.lock();
}
}
pub(crate) fn pending_oneway_finished(&self) {
let mut guard = self.owner.inner.lock();
if guard.is_dead {
// Cleanup will happen in `Process::deferred_release`.
return;
}
let inner = self.inner.access_mut(&mut guard);
let transaction = inner.oneway_todo.pop_front();
inner.has_oneway_transaction = transaction.is_some();
if let Some(transaction) = transaction {
match guard.push_work(&self.owner, transaction) {
Ok(()) => {}
Err((_err, work)) => {
// Process is dead.
// This shouldn't happen due to the `is_dead` check, but if it does, just drop
// the transaction and return.
drop(guard);
drop(work);
}
}
}
}
/// Finds an outdated transaction that the given transaction can replace.
///
/// If one is found, it is removed from the list and returned.
pub(crate) fn take_outdated_transaction(
&self,
new: &Transaction,
guard: &mut Guard<'_, ProcessInner, SpinLockBackend>,
) -> Option<DLArc<Transaction>> {
let inner = self.inner.access_mut(guard);
let mut cursor = inner.oneway_todo.cursor_front();
while let Some(next) = cursor.peek_next() {
if new.can_replace(&next) {
return Some(next.remove());
}
cursor.move_next();
}
None
}
/// This is split into a separate function since it's called by both `Node::do_work` and
/// `NodeWrapper::do_work`.
fn do_work_locked(
&self,
writer: &mut BinderReturnWriter<'_>,
mut guard: Guard<'_, ProcessInner, SpinLockBackend>,
) -> Result<bool> {
let inner = self.inner.access_mut(&mut guard);
let strong = inner.strong.count > 0;
let has_strong = inner.strong.has_count;
let weak = strong || inner.weak.count > 0;
let has_weak = inner.weak.has_count;
if weak && !has_weak {
inner.weak.has_count = true;
inner.active_inc_refs += 1;
}
if strong && !has_strong {
inner.strong.has_count = true;
inner.active_inc_refs += 1;
}
let no_active_inc_refs = inner.active_inc_refs == 0;
let should_drop_weak = no_active_inc_refs && (!weak && has_weak);
let should_drop_strong = no_active_inc_refs && (!strong && has_strong);
if should_drop_weak {
inner.weak.has_count = false;
}
if should_drop_strong {
inner.strong.has_count = false;
}
if no_active_inc_refs && !weak {
// Remove the node if there are no references to it.
guard.remove_node(self.ptr);
}
drop(guard);
if weak && !has_weak {
self.write(writer, BR_INCREFS)?;
}
if strong && !has_strong {
self.write(writer, BR_ACQUIRE)?;
}
if should_drop_strong {
self.write(writer, BR_RELEASE)?;
}
if should_drop_weak {
self.write(writer, BR_DECREFS)?;
}
Ok(true)
}
pub(crate) fn add_freeze_listener(
&self,
process: &Arc<Process>,
// If the vector needs to be resized, it's done via this argument.
vec_alloc: &mut KVVec<Arc<Process>>,
) -> Result<Result<(), usize>> {
let mut guard = self.owner.inner.lock();
// Do not check for `guard.dead`. The `dead` flag that matters here is the owner of the
// listener, not the target.
let inner = self.inner.access_mut(&mut guard);
let len = inner.freeze_list.len();
if len == inner.freeze_list.capacity() {
if len >= vec_alloc.capacity() {
// Request the caller to reallocate.
return Ok(Err((1 + len).next_power_of_two()));
}
mem::swap(&mut inner.freeze_list, vec_alloc);
for elem in vec_alloc.drain_all() {
inner.freeze_list.push_within_capacity(elem)?;
}
}
inner.freeze_list.push_within_capacity(process.clone())?;
Ok(Ok(()))
}
pub(crate) fn remove_freeze_listener(&self, p: &Process) -> KVVec<Arc<Process>> {
let mut guard = self.owner.inner.lock();
let inner = self.inner.access_mut(&mut guard);
let len = inner.freeze_list.len();
inner
.freeze_list
.retain(|proc| !core::ptr::eq::<Process>(&**proc, p));
if len == inner.freeze_list.len() {
pr_warn!(
"Could not remove freeze listener for {}\n",
p.pid_in_current_ns()
);
}
// If the vector is empty it needs to be freed. However, we can't free it here because that
// might sleep, so return it to the caller.
if inner.freeze_list.is_empty() {
return mem::take(&mut inner.freeze_list);
}
KVVec::new()
}
pub(crate) fn freeze_list<'a>(&'a self, guard: &'a ProcessInner) -> &'a [Arc<Process>] {
&self.inner.access(guard).freeze_list
}
}
impl DeliverToRead for Node {
fn do_work(
self: DArc<Self>,
_thread: &Thread,
writer: &mut BinderReturnWriter<'_>,
) -> Result<bool> {
let mut owner_inner = self.owner.inner.lock();
let inner = self.inner.access_mut(&mut owner_inner);
assert!(inner.delivery_state.has_pushed_node);
if inner.delivery_state.has_pushed_wrapper {
// If the wrapper is scheduled, then we are either a normal push or weak zero2one
// increment, and the wrapper is a strong zero2one increment, so the wrapper always
// takes precedence over us.
assert!(inner.delivery_state.has_strong_zero2one);
inner.delivery_state.has_pushed_node = false;
inner.delivery_state.has_weak_zero2one = false;
return Ok(true);
}
inner.delivery_state.has_pushed_node = false;
inner.delivery_state.has_weak_zero2one = false;
inner.delivery_state.has_strong_zero2one = false;
self.do_work_locked(writer, owner_inner)
}
fn cancel(self: DArc<Self>) {}
fn should_sync_wakeup(&self) -> bool {
false
}
#[inline(never)]
fn debug_print(&self, m: &SeqFile, prefix: &str, _tprefix: &str) -> Result<()> {
seq_print!(
m,
"{}node work {}: u{:016x} c{:016x}\n",
prefix,
self.debug_id,
self.ptr,
self.cookie,
);
Ok(())
}
}
/// Represents something that holds one or more ref-counts to a `Node`.
///
/// Whenever process A holds a refcount to a node owned by a different process B, then process A
/// will store a `NodeRef` that refers to the `Node` in process B. When process A releases the
/// refcount, we destroy the NodeRef, which decrements the ref-count in process A.
///
/// This type is also used for some other cases. For example, a transaction allocation holds a
/// refcount on the target node, and this is implemented by storing a `NodeRef` in the allocation
/// so that the destructor of the allocation will drop a refcount of the `Node`.
pub(crate) struct NodeRef {
pub(crate) node: DArc<Node>,
/// How many times does this NodeRef hold a refcount on the Node?
strong_node_count: usize,
weak_node_count: usize,
/// How many times does userspace hold a refcount on this NodeRef?
strong_count: usize,
weak_count: usize,
}
impl NodeRef {
pub(crate) fn new(node: DArc<Node>, strong_count: usize, weak_count: usize) -> Self {
Self {
node,
strong_node_count: strong_count,
weak_node_count: weak_count,
strong_count,
weak_count,
}
}
pub(crate) fn absorb(&mut self, mut other: Self) {
assert!(
Arc::ptr_eq(&self.node, &other.node),
"absorb called with differing nodes"
);
self.strong_node_count += other.strong_node_count;
self.weak_node_count += other.weak_node_count;
self.strong_count += other.strong_count;
self.weak_count += other.weak_count;
other.strong_count = 0;
other.weak_count = 0;
other.strong_node_count = 0;
other.weak_node_count = 0;
if self.strong_node_count >= 2 || self.weak_node_count >= 2 {
let mut guard = self.node.owner.inner.lock();
let inner = self.node.inner.access_mut(&mut guard);
if self.strong_node_count >= 2 {
inner.strong.count -= self.strong_node_count - 1;
self.strong_node_count = 1;
assert_ne!(inner.strong.count, 0);
}
if self.weak_node_count >= 2 {
inner.weak.count -= self.weak_node_count - 1;
self.weak_node_count = 1;
assert_ne!(inner.weak.count, 0);
}
}
}
pub(crate) fn get_count(&self) -> (usize, usize) {
(self.strong_count, self.weak_count)
}
pub(crate) fn clone(&self, strong: bool) -> Result<NodeRef> {
if strong && self.strong_count == 0 {
binder_debug!(UserError, "tried to use weak ref as strong ref");
return Err(EINVAL);
}
Ok(self
.node
.owner
.inner
.lock()
.new_node_ref(self.node.clone(), strong, None))
}
/// Updates (increments or decrements) the number of references held against the node. If the
/// count being updated transitions from 0 to 1 or from 1 to 0, the node is notified by having
/// its `update_refcount` function called.
///
/// Returns whether `self` should be removed (when both counts are zero).
pub(crate) fn update(&mut self, inc: bool, strong: bool) -> bool {
if strong && self.strong_count == 0 {
return false;
}
let (count, node_count, other_count) = if strong {
(
&mut self.strong_count,
&mut self.strong_node_count,
self.weak_count,
)
} else {
(
&mut self.weak_count,
&mut self.weak_node_count,
self.strong_count,
)
};
if inc {
if *count == 0 {
*node_count = 1;
self.node.update_refcount(true, 1, strong);
}
*count += 1;
} else {
if *count == 0 {
binder_debug!(
UserError,
"performed invalid {} decrement on ref",
if strong { "strong" } else { "weak" }
);
return false;
}
*count -= 1;
if *count == 0 {
self.node.update_refcount(false, *node_count, strong);
*node_count = 0;
return other_count == 0;
}
}
false
}
}
impl Drop for NodeRef {
// This destructor is called conditionally from `Allocation::drop`. That branch is often
// mispredicted. Inlining this method call reduces the cost of those branch mispredictions.
#[inline(always)]
fn drop(&mut self) {
if self.strong_node_count > 0 {
self.node
.update_refcount(false, self.strong_node_count, true);
}
if self.weak_node_count > 0 {
self.node
.update_refcount(false, self.weak_node_count, false);
}
}
}
struct NodeDeathInner {
dead: bool,
cleared: bool,
notification_done: bool,
/// Indicates whether the normal flow was interrupted by removing the handle. In this case, we
/// need behave as if the death notification didn't exist (i.e., we don't deliver anything to
/// the user.
aborted: bool,
}
/// Used to deliver notifications when a process dies.
///
/// A process can request to be notified when a process dies using `BC_REQUEST_DEATH_NOTIFICATION`.
/// This will make the driver send a `BR_DEAD_BINDER` to userspace when the process dies (or
/// immediately if it is already dead). Userspace is supposed to respond with `BC_DEAD_BINDER_DONE`
/// once it has processed the notification.
///
/// Userspace can unregister from death notifications using the `BC_CLEAR_DEATH_NOTIFICATION`
/// command. In this case, the kernel will respond with `BR_CLEAR_DEATH_NOTIFICATION_DONE` once the
/// notification has been removed. Note that if the remote process dies before the kernel has
/// responded with `BR_CLEAR_DEATH_NOTIFICATION_DONE`, then the kernel will still send a
/// `BR_DEAD_BINDER`, which userspace must be able to process. In this case, the kernel will wait
/// for the `BC_DEAD_BINDER_DONE` command before it sends `BR_CLEAR_DEATH_NOTIFICATION_DONE`.
///
/// Note that even if the kernel sends a `BR_DEAD_BINDER`, this does not remove the death
/// notification. Userspace must still remove it manually using `BC_CLEAR_DEATH_NOTIFICATION`.
///
/// If a process uses `BC_RELEASE` to destroy its last refcount on a node that has an active death
/// registration, then the death registration is immediately deleted (we implement this using the
/// `aborted` field). However, userspace is not supposed to delete a `NodeRef` without first
/// deregistering death notifications, so this codepath is not executed under normal circumstances.
#[pin_data]
pub(crate) struct NodeDeath {
node: DArc<Node>,
process: Arc<Process>,
pub(crate) cookie: u64,
#[pin]
links_track: AtomicTracker<0>,
/// Used by the owner `Node` to store a list of registered death notifications.
///
/// # Invariants
///
/// Only ever used with the `death_list` list of `self.node`.
#[pin]
death_links: ListLinks<1>,
/// Used by the process to keep track of the death notifications for which we have sent a
/// `BR_DEAD_BINDER` but not yet received a `BC_DEAD_BINDER_DONE`.
///
/// # Invariants
///
/// Only ever used with the `delivered_deaths` list of `self.process`.
#[pin]
delivered_links: ListLinks<2>,
#[pin]
delivered_links_track: AtomicTracker<2>,
#[pin]
inner: SpinLock<NodeDeathInner>,
}
impl NodeDeath {
/// Constructs a new node death notification object.
pub(crate) fn new(
node: DArc<Node>,
process: Arc<Process>,
cookie: u64,
) -> impl PinInit<DTRWrap<Self>> {
DTRWrap::new(pin_init!(
Self {
node,
process,
cookie,
links_track <- AtomicTracker::new(),
death_links <- ListLinks::new(),
delivered_links <- ListLinks::new(),
delivered_links_track <- AtomicTracker::new(),
inner <- kernel::new_spinlock!(NodeDeathInner {
dead: false,
cleared: false,
notification_done: false,
aborted: false,
}, "NodeDeath::inner"),
}
))
}
/// Sets the cleared flag to `true`.
///
/// It removes `self` from the node's death notification list if needed.
///
/// Returns whether it needs to be queued.
pub(crate) fn set_cleared(self: &DArc<Self>, abort: bool) -> bool {
let (needs_removal, needs_queueing) = {
// Update state and determine if we need to queue a work item. We only need to do it
// when the node is not dead or if the user already completed the death notification.
let mut inner = self.inner.lock();
if abort {
inner.aborted = true;
}
if inner.cleared {
// Already cleared.
return false;
}
inner.cleared = true;
(!inner.dead, !inner.dead || inner.notification_done)
};
// Remove death notification from node.
if needs_removal {
let mut owner_inner = self.node.owner.inner.lock();
let node_inner = self.node.inner.access_mut(&mut owner_inner);
// SAFETY: A `NodeDeath` is never inserted into the death list of any node other than
// its owner, so it is either in this death list or in no death list.
unsafe { node_inner.death_list.remove(self) };
}
needs_queueing
}
/// Sets the 'notification done' flag to `true`.
pub(crate) fn set_notification_done(self: DArc<Self>, thread: &Thread) {
let needs_queueing = {
let mut inner = self.inner.lock();
inner.notification_done = true;
inner.cleared
};
if needs_queueing {
if let Some(death) = ListArc::try_from_arc_or_drop(self) {
let _ = thread.push_work_if_looper(death);
}
}
}
/// Sets the 'dead' flag to `true` and queues work item if needed.
pub(crate) fn set_dead(self: DArc<Self>) {
let needs_queueing = {
let mut inner = self.inner.lock();
if inner.cleared {
false
} else {
inner.dead = true;
true
}
};
if needs_queueing {
// Push the death notification to the target process. There is nothing else to do if
// it's already dead.
if let Some(death) = ListArc::try_from_arc_or_drop(self) {
let process = death.process.clone();
let _ = process.push_work(death);
}
}
}
}
kernel::list::impl_list_arc_safe! {
impl ListArcSafe<0> for NodeDeath {
tracked_by links_track: AtomicTracker;
}
}
kernel::list::impl_list_arc_safe! {
impl ListArcSafe<1> for DTRWrap<NodeDeath> { untracked; }
}
kernel::list::impl_list_item! {
impl ListItem<1> for DTRWrap<NodeDeath> {
using ListLinks { self.wrapped.death_links };
}
}
kernel::list::impl_list_arc_safe! {
impl ListArcSafe<2> for DTRWrap<NodeDeath> {
tracked_by wrapped: NodeDeath;
}
}
kernel::list::impl_list_arc_safe! {
impl ListArcSafe<2> for NodeDeath {
tracked_by delivered_links_track: AtomicTracker<2>;
}
}
kernel::list::impl_list_item! {
impl ListItem<2> for DTRWrap<NodeDeath> {
using ListLinks { self.wrapped.delivered_links };
}
}
impl DeliverToRead for NodeDeath {
fn do_work(
self: DArc<Self>,
_thread: &Thread,
writer: &mut BinderReturnWriter<'_>,
) -> Result<bool> {
let done = {
let inner = self.inner.lock();
if inner.aborted {
return Ok(true);
}
inner.cleared && (!inner.dead || inner.notification_done)
};
let cookie = self.cookie;
let cmd = if done {
BR_CLEAR_DEATH_NOTIFICATION_DONE
} else {
let process = self.process.clone();
let mut process_inner = process.inner.lock();
let inner = self.inner.lock();
if inner.aborted {
return Ok(true);
}
// We're still holding the inner lock, so it cannot be aborted while we insert it into
// the delivered list.
process_inner.death_delivered(self.clone());
binder_debug!(
DeathNotification,
"sending death notification, cookie {:016x}",
cookie
);
BR_DEAD_BINDER
};
writer.write_code(cmd)?;
writer.write_payload(&cookie)?;
// DEAD_BINDER notifications can cause transactions, so stop processing work items when we
// get to a death notification.
Ok(cmd != BR_DEAD_BINDER)
}
fn cancel(self: DArc<Self>) {
binder_debug!(
pid = self.process.task.pid(),
DeadTransaction,
"undelivered death notification, {:016x}",
self.cookie
);
}
fn should_sync_wakeup(&self) -> bool {
false
}
#[inline(never)]
fn debug_print(&self, m: &SeqFile, prefix: &str, _tprefix: &str) -> Result<()> {
let inner = self.inner.lock();
let dead_binder = inner.dead && !inner.notification_done;
if dead_binder {
if inner.cleared {
seq_print!(m, "{}has cleared dead binder\n", prefix);
} else {
seq_print!(m, "{}has dead binder\n", prefix);
}
} else {
seq_print!(m, "{}has cleared death notification\n", prefix);
}
Ok(())
}
}
