/* SPDX-License-Identifier: GPL-2.0-only */
/* Copyright (C) 2025 Intel Corporation */
#ifndef __LIBETH_XDP_H
#define __LIBETH_XDP_H
#include <linux/bpf_trace.h>
#include <linux/unroll.h>
#include <net/libeth/rx.h>
#include <net/libeth/tx.h>
#include <net/xsk_buff_pool.h>
/*
* Defined as bits to be able to use them as a mask on Rx.
* Also used as internal return values on Tx.
*/
enum {
LIBETH_XDP_PASS = 0U,
LIBETH_XDP_DROP = BIT(0),
LIBETH_XDP_ABORTED = BIT(1),
LIBETH_XDP_TX = BIT(2),
LIBETH_XDP_REDIRECT = BIT(3),
};
/*
* &xdp_buff_xsk is the largest structure &libeth_xdp_buff gets casted to,
* pick maximum pointer-compatible alignment.
*/
#define __LIBETH_XDP_BUFF_ALIGN \
(IS_ALIGNED(sizeof(struct xdp_buff_xsk), 16) ? 16 : \
IS_ALIGNED(sizeof(struct xdp_buff_xsk), 8) ? 8 : \
sizeof(long))
/**
* struct libeth_xdp_buff - libeth extension over &xdp_buff
* @base: main &xdp_buff
* @data: shortcut for @base.data
* @desc: RQ descriptor containing metadata for this buffer
* @priv: driver-private scratchspace
*
* The main reason for this is to have a pointer to the descriptor to be able
* to quickly get frame metadata from xdpmo and driver buff-to-xdp callbacks
* (as well as bigger alignment).
* Pointer/layout-compatible with &xdp_buff and &xdp_buff_xsk.
*/
struct libeth_xdp_buff {
union {
struct xdp_buff base;
void *data;
};
const void *desc;
unsigned long priv[]
__aligned(__LIBETH_XDP_BUFF_ALIGN);
} __aligned(__LIBETH_XDP_BUFF_ALIGN);
static_assert(offsetof(struct libeth_xdp_buff, data) ==
offsetof(struct xdp_buff_xsk, xdp.data));
static_assert(offsetof(struct libeth_xdp_buff, desc) ==
offsetof(struct xdp_buff_xsk, cb));
static_assert(IS_ALIGNED(sizeof(struct xdp_buff_xsk),
__alignof(struct libeth_xdp_buff)));
/**
* __LIBETH_XDP_ONSTACK_BUFF - declare a &libeth_xdp_buff on the stack
* @name: name of the variable to declare
* @...: sizeof() of the driver-private data
*/
#define __LIBETH_XDP_ONSTACK_BUFF(name, ...) \
___LIBETH_XDP_ONSTACK_BUFF(name, ##__VA_ARGS__)
/**
* LIBETH_XDP_ONSTACK_BUFF - declare a &libeth_xdp_buff on the stack
* @name: name of the variable to declare
* @...: type or variable name of the driver-private data
*/
#define LIBETH_XDP_ONSTACK_BUFF(name, ...) \
__LIBETH_XDP_ONSTACK_BUFF(name, __libeth_xdp_priv_sz(__VA_ARGS__))
#define ___LIBETH_XDP_ONSTACK_BUFF(name, ...) \
__DEFINE_FLEX(struct libeth_xdp_buff, name, priv, \
LIBETH_XDP_PRIV_SZ(__VA_ARGS__ + 0), \
__uninitialized); \
LIBETH_XDP_ASSERT_PRIV_SZ(__VA_ARGS__ + 0)
#define __libeth_xdp_priv_sz(...) \
CONCATENATE(__libeth_xdp_psz, COUNT_ARGS(__VA_ARGS__))(__VA_ARGS__)
#define __libeth_xdp_psz0(...)
#define __libeth_xdp_psz1(...) sizeof(__VA_ARGS__)
#define LIBETH_XDP_PRIV_SZ(sz) \
(ALIGN(sz, __alignof(struct libeth_xdp_buff)) / sizeof(long))
/* Performs XSK_CHECK_PRIV_TYPE() */
#define LIBETH_XDP_ASSERT_PRIV_SZ(sz) \
static_assert(offsetofend(struct xdp_buff_xsk, cb) >= \
struct_size_t(struct libeth_xdp_buff, priv, \
LIBETH_XDP_PRIV_SZ(sz)))
/* XDPSQ sharing */
DECLARE_STATIC_KEY_FALSE(libeth_xdpsq_share);
/**
* libeth_xdpsq_num - calculate optimal number of XDPSQs for this device + sys
* @rxq: current number of active Rx queues
* @txq: current number of active Tx queues
* @max: maximum number of Tx queues
*
* Each RQ must have its own XDPSQ for XSk pairs, each CPU must have own XDPSQ
* for lockless sending (``XDP_TX``, .ndo_xdp_xmit()). Cap the maximum of these
* two with the number of SQs the device can have (minus used ones).
*
* Return: number of XDP Tx queues the device needs to use.
*/
static inline u32 libeth_xdpsq_num(u32 rxq, u32 txq, u32 max)
{
return min(max(nr_cpu_ids, rxq), max - txq);
}
/**
* libeth_xdpsq_shared - whether XDPSQs can be shared between several CPUs
* @num: number of active XDPSQs
*
* Return: true if there's no 1:1 XDPSQ/CPU association, false otherwise.
*/
static inline bool libeth_xdpsq_shared(u32 num)
{
return num < nr_cpu_ids;
}
/**
* libeth_xdpsq_id - get XDPSQ index corresponding to this CPU
* @num: number of active XDPSQs
*
* Helper for libeth_xdp routines, do not use in drivers directly.
*
* Return: XDPSQ index needs to be used on this CPU.
*/
static inline u32 libeth_xdpsq_id(u32 num)
{
u32 ret = raw_smp_processor_id();
if (static_branch_unlikely(&libeth_xdpsq_share) &&
libeth_xdpsq_shared(num))
ret %= num;
return ret;
}
void __libeth_xdpsq_get(struct libeth_xdpsq_lock *lock,
const struct net_device *dev);
void __libeth_xdpsq_put(struct libeth_xdpsq_lock *lock,
const struct net_device *dev);
/**
* libeth_xdpsq_get - initialize &libeth_xdpsq_lock
* @lock: lock to initialize
* @dev: netdev which this lock belongs to
* @share: whether XDPSQs can be shared
*
* Tracks the current XDPSQ association and enables the static lock
* if needed.
*/
static inline void libeth_xdpsq_get(struct libeth_xdpsq_lock *lock,
const struct net_device *dev,
bool share)
{
if (unlikely(share))
__libeth_xdpsq_get(lock, dev);
}
/**
* libeth_xdpsq_put - deinitialize &libeth_xdpsq_lock
* @lock: lock to deinitialize
* @dev: netdev which this lock belongs to
*
* Tracks the current XDPSQ association and disables the static lock
* if needed.
*/
static inline void libeth_xdpsq_put(struct libeth_xdpsq_lock *lock,
const struct net_device *dev)
{
if (static_branch_unlikely(&libeth_xdpsq_share) && lock->share)
__libeth_xdpsq_put(lock, dev);
}
void __libeth_xdpsq_lock(struct libeth_xdpsq_lock *lock);
void __libeth_xdpsq_unlock(struct libeth_xdpsq_lock *lock);
/**
* libeth_xdpsq_lock - grab &libeth_xdpsq_lock if needed
* @lock: lock to take
*
* Touches the underlying spinlock only if the static key is enabled
* and the queue itself is marked as shareable.
*/
static inline void libeth_xdpsq_lock(struct libeth_xdpsq_lock *lock)
{
if (static_branch_unlikely(&libeth_xdpsq_share) && lock->share)
__libeth_xdpsq_lock(lock);
}
/**
* libeth_xdpsq_unlock - free &libeth_xdpsq_lock if needed
* @lock: lock to free
*
* Touches the underlying spinlock only if the static key is enabled
* and the queue itself is marked as shareable.
*/
static inline void libeth_xdpsq_unlock(struct libeth_xdpsq_lock *lock)
{
if (static_branch_unlikely(&libeth_xdpsq_share) && lock->share)
__libeth_xdpsq_unlock(lock);
}
/* XDPSQ clean-up timers */
void libeth_xdpsq_init_timer(struct libeth_xdpsq_timer *timer, void *xdpsq,
struct libeth_xdpsq_lock *lock,
void (*poll)(struct work_struct *work));
/**
* libeth_xdpsq_deinit_timer - deinitialize &libeth_xdpsq_timer
* @timer: timer to deinitialize
*
* Flush and disable the underlying workqueue.
*/
static inline void libeth_xdpsq_deinit_timer(struct libeth_xdpsq_timer *timer)
{
cancel_delayed_work_sync(&timer->dwork);
}
/**
* libeth_xdpsq_queue_timer - run &libeth_xdpsq_timer
* @timer: timer to queue
*
* Should be called after the queue was filled and the transmission was run
* to complete the pending buffers if no further sending will be done in a
* second (-> lazy cleaning won't happen).
* If the timer was already run, it will be requeued back to one second
* timeout again.
*/
static inline void libeth_xdpsq_queue_timer(struct libeth_xdpsq_timer *timer)
{
mod_delayed_work_on(raw_smp_processor_id(), system_bh_highpri_wq,
&timer->dwork, HZ);
}
/**
* libeth_xdpsq_run_timer - wrapper to run a queue clean-up on a timer event
* @work: workqueue belonging to the corresponding timer
* @poll: driver-specific completion queue poll function
*
* Run the polling function on the locked queue and requeue the timer if
* there's more work to do.
* Designed to be used via LIBETH_XDP_DEFINE_TIMER() below.
*/
static __always_inline void
libeth_xdpsq_run_timer(struct work_struct *work,
u32 (*poll)(void *xdpsq, u32 budget))
{
struct libeth_xdpsq_timer *timer = container_of(work, typeof(*timer),
dwork.work);
libeth_xdpsq_lock(timer->lock);
if (poll(timer->xdpsq, U32_MAX))
libeth_xdpsq_queue_timer(timer);
libeth_xdpsq_unlock(timer->lock);
}
/* Common Tx bits */
/**
* enum - libeth_xdp internal Tx flags
* @LIBETH_XDP_TX_BULK: one bulk size at which it will be flushed to the queue
* @LIBETH_XDP_TX_BATCH: batch size for which the queue fill loop is unrolled
* @LIBETH_XDP_TX_DROP: indicates the send function must drop frames not sent
* @LIBETH_XDP_TX_NDO: whether the send function is called from .ndo_xdp_xmit()
* @LIBETH_XDP_TX_XSK: whether the function is called for ``XDP_TX`` for XSk
*/
enum {
LIBETH_XDP_TX_BULK = DEV_MAP_BULK_SIZE,
LIBETH_XDP_TX_BATCH = 8,
LIBETH_XDP_TX_DROP = BIT(0),
LIBETH_XDP_TX_NDO = BIT(1),
LIBETH_XDP_TX_XSK = BIT(2),
};
/**
* enum - &libeth_xdp_tx_frame and &libeth_xdp_tx_desc flags
* @LIBETH_XDP_TX_LEN: only for ``XDP_TX``, [15:0] of ::len_fl is actual length
* @LIBETH_XDP_TX_CSUM: for XSk xmit, enable checksum offload
* @LIBETH_XDP_TX_XSKMD: for XSk xmit, mask of the metadata bits
* @LIBETH_XDP_TX_FIRST: indicates the frag is the first one of the frame
* @LIBETH_XDP_TX_LAST: whether the frag is the last one of the frame
* @LIBETH_XDP_TX_MULTI: whether the frame contains several frags
* @LIBETH_XDP_TX_FLAGS: only for ``XDP_TX``, [31:16] of ::len_fl is flags
*/
enum {
LIBETH_XDP_TX_LEN = GENMASK(15, 0),
LIBETH_XDP_TX_CSUM = XDP_TXMD_FLAGS_CHECKSUM,
LIBETH_XDP_TX_XSKMD = LIBETH_XDP_TX_LEN,
LIBETH_XDP_TX_FIRST = BIT(16),
LIBETH_XDP_TX_LAST = BIT(17),
LIBETH_XDP_TX_MULTI = BIT(18),
LIBETH_XDP_TX_FLAGS = GENMASK(31, 16),
};
/**
* struct libeth_xdp_tx_frame - represents one XDP Tx element
* @data: frame start pointer for ``XDP_TX``
* @len_fl: ``XDP_TX``, combined flags [31:16] and len [15:0] field for speed
* @soff: ``XDP_TX``, offset from @data to the start of &skb_shared_info
* @frag: one (non-head) frag for ``XDP_TX``
* @xdpf: &xdp_frame for the head frag for .ndo_xdp_xmit()
* @dma: DMA address of the non-head frag for .ndo_xdp_xmit()
* @xsk: ``XDP_TX`` for XSk, XDP buffer for any frag
* @len: frag length for XSk ``XDP_TX`` and .ndo_xdp_xmit()
* @flags: Tx flags for the above
* @opts: combined @len + @flags for the above for speed
* @desc: XSk xmit descriptor for direct casting
*/
struct libeth_xdp_tx_frame {
union {
/* ``XDP_TX`` */
struct {
void *data;
u32 len_fl;
u32 soff;
};
/* ``XDP_TX`` frag */
skb_frag_t frag;
/* .ndo_xdp_xmit(), XSk ``XDP_TX`` */
struct {
union {
struct xdp_frame *xdpf;
dma_addr_t dma;
struct libeth_xdp_buff *xsk;
};
union {
struct {
u32 len;
u32 flags;
};
aligned_u64 opts;
};
};
/* XSk xmit */
struct xdp_desc desc;
};
} __aligned(sizeof(struct xdp_desc));
static_assert(offsetof(struct libeth_xdp_tx_frame, frag.len) ==
offsetof(struct libeth_xdp_tx_frame, len_fl));
static_assert(sizeof(struct libeth_xdp_tx_frame) == sizeof(struct xdp_desc));
/**
* struct libeth_xdp_tx_bulk - XDP Tx frame bulk for bulk sending
* @prog: corresponding active XDP program, %NULL for .ndo_xdp_xmit()
* @dev: &net_device which the frames are transmitted on
* @xdpsq: shortcut to the corresponding driver-specific XDPSQ structure
* @act_mask: Rx only, mask of all the XDP prog verdicts for that NAPI session
* @count: current number of frames in @bulk
* @bulk: array of queued frames for bulk Tx
*
* All XDP Tx operations except XSk xmit queue each frame to the bulk first
* and flush it when @count reaches the array end. Bulk is always placed on
* the stack for performance. One bulk element contains all the data necessary
* for sending a frame and then freeing it on completion.
* For XSk xmit, Tx descriptor array from &xsk_buff_pool is casted directly
* to &libeth_xdp_tx_frame as they are compatible and the bulk structure is
* not used.
*/
struct libeth_xdp_tx_bulk {
const struct bpf_prog *prog;
struct net_device *dev;
void *xdpsq;
u32 act_mask;
u32 count;
struct libeth_xdp_tx_frame bulk[LIBETH_XDP_TX_BULK];
} __aligned(sizeof(struct libeth_xdp_tx_frame));
/**
* LIBETH_XDP_ONSTACK_BULK - declare &libeth_xdp_tx_bulk on the stack
* @bq: name of the variable to declare
*
* Helper to declare a bulk on the stack with a compiler hint that it should
* not be initialized automatically (with `CONFIG_INIT_STACK_ALL_*`) for
* performance reasons.
*/
#define LIBETH_XDP_ONSTACK_BULK(bq) \
struct libeth_xdp_tx_bulk bq __uninitialized
/**
* struct libeth_xdpsq - abstraction for an XDPSQ
* @pool: XSk buffer pool for XSk ``XDP_TX`` and xmit
* @sqes: array of Tx buffers from the actual queue struct
* @descs: opaque pointer to the HW descriptor array
* @ntu: pointer to the next free descriptor index
* @count: number of descriptors on that queue
* @pending: pointer to the number of sent-not-completed descs on that queue
* @xdp_tx: pointer to the above, but only for non-XSk-xmit frames
* @lock: corresponding XDPSQ lock
*
* Abstraction for driver-independent implementation of Tx. Placed on the stack
* and filled by the driver before the transmission, so that the generic
* functions can access and modify driver-specific resources.
*/
struct libeth_xdpsq {
struct xsk_buff_pool *pool;
struct libeth_sqe *sqes;
void *descs;
u32 *ntu;
u32 count;
u32 *pending;
u32 *xdp_tx;
struct libeth_xdpsq_lock *lock;
};
/**
* struct libeth_xdp_tx_desc - abstraction for an XDP Tx descriptor
* @addr: DMA address of the frame
* @len: length of the frame
* @flags: XDP Tx flags
* @opts: combined @len + @flags for speed
*
* Filled by the generic functions and then passed to driver-specific functions
* to fill a HW Tx descriptor, always placed on the [function] stack.
*/
struct libeth_xdp_tx_desc {
dma_addr_t addr;
union {
struct {
u32 len;
u32 flags;
};
aligned_u64 opts;
};
} __aligned_largest;
/**
* libeth_xdp_ptr_to_priv - convert pointer to a libeth_xdp u64 priv
* @ptr: pointer to convert
*
* The main sending function passes private data as the largest scalar, u64.
* Use this helper when you want to pass a pointer there.
*/
#define libeth_xdp_ptr_to_priv(ptr) ({ \
typecheck_pointer(ptr); \
((u64)(uintptr_t)(ptr)); \
})
/**
* libeth_xdp_priv_to_ptr - convert libeth_xdp u64 priv to a pointer
* @priv: private data to convert
*
* The main sending function passes private data as the largest scalar, u64.
* Use this helper when your callback takes this u64 and you want to convert
* it back to a pointer.
*/
#define libeth_xdp_priv_to_ptr(priv) ({ \
static_assert(__same_type(priv, u64)); \
((const void *)(uintptr_t)(priv)); \
})
/*
* On 64-bit systems, assigning one u64 is faster than two u32s. When ::len
* occupies lowest 32 bits (LE), whole ::opts can be assigned directly instead.
*/
#ifdef __LITTLE_ENDIAN
#define __LIBETH_WORD_ACCESS 1
#endif
#ifdef __LIBETH_WORD_ACCESS
#define __libeth_xdp_tx_len(flen, ...) \
.opts = ((flen) | FIELD_PREP(GENMASK_ULL(63, 32), (__VA_ARGS__ + 0)))
#else
#define __libeth_xdp_tx_len(flen, ...) \
.len = (flen), .flags = (__VA_ARGS__ + 0)
#endif
/**
* libeth_xdp_tx_xmit_bulk - main XDP Tx function
* @bulk: array of frames to send
* @xdpsq: pointer to the driver-specific XDPSQ struct
* @n: number of frames to send
* @unroll: whether to unroll the queue filling loop for speed
* @priv: driver-specific private data
* @prep: callback for cleaning the queue and filling abstract &libeth_xdpsq
* @fill: internal callback for filling &libeth_sqe and &libeth_xdp_tx_desc
* @xmit: callback for filling a HW descriptor with the frame info
*
* Internal abstraction for placing @n XDP Tx frames on the HW XDPSQ. Used for
* all types of frames: ``XDP_TX``, .ndo_xdp_xmit(), XSk ``XDP_TX``, and XSk
* xmit.
* @prep must lock the queue as this function releases it at the end. @unroll
* greatly increases the object code size, but also greatly increases XSk xmit
* performance; for other types of frames, it's not enabled.
* The compilers inline all those onstack abstractions to direct data accesses.
*
* Return: number of frames actually placed on the queue, <= @n. The function
* can't fail, but can send less frames if there's no enough free descriptors
* available. The actual free space is returned by @prep from the driver.
*/
static __always_inline __nocfi_generic u32
libeth_xdp_tx_xmit_bulk(const struct libeth_xdp_tx_frame *bulk, void *xdpsq,
u32 n, bool unroll, u64 priv,
u32 (*prep)(void *xdpsq, struct libeth_xdpsq *sq),
struct libeth_xdp_tx_desc
(*fill)(struct libeth_xdp_tx_frame frm, u32 i,
const struct libeth_xdpsq *sq, u64 priv),
void (*xmit)(struct libeth_xdp_tx_desc desc, u32 i,
const struct libeth_xdpsq *sq, u64 priv))
{
struct libeth_xdpsq sq __uninitialized;
u32 this, batched, off = 0;
u32 ntu, i = 0;
n = min(n, prep(xdpsq, &sq));
if (unlikely(!n))
goto unlock;
ntu = *sq.ntu;
this = sq.count - ntu;
if (likely(this > n))
this = n;
again:
if (!unroll)
goto linear;
batched = ALIGN_DOWN(this, LIBETH_XDP_TX_BATCH);
for ( ; i < off + batched; i += LIBETH_XDP_TX_BATCH) {
u32 base = ntu + i - off;
unrolled_count(LIBETH_XDP_TX_BATCH)
for (u32 j = 0; j < LIBETH_XDP_TX_BATCH; j++)
xmit(fill(bulk[i + j], base + j, &sq, priv),
base + j, &sq, priv);
}
if (batched < this) {
linear:
for ( ; i < off + this; i++)
xmit(fill(bulk[i], ntu + i - off, &sq, priv),
ntu + i - off, &sq, priv);
}
ntu += this;
if (likely(ntu < sq.count))
goto out;
ntu = 0;
if (i < n) {
this = n - i;
off = i;
goto again;
}
out:
*sq.ntu = ntu;
*sq.pending += n;
if (sq.xdp_tx)
*sq.xdp_tx += n;
unlock:
libeth_xdpsq_unlock(sq.lock);
return n;
}
/* ``XDP_TX`` bulking */
void libeth_xdp_return_buff_slow(struct libeth_xdp_buff *xdp);
/**
* libeth_xdp_tx_queue_head - internal helper for queueing one ``XDP_TX`` head
* @bq: XDP Tx bulk to queue the head frag to
* @xdp: XDP buffer with the head to queue
*
* Return: false if it's the only frag of the frame, true if it's an S/G frame.
*/
static inline bool libeth_xdp_tx_queue_head(struct libeth_xdp_tx_bulk *bq,
const struct libeth_xdp_buff *xdp)
{
const struct xdp_buff *base = &xdp->base;
bq->bulk[bq->count++] = (typeof(*bq->bulk)){
.data = xdp->data,
.len_fl = (base->data_end - xdp->data) | LIBETH_XDP_TX_FIRST,
.soff = xdp_data_hard_end(base) - xdp->data,
};
if (!xdp_buff_has_frags(base))
return false;
bq->bulk[bq->count - 1].len_fl |= LIBETH_XDP_TX_MULTI;
return true;
}
/**
* libeth_xdp_tx_queue_frag - internal helper for queueing one ``XDP_TX`` frag
* @bq: XDP Tx bulk to queue the frag to
* @frag: frag to queue
*/
static inline void libeth_xdp_tx_queue_frag(struct libeth_xdp_tx_bulk *bq,
const skb_frag_t *frag)
{
bq->bulk[bq->count++].frag = *frag;
}
/**
* libeth_xdp_tx_queue_bulk - internal helper for queueing one ``XDP_TX`` frame
* @bq: XDP Tx bulk to queue the frame to
* @xdp: XDP buffer to queue
* @flush_bulk: driver callback to flush the bulk to the HW queue
*
* Return: true on success, false on flush error.
*/
static __always_inline bool
libeth_xdp_tx_queue_bulk(struct libeth_xdp_tx_bulk *bq,
struct libeth_xdp_buff *xdp,
bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq,
u32 flags))
{
const struct skb_shared_info *sinfo;
bool ret = true;
u32 nr_frags;
if (unlikely(bq->count == LIBETH_XDP_TX_BULK) &&
unlikely(!flush_bulk(bq, 0))) {
libeth_xdp_return_buff_slow(xdp);
return false;
}
if (!libeth_xdp_tx_queue_head(bq, xdp))
goto out;
sinfo = xdp_get_shared_info_from_buff(&xdp->base);
nr_frags = sinfo->nr_frags;
for (u32 i = 0; i < nr_frags; i++) {
if (unlikely(bq->count == LIBETH_XDP_TX_BULK) &&
unlikely(!flush_bulk(bq, 0))) {
ret = false;
break;
}
libeth_xdp_tx_queue_frag(bq, &sinfo->frags[i]);
}
out:
bq->bulk[bq->count - 1].len_fl |= LIBETH_XDP_TX_LAST;
xdp->data = NULL;
return ret;
}
/**
* libeth_xdp_tx_fill_stats - fill &libeth_sqe with ``XDP_TX`` frame stats
* @sqe: SQ element to fill
* @desc: libeth_xdp Tx descriptor
* @sinfo: &skb_shared_info for this frame
*
* Internal helper for filling an SQE with the frame stats, do not use in
* drivers. Fills the number of frags and bytes for this frame.
*/
#define libeth_xdp_tx_fill_stats(sqe, desc, sinfo) \
__libeth_xdp_tx_fill_stats(sqe, desc, sinfo, __UNIQUE_ID(sqe_), \
__UNIQUE_ID(desc_), __UNIQUE_ID(sinfo_))
#define __libeth_xdp_tx_fill_stats(sqe, desc, sinfo, ue, ud, us) do { \
const struct libeth_xdp_tx_desc *ud = (desc); \
const struct skb_shared_info *us; \
struct libeth_sqe *ue = (sqe); \
\
ue->nr_frags = 1; \
ue->bytes = ud->len; \
\
if (ud->flags & LIBETH_XDP_TX_MULTI) { \
us = (sinfo); \
ue->nr_frags += us->nr_frags; \
ue->bytes += us->xdp_frags_size; \
} \
} while (0)
/**
* libeth_xdp_tx_fill_buf - internal helper to fill one ``XDP_TX`` &libeth_sqe
* @frm: XDP Tx frame from the bulk
* @i: index on the HW queue
* @sq: XDPSQ abstraction for the queue
* @priv: private data
*
* Return: XDP Tx descriptor with the synced DMA and other info to pass to
* the driver callback.
*/
static inline struct libeth_xdp_tx_desc
libeth_xdp_tx_fill_buf(struct libeth_xdp_tx_frame frm, u32 i,
const struct libeth_xdpsq *sq, u64 priv)
{
struct libeth_xdp_tx_desc desc;
struct skb_shared_info *sinfo;
skb_frag_t *frag = &frm.frag;
struct libeth_sqe *sqe;
netmem_ref netmem;
if (frm.len_fl & LIBETH_XDP_TX_FIRST) {
sinfo = frm.data + frm.soff;
skb_frag_fill_netmem_desc(frag, virt_to_netmem(frm.data),
offset_in_page(frm.data),
frm.len_fl);
} else {
sinfo = NULL;
}
netmem = skb_frag_netmem(frag);
desc = (typeof(desc)){
.addr = page_pool_get_dma_addr_netmem(netmem) +
skb_frag_off(frag),
.len = skb_frag_size(frag) & LIBETH_XDP_TX_LEN,
.flags = skb_frag_size(frag) & LIBETH_XDP_TX_FLAGS,
};
dma_sync_single_for_device(__netmem_get_pp(netmem)->p.dev, desc.addr,
desc.len, DMA_BIDIRECTIONAL);
if (!sinfo)
return desc;
sqe = &sq->sqes[i];
sqe->type = LIBETH_SQE_XDP_TX;
sqe->sinfo = sinfo;
libeth_xdp_tx_fill_stats(sqe, &desc, sinfo);
return desc;
}
void libeth_xdp_tx_exception(struct libeth_xdp_tx_bulk *bq, u32 sent,
u32 flags);
/**
* __libeth_xdp_tx_flush_bulk - internal helper to flush one XDP Tx bulk
* @bq: bulk to flush
* @flags: XDP TX flags (.ndo_xdp_xmit(), XSk etc.)
* @prep: driver-specific callback to prepare the queue for sending
* @fill: libeth_xdp callback to fill &libeth_sqe and &libeth_xdp_tx_desc
* @xmit: driver callback to fill a HW descriptor
*
* Internal abstraction to create bulk flush functions for drivers. Used for
* everything except XSk xmit.
*
* Return: true if anything was sent, false otherwise.
*/
static __always_inline bool
__libeth_xdp_tx_flush_bulk(struct libeth_xdp_tx_bulk *bq, u32 flags,
u32 (*prep)(void *xdpsq, struct libeth_xdpsq *sq),
struct libeth_xdp_tx_desc
(*fill)(struct libeth_xdp_tx_frame frm, u32 i,
const struct libeth_xdpsq *sq, u64 priv),
void (*xmit)(struct libeth_xdp_tx_desc desc, u32 i,
const struct libeth_xdpsq *sq,
u64 priv))
{
u32 sent, drops;
int err = 0;
sent = libeth_xdp_tx_xmit_bulk(bq->bulk, bq->xdpsq,
min(bq->count, LIBETH_XDP_TX_BULK),
false, 0, prep, fill, xmit);
drops = bq->count - sent;
if (unlikely(drops)) {
libeth_xdp_tx_exception(bq, sent, flags);
err = -ENXIO;
} else {
bq->count = 0;
}
trace_xdp_bulk_tx(bq->dev, sent, drops, err);
return likely(sent);
}
/**
* libeth_xdp_tx_flush_bulk - wrapper to define flush of one ``XDP_TX`` bulk
* @bq: bulk to flush
* @flags: Tx flags, see above
* @prep: driver callback to prepare the queue
* @xmit: driver callback to fill a HW descriptor
*
* Use via LIBETH_XDP_DEFINE_FLUSH_TX() to define an ``XDP_TX`` driver
* callback.
*/
#define libeth_xdp_tx_flush_bulk(bq, flags, prep, xmit) \
__libeth_xdp_tx_flush_bulk(bq, flags, prep, libeth_xdp_tx_fill_buf, \
xmit)
/* .ndo_xdp_xmit() implementation */
/**
* libeth_xdp_xmit_init_bulk - internal helper to initialize bulk for XDP xmit
* @bq: bulk to initialize
* @dev: target &net_device
* @xdpsqs: array of driver-specific XDPSQ structs
* @num: number of active XDPSQs (the above array length)
*/
#define libeth_xdp_xmit_init_bulk(bq, dev, xdpsqs, num) \
__libeth_xdp_xmit_init_bulk(bq, dev, (xdpsqs)[libeth_xdpsq_id(num)])
static inline void __libeth_xdp_xmit_init_bulk(struct libeth_xdp_tx_bulk *bq,
struct net_device *dev,
void *xdpsq)
{
bq->dev = dev;
bq->xdpsq = xdpsq;
bq->count = 0;
}
/**
* libeth_xdp_xmit_frame_dma - internal helper to access DMA of an &xdp_frame
* @xf: pointer to the XDP frame
*
* There's no place in &libeth_xdp_tx_frame to store DMA address for an
* &xdp_frame head. The headroom is used then, the address is placed right
* after the frame struct, naturally aligned.
*
* Return: pointer to the DMA address to use.
*/
#define libeth_xdp_xmit_frame_dma(xf) \
_Generic((xf), \
const struct xdp_frame *: \
(const dma_addr_t *)__libeth_xdp_xmit_frame_dma(xf), \
struct xdp_frame *: \
(dma_addr_t *)__libeth_xdp_xmit_frame_dma(xf) \
)
static inline void *__libeth_xdp_xmit_frame_dma(const struct xdp_frame *xdpf)
{
void *addr = (void *)(xdpf + 1);
if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) &&
__alignof(*xdpf) < sizeof(dma_addr_t))
addr = PTR_ALIGN(addr, sizeof(dma_addr_t));
return addr;
}
/**
* libeth_xdp_xmit_queue_head - internal helper for queueing one XDP xmit head
* @bq: XDP Tx bulk to queue the head frag to
* @xdpf: XDP frame with the head to queue
* @dev: device to perform DMA mapping
*
* Return: ``LIBETH_XDP_DROP`` on DMA mapping error,
* ``LIBETH_XDP_PASS`` if it's the only frag in the frame,
* ``LIBETH_XDP_TX`` if it's an S/G frame.
*/
static inline u32 libeth_xdp_xmit_queue_head(struct libeth_xdp_tx_bulk *bq,
struct xdp_frame *xdpf,
struct device *dev)
{
dma_addr_t dma;
dma = dma_map_single(dev, xdpf->data, xdpf->len, DMA_TO_DEVICE);
if (dma_mapping_error(dev, dma))
return LIBETH_XDP_DROP;
*libeth_xdp_xmit_frame_dma(xdpf) = dma;
bq->bulk[bq->count++] = (typeof(*bq->bulk)){
.xdpf = xdpf,
__libeth_xdp_tx_len(xdpf->len, LIBETH_XDP_TX_FIRST),
};
if (!xdp_frame_has_frags(xdpf))
return LIBETH_XDP_PASS;
bq->bulk[bq->count - 1].flags |= LIBETH_XDP_TX_MULTI;
return LIBETH_XDP_TX;
}
/**
* libeth_xdp_xmit_queue_frag - internal helper for queueing one XDP xmit frag
* @bq: XDP Tx bulk to queue the frag to
* @frag: frag to queue
* @dev: device to perform DMA mapping
*
* Return: true on success, false on DMA mapping error.
*/
static inline bool libeth_xdp_xmit_queue_frag(struct libeth_xdp_tx_bulk *bq,
const skb_frag_t *frag,
struct device *dev)
{
dma_addr_t dma;
dma = skb_frag_dma_map(dev, frag);
if (dma_mapping_error(dev, dma))
return false;
bq->bulk[bq->count++] = (typeof(*bq->bulk)){
.dma = dma,
__libeth_xdp_tx_len(skb_frag_size(frag)),
};
return true;
}
/**
* libeth_xdp_xmit_queue_bulk - internal helper for queueing one XDP xmit frame
* @bq: XDP Tx bulk to queue the frame to
* @xdpf: XDP frame to queue
* @flush_bulk: driver callback to flush the bulk to the HW queue
*
* Return: ``LIBETH_XDP_TX`` on success,
* ``LIBETH_XDP_DROP`` if the frame should be dropped by the stack,
* ``LIBETH_XDP_ABORTED`` if the frame will be dropped by libeth_xdp.
*/
static __always_inline u32
libeth_xdp_xmit_queue_bulk(struct libeth_xdp_tx_bulk *bq,
struct xdp_frame *xdpf,
bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq,
u32 flags))
{
u32 head, nr_frags, i, ret = LIBETH_XDP_TX;
struct device *dev = bq->dev->dev.parent;
const struct skb_shared_info *sinfo;
if (unlikely(bq->count == LIBETH_XDP_TX_BULK) &&
unlikely(!flush_bulk(bq, LIBETH_XDP_TX_NDO)))
return LIBETH_XDP_DROP;
head = libeth_xdp_xmit_queue_head(bq, xdpf, dev);
if (head == LIBETH_XDP_PASS)
goto out;
else if (head == LIBETH_XDP_DROP)
return LIBETH_XDP_DROP;
sinfo = xdp_get_shared_info_from_frame(xdpf);
nr_frags = sinfo->nr_frags;
for (i = 0; i < nr_frags; i++) {
if (unlikely(bq->count == LIBETH_XDP_TX_BULK) &&
unlikely(!flush_bulk(bq, LIBETH_XDP_TX_NDO)))
break;
if (!libeth_xdp_xmit_queue_frag(bq, &sinfo->frags[i], dev))
break;
}
if (unlikely(i < nr_frags))
ret = LIBETH_XDP_ABORTED;
out:
bq->bulk[bq->count - 1].flags |= LIBETH_XDP_TX_LAST;
return ret;
}
/**
* libeth_xdp_xmit_fill_buf - internal helper to fill one XDP xmit &libeth_sqe
* @frm: XDP Tx frame from the bulk
* @i: index on the HW queue
* @sq: XDPSQ abstraction for the queue
* @priv: private data
*
* Return: XDP Tx descriptor with the mapped DMA and other info to pass to
* the driver callback.
*/
static inline struct libeth_xdp_tx_desc
libeth_xdp_xmit_fill_buf(struct libeth_xdp_tx_frame frm, u32 i,
const struct libeth_xdpsq *sq, u64 priv)
{
struct libeth_xdp_tx_desc desc;
struct libeth_sqe *sqe;
struct xdp_frame *xdpf;
if (frm.flags & LIBETH_XDP_TX_FIRST) {
xdpf = frm.xdpf;
desc.addr = *libeth_xdp_xmit_frame_dma(xdpf);
} else {
xdpf = NULL;
desc.addr = frm.dma;
}
desc.opts = frm.opts;
sqe = &sq->sqes[i];
dma_unmap_addr_set(sqe, dma, desc.addr);
dma_unmap_len_set(sqe, len, desc.len);
if (!xdpf) {
sqe->type = LIBETH_SQE_XDP_XMIT_FRAG;
return desc;
}
sqe->type = LIBETH_SQE_XDP_XMIT;
sqe->xdpf = xdpf;
libeth_xdp_tx_fill_stats(sqe, &desc,
xdp_get_shared_info_from_frame(xdpf));
return desc;
}
/**
* libeth_xdp_xmit_flush_bulk - wrapper to define flush of one XDP xmit bulk
* @bq: bulk to flush
* @flags: Tx flags, see __libeth_xdp_tx_flush_bulk()
* @prep: driver callback to prepare the queue
* @xmit: driver callback to fill a HW descriptor
*
* Use via LIBETH_XDP_DEFINE_FLUSH_XMIT() to define an XDP xmit driver
* callback.
*/
#define libeth_xdp_xmit_flush_bulk(bq, flags, prep, xmit) \
__libeth_xdp_tx_flush_bulk(bq, (flags) | LIBETH_XDP_TX_NDO, prep, \
libeth_xdp_xmit_fill_buf, xmit)
u32 libeth_xdp_xmit_return_bulk(const struct libeth_xdp_tx_frame *bq,
u32 count, const struct net_device *dev);
/**
* __libeth_xdp_xmit_do_bulk - internal function to implement .ndo_xdp_xmit()
* @bq: XDP Tx bulk to queue frames to
* @frames: XDP frames passed by the stack
* @n: number of frames
* @flags: flags passed by the stack
* @flush_bulk: driver callback to flush an XDP xmit bulk
* @finalize: driver callback to finalize sending XDP Tx frames on the queue
*
* Perform common checks, map the frags and queue them to the bulk, then flush
* the bulk to the XDPSQ. If requested by the stack, finalize the queue.
*
* Return: number of frames send or -errno on error.
*/
static __always_inline int
__libeth_xdp_xmit_do_bulk(struct libeth_xdp_tx_bulk *bq,
struct xdp_frame **frames, u32 n, u32 flags,
bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq,
u32 flags),
void (*finalize)(void *xdpsq, bool sent, bool flush))
{
u32 nxmit = 0;
if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
return -EINVAL;
for (u32 i = 0; likely(i < n); i++) {
u32 ret;
ret = libeth_xdp_xmit_queue_bulk(bq, frames[i], flush_bulk);
if (unlikely(ret != LIBETH_XDP_TX)) {
nxmit += ret == LIBETH_XDP_ABORTED;
break;
}
nxmit++;
}
if (bq->count) {
flush_bulk(bq, LIBETH_XDP_TX_NDO);
if (unlikely(bq->count))
nxmit -= libeth_xdp_xmit_return_bulk(bq->bulk,
bq->count,
bq->dev);
}
finalize(bq->xdpsq, nxmit, flags & XDP_XMIT_FLUSH);
return nxmit;
}
/**
* libeth_xdp_xmit_do_bulk - implement full .ndo_xdp_xmit() in driver
* @dev: target &net_device
* @n: number of frames to send
* @fr: XDP frames to send
* @f: flags passed by the stack
* @xqs: array of XDPSQs driver structs
* @nqs: number of active XDPSQs, the above array length
* @fl: driver callback to flush an XDP xmit bulk
* @fin: driver cabback to finalize the queue
*
* If the driver has active XDPSQs, perform common checks and send the frames.
* Finalize the queue, if requested.
*
* Return: number of frames sent or -errno on error.
*/
#define libeth_xdp_xmit_do_bulk(dev, n, fr, f, xqs, nqs, fl, fin) \
_libeth_xdp_xmit_do_bulk(dev, n, fr, f, xqs, nqs, fl, fin, \
__UNIQUE_ID(bq_), __UNIQUE_ID(ret_), \
__UNIQUE_ID(nqs_))
#define _libeth_xdp_xmit_do_bulk(d, n, fr, f, xqs, nqs, fl, fin, ub, ur, un) \
({ \
u32 un = (nqs); \
int ur; \
\
if (likely(un)) { \
LIBETH_XDP_ONSTACK_BULK(ub); \
\
libeth_xdp_xmit_init_bulk(&ub, d, xqs, un); \
ur = __libeth_xdp_xmit_do_bulk(&ub, fr, n, f, fl, fin); \
} else { \
ur = -ENXIO; \
} \
\
ur; \
})
/* Rx polling path */
/**
* libeth_xdp_tx_init_bulk - initialize an XDP Tx bulk for Rx NAPI poll
* @bq: bulk to initialize
* @prog: RCU pointer to the XDP program (can be %NULL)
* @dev: target &net_device
* @xdpsqs: array of driver XDPSQ structs
* @num: number of active XDPSQs, the above array length
*
* Should be called on an onstack XDP Tx bulk before the NAPI polling loop.
* Initializes all the needed fields to run libeth_xdp functions. If @num == 0,
* assumes XDP is not enabled.
* Do not use for XSk, it has its own optimized helper.
*/
#define libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num) \
__libeth_xdp_tx_init_bulk(bq, prog, dev, xdpsqs, num, false, \
__UNIQUE_ID(bq_), __UNIQUE_ID(nqs_))
#define __libeth_xdp_tx_init_bulk(bq, pr, d, xdpsqs, num, xsk, ub, un) do { \
typeof(bq) ub = (bq); \
u32 un = (num); \
\
rcu_read_lock(); \
\
if (un || (xsk)) { \
ub->prog = rcu_dereference(pr); \
ub->dev = (d); \
ub->xdpsq = (xdpsqs)[libeth_xdpsq_id(un)]; \
} else { \
ub->prog = NULL; \
} \
\
ub->act_mask = 0; \
ub->count = 0; \
} while (0)
void libeth_xdp_load_stash(struct libeth_xdp_buff *dst,
const struct libeth_xdp_buff_stash *src);
void libeth_xdp_save_stash(struct libeth_xdp_buff_stash *dst,
const struct libeth_xdp_buff *src);
void __libeth_xdp_return_stash(struct libeth_xdp_buff_stash *stash);
/**
* libeth_xdp_init_buff - initialize a &libeth_xdp_buff for Rx NAPI poll
* @dst: onstack buffer to initialize
* @src: XDP buffer stash placed on the queue
* @rxq: registered &xdp_rxq_info corresponding to this queue
*
* Should be called before the main NAPI polling loop. Loads the content of
* the previously saved stash or initializes the buffer from scratch.
* Do not use for XSk.
*/
static inline void
libeth_xdp_init_buff(struct libeth_xdp_buff *dst,
const struct libeth_xdp_buff_stash *src,
struct xdp_rxq_info *rxq)
{
if (likely(!src->data))
dst->data = NULL;
else
libeth_xdp_load_stash(dst, src);
dst->base.rxq = rxq;
}
/**
* libeth_xdp_save_buff - save a partially built buffer on a queue
* @dst: XDP buffer stash placed on the queue
* @src: onstack buffer to save
*
* Should be called after the main NAPI polling loop. If the loop exited before
* the buffer was finished, saves its content on the queue, so that it can be
* completed during the next poll. Otherwise, clears the stash.
*/
static inline void libeth_xdp_save_buff(struct libeth_xdp_buff_stash *dst,
const struct libeth_xdp_buff *src)
{
if (likely(!src->data))
dst->data = NULL;
else
libeth_xdp_save_stash(dst, src);
}
/**
* libeth_xdp_return_stash - free an XDP buffer stash from a queue
* @stash: stash to free
*
* If the queue is about to be destroyed, but it still has an incompleted
* buffer stash, this helper should be called to free it.
*/
static inline void libeth_xdp_return_stash(struct libeth_xdp_buff_stash *stash)
{
if (stash->data)
__libeth_xdp_return_stash(stash);
}
static inline void libeth_xdp_return_va(const void *data, bool napi)
{
netmem_ref netmem = virt_to_netmem(data);
page_pool_put_full_netmem(__netmem_get_pp(netmem), netmem, napi);
}
static inline void libeth_xdp_return_frags(const struct skb_shared_info *sinfo,
bool napi)
{
for (u32 i = 0; i < sinfo->nr_frags; i++) {
netmem_ref netmem = skb_frag_netmem(&sinfo->frags[i]);
page_pool_put_full_netmem(netmem_get_pp(netmem), netmem, napi);
}
}
/**
* libeth_xdp_return_buff - free/recycle &libeth_xdp_buff
* @xdp: buffer to free
*
* Hotpath helper to free &libeth_xdp_buff. Comparing to xdp_return_buff(),
* it's faster as it gets inlined and always assumes order-0 pages and safe
* direct recycling. Zeroes @xdp->data to avoid UAFs.
*/
#define libeth_xdp_return_buff(xdp) __libeth_xdp_return_buff(xdp, true)
static inline void __libeth_xdp_return_buff(struct libeth_xdp_buff *xdp,
bool napi)
{
if (!xdp_buff_has_frags(&xdp->base))
goto out;
libeth_xdp_return_frags(xdp_get_shared_info_from_buff(&xdp->base),
napi);
out:
libeth_xdp_return_va(xdp->data, napi);
xdp->data = NULL;
}
bool libeth_xdp_buff_add_frag(struct libeth_xdp_buff *xdp,
const struct libeth_fqe *fqe,
u32 len);
/**
* libeth_xdp_prepare_buff - fill &libeth_xdp_buff with head FQE data
* @xdp: XDP buffer to attach the head to
* @fqe: FQE containing the head buffer
* @len: buffer len passed from HW
*
* Internal, use libeth_xdp_process_buff() instead. Initializes XDP buffer
* head with the Rx buffer data: data pointer, length, headroom, and
* truesize/tailroom. Zeroes the flags.
*/
static inline void libeth_xdp_prepare_buff(struct libeth_xdp_buff *xdp,
const struct libeth_fqe *fqe,
u32 len)
{
const struct page *page = __netmem_to_page(fqe->netmem);
xdp_prepare_buff(&xdp->base, page_address(page) + fqe->offset,
pp_page_to_nmdesc(page)->pp->p.offset, len, true);
xdp_init_buff(&xdp->base, fqe->truesize, xdp->base.rxq);
}
/**
* libeth_xdp_process_buff - attach Rx buffer to &libeth_xdp_buff
* @xdp: XDP buffer to attach the Rx buffer to
* @fqe: Rx buffer to process
* @len: received data length from the descriptor
*
* If the XDP buffer is empty, attaches the Rx buffer as head and initializes
* the required fields. Otherwise, attaches the buffer as a frag.
* Already performs DMA sync-for-CPU and frame start prefetch
* (for head buffers only).
*
* Return: true on success, false if the descriptor must be skipped (empty or
* no space for a new frag).
*/
static inline bool libeth_xdp_process_buff(struct libeth_xdp_buff *xdp,
const struct libeth_fqe *fqe,
u32 len)
{
if (!libeth_rx_sync_for_cpu(fqe, len))
return false;
if (xdp->data)
return libeth_xdp_buff_add_frag(xdp, fqe, len);
libeth_xdp_prepare_buff(xdp, fqe, len);
prefetch(xdp->data);
return true;
}
/**
* libeth_xdp_buff_stats_frags - update onstack RQ stats with XDP frags info
* @ss: onstack stats to update
* @xdp: buffer to account
*
* Internal helper used by __libeth_xdp_run_pass(), do not call directly.
* Adds buffer's frags count and total len to the onstack stats.
*/
static inline void
libeth_xdp_buff_stats_frags(struct libeth_rq_napi_stats *ss,
const struct libeth_xdp_buff *xdp)
{
const struct skb_shared_info *sinfo;
sinfo = xdp_get_shared_info_from_buff(&xdp->base);
ss->bytes += sinfo->xdp_frags_size;
ss->fragments += sinfo->nr_frags + 1;
}
u32 libeth_xdp_prog_exception(const struct libeth_xdp_tx_bulk *bq,
struct libeth_xdp_buff *xdp,
enum xdp_action act, int ret);
/**
* __libeth_xdp_run_prog - run XDP program on an XDP buffer
* @xdp: XDP buffer to run the prog on
* @bq: buffer bulk for ``XDP_TX`` queueing
*
* Internal inline abstraction to run XDP program. Handles ``XDP_DROP``
* and ``XDP_REDIRECT`` only, the rest is processed levels up.
* Reports an XDP prog exception on errors.
*
* Return: libeth_xdp prog verdict depending on the prog's verdict.
*/
static __always_inline u32
__libeth_xdp_run_prog(struct libeth_xdp_buff *xdp,
const struct libeth_xdp_tx_bulk *bq)
{
enum xdp_action act;
act = bpf_prog_run_xdp(bq->prog, &xdp->base);
if (unlikely(act < XDP_DROP || act > XDP_REDIRECT))
goto out;
switch (act) {
case XDP_PASS:
return LIBETH_XDP_PASS;
case XDP_DROP:
libeth_xdp_return_buff(xdp);
return LIBETH_XDP_DROP;
case XDP_TX:
return LIBETH_XDP_TX;
case XDP_REDIRECT:
if (unlikely(xdp_do_redirect(bq->dev, &xdp->base, bq->prog)))
break;
xdp->data = NULL;
return LIBETH_XDP_REDIRECT;
default:
break;
}
out:
return libeth_xdp_prog_exception(bq, xdp, act, 0);
}
/**
* __libeth_xdp_run_flush - run XDP program and handle ``XDP_TX`` verdict
* @xdp: XDP buffer to run the prog on
* @bq: buffer bulk for ``XDP_TX`` queueing
* @run: internal callback for running XDP program
* @queue: internal callback for queuing ``XDP_TX`` frame
* @flush_bulk: driver callback for flushing a bulk
*
* Internal inline abstraction to run XDP program and additionally handle
* ``XDP_TX`` verdict. Used by both XDP and XSk, hence @run and @queue.
* Do not use directly.
*
* Return: libeth_xdp prog verdict depending on the prog's verdict.
*/
static __always_inline u32
__libeth_xdp_run_flush(struct libeth_xdp_buff *xdp,
struct libeth_xdp_tx_bulk *bq,
u32 (*run)(struct libeth_xdp_buff *xdp,
const struct libeth_xdp_tx_bulk *bq),
bool (*queue)(struct libeth_xdp_tx_bulk *bq,
struct libeth_xdp_buff *xdp,
bool (*flush_bulk)
(struct libeth_xdp_tx_bulk *bq,
u32 flags)),
bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq,
u32 flags))
{
u32 act;
act = run(xdp, bq);
if (act == LIBETH_XDP_TX && unlikely(!queue(bq, xdp, flush_bulk)))
act = LIBETH_XDP_DROP;
bq->act_mask |= act;
return act;
}
/**
* libeth_xdp_run_prog - run XDP program (non-XSk path) and handle all verdicts
* @xdp: XDP buffer to process
* @bq: XDP Tx bulk to queue ``XDP_TX`` buffers
* @fl: driver ``XDP_TX`` bulk flush callback
*
* Run the attached XDP program and handle all possible verdicts. XSk has its
* own version.
* Prefer using it via LIBETH_XDP_DEFINE_RUN{,_PASS,_PROG}().
*
* Return: true if the buffer should be passed up the stack, false if the poll
* should go to the next buffer.
*/
#define libeth_xdp_run_prog(xdp, bq, fl) \
(__libeth_xdp_run_flush(xdp, bq, __libeth_xdp_run_prog, \
libeth_xdp_tx_queue_bulk, \
fl) == LIBETH_XDP_PASS)
/**
* __libeth_xdp_run_pass - helper to run XDP program and handle the result
* @xdp: XDP buffer to process
* @bq: XDP Tx bulk to queue ``XDP_TX`` frames
* @napi: NAPI to build an skb and pass it up the stack
* @rs: onstack libeth RQ stats
* @md: metadata that should be filled to the XDP buffer
* @prep: callback for filling the metadata
* @run: driver wrapper to run XDP program
* @populate: driver callback to populate an skb with the HW descriptor data
*
* Inline abstraction that does the following (non-XSk path):
* 1) adds frame size and frag number (if needed) to the onstack stats;
* 2) fills the descriptor metadata to the onstack &libeth_xdp_buff
* 3) runs XDP program if present;
* 4) handles all possible verdicts;
* 5) on ``XDP_PASS`, builds an skb from the buffer;
* 6) populates it with the descriptor metadata;
* 7) passes it up the stack.
*
* In most cases, number 2 means just writing the pointer to the HW descriptor
* to the XDP buffer. If so, please use LIBETH_XDP_DEFINE_RUN{,_PASS}()
* wrappers to build a driver function.
*/
static __always_inline void
__libeth_xdp_run_pass(struct libeth_xdp_buff *xdp,
struct libeth_xdp_tx_bulk *bq, struct napi_struct *napi,
struct libeth_rq_napi_stats *rs, const void *md,
void (*prep)(struct libeth_xdp_buff *xdp,
const void *md),
bool (*run)(struct libeth_xdp_buff *xdp,
struct libeth_xdp_tx_bulk *bq),
bool (*populate)(struct sk_buff *skb,
const struct libeth_xdp_buff *xdp,
struct libeth_rq_napi_stats *rs))
{
struct sk_buff *skb;
rs->bytes += xdp->base.data_end - xdp->data;
rs->packets++;
if (xdp_buff_has_frags(&xdp->base))
libeth_xdp_buff_stats_frags(rs, xdp);
if (prep && (!__builtin_constant_p(!!md) || md))
prep(xdp, md);
if (!bq || !run || !bq->prog)
goto build;
if (!run(xdp, bq))
return;
build:
skb = xdp_build_skb_from_buff(&xdp->base);
if (unlikely(!skb)) {
libeth_xdp_return_buff_slow(xdp);
return;
}
xdp->data = NULL;
if (unlikely(!populate(skb, xdp, rs))) {
napi_consume_skb(skb, true);
return;
}
napi_gro_receive(napi, skb);
}
static inline void libeth_xdp_prep_desc(struct libeth_xdp_buff *xdp,
const void *desc)
{
xdp->desc = desc;
}
/**
* libeth_xdp_run_pass - helper to run XDP program and handle the result
* @xdp: XDP buffer to process
* @bq: XDP Tx bulk to queue ``XDP_TX`` frames
* @napi: NAPI to build an skb and pass it up the stack
* @ss: onstack libeth RQ stats
* @desc: pointer to the HW descriptor for that frame
* @run: driver wrapper to run XDP program
* @populate: driver callback to populate an skb with the HW descriptor data
*
* Wrapper around the underscored version when "fill the descriptor metadata"
* means just writing the pointer to the HW descriptor as @xdp->desc.
*/
#define libeth_xdp_run_pass(xdp, bq, napi, ss, desc, run, populate) \
__libeth_xdp_run_pass(xdp, bq, napi, ss, desc, libeth_xdp_prep_desc, \
run, populate)
/**
* libeth_xdp_finalize_rx - finalize XDPSQ after a NAPI polling loop (non-XSk)
* @bq: ``XDP_TX`` frame bulk
* @flush: driver callback to flush the bulk
* @finalize: driver callback to start sending the frames and run the timer
*
* Flush the bulk if there are frames left to send, kick the queue and flush
* the XDP maps.
*/
#define libeth_xdp_finalize_rx(bq, flush, finalize) \
__libeth_xdp_finalize_rx(bq, 0, flush, finalize)
static __always_inline void
__libeth_xdp_finalize_rx(struct libeth_xdp_tx_bulk *bq, u32 flags,
bool (*flush_bulk)(struct libeth_xdp_tx_bulk *bq,
u32 flags),
void (*finalize)(void *xdpsq, bool sent, bool flush))
{
if (bq->act_mask & LIBETH_XDP_TX) {
if (bq->count)
flush_bulk(bq, flags | LIBETH_XDP_TX_DROP);
finalize(bq->xdpsq, true, true);
}
if (bq->act_mask & LIBETH_XDP_REDIRECT)
xdp_do_flush();
rcu_read_unlock();
}
/*
* Helpers to reduce boilerplate code in drivers.
*
* Typical driver Rx flow would be (excl. bulk and buff init, frag attach):
*
* LIBETH_XDP_DEFINE_START();
* LIBETH_XDP_DEFINE_FLUSH_TX(static driver_xdp_flush_tx, driver_xdp_tx_prep,
* driver_xdp_xmit);
* LIBETH_XDP_DEFINE_RUN(static driver_xdp_run, driver_xdp_run_prog,
* driver_xdp_flush_tx, driver_populate_skb);
* LIBETH_XDP_DEFINE_FINALIZE(static driver_xdp_finalize_rx,
* driver_xdp_flush_tx, driver_xdp_finalize_sq);
* LIBETH_XDP_DEFINE_END();
*
* This will build a set of 4 static functions. The compiler is free to decide
* whether to inline them.
* Then, in the NAPI polling function:
*
* while (packets < budget) {
* // ...
* driver_xdp_run(xdp, &bq, napi, &rs, desc);
* }
* driver_xdp_finalize_rx(&bq);
*/
#define LIBETH_XDP_DEFINE_START() \
__diag_push(); \
__diag_ignore(GCC, 8, "-Wold-style-declaration", \
"Allow specifying \'static\' after the return type")
/**
* LIBETH_XDP_DEFINE_TIMER - define a driver XDPSQ cleanup timer callback
* @name: name of the function to define
* @poll: Tx polling/completion function
*/
#define LIBETH_XDP_DEFINE_TIMER(name, poll) \
void name(struct work_struct *work) \
{ \
libeth_xdpsq_run_timer(work, poll); \
}
/**
* LIBETH_XDP_DEFINE_FLUSH_TX - define a driver ``XDP_TX`` bulk flush function
* @name: name of the function to define
* @prep: driver callback to clean an XDPSQ
* @xmit: driver callback to write a HW Tx descriptor
*/
#define LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit) \
__LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit, xdp)
#define __LIBETH_XDP_DEFINE_FLUSH_TX(name, prep, xmit, pfx) \
bool name(struct libeth_xdp_tx_bulk *bq, u32 flags) \
{ \
return libeth_##pfx##_tx_flush_bulk(bq, flags, prep, xmit); \
}
/**
* LIBETH_XDP_DEFINE_FLUSH_XMIT - define a driver XDP xmit bulk flush function
* @name: name of the function to define
* @prep: driver callback to clean an XDPSQ
* @xmit: driver callback to write a HW Tx descriptor
*/
#define LIBETH_XDP_DEFINE_FLUSH_XMIT(name, prep, xmit) \
bool name(struct libeth_xdp_tx_bulk *bq, u32 flags) \
{ \
return libeth_xdp_xmit_flush_bulk(bq, flags, prep, xmit); \
}
/**
* LIBETH_XDP_DEFINE_RUN_PROG - define a driver XDP program run function
* @name: name of the function to define
* @flush: driver callback to flush an ``XDP_TX`` bulk
*/
#define LIBETH_XDP_DEFINE_RUN_PROG(name, flush) \
bool __LIBETH_XDP_DEFINE_RUN_PROG(name, flush, xdp)
#define __LIBETH_XDP_DEFINE_RUN_PROG(name, flush, pfx) \
name(struct libeth_xdp_buff *xdp, struct libeth_xdp_tx_bulk *bq) \
{ \
return libeth_##pfx##_run_prog(xdp, bq, flush); \
}
/**
* LIBETH_XDP_DEFINE_RUN_PASS - define a driver buffer process + pass function
* @name: name of the function to define
* @run: driver callback to run XDP program (above)
* @populate: driver callback to fill an skb with HW descriptor info
*/
#define LIBETH_XDP_DEFINE_RUN_PASS(name, run, populate) \
void __LIBETH_XDP_DEFINE_RUN_PASS(name, run, populate, xdp)
#define __LIBETH_XDP_DEFINE_RUN_PASS(name, run, populate, pfx) \
name(struct libeth_xdp_buff *xdp, struct libeth_xdp_tx_bulk *bq, \
struct napi_struct *napi, struct libeth_rq_napi_stats *ss, \
const void *desc) \
{ \
return libeth_##pfx##_run_pass(xdp, bq, napi, ss, desc, run, \
populate); \
}
/**
* LIBETH_XDP_DEFINE_RUN - define a driver buffer process, run + pass function
* @name: name of the function to define
* @run: name of the XDP prog run function to define
* @flush: driver callback to flush an ``XDP_TX`` bulk
* @populate: driver callback to fill an skb with HW descriptor info
*/
#define LIBETH_XDP_DEFINE_RUN(name, run, flush, populate) \
__LIBETH_XDP_DEFINE_RUN(name, run, flush, populate, XDP)
#define __LIBETH_XDP_DEFINE_RUN(name, run, flush, populate, pfx) \
LIBETH_##pfx##_DEFINE_RUN_PROG(static run, flush); \
LIBETH_##pfx##_DEFINE_RUN_PASS(name, run, populate)
/**
* LIBETH_XDP_DEFINE_FINALIZE - define a driver Rx NAPI poll finalize function
* @name: name of the function to define
* @flush: driver callback to flush an ``XDP_TX`` bulk
* @finalize: driver callback to finalize an XDPSQ and run the timer
*/
#define LIBETH_XDP_DEFINE_FINALIZE(name, flush, finalize) \
__LIBETH_XDP_DEFINE_FINALIZE(name, flush, finalize, xdp)
#define __LIBETH_XDP_DEFINE_FINALIZE(name, flush, finalize, pfx) \
void name(struct libeth_xdp_tx_bulk *bq) \
{ \
libeth_##pfx##_finalize_rx(bq, flush, finalize); \
}
#define LIBETH_XDP_DEFINE_END() __diag_pop()
/* XMO */
/**
* libeth_xdp_buff_to_rq - get RQ pointer from an XDP buffer pointer
* @xdp: &libeth_xdp_buff corresponding to the queue
* @type: typeof() of the driver Rx queue structure
* @member: name of &xdp_rxq_info inside @type
*
* Often times, pointer to the RQ is needed when reading/filling metadata from
* HW descriptors. The helper can be used to quickly jump from an XDP buffer
* to the queue corresponding to its &xdp_rxq_info without introducing
* additional fields (&libeth_xdp_buff is precisely 1 cacheline long on x64).
*/
#define libeth_xdp_buff_to_rq(xdp, type, member) \
container_of_const((xdp)->base.rxq, type, member)
/**
* libeth_xdpmo_rx_hash - convert &libeth_rx_pt to an XDP RSS hash metadata
* @hash: pointer to the variable to write the hash to
* @rss_type: pointer to the variable to write the hash type to
* @val: hash value from the HW descriptor
* @pt: libeth parsed packet type
*
* Handle zeroed/non-available hash and convert libeth parsed packet type to
* the corresponding XDP RSS hash type. To be called at the end of
* xdp_metadata_ops idpf_xdpmo::xmo_rx_hash() implementation.
* Note that if the driver doesn't use a constant packet type lookup table but
* generates it at runtime, it must call libeth_rx_pt_gen_hash_type(pt) to
* generate XDP RSS hash type for each packet type.
*
* Return: 0 on success, -ENODATA when the hash is not available.
*/
static inline int libeth_xdpmo_rx_hash(u32 *hash,
enum xdp_rss_hash_type *rss_type,
u32 val, struct libeth_rx_pt pt)
{
if (unlikely(!val))
return -ENODATA;
*hash = val;
*rss_type = pt.hash_type;
return 0;
}
/* Tx buffer completion */
void libeth_xdp_return_buff_bulk(const struct skb_shared_info *sinfo,
struct xdp_frame_bulk *bq, bool frags);
void libeth_xsk_buff_free_slow(struct libeth_xdp_buff *xdp);
/**
* __libeth_xdp_complete_tx - complete sent XDPSQE
* @sqe: SQ element / Tx buffer to complete
* @cp: Tx polling/completion params
* @bulk: internal callback to bulk-free ``XDP_TX`` buffers
* @xsk: internal callback to free XSk ``XDP_TX`` buffers
*
* Use the non-underscored version in drivers instead. This one is shared
* internally with libeth_tx_complete_any().
* Complete an XDPSQE of any type of XDP frame. This includes DMA unmapping
* when needed, buffer freeing, stats update, and SQE invalidation.
*/
static __always_inline void
__libeth_xdp_complete_tx(struct libeth_sqe *sqe, struct libeth_cq_pp *cp,
typeof(libeth_xdp_return_buff_bulk) bulk,
typeof(libeth_xsk_buff_free_slow) xsk)
{
enum libeth_sqe_type type = sqe->type;
switch (type) {
case LIBETH_SQE_EMPTY:
return;
case LIBETH_SQE_XDP_XMIT:
case LIBETH_SQE_XDP_XMIT_FRAG:
dma_unmap_page(cp->dev, dma_unmap_addr(sqe, dma),
dma_unmap_len(sqe, len), DMA_TO_DEVICE);
break;
default:
break;
}
switch (type) {
case LIBETH_SQE_XDP_TX:
bulk(sqe->sinfo, cp->bq, sqe->nr_frags != 1);
break;
case LIBETH_SQE_XDP_XMIT:
xdp_return_frame_bulk(sqe->xdpf, cp->bq);
break;
case LIBETH_SQE_XSK_TX:
case LIBETH_SQE_XSK_TX_FRAG:
xsk(sqe->xsk);
break;
default:
break;
}
switch (type) {
case LIBETH_SQE_XDP_TX:
case LIBETH_SQE_XDP_XMIT:
case LIBETH_SQE_XSK_TX:
cp->xdp_tx -= sqe->nr_frags;
cp->xss->packets++;
cp->xss->bytes += sqe->bytes;
break;
default:
break;
}
sqe->type = LIBETH_SQE_EMPTY;
}
static inline void libeth_xdp_complete_tx(struct libeth_sqe *sqe,
struct libeth_cq_pp *cp)
{
__libeth_xdp_complete_tx(sqe, cp, libeth_xdp_return_buff_bulk,
libeth_xsk_buff_free_slow);
}
/* Misc */
u32 libeth_xdp_queue_threshold(u32 count);
void __libeth_xdp_set_features(struct net_device *dev,
const struct xdp_metadata_ops *xmo,
u32 zc_segs,
const struct xsk_tx_metadata_ops *tmo);
void libeth_xdp_set_redirect(struct net_device *dev, bool enable);
/**
* libeth_xdp_set_features - set XDP features for netdev
* @dev: &net_device to configure
* @...: optional params, see __libeth_xdp_set_features()
*
* Set all the features libeth_xdp supports, including .ndo_xdp_xmit(). That
* said, it should be used only when XDPSQs are always available regardless
* of whether an XDP prog is attached to @dev.
*/
#define libeth_xdp_set_features(dev, ...) \
CONCATENATE(__libeth_xdp_feat, \
COUNT_ARGS(__VA_ARGS__))(dev, ##__VA_ARGS__)
#define __libeth_xdp_feat0(dev) \
__libeth_xdp_set_features(dev, NULL, 0, NULL)
#define __libeth_xdp_feat1(dev, xmo) \
__libeth_xdp_set_features(dev, xmo, 0, NULL)
#define __libeth_xdp_feat2(dev, xmo, zc_segs) \
__libeth_xdp_set_features(dev, xmo, zc_segs, NULL)
#define __libeth_xdp_feat3(dev, xmo, zc_segs, tmo) \
__libeth_xdp_set_features(dev, xmo, zc_segs, tmo)
/**
* libeth_xdp_set_features_noredir - enable all libeth_xdp features w/o redir
* @dev: target &net_device
* @...: optional params, see __libeth_xdp_set_features()
*
* Enable everything except the .ndo_xdp_xmit() feature, use when XDPSQs are
* not available right after netdev registration.
*/
#define libeth_xdp_set_features_noredir(dev, ...) \
__libeth_xdp_set_features_noredir(dev, __UNIQUE_ID(dev_), \
##__VA_ARGS__)
#define __libeth_xdp_set_features_noredir(dev, ud, ...) do { \
struct net_device *ud = (dev); \
\
libeth_xdp_set_features(ud, ##__VA_ARGS__); \
libeth_xdp_set_redirect(ud, false); \
} while (0)
#define libeth_xsktmo ((const void *)GOLDEN_RATIO_PRIME)
#endif /* __LIBETH_XDP_H */
