Skip to content

Commit

Permalink
ionic: Add XDP_TX support
Browse files Browse the repository at this point in the history
The XDP_TX packets get fed back into the Rx queue's partnered
Tx queue as an xdp_frame.

Co-developed-by: Brett Creeley <[email protected]>
Signed-off-by: Brett Creeley <[email protected]>
Signed-off-by: Shannon Nelson <[email protected]>
Reviewed-by: Jacob Keller <[email protected]>
Signed-off-by: David S. Miller <[email protected]>
  • Loading branch information
emusln authored and davem330 committed Feb 16, 2024
1 parent f81da39 commit 8eeed83
Show file tree
Hide file tree
Showing 5 changed files with 150 additions and 3 deletions.
3 changes: 3 additions & 0 deletions drivers/net/ethernet/pensando/ionic/ionic_dev.h
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,8 @@ struct ionic_desc_info {
struct ionic_buf_info bufs[MAX_SKB_FRAGS + 1];
ionic_desc_cb cb;
void *cb_arg;
struct xdp_frame *xdpf;
enum xdp_action act;
};

#define IONIC_QUEUE_NAME_MAX_SZ 16
Expand Down Expand Up @@ -263,6 +265,7 @@ struct ionic_queue {
struct ionic_rxq_sg_desc *rxq_sgl;
};
struct xdp_rxq_info *xdp_rxq_info;
struct ionic_queue *partner;
dma_addr_t base_pa;
dma_addr_t cmb_base_pa;
dma_addr_t sg_base_pa;
Expand Down
4 changes: 4 additions & 0 deletions drivers/net/ethernet/pensando/ionic/ionic_lif.c
Original file line number Diff line number Diff line change
Expand Up @@ -878,6 +878,9 @@ static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
};
int err;

q->partner = &lif->txqcqs[q->index]->q;
q->partner->partner = q;

if (!lif->xdp_prog)
ctx.cmd.q_init.flags |= cpu_to_le16(IONIC_QINIT_F_SG);

Expand Down Expand Up @@ -2926,6 +2929,7 @@ static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b)
swap(a->q.base_pa, b->q.base_pa);
swap(a->q.info, b->q.info);
swap(a->q.xdp_rxq_info, b->q.xdp_rxq_info);
swap(a->q.partner, b->q.partner);
swap(a->q_base, b->q_base);
swap(a->q_base_pa, b->q_base_pa);
swap(a->q_size, b->q_size);
Expand Down
4 changes: 4 additions & 0 deletions drivers/net/ethernet/pensando/ionic/ionic_lif.h
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ struct ionic_tx_stats {
u64 dma_map_err;
u64 hwstamp_valid;
u64 hwstamp_invalid;
u64 xdp_frames;
};

struct ionic_rx_stats {
Expand All @@ -54,6 +55,7 @@ struct ionic_rx_stats {
u64 xdp_drop;
u64 xdp_aborted;
u64 xdp_pass;
u64 xdp_tx;
};

#define IONIC_QCQ_F_INITED BIT(0)
Expand Down Expand Up @@ -141,6 +143,8 @@ struct ionic_lif_sw_stats {
u64 xdp_drop;
u64 xdp_aborted;
u64 xdp_pass;
u64 xdp_tx;
u64 xdp_frames;
};

enum ionic_lif_state_flags {
Expand Down
6 changes: 6 additions & 0 deletions drivers/net/ethernet/pensando/ionic/ionic_stats.c
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ static const struct ionic_stat_desc ionic_lif_stats_desc[] = {
IONIC_LIF_STAT_DESC(xdp_drop),
IONIC_LIF_STAT_DESC(xdp_aborted),
IONIC_LIF_STAT_DESC(xdp_pass),
IONIC_LIF_STAT_DESC(xdp_tx),
IONIC_LIF_STAT_DESC(xdp_frames),
};

static const struct ionic_stat_desc ionic_port_stats_desc[] = {
Expand Down Expand Up @@ -138,6 +140,7 @@ static const struct ionic_stat_desc ionic_tx_stats_desc[] = {
IONIC_TX_STAT_DESC(csum_none),
IONIC_TX_STAT_DESC(csum),
IONIC_TX_STAT_DESC(vlan_inserted),
IONIC_TX_STAT_DESC(xdp_frames),
};

static const struct ionic_stat_desc ionic_rx_stats_desc[] = {
Expand All @@ -155,6 +158,7 @@ static const struct ionic_stat_desc ionic_rx_stats_desc[] = {
IONIC_RX_STAT_DESC(xdp_drop),
IONIC_RX_STAT_DESC(xdp_aborted),
IONIC_RX_STAT_DESC(xdp_pass),
IONIC_RX_STAT_DESC(xdp_tx),
};

#define IONIC_NUM_LIF_STATS ARRAY_SIZE(ionic_lif_stats_desc)
Expand All @@ -177,6 +181,7 @@ static void ionic_add_lif_txq_stats(struct ionic_lif *lif, int q_num,
stats->tx_csum += txstats->csum;
stats->tx_hwstamp_valid += txstats->hwstamp_valid;
stats->tx_hwstamp_invalid += txstats->hwstamp_invalid;
stats->xdp_frames += txstats->xdp_frames;
}

static void ionic_add_lif_rxq_stats(struct ionic_lif *lif, int q_num,
Expand All @@ -194,6 +199,7 @@ static void ionic_add_lif_rxq_stats(struct ionic_lif *lif, int q_num,
stats->xdp_drop += rxstats->xdp_drop;
stats->xdp_aborted += rxstats->xdp_aborted;
stats->xdp_pass += rxstats->xdp_pass;
stats->xdp_tx += rxstats->xdp_tx;
}

static void ionic_get_lif_stats(struct ionic_lif *lif,
Expand Down
136 changes: 133 additions & 3 deletions drivers/net/ethernet/pensando/ionic/ionic_txrx.c
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,16 @@
#include "ionic_lif.h"
#include "ionic_txrx.h"

static int ionic_maybe_stop_tx(struct ionic_queue *q, int ndescs);

static dma_addr_t ionic_tx_map_single(struct ionic_queue *q,
void *data, size_t len);

static void ionic_tx_clean(struct ionic_queue *q,
struct ionic_desc_info *desc_info,
struct ionic_cq_info *cq_info,
void *cb_arg);

static inline void ionic_txq_post(struct ionic_queue *q, bool ring_dbell,
ionic_desc_cb cb_func, void *cb_arg)
{
Expand Down Expand Up @@ -297,6 +307,75 @@ static struct sk_buff *ionic_rx_copybreak(struct ionic_queue *q,
return skb;
}

static void ionic_xdp_tx_desc_clean(struct ionic_queue *q,
struct ionic_desc_info *desc_info)
{
unsigned int nbufs = desc_info->nbufs;
struct ionic_buf_info *buf_info;
struct device *dev = q->dev;

if (!nbufs)
return;

buf_info = desc_info->bufs;
dma_unmap_single(dev, buf_info->dma_addr,
buf_info->len, DMA_TO_DEVICE);
__free_pages(buf_info->page, 0);
buf_info->page = NULL;

desc_info->nbufs = 0;
desc_info->xdpf = NULL;
desc_info->act = 0;
}

static int ionic_xdp_post_frame(struct net_device *netdev,
struct ionic_queue *q, struct xdp_frame *frame,
enum xdp_action act, struct page *page, int off,
bool ring_doorbell)
{
struct ionic_desc_info *desc_info;
struct ionic_buf_info *buf_info;
struct ionic_tx_stats *stats;
struct ionic_txq_desc *desc;
size_t len = frame->len;
dma_addr_t dma_addr;
u64 cmd;

desc_info = &q->info[q->head_idx];
desc = desc_info->txq_desc;
buf_info = desc_info->bufs;
stats = q_to_tx_stats(q);

dma_addr = ionic_tx_map_single(q, frame->data, len);
if (dma_mapping_error(q->dev, dma_addr)) {
stats->dma_map_err++;
return -EIO;
}
buf_info->dma_addr = dma_addr;
buf_info->len = len;
buf_info->page = page;
buf_info->page_offset = off;

desc_info->nbufs = 1;
desc_info->xdpf = frame;
desc_info->act = act;

cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_CSUM_NONE,
0, 0, buf_info->dma_addr);
desc->cmd = cpu_to_le64(cmd);
desc->len = cpu_to_le16(len);
desc->csum_start = 0;
desc->csum_offset = 0;

stats->xdp_frames++;
stats->pkts++;
stats->bytes += len;

ionic_txq_post(q, ring_doorbell, ionic_tx_clean, NULL);

return 0;
}

static bool ionic_run_xdp(struct ionic_rx_stats *stats,
struct net_device *netdev,
struct bpf_prog *xdp_prog,
Expand All @@ -306,6 +385,10 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats,
{
u32 xdp_action = XDP_ABORTED;
struct xdp_buff xdp_buf;
struct ionic_queue *txq;
struct netdev_queue *nq;
struct xdp_frame *xdpf;
int err = 0;

xdp_init_buff(&xdp_buf, IONIC_PAGE_SIZE, rxq->xdp_rxq_info);
xdp_prepare_buff(&xdp_buf, ionic_rx_buf_va(buf_info),
Expand All @@ -330,14 +413,51 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats,
break;

case XDP_TX:
xdpf = xdp_convert_buff_to_frame(&xdp_buf);
if (!xdpf)
goto out_xdp_abort;

txq = rxq->partner;
nq = netdev_get_tx_queue(netdev, txq->index);
__netif_tx_lock(nq, smp_processor_id());
txq_trans_cond_update(nq);

if (netif_tx_queue_stopped(nq) ||
unlikely(ionic_maybe_stop_tx(txq, 1))) {
__netif_tx_unlock(nq);
goto out_xdp_abort;
}

dma_unmap_page(rxq->dev, buf_info->dma_addr,
IONIC_PAGE_SIZE, DMA_FROM_DEVICE);

err = ionic_xdp_post_frame(netdev, txq, xdpf, XDP_TX,
buf_info->page,
buf_info->page_offset,
true);
__netif_tx_unlock(nq);
if (err) {
netdev_dbg(netdev, "tx ionic_xdp_post_frame err %d\n", err);
goto out_xdp_abort;
}
stats->xdp_tx++;

/* the Tx completion will free the buffers */
break;

case XDP_REDIRECT:
case XDP_ABORTED:
default:
trace_xdp_exception(netdev, xdp_prog, xdp_action);
ionic_rx_page_free(rxq, buf_info);
stats->xdp_aborted++;
goto out_xdp_abort;
}

return true;

out_xdp_abort:
trace_xdp_exception(netdev, xdp_prog, xdp_action);
ionic_rx_page_free(rxq, buf_info);
stats->xdp_aborted++;

return true;
}

Expand Down Expand Up @@ -893,6 +1013,16 @@ static void ionic_tx_clean(struct ionic_queue *q,
struct sk_buff *skb = cb_arg;
u16 qi;

if (desc_info->xdpf) {
ionic_xdp_tx_desc_clean(q->partner, desc_info);
stats->clean++;

if (unlikely(__netif_subqueue_stopped(q->lif->netdev, q->index)))
netif_wake_subqueue(q->lif->netdev, q->index);

return;
}

ionic_tx_desc_unmap_bufs(q, desc_info);

if (!skb)
Expand Down

0 comments on commit 8eeed83

Please sign in to comment.