net/nfp: add flower representor framework

Adds the framework to support flower representors. The number of VF
representors are parsed from the command line. For physical port
representors the current logic aims to create a representor for
each physical port present on the hardware.

An eth_dev is created for each physical port and VF, and flower
firmware requires a MAC repr cmsg to be transmitted to firmware
with info about the number of physical ports configured.

Reify messages are sent to hardware for each physical port representor.
An rte_ring is also created per representor so that traffic can be
pushed and pulled to this interface.

To up and down the real device represented by a flower representor port
a port mod message is used to convey that info to the firmware. This
message will be used in the dev_ops callbacks of flower representors.

Each cmsg generated by the driver is prepended with a cmsg header.
This commit also adds the logic to fill in the header of cmsgs.

Also add the Rx and Tx path for flower representors. For Rx packets are
dequeued from the representor ring and passed to the eth_dev. For Tx
the first queue of the PF vNIC is used. Metadata about the representor
is added before the packet is sent down to firmware.

Signed-off-by: Chaoyong He <[email protected]>
Reviewed-by: Niklas Söderlund <[email protected]>

doc/guides/nics/nfp.rst

@@ -194,3 +194,9 @@ The flower firmware application requires the PMD running two services:
 	* PF vNIC service: handling the feedback traffic.
 	* ctrl vNIC service: communicate between PMD and firmware through
 	  control message.
+
+To achieve the offload of flow, the representor ports are exposed to OVS.
+The flower firmware application support representor port for VF and physical
+port. There will always exist a representor port for each physical port,
+and the number of the representor port for VF is specified by the user through
+parameter.

doc/guides/rel_notes/release_22_11.rst

@@ -101,6 +101,7 @@ New Features
   * Added the support of flower firmware.
   * Added the flower service infrastructure.
   * Added the control message interactive channels between PMD and firmware.
+  * Added the support of representor port.
 
 * **Updated NXP dpaa2 driver.**
 

drivers/net/nfp/flower/nfp_flower.c

@@ -19,6 +19,7 @@
 #include "../nfpcore/nfp_nsp.h"
 #include "nfp_flower.h"
 #include "nfp_flower_ctrl.h"
+#include "nfp_flower_representor.h"
 
 #define CTRL_VNIC_NB_DESC 512
 #define DEFAULT_FLBUF_SIZE 9216
@@ -568,6 +569,12 @@ nfp_init_app_fw_flower(struct nfp_pf_dev *pf_dev)
 		goto ctrl_vnic_cleanup;
 	}
 
+	ret = nfp_flower_repr_create(app_fw_flower);
+	if (ret != 0) {
+		PMD_INIT_LOG(ERR, "Could not create representor ports");
+		goto ctrl_vnic_cleanup;
+	}
+
 	return 0;
 
 ctrl_vnic_cleanup:

drivers/net/nfp/flower/nfp_flower.h

@@ -14,8 +14,20 @@
  */
 #define FLOWER_PKT_DATA_OFFSET 8
 
+#define MAX_FLOWER_PHYPORTS 8
+#define MAX_FLOWER_VFS 64
+
 /* The flower application's private structure */
 struct nfp_app_fw_flower {
+	/* switch domain for this app */
+	uint16_t switch_domain_id;
+
+	/* Number of VF representors */
+	uint8_t num_vf_reprs;
+
+	/* Number of phyport representors */
+	uint8_t num_phyport_reprs;
+
 	/* Pointer to the PF vNIC */
 	struct nfp_net_hw *pf_hw;
 
@@ -30,6 +42,15 @@ struct nfp_app_fw_flower {
 
 	/* Ctrl vNIC Tx counter */
 	uint64_t ctrl_vnic_tx_count;
+
+	/* Array of phyport representors */
+	struct nfp_flower_representor *phy_reprs[MAX_FLOWER_PHYPORTS];
+
+	/* Array of VF representors */
+	struct nfp_flower_representor *vf_reprs[MAX_FLOWER_VFS];
+
+	/* PF representor */
+	struct nfp_flower_representor *pf_repr;
 };
 
 int nfp_init_app_fw_flower(struct nfp_pf_dev *pf_dev);

drivers/net/nfp/flower/nfp_flower_cmsg.c

@@ -0,0 +1,179 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2022 Corigine, Inc.
+ * All rights reserved.
+ */
+
+#include "../nfpcore/nfp_nsp.h"
+#include "../nfp_logs.h"
+#include "../nfp_common.h"
+#include "nfp_flower.h"
+#include "nfp_flower_cmsg.h"
+#include "nfp_flower_ctrl.h"
+#include "nfp_flower_representor.h"
+
+static void *
+nfp_flower_cmsg_init(struct rte_mbuf *m,
+		enum nfp_flower_cmsg_type type,
+		uint32_t size)
+{
+	char *pkt;
+	uint32_t data;
+	uint32_t new_size = size;
+	struct nfp_flower_cmsg_hdr *hdr;
+
+	pkt = rte_pktmbuf_mtod(m, char *);
+	PMD_DRV_LOG(DEBUG, "flower_cmsg_init using pkt at %p", pkt);
+
+	data = rte_cpu_to_be_32(NFP_NET_META_PORTID);
+	rte_memcpy(pkt, &data, 4);
+	pkt += 4;
+	new_size += 4;
+
+	/* First the metadata as flower requires it */
+	data = rte_cpu_to_be_32(NFP_META_PORT_ID_CTRL);
+	rte_memcpy(pkt, &data, 4);
+	pkt += 4;
+	new_size += 4;
+
+	/* Now the ctrl header */
+	hdr = (struct nfp_flower_cmsg_hdr *)pkt;
+	hdr->pad     = 0;
+	hdr->type    = type;
+	hdr->version = NFP_FLOWER_CMSG_VER1;
+
+	pkt = (char *)hdr + NFP_FLOWER_CMSG_HLEN;
+	new_size += NFP_FLOWER_CMSG_HLEN;
+
+	m->pkt_len = new_size;
+	m->data_len = m->pkt_len;
+
+	return pkt;
+}
+
+static void
+nfp_flower_cmsg_mac_repr_init(struct rte_mbuf *mbuf, int num_ports)
+{
+	uint32_t size;
+	struct nfp_flower_cmsg_mac_repr *msg;
+	enum nfp_flower_cmsg_type type = NFP_FLOWER_CMSG_TYPE_MAC_REPR;
+
+	size = sizeof(*msg) + (num_ports * sizeof(msg->ports[0]));
+	msg = nfp_flower_cmsg_init(mbuf, type, size);
+	memset(msg->reserved, 0, sizeof(msg->reserved));
+	msg->num_ports = num_ports;
+}
+
+static void
+nfp_flower_cmsg_mac_repr_fill(struct rte_mbuf *m,
+		unsigned int idx,
+		unsigned int nbi,
+		unsigned int nbi_port,
+		unsigned int phys_port)
+{
+	struct nfp_flower_cmsg_mac_repr *msg;
+
+	msg = (struct nfp_flower_cmsg_mac_repr *)nfp_flower_cmsg_get_data(m);
+	msg->ports[idx].idx       = idx;
+	msg->ports[idx].info      = nbi & NFP_FLOWER_CMSG_MAC_REPR_NBI;
+	msg->ports[idx].nbi_port  = nbi_port;
+	msg->ports[idx].phys_port = phys_port;
+}
+
+int
+nfp_flower_cmsg_mac_repr(struct nfp_app_fw_flower *app_fw_flower)
+{
+	int i;
+	uint16_t cnt;
+	unsigned int nbi;
+	unsigned int nbi_port;
+	unsigned int phys_port;
+	struct rte_mbuf *mbuf;
+	struct nfp_eth_table *nfp_eth_table;
+
+	mbuf = rte_pktmbuf_alloc(app_fw_flower->ctrl_pktmbuf_pool);
+	if (mbuf == NULL) {
+		PMD_DRV_LOG(ERR, "Could not allocate mac repr cmsg");
+		return -ENOMEM;
+	}
+
+	nfp_flower_cmsg_mac_repr_init(mbuf, app_fw_flower->num_phyport_reprs);
+
+	/* Fill in the mac repr cmsg */
+	nfp_eth_table = app_fw_flower->pf_hw->pf_dev->nfp_eth_table;
+	for (i = 0; i < app_fw_flower->num_phyport_reprs; i++) {
+		nbi = nfp_eth_table->ports[i].nbi;
+		nbi_port = nfp_eth_table->ports[i].base;
+		phys_port = nfp_eth_table->ports[i].index;
+
+		nfp_flower_cmsg_mac_repr_fill(mbuf, i, nbi, nbi_port, phys_port);
+	}
+
+	/* Send the cmsg via the ctrl vNIC */
+	cnt = nfp_flower_ctrl_vnic_xmit(app_fw_flower, mbuf);
+	if (cnt == 0) {
+		PMD_DRV_LOG(ERR, "Send cmsg through ctrl vnic failed.");
+		rte_pktmbuf_free(mbuf);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+int
+nfp_flower_cmsg_repr_reify(struct nfp_app_fw_flower *app_fw_flower,
+		struct nfp_flower_representor *repr)
+{
+	uint16_t cnt;
+	struct rte_mbuf *mbuf;
+	struct nfp_flower_cmsg_port_reify *msg;
+
+	mbuf = rte_pktmbuf_alloc(app_fw_flower->ctrl_pktmbuf_pool);
+	if (mbuf == NULL) {
+		PMD_DRV_LOG(DEBUG, "alloc mbuf for repr reify failed");
+		return -ENOMEM;
+	}
+
+	msg = nfp_flower_cmsg_init(mbuf, NFP_FLOWER_CMSG_TYPE_PORT_REIFY, sizeof(*msg));
+	msg->portnum  = rte_cpu_to_be_32(repr->port_id);
+	msg->reserved = 0;
+	msg->info     = rte_cpu_to_be_16(1);
+
+	cnt = nfp_flower_ctrl_vnic_xmit(app_fw_flower, mbuf);
+	if (cnt == 0) {
+		PMD_DRV_LOG(ERR, "Send cmsg through ctrl vnic failed.");
+		rte_pktmbuf_free(mbuf);
+		return -EIO;
+	}
+
+	return 0;
+}
+
+int
+nfp_flower_cmsg_port_mod(struct nfp_app_fw_flower *app_fw_flower,
+		uint32_t port_id, bool carrier_ok)
+{
+	uint16_t cnt;
+	struct rte_mbuf *mbuf;
+	struct nfp_flower_cmsg_port_mod *msg;
+
+	mbuf = rte_pktmbuf_alloc(app_fw_flower->ctrl_pktmbuf_pool);
+	if (mbuf == NULL) {
+		PMD_DRV_LOG(DEBUG, "alloc mbuf for repr portmod failed");
+		return -ENOMEM;
+	}
+
+	msg = nfp_flower_cmsg_init(mbuf, NFP_FLOWER_CMSG_TYPE_PORT_MOD, sizeof(*msg));
+	msg->portnum  = rte_cpu_to_be_32(port_id);
+	msg->reserved = 0;
+	msg->info     = carrier_ok;
+	msg->mtu      = 9000;
+
+	cnt = nfp_flower_ctrl_vnic_xmit(app_fw_flower, mbuf);
+	if (cnt == 0) {
+		PMD_DRV_LOG(ERR, "Send cmsg through ctrl vnic failed.");
+		rte_pktmbuf_free(mbuf);
+		return -EIO;
+	}
+
+	return 0;
+}