From: Aaron Williams <awilli...@marvell.com>

Import cvmx-pko3.c from 2013 U-Boot. It will be used by the later
added drivers to support networking on the MIPS Octeon II / III
platforms.

Signed-off-by: Aaron Williams <awilli...@marvell.com>
Signed-off-by: Stefan Roese <s...@denx.de>
---
 arch/mips/mach-octeon/cvmx-pko3.c | 2143 +++++++++++++++++++++++++++++
 1 file changed, 2143 insertions(+)
 create mode 100644 arch/mips/mach-octeon/cvmx-pko3.c

diff --git a/arch/mips/mach-octeon/cvmx-pko3.c 
b/arch/mips/mach-octeon/cvmx-pko3.c
new file mode 100644
index 000000000000..dc180ea83ac3
--- /dev/null
+++ b/arch/mips/mach-octeon/cvmx-pko3.c
@@ -0,0 +1,2143 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2018-2022 Marvell International Ltd.
+ */
+
+#include <errno.h>
+#include <log.h>
+#include <time.h>
+#include <linux/delay.h>
+
+#include <mach/cvmx-regs.h>
+#include <mach/cvmx-csr.h>
+#include <mach/cvmx-bootmem.h>
+#include <mach/octeon-model.h>
+#include <mach/cvmx-fuse.h>
+#include <mach/octeon-feature.h>
+#include <mach/cvmx-qlm.h>
+#include <mach/octeon_qlm.h>
+#include <mach/cvmx-pcie.h>
+#include <mach/cvmx-coremask.h>
+
+#include <mach/cvmx-agl-defs.h>
+#include <mach/cvmx-bgxx-defs.h>
+#include <mach/cvmx-ciu-defs.h>
+#include <mach/cvmx-gmxx-defs.h>
+#include <mach/cvmx-gserx-defs.h>
+#include <mach/cvmx-ilk-defs.h>
+#include <mach/cvmx-ipd-defs.h>
+#include <mach/cvmx-pcsx-defs.h>
+#include <mach/cvmx-pcsxx-defs.h>
+#include <mach/cvmx-pki-defs.h>
+#include <mach/cvmx-pko-defs.h>
+#include <mach/cvmx-xcv-defs.h>
+
+#include <mach/cvmx-hwpko.h>
+#include <mach/cvmx-ilk.h>
+#include <mach/cvmx-pki.h>
+#include <mach/cvmx-pko3.h>
+#include <mach/cvmx-pko3-queue.h>
+#include <mach/cvmx-pko3-resources.h>
+
+#include <mach/cvmx-helper.h>
+#include <mach/cvmx-helper-board.h>
+#include <mach/cvmx-helper-cfg.h>
+
+#include <mach/cvmx-helper-bgx.h>
+#include <mach/cvmx-helper-cfg.h>
+#include <mach/cvmx-helper-util.h>
+#include <mach/cvmx-helper-pki.h>
+
+static const int debug;
+static const bool __native_le;
+
+#define CVMX_DUMP_REGX(reg)                                            \
+       if (debug)                                                      \
+               debug("%s=%#llx\n", #reg, (long long)csr_rd_node(node, reg))
+
+static int cvmx_pko_setup_macs(int node);
+
+/*
+ * PKO descriptor queue operation error string
+ *
+ * @param dqstatus is the enumeration returned from hardware,
+ *       PKO_QUERY_RTN_S[DQSTATUS].
+ *
+ * @return static constant string error description
+ */
+const char *pko_dqstatus_error(pko_query_dqstatus_t dqstatus)
+{
+       char *str = "PKO Undefined error";
+
+       switch (dqstatus) {
+       case PKO_DQSTATUS_PASS:
+               str = "No error";
+               break;
+       case PKO_DQSTATUS_BADSTATE:
+               str = "PKO queue not ready";
+               break;
+       case PKO_DQSTATUS_NOFPABUF:
+               str = "PKO failed to allocate buffer from FPA";
+               break;
+       case PKO_DQSTATUS_NOPKOBUF:
+               str = "PKO out of buffers";
+               break;
+       case PKO_DQSTATUS_FAILRTNPTR:
+               str = "PKO failed to return buffer to FPA";
+               break;
+       case PKO_DQSTATUS_ALREADY:
+               str = "PKO queue already opened";
+               break;
+       case PKO_DQSTATUS_NOTCREATED:
+               str = "PKO queue has not been created";
+               break;
+       case PKO_DQSTATUS_NOTEMPTY:
+               str = "PKO queue is not empty";
+               break;
+       case PKO_DQSTATUS_SENDPKTDROP:
+               str = "Illegal PKO command construct";
+               break;
+       }
+       return str;
+}
+
+/*
+ * PKO global initialization for 78XX.
+ *
+ * @param node is the node on which PKO block is initialized.
+ * @return none.
+ */
+int cvmx_pko3_hw_init_global(int node, uint16_t aura)
+{
+       cvmx_pko_dpfi_flush_t pko_flush;
+       cvmx_pko_dpfi_fpa_aura_t pko_aura;
+       cvmx_pko_dpfi_ena_t dpfi_enable;
+       cvmx_pko_ptf_iobp_cfg_t ptf_iobp_cfg;
+       cvmx_pko_pdm_cfg_t pko_pdm_cfg;
+       cvmx_pko_enable_t pko_enable;
+       cvmx_pko_dpfi_status_t dpfi_status;
+       cvmx_pko_status_t pko_status;
+       cvmx_pko_shaper_cfg_t shaper_cfg;
+       u64 cycles;
+       const unsigned int timeout = 100; /* 100 milliseconds */
+
+       if (node != (aura >> 10))
+               cvmx_printf("WARNING: AURA vs PKO node mismatch\n");
+
+       pko_enable.u64 = csr_rd_node(node, CVMX_PKO_ENABLE);
+       if (pko_enable.s.enable) {
+               cvmx_printf("WARNING: %s: PKO already enabled on node %u\n",
+                           __func__, node);
+               return 0;
+       }
+       /* Enable color awareness. */
+       shaper_cfg.u64 = csr_rd_node(node, CVMX_PKO_SHAPER_CFG);
+       shaper_cfg.s.color_aware = 1;
+       csr_wr_node(node, CVMX_PKO_SHAPER_CFG, shaper_cfg.u64);
+
+       /* Clear FLUSH command to be sure */
+       pko_flush.u64 = 0;
+       pko_flush.s.flush_en = 0;
+       csr_wr_node(node, CVMX_PKO_DPFI_FLUSH, pko_flush.u64);
+
+       /* set the aura number in pko, use aura node from parameter */
+       pko_aura.u64 = 0;
+       pko_aura.s.node = aura >> 10;
+       pko_aura.s.laura = aura;
+       csr_wr_node(node, CVMX_PKO_DPFI_FPA_AURA, pko_aura.u64);
+
+       CVMX_DUMP_REGX(CVMX_PKO_DPFI_FPA_AURA);
+
+       dpfi_enable.u64 = 0;
+       dpfi_enable.s.enable = 1;
+       csr_wr_node(node, CVMX_PKO_DPFI_ENA, dpfi_enable.u64);
+
+       /* Prepare timeout */
+       cycles = get_timer(0);
+
+       /* Wait until all pointers have been returned */
+       do {
+               pko_status.u64 = csr_rd_node(node, CVMX_PKO_STATUS);
+               if (get_timer(cycles) > timeout)
+                       break;
+       } while (!pko_status.s.pko_rdy);
+
+       if (!pko_status.s.pko_rdy) {
+               dpfi_status.u64 = csr_rd_node(node, CVMX_PKO_DPFI_STATUS);
+               cvmx_printf("ERROR: %s: PKO DFPI failed, PKO_STATUS=%#llx 
DPFI_STATUS=%#llx\n",
+                           __func__, (unsigned long long)pko_status.u64,
+                           (unsigned long long)dpfi_status.u64);
+               return -1;
+       }
+
+       /* Set max outstanding requests in IOBP for any FIFO.*/
+       ptf_iobp_cfg.u64 = csr_rd_node(node, CVMX_PKO_PTF_IOBP_CFG);
+       if (OCTEON_IS_MODEL(OCTEON_CN78XX))
+               ptf_iobp_cfg.s.max_read_size = 0x10; /* Recommended by HRM.*/
+       else
+               /* Reduce the value from recommended 0x10 to avoid
+                * getting "underflow" condition in the BGX TX FIFO.
+                */
+               ptf_iobp_cfg.s.max_read_size = 3;
+       csr_wr_node(node, CVMX_PKO_PTF_IOBP_CFG, ptf_iobp_cfg.u64);
+
+       /* Set minimum packet size per Ethernet standard */
+       pko_pdm_cfg.u64 = 0;
+       pko_pdm_cfg.s.pko_pad_minlen = 0x3c; /* 60 bytes before FCS */
+       csr_wr_node(node, CVMX_PKO_PDM_CFG, pko_pdm_cfg.u64);
+
+       /* Initialize MACs and FIFOs */
+       cvmx_pko_setup_macs(node);
+
+       /* enable PKO, although interfaces and queues are not up yet */
+       pko_enable.u64 = 0;
+       pko_enable.s.enable = 1;
+       csr_wr_node(node, CVMX_PKO_ENABLE, pko_enable.u64);
+
+       /* PKO_RDY set indicates successful initialization */
+       pko_status.u64 = csr_rd_node(node, CVMX_PKO_STATUS);
+       if (pko_status.s.pko_rdy)
+               return 0;
+
+       cvmx_printf("ERROR: %s: failed, PKO_STATUS=%#llx\n", __func__,
+                   (unsigned long long)pko_status.u64);
+       return -1;
+}
+
+/**
+ * Shutdown the entire PKO
+ */
+int cvmx_pko3_hw_disable(int node)
+{
+       cvmx_pko_dpfi_flush_t pko_flush;
+       cvmx_pko_dpfi_status_t dpfi_status;
+       cvmx_pko_dpfi_ena_t dpfi_enable;
+       cvmx_pko_enable_t pko_enable;
+       cvmx_pko_status_t pko_status;
+       u64 cycles;
+       const unsigned int timeout = 10; /* 10 milliseconds */
+       unsigned int mac_num, fifo, i;
+       unsigned int null_mac_num, null_fifo_num, fifo_grp_count, pq_count;
+
+       (void)pko_status;
+
+       /* Wait until there are no in-flight packets */
+       for (i = mac_num = 0; mac_num < __cvmx_pko3_num_macs(); mac_num++) {
+               cvmx_pko_ptfx_status_t ptf_status;
+
+               ptf_status.u64 =
+                       csr_rd_node(node, CVMX_PKO_PTFX_STATUS(mac_num));
+               if (debug)
+                       debug("%s: MAC %u in-flight %u total %u\n", __func__,
+                             mac_num, ptf_status.s.in_flight_cnt,
+                             ptf_status.s.total_in_flight_cnt);
+               if (ptf_status.s.mac_num == 0x1f)
+                       continue;
+               if (ptf_status.s.in_flight_cnt != 0) {
+                       cvmx_printf("WARNING: %s: MAC %d in-flight %d\n",
+                                   __func__, mac_num,
+                                   ptf_status.s.in_flight_cnt);
+                       mac_num--;
+                       udelay(1000);
+               }
+       }
+
+       /* disable PKO - all packets should be out by now */
+       pko_enable.u64 = 0;
+       pko_enable.s.enable = 0;
+       csr_wr_node(node, CVMX_PKO_ENABLE, pko_enable.u64);
+
+       /* Assign NULL MAC# for L1/SQ disabled state */
+       if (OCTEON_IS_MODEL(OCTEON_CN73XX)) {
+               null_mac_num = 0x0f;
+               null_fifo_num = 0x1f;
+               fifo_grp_count = 4;
+               pq_count = 16;
+       } else if (OCTEON_IS_MODEL(OCTEON_CNF75XX)) {
+               null_mac_num = 0x0a;
+               null_fifo_num = 0x1f;
+               fifo_grp_count = 4;
+               pq_count = 16;
+       } else {
+               null_mac_num = 0x1c;
+               null_fifo_num = 0x1f;
+               fifo_grp_count = 8;
+               pq_count = 32;
+       }
+
+       /* Reset L1_PQ */
+       for (i = 0; i < pq_count; i++) {
+               cvmx_pko_l1_sqx_topology_t pko_l1_topology;
+               cvmx_pko_l1_sqx_shape_t pko_l1_shape;
+               cvmx_pko_l1_sqx_link_t pko_l1_link;
+
+               pko_l1_topology.u64 = 0;
+               pko_l1_topology.s.link = null_mac_num;
+               csr_wr_node(node, CVMX_PKO_L1_SQX_TOPOLOGY(i),
+                           pko_l1_topology.u64);
+
+               pko_l1_shape.u64 = 0;
+               pko_l1_shape.s.link = null_mac_num;
+               csr_wr_node(node, CVMX_PKO_L1_SQX_SHAPE(i), pko_l1_shape.u64);
+
+               pko_l1_link.u64 = 0;
+               pko_l1_link.s.link = null_mac_num;
+               csr_wr_node(node, CVMX_PKO_L1_SQX_LINK(i), pko_l1_link.u64);
+       }
+
+       /* Reset all MAC configurations */
+       for (mac_num = 0; mac_num < __cvmx_pko3_num_macs(); mac_num++) {
+               cvmx_pko_macx_cfg_t pko_mac_cfg;
+
+               pko_mac_cfg.u64 = 0;
+               pko_mac_cfg.s.fifo_num = null_fifo_num;
+               csr_wr_node(node, CVMX_PKO_MACX_CFG(mac_num), pko_mac_cfg.u64);
+       }
+
+       /* Reset all FIFO groups */
+       for (fifo = 0; fifo < fifo_grp_count; fifo++) {
+               cvmx_pko_ptgfx_cfg_t pko_ptgfx_cfg;
+
+               pko_ptgfx_cfg.u64 = csr_rd_node(node, CVMX_PKO_PTGFX_CFG(fifo));
+               /* Simulator asserts if an unused group is reset */
+               if (pko_ptgfx_cfg.u64 == 0)
+                       continue;
+               pko_ptgfx_cfg.u64 = 0;
+               pko_ptgfx_cfg.s.reset = 1;
+               csr_wr_node(node, CVMX_PKO_PTGFX_CFG(fifo), pko_ptgfx_cfg.u64);
+       }
+
+       /* Set FLUSH_EN to return cached pointers to FPA */
+       pko_flush.u64 = 0;
+       pko_flush.s.flush_en = 1;
+       csr_wr_node(node, CVMX_PKO_DPFI_FLUSH, pko_flush.u64);
+
+       /* Prepare timeout */
+       //      cycles = cvmx_get_cycle();
+       //      cycles += cvmx_clock_get_rate(CVMX_CLOCK_CORE)/1000 * timeout;
+       cycles = get_timer(0);
+
+       /* Wait until all pointers have been returned */
+       do {
+               dpfi_status.u64 = csr_rd_node(node, CVMX_PKO_DPFI_STATUS);
+               //              if (cycles < cvmx_get_cycle())
+               if (get_timer(cycles) > timeout)
+                       break;
+       } while (!dpfi_status.s.cache_flushed);
+
+       /* disable PKO buffer manager, should return all buffers to FPA */
+       dpfi_enable.u64 = 0;
+       dpfi_enable.s.enable = 0;
+       csr_wr_node(node, CVMX_PKO_DPFI_ENA, dpfi_enable.u64);
+
+       CVMX_DUMP_REGX(CVMX_PKO_DPFI_ENA);
+       CVMX_DUMP_REGX(CVMX_PKO_DPFI_STATUS);
+       CVMX_DUMP_REGX(CVMX_PKO_STATUS);
+
+       /* Clear the FLUSH_EN bit, as we are done */
+       pko_flush.u64 = 0;
+       csr_wr_node(node, CVMX_PKO_DPFI_FLUSH, pko_flush.u64);
+       CVMX_DUMP_REGX(CVMX_PKO_DPFI_FLUSH);
+
+       if (dpfi_status.s.cache_flushed == 0) {
+               cvmx_printf("%s: ERROR: timeout waiting for PKO3 ptr flush\n",
+                           __func__);
+               return -1;
+       }
+
+       return 0;
+}
+
+/*
+ * Configure Channel credit level in PKO.
+ *
+ * @param node is to specify the node to which this configuration is applied.
+ * @param level specifies the level at which pko channel queues will be 
configured,
+ * @return returns 0 if successful and -1 on failure.
+ */
+int cvmx_pko3_channel_credit_level(int node, enum cvmx_pko3_level_e level)
+{
+       union cvmx_pko_channel_level channel_level;
+
+       channel_level.u64 = 0;
+
+       if (level == CVMX_PKO_L2_QUEUES)
+               channel_level.s.cc_level = 0;
+       else if (level == CVMX_PKO_L3_QUEUES)
+               channel_level.s.cc_level = 1;
+       else
+               return -1;
+
+       csr_wr_node(node, CVMX_PKO_CHANNEL_LEVEL, channel_level.u64);
+
+       return 0;
+}
+
+/** Open configured descriptor queues before queueing packets into them.
+ *
+ * @param node is to specify the node to which this configuration is applied.
+ * @param dq is the descriptor queue number to be opened.
+ * @return returns 0 on success or -1 on failure.
+ */
+int cvmx_pko_dq_open(int node, int dq)
+{
+       cvmx_pko_query_rtn_t pko_status;
+       pko_query_dqstatus_t dqstatus;
+       cvmx_pko3_dq_params_t *p_param;
+
+       if (debug)
+               debug("%s: DEBUG: dq %u\n", __func__, dq);
+
+       __cvmx_pko3_dq_param_setup(node);
+
+       pko_status = __cvmx_pko3_do_dma(node, dq, NULL, 0, CVMX_PKO_DQ_OPEN);
+
+       dqstatus = pko_status.s.dqstatus;
+
+       if (dqstatus == PKO_DQSTATUS_ALREADY)
+               return 0;
+       if (dqstatus != PKO_DQSTATUS_PASS) {
+               cvmx_printf("%s: ERROR: Failed to open dq :%u: %s\n", __func__,
+                           dq, pko_dqstatus_error(dqstatus));
+               return -1;
+       }
+
+       /* Setup the descriptor queue software parameters */
+       p_param = cvmx_pko3_dq_parameters(node, dq);
+       if (p_param) {
+               p_param->depth = pko_status.s.depth;
+               if (p_param->limit == 0)
+                       p_param->limit = 1024; /* last-resort default */
+       }
+
+       return 0;
+}
+
+/**
+ * Close a descriptor queue
+ *
+ * @param node is to specify the node to which this configuration is applied.
+ * @param dq is the descriptor queue number to be opened.
+ * @return returns 0 on success or -1 on failure.
+ *
+ * This should be called before changing the DQ parent link, topology,
+ * or when shutting down the PKO.
+ */
+int cvmx_pko3_dq_close(int node, int dq)
+{
+       cvmx_pko_query_rtn_t pko_status;
+       pko_query_dqstatus_t dqstatus;
+
+       if (debug)
+               debug("%s: DEBUG: dq %u\n", __func__, dq);
+
+       pko_status = __cvmx_pko3_do_dma(node, dq, NULL, 0, CVMX_PKO_DQ_CLOSE);
+
+       dqstatus = pko_status.s.dqstatus;
+
+       if (dqstatus == PKO_DQSTATUS_NOTCREATED)
+               return 0;
+
+       if (dqstatus != PKO_DQSTATUS_PASS) {
+               cvmx_printf("WARNING: %s: Failed to close dq :%u: %s\n",
+                           __func__, dq, pko_dqstatus_error(dqstatus));
+               debug("DEBUG: %s: dq %u depth %u\n", __func__, dq,
+                     (unsigned int)pko_status.s.depth);
+       }
+
+       return 0;
+}
+
+/**
+ * Drain a descriptor queue
+ *
+ * Before closing a DQ, this call will drain all pending traffic
+ * on the DQ to the NULL MAC, which will circumvent any traffic
+ * shaping and flow control to quickly reclaim all packet buffers.
+ *
+ * @param node is to specify the node to which this configuration is applied.
+ * @param dq is the descriptor queue number to be drained.
+ */
+void cvmx_pko3_dq_drain(int node, int dq)
+{
+       cvmx_pko_dqx_sw_xoff_t rxoff;
+
+       rxoff.u64 = 0;
+       rxoff.s.drain_null_link = 1;
+       rxoff.s.drain = 1;
+       rxoff.s.xoff = 0;
+
+       csr_wr_node(node, CVMX_PKO_DQX_SW_XOFF(dq), rxoff.u64);
+
+       udelay(100);
+
+       rxoff.u64 = 0;
+       csr_wr_node(node, CVMX_PKO_DQX_SW_XOFF(dq), rxoff.u64);
+}
+
+/**
+ * Query a descriptor queue
+ *
+ * @param node is to specify the node to which this configuration is applied.
+ * @param dq is the descriptor queue number to be opened.
+ * @return returns the descriptor queue depth on success or -1 on failure.
+ *
+ * This should be called before changing the DQ parent link, topology,
+ * or when shutting down the PKO.
+ */
+int cvmx_pko3_dq_query(int node, int dq)
+{
+       cvmx_pko_query_rtn_t pko_status;
+       pko_query_dqstatus_t dqstatus;
+
+       pko_status = __cvmx_pko3_do_dma(node, dq, NULL, 0, CVMX_PKO_DQ_QUERY);
+
+       dqstatus = pko_status.s.dqstatus;
+
+       if (dqstatus != PKO_DQSTATUS_PASS) {
+               cvmx_printf("%s: ERROR: Failed to query dq :%u: %s\n", __func__,
+                           dq, pko_dqstatus_error(dqstatus));
+               return -1;
+       }
+
+       /* Temp: debug for HW */
+       if (pko_status.s.depth > 0)
+               debug("%s: DEBUG: dq %u depth %u\n", __func__, dq,
+                     (unsigned int)pko_status.s.depth);
+
+       return pko_status.s.depth;
+}
+
+/*
+ * PKO initialization of MACs and FIFOs
+ *
+ * All MACs are configured and assigned a specific FIFO,
+ * and each FIFO is configured with size for a best utilization
+ * of available FIFO resources.
+ *
+ * @param node is to specify which node's pko block for this setup.
+ * @return returns 0 if successful and -1 on failure.
+ *
+ * Note: This function contains model-specific code.
+ */
+static int cvmx_pko_setup_macs(int node)
+{
+       unsigned int interface;
+       unsigned int port, num_ports;
+       unsigned int mac_num, fifo, pri, cnt;
+       cvmx_helper_interface_mode_t mode;
+       const unsigned int num_interfaces =
+               cvmx_helper_get_number_of_interfaces();
+       u8 fifo_group_cfg[8];
+       u8 fifo_group_spd[8];
+       unsigned int fifo_count = 0;
+       unsigned int max_fifos = 0, fifo_groups = 0;
+       struct {
+               u8 fifo_cnt;
+               u8 fifo_id;
+               u8 pri;
+               u8 spd;
+               u8 mac_fifo_cnt;
+       } cvmx_pko3_mac_table[32];
+
+       if (OCTEON_IS_MODEL(OCTEON_CN78XX)) {
+               max_fifos = 28;  /* exclusive of NULL FIFO */
+               fifo_groups = 8; /* inclusive of NULL PTGF */
+       }
+       if (OCTEON_IS_MODEL(OCTEON_CN73XX) || OCTEON_IS_MODEL(OCTEON_CNF75XX)) {
+               max_fifos = 16;
+               fifo_groups = 5;
+       }
+
+       /* Initialize FIFO allocation table */
+       memset(&fifo_group_cfg, 0, sizeof(fifo_group_cfg));
+       memset(&fifo_group_spd, 0, sizeof(fifo_group_spd));
+       memset(cvmx_pko3_mac_table, 0, sizeof(cvmx_pko3_mac_table));
+
+       /* Initialize all MACs as disabled */
+       for (mac_num = 0; mac_num < __cvmx_pko3_num_macs(); mac_num++) {
+               cvmx_pko3_mac_table[mac_num].pri = 0;
+               cvmx_pko3_mac_table[mac_num].fifo_cnt = 0;
+               cvmx_pko3_mac_table[mac_num].fifo_id = 0x1f;
+       }
+
+       for (interface = 0; interface < num_interfaces; interface++) {
+               int xiface =
+                       cvmx_helper_node_interface_to_xiface(node, interface);
+               /* Interface type for ALL interfaces */
+               mode = cvmx_helper_interface_get_mode(xiface);
+               num_ports = cvmx_helper_interface_enumerate(xiface);
+
+               if (mode == CVMX_HELPER_INTERFACE_MODE_DISABLED)
+                       continue;
+               /*
+                * Non-BGX interfaces:
+                * Each of these interfaces has a single MAC really.
+                */
+               if (mode == CVMX_HELPER_INTERFACE_MODE_ILK ||
+                   mode == CVMX_HELPER_INTERFACE_MODE_NPI ||
+                   mode == CVMX_HELPER_INTERFACE_MODE_LOOP)
+                       num_ports = 1;
+
+               for (port = 0; port < num_ports; port++) {
+                       int i;
+
+                       /* Get the per-port mode for BGX-interfaces */
+                       if (interface < CVMX_HELPER_MAX_GMX)
+                               mode = cvmx_helper_bgx_get_mode(xiface, port);
+                       /* In MIXED mode, LMACs can run different protocols */
+
+                       /* convert interface/port to mac number */
+                       i = __cvmx_pko3_get_mac_num(xiface, port);
+                       if (i < 0 || i >= (int)__cvmx_pko3_num_macs()) {
+                               cvmx_printf("%s: ERROR: interface %d:%u port %d 
has no MAC %d/%d\n",
+                                           __func__, node, interface, port, i,
+                                           __cvmx_pko3_num_macs());
+                               continue;
+                       }
+
+                       if (mode == CVMX_HELPER_INTERFACE_MODE_RXAUI) {
+                               unsigned int bgx_fifo_size =
+                                       __cvmx_helper_bgx_fifo_size(xiface,
+                                                                   port);
+
+                               cvmx_pko3_mac_table[i].mac_fifo_cnt =
+                                       bgx_fifo_size /
+                                       (CVMX_BGX_TX_FIFO_SIZE / 4);
+                               cvmx_pko3_mac_table[i].pri = 2;
+                               cvmx_pko3_mac_table[i].spd = 10;
+                               cvmx_pko3_mac_table[i].fifo_cnt = 2;
+                       } else if (mode == CVMX_HELPER_INTERFACE_MODE_XLAUI) {
+                               unsigned int bgx_fifo_size =
+                                       __cvmx_helper_bgx_fifo_size(xiface,
+                                                                   port);
+
+                               cvmx_pko3_mac_table[i].mac_fifo_cnt =
+                                       bgx_fifo_size /
+                                       (CVMX_BGX_TX_FIFO_SIZE / 4);
+                               cvmx_pko3_mac_table[i].pri = 4;
+                               cvmx_pko3_mac_table[i].spd = 40;
+                               cvmx_pko3_mac_table[i].fifo_cnt = 4;
+                       } else if (mode == CVMX_HELPER_INTERFACE_MODE_XAUI) {
+                               unsigned int bgx_fifo_size =
+                                       __cvmx_helper_bgx_fifo_size(xiface,
+                                                                   port);
+
+                               cvmx_pko3_mac_table[i].mac_fifo_cnt =
+                                       bgx_fifo_size /
+                                       (CVMX_BGX_TX_FIFO_SIZE / 4);
+                               cvmx_pko3_mac_table[i].pri = 3;
+                               cvmx_pko3_mac_table[i].fifo_cnt = 4;
+                               /* DXAUI at 20G, or XAU at 10G */
+                               cvmx_pko3_mac_table[i].spd = 20;
+                       } else if (mode == CVMX_HELPER_INTERFACE_MODE_XFI) {
+                               unsigned int bgx_fifo_size =
+                                       __cvmx_helper_bgx_fifo_size(xiface,
+                                                                   port);
+
+                               cvmx_pko3_mac_table[i].mac_fifo_cnt =
+                                       bgx_fifo_size /
+                                       (CVMX_BGX_TX_FIFO_SIZE / 4);
+                               cvmx_pko3_mac_table[i].pri = 3;
+                               cvmx_pko3_mac_table[i].fifo_cnt = 4;
+                               cvmx_pko3_mac_table[i].spd = 10;
+                       } else if (mode == CVMX_HELPER_INTERFACE_MODE_LOOP) {
+                               cvmx_pko3_mac_table[i].fifo_cnt = 1;
+                               cvmx_pko3_mac_table[i].pri = 1;
+                               cvmx_pko3_mac_table[i].spd = 1;
+                               cvmx_pko3_mac_table[i].mac_fifo_cnt = 1;
+                       } else if (mode == CVMX_HELPER_INTERFACE_MODE_ILK ||
+                                  mode == CVMX_HELPER_INTERFACE_MODE_SRIO) {
+                               cvmx_pko3_mac_table[i].fifo_cnt = 4;
+                               cvmx_pko3_mac_table[i].pri = 3;
+                               /* ILK/SRIO: speed depends on lane count */
+                               cvmx_pko3_mac_table[i].spd = 40;
+                               cvmx_pko3_mac_table[i].mac_fifo_cnt = 4;
+                       } else if (mode == CVMX_HELPER_INTERFACE_MODE_NPI) {
+                               cvmx_pko3_mac_table[i].fifo_cnt = 4;
+                               cvmx_pko3_mac_table[i].pri = 2;
+                               /* Actual speed depends on PCIe lanes/mode */
+                               cvmx_pko3_mac_table[i].spd = 50;
+                               /* SLI Tx FIFO size to be revisitted */
+                               cvmx_pko3_mac_table[i].mac_fifo_cnt = 1;
+                       } else {
+                               /* Other BGX interface modes: SGMII/RGMII */
+                               unsigned int bgx_fifo_size =
+                                       __cvmx_helper_bgx_fifo_size(xiface,
+                                                                   port);
+
+                               cvmx_pko3_mac_table[i].mac_fifo_cnt =
+                                       bgx_fifo_size /
+                                       (CVMX_BGX_TX_FIFO_SIZE / 4);
+                               cvmx_pko3_mac_table[i].fifo_cnt = 1;
+                               cvmx_pko3_mac_table[i].pri = 1;
+                               cvmx_pko3_mac_table[i].spd = 1;
+                       }
+
+                       if (debug)
+                               debug("%s: intf %d:%u port %u %s mac %02u cnt 
%u macfifo %uk spd %u\n",
+                                     __func__, node, interface, port,
+                                     
cvmx_helper_interface_mode_to_string(mode),
+                                     i, cvmx_pko3_mac_table[i].fifo_cnt,
+                                     cvmx_pko3_mac_table[i].mac_fifo_cnt * 8,
+                                     cvmx_pko3_mac_table[i].spd);
+
+               } /* for port */
+       }         /* for interface */
+
+       /* Count the number of requested FIFOs */
+       for (fifo_count = mac_num = 0; mac_num < __cvmx_pko3_num_macs();
+            mac_num++)
+               fifo_count += cvmx_pko3_mac_table[mac_num].fifo_cnt;
+
+       if (debug)
+               debug("%s: initially requested FIFO count %u\n", __func__,
+                     fifo_count);
+
+       /* Heuristically trim FIFO count to fit in available number */
+       pri = 1;
+       cnt = 4;
+       while (fifo_count > max_fifos) {
+               for (mac_num = 0; mac_num < __cvmx_pko3_num_macs(); mac_num++) {
+                       if (cvmx_pko3_mac_table[mac_num].fifo_cnt == cnt &&
+                           cvmx_pko3_mac_table[mac_num].pri <= pri) {
+                               cvmx_pko3_mac_table[mac_num].fifo_cnt >>= 1;
+                               fifo_count -=
+                                       cvmx_pko3_mac_table[mac_num].fifo_cnt;
+                       }
+                       if (fifo_count <= max_fifos)
+                               break;
+               }
+               if (pri >= 4) {
+                       pri = 1;
+                       cnt >>= 1;
+               } else {
+                       pri++;
+               }
+               if (cnt == 0)
+                       break;
+       }
+
+       if (debug)
+               debug("%s: adjusted FIFO count %u\n", __func__, fifo_count);
+
+       /* Special case for NULL Virtual FIFO */
+       fifo_group_cfg[fifo_groups - 1] = 0;
+       /* there is no MAC connected to NULL FIFO */
+
+       /* Configure MAC units, and attach a FIFO to each */
+       for (fifo = 0, cnt = 4; cnt > 0; cnt >>= 1) {
+               unsigned int g;
+
+               for (mac_num = 0; mac_num < __cvmx_pko3_num_macs(); mac_num++) {
+                       if (cvmx_pko3_mac_table[mac_num].fifo_cnt < cnt ||
+                           cvmx_pko3_mac_table[mac_num].fifo_id != 0x1f)
+                               continue;
+
+                       /* Attach FIFO to MAC */
+                       cvmx_pko3_mac_table[mac_num].fifo_id = fifo;
+                       g = fifo >> 2;
+                       /* Sum speed for FIFO group */
+                       fifo_group_spd[g] += cvmx_pko3_mac_table[mac_num].spd;
+
+                       if (cnt == 4)
+                               fifo_group_cfg[g] = 4; /* 10k,0,0,0 */
+                       else if (cnt == 2 && (fifo & 0x3) == 0)
+                               fifo_group_cfg[g] = 3; /* 5k,0,5k,0 */
+                       else if (cnt == 2 && fifo_group_cfg[g] == 3)
+                               /* no change */;
+                       else if (cnt == 1 && (fifo & 0x2) &&
+                                fifo_group_cfg[g] == 3)
+                               fifo_group_cfg[g] = 1; /* 5k,0,2.5k 2.5k*/
+                       else if (cnt == 1 && (fifo & 0x3) == 0x3)
+                               /* no change */;
+                       else if (cnt == 1)
+                               fifo_group_cfg[g] = 0; /* 2.5k x 4 */
+                       else
+                               cvmx_printf("ERROR: %s: internal error\n",
+                                           __func__);
+
+                       fifo += cnt;
+               }
+       }
+
+       /* Check if there was no error in FIFO allocation */
+       if (fifo > max_fifos) {
+               cvmx_printf("ERROR: %s: Internal error FIFO %u\n", __func__,
+                           fifo);
+               return -1;
+       }
+
+       if (debug)
+               debug("%s: used %u of FIFOs\n", __func__, fifo);
+
+       /* Now configure all FIFO groups */
+       for (fifo = 0; fifo < fifo_groups; fifo++) {
+               cvmx_pko_ptgfx_cfg_t pko_ptgfx_cfg;
+
+               pko_ptgfx_cfg.u64 = csr_rd_node(node, CVMX_PKO_PTGFX_CFG(fifo));
+               if (pko_ptgfx_cfg.s.size != fifo_group_cfg[fifo])
+                       pko_ptgfx_cfg.s.reset = 1;
+               pko_ptgfx_cfg.s.size = fifo_group_cfg[fifo];
+               if (fifo_group_spd[fifo] >= 40)
+                       if (pko_ptgfx_cfg.s.size >= 3)
+                               pko_ptgfx_cfg.s.rate = 3; /* 50 Gbps */
+                       else
+                               pko_ptgfx_cfg.s.rate = 2; /* 25 Gbps */
+               else if (fifo_group_spd[fifo] >= 20)
+                       pko_ptgfx_cfg.s.rate = 2; /* 25 Gbps */
+               else if (fifo_group_spd[fifo] >= 10)
+                       pko_ptgfx_cfg.s.rate = 1; /* 12.5 Gbps */
+               else
+                       pko_ptgfx_cfg.s.rate = 0; /* 6.25 Gbps */
+
+               if (debug)
+                       debug("%s: FIFO %#x-%#x size=%u speed=%d rate=%d\n",
+                             __func__, fifo * 4, fifo * 4 + 3,
+                             pko_ptgfx_cfg.s.size, fifo_group_spd[fifo],
+                             pko_ptgfx_cfg.s.rate);
+
+               csr_wr_node(node, CVMX_PKO_PTGFX_CFG(fifo), pko_ptgfx_cfg.u64);
+               pko_ptgfx_cfg.s.reset = 0;
+               csr_wr_node(node, CVMX_PKO_PTGFX_CFG(fifo), pko_ptgfx_cfg.u64);
+       }
+
+       /* Configure all MACs assigned FIFO number */
+       for (mac_num = 0; mac_num < __cvmx_pko3_num_macs(); mac_num++) {
+               cvmx_pko_macx_cfg_t pko_mac_cfg;
+
+               if (debug)
+                       debug("%s: mac#%02u: fifo=%#x cnt=%u speed=%d\n",
+                             __func__, mac_num,
+                             cvmx_pko3_mac_table[mac_num].fifo_id,
+                             cvmx_pko3_mac_table[mac_num].fifo_cnt,
+                             cvmx_pko3_mac_table[mac_num].spd);
+
+               pko_mac_cfg.u64 = csr_rd_node(node, CVMX_PKO_MACX_CFG(mac_num));
+               pko_mac_cfg.s.fifo_num = cvmx_pko3_mac_table[mac_num].fifo_id;
+               csr_wr_node(node, CVMX_PKO_MACX_CFG(mac_num), pko_mac_cfg.u64);
+       }
+
+       /* Setup PKO MCI0/MCI1/SKID credits */
+       for (mac_num = 0; mac_num < __cvmx_pko3_num_macs(); mac_num++) {
+               cvmx_pko_mci0_max_credx_t pko_mci0_max_cred;
+               cvmx_pko_mci1_max_credx_t pko_mci1_max_cred;
+               cvmx_pko_macx_cfg_t pko_mac_cfg;
+               unsigned int fifo_credit, mac_credit, skid_credit;
+               unsigned int pko_fifo_cnt, fifo_size;
+               unsigned int mac_fifo_cnt;
+               unsigned int tmp;
+               int saved_fifo_num;
+
+               pko_fifo_cnt = cvmx_pko3_mac_table[mac_num].fifo_cnt;
+               mac_fifo_cnt = cvmx_pko3_mac_table[mac_num].mac_fifo_cnt;
+
+               /* Skip unused MACs */
+               if (pko_fifo_cnt == 0)
+                       continue;
+
+               /* Check for sanity */
+               if (pko_fifo_cnt > 4)
+                       pko_fifo_cnt = 1;
+
+               fifo_size = (2 * 1024) + (1024 / 2); /* 2.5KiB */
+               fifo_credit = pko_fifo_cnt * fifo_size;
+
+               if (mac_num == 0) {
+                       /* loopback */
+                       mac_credit = 4096; /* From HRM Sec 13.0 */
+                       skid_credit = 0;
+               } else if (mac_num == 1) {
+                       /* DPI */
+                       mac_credit = 2 * 1024;
+                       skid_credit = 0;
+               } else if (octeon_has_feature(OCTEON_FEATURE_ILK) &&
+                          (mac_num & 0xfe) == 2) {
+                       /* ILK0, ILK1: MAC 2,3 */
+                       mac_credit = 4 * 1024; /* 4KB fifo */
+                       skid_credit = 0;
+               } else if (octeon_has_feature(OCTEON_FEATURE_SRIO) &&
+                          (mac_num >= 6) && (mac_num <= 9)) {
+                       /* SRIO0, SRIO1: MAC 6..9 */
+                       mac_credit = 1024 / 2;
+                       skid_credit = 0;
+               } else {
+                       /* BGX */
+                       mac_credit = mac_fifo_cnt * 8 * 1024;
+                       skid_credit = mac_fifo_cnt * 256;
+               }
+
+               if (debug)
+                       debug("%s: mac %u pko_fifo_credit=%u mac_credit=%u\n",
+                             __func__, mac_num, fifo_credit, mac_credit);
+
+               tmp = (fifo_credit + mac_credit) / 16;
+               pko_mci0_max_cred.u64 = 0;
+               pko_mci0_max_cred.s.max_cred_lim = tmp;
+
+               /* Check for overflow */
+               if (pko_mci0_max_cred.s.max_cred_lim != tmp) {
+                       cvmx_printf("WARNING: %s: MCI0 credit overflow\n",
+                                   __func__);
+                       pko_mci0_max_cred.s.max_cred_lim = 0xfff;
+               }
+
+               /* Pass 2 PKO hardware does not use the MCI0 credits */
+               if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X))
+                       csr_wr_node(node, CVMX_PKO_MCI0_MAX_CREDX(mac_num),
+                                   pko_mci0_max_cred.u64);
+
+               /* The original CSR formula is the correct one after all */
+               tmp = (mac_credit) / 16;
+               pko_mci1_max_cred.u64 = 0;
+               pko_mci1_max_cred.s.max_cred_lim = tmp;
+
+               /* Check for overflow */
+               if (pko_mci1_max_cred.s.max_cred_lim != tmp) {
+                       cvmx_printf("WARNING: %s: MCI1 credit overflow\n",
+                                   __func__);
+                       pko_mci1_max_cred.s.max_cred_lim = 0xfff;
+               }
+
+               csr_wr_node(node, CVMX_PKO_MCI1_MAX_CREDX(mac_num),
+                           pko_mci1_max_cred.u64);
+
+               tmp = (skid_credit / 256) >> 1; /* valid 0,1,2 */
+               pko_mac_cfg.u64 = csr_rd_node(node, CVMX_PKO_MACX_CFG(mac_num));
+
+               /* The PKO_MACX_CFG bits cannot be changed unless FIFO_MUM=0x1f 
(unused fifo) */
+               saved_fifo_num = pko_mac_cfg.s.fifo_num;
+               pko_mac_cfg.s.fifo_num = 0x1f;
+               pko_mac_cfg.s.skid_max_cnt = tmp;
+               csr_wr_node(node, CVMX_PKO_MACX_CFG(mac_num), pko_mac_cfg.u64);
+
+               pko_mac_cfg.u64 = csr_rd_node(node, CVMX_PKO_MACX_CFG(mac_num));
+               pko_mac_cfg.s.fifo_num = saved_fifo_num;
+               csr_wr_node(node, CVMX_PKO_MACX_CFG(mac_num), pko_mac_cfg.u64);
+
+               if (debug) {
+                       pko_mci0_max_cred.u64 =
+                               csr_rd_node(node, 
CVMX_PKO_MCI0_MAX_CREDX(mac_num));
+                       pko_mci1_max_cred.u64 =
+                               csr_rd_node(node, 
CVMX_PKO_MCI1_MAX_CREDX(mac_num));
+                       pko_mac_cfg.u64 =
+                               csr_rd_node(node, CVMX_PKO_MACX_CFG(mac_num));
+                       debug("%s: mac %u PKO_MCI0_MAX_CREDX=%u 
PKO_MCI1_MAX_CREDX=%u PKO_MACX_CFG[SKID_MAX_CNT]=%u\n",
+                             __func__, mac_num,
+                             pko_mci0_max_cred.s.max_cred_lim,
+                             pko_mci1_max_cred.s.max_cred_lim,
+                             pko_mac_cfg.s.skid_max_cnt);
+               }
+       } /* for mac_num */
+
+       return 0;
+}
+
+/**
+ * @INTERNAL
+ * Backward compatibility for collecting statistics from PKO3
+ *
+ * NOTE:
+ * The good stats are in BGX block.
+ */
+void cvmx_pko3_get_legacy_port_stats(u16 ipd_port, unsigned int clear,
+                                    cvmx_pko_port_status_t *stats)
+{
+       unsigned int dq, dq_base, dq_num;
+       unsigned int node = cvmx_get_node_num();
+
+       dq_base = cvmx_pko3_get_queue_base(ipd_port);
+       dq_num = cvmx_pko3_get_queue_num(ipd_port);
+
+       stats->packets = 0;
+       stats->octets = 0;
+       stats->doorbell = 0; /* NOTE: PKO3 does not have a doorbell */
+
+       for (dq = dq_base; dq < (dq_base + dq_num); dq++) {
+               cvmx_pko_dqx_packets_t pkts;
+               cvmx_pko_dqx_bytes_t byts;
+
+               /* NOTE: clearing of these counters is non-atomic */
+               pkts.u64 = csr_rd_node(node, CVMX_PKO_DQX_PACKETS(dq));
+               if (clear)
+                       csr_wr_node(node, CVMX_PKO_DQX_PACKETS(dq), 0ull);
+
+               byts.u64 = csr_rd_node(node, CVMX_PKO_DQX_BYTES(dq));
+               if (clear)
+                       csr_wr_node(node, CVMX_PKO_DQX_BYTES(dq), 0ull);
+
+               stats->packets += pkts.s.count;
+               stats->octets += byts.s.count;
+       } /* for dq */
+}
+
+/** Set MAC options
+ *
+ * The options supported are the parameters below:
+ *
+ * @param xiface The physical interface number
+ * @param index The physical sub-interface port
+ * @param fcs_enable Enable FCS generation
+ * @param pad_enable Enable padding to minimum packet size
+ * @param fcs_sop_off Number of bytes at start of packet to exclude from FCS
+ *
+ * The typical use for `fcs_sop_off` is when the interface is configured
+ * to use a header such as HighGig to precede every Ethernet packet,
+ * such a header usually does not partake in the CRC32 computation stream,
+ * and its size must be set with this parameter.
+ *
+ * @return Returns 0 on success, -1 if interface/port is invalid.
+ */
+int cvmx_pko3_interface_options(int xiface, int index, bool fcs_enable,
+                               bool pad_enable, unsigned int fcs_sop_off)
+{
+       int mac_num;
+       cvmx_pko_macx_cfg_t pko_mac_cfg;
+       unsigned int fifo_num;
+       struct cvmx_xiface xi = cvmx_helper_xiface_to_node_interface(xiface);
+
+       if (debug)
+               debug("%s: intf %u:%u/%u fcs=%d pad=%d\n", __func__, xi.node,
+                     xi.interface, index, fcs_enable, pad_enable);
+
+       mac_num = __cvmx_pko3_get_mac_num(xiface, index);
+       if (mac_num < 0) {
+               cvmx_printf("ERROR: %s: invalid interface %u:%u/%u\n", __func__,
+                           xi.node, xi.interface, index);
+               return -1;
+       }
+
+       pko_mac_cfg.u64 = csr_rd_node(xi.node, CVMX_PKO_MACX_CFG(mac_num));
+
+       /* If MAC is not assigned, return an error */
+       if (pko_mac_cfg.s.fifo_num == 0x1f) {
+               cvmx_printf("ERROR: %s: unused interface %u:%u/%u\n", __func__,
+                           xi.node, xi.interface, index);
+               return -1;
+       }
+
+       if (pko_mac_cfg.s.min_pad_ena == pad_enable &&
+           pko_mac_cfg.s.fcs_ena == fcs_enable) {
+               if (debug)
+                       debug("%s: mac %#x unchanged\n", __func__, mac_num);
+               return 0;
+       }
+
+       /* WORKAROUND: Pass1 won't allow change any bits unless FIFO_NUM=0x1f */
+       fifo_num = pko_mac_cfg.s.fifo_num;
+       pko_mac_cfg.s.fifo_num = 0x1f;
+
+       pko_mac_cfg.s.min_pad_ena = pad_enable;
+       pko_mac_cfg.s.fcs_ena = fcs_enable;
+       pko_mac_cfg.s.fcs_sop_off = fcs_sop_off;
+
+       csr_wr_node(xi.node, CVMX_PKO_MACX_CFG(mac_num), pko_mac_cfg.u64);
+
+       pko_mac_cfg.s.fifo_num = fifo_num;
+       csr_wr_node(xi.node, CVMX_PKO_MACX_CFG(mac_num), pko_mac_cfg.u64);
+
+       if (debug)
+               debug("%s: PKO_MAC[%u]CFG=%#llx\n", __func__, mac_num,
+                     (unsigned long long)csr_rd_node(xi.node, 
CVMX_PKO_MACX_CFG(mac_num)));
+
+       return 0;
+}
+
+/** Set Descriptor Queue options
+ *
+ * The `min_pad` parameter must be in agreement with the interface-level
+ * padding option for all descriptor queues assigned to that particular
+ * interface/port.
+ *
+ * @param node on which to operate
+ * @param dq descriptor queue to set
+ * @param min_pad minimum padding to set for dq
+ */
+void cvmx_pko3_dq_options(unsigned int node, unsigned int dq, bool min_pad)
+{
+       cvmx_pko_pdm_dqx_minpad_t reg;
+
+       dq &= (1 << 10) - 1;
+       reg.u64 = csr_rd_node(node, CVMX_PKO_PDM_DQX_MINPAD(dq));
+       reg.s.minpad = min_pad;
+       csr_wr_node(node, CVMX_PKO_PDM_DQX_MINPAD(dq), reg.u64);
+}
+
+/**
+ * Get number of PKO internal buffers available
+ *
+ * This function may be used to throttle output processing
+ * when the PKO runs out of internal buffers, to avoid discarding
+ * of packets or returning error results from transmission function.
+ *
+ * Returns negative numbers on error, positive number ti nidicate the
+ * number of buffers available, or 0 when no more buffers are available.
+ *
+ * @INTERNAL
+ */
+int cvmx_pko3_internal_buffer_count(unsigned int node)
+{
+       cvmx_pko_dpfi_fpa_aura_t pko_aura;
+       unsigned int laura, pool;
+       long long avail1, avail2;
+
+       /* get the aura number in pko, use aura node from parameter */
+       pko_aura.u64 = csr_rd_node(node, CVMX_PKO_DPFI_FPA_AURA);
+       laura = pko_aura.s.laura;
+
+       /* form here on, node is the AURA node */
+       node = pko_aura.s.node;
+
+       /* get the POOL number for this AURA */
+       pool = csr_rd_node(node, CVMX_FPA_AURAX_POOL(laura));
+
+       avail1 = csr_rd_node(node, CVMX_FPA_POOLX_AVAILABLE(pool));
+
+       avail2 = csr_rd_node(node, CVMX_FPA_AURAX_CNT_LIMIT(laura)) -
+                csr_rd_node(node, CVMX_FPA_AURAX_CNT(laura));
+
+       if (avail1 < avail2)
+               return avail1;
+
+       return avail2;
+}
+
+/**
+ * @INTERNAL
+ *
+ * Get actual PKO FIFO buffer size for a given port
+ *
+ * Since the FIFOs are allocated dynamically based on supply/demand
+ * heuristics, it may be useful in some instances to know the
+ * actual FIFO size allocated to any specific port at run time.
+ *
+ * @param xiface global interface number
+ * @param index port index on interface
+ * @return Returns the per-port FIFO size in bytes, or 0 if the port
+ * has not been configured, or a negative number if the interface and
+ * index numbers are not valid.
+ */
+int cvmx_pko3_port_fifo_size(unsigned int xiface, unsigned int index)
+{
+       unsigned int node;
+       unsigned int mac_num;
+       unsigned int fifo_grp, fifo_off;
+       cvmx_pko_macx_cfg_t pko_mac_cfg;
+       cvmx_pko_ptgfx_cfg_t pko_ptgfx_cfg;
+       cvmx_xiface_t xi = cvmx_helper_xiface_to_node_interface(xiface);
+       int ret;
+
+       node = xi.node;
+       ret = __cvmx_pko3_get_mac_num(xiface, index);
+       if (ret < 0)
+               return ret;
+
+       if (debug)
+               debug("%s: iface=%u:%u/%u mac %d\n", __func__, xi.node,
+                     xi.interface, index, ret);
+
+       mac_num = ret;
+
+       /* Check for the special value on onused MACs */
+       if (mac_num == 0x1f)
+               return 0;
+
+       pko_mac_cfg.u64 = csr_rd_node(node, CVMX_PKO_MACX_CFG(mac_num));
+
+       fifo_grp = pko_mac_cfg.s.fifo_num >> 2;
+       fifo_off = pko_mac_cfg.s.fifo_num & 0x3;
+       pko_ptgfx_cfg.u64 = csr_rd_node(node, CVMX_PKO_PTGFX_CFG(fifo_grp));
+
+       ret = (2 << 10) + (1 << 9); /* set 2.5KBytes base FIFO size */
+
+       switch (pko_ptgfx_cfg.s.size) {
+       case 0:
+               /* 2.5l, 2.5k, 2.5k, 2.5k */
+               break;
+       case 1:
+               /* 5.0k, 0.0k, 2.5k, 2.5k */
+               if (fifo_off == 1)
+                       ret = 0;
+               if (fifo_off == 0)
+                       ret *= 2;
+               break;
+       case 2:
+               /* 2.5k, 2.5k, 5.0k, 0.0k */
+               if (fifo_off == 3)
+                       ret = 0;
+               if (fifo_off == 2)
+                       ret *= 2;
+               break;
+       case 3:
+               /* 5k, 0, 5k, 0 */
+               if ((fifo_off & 1) != 0)
+                       ret = 0;
+               ret *= 2;
+               break;
+       case 4:
+               /* 10k, 0, 0, 0 */
+               if (fifo_off != 0)
+                       ret = 0;
+               ret *= 4;
+               break;
+       default:
+               ret = -1;
+       }
+       return ret;
+}
+
+/**
+ * @INTERNAL
+ *
+ * Stop an interface port transmission and wait until its FIFO is empty.
+ *
+ */
+int cvmx_pko3_port_xoff(unsigned int xiface, unsigned int index)
+{
+       cvmx_pko_l1_sqx_topology_t pko_l1_topology;
+       cvmx_pko_l1_sqx_sw_xoff_t pko_l1_xoff;
+       cvmx_pko_ptfx_status_t pko_ptfx_status;
+       cvmx_pko_macx_cfg_t pko_mac_cfg;
+       cvmx_pko_mci1_cred_cntx_t cred_cnt;
+       unsigned int node, pq, num_pq, mac_num, fifo_num;
+       int ret;
+       cvmx_xiface_t xi = cvmx_helper_xiface_to_node_interface(xiface);
+
+       node = xi.node;
+       ret = __cvmx_pko3_get_mac_num(xiface, index);
+
+       if (debug)
+               debug("%s: iface=%u:%u/%u mac %d\n", __func__, xi.node,
+                     xi.interface, index, ret);
+
+       if (ret < 0)
+               return ret;
+
+       mac_num = ret;
+
+       if (mac_num == 0x1f)
+               return 0;
+
+       pko_mac_cfg.u64 = csr_rd_node(node, CVMX_PKO_MACX_CFG(mac_num));
+       fifo_num = pko_mac_cfg.s.fifo_num;
+
+       /* Verify the FIFO number is correct */
+       pko_ptfx_status.u64 = csr_rd_node(node, CVMX_PKO_PTFX_STATUS(fifo_num));
+
+       if (debug)
+               debug("%s: mac %d fifo %d, fifo mac %d\n", __func__, mac_num,
+                     fifo_num, pko_ptfx_status.s.mac_num);
+
+       cvmx_warn_if(pko_ptfx_status.s.mac_num != mac_num,
+                    "PKO3 FIFO number does not match MAC\n");
+
+       num_pq = cvmx_pko3_num_level_queues(CVMX_PKO_PORT_QUEUES);
+       /* Find the L1/PQ connected to the MAC for this interface */
+       for (pq = 0; pq < num_pq; pq++) {
+               pko_l1_topology.u64 =
+                       csr_rd_node(node, CVMX_PKO_L1_SQX_TOPOLOGY(pq));
+               if (pko_l1_topology.s.link == mac_num)
+                       break;
+       }
+
+       if (debug)
+               debug("%s: L1_PQ%u LINK %d MAC_NUM %d\n", __func__, pq,
+                     pko_l1_topology.s.link, mac_num);
+
+       if (pq >= num_pq)
+               return -1;
+
+       if (debug) {
+               pko_ptfx_status.u64 =
+                       csr_rd_node(node, CVMX_PKO_PTFX_STATUS(fifo_num));
+               ret = pko_ptfx_status.s.in_flight_cnt;
+               debug("%s: FIFO %d in-flight %d packets\n", __func__, fifo_num,
+                     ret);
+       }
+
+       /* Turn the XOFF bit on */
+       pko_l1_xoff.u64 = csr_rd_node(node, CVMX_PKO_L1_SQX_SW_XOFF(pq));
+       pko_l1_xoff.s.xoff = 1;
+       csr_wr_node(node, CVMX_PKO_L1_SQX_SW_XOFF(pq), pko_l1_xoff.u64);
+
+       ret = 1 << 22;
+       /* Wait for PKO TX FIFO to drain */
+       do {
+               CVMX_SYNC;
+               pko_ptfx_status.u64 =
+                       csr_rd_node(node, CVMX_PKO_PTFX_STATUS(fifo_num));
+       } while (pko_ptfx_status.s.in_flight_cnt != 0 && ret--);
+
+       if (pko_ptfx_status.s.in_flight_cnt != 0)
+               cvmx_warn("%s: FIFO %d failed to drain\n", __func__, fifo_num);
+
+       if (debug)
+               debug("%s: FIFO %d drained in %d cycles\n", __func__, fifo_num,
+                     (1 << 22) - ret);
+
+       /* Wait for MAC TX FIFO to drain. */
+       do {
+               cred_cnt.u64 =
+                       csr_rd_node(node, CVMX_PKO_MCI1_CRED_CNTX(mac_num));
+       } while (cred_cnt.s.cred_cnt != 0);
+
+       return 0;
+}
+
+/**
+ * @INTERNAL
+ *
+ * Resume transmission on an interface port.
+ *
+ */
+int cvmx_pko3_port_xon(unsigned int xiface, unsigned int index)
+{
+       cvmx_pko_l1_sqx_topology_t pko_l1_topology;
+       cvmx_pko_l1_sqx_sw_xoff_t pko_l1_xoff;
+       unsigned int node, pq, num_pq, mac_num;
+       int ret;
+       cvmx_xiface_t xi = cvmx_helper_xiface_to_node_interface(xiface);
+
+       num_pq = cvmx_pko3_num_level_queues(CVMX_PKO_PORT_QUEUES);
+       node = xi.node;
+       ret = __cvmx_pko3_get_mac_num(xiface, index);
+
+       if (debug)
+               debug("%s: iface=%u:%u/%u mac %d\n", __func__, xi.node,
+                     xi.interface, index, ret);
+
+       if (ret < 0)
+               return ret;
+
+       mac_num = ret;
+
+       if (mac_num == 0x1f)
+               return 0;
+
+       /* Find the L1/PQ connected to the MAC for this interface */
+       for (pq = 0; pq < num_pq; pq++) {
+               pko_l1_topology.u64 =
+                       csr_rd_node(node, CVMX_PKO_L1_SQX_TOPOLOGY(pq));
+               if (pko_l1_topology.s.link == mac_num)
+                       break;
+       }
+
+       if (debug)
+               debug("%s: L1_PQ%u LINK %d MAC_NUM %d\n", __func__, pq,
+                     pko_l1_topology.s.link, mac_num);
+
+       if (pq >= num_pq)
+               return -1;
+
+       /* Turn the XOFF bit off */
+       pko_l1_xoff.u64 = csr_rd_node(node, CVMX_PKO_L1_SQX_SW_XOFF(pq));
+       ret = pko_l1_xoff.s.xoff;
+       pko_l1_xoff.s.xoff = 0;
+       csr_wr_node(node, CVMX_PKO_L1_SQX_SW_XOFF(pq), pko_l1_xoff.u64);
+
+       return ret;
+}
+
+/******************************************************************************
+ * New PKO3 API - Experimental
+ 
******************************************************************************/
+
+/**
+ * Initialize packet descriptor
+ *
+ * Descriptor storage is provided by the caller,
+ * use this function to initialize the descriptor to a known
+ * empty state.
+ *
+ * @param pdesc Packet Descriptor.
+ *
+ * Do not use this function when creating a descriptor from a
+ * Work Queue Entry.
+ *
+ * The default setting of the 'free_bufs' attribute is 'false'.
+ */
+void cvmx_pko3_pdesc_init(cvmx_pko3_pdesc_t *pdesc)
+{
+       cvmx_pko_send_aura_t *ext_s;
+
+       memset(pdesc, 0, sizeof(*pdesc));
+
+       /* Start with HDR_S and HDR_EXT_S in first two words, all 0's */
+       pdesc->num_words = 2;
+
+       pdesc->hdr_s = (void *)&pdesc->word[0];
+       ext_s = (void *)&pdesc->word[1];
+       ext_s->s.subdc4 = CVMX_PKO_SENDSUBDC_EXT;
+
+       pdesc->last_aura = -1;
+       pdesc->jb_aura = -1;
+
+       /* Empty packets, can not decode header offsets (yet) */
+       pdesc->hdr_offsets = 1;
+}
+
+/**
+ * @INTERNAL
+ *
+ * Add arbitrary subcommand to a packet descriptor.
+ *
+ * This function will also allocate a jump buffer when
+ * the primary LMTDMA buffer is exhausted.
+ * The jump buffer is allocated from the internal PKO3 aura
+ * on the node where this function is running.
+ */
+static int cvmx_pko3_pdesc_subdc_add(cvmx_pko3_pdesc_t *pdesc, uint64_t subdc)
+{
+       cvmx_pko_send_hdr_t *hdr_s;
+       cvmx_pko_send_aura_t *ext_s;
+       cvmx_pko_buf_ptr_t *jump_s;
+       const unsigned int jump_buf_size = 4 * 1024 / sizeof(uint64_t);
+       unsigned int i;
+
+       /* Simple handling while fitting the command buffer */
+       if (cvmx_likely(pdesc->num_words < 15 && !pdesc->jump_buf)) {
+               pdesc->word[pdesc->num_words] = subdc;
+               pdesc->num_words++;
+               return pdesc->num_words;
+       }
+
+       /* SEND_JUMP_S missing on Pass1 */
+       if (OCTEON_IS_MODEL(OCTEON_CN78XX_PASS1_X)) {
+               cvmx_printf("%s: ERROR: too many segments\n", __func__);
+               return -E2BIG;
+       }
+
+       hdr_s = (void *)&pdesc->word[0];
+       ext_s = (void *)&pdesc->word[1];
+
+       /* Allocate jump buffer */
+       if (cvmx_unlikely(!pdesc->jump_buf)) {
+               u16 pko_gaura;
+               cvmx_fpa3_gaura_t aura;
+               unsigned int fpa_node = cvmx_get_node_num();
+
+               /* Allocate jump buffer from PKO internal FPA AURA, size=4KiB */
+               pko_gaura = __cvmx_pko3_aura_get(fpa_node);
+               aura = __cvmx_fpa3_gaura(pko_gaura >> 10, pko_gaura & 0x3ff);
+
+               pdesc->jump_buf = cvmx_fpa3_alloc(aura);
+               if (!pdesc->jump_buf)
+                       return -EINVAL;
+
+               /* Save the JB aura for later */
+               pdesc->jb_aura = pko_gaura;
+
+               /* Move most of the command to the jump buffer */
+               memcpy(pdesc->jump_buf, &pdesc->word[2],
+                      (pdesc->num_words - 2) * sizeof(uint64_t));
+               jump_s = (void *)&pdesc->word[2];
+               jump_s->u64 = 0;
+               jump_s->s.addr = cvmx_ptr_to_phys(pdesc->jump_buf);
+               jump_s->s.i = !hdr_s->s.df; /* F= ~DF */
+               jump_s->s.size = pdesc->num_words - 2;
+               jump_s->s.subdc3 = CVMX_PKO_SENDSUBDC_JUMP;
+
+               /* Now the LMTDMA buffer has only HDR_S, EXT_S, JUMP_S */
+               pdesc->num_words = 3;
+       }
+
+       /* Add the new subcommand to the jump buffer */
+       jump_s = (void *)&pdesc->word[2];
+       i = jump_s->s.size;
+
+       /* Avoid overrunning jump buffer */
+       if (i >= (jump_buf_size - 2)) {
+               cvmx_printf("%s: ERROR: too many segments\n", __func__);
+               return -E2BIG;
+       }
+
+       pdesc->jump_buf[i] = subdc;
+       jump_s->s.size++;
+
+       (void)ext_s;
+
+       return (i + pdesc->num_words);
+}
+
+/**
+ * Send a packet in a descriptor to an output port via an output queue.
+ *
+ * A call to this function must follow all other functions that
+ * create a packet descriptor from WQE, or after initializing an
+ * empty descriptor and filling it with one or more data fragments.
+ * After this function is called, the content of the packet descriptor
+ * can no longer be used, and are undefined.
+ *
+ * @param pdesc Packet Descriptor.
+ * @param dq Descriptor Queue associated with the desired output port
+ * @param tag Flow Tag pointer for packet ordering or NULL
+ * @return Returns 0 on success, -1 on error.
+ *
+ */
+int cvmx_pko3_pdesc_transmit(cvmx_pko3_pdesc_t *pdesc, uint16_t dq,
+                            uint32_t *tag)
+{
+       cvmx_pko_query_rtn_t pko_status;
+       cvmx_pko_send_aura_t aura_s;
+       u8 port_node;
+       int rc;
+
+       /* Add last AURA_S for jump_buf, if present */
+       if (cvmx_unlikely(pdesc->jump_buf) &&
+           pdesc->last_aura != pdesc->jb_aura) {
+               /* The last AURA_S subdc refers to the jump_buf itself */
+               aura_s.s.aura = pdesc->jb_aura;
+               aura_s.s.offset = 0;
+               aura_s.s.alg = AURAALG_NOP;
+               aura_s.s.subdc4 = CVMX_PKO_SENDSUBDC_AURA;
+               pdesc->last_aura = pdesc->jb_aura;
+
+               rc = cvmx_pko3_pdesc_subdc_add(pdesc, aura_s.u64);
+               if (rc < 0)
+                       return -1;
+       }
+
+       /* SEND_WORK_S must be the very last subdc */
+       if (cvmx_unlikely(pdesc->send_work_s != 0ULL)) {
+               rc = cvmx_pko3_pdesc_subdc_add(pdesc, pdesc->send_work_s);
+               if (rc < 0)
+                       return -1;
+               pdesc->send_work_s = 0ULL;
+       }
+
+       /* Derive destination node from dq */
+       port_node = dq >> 10;
+       dq &= (1 << 10) - 1;
+
+       /* To preserve packet order, go atomic with DQ-specific tag */
+       if (tag)
+               cvmx_pow_tag_sw(*tag ^ dq, CVMX_POW_TAG_TYPE_ATOMIC);
+
+       /* Send the PKO3 command into the Descriptor Queue */
+       pko_status = __cvmx_pko3_do_dma(port_node, dq, pdesc->word,
+                                       pdesc->num_words, CVMX_PKO_DQ_SEND);
+
+       /* Map PKO3 result codes to legacy return values */
+       if (pko_status.s.dqstatus == PKO_DQSTATUS_PASS)
+               return 0;
+
+       return -1;
+}
+
+int cvmx_pko3_pdesc_append_free(cvmx_pko3_pdesc_t *pdesc, uint64_t addr,
+                               unsigned int gaura)
+{
+       cvmx_pko_send_hdr_t *hdr_s;
+       cvmx_pko_send_free_t free_s;
+       cvmx_pko_send_aura_t aura_s;
+
+       hdr_s = (void *)&pdesc->word[0];
+
+       if (pdesc->last_aura == -1) {
+               hdr_s->s.aura = gaura;
+               pdesc->last_aura = hdr_s->s.aura;
+       } else if (pdesc->last_aura != (short)gaura) {
+               aura_s.s.aura = gaura;
+               aura_s.s.offset = 0;
+               aura_s.s.alg = AURAALG_NOP;
+               aura_s.s.subdc4 = CVMX_PKO_SENDSUBDC_AURA;
+               pdesc->last_aura = gaura;
+               if (cvmx_pko3_pdesc_subdc_add(pdesc, aura_s.u64) < 0)
+                       return -1;
+       }
+
+       free_s.u64 = 0;
+       free_s.s.subdc4 = CVMX_PKO_SENDSUBDC_FREE;
+       free_s.s.addr = addr;
+
+       return cvmx_pko3_pdesc_subdc_add(pdesc, free_s.u64);
+}
+
+/**
+ * Append a packet segment to a packet descriptor
+ *
+ * After a packet descriptor is initialized, one or more
+ * packet data segments can be added to the packet,
+ * in the order in which they should be transmitted.
+ *
+ * The size of the resulting packet will be equal to the
+ * sum of the segments appended by this function.
+ * Every segment may be contained in a buffer that belongs
+ * to a different FPA 'aura', and may be automatically
+ * released back to that aura, if required.
+ *
+ * @param pdesc Packet Descriptor.
+ * @param p_data Address of the segment first byte (virtual).
+ * @param data_bytes Size of the data segment (in bytes).
+ * @param gaura A global FPA 'aura' where the packet buffer was allocated from.
+ *
+ * The 'gaura' parameter contains the node number where the buffer pool
+ * is located, and has only a meaning if the 'free_buf' argument is 'true'.
+ * The buffer being added will be automatically freed upon transmission
+ * along with all other buffers in this descriptor, or not, depending
+ * on the descriptor 'free_bufs' attribute that is set during
+ * descriptor creation, or changed subsequently with a call to
+ * 'cvmx_pko3_pdesc_set_free()'.
+ *
+ * @return Returns 0 on success, -1 on error.
+ */
+int cvmx_pko3_pdesc_buf_append(cvmx_pko3_pdesc_t *pdesc, void *p_data,
+                              unsigned int data_bytes, unsigned int gaura)
+{
+       cvmx_pko_send_hdr_t *hdr_s;
+       cvmx_pko_buf_ptr_t gather_s;
+       cvmx_pko_send_aura_t aura_s;
+       int rc;
+
+       if (pdesc->mem_s_ix > 0) {
+               cvmx_printf("ERROR: %s: subcommand restriction violated\n",
+                           __func__);
+               return -1;
+       }
+
+       hdr_s = (void *)&pdesc->word[0];
+
+       if (gaura != (unsigned int)-1) {
+               if (pdesc->last_aura == -1) {
+                       unsigned int buf_sz = 128;
+
+                       /* First mbuf, calculate headroom */
+                       cvmx_fpa3_gaura_t aura;
+
+                       aura = __cvmx_fpa3_gaura(gaura >> 10, gaura & 0x3ff);
+                       buf_sz = cvmx_fpa3_get_aura_buf_size(aura);
+                       pdesc->headroom = (unsigned long)p_data & (buf_sz - 1);
+                       hdr_s->s.aura = gaura;
+                       pdesc->last_aura = hdr_s->s.aura;
+               } else if (pdesc->last_aura != (short)gaura) {
+                       aura_s.s.aura = gaura;
+                       aura_s.s.offset = 0;
+                       aura_s.s.alg = AURAALG_NOP;
+                       aura_s.s.subdc4 = CVMX_PKO_SENDSUBDC_AURA;
+                       pdesc->last_aura = gaura;
+
+                       rc = cvmx_pko3_pdesc_subdc_add(pdesc, aura_s.u64);
+                       if (rc < 0)
+                               return -1;
+               }
+       }
+
+       gather_s.u64 = 0;
+       gather_s.s.addr = cvmx_ptr_to_phys(p_data);
+       gather_s.s.size = data_bytes;
+       hdr_s->s.total += data_bytes;
+       gather_s.s.i = 0; /* follow HDR_S[DF] setting */
+       gather_s.s.subdc3 = CVMX_PKO_SENDSUBDC_GATHER;
+
+       rc = cvmx_pko3_pdesc_subdc_add(pdesc, gather_s.u64);
+       if (rc < 0)
+               return -1;
+
+       return rc;
+}
+
+/**
+ * Add a Work Entry for packet transmission notification
+ *
+ * Add a subcommand to notify of packet transmission completion
+ * via a Work Queue entry over the SSO.
+ * The Work Queue entry may be a 'software' event, or the content
+ * of a packet.
+ *
+ * @param pdesc Packet Descriptor, memory provided by caller.
+ * @param wqe Work Queue Entry in a model-native format.
+ * @param node The OCI node of the SSO where the WQE will be delivered.
+ * @param group The SSO group where the WQE is delivered.
+ * @param tt The SSO Tag Type for the WQE. If tt is not NULL, tag should be a
+ * valid tag value.
+ * @param tag Valid tag value to assign to WQE
+ *
+ * @return Returns 0 on success, -1 on error.
+ *
+ * Restrictions:
+ * There can be only one such notification per packet descriptor,
+ * but this function may be called at any time after the descriptor
+ * is first created from WQE or initialized, and before
+ * starting transmission.
+ *
+ */
+int cvmx_pko3_pdesc_notify_wqe(cvmx_pko3_pdesc_t *pdesc, cvmx_wqe_78xx_t *wqe,
+                              u8 node, u8 group, uint8_t tt, uint32_t tag)
+{
+       cvmx_pko_send_work_t work_s;
+
+       /*
+        * There can be only one SEND_WORK_S entry in the command
+        * and it must be the very last subcommand
+        */
+       if (pdesc->send_work_s != 0) {
+               cvmx_printf("ERROR: %s: Only one SEND_WORK_S is allowed\n",
+                           __func__);
+               return -1;
+       }
+
+       work_s.u64 = 0;
+       work_s.s.subdc4 = CVMX_PKO_SENDSUBDC_WORK;
+       work_s.s.addr = cvmx_ptr_to_phys(wqe);
+       work_s.s.grp = (group & 0xff) | (node << 8);
+       work_s.s.tt = tt;
+
+       wqe->word1.rsvd_0 = 0;
+       wqe->word1.rsvd_1 = 0;
+       wqe->word1.tag = tag;
+       wqe->word1.tag_type = tt;
+       wqe->word1.grp = work_s.s.grp;
+
+       /* Store in descriptor for now, apply just before LMTDMA-ing */
+       pdesc->send_work_s = work_s.u64;
+
+       return 0;
+}
+
+/**
+ * Request atomic memory decrement at transmission completion
+ *
+ * Each packet descriptor may contain several decrement notification
+ * requests, but these requests must only be made after all of the
+ * packet data segments have been added, and before packet transmission
+ * commences.
+ *
+ * Only decrement of a 64-bit memory location is supported.
+ *
+ * @param pdesc Packet Descriptor.
+ * @param p_counter A pointer to an atomic 64-bit memory location.
+ *
+ * @return Returns 0 on success, -1 on failure.
+ */
+int cvmx_pko3_pdesc_notify_decrement(cvmx_pko3_pdesc_t *pdesc,
+                                    volatile uint64_t *p_counter)
+{
+       int rc;
+       /* 64-bit decrement is the only supported operation */
+       cvmx_pko_send_mem_t mem_s = {
+               .s = { .subdc4 = CVMX_PKO_SENDSUBDC_MEM,
+                      .dsz = MEMDSZ_B64,
+                      .alg = MEMALG_SUB,
+                      .offset = 1,
+#ifdef _NOT_IN_SIM_
+                      /* Enforce MEM before SSO submission if both present */
+                      .wmem = 1
+#endif
+               }
+       };
+
+       mem_s.s.addr = cvmx_ptr_to_phys(CASTPTR(void, p_counter));
+
+       rc = cvmx_pko3_pdesc_subdc_add(pdesc, mem_s.u64);
+
+       /*
+        * SEND_MEM_S must be after all LINK_S/FATHER_S/IMM_S
+        * subcommands, set the index to prevent further data
+        * subcommands.
+        */
+       if (rc > 0)
+               pdesc->mem_s_ix = rc;
+
+       return rc;
+}
+
+/**
+ * Request atomic memory clear at transmission completion
+ *
+ * Each packet descriptor may contain several notification
+ * requests, but these request must only be made after all of the
+ * packet data segments have been added, and before packet transmission
+ * commences.
+ *
+ * Clearing of a single byte is requested by this function.
+ *
+ * @param pdesc Packet Descriptor.
+ * @param p_mem A pointer to a byte location.
+ *
+ * @return Returns 0 on success, -1 on failure.
+ */
+int cvmx_pko3_pdesc_notify_memclr(cvmx_pko3_pdesc_t *pdesc,
+                                 volatile uint8_t *p_mem)
+{
+       int rc;
+       /* 640bit decrement is the only supported operation */
+       cvmx_pko_send_mem_t mem_s = { .s = {
+                                             .subdc4 = CVMX_PKO_SENDSUBDC_MEM,
+                                             .dsz = MEMDSZ_B8,
+                                             .alg = MEMALG_SET,
+                                             .offset = 0,
+                                     } };
+
+       mem_s.s.addr = cvmx_ptr_to_phys(CASTPTR(void, p_mem));
+
+       rc = cvmx_pko3_pdesc_subdc_add(pdesc, mem_s.u64);
+
+       /*
+        * SEND_MEM_S must be after all LINK_S/FATHER_S/IMM_S
+        * subcommands, set the index to prevent further data
+        * subcommands.
+        */
+       if (rc > 0)
+               pdesc->mem_s_ix = rc;
+
+       return rc;
+}
+
+/**
+ * @INTERNAL
+ *
+ * Decode packet header and calculate protocol header offsets
+ *
+ * The protocol information and layer offset is derived
+ * from the results if decoding done by the PKI,
+ * and the appropriate PKO fields are filled.
+ *
+ * The function assumes the headers have not been modified
+ * since converted from WQE, and does not (yet) implement
+ * software-based decoding to handle modified or originated
+ * packets correctly.
+ *
+ * @note
+ * Need to add simple accessors to read the decoded protocol fields.
+ */
+static int cvmx_pko3_pdesc_hdr_offsets(cvmx_pko3_pdesc_t *pdesc)
+{
+       cvmx_pko_send_hdr_t *hdr_s;
+
+       if (pdesc->hdr_offsets)
+               return 0;
+
+       if (!pdesc->pki_word4_present)
+               return -EINVAL;
+
+       hdr_s = (void *)&pdesc->word[0];
+       pdesc->hdr_s = hdr_s;
+
+       /* Match IPv5/IPv6 protocols with/without options */
+       if ((pdesc->pki_word2.lc_hdr_type & 0x1c) == CVMX_PKI_LTYPE_E_IP4) {
+               hdr_s->s.l3ptr = pdesc->pki_word4.ptr_layer_c;
+
+               /* Match TCP/UDP/SCTP group */
+               if ((pdesc->pki_word2.lf_hdr_type & 0x18) ==
+                   CVMX_PKI_LTYPE_E_TCP)
+                       hdr_s->s.l4ptr = pdesc->pki_word4.ptr_layer_f;
+
+               if (pdesc->pki_word2.lf_hdr_type == CVMX_PKI_LTYPE_E_UDP)
+                       pdesc->ckl4_alg = CKL4ALG_UDP;
+               if (pdesc->pki_word2.lf_hdr_type == CVMX_PKI_LTYPE_E_TCP)
+                       pdesc->ckl4_alg = CKL4ALG_TCP;
+               if (pdesc->pki_word2.lf_hdr_type == CVMX_PKI_LTYPE_E_SCTP)
+                       pdesc->ckl4_alg = CKL4ALG_SCTP;
+       }
+       /* May need to add logic for ARP, IPfrag packets here */
+
+       pdesc->hdr_offsets = 1; /* make sure its done once */
+       return 0;
+}
+
+/*
+ * @INTERNAL
+ *
+ * memcpy() a reverse endian memory region.
+ * where both the source and destination are the reverse endianness
+ * with respect to native byte order.
+ */
+static void memcpy_swap(void *dst, const void *src, unsigned int bytes)
+{
+       u8 *d = dst;
+       const u8 *s = src;
+       unsigned int i;
+       const unsigned int swizzle = 0x7; /* 64-bit invariant endianness */
+
+       for (i = 0; i < bytes; i++)
+               d[i ^ swizzle] = s[i ^ swizzle];
+}
+
+/*
+ * @INTERNAL
+ *
+ * memcpy() with swizzling, from reverse endianness to native byte order.
+ */
+static void memcpy_from_swap(void *dst, const void *src, unsigned int bytes)
+{
+       u8 *d = dst;
+       const u8 *s = src;
+       unsigned int i;
+       const unsigned int swizzle = 0x7; /* 64-bit invariant endianness */
+
+       for (i = 0; i < bytes; i++)
+               d[i] = s[i ^ swizzle];
+}
+
+/*
+ * @INTERNAL
+ *
+ * memcpy() with swizzling, from native byte order to the reverse endianness.
+ */
+static void memcpy_to_swap(void *dst, const void *src, unsigned int bytes)
+{
+       u8 *d = dst;
+       const u8 *s = src;
+       unsigned int i;
+       const unsigned int swizzle = 0x7; /* 64-bit invariant endianness */
+
+       for (i = 0; i < bytes; i++)
+               d[i ^ swizzle] = s[i];
+}
+
+/**
+ * Prepend a data segment to the packet descriptor
+ *
+ * Useful for pushing additional headers
+ *
+ * The initial implementation is confined by the size of the
+ * "headroom" in the first packet buffer attached to the descriptor.
+ * Future version may prepend additional buffers when this head room
+ * is insufficient, but currently will return -1 when headrom is
+ * insufficient.
+ *
+ * On success, the function returns the remaining headroom in the buffer.
+ *
+ */
+int cvmx_pko3_pdesc_hdr_push(cvmx_pko3_pdesc_t *pdesc, const void *p_data,
+                            u8 data_bytes, uint8_t layer)
+{
+       cvmx_pko_send_hdr_t *hdr_s;
+       cvmx_pko_buf_ptr_t *gather_s;
+       short headroom;
+       void *p; /* old data location */
+       void *q; /* new data location */
+       bool endian_swap;
+
+       headroom = pdesc->headroom;
+
+       if ((short)data_bytes > headroom)
+               return -ENOSPC;
+
+       hdr_s = (void *)&pdesc->word[0];
+       endian_swap = (hdr_s->s.le != __native_le);
+
+       /* Get GATTHER_S/LINK_S subcommand location */
+       if (cvmx_likely(!pdesc->jump_buf))
+               /* Without JB, first data buf is in 3rd command word */
+               gather_s = (void *)&pdesc->word[2];
+       else
+               /* With JB, its first word is the first buffer */
+               gather_s = (void *)pdesc->jump_buf;
+
+       /* Verify the subcommand is of the expected type */
+       if (cvmx_unlikely(gather_s->s.subdc3 != CVMX_PKO_SENDSUBDC_LINK &&
+                         gather_s->s.subdc3 != CVMX_PKO_SENDSUBDC_GATHER))
+               return -EINVAL;
+
+       /* adjust  address and size values */
+       p = cvmx_phys_to_ptr(gather_s->s.addr);
+       q = p - data_bytes;
+       gather_s->s.addr -= data_bytes;
+       gather_s->s.size += data_bytes;
+       hdr_s->s.total += data_bytes;
+       headroom -= data_bytes;
+
+       /* Move link pointer if the descriptor is SEND_LINK_S */
+       if (gather_s->s.subdc3 == CVMX_PKO_SENDSUBDC_LINK) {
+               if (cvmx_likely(!endian_swap))
+                       memcpy(q - 8, p - 8, 8);
+               else
+                       memcpy_swap(q - 8, p - 8, 8);
+       }
+
+       if (cvmx_likely(!endian_swap))
+               memcpy(q, p_data, data_bytes);
+       else
+               memcpy_to_swap(q, p_data, data_bytes);
+
+       pdesc->headroom = headroom;
+
+       /* Adjust higher level protocol header offset */
+       cvmx_pko3_pdesc_hdr_offsets(pdesc);
+       if (layer <= 4)
+               pdesc->hdr_s->s.l4ptr += data_bytes;
+
+       if (layer <= 3)
+               pdesc->hdr_s->s.l3ptr += data_bytes;
+
+       if (layer >= 3) {
+               /* Set CKL3 only for IPv4 */
+               if ((pdesc->pki_word2.lc_hdr_type & 0x1e) ==
+                   CVMX_PKI_LTYPE_E_IP4)
+                       hdr_s->s.ckl3 = 1;
+               hdr_s->s.ckl4 = pdesc->ckl4_alg;
+       }
+
+       return headroom;
+}
+
+/**
+ * Remove some bytes from start of packet
+ *
+ * Useful for popping a header from a packet.
+ * It only needs to find the first segment, and adjust its address,
+ * as well as segment and total sizes.
+ *
+ * Returns new packet size, or -1 if the trimmed size exceeds the
+ * size of the first data segment.
+ */
+int cvmx_pko3_pdesc_hdr_pop(cvmx_pko3_pdesc_t *pdesc, void *hdr_buf,
+                           unsigned int num_bytes)
+{
+       cvmx_pko_send_hdr_t *hdr_s;
+       cvmx_pko_buf_ptr_t *gather_s;
+       short headroom;
+       void *p;
+       void *q;
+       bool endian_swap;
+
+       headroom = pdesc->headroom;
+
+       hdr_s = (void *)&pdesc->word[0];
+       endian_swap = (hdr_s->s.le != __native_le);
+
+       if (hdr_s->s.total < num_bytes)
+               return -ENOSPC;
+
+       /* Get GATTHER_S/LINK_S subcommand location */
+       if (cvmx_likely(!pdesc->jump_buf))
+               /* Without JB, first data buf is in 3rd command word */
+               gather_s = (void *)&pdesc->word[2];
+       else
+               /* With JB, its first word is the first buffer */
+               gather_s = (void *)pdesc->jump_buf;
+
+       /* Verify the subcommand is of the expected type */
+       if (cvmx_unlikely(gather_s->s.subdc3 != CVMX_PKO_SENDSUBDC_LINK &&
+                         gather_s->s.subdc3 != CVMX_PKO_SENDSUBDC_GATHER))
+               return -EINVAL;
+
+       /* Can't trim more than the content of the first buffer */
+       if (gather_s->s.size < num_bytes)
+               return -ENOMEM;
+
+       /* adjust  address and size values */
+       p = cvmx_phys_to_ptr(gather_s->s.addr);
+       q = p + num_bytes;
+       gather_s->s.addr += num_bytes;
+       gather_s->s.size -= num_bytes;
+       hdr_s->s.total -= num_bytes;
+       headroom += num_bytes;
+
+       if (hdr_buf) {
+               /* Retrieve popped header to user buffer */
+               if (cvmx_likely(!endian_swap))
+                       memcpy(hdr_buf, p, num_bytes);
+               else
+                       memcpy_from_swap(hdr_buf, p, num_bytes);
+       }
+
+       /* Move link pointer if the descriptor is SEND_LINK_S */
+       if (gather_s->s.subdc3 == CVMX_PKO_SENDSUBDC_LINK) {
+               if (cvmx_likely(!endian_swap))
+                       memcpy(q - 8, p - 8, 8);
+               else
+                       memcpy_swap(q - 8, p - 8, 8);
+       }
+
+       pdesc->headroom = headroom;
+
+       /* Adjust higher level protocol header offset */
+       cvmx_pko3_pdesc_hdr_offsets(pdesc);
+       if (num_bytes < pdesc->hdr_s->s.l3ptr) {
+               pdesc->hdr_s->s.l3ptr -= num_bytes;
+               pdesc->hdr_s->s.l4ptr -= num_bytes;
+       } else if (num_bytes < pdesc->hdr_s->s.l4ptr) {
+               pdesc->hdr_s->s.l3ptr = 0;
+               pdesc->hdr_s->s.l4ptr -= num_bytes;
+       } else {
+               pdesc->hdr_s->s.l3ptr = 0;
+               pdesc->hdr_s->s.l4ptr = 0;
+               hdr_s->s.ckl4 = CKL4ALG_NONE;
+       }
+
+       return hdr_s->s.total;
+}
+
+/**
+ * Peek into some header field of a packet
+ *
+ * Will return a number of bytes of packet header data at an arbitrary offset
+ * which must reside within the first packet data buffer.
+ *
+ */
+int cvmx_pko3_pdesc_hdr_peek(cvmx_pko3_pdesc_t *pdesc, void *hdr_buf,
+                            unsigned int num_bytes, unsigned int offset)
+{
+       cvmx_pko_send_hdr_t *hdr_s;
+       cvmx_pko_buf_ptr_t *gather_s;
+       void *p;
+       bool endian_swap;
+
+       hdr_s = (void *)&pdesc->word[0];
+       endian_swap = (hdr_s->s.le != __native_le);
+
+       if (hdr_s->s.total < (num_bytes + offset))
+               return -ENOSPC;
+
+       /* Get GATTHER_S/LINK_S subcommand location */
+       if (cvmx_likely(!pdesc->jump_buf))
+               /* Without JB, first data buf is in 3rd command word */
+               gather_s = (void *)&pdesc->word[2];
+       else
+               /* With JB, its first word is the first buffer */
+               gather_s = (void *)pdesc->jump_buf;
+
+       /* Verify the subcommand is of the expected type */
+       if (cvmx_unlikely(gather_s->s.subdc3 != CVMX_PKO_SENDSUBDC_LINK &&
+                         gather_s->s.subdc3 != CVMX_PKO_SENDSUBDC_GATHER))
+               return -EINVAL;
+
+       /* Can't peek more than the content of the first buffer */
+       if (gather_s->s.size <= offset)
+               return -ENOMEM;
+       if ((gather_s->s.size - offset) < num_bytes)
+               num_bytes = gather_s->s.size - offset;
+
+       /* adjust address */
+       p = cvmx_phys_to_ptr(gather_s->s.addr) + offset;
+
+       if (!hdr_buf)
+               return -EINVAL;
+
+       /* Copy requested bytes */
+       if (cvmx_likely(!endian_swap))
+               memcpy(hdr_buf, p, num_bytes);
+       else
+               memcpy_from_swap(hdr_buf, p, num_bytes);
+
+       return num_bytes;
+}
+
+/**
+ * Set the packet descriptor automatic-free attribute
+ *
+ * Override the 'free_bufs' attribute that was set during
+ * packet descriptor creation, or by an earlier call to
+ * this function.
+ * Setting the 'buf_free" attribute to 'true' will cause
+ * the PKO3 to free all buffers associated with this packet
+ * descriptor to be released upon transmission complete.
+ * Setting this attribute to 'false' allows e.g. using the
+ * same descriptor to transmit a packet out of several ports
+ * with a minimum overhead.
+ */
+void cvmx_pko3_pdesc_set_free(cvmx_pko3_pdesc_t *pdesc, bool free_bufs)
+{
+       cvmx_pko_send_hdr_t *hdr_s;
+       cvmx_pko_buf_ptr_t *jump_s;
+
+       hdr_s = (void *)&pdesc->word[0];
+       hdr_s->s.df = !free_bufs;
+
+       if (cvmx_likely(!pdesc->jump_buf))
+               return;
+       jump_s = (void *)&pdesc->word[2];
+       jump_s->s.i = free_bufs; /* F=free */
+}
-- 
2.35.1

Reply via email to