Add rawdev dequeue operation for gdtc devices.
Signed-off-by: Yong Zhang <zhang.yon...@zte.com.cn>
---
drivers/raw/gdtc/gdtc_rawdev.c | 168 ++++++++++++++++++++++++++++-----
1 file changed, 147 insertions(+), 21 deletions(-)
diff --git a/drivers/raw/gdtc/gdtc_rawdev.c b/drivers/raw/gdtc/gdtc_rawdev.c
index f58b034e58..bc94bad827 100644
--- a/drivers/raw/gdtc/gdtc_rawdev.c
+++ b/drivers/raw/gdtc/gdtc_rawdev.c
@@ -88,6 +88,8 @@
#define LOW32_MASK 0xffffffff
#define LOW16_MASK 0xffff
+#define ZXDH_GDMA_TC_CNT_MAX 0x10000
+
#define IDX_TO_ADDR(addr, idx, t) \
((t)((uintptr_t)(addr) + (idx) * sizeof(struct zxdh_gdma_buff_desc)))
@@ -116,6 +118,19 @@ zxdh_gdma_get_queue(struct rte_rawdev *dev, uint16_t
queue_id)
return &(gdmadev->vqs[queue_id]);
}
+static uint32_t
+zxdh_gdma_read_reg(struct rte_rawdev *dev, uint16_t queue_id, uint32_t offset)
+{
+ struct zxdh_gdma_rawdev *gdmadev = zxdh_gdma_rawdev_get_priv(dev);
+ uint32_t addr = 0;
+ uint32_t val = 0;
+
+ addr = offset + queue_id * ZXDH_GDMA_CHAN_SHIFT;
+ val = *(uint32_t *)(gdmadev->base_addr + addr);
+
+ return val;
+}
+
static void
zxdh_gdma_write_reg(struct rte_rawdev *dev, uint16_t queue_id, uint32_t
offset, uint32_t val)
{
@@ -264,11 +279,11 @@ zxdh_gdma_rawdev_queue_setup(struct rte_rawdev *dev,
if (rbp->svfid != 0)
src_user |= (ZXDH_GDMA_VF_EN |
- ((rbp->svfid - 1) <<
ZXDH_GDMA_VF_NUM_SHIFT));
+ ((rbp->svfid - 1) <<
ZXDH_GDMA_VF_NUM_SHIFT));
ZXDH_PMD_LOG(DEBUG, "rxq->qidx:%d setup src_user(ep:%d pf:%d
vf:%d) success",
- queue_id, (uint8_t)rbp->sportid,
(uint8_t)rbp->spfid,
- (uint8_t)rbp->svfid);
+ queue_id, (uint8_t)rbp->sportid,
(uint8_t)rbp->spfid,
+ (uint8_t)rbp->svfid);
} else if ((rbp->srbp == 0) && (rbp->drbp != 0)) {
is_txq = 1;
src_user = ZXDH_GDMA_ZF_USER;
@@ -277,11 +292,11 @@ zxdh_gdma_rawdev_queue_setup(struct rte_rawdev *dev,
if (rbp->dvfid != 0)
dst_user |= (ZXDH_GDMA_VF_EN |
- ((rbp->dvfid - 1) <<
ZXDH_GDMA_VF_NUM_SHIFT));
+ ((rbp->dvfid - 1) <<
ZXDH_GDMA_VF_NUM_SHIFT));
ZXDH_PMD_LOG(DEBUG, "txq->qidx:%d setup dst_user(ep:%d pf:%d
vf:%d) success",
- queue_id, (uint8_t)rbp->dportid,
(uint8_t)rbp->dpfid,
- (uint8_t)rbp->dvfid);
+ queue_id, (uint8_t)rbp->dportid,
(uint8_t)rbp->dpfid,
+ (uint8_t)rbp->dvfid);
} else {
ZXDH_PMD_LOG(ERR, "Failed to setup queue, srbp/drbp is
invalid");
return -EINVAL;
@@ -353,7 +368,7 @@ zxdh_gdma_user_get(struct zxdh_gdma_queue *queue, struct
zxdh_gdma_job *job)
if ((job->flags & ZXDH_GDMA_JOB_DIR_MASK) == 0) {
ZXDH_PMD_LOG(DEBUG, "job flags:0x%x default user:0x%x",
- job->flags,
queue->user);
+ job->flags, queue->user);
return queue->user;
} else if ((job->flags & ZXDH_GDMA_JOB_DIR_TX) != 0) {
src_user = ZXDH_GDMA_ZF_USER;
@@ -362,7 +377,7 @@ zxdh_gdma_user_get(struct zxdh_gdma_queue *queue, struct
zxdh_gdma_job *job)
if (job->vf_id != 0)
dst_user |= (ZXDH_GDMA_VF_EN |
- ((job->vf_id - 1) <<
ZXDH_GDMA_VF_NUM_SHIFT));
+ ((job->vf_id - 1) <<
ZXDH_GDMA_VF_NUM_SHIFT));
} else {
dst_user = ZXDH_GDMA_ZF_USER;
src_user = ((job->pf_id << ZXDH_GDMA_PF_NUM_SHIFT) |
@@ -370,11 +385,11 @@ zxdh_gdma_user_get(struct zxdh_gdma_queue *queue, struct
zxdh_gdma_job *job)
if (job->vf_id != 0)
src_user |= (ZXDH_GDMA_VF_EN |
- ((job->vf_id - 1) <<
ZXDH_GDMA_VF_NUM_SHIFT));
+ ((job->vf_id - 1) <<
ZXDH_GDMA_VF_NUM_SHIFT));
}
ZXDH_PMD_LOG(DEBUG, "job flags:0x%x ep_id:%u, pf_id:%u, vf_id:%u,
user:0x%x",
- job->flags, job->ep_id,
job->pf_id, job->vf_id,
- (src_user & LOW16_MASK) |
(dst_user << 16));
+ job->flags, job->ep_id, job->pf_id, job->vf_id,
+ (src_user & LOW16_MASK) | (dst_user << 16));
return (src_user & LOW16_MASK) | (dst_user << 16);
}
@@ -395,8 +410,8 @@ zxdh_gdma_fill_bd(struct zxdh_gdma_queue *queue, struct
zxdh_gdma_job *job)
if (job != NULL) {
zxdh_gdma_control_cal(&val, 1);
next_bd_addr = IDX_TO_ADDR(queue->ring.ring_mem,
- (avail_idx + 1) %
ZXDH_GDMA_RING_SIZE,
- uint64_t);
+ (avail_idx + 1) %
ZXDH_GDMA_RING_SIZE,
+ uint64_t);
bd->SrcAddr_L = job->src & LOW32_MASK;
bd->DstAddr_L = job->dest & LOW32_MASK;
bd->SrcAddr_H = (job->src >> 32) & LOW32_MASK;
@@ -473,8 +488,8 @@ zxdh_gdma_rawdev_enqueue_bufs(struct rte_rawdev *dev,
free_cnt = queue->sw_ring.free_cnt;
if (free_cnt == 0) {
ZXDH_PMD_LOG(ERR, "queue %u is full, enq_idx:%u deq_idx:%u
used_idx:%u",
- queue_id,
queue->sw_ring.enq_idx,
- queue->sw_ring.deq_idx,
queue->sw_ring.used_idx);
+ queue_id, queue->sw_ring.enq_idx,
+ queue->sw_ring.deq_idx,
queue->sw_ring.used_idx);
return 0;
} else if (free_cnt < count) {
ZXDH_PMD_LOG(DEBUG, "job num %u > free_cnt, change to %u",
count, free_cnt);
@@ -519,6 +534,116 @@ zxdh_gdma_rawdev_enqueue_bufs(struct rte_rawdev *dev,
return count;
}
+
+static inline void
+zxdh_gdma_used_idx_update(struct zxdh_gdma_queue *queue, uint16_t cnt, uint8_t
data_bd_err)
+{
+ uint16_t idx = 0;
+
+ if (queue->sw_ring.used_idx + cnt < queue->queue_size)
+ queue->sw_ring.used_idx += cnt;
+ else
+ queue->sw_ring.used_idx = queue->sw_ring.used_idx + cnt -
queue->queue_size;
+
+ if (data_bd_err == 1) {
+ /* Update job status, the last job status is error */
+ if (queue->sw_ring.used_idx == 0)
+ idx = queue->queue_size - 1;
+ else
+ idx = queue->sw_ring.used_idx - 1;
+
+ queue->sw_ring.job[idx]->status = 1;
+ }
+}
+
+static int
+zxdh_gdma_rawdev_dequeue_bufs(struct rte_rawdev *dev,
+ __rte_unused struct rte_rawdev_buf **buffers,
+ uint32_t count,
+ rte_rawdev_obj_t context)
+{
+ struct zxdh_gdma_queue *queue = NULL;
+ struct zxdh_gdma_enqdeq *e_context = NULL;
+ uint16_t queue_id = 0;
+ uint32_t val = 0;
+ uint16_t tc_cnt = 0;
+ uint16_t diff_cnt = 0;
+ uint16_t i = 0;
+ uint16_t bd_idx = 0;
+ uint64_t next_bd_addr = 0;
+ uint8_t data_bd_err = 0;
+
+ if ((dev == NULL) || (context == NULL))
+ return -EINVAL;
+
+ e_context = (struct zxdh_gdma_enqdeq *)context;
+ queue_id = e_context->vq_id;
+ queue = zxdh_gdma_get_queue(dev, queue_id);
+ if ((queue == NULL) || (queue->enable == 0))
+ return -EINVAL;
+
+ if (queue->sw_ring.pend_cnt == 0)
+ goto deq_job;
+
+ /* Get data transmit count */
+ val = zxdh_gdma_read_reg(dev, queue_id, ZXDH_GDMA_TC_CNT_OFFSET);
+ tc_cnt = val & LOW16_MASK;
+ if (tc_cnt >= queue->tc_cnt)
+ diff_cnt = tc_cnt - queue->tc_cnt;
+ else
+ diff_cnt = tc_cnt + ZXDH_GDMA_TC_CNT_MAX - queue->tc_cnt;
+
+ queue->tc_cnt = tc_cnt;
+
+ /* Data transmit error, channel stopped */
+ if ((val & ZXDH_GDMA_ERR_STATUS) != 0) {
+ next_bd_addr = zxdh_gdma_read_reg(dev, queue_id,
ZXDH_GDMA_LLI_L_OFFSET);
+ next_bd_addr |= ((uint64_t)zxdh_gdma_read_reg(dev, queue_id,
+ ZXDH_GDMA_LLI_H_OFFSET)
<< 32);
+ next_bd_addr = next_bd_addr << 6;
+ bd_idx = (next_bd_addr - queue->ring.ring_mem) / sizeof(struct
zxdh_gdma_buff_desc);
+ if ((val & ZXDH_GDMA_SRC_DATA_ERR) || (val &
ZXDH_GDMA_DST_ADDR_ERR)) {
+ diff_cnt++;
+ data_bd_err = 1;
+ }
+ ZXDH_PMD_LOG(INFO, "queue%d is err(0x%x) next_bd_idx:%u
ll_addr:0x%"PRIx64" def user:0x%x",
+ queue_id, val, bd_idx, next_bd_addr,
queue->user);
+
+ ZXDH_PMD_LOG(INFO, "Clean up error status");
+ val = ZXDH_GDMA_ERR_STATUS | ZXDH_GDMA_ERR_INTR_ENABLE;
+ zxdh_gdma_write_reg(dev, queue_id, ZXDH_GDMA_TC_CNT_OFFSET,
val);
+
+ ZXDH_PMD_LOG(INFO, "Restart channel");
+ zxdh_gdma_write_reg(dev, queue_id, ZXDH_GDMA_XFERSIZE_OFFSET,
0);
+ zxdh_gdma_control_cal(&val, 0);
+ zxdh_gdma_write_reg(dev, queue_id, ZXDH_GDMA_CONTROL_OFFSET,
val);
+ }
+
+ if (diff_cnt != 0) {
+ zxdh_gdma_used_idx_update(queue, diff_cnt, data_bd_err);
+ queue->sw_ring.deq_cnt += diff_cnt;
+ queue->sw_ring.pend_cnt -= diff_cnt;
+ }
+
+deq_job:
+ if (queue->sw_ring.deq_cnt == 0)
+ return 0;
+ else if (queue->sw_ring.deq_cnt < count)
+ count = queue->sw_ring.deq_cnt;
+
+ queue->sw_ring.deq_cnt -= count;
+
+ for (i = 0; i < count; i++) {
+ e_context->job[i] = queue->sw_ring.job[queue->sw_ring.deq_idx];
+ queue->sw_ring.job[queue->sw_ring.deq_idx] = NULL;
+ if (++queue->sw_ring.deq_idx >= queue->queue_size)
+ queue->sw_ring.deq_idx -= queue->queue_size;
+ }
+ queue->sw_ring.free_cnt += count;
+
+ return count;
+}
+
static const struct rte_rawdev_ops zxdh_gdma_rawdev_ops = {
.dev_info_get = zxdh_gdma_rawdev_info_get,
.dev_configure = zxdh_gdma_rawdev_configure,
@@ -533,6 +658,7 @@ static const struct rte_rawdev_ops zxdh_gdma_rawdev_ops = {
.attr_get = zxdh_gdma_rawdev_get_attr,
.enqueue_bufs = zxdh_gdma_rawdev_enqueue_bufs,
+ .dequeue_bufs = zxdh_gdma_rawdev_dequeue_bufs,
};
static int
@@ -573,7 +699,7 @@ zxdh_gdma_queue_init(struct rte_rawdev *dev, uint16_t
queue_id)
snprintf(name, RTE_MEMZONE_NAMESIZE, "gdma_vq%d_ring", queue_id);
size = ZXDH_GDMA_RING_SIZE * sizeof(struct zxdh_gdma_buff_desc);
mz = rte_memzone_reserve_aligned(name, size, rte_socket_id(),
-
RTE_MEMZONE_IOVA_CONTIG, size);
+ RTE_MEMZONE_IOVA_CONTIG, size);
if (mz == NULL) {
if (rte_errno == EEXIST)
mz = rte_memzone_lookup(name);
@@ -589,7 +715,7 @@ zxdh_gdma_queue_init(struct rte_rawdev *dev, uint16_t
queue_id)
queue->ring.ring_mem = mz->iova;
queue->ring.avail_idx = 0;
ZXDH_PMD_LOG(INFO, "queue%u ring phy addr:0x%"PRIx64" virt addr:%p",
- queue_id, mz->iova, mz->addr);
+ queue_id, mz->iova, mz->addr);
/* Configure the hardware channel to the initial state */
zxdh_gdma_write_reg(dev, queue_id, ZXDH_GDMA_CONTROL_OFFSET,
@@ -689,7 +815,7 @@ zxdh_gdma_unmap_resource(void *requested_addr, size_t size)
/* Unmap the PCI memory resource of device */
if (rte_mem_unmap(requested_addr, size))
ZXDH_PMD_LOG(ERR, "cannot mem unmap(%p, %#zx): %s",
- requested_addr, size, rte_strerror(rte_errno));
+ requested_addr, size, rte_strerror(rte_errno));
else
ZXDH_PMD_LOG(DEBUG, "PCI memory unmapped at %p",
requested_addr);
}
@@ -715,8 +841,8 @@ zxdh_gdma_rawdev_probe(struct rte_pci_driver *pci_drv
__rte_unused,
return -1;
}
ZXDH_PMD_LOG(INFO, "%s bar0 0x%"PRIx64" mapped at %p",
- pci_dev->name,
pci_dev->mem_resource[0].phys_addr,
- pci_dev->mem_resource[0].addr);
+ pci_dev->name, pci_dev->mem_resource[0].phys_addr,
+ pci_dev->mem_resource[0].addr);
dev = rte_rawdev_pmd_allocate(dev_name, sizeof(struct
zxdh_gdma_rawdev), rte_socket_id());
if (dev == NULL) {
@@ -747,7 +873,7 @@ zxdh_gdma_rawdev_probe(struct rte_pci_driver *pci_drv
__rte_unused,
err_out:
zxdh_gdma_unmap_resource(pci_dev->mem_resource[0].addr,
- (size_t)pci_dev->mem_resource[0].len);
+ (size_t)pci_dev->mem_resource[0].len);
return -1;
}
--
2.43.0
