One quick question
>-----Original Message-----
>From: Pablo de Lara [mailto:pablo.de.lara.gua...@intel.com]
>Sent: 27 April 2018 18:54
>To: dev@dpdk.org
>Cc: fiona.tr...@intel.com; Verma, Shally <shally.ve...@cavium.com>;
>ahmed.mans...@nxp.com; Gupta, Ashish
><ashish.gu...@cavium.com>; Pablo de Lara <pablo.de.lara.gua...@intel.com>;
>Verma, Shally <shally.ve...@cavium.com>; Gupta,
>Ashish <ashish.gu...@cavium.com>
>Subject: [PATCH v6 02/14] compressdev: add queue pair management
>
>From: Fiona Trahe <fiona.tr...@intel.com>
>
>Add functions to manage device queue pairs.
>
>Signed-off-by: Fiona Trahe <fiona.tr...@intel.com>
>Signed-off-by: Pablo de Lara <pablo.de.lara.gua...@intel.com>
>Signed-off-by: Shally Verma <shally.ve...@caviumnetworks.com>
>Signed-off-by: Ashish Gupta <ashish.gu...@caviumnetworks.com>
>---
> lib/librte_compressdev/rte_compressdev.c | 178 ++++++++++++++++++++-
> lib/librte_compressdev/rte_compressdev.h | 45 ++++++
> lib/librte_compressdev/rte_compressdev_internal.h | 5 +
> lib/librte_compressdev/rte_compressdev_pmd.h | 47 ++++++
> lib/librte_compressdev/rte_compressdev_version.map | 2 +
> 5 files changed, 276 insertions(+), 1 deletion(-)
>
>diff --git a/lib/librte_compressdev/rte_compressdev.c
>b/lib/librte_compressdev/rte_compressdev.c
>index 751517c3d..6667528b5 100644
>--- a/lib/librte_compressdev/rte_compressdev.c
>+++ b/lib/librte_compressdev/rte_compressdev.c
>@@ -227,10 +227,136 @@ rte_compressdev_pmd_release_device(struct
>rte_compressdev *compressdev)
> return 0;
> }
>
>+uint16_t __rte_experimental
>+rte_compressdev_queue_pair_count(uint8_t dev_id)
>+{
>+ struct rte_compressdev *dev;
>+
>+ dev = &rte_comp_devices[dev_id];
>+ return dev->data->nb_queue_pairs;
>+}
>+
>+static int
>+rte_compressdev_queue_pairs_config(struct rte_compressdev *dev,
>+ uint16_t nb_qpairs, int socket_id)
>+{
>+ struct rte_compressdev_info dev_info;
>+ void **qp;
>+ unsigned int i;
>+
>+ if ((dev == NULL) || (nb_qpairs < 1)) {
>+ COMPRESSDEV_LOG(ERR, "invalid param: dev %p, nb_queues %u",
>+ dev, nb_qpairs);
>+ return -EINVAL;
>+ }
>+
>+ COMPRESSDEV_LOG(DEBUG, "Setup %d queues pairs on device %u",
>+ nb_qpairs, dev->data->dev_id);
>+
>+ memset(&dev_info, 0, sizeof(struct rte_compressdev_info));
>+
>+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
>+ (*dev->dev_ops->dev_infos_get)(dev, &dev_info);
>+
>+ if ((dev_info.max_nb_queue_pairs != 0) &&
>+ (nb_qpairs > dev_info.max_nb_queue_pairs)) {
>+ COMPRESSDEV_LOG(ERR, "Invalid num queue_pairs (%u) for dev %u",
>+ nb_qpairs, dev->data->dev_id);
>+ return -EINVAL;
>+ }
>+
>+ if (dev->data->queue_pairs == NULL) { /* first time configuration */
>+ dev->data->queue_pairs = rte_zmalloc_socket(
>+ "compressdev->queue_pairs",
>+ sizeof(dev->data->queue_pairs[0]) * nb_qpairs,
>+ RTE_CACHE_LINE_SIZE, socket_id);
>+
>+ if (dev->data->queue_pairs == NULL) {
>+ dev->data->nb_queue_pairs = 0;
>+ COMPRESSDEV_LOG(ERR,
>+ "failed to get memory for qp meta data, nb_queues %u",
>+ nb_qpairs);
>+ return -(ENOMEM);
>+ }
>+ } else { /* re-configure */
>+ int ret;
>+ uint16_t old_nb_queues = dev->data->nb_queue_pairs;
>+
[Shally] dev->data is shared entity among processes. So, does it mean if
multiple processes call for device_configure, then every new process overrides
qp allocations (and other resources) already done by any other process, leaving
it unusable for them?
Thanks
Shally
>+ qp = dev->data->queue_pairs;
>+
>+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_release,
>+ -ENOTSUP);
>+
>+ for (i = nb_qpairs; i < old_nb_queues; i++) {
>+ ret = (*dev->dev_ops->queue_pair_release)(dev, i);
>+ if (ret < 0)
>+ return ret;
>+ }
>+
>+ qp = rte_realloc(qp, sizeof(qp[0]) * nb_qpairs,
>+ RTE_CACHE_LINE_SIZE);
>+ if (qp == NULL) {
>+ COMPRESSDEV_LOG(ERR,
>+ "failed to realloc qp meta data, nb_queues %u",
>+ nb_qpairs);
>+ return -(ENOMEM);
>+ }
>+
>+ if (nb_qpairs > old_nb_queues) {
>+ uint16_t new_qs = nb_qpairs - old_nb_queues;
>+
>+ memset(qp + old_nb_queues, 0,
>+ sizeof(qp[0]) * new_qs);
>+ }
>+
>+ dev->data->queue_pairs = qp;
>+
>+ }
>+ dev->data->nb_queue_pairs = nb_qpairs;
>+ return 0;
>+}
>+
>+static int
>+rte_compressdev_queue_pairs_release(struct rte_compressdev *dev)
>+{
>+ uint16_t num_qps, i;
>+ int ret;
>+
>+ if (dev == NULL) {
>+ COMPRESSDEV_LOG(ERR, "invalid param: dev %p", dev);
>+ return -EINVAL;
>+ }
>+
>+ num_qps = dev->data->nb_queue_pairs;
>+
>+ if (num_qps == 0)
>+ return 0;
>+
>+ COMPRESSDEV_LOG(DEBUG, "Free %d queues pairs on device %u",
>+ dev->data->nb_queue_pairs, dev->data->dev_id);
>+
>+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_release,
>+ -ENOTSUP);
>+
>+ for (i = 0; i < num_qps; i++) {
>+ ret = (*dev->dev_ops->queue_pair_release)(dev, i);
>+ if (ret < 0)
>+ return ret;
>+ }
>+
>+ if (dev->data->queue_pairs != NULL)
>+ rte_free(dev->data->queue_pairs);
>+ dev->data->queue_pairs = NULL;
>+ dev->data->nb_queue_pairs = 0;
>+
>+ return 0;
>+}
>+
> int __rte_experimental
> rte_compressdev_configure(uint8_t dev_id, struct rte_compressdev_config
> *config)
> {
> struct rte_compressdev *dev;
>+ int diag;
>
> if (!rte_compressdev_is_valid_dev(dev_id)) {
> COMPRESSDEV_LOG(ERR, "Invalid dev_id=%" PRIu8, dev_id);
>@@ -247,10 +373,19 @@ rte_compressdev_configure(uint8_t dev_id, struct
>rte_compressdev_config *config)
>
> RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
>
>+ /* Setup new number of queue pairs and reconfigure device. */
>+ diag = rte_compressdev_queue_pairs_config(dev, config->nb_queue_pairs,
>+ config->socket_id);
>+ if (diag != 0) {
>+ COMPRESSDEV_LOG(ERR,
>+ "dev%d rte_comp_dev_queue_pairs_config = %d",
>+ dev_id, diag);
>+ return diag;
>+ }
>+
> return (*dev->dev_ops->dev_configure)(dev, config);
> }
>
>-
> int __rte_experimental
> rte_compressdev_start(uint8_t dev_id)
> {
>@@ -327,6 +462,12 @@ rte_compressdev_close(uint8_t dev_id)
> return -EBUSY;
> }
>
>+ /* Free queue pairs memory */
>+ retval = rte_compressdev_queue_pairs_release(dev);
>+
>+ if (retval < 0)
>+ return retval;
>+
> RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_close, -ENOTSUP);
> retval = (*dev->dev_ops->dev_close)(dev);
>
>@@ -336,6 +477,41 @@ rte_compressdev_close(uint8_t dev_id)
> return 0;
> }
>
>+int __rte_experimental
>+rte_compressdev_queue_pair_setup(uint8_t dev_id, uint16_t queue_pair_id,
>+ uint32_t max_inflight_ops, int socket_id)
>+{
>+ struct rte_compressdev *dev;
>+
>+ if (!rte_compressdev_is_valid_dev(dev_id)) {
>+ COMPRESSDEV_LOG(ERR, "Invalid dev_id=%" PRIu8, dev_id);
>+ return -EINVAL;
>+ }
>+
>+ dev = &rte_comp_devices[dev_id];
>+ if (queue_pair_id >= dev->data->nb_queue_pairs) {
>+ COMPRESSDEV_LOG(ERR, "Invalid queue_pair_id=%d", queue_pair_id);
>+ return -EINVAL;
>+ }
>+
>+ if (dev->data->dev_started) {
>+ COMPRESSDEV_LOG(ERR,
>+ "device %d must be stopped to allow configuration", dev_id);
>+ return -EBUSY;
>+ }
>+
>+ if (max_inflight_ops == 0) {
>+ COMPRESSDEV_LOG(ERR,
>+ "Invalid maximum number of inflight operations");
>+ return -EINVAL;
>+ }
>+
>+ RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_pair_setup, -ENOTSUP);
>+
>+ return (*dev->dev_ops->queue_pair_setup)(dev, queue_pair_id,
>+ max_inflight_ops, socket_id);
>+}
>+
> void __rte_experimental
> rte_compressdev_info_get(uint8_t dev_id, struct rte_compressdev_info
> *dev_info)
> {
>diff --git a/lib/librte_compressdev/rte_compressdev.h
>b/lib/librte_compressdev/rte_compressdev.h
>index 5be5973bf..81710bbea 100644
>--- a/lib/librte_compressdev/rte_compressdev.h
>+++ b/lib/librte_compressdev/rte_compressdev.h
>@@ -22,6 +22,10 @@ extern "C" {
> /** comp device information */
> struct rte_compressdev_info {
> const char *driver_name; /**< Driver name. */
>+ uint16_t max_nb_queue_pairs;
>+ /**< Maximum number of queues pairs supported by device.
>+ * (If 0, there is no limit in maximum number of queue pairs)
>+ */
> };
>
> /**
>@@ -80,6 +84,9 @@ rte_compressdev_socket_id(uint8_t dev_id);
> /** Compress device configuration structure */
> struct rte_compressdev_config {
> int socket_id;
>+ /**< Socket on which to allocate resources */
>+ uint16_t nb_queue_pairs;
>+ /**< Total number of queue pairs to configure on a device */
> };
>
> /**
>@@ -145,6 +152,44 @@ rte_compressdev_stop(uint8_t dev_id);
> int __rte_experimental
> rte_compressdev_close(uint8_t dev_id);
>
>+/**
>+ * Allocate and set up a receive queue pair for a device.
>+ * This should only be called when the device is stopped.
>+ *
>+ *
>+ * @param dev_id
>+ * Compress device identifier
>+ * @param queue_pair_id
>+ * The index of the queue pairs to set up. The
>+ * value must be in the range [0, nb_queue_pair - 1]
>+ * previously supplied to rte_compressdev_configure()
>+ * @param max_inflight_ops
>+ * Max number of ops which the qp will have to
>+ * accommodate simultaneously
>+ * @param socket_id
>+ * The *socket_id* argument is the socket identifier
>+ * in case of NUMA. The value can be *SOCKET_ID_ANY*
>+ * if there is no NUMA constraint for the DMA memory
>+ * allocated for the receive queue pair
>+ * @return
>+ * - 0: Success, queue pair correctly set up.
>+ * - <0: Queue pair configuration failed
>+ */
>+int __rte_experimental
>+rte_compressdev_queue_pair_setup(uint8_t dev_id, uint16_t queue_pair_id,
>+ uint32_t max_inflight_ops, int socket_id);
>+
>+/**
>+ * Get the number of queue pairs on a specific comp device
>+ *
>+ * @param dev_id
>+ * Compress device identifier
>+ * @return
>+ * - The number of configured queue pairs.
>+ */
>+uint16_t __rte_experimental
>+rte_compressdev_queue_pair_count(uint8_t dev_id);
>+
> /**
> * Retrieve the contextual information of a device.
> *
>diff --git a/lib/librte_compressdev/rte_compressdev_internal.h
>b/lib/librte_compressdev/rte_compressdev_internal.h
>index 0a2ddcb2b..57af163c1 100644
>--- a/lib/librte_compressdev/rte_compressdev_internal.h
>+++ b/lib/librte_compressdev/rte_compressdev_internal.h
>@@ -58,6 +58,11 @@ struct rte_compressdev_data {
> uint8_t dev_started : 1;
> /**< Device state: STARTED(1)/STOPPED(0) */
>
>+ void **queue_pairs;
>+ /**< Array of pointers to queue pairs. */
>+ uint16_t nb_queue_pairs;
>+ /**< Number of device queue pairs */
>+
> void *dev_private;
> /**< PMD-specific private data */
> } __rte_cache_aligned;
>diff --git a/lib/librte_compressdev/rte_compressdev_pmd.h
>b/lib/librte_compressdev/rte_compressdev_pmd.h
>index 43307ee8e..14ce76b44 100644
>--- a/lib/librte_compressdev/rte_compressdev_pmd.h
>+++ b/lib/librte_compressdev/rte_compressdev_pmd.h
>@@ -125,6 +125,48 @@ typedef int (*compressdev_close_t)(struct rte_compressdev
>*dev);
> typedef void (*compressdev_info_get_t)(struct rte_compressdev *dev,
> struct rte_compressdev_info *dev_info);
>
>+/**
>+ * Setup a queue pair for a device.
>+ *
>+ * @param dev
>+ * Compress device
>+ * @param qp_id
>+ * Queue pair identifier
>+ * @param max_inflight_ops
>+ * Max inflight ops which qp must accommodate
>+ * @param socket_id
>+ * Socket identifier
>+ * @return
>+ * Returns 0 on success.
>+ */
>+typedef int (*compressdev_queue_pair_setup_t)(struct rte_compressdev *dev,
>+ uint16_t qp_id, uint32_t max_inflight_ops, int socket_id);
>+
>+/**
>+ * Release memory resources allocated by given queue pair.
>+ *
>+ * @param dev
>+ * Compress device
>+ * @param qp_id
>+ * Queue pair identifier
>+ * @return
>+ * - 0 on success.
>+ * - EAGAIN if can't close as device is busy
>+ */
>+typedef int (*compressdev_queue_pair_release_t)(struct rte_compressdev *dev,
>+ uint16_t qp_id);
>+
>+/**
>+ * Get number of available queue pairs of a device.
>+ *
>+ * @param dev
>+ * Compress device
>+ * @return
>+ * Returns number of queue pairs on success.
>+ */
>+typedef uint32_t (*compressdev_queue_pair_count_t)(struct rte_compressdev
>*dev);
>+
>+
> /** comp device operations function pointer table */
> struct rte_compressdev_ops {
> compressdev_configure_t dev_configure; /**< Configure device. */
>@@ -133,6 +175,11 @@ struct rte_compressdev_ops {
> compressdev_close_t dev_close; /**< Close device. */
>
> compressdev_info_get_t dev_infos_get; /**< Get device info. */
>+
>+ compressdev_queue_pair_setup_t queue_pair_setup;
>+ /**< Set up a device queue pair. */
>+ compressdev_queue_pair_release_t queue_pair_release;
>+ /**< Release a queue pair. */
> };
>
> /**
>diff --git a/lib/librte_compressdev/rte_compressdev_version.map
>b/lib/librte_compressdev/rte_compressdev_version.map
>index a996abcd2..182a371d8 100644
>--- a/lib/librte_compressdev/rte_compressdev_version.map
>+++ b/lib/librte_compressdev/rte_compressdev_version.map
>@@ -16,6 +16,8 @@ EXPERIMENTAL {
> rte_compressdev_pmd_get_named_dev;
> rte_compressdev_pmd_parse_input_args;
> rte_compressdev_pmd_release_device;
>+ rte_compressdev_queue_pair_count;
>+ rte_compressdev_queue_pair_setup;
> rte_compressdev_socket_id;
> rte_compressdev_start;
> rte_compressdev_stop;
>--
>2.14.3
