On 26/08/2020 06:52, rohit....@nxp.com wrote:
> From: Rohit Raj <rohit....@nxp.com>
>
> This patch add support for closing the bus objects which
> were acquired In the bus probe.
>
> Some devices need to be cleaned while in both primary and
> secondary process and while some need to be cleaned only in
> case of primary process.
>
> The devices are closed as per the white list used while
> creating the objects in a particular process.
>
> Signed-off-by: Rohit Raj <rohit....@nxp.com>
> Acked-by: Sachin Saxena <sachin.sax...@oss.nxp.com>
> ---
> drivers/bus/fslmc/fslmc_bus.c | 15 +++-
> drivers/bus/fslmc/fslmc_vfio.c | 89 ++++++++++++++++++++-
> drivers/bus/fslmc/fslmc_vfio.h | 3 +-
> drivers/bus/fslmc/mc/fsl_dpcon.h | 1 +
> drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c | 31 ++++++-
> drivers/bus/fslmc/portal/dpaa2_hw_dpci.c | 32 +++++++-
> drivers/bus/fslmc/portal/dpaa2_hw_dpio.c | 34 ++++++++
> drivers/bus/fslmc/rte_bus_fslmc_version.map | 1 +
> drivers/bus/fslmc/rte_fslmc.h | 5 +-
> drivers/event/dpaa2/dpaa2_hw_dpcon.c | 32 +++++++-
> drivers/net/dpaa2/dpaa2_mux.c | 18 ++++-
> 11 files changed, 253 insertions(+), 8 deletions(-)
>
> diff --git a/drivers/bus/fslmc/fslmc_bus.c b/drivers/bus/fslmc/fslmc_bus.c
> index beb3dd008..9d3da6be3 100644
> --- a/drivers/bus/fslmc/fslmc_bus.c
> +++ b/drivers/bus/fslmc/fslmc_bus.c
> @@ -1,6 +1,6 @@
> /* SPDX-License-Identifier: BSD-3-Clause
> *
> - * Copyright 2016,2018-2019 NXP
> + * Copyright 2016,2018-2020 NXP
> *
> */
>
> @@ -365,6 +365,18 @@ rte_fslmc_match(struct rte_dpaa2_driver *dpaa2_drv,
> return 1;
> }
>
> +static int
> +rte_fslmc_close(void)
> +{
> + int ret = 0;
> +
> + ret = fslmc_vfio_close_group();
> + if (ret)
> + DPAA2_BUS_ERR("Unable to close devices %d", ret);
> +
> + return 0;
> +}
> +
> static int
> rte_fslmc_probe(void)
> {
> @@ -637,6 +649,7 @@ struct rte_fslmc_bus rte_fslmc_bus = {
> .bus = {
> .scan = rte_fslmc_scan,
> .probe = rte_fslmc_probe,
> + .close = rte_fslmc_close,
> .parse = rte_fslmc_parse,
> .find_device = rte_fslmc_find_device,
> .get_iommu_class = rte_dpaa2_get_iommu_class,
> diff --git a/drivers/bus/fslmc/fslmc_vfio.c b/drivers/bus/fslmc/fslmc_vfio.c
> index 9134ec552..34ccc9db3 100644
> --- a/drivers/bus/fslmc/fslmc_vfio.c
> +++ b/drivers/bus/fslmc/fslmc_vfio.c
> @@ -1,7 +1,7 @@
> /* SPDX-License-Identifier: BSD-3-Clause
> *
> * Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights
> reserved.
> - * Copyright 2016-2019 NXP
> + * Copyright 2016-2020 NXP
> *
> */
>
> @@ -692,6 +692,54 @@ rte_dpaa2_vfio_setup_intr(struct rte_intr_handle
> *intr_handle,
> return -1;
> }
>
> +static void
> +fslmc_close_iodevices(struct rte_dpaa2_device *dev)
> +{
> + struct rte_dpaa2_object *object = NULL;
> + struct rte_dpaa2_driver *drv;
> + int ret, probe_all;
> +
> + switch (dev->dev_type) {
> + case DPAA2_IO:
> + case DPAA2_CON:
> + case DPAA2_CI:
> + case DPAA2_BPOOL:
> + case DPAA2_MUX:
> + TAILQ_FOREACH(object, &dpaa2_obj_list, next) {
> + if (dev->dev_type == object->dev_type)
> + object->close(dev->object_id);
> + else
> + continue;
> + }
> + break;
> + case DPAA2_ETH:
> + case DPAA2_CRYPTO:
> + case DPAA2_QDMA:
> + probe_all = rte_fslmc_bus.bus.conf.scan_mode !=
> + RTE_BUS_SCAN_WHITELIST;
> + TAILQ_FOREACH(drv, &rte_fslmc_bus.driver_list, next) {
> + if (drv->drv_type != dev->dev_type)
> + continue;
> + if (rte_dev_is_probed(&dev->device))
> + continue;
> + if (probe_all ||
> + (dev->device.devargs &&
> + dev->device.devargs->policy ==
> + RTE_DEV_WHITELISTED)) {
> + ret = drv->remove(dev);
> + if (ret)
> + DPAA2_BUS_ERR("Unable to remove");
> + }
> + }
> + break;
> + default:
> + break;
> + }
> +
> + DPAA2_BUS_LOG(DEBUG, "Device (%s) Closed",
> + dev->device.name);
> +}
> +
> /*
> * fslmc_process_iodevices for processing only IO (ETH, CRYPTO, and possibly
> * EVENT) devices.
> @@ -796,6 +844,45 @@ fslmc_process_mcp(struct rte_dpaa2_device *dev)
> return ret;
> }
>
> +int
> +fslmc_vfio_close_group(void)
> +{
> + struct rte_dpaa2_device *dev, *dev_temp;
> +
> + TAILQ_FOREACH_SAFE(dev, &rte_fslmc_bus.device_list, next, dev_temp) {
> + if (dev->device.devargs &&
> + dev->device.devargs->policy == RTE_DEV_BLACKLISTED) {
> + DPAA2_BUS_LOG(DEBUG, "%s Blacklisted, skipping",
> + dev->device.name);
> + TAILQ_REMOVE(&rte_fslmc_bus.device_list, dev, next);
> + continue;
> + }
> + switch (dev->dev_type) {
> + case DPAA2_ETH:
> + case DPAA2_CRYPTO:
> + case DPAA2_QDMA:
> + case DPAA2_IO:
> + fslmc_close_iodevices(dev);
> + break;
> + case DPAA2_CON:
> + case DPAA2_CI:
> + case DPAA2_BPOOL:
> + case DPAA2_MUX:
> + if (rte_eal_process_type() == RTE_PROC_SECONDARY)
> + continue;
> +
> + fslmc_close_iodevices(dev);
> + break;
> + case DPAA2_DPRTC:
> + default:
> + DPAA2_BUS_DEBUG("Device cannot be closed: Not supported
> (%s)",
> + dev->device.name);
> + }
> + }
> +
> + return 0;
> +}
> +
> int
> fslmc_vfio_process_group(void)
> {
> diff --git a/drivers/bus/fslmc/fslmc_vfio.h b/drivers/bus/fslmc/fslmc_vfio.h
> index bc7c6f62d..0a29fe739 100644
> --- a/drivers/bus/fslmc/fslmc_vfio.h
> +++ b/drivers/bus/fslmc/fslmc_vfio.h
> @@ -1,7 +1,7 @@
> /* SPDX-License-Identifier: BSD-3-Clause
> *
> * Copyright (c) 2015-2016 Freescale Semiconductor, Inc. All rights
> reserved.
> - * Copyright 2016,2019 NXP
> + * Copyright 2016,2019-2020 NXP
> *
> */
>
> @@ -55,6 +55,7 @@ int rte_dpaa2_vfio_setup_intr(struct rte_intr_handle
> *intr_handle,
>
> int fslmc_vfio_setup_group(void);
> int fslmc_vfio_process_group(void);
> +int fslmc_vfio_close_group(void);
> char *fslmc_get_container(void);
> int fslmc_get_container_group(int *gropuid);
> int rte_fslmc_vfio_dmamap(void);
> diff --git a/drivers/bus/fslmc/mc/fsl_dpcon.h
> b/drivers/bus/fslmc/mc/fsl_dpcon.h
> index 7caa6c68a..8053a6db1 100644
> --- a/drivers/bus/fslmc/mc/fsl_dpcon.h
> +++ b/drivers/bus/fslmc/mc/fsl_dpcon.h
> @@ -26,6 +26,7 @@ int dpcon_open(struct fsl_mc_io *mc_io,
> int dpcon_id,
> uint16_t *token);
>
> +__rte_internal
> int dpcon_close(struct fsl_mc_io *mc_io,
> uint32_t cmd_flags,
> uint16_t token);
> diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
> b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
> index d9619848d..9a4cdfd7a 100644
> --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
> +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpbp.c
> @@ -1,7 +1,7 @@
> /* SPDX-License-Identifier: BSD-3-Clause
> *
> * Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
> - * Copyright 2016 NXP
> + * Copyright 2016,2020 NXP
> *
> */
>
> @@ -33,6 +33,19 @@ TAILQ_HEAD(dpbp_dev_list, dpaa2_dpbp_dev);
> static struct dpbp_dev_list dpbp_dev_list
> = TAILQ_HEAD_INITIALIZER(dpbp_dev_list); /*!< DPBP device list */
>
> +static struct dpaa2_dpbp_dev *get_dpbp_from_id(uint32_t dpbp_id)
> +{
> + struct dpaa2_dpbp_dev *dpbp_dev = NULL;
> +
> + /* Get DPBP dev handle from list using index */
> + TAILQ_FOREACH(dpbp_dev, &dpbp_dev_list, next) {
> + if (dpbp_dev->dpbp_id == dpbp_id)
> + break;
> + }
> +
> + return dpbp_dev;
> +}
> +
> static int
> dpaa2_create_dpbp_device(int vdev_fd __rte_unused,
> struct vfio_device_info *obj_info __rte_unused,
> @@ -116,9 +129,25 @@ int dpaa2_dpbp_supported(void)
> return 0;
> }
>
> +static void
> +dpaa2_close_dpbp_device(int object_id)
> +{
> + struct dpaa2_dpbp_dev *dpbp_dev = NULL;
> +
> + dpbp_dev = get_dpbp_from_id((uint32_t)object_id);
> +
> + if (dpbp_dev) {
> + dpaa2_free_dpbp_dev(dpbp_dev);
> + dpbp_close(&dpbp_dev->dpbp, CMD_PRI_LOW, dpbp_dev->token);
> + TAILQ_REMOVE(&dpbp_dev_list, dpbp_dev, next);
> + rte_free(dpbp_dev);
> + }
> +}
> +
> static struct rte_dpaa2_object rte_dpaa2_dpbp_obj = {
> .dev_type = DPAA2_BPOOL,
> .create = dpaa2_create_dpbp_device,
> + .close = dpaa2_close_dpbp_device,
> };
>
> RTE_PMD_REGISTER_DPAA2_OBJECT(dpbp, rte_dpaa2_dpbp_obj);
> diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
> b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
> index d393ce618..9902f38ce 100644
> --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
> +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpci.c
> @@ -1,6 +1,6 @@
> /* SPDX-License-Identifier: BSD-3-Clause
> *
> - * Copyright 2017 NXP
> + * Copyright 2017,2020 NXP
> *
> */
>
> @@ -30,6 +30,19 @@ TAILQ_HEAD(dpci_dev_list, dpaa2_dpci_dev);
> static struct dpci_dev_list dpci_dev_list
> = TAILQ_HEAD_INITIALIZER(dpci_dev_list); /*!< DPCI device list */
>
> +static struct dpaa2_dpci_dev *get_dpci_from_id(uint32_t dpci_id)
> +{
> + struct dpaa2_dpci_dev *dpci_dev = NULL;
> +
> + /* Get DPCI dev handle from list using index */
> + TAILQ_FOREACH(dpci_dev, &dpci_dev_list, next) {
> + if (dpci_dev->dpci_id == dpci_id)
> + break;
> + }
> +
> + return dpci_dev;
> +}
> +
> static int
> rte_dpaa2_create_dpci_device(int vdev_fd __rte_unused,
> struct vfio_device_info *obj_info __rte_unused,
> @@ -179,9 +192,26 @@ void rte_dpaa2_free_dpci_dev(struct dpaa2_dpci_dev *dpci)
> }
> }
>
> +
> +static void
> +rte_dpaa2_close_dpci_device(int object_id)
> +{
> + struct dpaa2_dpci_dev *dpci_dev = NULL;
> +
> + dpci_dev = get_dpci_from_id((uint32_t)object_id);
> +
> + if (dpci_dev) {
> + rte_dpaa2_free_dpci_dev(dpci_dev);
> + dpci_close(&dpci_dev->dpci, CMD_PRI_LOW, dpci_dev->token);
> + TAILQ_REMOVE(&dpci_dev_list, dpci_dev, next);
> + rte_free(dpci_dev);
> + }
> +}
> +
> static struct rte_dpaa2_object rte_dpaa2_dpci_obj = {
> .dev_type = DPAA2_CI,
> .create = rte_dpaa2_create_dpci_device,
> + .close = rte_dpaa2_close_dpci_device,
> };
>
> RTE_PMD_REGISTER_DPAA2_OBJECT(dpci, rte_dpaa2_dpci_obj);
> diff --git a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
> b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
> index 97be76116..9563cd620 100644
> --- a/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
> +++ b/drivers/bus/fslmc/portal/dpaa2_hw_dpio.c
> @@ -86,6 +86,19 @@ static int dpaa2_cluster_sz = 2;
> * Cluster 4 (ID = x07) : CPU14, CPU15;
> */
>
> +static struct dpaa2_dpio_dev *get_dpio_dev_from_id(int32_t dpio_id)
> +{
> + struct dpaa2_dpio_dev *dpio_dev = NULL;
> +
> + /* Get DPIO dev handle from list using index */
> + TAILQ_FOREACH(dpio_dev, &dpio_dev_list, next) {
> + if (dpio_dev->hw_id == dpio_id)
> + break;
> + }
> +
> + return dpio_dev;
> +}
> +
> static int
> dpaa2_get_core_id(void)
> {
> @@ -360,6 +373,26 @@ static void dpaa2_portal_finish(void *arg)
> pthread_setspecific(dpaa2_portal_key, NULL);
> }
>
> +static void
> +dpaa2_close_dpio_device(int object_id)
> +{
> + struct dpaa2_dpio_dev *dpio_dev = NULL;
> +
> + dpio_dev = get_dpio_dev_from_id((int32_t)object_id);
> +
> + if (dpio_dev) {
> + if (dpio_dev->dpio) {
> + dpio_disable(dpio_dev->dpio, CMD_PRI_LOW,
> + dpio_dev->token);
> + dpio_close(dpio_dev->dpio, CMD_PRI_LOW,
> + dpio_dev->token);
> + rte_free(dpio_dev->dpio);
> + }
> + TAILQ_REMOVE(&dpio_dev_list, dpio_dev, next);
> + rte_free(dpio_dev);
> + }
> +}
> +
> static int
> dpaa2_create_dpio_device(int vdev_fd,
> struct vfio_device_info *obj_info,
> @@ -623,6 +656,7 @@ dpaa2_free_eq_descriptors(void)
> static struct rte_dpaa2_object rte_dpaa2_dpio_obj = {
> .dev_type = DPAA2_IO,
> .create = dpaa2_create_dpio_device,
> + .close = dpaa2_close_dpio_device,
> };
>
> RTE_PMD_REGISTER_DPAA2_OBJECT(dpio, rte_dpaa2_dpio_obj);
> diff --git a/drivers/bus/fslmc/rte_bus_fslmc_version.map
> b/drivers/bus/fslmc/rte_bus_fslmc_version.map
> index b169f5228..ac4ee2bf8 100644
> --- a/drivers/bus/fslmc/rte_bus_fslmc_version.map
> +++ b/drivers/bus/fslmc/rte_bus_fslmc_version.map
> @@ -30,6 +30,7 @@ INTERNAL {
> dpci_get_opr;
> dpci_set_opr;
> dpci_set_rx_queue;
> + dpcon_close;
> dpcon_get_attributes;
> dpcon_open;
> dpdmai_close;
> diff --git a/drivers/bus/fslmc/rte_fslmc.h b/drivers/bus/fslmc/rte_fslmc.h
> index 80873fffc..a938ab8f9 100644
> --- a/drivers/bus/fslmc/rte_fslmc.h
> +++ b/drivers/bus/fslmc/rte_fslmc.h
> @@ -1,6 +1,6 @@
> /* SPDX-License-Identifier: BSD-3-Clause
> *
> - * Copyright 2016,2019 NXP
> + * Copyright 2016,2019-2020 NXP
> *
> */
>
> @@ -79,6 +79,8 @@ typedef int (*rte_dpaa2_obj_create_t)(int vdev_fd,
> struct vfio_device_info *obj_info,
> int object_id);
>
> +typedef void (*rte_dpaa2_obj_close_t)(int object_id);
> +
> /**
> * A structure describing a DPAA2 object.
> */
> @@ -87,6 +89,7 @@ struct rte_dpaa2_object {
> const char *name; /**< Name of Object. */
> enum rte_dpaa2_dev_type dev_type; /**< Type of device */
> rte_dpaa2_obj_create_t create;
> + rte_dpaa2_obj_close_t close;
> };
>
> /**
> diff --git a/drivers/event/dpaa2/dpaa2_hw_dpcon.c
> b/drivers/event/dpaa2/dpaa2_hw_dpcon.c
> index 200b71640..86844f49f 100644
> --- a/drivers/event/dpaa2/dpaa2_hw_dpcon.c
> +++ b/drivers/event/dpaa2/dpaa2_hw_dpcon.c
> @@ -1,6 +1,6 @@
> /* SPDX-License-Identifier: BSD-3-Clause
> *
> - * Copyright 2017 NXP
> + * Copyright 2017,2020 NXP
> *
> */
>
> @@ -30,6 +30,19 @@ TAILQ_HEAD(dpcon_dev_list, dpaa2_dpcon_dev);
> static struct dpcon_dev_list dpcon_dev_list
> = TAILQ_HEAD_INITIALIZER(dpcon_dev_list); /*!< DPCON device list */
>
> +static struct dpaa2_dpcon_dev *get_dpcon_from_id(uint32_t dpcon_id)
> +{
> + struct dpaa2_dpcon_dev *dpcon_dev = NULL;
> +
> + /* Get DPCONC dev handle from list using index */
> + TAILQ_FOREACH(dpcon_dev, &dpcon_dev_list, next) {
> + if (dpcon_dev->dpcon_id == dpcon_id)
> + break;
> + }
> +
> + return dpcon_dev;
> +}
> +
> static int
> rte_dpaa2_create_dpcon_device(int dev_fd __rte_unused,
> struct vfio_device_info *obj_info __rte_unused,
> @@ -105,9 +118,26 @@ void rte_dpaa2_free_dpcon_dev(struct dpaa2_dpcon_dev
> *dpcon)
> }
> }
>
> +
> +static void
> +rte_dpaa2_close_dpcon_device(int object_id)
> +{
> + struct dpaa2_dpcon_dev *dpcon_dev = NULL;
> +
> + dpcon_dev = get_dpcon_from_id((uint32_t)object_id);
> +
> + if (dpcon_dev) {
> + rte_dpaa2_free_dpcon_dev(dpcon_dev);
> + dpcon_close(&dpcon_dev->dpcon, CMD_PRI_LOW, dpcon_dev->token);
> + TAILQ_REMOVE(&dpcon_dev_list, dpcon_dev, next);
> + rte_free(dpcon_dev);
> + }
> +}
> +
> static struct rte_dpaa2_object rte_dpaa2_dpcon_obj = {
> .dev_type = DPAA2_CON,
> .create = rte_dpaa2_create_dpcon_device,
> + .close = rte_dpaa2_close_dpcon_device,
> };
>
> RTE_PMD_REGISTER_DPAA2_OBJECT(dpcon, rte_dpaa2_dpcon_obj);
> diff --git a/drivers/net/dpaa2/dpaa2_mux.c b/drivers/net/dpaa2/dpaa2_mux.c
> index f8366e839..15114dac9 100644
> --- a/drivers/net/dpaa2/dpaa2_mux.c
> +++ b/drivers/net/dpaa2/dpaa2_mux.c
> @@ -44,7 +44,7 @@ static struct dpaa2_dpdmux_dev *get_dpdmux_from_id(uint32_t
> dpdmux_id)
> {
> struct dpaa2_dpdmux_dev *dpdmux_dev = NULL;
>
> - /* Get DPBP dev handle from list using index */
> + /* Get DPDMUX dev handle from list using index */
> TAILQ_FOREACH(dpdmux_dev, &dpdmux_dev_list, next) {
> if (dpdmux_dev->dpdmux_id == dpdmux_id)
> break;
> @@ -261,9 +261,25 @@ dpaa2_create_dpdmux_device(int vdev_fd __rte_unused,
> return -1;
> }
>
> +static void
> +dpaa2_close_dpdmux_device(int object_id)
> +{
> + struct dpaa2_dpdmux_dev *dpdmux_dev;
> +
> + dpdmux_dev = get_dpdmux_from_id((uint32_t)object_id);
> +
> + if (dpdmux_dev) {
> + dpdmux_close(&dpdmux_dev->dpdmux, CMD_PRI_LOW,
> + dpdmux_dev->token);
> + TAILQ_REMOVE(&dpdmux_dev_list, dpdmux_dev, next);
> + rte_free(dpdmux_dev);
> + }
> +}
> +
> static struct rte_dpaa2_object rte_dpaa2_dpdmux_obj = {
> .dev_type = DPAA2_MUX,
> .create = dpaa2_create_dpdmux_device,
> + .close = dpaa2_close_dpdmux_device,
> };
>
> RTE_PMD_REGISTER_DPAA2_OBJECT(dpdmux, rte_dpaa2_dpdmux_obj);
>
Acked-by: Ray Kinsella <m...@ashroe.eu>
