On Wed, Jan 23, 2019 at 02:00:46AM +0200, Oded Gabbay wrote:
> This patch adds two modules - ASID and context.
>
> Each user process the opens a device's file must have at least one context
^that
> before it is able to "work" with the device. Each context has its own
> device address-space and contains information about its runtime state (its
> active command submissions).
>
> To have address-space separation between contexts, each context is assigned
> a unique ASID, which stands for "address-space id". Goya supports up to
> 1024 ASIDs.
>
> Currently, the driver doesn't support multiple contexts. Therefore, the
> user doesn't need to actively create a context. A "primary context" is
> created automatically when the user opens the device's file.
>
> Signed-off-by: Oded Gabbay <oded.gab...@gmail.com>
> ---
> drivers/misc/habanalabs/Makefile | 2 +-
> drivers/misc/habanalabs/asid.c | 58 +++++++++
> drivers/misc/habanalabs/context.c | 155 +++++++++++++++++++++++
> drivers/misc/habanalabs/device.c | 47 +++++++
> drivers/misc/habanalabs/habanalabs.h | 70 ++++++++++
> drivers/misc/habanalabs/habanalabs_drv.c | 46 ++++++-
> 6 files changed, 375 insertions(+), 3 deletions(-)
> create mode 100644 drivers/misc/habanalabs/asid.c
> create mode 100644 drivers/misc/habanalabs/context.c
>
> diff --git a/drivers/misc/habanalabs/Makefile
> b/drivers/misc/habanalabs/Makefile
> index 6f1ead69bd77..3ffbadc2ca01 100644
> --- a/drivers/misc/habanalabs/Makefile
> +++ b/drivers/misc/habanalabs/Makefile
> @@ -4,7 +4,7 @@
>
> obj-m := habanalabs.o
>
> -habanalabs-y := habanalabs_drv.o device.o
> +habanalabs-y := habanalabs_drv.o device.o context.o asid.o
>
> include $(src)/goya/Makefile
> habanalabs-y += $(HL_GOYA_FILES)
> diff --git a/drivers/misc/habanalabs/asid.c b/drivers/misc/habanalabs/asid.c
> new file mode 100644
> index 000000000000..0ce84c8f5a47
> --- /dev/null
> +++ b/drivers/misc/habanalabs/asid.c
> @@ -0,0 +1,58 @@
> +// SPDX-License-Identifier: GPL-2.0
> +
> +/*
> + * Copyright 2016-2018 HabanaLabs, Ltd.
> + * All Rights Reserved.
> + */
> +
> +#include "habanalabs.h"
> +
> +#include <linux/slab.h>
> +#include <linux/types.h>
> +
> +int hl_asid_init(struct hl_device *hdev)
> +{
> + hdev->asid_bitmap = kcalloc(BITS_TO_LONGS(hdev->asic_prop.max_asid),
> + sizeof(*hdev->asid_bitmap), GFP_KERNEL);
> + if (!hdev->asid_bitmap)
> + return -ENOMEM;
> +
> + mutex_init(&hdev->asid_mutex);
> +
> + /* ASID 0 is reserved for KMD */
> + set_bit(0, hdev->asid_bitmap);
> +
> + return 0;
> +}
> +
> +void hl_asid_fini(struct hl_device *hdev)
> +{
> + mutex_destroy(&hdev->asid_mutex);
> + kfree(hdev->asid_bitmap);
> +}
> +
> +unsigned long hl_asid_alloc(struct hl_device *hdev)
> +{
> + unsigned long found;
> +
> + mutex_lock(&hdev->asid_mutex);
> +
> + found = find_first_zero_bit(hdev->asid_bitmap,
> + hdev->asic_prop.max_asid);
> + if (found == hdev->asic_prop.max_asid)
> + found = 0;
> + else
> + set_bit(found, hdev->asid_bitmap);
> +
> + mutex_unlock(&hdev->asid_mutex);
> +
> + return found;
> +}
> +
> +void hl_asid_free(struct hl_device *hdev, unsigned long asid)
> +{
> + if (WARN((asid == 0 || asid >= hdev->asic_prop.max_asid),
> + "Invalid ASID %lu", asid))
> + return;
> + clear_bit(asid, hdev->asid_bitmap);
> +}
> diff --git a/drivers/misc/habanalabs/context.c
> b/drivers/misc/habanalabs/context.c
> new file mode 100644
> index 000000000000..cdcad077e5cf
> --- /dev/null
> +++ b/drivers/misc/habanalabs/context.c
> @@ -0,0 +1,155 @@
> +// SPDX-License-Identifier: GPL-2.0
> +
> +/*
> + * Copyright 2016-2018 HabanaLabs, Ltd.
> + * All Rights Reserved.
> + */
> +
> +#include "habanalabs.h"
> +
> +#include <linux/sched.h>
> +#include <linux/delay.h>
> +
> +static void hl_ctx_fini(struct hl_ctx *ctx)
> +{
> + struct hl_device *hdev = ctx->hdev;
> +
> + if (ctx->asid != HL_KERNEL_ASID_ID)
> + hl_asid_free(hdev, ctx->asid);
> +}
> +
> +void hl_ctx_do_release(struct kref *ref)
> +{
> + struct hl_ctx *ctx;
> +
> + ctx = container_of(ref, struct hl_ctx, refcount);
> +
> + dev_dbg(ctx->hdev->dev, "Now really releasing context %d\n", ctx->asid);
> +
> + hl_ctx_fini(ctx);
> +
> + if (ctx->hpriv)
> + hl_hpriv_put(ctx->hpriv);
> +
> + kfree(ctx);
> +}
> +
> +int hl_ctx_create(struct hl_device *hdev, struct hl_fpriv *hpriv)
> +{
> + struct hl_ctx_mgr *mgr = &hpriv->ctx_mgr;
> + struct hl_ctx *ctx;
> + int rc;
> +
> + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
> + if (!ctx) {
> + rc = -ENOMEM;
> + goto out_err;
> + }
> +
> + rc = hl_ctx_init(hdev, ctx, false);
> + if (rc)
> + goto free_ctx;
> +
> + hl_hpriv_get(hpriv);
> + ctx->hpriv = hpriv;
> +
> + /* TODO: remove for multiple contexts */
> + hpriv->ctx = ctx;
> + hdev->user_ctx = ctx;
> +
> + mutex_lock(&mgr->ctx_lock);
> + rc = idr_alloc(&mgr->ctx_handles, ctx, 1, 0, GFP_KERNEL);
> + mutex_unlock(&mgr->ctx_lock);
> +
> + if (rc < 0) {
> + dev_err(hdev->dev, "Failed to allocate IDR for a new CTX\n");
> + hl_ctx_free(hdev, ctx);
> + goto out_err;
> + }
> +
> + return 0;
> +
> +free_ctx:
> + kfree(ctx);
> +out_err:
> + return rc;
> +}
> +
> +void hl_ctx_free(struct hl_device *hdev, struct hl_ctx *ctx)
> +{
> + if (kref_put(&ctx->refcount, hl_ctx_do_release) == 1)
> + return;
> +
> + dev_warn(hdev->dev,
> + "Context %d closed or terminated but its CS are executing\n",
> + ctx->asid);
> +}
> +
> +int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool
> is_kernel_ctx)
> +{
> + ctx->hdev = hdev;
> +
> + kref_init(&ctx->refcount);
> +
> + if (is_kernel_ctx) {
> + ctx->asid = HL_KERNEL_ASID_ID; /* KMD gets ASID 0 */
> + } else {
> + ctx->asid = hl_asid_alloc(hdev);
> + if (!ctx->asid) {
> + dev_err(hdev->dev, "No free ASID, failed to create
> context\n");
> + return -ENOMEM;
> + }
> + }
> +
> + dev_dbg(hdev->dev, "Created context with ASID %u\n", ctx->asid);
> +
> + return 0;
> +}
> +
> +void hl_ctx_get(struct hl_device *hdev, struct hl_ctx *ctx)
> +{
> + kref_get(&ctx->refcount);
> +}
> +
> +int hl_ctx_put(struct hl_ctx *ctx)
> +{
> + return kref_put(&ctx->refcount, hl_ctx_do_release);
> +}
> +
> +/**
> + * hl_ctx_mgr_init - initialize the context manager
> + *
> + * @mgr: pointer to context manager structure
> + *
> + * This manager is an object inside the hpriv object of the user process.
> + * The function is called when a user process opens the FD.
> + */
> +void hl_ctx_mgr_init(struct hl_ctx_mgr *mgr)
> +{
> + mutex_init(&mgr->ctx_lock);
> + idr_init(&mgr->ctx_handles);
> +}
> +
> +/**
> + * hl_ctx_mgr_fini - finalize the context manager
> + *
> + * @hdev: pointer to device structure
> + * @mgr: pointer to context manager structure
> + *
> + * This function goes over all the contexts in the manager and frees them.
> + * It is called when a process closes the FD.
> + */
> +void hl_ctx_mgr_fini(struct hl_device *hdev, struct hl_ctx_mgr *mgr)
> +{
> + struct hl_ctx *ctx;
> + struct idr *idp;
> + u32 id;
> +
> + idp = &mgr->ctx_handles;
> +
> + idr_for_each_entry(idp, ctx, id)
> + hl_ctx_free(hdev, ctx);
> +
> + idr_destroy(&mgr->ctx_handles);
> + mutex_destroy(&mgr->ctx_lock);
> +}
> diff --git a/drivers/misc/habanalabs/device.c
> b/drivers/misc/habanalabs/device.c
> index a4276ef559b3..84ce9fcb52da 100644
> --- a/drivers/misc/habanalabs/device.c
> +++ b/drivers/misc/habanalabs/device.c
> @@ -23,6 +23,12 @@ static void hpriv_release(struct kref *ref)
> put_pid(hpriv->taskpid);
>
> kfree(hpriv);
> +
> + /* Now the FD is really closed */
> + atomic_dec(&hdev->fd_open_cnt);
> +
> + /* This allows a new user context to open the device */
> + hdev->user_ctx = NULL;
> }
>
> void hl_hpriv_get(struct hl_fpriv *hpriv)
> @@ -47,6 +53,8 @@ static int hl_device_release(struct inode *inode, struct
> file *filp)
> {
> struct hl_fpriv *hpriv = filp->private_data;
>
> + hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr);
> +
> filp->private_data = NULL;
>
> hl_hpriv_put(hpriv);
> @@ -133,7 +141,20 @@ static int device_early_init(struct hl_device *hdev)
> if (rc)
> return rc;
>
> + rc = hl_asid_init(hdev);
> + if (rc)
> + goto early_fini;
> +
> + mutex_init(&hdev->device_open);
> + atomic_set(&hdev->fd_open_cnt, 0);
> +
> return 0;
> +
> +early_fini:
> + if (hdev->asic_funcs->early_fini)
> + hdev->asic_funcs->early_fini(hdev);
> +
> + return rc;
> }
>
> /**
> @@ -145,9 +166,12 @@ static int device_early_init(struct hl_device *hdev)
> static void device_early_fini(struct hl_device *hdev)
> {
>
> + hl_asid_fini(hdev);
> +
> if (hdev->asic_funcs->early_fini)
> hdev->asic_funcs->early_fini(hdev);
>
> + mutex_destroy(&hdev->device_open);
> }
>
> /**
> @@ -241,11 +265,30 @@ int hl_device_init(struct hl_device *hdev, struct class
> *hclass)
> if (rc)
> goto early_fini;
>
> + /* Allocate the kernel context */
> + hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL);
> + if (!hdev->kernel_ctx) {
> + rc = -ENOMEM;
> + goto sw_fini;
> + }
> +
> + hdev->user_ctx = NULL;
> +
> + rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
> + if (rc) {
> + dev_err(hdev->dev, "failed to initialize kernel context\n");
> + goto free_ctx;
> + }
> +
> dev_notice(hdev->dev,
> "Successfully added device to habanalabs driver\n");
>
> return 0;
>
> +free_ctx:
> + kfree(hdev->kernel_ctx);
> +sw_fini:
> + hdev->asic_funcs->sw_fini(hdev);
> early_fini:
> device_early_fini(hdev);
> release_device:
> @@ -278,6 +321,10 @@ void hl_device_fini(struct hl_device *hdev)
> /* Mark device as disabled */
> hdev->disabled = true;
>
> + /* Release kernel context */
> + if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1))
> + dev_err(hdev->dev, "kernel ctx is still alive\n");
> +
> /* Call ASIC S/W finalize function */
> hdev->asic_funcs->sw_fini(hdev);
>
> diff --git a/drivers/misc/habanalabs/habanalabs.h
> b/drivers/misc/habanalabs/habanalabs.h
> index 97844825f7a8..d003a6af2131 100644
> --- a/drivers/misc/habanalabs/habanalabs.h
> +++ b/drivers/misc/habanalabs/habanalabs.h
> @@ -125,6 +125,45 @@ struct hl_asic_funcs {
> void *cpu_addr, dma_addr_t dma_handle);
> };
>
> +
> +
> +
> +
> +/*
> + * CONTEXTS
> + */
> +
> +#define HL_KERNEL_ASID_ID 0
> +
> +/**
> + * struct hl_ctx - user/kernel context.
> + * @hpriv: pointer to the private (KMD) data of the process (fd).
> + * @hdev: pointer to the device structure.
> + * @refcount: reference counter for the context. Context is released only
> when
> + * this hits 0l. It is incremented on CS and CS_WAIT.
> + * @asid: context's unique address space ID in the device's MMU.
> + */
> +struct hl_ctx {
> + struct hl_fpriv *hpriv;
> + struct hl_device *hdev;
> + struct kref refcount;
> + u32 asid;
> +};
> +
> +/**
> + * struct hl_ctx_mgr - for handling multiple contexts.
> + * @ctx_lock: protects ctx_handles.
> + * @ctx_handles: idr to hold all ctx handles.
> + */
> +struct hl_ctx_mgr {
> + struct mutex ctx_lock;
> + struct idr ctx_handles;
> +};
> +
> +
> +
> +
> +
> /*
> * FILE PRIVATE STRUCTURE
> */
> @@ -134,12 +173,16 @@ struct hl_asic_funcs {
> * @hdev: habanalabs device structure.
> * @filp: pointer to the given file structure.
> * @taskpid: current process ID.
> + * @ctx: current executing context.
> + * @ctx_mgr: context manager to handle multiple context for this FD.
> * @refcount: number of related contexts.
> */
> struct hl_fpriv {
> struct hl_device *hdev;
> struct file *filp;
> struct pid *taskpid;
> + struct hl_ctx *ctx; /* TODO: remove for multiple ctx */
> + struct hl_ctx_mgr ctx_mgr;
> struct kref refcount;
> };
>
> @@ -195,13 +238,19 @@ void hl_wreg(struct hl_device *hdev, u32 reg, u32 val);
> * @dev: realted kernel basic device structure.
> * @asic_name: ASIC specific nmae.
> * @asic_type: ASIC specific type.
> + * @kernel_ctx: KMD context structure.
> * @dma_pool: DMA pool for small allocations.
> * @cpu_accessible_dma_mem: KMD <-> ArmCP shared memory CPU address.
> * @cpu_accessible_dma_address: KMD <-> ArmCP shared memory DMA address.
> * @cpu_accessible_dma_pool: KMD <-> ArmCP shared memory pool.
> + * @asid_bitmap: holds used/available ASIDs.
> + * @asid_mutex: protects asid_bitmap.
> + * @device_open: lock for sanity checks upon FD open.
device_open is an ambiguous name for a lock
> * @asic_prop: ASIC specific immutable properties.
> * @asic_funcs: ASIC specific functions.
> * @asic_specific: ASIC specific information to use only from ASIC files.
> + * @user_ctx: current user context executing.
> + * @fd_open_cnt: number of open context executing.
> * @major: habanalabs KMD major.
> * @id: device minor.
> * @disabled: is device disabled.
> @@ -214,13 +263,21 @@ struct hl_device {
> struct device *dev;
> char asic_name[16];
> enum hl_asic_type asic_type;
> + struct hl_ctx *kernel_ctx;
> struct dma_pool *dma_pool;
> void *cpu_accessible_dma_mem;
> dma_addr_t cpu_accessible_dma_address;
> struct gen_pool *cpu_accessible_dma_pool;
> + unsigned long *asid_bitmap;
> + struct mutex asid_mutex;
> + /* TODO: change to rw_sem for multiple contexts (same as other IOCTL) */
> + struct mutex device_open;
> struct asic_fixed_properties asic_prop;
> const struct hl_asic_funcs *asic_funcs;
> void *asic_specific;
> + /* TODO: The following fields should be moved for multi-context */
> + struct hl_ctx *user_ctx;
> + atomic_t fd_open_cnt;
> u32 major;
> u16 id;
> u8 disabled;
> @@ -270,10 +327,23 @@ int hl_poll_timeout_memory(struct hl_device *hdev, u64
> addr, u32 timeout_us,
> int hl_poll_timeout_device_memory(struct hl_device *hdev, void __iomem *addr,
> u32 timeout_us, u32 *val);
>
> +int hl_asid_init(struct hl_device *hdev);
> +void hl_asid_fini(struct hl_device *hdev);
> +unsigned long hl_asid_alloc(struct hl_device *hdev);
> +void hl_asid_free(struct hl_device *hdev, unsigned long asid);
> +
> +int hl_ctx_create(struct hl_device *hdev, struct hl_fpriv *hpriv);
> +void hl_ctx_free(struct hl_device *hdev, struct hl_ctx *ctx);
> +int hl_ctx_init(struct hl_device *hdev, struct hl_ctx *ctx, bool
> is_kernel_ctx);
> +int hl_ctx_put(struct hl_ctx *ctx);
> +void hl_ctx_mgr_init(struct hl_ctx_mgr *mgr);
> +void hl_ctx_mgr_fini(struct hl_device *hdev, struct hl_ctx_mgr *mgr);
> int hl_device_init(struct hl_device *hdev, struct class *hclass);
> void hl_device_fini(struct hl_device *hdev);
> int hl_device_suspend(struct hl_device *hdev);
> int hl_device_resume(struct hl_device *hdev);
> +void hl_hpriv_get(struct hl_fpriv *hpriv);
> +void hl_hpriv_put(struct hl_fpriv *hpriv);
>
> void goya_set_asic_funcs(struct hl_device *hdev);
>
> diff --git a/drivers/misc/habanalabs/habanalabs_drv.c
> b/drivers/misc/habanalabs/habanalabs_drv.c
> index 79545003b7c2..0646da83eb53 100644
> --- a/drivers/misc/habanalabs/habanalabs_drv.c
> +++ b/drivers/misc/habanalabs/habanalabs_drv.c
> @@ -77,6 +77,7 @@ int hl_device_open(struct inode *inode, struct file *filp)
> {
> struct hl_device *hdev;
> struct hl_fpriv *hpriv;
> + int rc;
>
> mutex_lock(&hl_devs_idr_lock);
> hdev = idr_find(&hl_devs_idr, iminor(inode));
> @@ -88,9 +89,33 @@ int hl_device_open(struct inode *inode, struct file *filp)
> return -ENXIO;
> }
>
> + mutex_lock(&hdev->device_open);
> +
> + if (hdev->disabled) {
> + dev_err_ratelimited(hdev->dev,
> + "Can't open %s because it is disabled\n",
> + dev_name(hdev->dev));
> + mutex_unlock(&hdev->device_open);
> + return -EPERM;
> + }
> +
> + if (hdev->user_ctx) {
> + dev_info_ratelimited(hdev->dev,
> + "Device %s is already attached to application\n",
> + dev_name(hdev->dev));
> + mutex_unlock(&hdev->device_open);
> + return -EBUSY;
> + }
> +
> + atomic_inc(&hdev->fd_open_cnt);
> +
> + mutex_unlock(&hdev->device_open);
> +
> hpriv = kzalloc(sizeof(*hpriv), GFP_KERNEL);
> - if (!hpriv)
> - return -ENOMEM;
> + if (!hpriv) {
> + rc = -ENOMEM;
> + goto close_device;
> + }
>
> hpriv->hdev = hdev;
> filp->private_data = hpriv;
> @@ -98,9 +123,26 @@ int hl_device_open(struct inode *inode, struct file *filp)
> kref_init(&hpriv->refcount);
> nonseekable_open(inode, filp);
>
> + hl_ctx_mgr_init(&hpriv->ctx_mgr);
> +
> + rc = hl_ctx_create(hdev, hpriv);
> + if (rc) {
> + dev_err(hdev->dev, "Failed to open FD (CTX fail)\n");
> + goto out_err;
> + }
> +
> hpriv->taskpid = find_get_pid(current->pid);
>
> return 0;
> +
> +out_err:
> + filp->private_data = NULL;
> + hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr);
> + kfree(hpriv);
> +
> +close_device:
> + atomic_dec(&hdev->fd_open_cnt);
> + return rc;
> }
>
> /**
> --
> 2.17.1
>
--
Sincerely yours,
Mike.
