To allocate kernel BO from VRAM domain for MQD in the following patch. No functional change because kernel BO allocate all from GTT domain.
Rename amdgpu_amdkfd_alloc_gtt_mem to amdgpu_amdkfd_alloc_kernel_mem Rename amdgpu_amdkfd_free_gtt_mem to amdgpu_amdkfd_free_kernel_mem Rename mem_kfd_mem_obj gtt_mem to mem Signed-off-by: Philip Yang <[email protected]> Reviewed-by: Kent Russell <[email protected]> --- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c | 10 +++++----- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h | 6 +++--- drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 2 +- drivers/gpu/drm/amd/amdkfd/kfd_debug.c | 3 ++- drivers/gpu/drm/amd/amdkfd/kfd_device.c | 13 +++++++------ .../gpu/drm/amd/amdkfd/kfd_device_queue_manager.c | 7 ++++--- drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c | 14 +++++++------- drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c | 4 ++-- drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c | 9 +++++---- drivers/gpu/drm/amd/amdkfd/kfd_priv.h | 2 +- drivers/gpu/drm/amd/amdkfd/kfd_process.c | 2 +- .../gpu/drm/amd/amdkfd/kfd_process_queue_manager.c | 12 +++++++----- 12 files changed, 45 insertions(+), 39 deletions(-) diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c index a2879d2b7c8e..090d17911bc4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c @@ -316,8 +316,8 @@ void amdgpu_amdkfd_gpu_reset(struct amdgpu_device *adev) &adev->kfd.reset_work); } -int amdgpu_amdkfd_alloc_gtt_mem(struct amdgpu_device *adev, size_t size, - void **mem_obj, uint64_t *gpu_addr, +int amdgpu_amdkfd_alloc_kernel_mem(struct amdgpu_device *adev, size_t size, + u32 domain, void **mem_obj, uint64_t *gpu_addr, void **cpu_ptr, bool cp_mqd_gfx9) { struct amdgpu_bo *bo = NULL; @@ -328,7 +328,7 @@ int amdgpu_amdkfd_alloc_gtt_mem(struct amdgpu_device *adev, size_t size, memset(&bp, 0, sizeof(bp)); bp.size = size; bp.byte_align = PAGE_SIZE; - bp.domain = AMDGPU_GEM_DOMAIN_GTT; + bp.domain = domain; bp.flags = AMDGPU_GEM_CREATE_CPU_GTT_USWC; bp.type = ttm_bo_type_kernel; bp.resv = NULL; @@ -351,7 +351,7 @@ int amdgpu_amdkfd_alloc_gtt_mem(struct amdgpu_device *adev, size_t size, goto allocate_mem_reserve_bo_failed; } - r = amdgpu_bo_pin(bo, AMDGPU_GEM_DOMAIN_GTT); + r = amdgpu_bo_pin(bo, domain); if (r) { dev_err(adev->dev, "(%d) failed to pin bo for amdkfd\n", r); goto allocate_mem_pin_bo_failed; @@ -388,7 +388,7 @@ int amdgpu_amdkfd_alloc_gtt_mem(struct amdgpu_device *adev, size_t size, return r; } -void amdgpu_amdkfd_free_gtt_mem(struct amdgpu_device *adev, void **mem_obj) +void amdgpu_amdkfd_free_kernel_mem(struct amdgpu_device *adev, void **mem_obj) { struct amdgpu_bo **bo = (struct amdgpu_bo **) mem_obj; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h index da4575676335..274a99eb6d44 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h @@ -240,10 +240,10 @@ int amdgpu_amdkfd_bo_validate_and_fence(struct amdgpu_bo *bo, } #endif /* Shared API */ -int amdgpu_amdkfd_alloc_gtt_mem(struct amdgpu_device *adev, size_t size, - void **mem_obj, uint64_t *gpu_addr, +int amdgpu_amdkfd_alloc_kernel_mem(struct amdgpu_device *adev, size_t size, + u32 domain, void **mem_obj, uint64_t *gpu_addr, void **cpu_ptr, bool mqd_gfx9); -void amdgpu_amdkfd_free_gtt_mem(struct amdgpu_device *adev, void **mem_obj); +void amdgpu_amdkfd_free_kernel_mem(struct amdgpu_device *adev, void **mem_obj); int amdgpu_amdkfd_alloc_gws(struct amdgpu_device *adev, size_t size, void **mem_obj); void amdgpu_amdkfd_free_gws(struct amdgpu_device *adev, void *mem_obj); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c index 6585b8b9e709..b029a3e218b9 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c @@ -2214,7 +2214,7 @@ int amdgpu_amdkfd_gpuvm_sync_memory( * @bo_gart: Return bo reference * * Before return, bo reference count is incremented. To release the reference and unpin/ - * unmap the BO, call amdgpu_amdkfd_free_gtt_mem. + * unmap the BO, call amdgpu_amdkfd_free_kernel_mem. */ int amdgpu_amdkfd_map_gtt_bo_to_gart(struct amdgpu_bo *bo, struct amdgpu_bo **bo_gart) { diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c index f83e1238c1b3..903372ce29c9 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_debug.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_debug.c @@ -357,8 +357,9 @@ int kfd_dbg_set_mes_debug_mode(struct kfd_process_device *pdd, bool sq_trap_en) return 0; if (!pdd->proc_ctx_cpu_ptr) { - r = amdgpu_amdkfd_alloc_gtt_mem(adev, + r = amdgpu_amdkfd_alloc_kernel_mem(adev, AMDGPU_MES_PROC_CTX_SIZE, + AMDGPU_GEM_DOMAIN_GTT, &pdd->proc_ctx_bo, &pdd->proc_ctx_gpu_addr, &pdd->proc_ctx_cpu_ptr, diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device.c b/drivers/gpu/drm/amd/amdkfd/kfd_device.c index b91843b2af6a..23f4da4df3cd 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device.c @@ -808,12 +808,13 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd, /* add another 512KB for all other allocations on gart (HPD, fences) */ size += 512 * 1024; - if (amdgpu_amdkfd_alloc_gtt_mem( - kfd->adev, size, &kfd->gtt_mem, + if (amdgpu_amdkfd_alloc_kernel_mem( + kfd->adev, size, AMDGPU_GEM_DOMAIN_GTT, + &kfd->gtt_mem, &kfd->gtt_start_gpu_addr, &kfd->gtt_start_cpu_ptr, false)) { dev_err(kfd_device, "Could not allocate %d bytes\n", size); - goto alloc_gtt_mem_failure; + goto alloc_kernel_mem_failure; } dev_info(kfd_device, "Allocated %d bytes on gart\n", size); @@ -937,8 +938,8 @@ bool kgd2kfd_device_init(struct kfd_dev *kfd, kfd_doorbell_error: kfd_gtt_sa_fini(kfd); kfd_gtt_sa_init_error: - amdgpu_amdkfd_free_gtt_mem(kfd->adev, &kfd->gtt_mem); -alloc_gtt_mem_failure: + amdgpu_amdkfd_free_kernel_mem(kfd->adev, &kfd->gtt_mem); +alloc_kernel_mem_failure: dev_err(kfd_device, "device %x:%x NOT added due to errors\n", kfd->adev->pdev->vendor, kfd->adev->pdev->device); @@ -955,7 +956,7 @@ void kgd2kfd_device_exit(struct kfd_dev *kfd) kfd_doorbell_fini(kfd); ida_destroy(&kfd->doorbell_ida); kfd_gtt_sa_fini(kfd); - amdgpu_amdkfd_free_gtt_mem(kfd->adev, &kfd->gtt_mem); + amdgpu_amdkfd_free_kernel_mem(kfd->adev, &kfd->gtt_mem); } kfree(kfd); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index 1dbdf9465e59..3b846bc59e64 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -2907,8 +2907,9 @@ static int allocate_hiq_sdma_mqd(struct device_queue_manager *dqm) (dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size * NUM_XCC(dqm->dev->xcc_mask)); - retval = amdgpu_amdkfd_alloc_gtt_mem(dev->adev, size, - &(mem_obj->gtt_mem), &(mem_obj->gpu_addr), + retval = amdgpu_amdkfd_alloc_kernel_mem(dev->adev, size, + AMDGPU_GEM_DOMAIN_GTT, + &(mem_obj->mem), &(mem_obj->gpu_addr), (void *)&(mem_obj->cpu_ptr), false); return retval; @@ -3049,7 +3050,7 @@ static void deallocate_hiq_sdma_mqd(struct kfd_node *dev, { WARN(!mqd, "No hiq sdma mqd trunk to free"); - amdgpu_amdkfd_free_gtt_mem(dev->adev, &mqd->gtt_mem); + amdgpu_amdkfd_free_kernel_mem(dev->adev, &mqd->mem); } void device_queue_manager_uninit(struct device_queue_manager *dqm) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c index d9ae854b6908..f78b249e1a41 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager.c @@ -54,7 +54,7 @@ struct kfd_mem_obj *allocate_hiq_mqd(struct kfd_node *dev, struct queue_properti if (!mqd_mem_obj) return NULL; - mqd_mem_obj->gtt_mem = dev->dqm->hiq_sdma_mqd.gtt_mem; + mqd_mem_obj->mem = dev->dqm->hiq_sdma_mqd.mem; mqd_mem_obj->gpu_addr = dev->dqm->hiq_sdma_mqd.gpu_addr; mqd_mem_obj->cpu_ptr = dev->dqm->hiq_sdma_mqd.cpu_ptr; @@ -79,7 +79,7 @@ struct kfd_mem_obj *allocate_sdma_mqd(struct kfd_node *dev, offset += dev->dqm->mqd_mgrs[KFD_MQD_TYPE_HIQ]->mqd_size * NUM_XCC(dev->xcc_mask); - mqd_mem_obj->gtt_mem = (void *)((uint64_t)dev->dqm->hiq_sdma_mqd.gtt_mem + mqd_mem_obj->mem = (void *)((uint64_t)dev->dqm->hiq_sdma_mqd.mem + offset); mqd_mem_obj->gpu_addr = dev->dqm->hiq_sdma_mqd.gpu_addr + offset; mqd_mem_obj->cpu_ptr = (uint32_t *)((uint64_t) @@ -91,7 +91,7 @@ struct kfd_mem_obj *allocate_sdma_mqd(struct kfd_node *dev, void free_mqd_hiq_sdma(struct mqd_manager *mm, void *mqd, struct kfd_mem_obj *mqd_mem_obj) { - WARN_ON(!mqd_mem_obj->gtt_mem); + WARN_ON(!mqd_mem_obj->mem); kfree(mqd_mem_obj); } @@ -224,8 +224,8 @@ int kfd_destroy_mqd_cp(struct mqd_manager *mm, void *mqd, void kfd_free_mqd_cp(struct mqd_manager *mm, void *mqd, struct kfd_mem_obj *mqd_mem_obj) { - if (mqd_mem_obj->gtt_mem) { - amdgpu_amdkfd_free_gtt_mem(mm->dev->adev, &mqd_mem_obj->gtt_mem); + if (mqd_mem_obj->mem) { + amdgpu_amdkfd_free_kernel_mem(mm->dev->adev, &mqd_mem_obj->mem); kfree(mqd_mem_obj); } else { kfd_gtt_sa_free(mm->dev, mqd_mem_obj); @@ -280,8 +280,8 @@ void kfd_get_hiq_xcc_mqd(struct kfd_node *dev, struct kfd_mem_obj *mqd_mem_obj, offset = kfd_hiq_mqd_stride(dev) * virtual_xcc_id; - mqd_mem_obj->gtt_mem = (virtual_xcc_id == 0) ? - dev->dqm->hiq_sdma_mqd.gtt_mem : NULL; + mqd_mem_obj->mem = (virtual_xcc_id == 0) ? + dev->dqm->hiq_sdma_mqd.mem : NULL; mqd_mem_obj->gpu_addr = dev->dqm->hiq_sdma_mqd.gpu_addr + offset; mqd_mem_obj->cpu_ptr = (uint32_t *)((uintptr_t) dev->dqm->hiq_sdma_mqd.cpu_ptr + offset); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c index 06ecc86fcb4c..22f51df6f174 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v12_1.c @@ -370,8 +370,8 @@ static void get_xcc_mqd(struct kfd_mem_obj *mqd_mem_obj, struct kfd_mem_obj *xcc_mqd_mem_obj, uint64_t offset) { - xcc_mqd_mem_obj->gtt_mem = (offset == 0) ? - mqd_mem_obj->gtt_mem : NULL; + xcc_mqd_mem_obj->mem = (offset == 0) ? + mqd_mem_obj->mem : NULL; xcc_mqd_mem_obj->gpu_addr = mqd_mem_obj->gpu_addr + offset; xcc_mqd_mem_obj->cpu_ptr = (uint32_t *)((uintptr_t)mqd_mem_obj->cpu_ptr + offset); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c index 2e9b6bcf2704..d234db138182 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c @@ -135,11 +135,12 @@ static struct kfd_mem_obj *allocate_mqd(struct kfd_node *node, mqd_mem_obj = kzalloc(sizeof(struct kfd_mem_obj), GFP_KERNEL); if (!mqd_mem_obj) return NULL; - retval = amdgpu_amdkfd_alloc_gtt_mem(node->adev, + retval = amdgpu_amdkfd_alloc_kernel_mem(node->adev, (ALIGN(q->ctl_stack_size, PAGE_SIZE) + ALIGN(sizeof(struct v9_mqd), PAGE_SIZE)) * NUM_XCC(node->xcc_mask), - &(mqd_mem_obj->gtt_mem), + AMDGPU_GEM_DOMAIN_GTT, + &(mqd_mem_obj->mem), &(mqd_mem_obj->gpu_addr), (void *)&(mqd_mem_obj->cpu_ptr), true); @@ -665,8 +666,8 @@ static void get_xcc_mqd(struct kfd_mem_obj *mqd_mem_obj, struct kfd_mem_obj *xcc_mqd_mem_obj, uint64_t offset) { - xcc_mqd_mem_obj->gtt_mem = (offset == 0) ? - mqd_mem_obj->gtt_mem : NULL; + xcc_mqd_mem_obj->mem = (offset == 0) ? + mqd_mem_obj->mem : NULL; xcc_mqd_mem_obj->gpu_addr = mqd_mem_obj->gpu_addr + offset; xcc_mqd_mem_obj->cpu_ptr = (uint32_t *)((uintptr_t)mqd_mem_obj->cpu_ptr + offset); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h index ebc637c38c04..06cd675c9e74 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h +++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h @@ -252,7 +252,7 @@ struct kfd_mem_obj { uint32_t range_end; uint64_t gpu_addr; uint32_t *cpu_ptr; - void *gtt_mem; + void *mem; }; struct kfd_vmid_info { diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c index 2a72dc95cc0f..79b72ee8c8a1 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c @@ -1131,7 +1131,7 @@ static void kfd_process_destroy_pdds(struct kfd_process *p) if (pdd->dev->kfd->shared_resources.enable_mes && pdd->proc_ctx_cpu_ptr) - amdgpu_amdkfd_free_gtt_mem(pdd->dev->adev, + amdgpu_amdkfd_free_kernel_mem(pdd->dev->adev, &pdd->proc_ctx_bo); /* * before destroying pdd, make sure to report availability diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c index 5f8cda4733f9..232103742712 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c @@ -209,8 +209,8 @@ static void pqm_clean_queue_resource(struct process_queue_manager *pqm, } if (dev->kfd->shared_resources.enable_mes) { - amdgpu_amdkfd_free_gtt_mem(dev->adev, &pqn->q->gang_ctx_bo); - amdgpu_amdkfd_free_gtt_mem(dev->adev, (void **)&pqn->q->wptr_bo_gart); + amdgpu_amdkfd_free_kernel_mem(dev->adev, &pqn->q->gang_ctx_bo); + amdgpu_amdkfd_free_kernel_mem(dev->adev, (void **)&pqn->q->wptr_bo_gart); } } @@ -264,8 +264,9 @@ static int init_user_queue(struct process_queue_manager *pqm, (*q)->process = pqm->process; if (dev->kfd->shared_resources.enable_mes) { - retval = amdgpu_amdkfd_alloc_gtt_mem(dev->adev, + retval = amdgpu_amdkfd_alloc_kernel_mem(dev->adev, AMDGPU_MES_GANG_CTX_SIZE, + AMDGPU_GEM_DOMAIN_GTT, &(*q)->gang_ctx_bo, &(*q)->gang_ctx_gpu_addr, &(*q)->gang_ctx_cpu_ptr, @@ -297,7 +298,7 @@ static int init_user_queue(struct process_queue_manager *pqm, return 0; free_gang_ctx_bo: - amdgpu_amdkfd_free_gtt_mem(dev->adev, &(*q)->gang_ctx_bo); + amdgpu_amdkfd_free_kernel_mem(dev->adev, &(*q)->gang_ctx_bo); cleanup: uninit_queue(*q); *q = NULL; @@ -367,8 +368,9 @@ int pqm_create_queue(struct process_queue_manager *pqm, /* Allocate proc_ctx_bo only if MES is enabled and this is the first queue */ if (!pdd->proc_ctx_cpu_ptr && dev->kfd->shared_resources.enable_mes) { - retval = amdgpu_amdkfd_alloc_gtt_mem(dev->adev, + retval = amdgpu_amdkfd_alloc_kernel_mem(dev->adev, AMDGPU_MES_PROC_CTX_SIZE, + AMDGPU_GEM_DOMAIN_GTT, &pdd->proc_ctx_bo, &pdd->proc_ctx_gpu_addr, &pdd->proc_ctx_cpu_ptr, -- 2.50.1
