On 5/12/2025 12:11 PM, Samuel Zhang wrote:
> add amdgpu_bo_fb_aper_addr() and update the cached GPU addresses to use
> the FB aperture address for SMU and PSP.
>
> 2 reasons for this change:
> 1. when pdb0 is enabled, gpu addr from amdgpu_bo_create_kernel() is GART
> aperture address, it is not compatible with SMU and PSP, it need to be
> updated to use FB aperture address.
> 2. Since FB aperture address will change after switching to new GPU
> index after hibernation, it need to be updated on resume.
>
> Signed-off-by: Jiang Liu <ge...@linux.alibaba.com>
> Signed-off-by: Samuel Zhang <guoqing.zh...@amd.com>
> ---
> drivers/gpu/drm/amd/amdgpu/amdgpu_object.c | 20 ++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_object.h | 1 +
> drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c | 27 ++++++++++++++++++++++
> drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c | 3 +++
> drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c | 18 +++++++++++++++
> 5 files changed, 69 insertions(+)
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
> index 4e794d546b61..3dde57cd5b81 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
> @@ -1513,6 +1513,26 @@ u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo)
> return amdgpu_bo_gpu_offset_no_check(bo);
> }
>
> +/**
> + * amdgpu_bo_fb_aper_addr - return FB aperture GPU offset of the VRAM bo
> + * @bo: amdgpu VRAM buffer object for which we query the offset
> + *
> + * Returns:
> + * current FB aperture GPU offset of the object.
> + */
> +u64 amdgpu_bo_fb_aper_addr(struct amdgpu_bo *bo)
> +{
> + struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
> + uint64_t offset, fb_base;
> +
> + WARN_ON_ONCE(bo->tbo.resource->mem_type != TTM_PL_VRAM);
> +
> + fb_base = adev->mmhub.funcs->get_fb_location(adev);
> + fb_base += adev->gmc.xgmi.physical_node_id *
> adev->gmc.xgmi.node_segment_size;
> + offset = (bo->tbo.resource->start << PAGE_SHIFT) + fb_base;
> + return amdgpu_gmc_sign_extend(offset);
> +}
> +
> /**
> * amdgpu_bo_gpu_offset_no_check - return GPU offset of bo
> * @bo: amdgpu object for which we query the offset
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
> index dcce362bfad3..c8a63e38f5d9 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.h
> @@ -320,6 +320,7 @@ int amdgpu_bo_sync_wait_resv(struct amdgpu_device *adev,
> struct dma_resv *resv,
> bool intr);
> int amdgpu_bo_sync_wait(struct amdgpu_bo *bo, void *owner, bool intr);
> u64 amdgpu_bo_gpu_offset(struct amdgpu_bo *bo);
> +u64 amdgpu_bo_fb_aper_addr(struct amdgpu_bo *bo);
> u64 amdgpu_bo_gpu_offset_no_check(struct amdgpu_bo *bo);
> uint32_t amdgpu_bo_mem_stats_placement(struct amdgpu_bo *bo);
> uint32_t amdgpu_bo_get_preferred_domain(struct amdgpu_device *adev,
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
> index e1e658a97b36..14271f62ca8e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
> @@ -871,6 +871,8 @@ static int psp_tmr_init(struct psp_context *psp)
> &psp->tmr_bo, &psp->tmr_mc_addr,
> pptr);
> }
> + if (amdgpu_virt_xgmi_migrate_enabled(psp->adev) && psp->tmr_bo)
> + psp->tmr_mc_addr = amdgpu_bo_fb_aper_addr(psp->tmr_bo);
>
> return ret;
> }
> @@ -1270,6 +1272,11 @@ int psp_ta_load(struct psp_context *psp, struct
> ta_context *context)
> psp_copy_fw(psp, context->bin_desc.start_addr,
> context->bin_desc.size_bytes);
>
> + if (amdgpu_virt_xgmi_migrate_enabled(psp->adev) &&
> + context->mem_context.shared_bo)
> + context->mem_context.shared_mc_addr =
> + amdgpu_bo_fb_aper_addr(context->mem_context.shared_bo);
> +
> psp_prep_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr, context);
>
> ret = psp_cmd_submit_buf(psp, NULL, cmd,
> @@ -2336,11 +2343,27 @@ bool amdgpu_psp_tos_reload_needed(struct
> amdgpu_device *adev)
> return false;
> }
>
> +static void psp_update_gpu_addresses(struct amdgpu_device *adev)
> +{
> + struct psp_context *psp = &adev->psp;
> +
> + if (psp->cmd_buf_bo && psp->cmd_buf_mem) {
> + psp->fw_pri_mc_addr = amdgpu_bo_fb_aper_addr(psp->fw_pri_bo);
> + psp->fence_buf_mc_addr =
> amdgpu_bo_fb_aper_addr(psp->fence_buf_bo);
> + psp->cmd_buf_mc_addr = amdgpu_bo_fb_aper_addr(psp->cmd_buf_bo);
> + }
> + if (adev->firmware.rbuf && psp->km_ring.ring_mem)
> + psp->km_ring.ring_mem_mc_addr =
> amdgpu_bo_fb_aper_addr(adev->firmware.rbuf);
> +}
> +
> static int psp_hw_start(struct psp_context *psp)
> {
> struct amdgpu_device *adev = psp->adev;
> int ret;
>
> + if (amdgpu_virt_xgmi_migrate_enabled(adev))
> + psp_update_gpu_addresses(adev);
> +
> if (!amdgpu_sriov_vf(adev)) {
> if ((is_psp_fw_valid(psp->kdb)) &&
> (psp->funcs->bootloader_load_kdb != NULL)) {
> @@ -3976,6 +3999,8 @@ static ssize_t psp_usbc_pd_fw_sysfs_write(struct device
> *dev,
> memcpy_toio(fw_pri_cpu_addr, usbc_pd_fw->data, usbc_pd_fw->size);
>
> mutex_lock(&adev->psp.mutex);
> + if (amdgpu_virt_xgmi_migrate_enabled(adev))
> + fw_pri_mc_addr = amdgpu_bo_fb_aper_addr(fw_buf_bo);
Better to skip this, this command is not supported on VF.
> ret = psp_load_usbc_pd_fw(&adev->psp, fw_pri_mc_addr);
> mutex_unlock(&adev->psp.mutex);
>
> @@ -4085,6 +4110,8 @@ static ssize_t amdgpu_psp_vbflash_read(struct file
> *filp, struct kobject *kobj,
> memcpy_toio(fw_pri_cpu_addr, adev->psp.vbflash_tmp_buf,
> adev->psp.vbflash_image_size);
>
> mutex_lock(&adev->psp.mutex);
> + if (amdgpu_virt_xgmi_migrate_enabled(adev))
> + fw_pri_mc_addr = amdgpu_bo_fb_aper_addr(fw_buf_bo);
Same as above - no VF support.
Thanks,
Lijo
> ret = psp_update_spirom(&adev->psp, fw_pri_mc_addr);
> mutex_unlock(&adev->psp.mutex);
>
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
> b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
> index 3d9e9fdc10b4..bf9013f8d12e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.c
> @@ -1152,6 +1152,9 @@ int amdgpu_ucode_init_bo(struct amdgpu_device *adev)
> adev->firmware.max_ucodes = AMDGPU_UCODE_ID_MAXIMUM;
> }
>
> + if (amdgpu_virt_xgmi_migrate_enabled(adev) && adev->firmware.fw_buf)
> + adev->firmware.fw_buf_mc =
> amdgpu_bo_fb_aper_addr(adev->firmware.fw_buf);
> +
> for (i = 0; i < adev->firmware.max_ucodes; i++) {
> ucode = &adev->firmware.ucode[i];
> if (ucode->fw) {
> diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
> b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
> index 315b0856bf02..f9f49f37dfcd 100644
> --- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
> +++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c
> @@ -1000,6 +1000,21 @@ static int smu_fini_fb_allocations(struct smu_context
> *smu)
> return 0;
> }
>
> +static void smu_update_gpu_addresses(struct smu_context *smu)
> +{
> + struct smu_table_context *smu_table = &smu->smu_table;
> + struct smu_table *pm_status_table = smu_table->tables +
> SMU_TABLE_PMSTATUSLOG;
> + struct smu_table *driver_table = &(smu_table->driver_table);
> + struct smu_table *dummy_read_1_table = &smu_table->dummy_read_1_table;
> +
> + if (pm_status_table->bo)
> + pm_status_table->mc_address =
> amdgpu_bo_fb_aper_addr(pm_status_table->bo);
> + if (driver_table->bo)
> + driver_table->mc_address =
> amdgpu_bo_fb_aper_addr(driver_table->bo);
> + if (dummy_read_1_table->bo)
> + dummy_read_1_table->mc_address =
> amdgpu_bo_fb_aper_addr(dummy_read_1_table->bo);
> +}
> +
> /**
> * smu_alloc_memory_pool - allocate memory pool in the system memory
> *
> @@ -1789,6 +1804,9 @@ static int smu_start_smc_engine(struct smu_context *smu)
> struct amdgpu_device *adev = smu->adev;
> int ret = 0;
>
> + if (amdgpu_virt_xgmi_migrate_enabled(adev))
> + smu_update_gpu_addresses(smu);
> +
> smu->smc_fw_state = SMU_FW_INIT;
>
> if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
