llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-backend-amdgpu Author: Matt Arsenault (arsenm) <details> <summary>Changes</summary> Make sure the MFMA VGPR to AGPR InstrMapping table is complete. I think I got everything, except the full cross product of input types with the mfma scale intrinsics. Also makes sure we have coverage for smfmac and mfma_scale cases. --- Patch is 85.27 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/153026.diff 3 Files Affected: - (added) llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.gfx90a.ll (+141) - (added) llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.gfx950.ll (+664) - (modified) llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.ll (+867) ``````````diff diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.gfx90a.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.gfx90a.ll new file mode 100644 index 0000000000000..7d00b12e7334a --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.gfx90a.ll @@ -0,0 +1,141 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mcpu=gfx90a -amdgpu-mfma-vgpr-form < %s | FileCheck %s + +target triple = "amdgcn-amd-amdhsa" + +define void @test_rewrite_mfma_i32_32x32x8i8(i32 %arg0, i32 %arg1, ptr addrspace(1) %ptr) #0 { +; CHECK-LABEL: test_rewrite_mfma_i32_32x32x8i8: +; CHECK: ; %bb.0: +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: global_load_dwordx4 a[12:15], v[2:3], off offset:48 +; CHECK-NEXT: global_load_dwordx4 a[8:11], v[2:3], off offset:32 +; CHECK-NEXT: global_load_dwordx4 a[4:7], v[2:3], off offset:16 +; CHECK-NEXT: global_load_dwordx4 a[0:3], v[2:3], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: v_mfma_i32_32x32x8i8 a[0:15], v0, v1, a[0:15] +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ; use a[0:15] +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: s_setpc_b64 s[30:31] + %src2 = load <16 x i32>, ptr addrspace(1) %ptr + %mai = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x8i8(i32 %arg0, i32 %arg1, <16 x i32> %src2, i32 0, i32 0, i32 0) + call void asm sideeffect "; use $0", "a"(<16 x i32> %mai) + ret void +} + +define void @test_rewrite_mfma_i32_16x16x16i8(i32 %arg0, i32 %arg1, ptr addrspace(1) %ptr) #0 { +; CHECK-LABEL: test_rewrite_mfma_i32_16x16x16i8: +; CHECK: ; %bb.0: +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: global_load_dwordx4 a[0:3], v[2:3], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: v_mfma_i32_16x16x16i8 a[0:3], v0, v1, a[0:3] +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ; use a[0:3] +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: s_setpc_b64 s[30:31] + %src2 = load <4 x i32>, ptr addrspace(1) %ptr + %mai = call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x16i8(i32 %arg0, i32 %arg1, <4 x i32> %src2, i32 0, i32 0, i32 0) + call void asm sideeffect "; use $0", "a"(<4 x i32> %mai) + ret void +} + +define void @test_rewrite_mfma_f32_32x32x2bf16(<2 x i16> %arg0, <2 x i16> %arg1, ptr addrspace(1) %ptr) #0 { +; CHECK-LABEL: test_rewrite_mfma_f32_32x32x2bf16: +; CHECK: ; %bb.0: +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: global_load_dwordx4 a[28:31], v[2:3], off offset:112 +; CHECK-NEXT: global_load_dwordx4 a[24:27], v[2:3], off offset:96 +; CHECK-NEXT: global_load_dwordx4 a[20:23], v[2:3], off offset:80 +; CHECK-NEXT: global_load_dwordx4 a[16:19], v[2:3], off offset:64 +; CHECK-NEXT: global_load_dwordx4 a[12:15], v[2:3], off offset:48 +; CHECK-NEXT: global_load_dwordx4 a[8:11], v[2:3], off offset:32 +; CHECK-NEXT: global_load_dwordx4 a[4:7], v[2:3], off offset:16 +; CHECK-NEXT: global_load_dwordx4 a[0:3], v[2:3], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: v_mfma_f32_32x32x2bf16 a[0:31], v0, v1, a[0:31] +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ; use a[0:31] +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: s_setpc_b64 s[30:31] + %src2 = load <32 x float>, ptr addrspace(1) %ptr + %mai = call <32 x float> @llvm.amdgcn.mfma.f32.32x32x2bf16(<2 x i16> %arg0, <2 x i16> %arg1, <32 x float> %src2, i32 0, i32 0, i32 0) + call void asm sideeffect "; use $0", "a"(<32 x float> %mai) + ret void +} + +define void @test_rewrite_mfma_f32_16x16x2bf16(<2 x i16> %arg0, <2 x i16> %arg1, ptr addrspace(1) %ptr) #0 { +; CHECK-LABEL: test_rewrite_mfma_f32_16x16x2bf16: +; CHECK: ; %bb.0: +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: global_load_dwordx4 a[12:15], v[2:3], off offset:48 +; CHECK-NEXT: global_load_dwordx4 a[8:11], v[2:3], off offset:32 +; CHECK-NEXT: global_load_dwordx4 a[4:7], v[2:3], off offset:16 +; CHECK-NEXT: global_load_dwordx4 a[0:3], v[2:3], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: v_mfma_f32_16x16x2bf16 a[0:15], v0, v1, a[0:15] +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ; use a[0:15] +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: s_setpc_b64 s[30:31] + %src2 = load <16 x float>, ptr addrspace(1) %ptr + %mai = call <16 x float> @llvm.amdgcn.mfma.f32.16x16x2bf16(<2 x i16> %arg0, <2 x i16> %arg1, <16 x float> %src2, i32 0, i32 0, i32 0) + call void asm sideeffect "; use $0", "a"(<16 x float> %mai) + ret void +} + +define void @test_rewrite_mfma_f32_4x4x2bf16(<2 x i16> %arg0, <2 x i16> %arg1, ptr addrspace(1) %ptr) #0 { +; CHECK-LABEL: test_rewrite_mfma_f32_4x4x2bf16: +; CHECK: ; %bb.0: +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: global_load_dwordx4 a[0:3], v[2:3], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: v_mfma_f32_4x4x2bf16 a[0:3], v0, v1, a[0:3] +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ; use a[0:3] +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: s_setpc_b64 s[30:31] + %src2 = load <4 x float>, ptr addrspace(1) %ptr + %mai = call <4 x float> @llvm.amdgcn.mfma.f32.4x4x2bf16(<2 x i16> %arg0, <2 x i16> %arg1, <4 x float> %src2, i32 0, i32 0, i32 0) + call void asm sideeffect "; use $0", "a"(<4 x float> %mai) + ret void +} + +define void @test_rewrite_mfma_f32_32x32x4bf16(<2 x i16> %arg0, <2 x i16> %arg1, ptr addrspace(1) %ptr) #0 { +; CHECK-LABEL: test_rewrite_mfma_f32_32x32x4bf16: +; CHECK: ; %bb.0: +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: global_load_dwordx4 a[12:15], v[2:3], off offset:48 +; CHECK-NEXT: global_load_dwordx4 a[8:11], v[2:3], off offset:32 +; CHECK-NEXT: global_load_dwordx4 a[4:7], v[2:3], off offset:16 +; CHECK-NEXT: global_load_dwordx4 a[0:3], v[2:3], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: v_mfma_f32_32x32x4bf16 a[0:15], v0, v1, a[0:15] +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ; use a[0:15] +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: s_setpc_b64 s[30:31] + %src2 = load <16 x float>, ptr addrspace(1) %ptr + %mai = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x4bf16(<2 x i16> %arg0, <2 x i16> %arg1, <16 x float> %src2, i32 0, i32 0, i32 0) + call void asm sideeffect "; use $0", "a"(<16 x float> %mai) + ret void +} + +define void @test_rewrite_mfma_f32_16x16x8bf16(<2 x i16> %arg0, <2 x i16> %arg1, ptr addrspace(1) %ptr) #0 { +; CHECK-LABEL: test_rewrite_mfma_f32_16x16x8bf16: +; CHECK: ; %bb.0: +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: global_load_dwordx4 a[0:3], v[2:3], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: v_mfma_f32_16x16x8bf16 a[0:3], v0, v1, a[0:3] +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ; use a[0:3] +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: s_setpc_b64 s[30:31] + %src2 = load <4 x float>, ptr addrspace(1) %ptr + %mai = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x8bf16(<2 x i16> %arg0, <2 x i16> %arg1, <4 x float> %src2, i32 0, i32 0, i32 0) + call void asm sideeffect "; use $0", "a"(<4 x float> %mai) + ret void +} + +attributes #0 = { nounwind "amdgpu-flat-work-group-size"="1,256" "amdgpu-waves-per-eu"="4,4" } diff --git a/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.gfx950.ll b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.gfx950.ll new file mode 100644 index 0000000000000..b2465b02f2eee --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/rewrite-vgpr-mfma-to-agpr.gfx950.ll @@ -0,0 +1,664 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 +; RUN: llc -mcpu=gfx950 -amdgpu-mfma-vgpr-form < %s | FileCheck %s + +target triple = "amdgcn-amd-amdhsa" + +define void @test_rewrite_mfma_f32_16x16x32_f16(<8 x half> %arg0, <8 x half> %arg1, ptr addrspace(1) %ptr) #0 { +; CHECK-LABEL: test_rewrite_mfma_f32_16x16x32_f16: +; CHECK: ; %bb.0: +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: global_load_dwordx4 a[0:3], v[8:9], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: v_mfma_f32_16x16x32_f16 a[0:3], v[0:3], v[4:7], a[0:3] +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ; use a[0:3] +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: s_setpc_b64 s[30:31] + %src2 = load <4 x float>, ptr addrspace(1) %ptr + %mai = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.f16(<8 x half> %arg0, <8 x half> %arg1, <4 x float> %src2, i32 0, i32 0, i32 0) + call void asm sideeffect "; use $0", "a"(<4 x float> %mai) + ret void +} + +define void @test_rewrite_mfma_f32_32x32x16_f16(<8 x half> %arg0, <8 x half> %arg1, ptr addrspace(1) %ptr) #0 { +; CHECK-LABEL: test_rewrite_mfma_f32_32x32x16_f16: +; CHECK: ; %bb.0: +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: global_load_dwordx4 a[12:15], v[8:9], off offset:48 +; CHECK-NEXT: global_load_dwordx4 a[8:11], v[8:9], off offset:32 +; CHECK-NEXT: global_load_dwordx4 a[4:7], v[8:9], off offset:16 +; CHECK-NEXT: global_load_dwordx4 a[0:3], v[8:9], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: v_mfma_f32_32x32x16_f16 a[0:15], v[0:3], v[4:7], a[0:15] +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ; use a[0:15] +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: s_setpc_b64 s[30:31] + %src2 = load <16 x float>, ptr addrspace(1) %ptr + %mai = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.f16(<8 x half> %arg0, <8 x half> %arg1, <16 x float> %src2, i32 0, i32 0, i32 0) + call void asm sideeffect "; use $0", "a"(<16 x float> %mai) + ret void +} + +define void @test_rewrite_mfma_i32_16x16x64_i8(<4 x i32> %arg0, <4 x i32> %arg1, ptr addrspace(1) %ptr) #0 { +; CHECK-LABEL: test_rewrite_mfma_i32_16x16x64_i8: +; CHECK: ; %bb.0: +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: global_load_dwordx4 a[0:3], v[8:9], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: v_mfma_i32_16x16x64_i8 a[0:3], v[0:3], v[4:7], a[0:3] +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ; use a[0:3] +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: s_setpc_b64 s[30:31] + %src2 = load <4 x i32>, ptr addrspace(1) %ptr + %mai = call <4 x i32> @llvm.amdgcn.mfma.i32.16x16x64.i8(<4 x i32> %arg0, <4 x i32> %arg1, <4 x i32> %src2, i32 0, i32 0, i32 0) + call void asm sideeffect "; use $0", "a"(<4 x i32> %mai) + ret void +} + +define void @test_rewrite_mfma_i32_32x32x32_i8(<4 x i32> %arg0, <4 x i32> %arg1, ptr addrspace(1) %ptr) #0 { +; CHECK-LABEL: test_rewrite_mfma_i32_32x32x32_i8: +; CHECK: ; %bb.0: +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: global_load_dwordx4 a[12:15], v[8:9], off offset:48 +; CHECK-NEXT: global_load_dwordx4 a[8:11], v[8:9], off offset:32 +; CHECK-NEXT: global_load_dwordx4 a[4:7], v[8:9], off offset:16 +; CHECK-NEXT: global_load_dwordx4 a[0:3], v[8:9], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: v_mfma_i32_32x32x32_i8 a[0:15], v[0:3], v[4:7], a[0:15] +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ; use a[0:15] +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: s_setpc_b64 s[30:31] + %src2 = load <16 x i32>, ptr addrspace(1) %ptr + %mai = call <16 x i32> @llvm.amdgcn.mfma.i32.32x32x32.i8(<4 x i32> %arg0, <4 x i32> %arg1, <16 x i32> %src2, i32 0, i32 0, i32 0) + call void asm sideeffect "; use $0", "a"(<16 x i32> %mai) + ret void +} + +define void @test_rewrite_mfma_f32_16x16x32_bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, ptr addrspace(1) %ptr) #0 { +; CHECK-LABEL: test_rewrite_mfma_f32_16x16x32_bf16: +; CHECK: ; %bb.0: +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: global_load_dwordx4 a[0:3], v[8:9], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: v_mfma_f32_16x16x32_bf16 a[0:3], v[0:3], v[4:7], a[0:3] +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ; use a[0:3] +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: s_setpc_b64 s[30:31] + %src2 = load <4 x float>, ptr addrspace(1) %ptr + %mai = call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <4 x float> %src2, i32 0, i32 0, i32 0) + call void asm sideeffect "; use $0", "a"(<4 x float> %mai) + ret void +} + +define void @test_rewrite_mfma_f32_32x32x16_bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, ptr addrspace(1) %ptr) #0 { +; CHECK-LABEL: test_rewrite_mfma_f32_32x32x16_bf16: +; CHECK: ; %bb.0: +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: global_load_dwordx4 a[12:15], v[8:9], off offset:48 +; CHECK-NEXT: global_load_dwordx4 a[8:11], v[8:9], off offset:32 +; CHECK-NEXT: global_load_dwordx4 a[4:7], v[8:9], off offset:16 +; CHECK-NEXT: global_load_dwordx4 a[0:3], v[8:9], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: v_mfma_f32_32x32x16_bf16 a[0:15], v[0:3], v[4:7], a[0:15] +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ; use a[0:15] +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: s_setpc_b64 s[30:31] + %src2 = load <16 x float>, ptr addrspace(1) %ptr + %mai = call <16 x float> @llvm.amdgcn.mfma.f32.32x32x16.bf16(<8 x bfloat> %arg0, <8 x bfloat> %arg1, <16 x float> %src2, i32 0, i32 0, i32 0) + call void asm sideeffect "; use $0", "a"(<16 x float> %mai) + ret void +} + +; TODO: Full cross product of src0/src1 sizes not tested +define void @test_rewrite_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp0(<8 x i32> %arg0, <8 x i32> %arg1, i32 %scale0, i32 %scale1, ptr addrspace(1) %ptr) #0 { +; CHECK-LABEL: test_rewrite_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp0: +; CHECK: ; %bb.0: +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: global_load_dwordx4 a[0:3], v[18:19], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3], v16, v17 op_sel_hi:[0,0,0] +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ; use a[0:3] +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: s_setpc_b64 s[30:31] + %src2 = load <4 x float>, ptr addrspace(1) %ptr + %mai = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %src2, + i32 0, ; cbsz + i32 0, ; blgp + i32 0, i32 %scale0, i32 0, i32 %scale1) + call void asm sideeffect "; use $0", "a"(<4 x float> %mai) + ret void +} + +define void @test_rewrite_mfma_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp0(<8 x i32> %arg0, <8 x i32> %arg1, ptr addrspace(1) %ptr) #0 { +; CHECK-LABEL: test_rewrite_mfma_f32_16x16x128_f8f6f4_0_0__cbsz0__blgp0: +; CHECK: ; %bb.0: +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: global_load_dwordx4 a[0:3], v[16:17], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:7], v[8:15], a[0:3] +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ; use a[0:3] +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: s_setpc_b64 s[30:31] + %src2 = load <4 x float>, ptr addrspace(1) %ptr + %mai = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v8i32.v8i32(<8 x i32> %arg0, <8 x i32> %arg1, <4 x float> %src2, + i32 0, ; cbsz + i32 0, ; blgp + i32 0, i32 0, i32 0, i32 0) + call void asm sideeffect "; use $0", "a"(<4 x float> %mai) + ret void +} + +define void @test_rewrite_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp2(<6 x i32> %arg0, <6 x i32> %arg1, i32 %scale0, i32 %scale1, ptr addrspace(1) %ptr) #0 { +; CHECK-LABEL: test_rewrite_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp2: +; CHECK: ; %bb.0: +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: global_load_dwordx4 a[0:3], v[14:15], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:11], a[0:3], v12, v13 op_sel_hi:[0,0,0] cbsz:2 blgp:2 +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ; use a[0:3] +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: s_setpc_b64 s[30:31] + %src2 = load <4 x float>, ptr addrspace(1) %ptr + %mai = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v6i32(<6 x i32> %arg0, <6 x i32> %arg1, <4 x float> %src2, + i32 2, ; cbsz + i32 2, ; blgp + i32 0, i32 %scale0, i32 0, i32 %scale1) + call void asm sideeffect "; use $0", "a"(<4 x float> %mai) + ret void +} + +define void @test_rewrite_mfma_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp2(<6 x i32> %arg0, <6 x i32> %arg1, ptr addrspace(1) %ptr) #0 { +; CHECK-LABEL: test_rewrite_mfma_f32_16x16x128_f8f6f4_0_0__cbsz2__blgp2: +; CHECK: ; %bb.0: +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: global_load_dwordx4 a[0:3], v[12:13], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:5], v[6:11], a[0:3] cbsz:2 blgp:2 +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ; use a[0:3] +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: s_setpc_b64 s[30:31] + %src2 = load <4 x float>, ptr addrspace(1) %ptr + %mai = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v6i32.v6i32(<6 x i32> %arg0, <6 x i32> %arg1, <4 x float> %src2, + i32 2, ; cbsz + i32 2, ; blgp + i32 0, i32 0, i32 0, i32 0) + call void asm sideeffect "; use $0", "a"(<4 x float> %mai) + ret void +} + +define void @test_rewrite_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp4(<4 x i32> %arg0, <4 x i32> %arg1, i32 %scale0, i32 %scale1, ptr addrspace(1) %ptr) #0 { +; CHECK-LABEL: test_rewrite_mfma_scale_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp4: +; CHECK: ; %bb.0: +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: global_load_dwordx4 a[0:3], v[10:11], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 a[0:3], v[0:3], v[4:7], a[0:3], v8, v9 op_sel_hi:[0,0,0] cbsz:4 blgp:4 +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ; use a[0:3] +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: s_setpc_b64 s[30:31] + %src2 = load <4 x float>, ptr addrspace(1) %ptr + %mai = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v4i32.v4i32(<4 x i32> %arg0, <4 x i32> %arg1, <4 x float> %src2, + i32 4, ; cbsz + i32 4, ; blgp + i32 0, i32 %scale0, i32 0, i32 %scale1) + call void asm sideeffect "; use $0", "a"(<4 x float> %mai) + ret void +} + +define void @test_rewrite_mfma_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp4(<4 x i32> %arg0, <4 x i32> %arg1, ptr addrspace(1) %ptr) #0 { +; CHECK-LABEL: test_rewrite_mfma_f32_16x16x128_f8f6f4_0_0__cbsz4__blgp4: +; CHECK: ; %bb.0: +; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; CHECK-NEXT: global_load_dwordx4 a[0:3], v[8:9], off +; CHECK-NEXT: s_waitcnt vmcnt(0) +; CHECK-NEXT: v_mfma_f32_16x16x128_f8f6f4 a[0:3], v[0:3], v[4:7], a[0:3] cbsz:4 blgp:4 +; CHECK-NEXT: ;;#ASMSTART +; CHECK-NEXT: ; use a[0:3] +; CHECK-NEXT: ;;#ASMEND +; CHECK-NEXT: s_setpc_b64 s[30:31] + %src2 = load <4 x float>, ptr addrspace(1) %ptr + %mai = call <4 x float> @llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.v4i32.v4i32(<4 x i32> %arg0, <4 x i32> %arg1, <4 x float> %src2, + i32 4, ; cbsz + i32 4, ; blgp + ... [truncated] `````````` </details> https://github.com/llvm/llvm-project/pull/153026 _______________________________________________ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
