steffenlarsen updated this revision to Diff 336454.
steffenlarsen added a comment.
Following changes:
- Changed the type in the names of the intrinsics and builtins.
- Changed use of `IntrNoMem` to `IntrInaccessibleMemOnly`.
- Added `PTX70` as a requirement to the builtins.
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D100124/new/
https://reviews.llvm.org/D100124
Files:
clang/include/clang/Basic/BuiltinsNVPTX.def
clang/test/CodeGenCUDA/redux-builtins.cu
llvm/include/llvm/IR/IntrinsicsNVVM.td
llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
llvm/test/CodeGen/NVPTX/redux-sync.ll
Index: llvm/test/CodeGen/NVPTX/redux-sync.ll
===================================================================
--- /dev/null
+++ llvm/test/CodeGen/NVPTX/redux-sync.ll
@@ -0,0 +1,73 @@
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_80 -mattr=+ptx70 | FileCheck %s
+
+declare i32 @llvm.nvvm.redux.sync.add.u32(i32, i32)
+; CHECK-LABEL: .func{{.*}}redux_sync_add_u32
+define i32 @redux_sync_add_u32(i32 %src, i32 %mask) {
+ ; CHECK: redux.sync.add.u32
+ %val = call i32 @llvm.nvvm.redux.sync.add.u32(i32 %src, i32 %mask)
+ ret i32 %val
+}
+
+declare i32 @llvm.nvvm.redux.sync.min.u32(i32, i32)
+; CHECK-LABEL: .func{{.*}}redux_sync_min_u32
+define i32 @redux_sync_min_u32(i32 %src, i32 %mask) {
+ ; CHECK: redux.sync.min.u32
+ %val = call i32 @llvm.nvvm.redux.sync.min.u32(i32 %src, i32 %mask)
+ ret i32 %val
+}
+
+declare i32 @llvm.nvvm.redux.sync.max.u32(i32, i32)
+; CHECK-LABEL: .func{{.*}}redux_sync_max_u32
+define i32 @redux_sync_max_u32(i32 %src, i32 %mask) {
+ ; CHECK: redux.sync.max.u32
+ %val = call i32 @llvm.nvvm.redux.sync.max.u32(i32 %src, i32 %mask)
+ ret i32 %val
+}
+
+declare i32 @llvm.nvvm.redux.sync.add.s32(i32, i32)
+; CHECK-LABEL: .func{{.*}}redux_sync_add_s32
+define i32 @redux_sync_add_s32(i32 %src, i32 %mask) {
+ ; CHECK: redux.sync.add.s32
+ %val = call i32 @llvm.nvvm.redux.sync.add.s32(i32 %src, i32 %mask)
+ ret i32 %val
+}
+
+declare i32 @llvm.nvvm.redux.sync.min.s32(i32, i32)
+; CHECK-LABEL: .func{{.*}}redux_sync_min_s32
+define i32 @redux_sync_min_s32(i32 %src, i32 %mask) {
+ ; CHECK: redux.sync.min.s32
+ %val = call i32 @llvm.nvvm.redux.sync.min.s32(i32 %src, i32 %mask)
+ ret i32 %val
+}
+
+declare i32 @llvm.nvvm.redux.sync.max.s32(i32, i32)
+; CHECK-LABEL: .func{{.*}}redux_sync_max_s32
+define i32 @redux_sync_max_s32(i32 %src, i32 %mask) {
+ ; CHECK: redux.sync.max.s32
+ %val = call i32 @llvm.nvvm.redux.sync.max.s32(i32 %src, i32 %mask)
+ ret i32 %val
+}
+
+declare i32 @llvm.nvvm.redux.sync.and.b32(i32, i32)
+; CHECK-LABEL: .func{{.*}}redux_sync_and_b32
+define i32 @redux_sync_and_b32(i32 %src, i32 %mask) {
+ ; CHECK: redux.sync.and.b32
+ %val = call i32 @llvm.nvvm.redux.sync.and.b32(i32 %src, i32 %mask)
+ ret i32 %val
+}
+
+declare i32 @llvm.nvvm.redux.sync.xor.b32(i32, i32)
+; CHECK-LABEL: .func{{.*}}redux_sync_xor_b32
+define i32 @redux_sync_xor_b32(i32 %src, i32 %mask) {
+ ; CHECK: redux.sync.xor.b32
+ %val = call i32 @llvm.nvvm.redux.sync.xor.b32(i32 %src, i32 %mask)
+ ret i32 %val
+}
+
+declare i32 @llvm.nvvm.redux.sync.or.b32(i32, i32)
+; CHECK-LABEL: .func{{.*}}redux_sync_or_b32
+define i32 @redux_sync_or_b32(i32 %src, i32 %mask) {
+ ; CHECK: redux.sync.or.b32
+ %val = call i32 @llvm.nvvm.redux.sync.or.b32(i32 %src, i32 %mask)
+ ret i32 %val
+}
Index: llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
===================================================================
--- llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -274,6 +274,23 @@
defm MATCH_ALLP_SYNC_64 : MATCH_ALLP_SYNC<Int64Regs, "b64", int_nvvm_match_all_sync_i64p,
i64imm>;
+multiclass REDUX_SYNC<string BinOp, string PTXType, Intrinsic Intrin> {
+ def : NVPTXInst<(outs Int32Regs:$dst), (ins Int32Regs:$src, Int32Regs:$mask),
+ "redux.sync." # BinOp # "." # PTXType # " $dst, $src, $mask;",
+ [(set Int32Regs:$dst, (Intrin Int32Regs:$src, Int32Regs:$mask))]>,
+ Requires<[hasPTX70, hasSM80]>;
+}
+
+defm REDUX_SYNC_ADD_U32 : REDUX_SYNC<"add", "u32", int_nvvm_redux_sync_add_ui>;
+defm REDUX_SYNC_MIN_U32 : REDUX_SYNC<"min", "u32", int_nvvm_redux_sync_min_ui>;
+defm REDUX_SYNC_MAX_U32 : REDUX_SYNC<"max", "u32", int_nvvm_redux_sync_max_ui>;
+defm REDUX_SYNC_ADD_S32 : REDUX_SYNC<"add", "s32", int_nvvm_redux_sync_add_i>;
+defm REDUX_SYNC_MIN_S32 : REDUX_SYNC<"min", "s32", int_nvvm_redux_sync_min_i>;
+defm REDUX_SYNC_MAX_S32 : REDUX_SYNC<"max", "s32", int_nvvm_redux_sync_max_i>;
+defm REDUX_SYNC_AND_B32 : REDUX_SYNC<"and", "b32", int_nvvm_redux_sync_and_i>;
+defm REDUX_SYNC_XOR_B32 : REDUX_SYNC<"xor", "b32", int_nvvm_redux_sync_xor_i>;
+defm REDUX_SYNC_OR_B32 : REDUX_SYNC<"or", "b32", int_nvvm_redux_sync_or_i>;
+
} // isConvergent = true
//-----------------------------------
Index: llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
===================================================================
--- llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -144,11 +144,13 @@
def hasPTX61 : Predicate<"Subtarget->getPTXVersion() >= 61">;
def hasPTX63 : Predicate<"Subtarget->getPTXVersion() >= 63">;
def hasPTX64 : Predicate<"Subtarget->getPTXVersion() >= 64">;
+def hasPTX70 : Predicate<"Subtarget->getPTXVersion() >= 70">;
def hasSM30 : Predicate<"Subtarget->getSmVersion() >= 30">;
def hasSM70 : Predicate<"Subtarget->getSmVersion() >= 70">;
def hasSM72 : Predicate<"Subtarget->getSmVersion() >= 72">;
def hasSM75 : Predicate<"Subtarget->getSmVersion() >= 75">;
+def hasSM80 : Predicate<"Subtarget->getSmVersion() >= 80">;
// non-sync shfl instructions are not available on sm_70+ in PTX6.4+
def hasSHFL : Predicate<"!(Subtarget->getSmVersion() >= 70"
Index: llvm/include/llvm/IR/IntrinsicsNVVM.td
===================================================================
--- llvm/include/llvm/IR/IntrinsicsNVVM.td
+++ llvm/include/llvm/IR/IntrinsicsNVVM.td
@@ -4096,6 +4096,54 @@
Intrinsic<[llvm_i64_ty, llvm_i1_ty], [llvm_i32_ty, llvm_i64_ty],
[IntrInaccessibleMemOnly, IntrConvergent], "llvm.nvvm.match.all.sync.i64p">;
+//
+// REDUX.SYNC
+//
+// redux.sync.add.u32 dst, src, membermask;
+def int_nvvm_redux_sync_add_ui : GCCBuiltin<"__nvvm_redux_sync_add_ui">,
+ Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
+ [IntrConvergent, IntrInaccessibleMemOnly]>;
+
+// redux.sync.min.u32 dst, src, membermask;
+def int_nvvm_redux_sync_min_ui : GCCBuiltin<"__nvvm_redux_sync_min_ui">,
+ Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
+ [IntrConvergent, IntrInaccessibleMemOnly]>;
+
+// redux.sync.max.u32 dst, src, membermask;
+def int_nvvm_redux_sync_max_ui : GCCBuiltin<"__nvvm_redux_sync_max_ui">,
+ Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
+ [IntrConvergent, IntrInaccessibleMemOnly]>;
+
+// redux.sync.add.s32 dst, src, membermask;
+def int_nvvm_redux_sync_add_i : GCCBuiltin<"__nvvm_redux_sync_add_i">,
+ Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
+ [IntrConvergent, IntrInaccessibleMemOnly]>;
+
+// redux.sync.min.s32 dst, src, membermask;
+def int_nvvm_redux_sync_min_i : GCCBuiltin<"__nvvm_redux_sync_min_i">,
+ Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
+ [IntrConvergent, IntrInaccessibleMemOnly]>;
+
+// redux.sync.max.s32 dst, src, membermask;
+def int_nvvm_redux_sync_max_i : GCCBuiltin<"__nvvm_redux_sync_max_i">,
+ Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
+ [IntrConvergent, IntrInaccessibleMemOnly]>;
+
+// redux.sync.and.b32 dst, src, membermask;
+def int_nvvm_redux_sync_and_i : GCCBuiltin<"__nvvm_redux_sync_and_i">,
+ Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
+ [IntrConvergent, IntrInaccessibleMemOnly]>;
+
+// redux.sync.xor.b32 dst, src, membermask;
+def int_nvvm_redux_sync_xor_i : GCCBuiltin<"__nvvm_redux_sync_xor_i">,
+ Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
+ [IntrConvergent, IntrInaccessibleMemOnly]>;
+
+// redux.sync.or.b32 dst, src, membermask;
+def int_nvvm_redux_sync_or_i : GCCBuiltin<"__nvvm_redux_sync_or_i">,
+ Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
+ [IntrConvergent, IntrInaccessibleMemOnly]>;
+
//
// WMMA instructions
//
Index: clang/test/CodeGenCUDA/redux-builtins.cu
===================================================================
--- /dev/null
+++ clang/test/CodeGenCUDA/redux-builtins.cu
@@ -0,0 +1,47 @@
+// RUN: %clang_cc1 "-triple" "nvptx-nvidia-cuda" "-target-feature" "+ptx70" "-target-cpu" "sm_80" -emit-llvm -fcuda-is-device -o - %s | FileCheck %s
+// RUN: %clang_cc1 "-triple" "nvptx64-nvidia-cuda" "-target-feature" "+ptx70" "-target-cpu" "sm_80" -emit-llvm -fcuda-is-device -o - %s | FileCheck %s
+
+// CHECK: define{{.*}} void @_Z6kernelPi(i32* %out)
+__attribute__((global)) void kernel(int *out) {
+ int a = 1;
+ unsigned int b = 5;
+ int i = 0;
+
+ out[i++] = __nvvm_redux_sync_add_i(a, 0xFF);
+ // CHECK: call i32 @llvm.nvvm.redux.sync.add.i
+
+ out[i++] = __nvvm_redux_sync_min_i(a, 0x0F);
+ // CHECK: call i32 @llvm.nvvm.redux.sync.min.i
+
+ out[i++] = __nvvm_redux_sync_max_i(a, 0xF0);
+ // CHECK: call i32 @llvm.nvvm.redux.sync.max.i
+
+ out[i++] = __nvvm_redux_sync_add_ui(b, 0x01);
+ // CHECK: call i32 @llvm.nvvm.redux.sync.add.ui
+
+ out[i++] = __nvvm_redux_sync_min_ui(b, 0xF0);
+ // CHECK: call i32 @llvm.nvvm.redux.sync.min.ui
+
+ out[i++] = __nvvm_redux_sync_max_ui(b, 0x0F);
+ // CHECK: call i32 @llvm.nvvm.redux.sync.max.ui
+
+ out[i++] = __nvvm_redux_sync_and_i(a, 0xF0);
+ // CHECK: call i32 @llvm.nvvm.redux.sync.and.i
+
+ out[i++] = __nvvm_redux_sync_and_i(b, 0x0F);
+ // CHECK: call i32 @llvm.nvvm.redux.sync.and.i
+
+ out[i++] = __nvvm_redux_sync_xor_i(a, 0x10);
+ // CHECK: call i32 @llvm.nvvm.redux.sync.xor.i
+
+ out[i++] = __nvvm_redux_sync_xor_i(b, 0x01);
+ // CHECK: call i32 @llvm.nvvm.redux.sync.xor.i
+
+ out[i++] = __nvvm_redux_sync_or_i(a, 0xFF);
+ // CHECK: call i32 @llvm.nvvm.redux.sync.or.i
+
+ out[i++] = __nvvm_redux_sync_or_i(b, 0xFF);
+ // CHECK: call i32 @llvm.nvvm.redux.sync.or.i
+
+ // CHECK: ret void
+}
Index: clang/include/clang/Basic/BuiltinsNVPTX.def
===================================================================
--- clang/include/clang/Basic/BuiltinsNVPTX.def
+++ clang/include/clang/Basic/BuiltinsNVPTX.def
@@ -456,6 +456,17 @@
TARGET_BUILTIN(__nvvm_match_all_sync_i32p, "UiUiUii*", "", PTX60)
TARGET_BUILTIN(__nvvm_match_all_sync_i64p, "WiUiWii*", "", PTX60)
+// Redux
+TARGET_BUILTIN(__nvvm_redux_sync_add_i, "iii", "", AND(SM_80,PTX70))
+TARGET_BUILTIN(__nvvm_redux_sync_min_i, "iii", "", AND(SM_80,PTX70))
+TARGET_BUILTIN(__nvvm_redux_sync_max_i, "iii", "", AND(SM_80,PTX70))
+TARGET_BUILTIN(__nvvm_redux_sync_add_ui, "UiUii", "", AND(SM_80,PTX70))
+TARGET_BUILTIN(__nvvm_redux_sync_min_ui, "UiUii", "", AND(SM_80,PTX70))
+TARGET_BUILTIN(__nvvm_redux_sync_max_ui, "UiUii", "", AND(SM_80,PTX70))
+TARGET_BUILTIN(__nvvm_redux_sync_and_i, "iii", "", AND(SM_80,PTX70))
+TARGET_BUILTIN(__nvvm_redux_sync_xor_i, "iii", "", AND(SM_80,PTX70))
+TARGET_BUILTIN(__nvvm_redux_sync_or_i, "iii", "", AND(SM_80,PTX70))
+
// Membar
BUILTIN(__nvvm_membar_cta, "v", "")
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
