jhuber6 updated this revision to Diff 330352.
jhuber6 added a comment.
Herald added a subscriber: ormris.
Changed the RTL to have an argument that indicates if there is only one active
caller for a team. This makes it easier to optimize.
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D97680/new/
https://reviews.llvm.org/D97680
Files:
clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
clang/lib/CodeGen/CGOpenMPRuntimeGPU.h
llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
llvm/lib/Transforms/IPO/OpenMPOpt.cpp
llvm/test/Transforms/OpenMP/globalization_remarks.ll
openmp/libomptarget/deviceRTLs/common/src/data_sharing.cu
openmp/libomptarget/deviceRTLs/interface.h
Index: openmp/libomptarget/deviceRTLs/interface.h
===================================================================
--- openmp/libomptarget/deviceRTLs/interface.h
+++ openmp/libomptarget/deviceRTLs/interface.h
@@ -424,13 +424,8 @@
EXTERN bool __kmpc_kernel_parallel(void **WorkFn);
EXTERN void __kmpc_kernel_end_parallel();
-EXTERN void __kmpc_data_sharing_init_stack();
-EXTERN void __kmpc_data_sharing_init_stack_spmd();
-EXTERN void *__kmpc_data_sharing_coalesced_push_stack(size_t size,
- int16_t UseSharedMemory);
-EXTERN void *__kmpc_data_sharing_push_stack(size_t size,
- int16_t UseSharedMemory);
-EXTERN void __kmpc_data_sharing_pop_stack(void *a);
+EXTERN void *__kmpc_alloc_shared(size_t Size, int16_t OnePerTeam);
+EXTERN void __kmpc_free_shared(void *Data);
EXTERN void __kmpc_begin_sharing_variables(void ***GlobalArgs, size_t nArgs);
EXTERN void __kmpc_end_sharing_variables();
EXTERN void __kmpc_get_shared_variables(void ***GlobalArgs);
@@ -445,4 +440,11 @@
EXTERN void __kmpc_restore_team_static_memory(int16_t isSPMDExecutionMode,
int16_t is_shared);
+// Deprecated globalization interface
+EXTERN void *__kmpc_data_sharing_coalesced_push_stack(size_t size, int16_t s);
+EXTERN void *__kmpc_data_sharing_push_stack(size_t size, int16_t s);
+EXTERN void __kmpc_data_sharing_pop_stack(void *a);
+EXTERN void __kmpc_data_sharing_init_stack();
+EXTERN void __kmpc_data_sharing_init_stack_spmd();
+
#endif
Index: openmp/libomptarget/deviceRTLs/common/src/data_sharing.cu
===================================================================
--- openmp/libomptarget/deviceRTLs/common/src/data_sharing.cu
+++ openmp/libomptarget/deviceRTLs/common/src/data_sharing.cu
@@ -24,187 +24,16 @@
// Runtime functions for trunk data sharing scheme.
////////////////////////////////////////////////////////////////////////////////
-INLINE static void data_sharing_init_stack_common() {
- ASSERT0(LT_FUSSY, isRuntimeInitialized(), "Runtime must be initialized.");
- omptarget_nvptx_TeamDescr *teamDescr =
- &omptarget_nvptx_threadPrivateContext->TeamContext();
-
- for (int WID = 0; WID < DS_Max_Warp_Number; WID++) {
- __kmpc_data_sharing_slot *RootS = teamDescr->GetPreallocatedSlotAddr(WID);
- DataSharingState.SlotPtr[WID] = RootS;
- DataSharingState.StackPtr[WID] = (void *)&RootS->Data[0];
- }
+// Allocate memory that can be shared between the threads.
+// TODO: Add a small buffer of shared memory to allocate memory from
+// TODO: Add an INFO message to communicate with the user
+EXTERN void *__kmpc_alloc_shared(size_t DataSize, int16_t IsOnePerTeam) {
+ return (void *)SafeMalloc(DataSize, "Alloc Shared");
}
-// Initialize data sharing data structure. This function needs to be called
-// once at the beginning of a data sharing context (coincides with the kernel
-// initialization). This function is called only by the MASTER thread of each
-// team in non-SPMD mode.
-EXTERN void __kmpc_data_sharing_init_stack() {
- ASSERT0(LT_FUSSY, isRuntimeInitialized(), "Runtime must be initialized.");
- // This function initializes the stack pointer with the pointer to the
- // statically allocated shared memory slots. The size of a shared memory
- // slot is pre-determined to be 256 bytes.
- data_sharing_init_stack_common();
- omptarget_nvptx_globalArgs.Init();
-}
-
-// Initialize data sharing data structure. This function needs to be called
-// once at the beginning of a data sharing context (coincides with the kernel
-// initialization). This function is called in SPMD mode only.
-EXTERN void __kmpc_data_sharing_init_stack_spmd() {
- ASSERT0(LT_FUSSY, isRuntimeInitialized(), "Runtime must be initialized.");
- // This function initializes the stack pointer with the pointer to the
- // statically allocated shared memory slots. The size of a shared memory
- // slot is pre-determined to be 256 bytes.
- if (GetThreadIdInBlock() == 0)
- data_sharing_init_stack_common();
-
- __kmpc_impl_threadfence_block();
-}
-
-INLINE static void *data_sharing_push_stack_common(size_t PushSize) {
- ASSERT0(LT_FUSSY, isRuntimeInitialized(), "Expected initialized runtime.");
-
- // Only warp active master threads manage the stack.
- bool IsWarpMaster = (GetThreadIdInBlock() % WARPSIZE) == 0;
-
- // Add worst-case padding to DataSize so that future stack allocations are
- // correctly aligned.
- const size_t Alignment = 8;
- PushSize = (PushSize + (Alignment - 1)) / Alignment * Alignment;
-
- // Frame pointer must be visible to all workers in the same warp.
- const unsigned WID = GetWarpId();
- void *FrameP = 0;
- __kmpc_impl_lanemask_t CurActive = __kmpc_impl_activemask();
-
- if (IsWarpMaster) {
- // SlotP will point to either the shared memory slot or an existing
- // global memory slot.
- __kmpc_data_sharing_slot *&SlotP = DataSharingState.SlotPtr[WID];
- void *&StackP = DataSharingState.StackPtr[WID];
-
- // Check if we have room for the data in the current slot.
- const uintptr_t StartAddress = (uintptr_t)StackP;
- const uintptr_t EndAddress = (uintptr_t)SlotP->DataEnd;
- const uintptr_t RequestedEndAddress = StartAddress + (uintptr_t)PushSize;
-
- // If we requested more data than there is room for in the rest
- // of the slot then we need to either re-use the next slot, if one exists,
- // or create a new slot.
- if (EndAddress < RequestedEndAddress) {
- __kmpc_data_sharing_slot *NewSlot = 0;
- size_t NewSize = PushSize;
-
- // Allocate at least the default size for each type of slot.
- // Master is a special case and even though there is only one thread,
- // it can share more things with the workers. For uniformity, it uses
- // the full size of a worker warp slot.
- size_t DefaultSlotSize = DS_Worker_Warp_Slot_Size;
- if (DefaultSlotSize > NewSize)
- NewSize = DefaultSlotSize;
- NewSlot = (__kmpc_data_sharing_slot *)SafeMalloc(
- sizeof(__kmpc_data_sharing_slot) + NewSize,
- "Global memory slot allocation.");
-
- NewSlot->Next = 0;
- NewSlot->Prev = SlotP;
- NewSlot->PrevSlotStackPtr = StackP;
- NewSlot->DataEnd = &NewSlot->Data[0] + NewSize;
-
- // Make previous slot point to the newly allocated slot.
- SlotP->Next = NewSlot;
- // The current slot becomes the new slot.
- SlotP = NewSlot;
- // The stack pointer always points to the next free stack frame.
- StackP = &NewSlot->Data[0] + PushSize;
- // The frame pointer always points to the beginning of the frame.
- FrameP = DataSharingState.FramePtr[WID] = &NewSlot->Data[0];
- } else {
- // Add the data chunk to the current slot. The frame pointer is set to
- // point to the start of the new frame held in StackP.
- FrameP = DataSharingState.FramePtr[WID] = StackP;
- // Reset stack pointer to the requested address.
- StackP = (void *)RequestedEndAddress;
- }
- }
- // Get address from lane 0.
- int *FP = (int *)&FrameP;
- FP[0] = __kmpc_impl_shfl_sync(CurActive, FP[0], 0);
- if (sizeof(FrameP) == 8)
- FP[1] = __kmpc_impl_shfl_sync(CurActive, FP[1], 0);
-
- return FrameP;
-}
-
-EXTERN void *__kmpc_data_sharing_coalesced_push_stack(size_t DataSize,
- int16_t UseSharedMemory) {
- return data_sharing_push_stack_common(DataSize);
-}
-
-// Called at the time of the kernel initialization. This is used to initilize
-// the list of references to shared variables and to pre-allocate global storage
-// for holding the globalized variables.
-//
-// By default the globalized variables are stored in global memory. If the
-// UseSharedMemory is set to true, the runtime will attempt to use shared memory
-// as long as the size requested fits the pre-allocated size.
-EXTERN void *__kmpc_data_sharing_push_stack(size_t DataSize,
- int16_t UseSharedMemory) {
- // Compute the total memory footprint of the requested data.
- // The master thread requires a stack only for itself. A worker
- // thread (which at this point is a warp master) will require
- // space for the variables of each thread in the warp,
- // i.e. one DataSize chunk per warp lane.
- // TODO: change WARPSIZE to the number of active threads in the warp.
- size_t PushSize = (isRuntimeUninitialized() || IsMasterThread(isSPMDMode()))
- ? DataSize
- : WARPSIZE * DataSize;
-
- // Compute the start address of the frame of each thread in the warp.
- uintptr_t FrameStartAddress =
- (uintptr_t)data_sharing_push_stack_common(PushSize);
- FrameStartAddress += (uintptr_t)(GetLaneId() * DataSize);
- return (void *)FrameStartAddress;
-}
-
-// Pop the stack and free any memory which can be reclaimed.
-//
-// When the pop operation removes the last global memory slot,
-// reclaim all outstanding global memory slots since it is
-// likely we have reached the end of the kernel.
-EXTERN void __kmpc_data_sharing_pop_stack(void *FrameStart) {
- ASSERT0(LT_FUSSY, isRuntimeInitialized(), "Expected initialized runtime.");
-
- __kmpc_impl_threadfence_block();
-
- if (GetThreadIdInBlock() % WARPSIZE == 0) {
- unsigned WID = GetWarpId();
-
- // Current slot
- __kmpc_data_sharing_slot *&SlotP = DataSharingState.SlotPtr[WID];
-
- // Pointer to next available stack.
- void *&StackP = DataSharingState.StackPtr[WID];
-
- // Pop the frame.
- StackP = FrameStart;
-
- // If the current slot is empty, we need to free the slot after the
- // pop.
- bool SlotEmpty = (StackP == &SlotP->Data[0]);
-
- if (SlotEmpty && SlotP->Prev) {
- // Before removing the slot we need to reset StackP.
- StackP = SlotP->PrevSlotStackPtr;
-
- // Remove the slot.
- SlotP = SlotP->Prev;
- SafeFree(SlotP->Next, "Free slot.");
- SlotP->Next = 0;
- }
- }
+// Free the allocated memory.
+EXTERN void __kmpc_free_shared(void *FrameStart) {
+ SafeFree(FrameStart, "Free Shared");
}
// Begin a data sharing context. Maintain a list of references to shared
@@ -278,4 +107,21 @@
omptarget_nvptx_simpleMemoryManager.Release();
}
+// Deprecated globalization code
+EXTERN void __kmpc_data_sharing_init_stack() {}
+EXTERN void __kmpc_data_sharing_init_stack_spmd() {}
+
+EXTERN void *__kmpc_data_sharing_coalesced_push_stack(size_t DataSize,
+ int16_t) {
+ return (void *)SafeMalloc(DataSize, "Alloc Deprecated");
+}
+
+EXTERN void *__kmpc_data_sharing_push_stack(size_t DataSize, int16_t) {
+ return (void *)SafeMalloc(DataSize, "Alloc Deprecated");
+}
+
+EXTERN void __kmpc_data_sharing_pop_stack(void *FrameStart) {
+ SafeFree(FrameStart, "Free Shared");
+}
+
#pragma omp end declare target
Index: llvm/test/Transforms/OpenMP/globalization_remarks.ll
===================================================================
--- llvm/test/Transforms/OpenMP/globalization_remarks.ll
+++ llvm/test/Transforms/OpenMP/globalization_remarks.ll
@@ -1,145 +1,41 @@
+; RUN: opt -openmpopt -pass-remarks-analysis=openmp-opt -disable-output < %s 2>&1 | FileCheck %s
; RUN: opt -passes=openmpopt -pass-remarks-analysis=openmp-opt -disable-output < %s 2>&1 | FileCheck %s
-; ModuleID = 'declare_target_codegen_globalization.cpp'
-source_filename = "declare_target_codegen_globalization.cpp"
target datalayout = "e-i64:64-i128:128-v16:16-v32:32-n16:32:64"
-target triple = "nvptx64-nvidia-cuda"
+target triple = "nvptx64"
-%struct.ident_t = type { i32, i32, i32, i32, i8* }
-%struct._globalized_locals_ty = type { [32 x i32] }
+; CHECK: remark: globalization_remarks.c:5:7: Found thread data sharing on the GPU. Expect degraded performance due to data globalization.
-@0 = private unnamed_addr constant [56 x i8] c";declare_target_codegen_globalization.cpp;maini1;17;1;;\00", align 1
-@1 = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 1, i32 0, i8* getelementptr inbounds ([56 x i8], [56 x i8]* @0, i32 0, i32 0) }, align 8
-@__omp_offloading_801_3022563__Z6maini1v_l17_exec_mode = weak constant i8 0
-@llvm.compiler.used = appending global [1 x i8*] [i8* @__omp_offloading_801_3022563__Z6maini1v_l17_exec_mode], section "llvm.metadata"
+@S = external local_unnamed_addr global i8*
-; CHECK: remark: declare_target_codegen_globalization.cpp:17:1: Found thread data sharing on the GPU. Expect degraded performance due to data globalization.
-; CHECK: remark: declare_target_codegen_globalization.cpp:10:1: Found thread data sharing on the GPU. Expect degraded performance due to data globalization.
-
-; Function Attrs: norecurse nounwind
-define weak void @__omp_offloading_801_3022563__Z6maini1v_l17(i32* nonnull align 4 dereferenceable(4) %a) local_unnamed_addr #0 !dbg !10 {
+define void @foo() {
entry:
- %nvptx_num_threads = tail call i32 @llvm.nvvm.read.ptx.sreg.ntid.x(), !dbg !12, !range !13
- tail call void @__kmpc_spmd_kernel_init(i32 %nvptx_num_threads, i16 1, i16 0) #4, !dbg !12
- tail call void @__kmpc_data_sharing_init_stack_spmd() #4, !dbg !12
- %0 = tail call i32 @__kmpc_global_thread_num(%struct.ident_t* nonnull @1)
- %1 = tail call i8 @__kmpc_is_spmd_exec_mode() #4
- %.not.i.i = icmp eq i8 %1, 0
- br i1 %.not.i.i, label %.non-spmd2.i.i, label %__omp_outlined__.exit
-
-.non-spmd2.i.i: ; preds = %entry
- %2 = tail call i8* @__kmpc_data_sharing_coalesced_push_stack(i64 128, i16 0) #4, !dbg !12
- tail call void @__kmpc_data_sharing_pop_stack(i8* %2) #4, !dbg !14
- br label %__omp_outlined__.exit, !dbg !14
-
-__omp_outlined__.exit: ; preds = %entry, %.non-spmd2.i.i
- tail call void @__kmpc_spmd_kernel_deinit_v2(i16 1) #4, !dbg !19
- ret void, !dbg !20
+ %0 = call i8* @__kmpc_alloc_shared(i64 4, i16 0), !dbg !8
+ %x_on_stack = bitcast i8* %0 to i32*
+ %1 = bitcast i32* %x_on_stack to i8*
+ call void @share(i8* %1)
+ call void @__kmpc_free_shared(i8* %0)
+ ret void
}
-; Function Attrs: nounwind readnone
-declare i32 @llvm.nvvm.read.ptx.sreg.ntid.x() #1
-
-declare void @__kmpc_spmd_kernel_init(i32, i16, i16) local_unnamed_addr
-
-declare void @__kmpc_data_sharing_init_stack_spmd() local_unnamed_addr
-
-; Function Attrs: norecurse nounwind readonly
-define hidden i32 @_Z3fooRi(i32* nocapture nonnull readonly align 4 dereferenceable(4) %a) local_unnamed_addr #2 !dbg !21 {
+define void @share(i8* %x) {
entry:
- %0 = load i32, i32* %a, align 4, !dbg !22, !tbaa !23
- ret i32 %0, !dbg !27
+ store i8* %x, i8** @S
+ ret void
}
-; Function Attrs: nounwind
-define hidden i32 @_Z3barv() local_unnamed_addr #3 !dbg !15 {
-entry:
- %a1 = alloca i32, align 4
- %0 = tail call i8 @__kmpc_is_spmd_exec_mode() #4
- %.not = icmp eq i8 %0, 0
- br i1 %.not, label %.non-spmd, label %.exit
-
-.non-spmd: ; preds = %entry
- %1 = tail call i8* @__kmpc_data_sharing_push_stack(i64 128, i16 0) #4, !dbg !31
- %2 = bitcast i8* %1 to %struct._globalized_locals_ty*
- br label %.exit
+declare i8* @__kmpc_alloc_shared(i64, i16)
-.exit: ; preds = %entry, %.non-spmd
- %_select_stack = phi %struct._globalized_locals_ty* [ %2, %.non-spmd ], [ null, %entry ]
- %nvptx_tid = tail call i32 @llvm.nvvm.read.ptx.sreg.tid.x(), !range !28
- %nvptx_lane_id = and i32 %nvptx_tid, 31
- %3 = zext i32 %nvptx_lane_id to i64
- %4 = getelementptr inbounds %struct._globalized_locals_ty, %struct._globalized_locals_ty* %_select_stack, i64 0, i32 0, i64 %3
- %5 = select i1 %.not, i32* %4, i32* %a1
- %6 = load i32, i32* %5, align 4, !dbg !29, !tbaa !23
- br i1 %.not, label %.non-spmd2, label %.exit3, !dbg !31
+declare void @__kmpc_free_shared(i8*)
-.non-spmd2: ; preds = %.exit
- %7 = bitcast %struct._globalized_locals_ty* %_select_stack to i8*, !dbg !31
- tail call void @__kmpc_data_sharing_pop_stack(i8* %7) #4, !dbg !31
- br label %.exit3, !dbg !31
-
-.exit3: ; preds = %.non-spmd2, %.exit
- ret i32 %6, !dbg !31
-}
-
-declare i8 @__kmpc_is_spmd_exec_mode() local_unnamed_addr
-
-declare i8* @__kmpc_data_sharing_coalesced_push_stack(i64, i16) local_unnamed_addr
-
-declare i8* @__kmpc_data_sharing_push_stack(i64, i16) local_unnamed_addr
-
-; Function Attrs: nounwind readnone
-declare i32 @llvm.nvvm.read.ptx.sreg.tid.x() #1
-
-declare void @__kmpc_data_sharing_pop_stack(i8*) local_unnamed_addr
-
-; Function Attrs: nounwind
-declare i32 @__kmpc_global_thread_num(%struct.ident_t*) local_unnamed_addr #4
-
-declare void @__kmpc_spmd_kernel_deinit_v2(i16) local_unnamed_addr
-
-attributes #0 = { norecurse nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="sm_35" "target-features"="+ptx32,+sm_35" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #1 = { nounwind readnone }
-attributes #2 = { norecurse nounwind readonly "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="sm_35" "target-features"="+ptx32,+sm_35" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #3 = { nounwind "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="sm_35" "target-features"="+ptx32,+sm_35" "unsafe-fp-math"="false" "use-soft-float"="false" }
-attributes #4 = { nounwind }
!llvm.dbg.cu = !{!0}
-!omp_offload.info = !{!3}
-!nvvm.annotations = !{!4}
-!llvm.module.flags = !{!5, !6, !7, !8}
-!llvm.ident = !{!9}
+!llvm.module.flags = !{!3, !4}
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 12.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: DebugDirectivesOnly, enums: !2, splitDebugInlining: false, nameTableKind: None)
-!1 = !DIFile(filename: "declare_target_codegen_globalization.cpp", directory: "/home/jhuber/Documents/llvm-project/clang/test/OpenMP")
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 12.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !2, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "globalization_remarks.c", directory: "/tmp/globalization_remarks.c")
!2 = !{}
-!3 = !{i32 0, i32 2049, i32 50472291, !"_Z6maini1v", i32 17, i32 0}
-!4 = !{void (i32*)* @__omp_offloading_801_3022563__Z6maini1v_l17, !"kernel", i32 1}
-!5 = !{i32 7, !"Dwarf Version", i32 2}
-!6 = !{i32 2, !"Debug Info Version", i32 3}
-!7 = !{i32 1, !"wchar_size", i32 4}
-!8 = !{i32 7, !"PIC Level", i32 2}
-!9 = !{!"clang version 12.0.0"}
-!10 = distinct !DISubprogram(name: "__omp_offloading_801_3022563__Z6maini1v_l17", scope: !1, file: !1, line: 17, type: !11, scopeLine: 17, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
-!11 = !DISubroutineType(types: !2)
-!12 = !DILocation(line: 17, column: 1, scope: !10)
-!13 = !{i32 1, i32 1025}
-!14 = !DILocation(line: 10, column: 1, scope: !15, inlinedAt: !16)
-!15 = distinct !DISubprogram(name: "bar", scope: !1, file: !1, line: 7, type: !11, scopeLine: 7, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
-!16 = distinct !DILocation(line: 20, column: 18, scope: !17, inlinedAt: !18)
-!17 = distinct !DISubprogram(name: "__omp_outlined__", scope: !1, file: !1, line: 17, type: !11, scopeLine: 17, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
-!18 = distinct !DILocation(line: 17, column: 1, scope: !10)
-!19 = !DILocation(line: 17, column: 40, scope: !10)
-!20 = !DILocation(line: 21, column: 3, scope: !10)
-!21 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 5, type: !11, scopeLine: 5, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
-!22 = !DILocation(line: 5, column: 26, scope: !21)
-!23 = !{!24, !24, i64 0}
-!24 = !{!"int", !25, i64 0}
-!25 = !{!"omnipotent char", !26, i64 0}
-!26 = !{!"Simple C++ TBAA"}
-!27 = !DILocation(line: 5, column: 19, scope: !21)
-!28 = !{i32 0, i32 1024}
-!29 = !DILocation(line: 5, column: 26, scope: !21, inlinedAt: !30)
-!30 = distinct !DILocation(line: 9, column: 10, scope: !15)
-!31 = !DILocation(line: 10, column: 1, scope: !15)
-
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"wchar_size", i32 4}
+!6 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !7, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !2)
+!7 = !DISubroutineType(types: !2)
+!8 = !DILocation(line: 5, column: 7, scope: !6)
Index: llvm/lib/Transforms/IPO/OpenMPOpt.cpp
===================================================================
--- llvm/lib/Transforms/IPO/OpenMPOpt.cpp
+++ llvm/lib/Transforms/IPO/OpenMPOpt.cpp
@@ -1112,9 +1112,8 @@
}
void analysisGlobalization() {
- RuntimeFunction GlobalizationRuntimeIDs[] = {
- OMPRTL___kmpc_data_sharing_coalesced_push_stack,
- OMPRTL___kmpc_data_sharing_push_stack};
+ RuntimeFunction GlobalizationRuntimeIDs[] = {OMPRTL___kmpc_alloc_shared,
+ OMPRTL___kmpc_free_shared};
for (const auto GlobalizationCallID : GlobalizationRuntimeIDs) {
auto &RFI = OMPInfoCache.RFIs[GlobalizationCallID];
Index: llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
===================================================================
--- llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
+++ llvm/include/llvm/Frontend/OpenMP/OMPKinds.def
@@ -425,12 +425,9 @@
GlobalListPtr, GlobalListPtr, GlobalListPtr, GlobalListPtr)
__OMP_RTL(__kmpc_shuffle_int64, false, Int64, Int64, Int16, Int16)
-__OMP_RTL(__kmpc_data_sharing_init_stack, false, Void, )
-__OMP_RTL(__kmpc_data_sharing_init_stack_spmd, false, Void, )
-__OMP_RTL(__kmpc_data_sharing_coalesced_push_stack, false, VoidPtr, SizeTy, Int16)
-__OMP_RTL(__kmpc_data_sharing_push_stack, false, VoidPtr, SizeTy, Int16)
-__OMP_RTL(__kmpc_data_sharing_pop_stack, false, Void, VoidPtr)
+__OMP_RTL(__kmpc_alloc_shared, false, VoidPtr, SizeTy, Int16)
+__OMP_RTL(__kmpc_free_shared, false, Void, VoidPtr)
__OMP_RTL(__kmpc_begin_sharing_variables, false, Void, VoidPtrPtrPtr, SizeTy)
__OMP_RTL(__kmpc_end_sharing_variables, false, Void, )
__OMP_RTL(__kmpc_get_shared_variables, false, Void, VoidPtrPtrPtr)
Index: clang/lib/CodeGen/CGOpenMPRuntimeGPU.h
===================================================================
--- clang/lib/CodeGen/CGOpenMPRuntimeGPU.h
+++ clang/lib/CodeGen/CGOpenMPRuntimeGPU.h
@@ -440,15 +440,14 @@
/// The data for the single globalized variable.
struct MappedVarData {
/// Corresponding field in the global record.
- const FieldDecl *FD = nullptr;
+ llvm::Value *GlobalizedVal = nullptr;
/// Corresponding address.
Address PrivateAddr = Address::invalid();
/// true, if only one element is required (for latprivates in SPMD mode),
/// false, if need to create based on the warp-size.
bool IsOnePerTeam = false;
MappedVarData() = delete;
- MappedVarData(const FieldDecl *FD, bool IsOnePerTeam = false)
- : FD(FD), IsOnePerTeam(IsOnePerTeam) {}
+ MappedVarData(bool IsOnePerTeam = false) : IsOnePerTeam(IsOnePerTeam) {}
};
/// The map of local variables to their addresses in the global memory.
using DeclToAddrMapTy = llvm::MapVector<const Decl *, MappedVarData>;
@@ -460,29 +459,12 @@
EscapedParamsTy EscapedParameters;
llvm::SmallVector<const ValueDecl*, 4> EscapedVariableLengthDecls;
llvm::SmallVector<llvm::Value *, 4> EscapedVariableLengthDeclsAddrs;
- const RecordDecl *GlobalRecord = nullptr;
- llvm::Optional<const RecordDecl *> SecondaryGlobalRecord = llvm::None;
- llvm::Value *GlobalRecordAddr = nullptr;
llvm::Value *IsInSPMDModeFlag = nullptr;
std::unique_ptr<CodeGenFunction::OMPMapVars> MappedParams;
};
/// Maps the function to the list of the globalized variables with their
/// addresses.
llvm::SmallDenseMap<llvm::Function *, FunctionData> FunctionGlobalizedDecls;
- /// List of records for the globalized variables in target/teams/distribute
- /// contexts. Inner records are going to be joined into the single record,
- /// while those resulting records are going to be joined into the single
- /// union. This resulting union (one per CU) is the entry point for the static
- /// memory management runtime functions.
- struct GlobalPtrSizeRecsTy {
- llvm::GlobalVariable *UseSharedMemory = nullptr;
- llvm::GlobalVariable *RecSize = nullptr;
- llvm::GlobalVariable *Buffer = nullptr;
- SourceLocation Loc;
- llvm::SmallVector<const RecordDecl *, 2> Records;
- unsigned RegionCounter = 0;
- };
- llvm::SmallVector<GlobalPtrSizeRecsTy, 8> GlobalizedRecords;
llvm::GlobalVariable *KernelTeamsReductionPtr = nullptr;
/// List of the records with the list of fields for the reductions across the
/// teams. Used to build the intermediate buffer for the fast teams
Index: clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
===================================================================
--- clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
+++ clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp
@@ -1096,17 +1096,6 @@
} Action(EST, WST);
CodeGen.setAction(Action);
IsInTTDRegion = true;
- // Reserve place for the globalized memory.
- GlobalizedRecords.emplace_back();
- if (!KernelStaticGlobalized) {
- KernelStaticGlobalized = new llvm::GlobalVariable(
- CGM.getModule(), CGM.VoidPtrTy, /*isConstant=*/false,
- llvm::GlobalValue::InternalLinkage,
- llvm::UndefValue::get(CGM.VoidPtrTy),
- "_openmp_kernel_static_glob_rd$ptr", /*InsertBefore=*/nullptr,
- llvm::GlobalValue::NotThreadLocal,
- CGM.getContext().getTargetAddressSpace(LangAS::cuda_shared));
- }
emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,
IsOffloadEntry, CodeGen);
IsInTTDRegion = false;
@@ -1156,10 +1145,6 @@
CGM.getModule(), OMPRTL___kmpc_kernel_init),
Args);
- // For data sharing, we need to initialize the stack.
- CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
- CGM.getModule(), OMPRTL___kmpc_data_sharing_init_stack));
-
emitGenericVarsProlog(CGF, WST.Loc);
}
@@ -1228,17 +1213,6 @@
} Action(*this, EST, D);
CodeGen.setAction(Action);
IsInTTDRegion = true;
- // Reserve place for the globalized memory.
- GlobalizedRecords.emplace_back();
- if (!KernelStaticGlobalized) {
- KernelStaticGlobalized = new llvm::GlobalVariable(
- CGM.getModule(), CGM.VoidPtrTy, /*isConstant=*/false,
- llvm::GlobalValue::InternalLinkage,
- llvm::UndefValue::get(CGM.VoidPtrTy),
- "_openmp_kernel_static_glob_rd$ptr", /*InsertBefore=*/nullptr,
- llvm::GlobalValue::NotThreadLocal,
- CGM.getContext().getTargetAddressSpace(LangAS::cuda_shared));
- }
emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,
IsOffloadEntry, CodeGen);
IsInTTDRegion = false;
@@ -1260,12 +1234,6 @@
CGM.getModule(), OMPRTL___kmpc_spmd_kernel_init),
Args);
- if (RequiresFullRuntime) {
- // For data sharing, we need to initialize the stack.
- CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
- CGM.getModule(), OMPRTL___kmpc_data_sharing_init_stack_spmd));
- }
-
CGF.EmitBranch(ExecuteBB);
CGF.EmitBlock(ExecuteBB);
@@ -1671,7 +1639,6 @@
static_cast<CGOpenMPRuntimeGPU &>(CGF.CGM.getOpenMPRuntime());
if (GlobalizedRD) {
auto I = Rt.FunctionGlobalizedDecls.try_emplace(CGF.CurFn).first;
- I->getSecond().GlobalRecord = GlobalizedRD;
I->getSecond().MappedParams =
std::make_unique<CodeGenFunction::OMPMapVars>();
DeclToAddrMapTy &Data = I->getSecond().LocalVarData;
@@ -1679,8 +1646,7 @@
assert(Pair.getFirst()->isCanonicalDecl() &&
"Expected canonical declaration");
Data.insert(std::make_pair(Pair.getFirst(),
- MappedVarData(Pair.getSecond(),
- /*IsOnePerTeam=*/true)));
+ MappedVarData(/*IsOnePerTeam=*/true)));
}
}
Rt.emitGenericVarsProlog(CGF, Loc);
@@ -1709,282 +1675,71 @@
const auto I = FunctionGlobalizedDecls.find(CGF.CurFn);
if (I == FunctionGlobalizedDecls.end())
return;
- if (const RecordDecl *GlobalizedVarsRecord = I->getSecond().GlobalRecord) {
- QualType GlobalRecTy = CGM.getContext().getRecordType(GlobalizedVarsRecord);
- QualType SecGlobalRecTy;
- // Recover pointer to this function's global record. The runtime will
- // handle the specifics of the allocation of the memory.
- // Use actual memory size of the record including the padding
- // for alignment purposes.
- unsigned Alignment =
- CGM.getContext().getTypeAlignInChars(GlobalRecTy).getQuantity();
- unsigned GlobalRecordSize =
- CGM.getContext().getTypeSizeInChars(GlobalRecTy).getQuantity();
- GlobalRecordSize = llvm::alignTo(GlobalRecordSize, Alignment);
-
- llvm::PointerType *GlobalRecPtrTy =
- CGF.ConvertTypeForMem(GlobalRecTy)->getPointerTo();
- llvm::Value *GlobalRecCastAddr;
- llvm::Value *IsTTD = nullptr;
- if (!IsInTTDRegion &&
- (WithSPMDCheck ||
- getExecutionMode() == CGOpenMPRuntimeGPU::EM_Unknown)) {
- llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".exit");
- llvm::BasicBlock *SPMDBB = CGF.createBasicBlock(".spmd");
- llvm::BasicBlock *NonSPMDBB = CGF.createBasicBlock(".non-spmd");
- if (I->getSecond().SecondaryGlobalRecord.hasValue()) {
- llvm::Value *RTLoc = emitUpdateLocation(CGF, Loc);
- llvm::Value *ThreadID = getThreadID(CGF, Loc);
- llvm::Value *PL = CGF.EmitRuntimeCall(
- OMPBuilder.getOrCreateRuntimeFunction(CGM.getModule(),
- OMPRTL___kmpc_parallel_level),
- {RTLoc, ThreadID});
- IsTTD = Bld.CreateIsNull(PL);
- }
- llvm::Value *IsSPMD = Bld.CreateIsNotNull(
- CGF.EmitNounwindRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
- CGM.getModule(), OMPRTL___kmpc_is_spmd_exec_mode)));
- Bld.CreateCondBr(IsSPMD, SPMDBB, NonSPMDBB);
- // There is no need to emit line number for unconditional branch.
- (void)ApplyDebugLocation::CreateEmpty(CGF);
- CGF.EmitBlock(SPMDBB);
- Address RecPtr = Address(llvm::ConstantPointerNull::get(GlobalRecPtrTy),
- CharUnits::fromQuantity(Alignment));
- CGF.EmitBranch(ExitBB);
- // There is no need to emit line number for unconditional branch.
- (void)ApplyDebugLocation::CreateEmpty(CGF);
- CGF.EmitBlock(NonSPMDBB);
- llvm::Value *Size = llvm::ConstantInt::get(CGM.SizeTy, GlobalRecordSize);
- if (const RecordDecl *SecGlobalizedVarsRecord =
- I->getSecond().SecondaryGlobalRecord.getValueOr(nullptr)) {
- SecGlobalRecTy =
- CGM.getContext().getRecordType(SecGlobalizedVarsRecord);
-
- // Recover pointer to this function's global record. The runtime will
- // handle the specifics of the allocation of the memory.
- // Use actual memory size of the record including the padding
- // for alignment purposes.
- unsigned Alignment =
- CGM.getContext().getTypeAlignInChars(SecGlobalRecTy).getQuantity();
- unsigned GlobalRecordSize =
- CGM.getContext().getTypeSizeInChars(SecGlobalRecTy).getQuantity();
- GlobalRecordSize = llvm::alignTo(GlobalRecordSize, Alignment);
- Size = Bld.CreateSelect(
- IsTTD, llvm::ConstantInt::get(CGM.SizeTy, GlobalRecordSize), Size);
- }
- // TODO: allow the usage of shared memory to be controlled by
- // the user, for now, default to global.
- llvm::Value *GlobalRecordSizeArg[] = {
- Size, CGF.Builder.getInt16(/*UseSharedMemory=*/0)};
- llvm::Value *GlobalRecValue = CGF.EmitRuntimeCall(
- OMPBuilder.getOrCreateRuntimeFunction(
- CGM.getModule(), OMPRTL___kmpc_data_sharing_coalesced_push_stack),
- GlobalRecordSizeArg);
- GlobalRecCastAddr = Bld.CreatePointerBitCastOrAddrSpaceCast(
- GlobalRecValue, GlobalRecPtrTy);
- CGF.EmitBlock(ExitBB);
- auto *Phi = Bld.CreatePHI(GlobalRecPtrTy,
- /*NumReservedValues=*/2, "_select_stack");
- Phi->addIncoming(RecPtr.getPointer(), SPMDBB);
- Phi->addIncoming(GlobalRecCastAddr, NonSPMDBB);
- GlobalRecCastAddr = Phi;
- I->getSecond().GlobalRecordAddr = Phi;
- I->getSecond().IsInSPMDModeFlag = IsSPMD;
- } else if (!CGM.getLangOpts().OpenMPCUDATargetParallel && IsInTTDRegion) {
- assert(GlobalizedRecords.back().Records.size() < 2 &&
- "Expected less than 2 globalized records: one for target and one "
- "for teams.");
- unsigned Offset = 0;
- for (const RecordDecl *RD : GlobalizedRecords.back().Records) {
- QualType RDTy = CGM.getContext().getRecordType(RD);
- unsigned Alignment =
- CGM.getContext().getTypeAlignInChars(RDTy).getQuantity();
- unsigned Size = CGM.getContext().getTypeSizeInChars(RDTy).getQuantity();
- Offset =
- llvm::alignTo(llvm::alignTo(Offset, Alignment) + Size, Alignment);
- }
- unsigned Alignment =
- CGM.getContext().getTypeAlignInChars(GlobalRecTy).getQuantity();
- Offset = llvm::alignTo(Offset, Alignment);
- GlobalizedRecords.back().Records.push_back(GlobalizedVarsRecord);
- ++GlobalizedRecords.back().RegionCounter;
- if (GlobalizedRecords.back().Records.size() == 1) {
- assert(KernelStaticGlobalized &&
- "Kernel static pointer must be initialized already.");
- auto *UseSharedMemory = new llvm::GlobalVariable(
- CGM.getModule(), CGM.Int16Ty, /*isConstant=*/true,
- llvm::GlobalValue::InternalLinkage, nullptr,
- "_openmp_static_kernel$is_shared");
- UseSharedMemory->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
- QualType Int16Ty = CGM.getContext().getIntTypeForBitwidth(
- /*DestWidth=*/16, /*Signed=*/0);
- llvm::Value *IsInSharedMemory = CGF.EmitLoadOfScalar(
- Address(UseSharedMemory,
- CGM.getContext().getTypeAlignInChars(Int16Ty)),
- /*Volatile=*/false, Int16Ty, Loc);
- auto *StaticGlobalized = new llvm::GlobalVariable(
- CGM.getModule(), CGM.Int8Ty, /*isConstant=*/false,
- llvm::GlobalValue::CommonLinkage, nullptr);
- auto *RecSize = new llvm::GlobalVariable(
- CGM.getModule(), CGM.SizeTy, /*isConstant=*/true,
- llvm::GlobalValue::InternalLinkage, nullptr,
- "_openmp_static_kernel$size");
- RecSize->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
- llvm::Value *Ld = CGF.EmitLoadOfScalar(
- Address(RecSize, CGM.getSizeAlign()), /*Volatile=*/false,
- CGM.getContext().getSizeType(), Loc);
- llvm::Value *ResAddr = Bld.CreatePointerBitCastOrAddrSpaceCast(
- KernelStaticGlobalized, CGM.VoidPtrPtrTy);
- llvm::Value *GlobalRecordSizeArg[] = {
- llvm::ConstantInt::get(
- CGM.Int16Ty,
- getExecutionMode() == CGOpenMPRuntimeGPU::EM_SPMD ? 1 : 0),
- StaticGlobalized, Ld, IsInSharedMemory, ResAddr};
- CGF.EmitRuntimeCall(
- OMPBuilder.getOrCreateRuntimeFunction(
- CGM.getModule(), OMPRTL___kmpc_get_team_static_memory),
- GlobalRecordSizeArg);
- GlobalizedRecords.back().Buffer = StaticGlobalized;
- GlobalizedRecords.back().RecSize = RecSize;
- GlobalizedRecords.back().UseSharedMemory = UseSharedMemory;
- GlobalizedRecords.back().Loc = Loc;
- }
- assert(KernelStaticGlobalized && "Global address must be set already.");
- Address FrameAddr = CGF.EmitLoadOfPointer(
- Address(KernelStaticGlobalized, CGM.getPointerAlign()),
- CGM.getContext()
- .getPointerType(CGM.getContext().VoidPtrTy)
- .castAs<PointerType>());
- llvm::Value *GlobalRecValue =
- Bld.CreateConstInBoundsGEP(FrameAddr, Offset).getPointer();
- I->getSecond().GlobalRecordAddr = GlobalRecValue;
- I->getSecond().IsInSPMDModeFlag = nullptr;
- GlobalRecCastAddr = Bld.CreatePointerBitCastOrAddrSpaceCast(
- GlobalRecValue, CGF.ConvertTypeForMem(GlobalRecTy)->getPointerTo());
- } else {
- // TODO: allow the usage of shared memory to be controlled by
- // the user, for now, default to global.
- bool UseSharedMemory =
- IsInTTDRegion && GlobalRecordSize <= SharedMemorySize;
- llvm::Value *GlobalRecordSizeArg[] = {
- llvm::ConstantInt::get(CGM.SizeTy, GlobalRecordSize),
- CGF.Builder.getInt16(UseSharedMemory ? 1 : 0)};
- llvm::Value *GlobalRecValue = CGF.EmitRuntimeCall(
- OMPBuilder.getOrCreateRuntimeFunction(
- CGM.getModule(),
- IsInTTDRegion ? OMPRTL___kmpc_data_sharing_push_stack
- : OMPRTL___kmpc_data_sharing_coalesced_push_stack),
- GlobalRecordSizeArg);
- GlobalRecCastAddr = Bld.CreatePointerBitCastOrAddrSpaceCast(
- GlobalRecValue, GlobalRecPtrTy);
- I->getSecond().GlobalRecordAddr = GlobalRecValue;
- I->getSecond().IsInSPMDModeFlag = nullptr;
- }
- LValue Base =
- CGF.MakeNaturalAlignPointeeAddrLValue(GlobalRecCastAddr, GlobalRecTy);
-
- // Emit the "global alloca" which is a GEP from the global declaration
- // record using the pointer returned by the runtime.
- LValue SecBase;
- decltype(I->getSecond().LocalVarData)::const_iterator SecIt;
- if (IsTTD) {
- SecIt = I->getSecond().SecondaryLocalVarData->begin();
- llvm::PointerType *SecGlobalRecPtrTy =
- CGF.ConvertTypeForMem(SecGlobalRecTy)->getPointerTo();
- SecBase = CGF.MakeNaturalAlignPointeeAddrLValue(
- Bld.CreatePointerBitCastOrAddrSpaceCast(
- I->getSecond().GlobalRecordAddr, SecGlobalRecPtrTy),
- SecGlobalRecTy);
+ for (auto &Rec : I->getSecond().LocalVarData) {
+ const auto *VD = cast<VarDecl>(Rec.first);
+ bool EscapedParam = I->getSecond().EscapedParameters.count(Rec.first);
+ QualType VarTy = VD->getType();
+
+ // Get the local allocation of a firstprivate variable before sharing
+ llvm::Value *ParValue;
+ if (EscapedParam) {
+ LValue ParLVal =
+ CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
+ ParValue = CGF.EmitLoadOfScalar(ParLVal, Loc);
}
- for (auto &Rec : I->getSecond().LocalVarData) {
- bool EscapedParam = I->getSecond().EscapedParameters.count(Rec.first);
- llvm::Value *ParValue;
- if (EscapedParam) {
- const auto *VD = cast<VarDecl>(Rec.first);
- LValue ParLVal =
- CGF.MakeAddrLValue(CGF.GetAddrOfLocalVar(VD), VD->getType());
- ParValue = CGF.EmitLoadOfScalar(ParLVal, Loc);
- }
- LValue VarAddr = CGF.EmitLValueForField(Base, Rec.second.FD);
- // Emit VarAddr basing on lane-id if required.
- QualType VarTy;
- if (Rec.second.IsOnePerTeam) {
- VarTy = Rec.second.FD->getType();
- } else {
- Address Addr = VarAddr.getAddress(CGF);
- llvm::Value *Ptr = CGF.Builder.CreateInBoundsGEP(
- Addr.getElementType(), Addr.getPointer(),
- {Bld.getInt32(0), getNVPTXLaneID(CGF)});
- VarTy =
- Rec.second.FD->getType()->castAsArrayTypeUnsafe()->getElementType();
- VarAddr = CGF.MakeAddrLValue(
- Address(Ptr, CGM.getContext().getDeclAlign(Rec.first)), VarTy,
- AlignmentSource::Decl);
- }
- Rec.second.PrivateAddr = VarAddr.getAddress(CGF);
- if (!IsInTTDRegion &&
- (WithSPMDCheck ||
- getExecutionMode() == CGOpenMPRuntimeGPU::EM_Unknown)) {
- assert(I->getSecond().IsInSPMDModeFlag &&
- "Expected unknown execution mode or required SPMD check.");
- if (IsTTD) {
- assert(SecIt->second.IsOnePerTeam &&
- "Secondary glob data must be one per team.");
- LValue SecVarAddr = CGF.EmitLValueForField(SecBase, SecIt->second.FD);
- VarAddr.setAddress(
- Address(Bld.CreateSelect(IsTTD, SecVarAddr.getPointer(CGF),
- VarAddr.getPointer(CGF)),
- VarAddr.getAlignment()));
- Rec.second.PrivateAddr = VarAddr.getAddress(CGF);
- }
- Address GlobalPtr = Rec.second.PrivateAddr;
- Address LocalAddr = CGF.CreateMemTemp(VarTy, Rec.second.FD->getName());
- Rec.second.PrivateAddr = Address(
- Bld.CreateSelect(I->getSecond().IsInSPMDModeFlag,
- LocalAddr.getPointer(), GlobalPtr.getPointer()),
- LocalAddr.getAlignment());
- }
- if (EscapedParam) {
- const auto *VD = cast<VarDecl>(Rec.first);
- CGF.EmitStoreOfScalar(ParValue, VarAddr);
- I->getSecond().MappedParams->setVarAddr(CGF, VD,
- VarAddr.getAddress(CGF));
- }
- if (IsTTD)
- ++SecIt;
+
+ // Allocate space for the variable to be globalized
+ llvm::Value *AllocArgs[] = {CGF.getTypeSize(VD->getType()),
+ CGF.Builder.getInt16(Rec.second.IsOnePerTeam)};
+ llvm::Instruction *VoidPtr =
+ CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+ CGM.getModule(), OMPRTL___kmpc_alloc_shared),
+ AllocArgs, VD->getName());
+
+ // Cast the void pointer and get the address of the globalized variable.
+ llvm::PointerType *VarPtrTy = CGF.ConvertTypeForMem(VarTy)->getPointerTo();
+ llvm::Value *CastedVoidPtr = Bld.CreatePointerBitCastOrAddrSpaceCast(
+ VoidPtr, VarPtrTy, VD->getName() + "_on_stack");
+ LValue VarAddr = CGF.MakeNaturalAlignAddrLValue(CastedVoidPtr, VarTy);
+ Rec.second.PrivateAddr = VarAddr.getAddress(CGF);
+ Rec.second.GlobalizedVal = VoidPtr;
+
+ // Assign the local allocation to the newly globalized location.
+ if (EscapedParam) {
+ CGF.EmitStoreOfScalar(ParValue, VarAddr);
+ I->getSecond().MappedParams->setVarAddr(CGF, VD, VarAddr.getAddress(CGF));
}
+ if (auto *DI = CGF.getDebugInfo())
+ VoidPtr->setDebugLoc(DI->SourceLocToDebugLoc(VD->getLocation()));
}
- for (const ValueDecl *VD : I->getSecond().EscapedVariableLengthDecls) {
- // Recover pointer to this function's global record. The runtime will
- // handle the specifics of the allocation of the memory.
- // Use actual memory size of the record including the padding
+ for (const auto *VD : I->getSecond().EscapedVariableLengthDecls) {
+ // Use actual memory size of the VLA object including the padding
// for alignment purposes.
- CGBuilderTy &Bld = CGF.Builder;
llvm::Value *Size = CGF.getTypeSize(VD->getType());
CharUnits Align = CGM.getContext().getDeclAlign(VD);
Size = Bld.CreateNUWAdd(
Size, llvm::ConstantInt::get(CGF.SizeTy, Align.getQuantity() - 1));
llvm::Value *AlignVal =
llvm::ConstantInt::get(CGF.SizeTy, Align.getQuantity());
+
Size = Bld.CreateUDiv(Size, AlignVal);
Size = Bld.CreateNUWMul(Size, AlignVal);
- // TODO: allow the usage of shared memory to be controlled by
- // the user, for now, default to global.
- llvm::Value *GlobalRecordSizeArg[] = {
- Size, CGF.Builder.getInt16(/*UseSharedMemory=*/0)};
- llvm::Value *GlobalRecValue = CGF.EmitRuntimeCall(
- OMPBuilder.getOrCreateRuntimeFunction(
- CGM.getModule(), OMPRTL___kmpc_data_sharing_coalesced_push_stack),
- GlobalRecordSizeArg);
- llvm::Value *GlobalRecCastAddr = Bld.CreatePointerBitCastOrAddrSpaceCast(
- GlobalRecValue, CGF.ConvertTypeForMem(VD->getType())->getPointerTo());
- LValue Base = CGF.MakeAddrLValue(GlobalRecCastAddr, VD->getType(),
+
+ // Allocate space for this VLA object to be globalized.
+ llvm::Value *AllocArgs[] = {CGF.getTypeSize(VD->getType()),
+ CGF.Builder.getInt16(/*IsOnePerTeam=*/0)};
+ llvm::Instruction *VoidPtr =
+ CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+ CGM.getModule(), OMPRTL___kmpc_alloc_shared),
+ AllocArgs, VD->getName());
+
+ I->getSecond().EscapedVariableLengthDeclsAddrs.emplace_back(VoidPtr);
+ LValue Base = CGF.MakeAddrLValue(VoidPtr, VD->getType(),
CGM.getContext().getDeclAlign(VD),
AlignmentSource::Decl);
I->getSecond().MappedParams->setVarAddr(CGF, cast<VarDecl>(VD),
Base.getAddress(CGF));
- I->getSecond().EscapedVariableLengthDeclsAddrs.emplace_back(GlobalRecValue);
}
I->getSecond().MappedParams->apply(CGF);
}
@@ -1997,60 +1752,20 @@
const auto I = FunctionGlobalizedDecls.find(CGF.CurFn);
if (I != FunctionGlobalizedDecls.end()) {
- I->getSecond().MappedParams->restore(CGF);
- if (!CGF.HaveInsertPoint())
- return;
+ // Deallocate the memory for each globalized VLA object
for (llvm::Value *Addr :
llvm::reverse(I->getSecond().EscapedVariableLengthDeclsAddrs)) {
- CGF.EmitRuntimeCall(
- OMPBuilder.getOrCreateRuntimeFunction(
- CGM.getModule(), OMPRTL___kmpc_data_sharing_pop_stack),
- Addr);
+ CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+ CGM.getModule(), OMPRTL___kmpc_free_shared),
+ Addr);
}
- if (I->getSecond().GlobalRecordAddr) {
- if (!IsInTTDRegion &&
- (WithSPMDCheck ||
- getExecutionMode() == CGOpenMPRuntimeGPU::EM_Unknown)) {
- CGBuilderTy &Bld = CGF.Builder;
- llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".exit");
- llvm::BasicBlock *NonSPMDBB = CGF.createBasicBlock(".non-spmd");
- Bld.CreateCondBr(I->getSecond().IsInSPMDModeFlag, ExitBB, NonSPMDBB);
- // There is no need to emit line number for unconditional branch.
- (void)ApplyDebugLocation::CreateEmpty(CGF);
- CGF.EmitBlock(NonSPMDBB);
- CGF.EmitRuntimeCall(
- OMPBuilder.getOrCreateRuntimeFunction(
- CGM.getModule(), OMPRTL___kmpc_data_sharing_pop_stack),
- CGF.EmitCastToVoidPtr(I->getSecond().GlobalRecordAddr));
- CGF.EmitBlock(ExitBB);
- } else if (!CGM.getLangOpts().OpenMPCUDATargetParallel && IsInTTDRegion) {
- assert(GlobalizedRecords.back().RegionCounter > 0 &&
- "region counter must be > 0.");
- --GlobalizedRecords.back().RegionCounter;
- // Emit the restore function only in the target region.
- if (GlobalizedRecords.back().RegionCounter == 0) {
- QualType Int16Ty = CGM.getContext().getIntTypeForBitwidth(
- /*DestWidth=*/16, /*Signed=*/0);
- llvm::Value *IsInSharedMemory = CGF.EmitLoadOfScalar(
- Address(GlobalizedRecords.back().UseSharedMemory,
- CGM.getContext().getTypeAlignInChars(Int16Ty)),
- /*Volatile=*/false, Int16Ty, GlobalizedRecords.back().Loc);
- llvm::Value *Args[] = {
- llvm::ConstantInt::get(
- CGM.Int16Ty,
- getExecutionMode() == CGOpenMPRuntimeGPU::EM_SPMD ? 1 : 0),
- IsInSharedMemory};
- CGF.EmitRuntimeCall(
- OMPBuilder.getOrCreateRuntimeFunction(
- CGM.getModule(), OMPRTL___kmpc_restore_team_static_memory),
- Args);
- }
- } else {
- CGF.EmitRuntimeCall(
- OMPBuilder.getOrCreateRuntimeFunction(
- CGM.getModule(), OMPRTL___kmpc_data_sharing_pop_stack),
- I->getSecond().GlobalRecordAddr);
- }
+ // Deallocate the memory for each globalized value
+ for (auto &Rec : llvm::reverse(I->getSecond().LocalVarData)) {
+ I->getSecond().MappedParams->restore(CGF);
+
+ CGF.EmitRuntimeCall(OMPBuilder.getOrCreateRuntimeFunction(
+ CGM.getModule(), OMPRTL___kmpc_free_shared),
+ {Rec.second.GlobalizedVal});
}
}
}
@@ -4333,6 +4048,7 @@
}
if (!Body)
return;
+
CheckVarsEscapingDeclContext VarChecker(CGF, TeamAndReductions.second);
VarChecker.Visit(Body);
const RecordDecl *GlobalizedVarsRecord =
@@ -4346,7 +4062,6 @@
auto I = FunctionGlobalizedDecls.try_emplace(CGF.CurFn).first;
I->getSecond().MappedParams =
std::make_unique<CodeGenFunction::OMPMapVars>();
- I->getSecond().GlobalRecord = GlobalizedVarsRecord;
I->getSecond().EscapedParameters.insert(
VarChecker.getEscapedParameters().begin(),
VarChecker.getEscapedParameters().end());
@@ -4355,21 +4070,16 @@
DeclToAddrMapTy &Data = I->getSecond().LocalVarData;
for (const ValueDecl *VD : VarChecker.getEscapedDecls()) {
assert(VD->isCanonicalDecl() && "Expected canonical declaration");
- const FieldDecl *FD = VarChecker.getFieldForGlobalizedVar(VD);
- Data.insert(std::make_pair(VD, MappedVarData(FD, IsInTTDRegion)));
+ Data.insert(std::make_pair(VD, MappedVarData(IsInTTDRegion)));
}
if (!IsInTTDRegion && !NeedToDelayGlobalization && !IsInParallelRegion) {
CheckVarsEscapingDeclContext VarChecker(CGF, llvm::None);
VarChecker.Visit(Body);
- I->getSecond().SecondaryGlobalRecord =
- VarChecker.getGlobalizedRecord(/*IsInTTDRegion=*/true);
I->getSecond().SecondaryLocalVarData.emplace();
DeclToAddrMapTy &Data = I->getSecond().SecondaryLocalVarData.getValue();
for (const ValueDecl *VD : VarChecker.getEscapedDecls()) {
assert(VD->isCanonicalDecl() && "Expected canonical declaration");
- const FieldDecl *FD = VarChecker.getFieldForGlobalizedVar(VD);
- Data.insert(
- std::make_pair(VD, MappedVarData(FD, /*IsInTTDRegion=*/true)));
+ Data.insert(std::make_pair(VD, MappedVarData(/*IsInTTDRegion=*/true)));
}
}
if (!NeedToDelayGlobalization) {
@@ -4660,185 +4370,8 @@
CGOpenMPRuntime::processRequiresDirective(D);
}
-/// Get number of SMs and number of blocks per SM.
-static std::pair<unsigned, unsigned> getSMsBlocksPerSM(CodeGenModule &CGM) {
- std::pair<unsigned, unsigned> Data;
- if (CGM.getLangOpts().OpenMPCUDANumSMs)
- Data.first = CGM.getLangOpts().OpenMPCUDANumSMs;
- if (CGM.getLangOpts().OpenMPCUDABlocksPerSM)
- Data.second = CGM.getLangOpts().OpenMPCUDABlocksPerSM;
- if (Data.first && Data.second)
- return Data;
- switch (getCudaArch(CGM)) {
- case CudaArch::SM_20:
- case CudaArch::SM_21:
- case CudaArch::SM_30:
- case CudaArch::SM_32:
- case CudaArch::SM_35:
- case CudaArch::SM_37:
- case CudaArch::SM_50:
- case CudaArch::SM_52:
- case CudaArch::SM_53:
- return {16, 16};
- case CudaArch::SM_60:
- case CudaArch::SM_61:
- case CudaArch::SM_62:
- return {56, 32};
- case CudaArch::SM_70:
- case CudaArch::SM_72:
- case CudaArch::SM_75:
- case CudaArch::SM_80:
- case CudaArch::SM_86:
- return {84, 32};
- case CudaArch::GFX600:
- case CudaArch::GFX601:
- case CudaArch::GFX602:
- case CudaArch::GFX700:
- case CudaArch::GFX701:
- case CudaArch::GFX702:
- case CudaArch::GFX703:
- case CudaArch::GFX704:
- case CudaArch::GFX705:
- case CudaArch::GFX801:
- case CudaArch::GFX802:
- case CudaArch::GFX803:
- case CudaArch::GFX805:
- case CudaArch::GFX810:
- case CudaArch::GFX900:
- case CudaArch::GFX902:
- case CudaArch::GFX904:
- case CudaArch::GFX906:
- case CudaArch::GFX908:
- case CudaArch::GFX909:
- case CudaArch::GFX90a:
- case CudaArch::GFX90c:
- case CudaArch::GFX1010:
- case CudaArch::GFX1011:
- case CudaArch::GFX1012:
- case CudaArch::GFX1030:
- case CudaArch::GFX1031:
- case CudaArch::GFX1032:
- case CudaArch::GFX1033:
- case CudaArch::UNUSED:
- case CudaArch::UNKNOWN:
- break;
- case CudaArch::LAST:
- llvm_unreachable("Unexpected Cuda arch.");
- }
- llvm_unreachable("Unexpected NVPTX target without ptx feature.");
-}
-
void CGOpenMPRuntimeGPU::clear() {
- if (!GlobalizedRecords.empty() &&
- !CGM.getLangOpts().OpenMPCUDATargetParallel) {
- ASTContext &C = CGM.getContext();
- llvm::SmallVector<const GlobalPtrSizeRecsTy *, 4> GlobalRecs;
- llvm::SmallVector<const GlobalPtrSizeRecsTy *, 4> SharedRecs;
- RecordDecl *StaticRD = C.buildImplicitRecord(
- "_openmp_static_memory_type_$_", RecordDecl::TagKind::TTK_Union);
- StaticRD->startDefinition();
- RecordDecl *SharedStaticRD = C.buildImplicitRecord(
- "_shared_openmp_static_memory_type_$_", RecordDecl::TagKind::TTK_Union);
- SharedStaticRD->startDefinition();
- for (const GlobalPtrSizeRecsTy &Records : GlobalizedRecords) {
- if (Records.Records.empty())
- continue;
- unsigned Size = 0;
- unsigned RecAlignment = 0;
- for (const RecordDecl *RD : Records.Records) {
- QualType RDTy = C.getRecordType(RD);
- unsigned Alignment = C.getTypeAlignInChars(RDTy).getQuantity();
- RecAlignment = std::max(RecAlignment, Alignment);
- unsigned RecSize = C.getTypeSizeInChars(RDTy).getQuantity();
- Size =
- llvm::alignTo(llvm::alignTo(Size, Alignment) + RecSize, Alignment);
- }
- Size = llvm::alignTo(Size, RecAlignment);
- llvm::APInt ArySize(/*numBits=*/64, Size);
- QualType SubTy = C.getConstantArrayType(
- C.CharTy, ArySize, nullptr, ArrayType::Normal, /*IndexTypeQuals=*/0);
- const bool UseSharedMemory = Size <= SharedMemorySize;
- auto *Field =
- FieldDecl::Create(C, UseSharedMemory ? SharedStaticRD : StaticRD,
- SourceLocation(), SourceLocation(), nullptr, SubTy,
- C.getTrivialTypeSourceInfo(SubTy, SourceLocation()),
- /*BW=*/nullptr, /*Mutable=*/false,
- /*InitStyle=*/ICIS_NoInit);
- Field->setAccess(AS_public);
- if (UseSharedMemory) {
- SharedStaticRD->addDecl(Field);
- SharedRecs.push_back(&Records);
- } else {
- StaticRD->addDecl(Field);
- GlobalRecs.push_back(&Records);
- }
- Records.RecSize->setInitializer(llvm::ConstantInt::get(CGM.SizeTy, Size));
- Records.UseSharedMemory->setInitializer(
- llvm::ConstantInt::get(CGM.Int16Ty, UseSharedMemory ? 1 : 0));
- }
- // Allocate SharedMemorySize buffer for the shared memory.
- // FIXME: nvlink does not handle weak linkage correctly (object with the
- // different size are reported as erroneous).
- // Restore this code as sson as nvlink is fixed.
- if (!SharedStaticRD->field_empty()) {
- llvm::APInt ArySize(/*numBits=*/64, SharedMemorySize);
- QualType SubTy = C.getConstantArrayType(
- C.CharTy, ArySize, nullptr, ArrayType::Normal, /*IndexTypeQuals=*/0);
- auto *Field = FieldDecl::Create(
- C, SharedStaticRD, SourceLocation(), SourceLocation(), nullptr, SubTy,
- C.getTrivialTypeSourceInfo(SubTy, SourceLocation()),
- /*BW=*/nullptr, /*Mutable=*/false,
- /*InitStyle=*/ICIS_NoInit);
- Field->setAccess(AS_public);
- SharedStaticRD->addDecl(Field);
- }
- SharedStaticRD->completeDefinition();
- if (!SharedStaticRD->field_empty()) {
- QualType StaticTy = C.getRecordType(SharedStaticRD);
- llvm::Type *LLVMStaticTy = CGM.getTypes().ConvertTypeForMem(StaticTy);
- auto *GV = new llvm::GlobalVariable(
- CGM.getModule(), LLVMStaticTy,
- /*isConstant=*/false, llvm::GlobalValue::WeakAnyLinkage,
- llvm::UndefValue::get(LLVMStaticTy),
- "_openmp_shared_static_glob_rd_$_", /*InsertBefore=*/nullptr,
- llvm::GlobalValue::NotThreadLocal,
- C.getTargetAddressSpace(LangAS::cuda_shared));
- auto *Replacement = llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(
- GV, CGM.VoidPtrTy);
- for (const GlobalPtrSizeRecsTy *Rec : SharedRecs) {
- Rec->Buffer->replaceAllUsesWith(Replacement);
- Rec->Buffer->eraseFromParent();
- }
- }
- StaticRD->completeDefinition();
- if (!StaticRD->field_empty()) {
- QualType StaticTy = C.getRecordType(StaticRD);
- std::pair<unsigned, unsigned> SMsBlockPerSM = getSMsBlocksPerSM(CGM);
- llvm::APInt Size1(32, SMsBlockPerSM.second);
- QualType Arr1Ty =
- C.getConstantArrayType(StaticTy, Size1, nullptr, ArrayType::Normal,
- /*IndexTypeQuals=*/0);
- llvm::APInt Size2(32, SMsBlockPerSM.first);
- QualType Arr2Ty =
- C.getConstantArrayType(Arr1Ty, Size2, nullptr, ArrayType::Normal,
- /*IndexTypeQuals=*/0);
- llvm::Type *LLVMArr2Ty = CGM.getTypes().ConvertTypeForMem(Arr2Ty);
- // FIXME: nvlink does not handle weak linkage correctly (object with the
- // different size are reported as erroneous).
- // Restore CommonLinkage as soon as nvlink is fixed.
- auto *GV = new llvm::GlobalVariable(
- CGM.getModule(), LLVMArr2Ty,
- /*isConstant=*/false, llvm::GlobalValue::InternalLinkage,
- llvm::Constant::getNullValue(LLVMArr2Ty),
- "_openmp_static_glob_rd_$_");
- auto *Replacement = llvm::ConstantExpr::getPointerBitCastOrAddrSpaceCast(
- GV, CGM.VoidPtrTy);
- for (const GlobalPtrSizeRecsTy *Rec : GlobalRecs) {
- Rec->Buffer->replaceAllUsesWith(Replacement);
- Rec->Buffer->eraseFromParent();
- }
- }
- }
+
if (!TeamsReductions.empty()) {
ASTContext &C = CGM.getContext();
RecordDecl *StaticRD = C.buildImplicitRecord(
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
