grokos created this revision.
grokos added a reviewer: ABataev.
grokos added projects: clang, OpenMP.
Herald added a subscriber: guansong.
This patch implements CodeGen support for the "declare target" directive.
Code is generated for variables, functions and ctors/dtors.
I understand that the patch as a whole is somewhat large; if this is the case
and it cannot land in one go then let's discuss how it can be split. Due to
this uncertainty I haven't included any regression tests, I'll upload them once
the scope of each patch has been determined.
Repository:
rC Clang
https://reviews.llvm.org/D43026
Files:
lib/CodeGen/CGCXXABI.h
lib/CodeGen/CGDeclCXX.cpp
lib/CodeGen/CGOpenMPRuntime.cpp
lib/CodeGen/CGOpenMPRuntime.h
lib/CodeGen/CodeGenFunction.h
lib/CodeGen/CodeGenModule.cpp
lib/CodeGen/ItaniumCXXABI.cpp
lib/CodeGen/MicrosoftCXXABI.cpp
lib/Parse/ParseOpenMP.cpp
Index: lib/Parse/ParseOpenMP.cpp
===================================================================
--- lib/Parse/ParseOpenMP.cpp
+++ lib/Parse/ParseOpenMP.cpp
@@ -758,6 +758,7 @@
if (!Actions.ActOnStartOpenMPDeclareTargetDirective(DTLoc))
return DeclGroupPtrTy();
+ SmallVector<Decl *, 32> Decls;
DKind = ParseOpenMPDirectiveKind(*this);
while (DKind != OMPD_end_declare_target && DKind != OMPD_declare_target &&
Tok.isNot(tok::eof) && Tok.isNot(tok::r_brace)) {
@@ -781,6 +782,12 @@
else
TPA.Commit();
}
+
+ // Save the declarations so that we can create the declare target group
+ // later on.
+ if (Ptr)
+ for (auto *V : Ptr.get())
+ Decls.push_back(V);
}
if (DKind == OMPD_end_declare_target) {
@@ -795,8 +802,17 @@
} else {
Diag(Tok, diag::err_expected_end_declare_target);
Diag(DTLoc, diag::note_matching) << "'#pragma omp declare target'";
+ // We have an error, so we don't have to attempt to generate code for the
+ // declarations.
+ Decls.clear();
}
Actions.ActOnFinishOpenMPDeclareTargetDirective();
+
+ // If we have decls generate the group so that code can be generated for it
+ // later on.
+ if (!Decls.empty())
+ return Actions.BuildDeclaratorGroup(Decls);
+
return DeclGroupPtrTy();
}
case OMPD_unknown:
Index: lib/CodeGen/MicrosoftCXXABI.cpp
===================================================================
--- lib/CodeGen/MicrosoftCXXABI.cpp
+++ lib/CodeGen/MicrosoftCXXABI.cpp
@@ -387,8 +387,8 @@
QualType LValType) override;
void EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D,
- llvm::GlobalVariable *DeclPtr,
- bool PerformInit) override;
+ llvm::GlobalVariable *DeclPtr, bool PerformInit,
+ bool EmitInitOnly, bool EmitDtorOnly) override;
void registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D,
llvm::Constant *Dtor, llvm::Constant *Addr) override;
@@ -2387,15 +2387,17 @@
void MicrosoftCXXABI::EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D,
llvm::GlobalVariable *GV,
- bool PerformInit) {
+ bool PerformInit, bool EmitInitOnly,
+ bool EmitDtorOnly) {
// MSVC only uses guards for static locals.
if (!D.isStaticLocal()) {
assert(GV->hasWeakLinkage() || GV->hasLinkOnceLinkage());
// GlobalOpt is allowed to discard the initializer, so use linkonce_odr.
llvm::Function *F = CGF.CurFn;
F->setLinkage(llvm::GlobalValue::LinkOnceODRLinkage);
F->setComdat(CGM.getModule().getOrInsertComdat(F->getName()));
- CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit);
+ CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit, EmitInitOnly,
+ EmitDtorOnly);
return;
}
@@ -2496,7 +2498,8 @@
CGF.EmitBlock(InitBlock);
Builder.CreateStore(Builder.CreateOr(LI, Bit), GuardAddr);
CGF.EHStack.pushCleanup<ResetGuardBit>(EHCleanup, GuardAddr, GuardNum);
- CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit);
+ CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit, EmitInitOnly,
+ EmitDtorOnly);
CGF.PopCleanupBlock();
Builder.CreateBr(EndBlock);
@@ -2542,7 +2545,8 @@
// Ok, we ended up getting selected as the initializing thread.
CGF.EmitBlock(InitBlock);
CGF.EHStack.pushCleanup<CallInitThreadAbort>(EHCleanup, GuardAddr);
- CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit);
+ CGF.EmitCXXGlobalVarDeclInit(D, GV, PerformInit, EmitInitOnly,
+ EmitDtorOnly);
CGF.PopCleanupBlock();
CGF.EmitNounwindRuntimeCall(getInitThreadFooterFn(CGM),
GuardAddr.getPointer());
Index: lib/CodeGen/ItaniumCXXABI.cpp
===================================================================
--- lib/CodeGen/ItaniumCXXABI.cpp
+++ lib/CodeGen/ItaniumCXXABI.cpp
@@ -338,7 +338,8 @@
void EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D,
llvm::GlobalVariable *DeclPtr,
- bool PerformInit) override;
+ bool PerformInit,
+ bool EmitInitOnly, bool EmitDtorOnly) override;
void registerGlobalDtor(CodeGenFunction &CGF, const VarDecl &D,
llvm::Constant *dtor, llvm::Constant *addr) override;
@@ -1996,7 +1997,8 @@
void ItaniumCXXABI::EmitGuardedInit(CodeGenFunction &CGF,
const VarDecl &D,
llvm::GlobalVariable *var,
- bool shouldPerformInit) {
+ bool shouldPerformInit, bool EmitInitOnly,
+ bool EmitDtorOnly) {
CGBuilderTy &Builder = CGF.Builder;
// Inline variables that weren't instantiated from variable templates have
@@ -2162,7 +2164,8 @@
}
// Emit the initializer and add a global destructor if appropriate.
- CGF.EmitCXXGlobalVarDeclInit(D, var, shouldPerformInit);
+ CGF.EmitCXXGlobalVarDeclInit(D, var, shouldPerformInit, EmitInitOnly,
+ EmitDtorOnly);
if (threadsafe) {
// Pop the guard-abort cleanup if we pushed one.
Index: lib/CodeGen/CodeGenModule.cpp
===================================================================
--- lib/CodeGen/CodeGenModule.cpp
+++ lib/CodeGen/CodeGenModule.cpp
@@ -244,6 +244,12 @@
}
}
+ // Tell the OpenMP runtime we are replacing a global that can potentially be
+ // marked "declare target".
+ if (OpenMPRuntime)
+ if (auto *NewGV = dyn_cast<llvm::GlobalValue>(Replacement))
+ OpenMPRuntime->registerGlobalReplacement(MangledName, NewGV);
+
// Replace old with new, but keep the old order.
OldF->replaceAllUsesWith(Replacement);
if (NewF) {
@@ -266,6 +272,12 @@
GV->replaceAllUsesWith(C);
GV->eraseFromParent();
+
+ // Tell the OpenMP runtime we are replacing a global that can potentially be
+ // marked "declare target".
+ if (OpenMPRuntime)
+ if (auto *NewGV = dyn_cast<llvm::GlobalValue>(C))
+ OpenMPRuntime->registerGlobalReplacement(GV->getName(), NewGV);
}
}
@@ -1537,6 +1549,10 @@
assert(DeferredVTables.empty());
}
+ if (LangOpts.OpenMP && !LangOpts.OpenMPIsDevice) {
+ OpenMPRuntime->registerTrackedFunction();
+ }
+
// Stop if we're out of both deferred vtables and deferred declarations.
if (DeferredDeclsToEmit.empty())
return;
@@ -1756,6 +1772,11 @@
if (LangOpts.EmitAllDecls)
return true;
+ // In OpenMP device mode all declarations that were not filtered should be
+ // emitted.
+ if (LangOpts.OpenMPIsDevice)
+ return true;
+
return getContext().DeclMustBeEmitted(Global);
}
@@ -1879,6 +1900,21 @@
}
if (LangOpts.OpenMP) {
+ if (LangOpts.OpenMPIsDevice) {
+ if (const auto *FD = dyn_cast<FunctionDecl>(Global))
+ if (FD->isThisDeclarationADefinition()) {
+ if (OpenMPRuntime->MustBeEmittedForDevice(GD)) {
+ EmitGlobalDefinition(GD);
+ return;
+ }
+ }
+ }
+
+ if (!LangOpts.OpenMPIsDevice) {
+ StringRef MangledName = getMangledName(GD);
+ OpenMPRuntime->addTrackedFunction(MangledName, GD);
+ }
+
// If this is OpenMP device, check if it is legal to emit this global
// normally.
if (OpenMPRuntime && OpenMPRuntime->emitTargetGlobal(GD))
@@ -2136,6 +2172,11 @@
Context.getSourceManager(),
"Generating code for declaration");
+ // If this is OpenMP device, check if it is legal to emit this global
+ // normally.
+ if (OpenMPRuntime && OpenMPRuntime->emitTargetGlobal(GD))
+ return;
+
if (isa<FunctionDecl>(D)) {
// At -O0, don't generate IR for functions with available_externally
// linkage.
@@ -2145,11 +2186,15 @@
if (const auto *Method = dyn_cast<CXXMethodDecl>(D)) {
// Make sure to emit the definition(s) before we emit the thunks.
// This is necessary for the generation of certain thunks.
- if (const auto *CD = dyn_cast<CXXConstructorDecl>(Method))
+ if (const auto *CD = dyn_cast<CXXConstructorDecl>(Method)) {
+ if (OpenMPRuntime)
+ OpenMPRuntime->registerTargetFunctionDefinition(GD);
ABI->emitCXXStructor(CD, getFromCtorType(GD.getCtorType()));
- else if (const auto *DD = dyn_cast<CXXDestructorDecl>(Method))
+ } else if (const auto *DD = dyn_cast<CXXDestructorDecl>(Method)) {
+ if (OpenMPRuntime)
+ OpenMPRuntime->registerTargetFunctionDefinition(GD);
ABI->emitCXXStructor(DD, getFromDtorType(GD.getDtorType()));
- else
+ } else
EmitGlobalFunctionDefinition(GD, GV);
if (Method->isVirtual())
@@ -2264,6 +2309,11 @@
}
}
+ // Process function name as required by the OpenMP runtime
+ if (OpenMPRuntime) {
+ MangledName = OpenMPRuntime->RenameStandardFunction(MangledName);
+ }
+
// Lookup the entry, lazily creating it if necessary.
llvm::GlobalValue *Entry = GetGlobalValue(MangledName);
if (Entry) {
@@ -2824,6 +2874,11 @@
void CodeGenModule::EmitTentativeDefinition(const VarDecl *D) {
assert(!D->getInit() && "Cannot emit definite definitions here!");
+ // If this is OpenMP device, check if it is legal to emit this global
+ // normally.
+ if (OpenMPRuntime && OpenMPRuntime->emitTargetGlobal(D))
+ return;
+
StringRef MangledName = getMangledName(D);
llvm::GlobalValue *GV = GetGlobalValue(MangledName);
@@ -3091,6 +3146,9 @@
}
}
+ if (OpenMPRuntime)
+ OpenMPRuntime->registerTargetVariableDefinition(D, GV);
+
GV->setInitializer(Init);
if (emitter) emitter->finalize(GV);
@@ -3448,6 +3506,9 @@
if (!GV->isDeclaration())
return;
+ if (OpenMPRuntime)
+ OpenMPRuntime->registerTargetFunctionDefinition(GD);
+
// We need to set linkage and visibility on the function before
// generating code for it because various parts of IR generation
// want to propagate this information down (e.g. to local static
Index: lib/CodeGen/CodeGenFunction.h
===================================================================
--- lib/CodeGen/CodeGenFunction.h
+++ lib/CodeGen/CodeGenFunction.h
@@ -3633,9 +3633,12 @@
/// EmitCXXGlobalVarDeclInit - Create the initializer for a C++
- /// variable with global storage.
+ /// variable with global storage. If \a EmitInitOnly or \a EmitDtorOnly are
+ /// set to true, only the call to the initializer or destructor is emitted,
+ /// respectively.
void EmitCXXGlobalVarDeclInit(const VarDecl &D, llvm::Constant *DeclPtr,
- bool PerformInit);
+ bool PerformInit, bool EmitInitOnly,
+ bool EmitDtorOnly);
llvm::Constant *createAtExitStub(const VarDecl &VD, llvm::Constant *Dtor,
llvm::Constant *Addr);
@@ -3651,7 +3654,8 @@
/// once, e.g. with a static local variable or a static data member
/// of a class template.
void EmitCXXGuardedInit(const VarDecl &D, llvm::GlobalVariable *DeclPtr,
- bool PerformInit);
+ bool PerformInit, bool EmitInitOnly = false,
+ bool EmitDtorOnly = false);
enum class GuardKind { VariableGuard, TlsGuard };
@@ -3674,10 +3678,12 @@
const std::vector<std::pair<llvm::WeakTrackingVH, llvm::Constant *>>
&DtorsAndObjects);
- void GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn,
- const VarDecl *D,
+
+ void GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn, const VarDecl *D,
llvm::GlobalVariable *Addr,
- bool PerformInit);
+ bool PerformInit,
+ bool EmitInitOnly = false,
+ bool EmitDtorOnly = false);
void EmitCXXConstructExpr(const CXXConstructExpr *E, AggValueSlot Dest);
Index: lib/CodeGen/CGOpenMPRuntime.h
===================================================================
--- lib/CodeGen/CGOpenMPRuntime.h
+++ lib/CodeGen/CGOpenMPRuntime.h
@@ -241,6 +241,18 @@
/// \brief Returns pointer to ident_t type.
llvm::Type *getIdentTyPointerTy();
+ /// Register target region related with the launching of Ctor/Dtors entry.
+ /// \param DeviceID The device ID of the target region in the system.
+ /// \param FileID The file ID of the target region in the system.
+ /// \param RegionName The name of the region.
+ /// \param Line Line where the declaration the target region refers to is
+ /// defined.
+ /// \param Fn The function that implements the target region.
+ /// \param IsDtor True if what being registered is a destructor.
+ virtual void registerCtorDtorEntry(unsigned DeviceID, unsigned FileID,
+ StringRef RegionName, unsigned Line,
+ llvm::Function *Fn, bool IsDtor);
+
/// \brief Gets thread id value for the current thread.
///
llvm::Value *getThreadID(CodeGenFunction &CGF, SourceLocation Loc);
@@ -250,6 +262,9 @@
//
virtual StringRef getOutlinedHelperName() const { return ".omp_outlined."; }
+public:
+ virtual StringRef RenameStandardFunction(StringRef name);
+
/// Emits \p Callee function call with arguments \p Args with location \p Loc.
void emitCall(CodeGenFunction &CGF, llvm::Value *Callee,
ArrayRef<llvm::Value *> Args = llvm::None,
@@ -370,18 +385,22 @@
class OffloadEntriesInfoManagerTy {
CodeGenModule &CGM;
- /// \brief Number of entries registered so far.
- unsigned OffloadingEntriesNum;
+ /// \brief Number of ordered entries registered so far.
+ unsigned OffloadingOrderedEntriesNum = 0u;
public:
- /// Base class of the entries info.
+ /// \brief Base class of the entries info.
class OffloadEntryInfo {
public:
- /// Kind of a given entry. Currently, only target regions are
+ /// \brief Kind of a given entry. Currently, only target regions are
/// supported.
enum OffloadingEntryInfoKinds : unsigned {
// Entry is a target region.
OFFLOAD_ENTRY_INFO_TARGET_REGION = 0,
+ // Entry is a device global variable.
+ OFFLOAD_ENTRY_INFO_DEVICE_GLOBAL_VAR = 1,
+ // Entry is a device function.
+ OFFLOAD_ENTRY_INFO_DEVICE_FUNCTION = 2,
// Invalid entry info.
OFFLOAD_ENTRY_INFO_INVALID = ~0u
};
@@ -403,18 +422,19 @@
/// Flags associated with the device global.
int32_t Flags;
- /// Order this entry was emitted.
+ // \brief Order this entry was emitted.
unsigned Order;
OffloadingEntryInfoKinds Kind;
};
/// \brief Return true if a there are no entries defined.
bool empty() const;
- /// \brief Return number of entries defined so far.
- unsigned size() const { return OffloadingEntriesNum; }
- OffloadEntriesInfoManagerTy(CodeGenModule &CGM)
- : CGM(CGM), OffloadingEntriesNum(0) {}
+ /// \brief Return number of ordered entries defined so far.
+ unsigned getOrderedEntriesNum() const {
+ return OffloadingOrderedEntriesNum;
+ }
+ OffloadEntriesInfoManagerTy(CodeGenModule &CGM) : CGM(CGM) {}
///
/// Target region entries related.
@@ -471,6 +491,102 @@
void actOnTargetRegionEntriesInfo(
const OffloadTargetRegionEntryInfoActTy &Action);
+ ///
+ /// Device global variable entries related.
+ ///
+ /// \brief Device global variable entries info.
+ class OffloadEntryInfoDeviceGlobalVar : public OffloadEntryInfo {
+ // \brief Address of the entity that has to be mapped for offloading.
+ llvm::Constant *Addr;
+ // \brief Type of the global variable.
+ QualType Ty;
+ // \brief Only generate metadata for this offload entry
+ bool OnlyMetadataFlag = false;
+
+ public:
+ OffloadEntryInfoDeviceGlobalVar()
+ : OffloadEntryInfo(OFFLOAD_ENTRY_INFO_DEVICE_GLOBAL_VAR, ~0u,
+ /*Flags=*/0u),
+ Addr(nullptr) {}
+ explicit OffloadEntryInfoDeviceGlobalVar(unsigned Order,
+ llvm::Constant *Addr,
+ QualType Ty, int32_t Flags)
+ : OffloadEntryInfo(OFFLOAD_ENTRY_INFO_DEVICE_GLOBAL_VAR, Order,
+ Flags),
+ Addr(Addr), Ty(Ty) {}
+
+ llvm::Constant *getAddress() const { return Addr; }
+ void setAddress(llvm::Constant *V) {
+ assert(!Addr && "Address as been set before!");
+ Addr = V;
+ }
+ QualType getType() const { return Ty; }
+ void setType(QualType QTy) { Ty = QTy; }
+ bool getOnlyMetadataFlag() { return OnlyMetadataFlag; }
+ void setOnlyMetadataFlag(bool B) { OnlyMetadataFlag = B; }
+ static bool classof(const OffloadEntryInfo *Info) {
+ return Info->getKind() == OFFLOAD_ENTRY_INFO_DEVICE_GLOBAL_VAR;
+ }
+ };
+ /// \brief Initialize device global variable entry.
+ void initializeDeviceGlobalVarEntryInfo(StringRef MangledName,
+ unsigned Order);
+ /// \brief Register device global variable entry.
+ void registerDeviceGlobalVarEntryInfo(StringRef MangledName,
+ llvm::Constant *Addr, QualType Ty,
+ int32_t Flags,
+ bool isExternallyVisible);
+ /// \brief Return true if a device global variable entry with the provided
+ /// information exists.
+ bool hasDeviceGlobalVarEntryInfo(StringRef MangledName) const;
+ /// brief Applies action \a Action on all registered entries.
+ typedef llvm::function_ref<void(StringRef,
+ OffloadEntryInfoDeviceGlobalVar &)>
+ OffloadDeviceGlobalVarEntryInfoActTy;
+ void actOnDeviceGlobalVarEntriesInfo(
+ const OffloadDeviceGlobalVarEntryInfoActTy &Action);
+
+ ///
+ /// Device function entries related.
+ ///
+ /// \brief Device global variable entries info.
+ class OffloadEntryInfoDeviceFunction : public OffloadEntryInfo {
+ // \brief Set to true if this entry was registered.
+ bool IsRegistered = false;
+
+ public:
+ OffloadEntryInfoDeviceFunction()
+ : OffloadEntryInfo(OFFLOAD_ENTRY_INFO_DEVICE_FUNCTION, ~0u,
+ /*Flags=*/0u) {}
+ explicit OffloadEntryInfoDeviceFunction(bool IsRegistered)
+ : OffloadEntryInfo(OFFLOAD_ENTRY_INFO_DEVICE_FUNCTION, ~0u,
+ /*Flags=*/0u),
+ IsRegistered(IsRegistered) {}
+
+ bool isRegistered() const { return IsRegistered; }
+ void setIsRegistered(bool Val) {
+ assert(!IsRegistered && "It was registered before!");
+ IsRegistered = Val;
+ }
+
+ static bool classof(const OffloadEntryInfo *Info) {
+ return Info->getKind() == OFFLOAD_ENTRY_INFO_DEVICE_FUNCTION;
+ }
+ };
+ /// \brief Initialize device function entry.
+ void initializeDeviceFunctionEntryInfo(StringRef MangledName);
+ /// \brief Register device function entry.
+ void registerDeviceFunctionEntryInfo(StringRef MangledName);
+ /// \brief Return true if a device function entry with the provided
+ /// information exists.
+ bool hasDeviceFunctionEntryInfo(StringRef MangledName) const;
+ /// brief Applies action \a Action on all registered entries.
+ typedef llvm::function_ref<void(StringRef,
+ OffloadEntryInfoDeviceFunction &)>
+ OffloadDeviceFunctionEntryInfoActTy;
+ void actOnDeviceFunctionEntriesInfo(
+ const OffloadDeviceFunctionEntryInfoActTy &Action);
+
private:
// Storage for target region entries kind. The storage is to be indexed by
// file ID, device ID, parent function name and line number.
@@ -483,6 +599,18 @@
typedef llvm::DenseMap<unsigned, OffloadEntriesTargetRegionPerFile>
OffloadEntriesTargetRegionPerDevice;
typedef OffloadEntriesTargetRegionPerDevice OffloadEntriesTargetRegionTy;
+
+ // Storage for device global variable entries kind. The storage is to be
+ // indexed by mangled name.
+ typedef llvm::StringMap<OffloadEntryInfoDeviceGlobalVar>
+ OffloadEntriesDeviceGlobalVarTy;
+ OffloadEntriesDeviceGlobalVarTy OffloadEntriesDeviceGlobalVar;
+
+ // Storage for device function entries kind. The storage is to be indexed by
+ // mangled name.
+ typedef llvm::StringMap<OffloadEntryInfoDeviceFunction>
+ OffloadEntriesDeviceFunctionTy;
+ OffloadEntriesDeviceFunctionTy OffloadEntriesDeviceFunction;
OffloadEntriesTargetRegionTy OffloadEntriesTargetRegion;
};
OffloadEntriesInfoManagerTy OffloadEntriesInfoManager;
@@ -616,11 +744,43 @@
llvm::Value *TaskFunction, QualType SharedsTy,
Address Shareds, const OMPTaskDataTy &Data);
+ /// This contains all the decls which were not specified under declare target
+ /// region, which are deferred for device code emission.
+ /// If a decl is used in target region implicitly without specifying under
+ /// declare target, deferred decl is emitted during Codegen::Release for
+ /// device codegen.
+ llvm::StringMap<GlobalDecl> TrackedDecls;
+
+ /// Struct that keeps information about the emitted definitions and
+ /// ctors/dtors so that it can be revisited when emitting declare target
+ /// entries.
+ struct DeclareTargetEntryInfo {
+ /// The declaration associated with this information.
+ const Decl *Variable;
+ /// Address of the variable or null if there is no variable.
+ llvm::Constant *VariableAddr = nullptr;
+ /// True if the associated variables requires Ctor or Dtor.
+ bool RequiresCtorDtor = false;
+ /// True if the variable associated with this information required
+ /// initialization.
+ bool PerformInitialization = false;
+ };
+
+ /// Map between a declaration name and its declare target information.
+ llvm::StringMap<DeclareTargetEntryInfo> DeclareTargetEntryInfoMap;
+
public:
explicit CGOpenMPRuntime(CodeGenModule &CGM);
virtual ~CGOpenMPRuntime() {}
virtual void clear();
+ /// The function is added tracked functions list.
+ virtual void addTrackedFunction(StringRef MangledName, GlobalDecl GD);
+
+ /// The function register all tracked functions if they have
+ /// OMPDeclareTargetDeclAttr
+ virtual void registerTrackedFunction();
+
/// Emit code for the specified user defined reduction construct.
virtual void emitUserDefinedReduction(CodeGenFunction *CGF,
const OMPDeclareReductionDecl *D);
@@ -1229,6 +1389,48 @@
/// \param GD Global to scan.
virtual bool emitTargetGlobal(GlobalDecl GD);
+ /// \brief Emit the entry points (target regions) that implement the
+ /// initializers and destructors of the global \a D if that is meaningful for
+ /// the device. Returns true if the emission was successful.
+ /// \param D Global whose initializers destructors should be emitted.
+ /// \param Addr Address of the global being initialized/destroyed.
+ /// \param PerformInit True if the initializer should be emitted.
+ virtual bool emitDeviceCtorDtor(const VarDecl &D, llvm::GlobalVariable *Addr,
+ bool PerformInit);
+
+ /// \brief Register that a variable requires a Ctor/Dtor.
+ /// \param D Global whose initializers destructors should be emitted.
+ /// \param Addr Address of the global being initialized/destroyed.
+ /// \param PerformInit True if the initializer should be emitted.
+ virtual void registerDeviceCtorDtorLaunching(CodeGenFunction &CGF,
+ const VarDecl &D,
+ llvm::GlobalVariable *Addr,
+ bool PerformInit);
+
+ /// \brief Check whether the function definition in \a GD must be emitted for
+ /// the device or not.
+ /// \param GD Global declaration whose definition is being emitted.
+ virtual bool MustBeEmittedForDevice(GlobalDecl GD);
+
+ /// \brief Register the function definition \a GD as meaningful for the
+ /// target.
+ /// \param GD Global declaration whose definition is being emitted.
+ virtual void registerTargetFunctionDefinition(GlobalDecl GD);
+
+ /// \brief Register the global variable definition \a D as meaningful for the
+ /// target.
+ /// \param D Global declaration whose definition is being emitted.
+ /// \param Addr Address of the global.
+ virtual void registerTargetVariableDefinition(const VarDecl *D,
+ llvm::Constant *Addr);
+
+ /// \brief Register a global that is replacing some other. If the global being
+ /// declare has a declare target attribute, the new one is registered as such.
+ /// \param MangledName Name of the global being replaced.
+ /// \param NewVal Global that is used to replace.
+ virtual void registerGlobalReplacement(StringRef MangledName,
+ llvm::GlobalValue *NewVal);
+
/// \brief Creates the offloading descriptor in the event any target region
/// was emitted in the current module and return the function that registers
/// it.
Index: lib/CodeGen/CGOpenMPRuntime.cpp
===================================================================
--- lib/CodeGen/CGOpenMPRuntime.cpp
+++ lib/CodeGen/CGOpenMPRuntime.cpp
@@ -3325,7 +3325,9 @@
bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty() const {
// FIXME: Add other entries type when they become supported.
- return OffloadEntriesTargetRegion.empty();
+ return OffloadEntriesTargetRegion.empty() &&
+ OffloadEntriesDeviceGlobalVar.empty() &&
+ OffloadEntriesDeviceFunction.empty();
}
/// \brief Initialize target region entry.
@@ -3339,7 +3341,7 @@
OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] =
OffloadEntryInfoTargetRegion(Order, /*Addr=*/nullptr, /*ID=*/nullptr,
/*Flags=*/0);
- ++OffloadingEntriesNum;
+ ++OffloadingOrderedEntriesNum;
}
void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
@@ -3360,7 +3362,8 @@
Entry.setFlags(Flags);
return;
} else {
- OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum++, Addr, ID, Flags);
+ OffloadEntryInfoTargetRegion Entry(OffloadingOrderedEntriesNum++, Addr, ID,
+ Flags);
OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry;
}
}
@@ -3396,16 +3399,114 @@
Action(D.first, F.first, P.first(), L.first, L.second);
}
+/// \brief Initialize device global variable entry.
+void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
+ initializeDeviceGlobalVarEntryInfo(StringRef MangledName, unsigned Order) {
+ assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
+ "only required for the device "
+ "code generation.");
+ OffloadEntriesDeviceGlobalVar[MangledName] = OffloadEntryInfoDeviceGlobalVar(
+ Order, /*Addr=*/nullptr, QualType(), /*Flags=*/0);
+ ++OffloadingOrderedEntriesNum;
+}
+
+void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
+ registerDeviceGlobalVarEntryInfo(StringRef MangledName,
+ llvm::Constant *Addr, QualType Ty,
+ int32_t Flags, bool isExternallyVisible) {
+ // If we are emitting code for a target, the entry is already initialized,
+ // only has to be registered.
+ if (CGM.getLangOpts().OpenMPIsDevice) {
+ assert(hasDeviceGlobalVarEntryInfo(MangledName) && "Entry must exist.");
+ auto &Entry = OffloadEntriesDeviceGlobalVar[MangledName];
+ assert(Entry.isValid() && "Entry not initialized!");
+ Entry.setAddress(Addr);
+ Entry.setType(Ty);
+ Entry.setFlags(Flags);
+ Entry.setOnlyMetadataFlag(!isExternallyVisible);
+ return;
+ } else {
+ OffloadEntryInfoDeviceGlobalVar Entry(OffloadingOrderedEntriesNum++, Addr,
+ Ty, Flags);
+ Entry.setOnlyMetadataFlag(!isExternallyVisible);
+ OffloadEntriesDeviceGlobalVar[MangledName] = Entry;
+ }
+}
+
+bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasDeviceGlobalVarEntryInfo(
+ StringRef MangledName) const {
+ auto Entry = OffloadEntriesDeviceGlobalVar.find(MangledName);
+ if (Entry == OffloadEntriesDeviceGlobalVar.end())
+ return false;
+ // Fail if this entry is already registered.
+ if (Entry->second.getAddress())
+ return false;
+ return true;
+}
+
+void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
+ actOnDeviceGlobalVarEntriesInfo(
+ const OffloadDeviceGlobalVarEntryInfoActTy &Action) {
+ // Scan all target region entries and perform the provided action.
+ for (auto &E : OffloadEntriesDeviceGlobalVar)
+ Action(E.first(), E.second);
+}
+
+/// \brief Initialize device function entry.
+void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
+ initializeDeviceFunctionEntryInfo(StringRef MangledName) {
+ assert(CGM.getLangOpts().OpenMPIsDevice && "Initialization of entries is "
+ "only required for the device "
+ "code generation.");
+ OffloadEntriesDeviceFunction[MangledName] = OffloadEntryInfoDeviceFunction();
+}
+
+void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
+ registerDeviceFunctionEntryInfo(StringRef MangledName) {
+ // If we are emitting code for a target, the entry is already initialized,
+ // only has to be registered.
+ if (CGM.getLangOpts().OpenMPIsDevice) {
+ assert(hasDeviceFunctionEntryInfo(MangledName) && "Entry must exist.");
+ auto &Entry = OffloadEntriesDeviceFunction[MangledName];
+ Entry.setIsRegistered(/*Val=*/true);
+ return;
+ } else
+ OffloadEntriesDeviceFunction[MangledName] =
+ OffloadEntryInfoDeviceFunction(/*IsRegistred=*/true);
+}
+
+bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasDeviceFunctionEntryInfo(
+ StringRef MangledName) const {
+ auto Entry = OffloadEntriesDeviceFunction.find(MangledName);
+ if (Entry == OffloadEntriesDeviceFunction.end())
+ return false;
+ // Fail if this entry is already registered.
+ if (Entry->second.isRegistered())
+ return false;
+ return true;
+}
+
+void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
+ actOnDeviceFunctionEntriesInfo(
+ const OffloadDeviceFunctionEntryInfoActTy &Action) {
+ // Scan all target region entries and perform the provided action.
+ for (auto &E : OffloadEntriesDeviceFunction)
+ Action(E.first(), E.second);
+}
+
/// \brief Create a Ctor/Dtor-like function whose body is emitted through
-/// \a Codegen. This is used to emit the two functions that register and
-/// unregister the descriptor of the current compilation unit.
+/// \a Codegen. This is used to emit functions that register and
+/// unregister descriptors, initializers or destructors in the current
+/// compilation unit.
static llvm::Function *
-createOffloadingBinaryDescriptorFunction(CodeGenModule &CGM, StringRef Name,
- const RegionCodeGenTy &Codegen) {
+createOffloadingHelperFunction(CodeGenModule &CGM, StringRef Name,
+ bool RequiresArgument,
+ const RegionCodeGenTy &Codegen) {
auto &C = CGM.getContext();
FunctionArgList Args;
ImplicitParamDecl DummyPtr(C, C.VoidPtrTy, ImplicitParamDecl::Other);
- Args.push_back(&DummyPtr);
+ if (RequiresArgument)
+ Args.push_back(&DummyPtr);
CodeGenFunction CGF(CGM);
// Disable debug info for global (de-)initializer because they are not part of
@@ -3510,14 +3611,14 @@
ImplicitParamDecl RegUnregVar(C, C.getTranslationUnitDecl(), SourceLocation(),
IdentInfo, C.CharTy, ImplicitParamDecl::Other);
- auto *UnRegFn = createOffloadingBinaryDescriptorFunction(
- CGM, ".omp_offloading.descriptor_unreg",
+ auto *UnRegFn = createOffloadingHelperFunction(
+ CGM, ".omp_offloading.descriptor_unreg", /*RequiresArgument=*/true,
[&](CodeGenFunction &CGF, PrePostActionTy &) {
CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_unregister_lib),
Desc);
});
- auto *RegFn = createOffloadingBinaryDescriptorFunction(
- CGM, ".omp_offloading.descriptor_reg",
+ auto *RegFn = createOffloadingHelperFunction(
+ CGM, ".omp_offloading.descriptor_reg", /*RequiresArgument=*/false,
[&](CodeGenFunction &CGF, PrePostActionTy &) {
CGF.EmitRuntimeCall(createRuntimeFunction(OMPRTL__tgt_register_lib),
Desc);
@@ -3561,6 +3662,10 @@
Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
llvm::Constant *StrPtr = llvm::ConstantExpr::getBitCast(Str, CGM.Int8PtrTy);
+ // Decide linkage type of the entry struct by looking at the linkage type of
+ // the variable. By default the linkage is Link-Once.
+ auto EntryLinkage = llvm::GlobalValue::WeakAnyLinkage;
+
// We can't have any padding between symbols, so we need to have 1-byte
// alignment.
auto Align = CharUnits::fromQuantity(1);
@@ -3573,9 +3678,10 @@
EntryInit.addInt(CGM.SizeTy, Size);
EntryInit.addInt(CGM.Int32Ty, Flags);
EntryInit.addInt(CGM.Int32Ty, 0);
- llvm::GlobalVariable *Entry = EntryInit.finishAndCreateGlobal(
- Twine(".omp_offloading.entry.") + Name, Align,
- /*constant*/ true, llvm::GlobalValue::ExternalLinkage);
+ SmallString<128> EntryGblName(".omp_offloading.entry.");
+ EntryGblName += Name;
+ llvm::GlobalVariable *Entry = EntryInit.finishAndCreateGlobal(EntryGblName,
+ Align, /*constant*/ true, EntryLinkage);
// The entry has to be created in the section the linker expects it to be.
Entry->setSection(".omp_offloading.entries");
@@ -3598,7 +3704,7 @@
llvm::Module &M = CGM.getModule();
llvm::LLVMContext &C = M.getContext();
SmallVector<OffloadEntriesInfoManagerTy::OffloadEntryInfo *, 16>
- OrderedEntries(OffloadEntriesInfoManager.size());
+ OrderedEntries(OffloadEntriesInfoManager.getOrderedEntriesNum());
// Create the offloading info metadata node.
llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata("omp_offload.info");
@@ -3615,7 +3721,7 @@
auto &&TargetRegionMetadataEmitter = [&](
unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned Line,
OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion &E) {
- llvm::SmallVector<llvm::Metadata *, 32> Ops;
+ llvm::SmallVector<llvm::Metadata *, 6> Ops;
// Generate metadata for target regions. Each entry of this metadata
// contains:
// - Entry 0 -> Kind of this type of metadata (0).
@@ -3642,16 +3748,72 @@
OffloadEntriesInfoManager.actOnTargetRegionEntriesInfo(
TargetRegionMetadataEmitter);
+ // Create function that emits metadata for each device global variable entry;
+ auto &&DeviceGlobalVarMetadataEmitter =
+ [&](StringRef MangledName,
+ OffloadEntriesInfoManagerTy::OffloadEntryInfoDeviceGlobalVar &E) {
+ llvm::SmallVector<llvm::Metadata *, 3> Ops;
+ // Generate metadata for global variables. Each entry of this metadata
+ // contains:
+ // - Entry 0 -> Kind of this type of metadata (1).
+ // - Entry 1 -> Mangled name of the variable.
+ // - Entry 2 -> Order the entry was created.
+ // The first element of the metadata node is the kind.
+ Ops.push_back(getMDInt(E.getKind()));
+ Ops.push_back(getMDString(MangledName));
+ Ops.push_back(getMDInt(E.getOrder()));
+
+ // Save this entry in the right position of the ordered entries array.
+ OrderedEntries[E.getOrder()] = &E;
+
+ // Add metadata to the named metadata node.
+ MD->addOperand(llvm::MDNode::get(C, Ops));
+ };
+
+ OffloadEntriesInfoManager.actOnDeviceGlobalVarEntriesInfo(
+ DeviceGlobalVarMetadataEmitter);
+
+ // Create function that emits metadata for each device function entry;
+ auto &&DeviceFunctionMetadataEmitter =
+ [&](StringRef MangledName,
+ OffloadEntriesInfoManagerTy::OffloadEntryInfoDeviceFunction &E) {
+ llvm::SmallVector<llvm::Metadata *, 2> Ops;
+ // Generate metadata for global variables. Each entry of this metadata
+ // contains:
+ // - Entry 0 -> Kind of this type of metadata (2).
+ // - Entry 1 -> Mangled name of the variable.
+ // The first element of the metadata node is the kind.
+ Ops.push_back(getMDInt(E.getKind()));
+ Ops.push_back(getMDString(MangledName));
+
+ // Add metadata to the named metadata node.
+ MD->addOperand(llvm::MDNode::get(C, Ops));
+ };
+
+ OffloadEntriesInfoManager.actOnDeviceFunctionEntriesInfo(
+ DeviceFunctionMetadataEmitter);
+
for (auto *E : OrderedEntries) {
assert(E && "All ordered entries must exist!");
if (auto *CE =
dyn_cast<OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion>(
E)) {
assert(CE->getID() && CE->getAddress() &&
"Entry ID and Addr are invalid!");
- createOffloadEntry(CE->getID(), CE->getAddress(), /*Size=*/0);
+ createOffloadEntry(CE->getID(), CE->getAddress(), /*Size=*/0,
+ CE->getFlags());
+ } else if (auto *CE = dyn_cast<OffloadEntriesInfoManagerTy::
+ OffloadEntryInfoDeviceGlobalVar>(E)) {
+ assert(CE->getAddress() && "Entry Addr is invalid!");
+ // The global address can be used as ID.
+ if (!CE->getOnlyMetadataFlag()) {
+ createOffloadEntry(
+ CE->getAddress(), CE->getAddress(),
+ CGM.getContext().getTypeSizeInChars(CE->getType()).getQuantity(),
+ CE->getFlags());
+ }
} else
- llvm_unreachable("Unsupported entry kind.");
+ llvm_unreachable("Unsupported ordered entry kind.");
}
}
@@ -3707,6 +3869,17 @@
/*ParentName=*/getMDString(3), /*Line=*/getMDInt(4),
/*Order=*/getMDInt(5));
break;
+ case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
+ OFFLOAD_ENTRY_INFO_DEVICE_GLOBAL_VAR:
+ OffloadEntriesInfoManager.initializeDeviceGlobalVarEntryInfo(
+ /*MangledName=*/getMDString(1),
+ /*Order=*/getMDInt(2));
+ break;
+ case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
+ OFFLOAD_ENTRY_INFO_DEVICE_FUNCTION:
+ OffloadEntriesInfoManager.initializeDeviceFunctionEntryInfo(
+ /*MangledName=*/getMDString(1));
+ break;
}
}
}
@@ -7390,6 +7563,11 @@
if (!CGM.getLangOpts().OpenMPIsDevice)
return false;
+ // Emit this function normally if it is a device function.
+ if (OffloadEntriesInfoManager.hasDeviceFunctionEntryInfo(
+ CGM.getMangledName(GD)))
+ return false;
+
// Try to detect target regions in the function.
scanForTargetRegionsFunctions(FD.getBody(), CGM.getMangledName(GD));
@@ -7421,9 +7599,10 @@
}
}
- // If we are in target mode, we do not emit any global (declare target is not
- // implemented yet). Therefore we signal that GD was processed in this case.
- return true;
+ // If we are in target mode we only emit global variables that we could find
+ // in the host metadata.
+ return !OffloadEntriesInfoManager.hasDeviceGlobalVarEntryInfo(
+ CGM.getMangledName(GD));
}
bool CGOpenMPRuntime::emitTargetGlobal(GlobalDecl GD) {
@@ -7434,7 +7613,323 @@
return emitTargetGlobalVariable(GD);
}
+bool CGOpenMPRuntime::MustBeEmittedForDevice(GlobalDecl GD) {
+ if (!CGM.getLangOpts().OpenMPIsDevice &&
+ CGM.getLangOpts().OMPTargetTriples.empty())
+ return false;
+
+ // Should be in device if it has metadata
+ return OffloadEntriesInfoManager.hasDeviceFunctionEntryInfo(
+ CGM.getMangledName(GD));
+}
+
+/// \brief Return the declare target attribute if the declaration is marked as
+// 'declare target', i.e. the declaration itself, its template declaration, or
+/// any of its redeclarations have the 'declare target' attribute.
+static OMPDeclareTargetDeclAttr *
+IsDeclareTargetDeclaration(const ValueDecl *VD) {
+ const Decl *RelevantDecl = VD;
+
+ // Try to get the original template if any.
+ if (auto *FD = dyn_cast<FunctionDecl>(VD)) {
+ if (auto *Tmpl = FD->getPrimaryTemplate())
+ RelevantDecl = Tmpl;
+ }
+
+ // Check if the declaration or any of its redeclarations have a declare target
+ // attribute.
+ if (auto *Attr = RelevantDecl->getAttr<OMPDeclareTargetDeclAttr>())
+ return Attr;
+
+ if (auto *Attr = VD->getAttr<OMPDeclareTargetDeclAttr>())
+ return Attr;
+
+ for (const Decl *RD : RelevantDecl->redecls())
+ if (auto *Attr = RD->getAttr<OMPDeclareTargetDeclAttr>())
+ return Attr;
+
+ return nullptr;
+}
+
+namespace {
+enum OpenMPOffloadingDeclareTargetFlags {
+ /// \brief Mark the entry has having a 'link' attribute.
+ OMP_DECLARE_TARGET_LINK = 0x01,
+ /// \brief Mark the entry has being a global constructor.
+ OMP_DECLARE_TARGET_CTOR = 0x02,
+ /// \brief Mark the entry has being a global destructor.
+ OMP_DECLARE_TARGET_DTOR = 0x04,
+};
+}
+
+void CGOpenMPRuntime::registerCtorDtorEntry(unsigned DeviceID, unsigned FileID,
+ StringRef RegionName, unsigned Line,
+ llvm::Function *Fn, bool IsDtor) {
+ OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
+ DeviceID, FileID, RegionName, Line, Fn, Fn,
+ IsDtor ? OMP_DECLARE_TARGET_DTOR : OMP_DECLARE_TARGET_CTOR);
+}
+
+bool CGOpenMPRuntime::emitDeviceCtorDtor(const VarDecl &D,
+ llvm::GlobalVariable *Addr,
+ bool PerformInit) {
+ // If this is not an OpenMP device, don't have to generate anything.
+ if (!CGM.getLangOpts().OpenMPIsDevice)
+ return false;
+
+ // Produce the unique prefix to identify the new target regions. We use the
+ // source location of the variable declaration which we know to not conflict
+ // with any target region.
+ unsigned DeviceID;
+ unsigned FileID;
+ unsigned Line;
+ getTargetEntryUniqueInfo(CGM.getContext(), D.getLocStart(), DeviceID, FileID,
+ Line);
+
+ // Check if we have a dtor target region specified by the host. The parent
+ // name is the name of the declaration with suffix _dtor and/or _init if that
+ // is a destructor or initializer target region.
+ SmallString<128> ParentNameDtor;
+ ParentNameDtor += "__omp_offloading_dtor_";
+ ParentNameDtor += D.getName();
+
+ // If we don't have a Dtor specified by the host, we don't have anything to
+ // do, but we return true to prevent the default initializer codegen.
+ if (!OffloadEntriesInfoManager.hasTargetRegionEntryInfo(DeviceID, FileID,
+ ParentNameDtor, Line))
+ return true;
+
+ auto &Ctx = CGM.getContext();
+ SmallString<64> PrefixName;
+ {
+ llvm::raw_svector_ostream OS(PrefixName);
+ OS << "__omp_offloading" << llvm::format("_%x", DeviceID)
+ << llvm::format("_%x_", FileID) << D.getName() << "_l" << Line;
+ }
+
+ // If we need to perform an initialization, we need to create a function that
+ // calls the initializer as entry point.
+ if (PerformInit) {
+ SmallString<128> ParentNameInit;
+ ParentNameInit += "__omp_offloading_init_";
+ ParentNameInit += D.getName();
+
+ assert(OffloadEntriesInfoManager.hasTargetRegionEntryInfo(
+ DeviceID, FileID, ParentNameInit, Line) &&
+ "Expecting initializer to be defined by the OpenMP host!");
+
+ auto &FnInfo =
+ CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, None);
+ auto *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
+ auto *Fn =
+ llvm::Function::Create(FnTy, llvm::GlobalValue::ExternalLinkage,
+ Twine(PrefixName) + "_init", &CGM.getModule());
+ CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, FnInfo, Fn);
+ CodeGenFunction(CGM).GenerateCXXGlobalVarDeclInitFunc(
+ Fn, &D, Addr, /*PerformInit=*/true, /*emitInitOnly=*/true);
+
+ // Register the information for the entry associated with this target
+ // region.
+ registerCtorDtorEntry(DeviceID, FileID, ParentNameInit, Line, Fn,
+ /*IsDtor=*/false);
+ }
+
+ // Create the target region for the destructor.
+ auto &FnInfo =
+ CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, None);
+ auto *FnTy = CGM.getTypes().GetFunctionType(FnInfo);
+ auto *Fn =
+ llvm::Function::Create(FnTy, llvm::GlobalValue::ExternalLinkage,
+ Twine(PrefixName) + "_dtor", &CGM.getModule());
+ CGM.SetLLVMFunctionAttributes(/*D=*/nullptr, FnInfo, Fn);
+ CodeGenFunction(CGM).GenerateCXXGlobalVarDeclInitFunc(
+ Fn, &D, Addr, PerformInit, /*emitInitOnly=*/false, /*emitDtorOnly=*/true);
+
+ // Register the information for the entry associated with this target region.
+ registerCtorDtorEntry(DeviceID, FileID, ParentNameDtor, Line, Fn,
+ /*IsDtor=*/true);
+
+ // We successfully generated the target regions to initialize and destroy the
+ // global.
+ return true;
+}
+
+void CGOpenMPRuntime::registerDeviceCtorDtorLaunching(
+ CodeGenFunction &CGF, const VarDecl &D, llvm::GlobalVariable *Addr,
+ bool PerformInit) {
+ // If we are not producing code for the host or we don't have any devices
+ // specified, we don't have to launch anything.
+ if (CGM.getLangOpts().OpenMPIsDevice ||
+ CGM.getLangOpts().OMPTargetTriples.empty())
+ return;
+
+ // Save the information in the declare target info map.
+ auto &Info = DeclareTargetEntryInfoMap[CGM.getMangledName(GlobalDecl(&D))];
+ Info.RequiresCtorDtor = true;
+ Info.VariableAddr = Addr;
+ Info.Variable = &D;
+ Info.PerformInitialization = PerformInit;
+
+ return;
+}
+
+void CGOpenMPRuntime::registerTargetFunctionDefinition(GlobalDecl GD) {
+ // We don't have to register anything if compiling for the host with no target
+ // devices specified.
+ if (!CGM.getLangOpts().OpenMPIsDevice &&
+ CGM.getLangOpts().OMPTargetTriples.empty())
+ return;
+
+ if (auto *FD = dyn_cast<FunctionDecl>(GD.getDecl())) {
+ auto &Info = DeclareTargetEntryInfoMap[CGM.getMangledName(GD)];
+ Info.Variable = FD;
+ }
+}
+
+void CGOpenMPRuntime::registerTargetVariableDefinition(const VarDecl *D,
+ llvm::Constant *Addr) {
+ // We don't have to register anything if compiling for the host with no target
+ // devices specified.
+ if (!CGM.getLangOpts().OpenMPIsDevice &&
+ CGM.getLangOpts().OMPTargetTriples.empty())
+ return;
+
+ assert(D && Addr && "Invalid variable information!");
+
+ auto &Info = DeclareTargetEntryInfoMap[CGM.getMangledName(GlobalDecl(D))];
+ Info.Variable = D;
+ Info.VariableAddr = Addr;
+}
+
+void CGOpenMPRuntime::registerGlobalReplacement(StringRef MangledName,
+ llvm::GlobalValue *NewVal) {
+ // We don't have to register anything if compiling for the host with no target
+ // devices specified.
+ if (!CGM.getLangOpts().OpenMPIsDevice &&
+ CGM.getLangOpts().OMPTargetTriples.empty())
+ return;
+
+ assert(MangledName != NewVal->getName() &&
+ "Replacing global with the same name??");
+
+ // If the existing global is registered, make sure the new global also is.
+ auto It = DeclareTargetEntryInfoMap.find(MangledName);
+ if (It == DeclareTargetEntryInfoMap.end())
+ return;
+
+ // Register the global. Copy the existing declare target record, but use the
+ // new address.
+ auto &Info = DeclareTargetEntryInfoMap[NewVal->getName()];
+ Info = It->second;
+ Info.VariableAddr = NewVal;
+}
+
llvm::Function *CGOpenMPRuntime::emitRegistrationFunction() {
+ auto &Ctx = CGM.getContext();
+
+ // Figure out all the declare target data that should be registered as such.
+ for (auto II = DeclareTargetEntryInfoMap.begin(),
+ IE = DeclareTargetEntryInfoMap.end();
+ II != IE; ++II) {
+ DeclareTargetEntryInfo &Info = II->second;
+
+ // Utility function to register the Ctors/Dtors.
+ auto &&RegisterCtorDtor = [&Info, &Ctx, this]() {
+ const VarDecl &D = *cast<VarDecl>(Info.Variable);
+ llvm::Constant *Addr = Info.VariableAddr;
+ assert(Addr && "declaration address is not defined??");
+
+ // Produce the unique prefix to identify the new target regions. We use
+ // the source location of the variable declaration which we know to not
+ // conflict with any target region.
+ unsigned DeviceID;
+ unsigned FileID;
+ unsigned Line;
+ getTargetEntryUniqueInfo(CGM.getContext(), D.getLocStart(), DeviceID,
+ FileID, Line);
+ SmallString<64> PrefixName;
+ {
+ llvm::raw_svector_ostream OS(PrefixName);
+ OS << "__omp_offloading" << llvm::format("_%x", DeviceID)
+ << llvm::format("_%x_", FileID) << D.getName() << "_l" << Line;
+ }
+
+ // If we have to perform an initialization, create a target region to
+ // launch that on the device.
+ if (Info.PerformInitialization) {
+ // Produce an ID whose address uniquely identifies the target region.
+ auto ID = new llvm::GlobalVariable(
+ CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
+ llvm::GlobalValue::PrivateLinkage,
+ llvm::Constant::getNullValue(CGM.Int8Ty),
+ Twine(PrefixName) + "_init");
+
+ // We define the parent name of this target region as the name of the
+ // global
+ // followed by the suffix _init. This suffix is what distinguishes a
+ // initializer from the destructor.
+ SmallString<128> ParentName;
+ ParentName += "__omp_offloading_init_";
+ ParentName += D.getName();
+
+ // Register the information for the entry associated with this target
+ // region.
+ OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
+ DeviceID, FileID, ParentName, Line, ID, ID,
+ OMP_DECLARE_TARGET_CTOR);
+ }
+
+ // Produce an ID whose address uniquely identifies the target dtor.
+ auto ID = new llvm::GlobalVariable(
+ CGM.getModule(), CGM.Int8Ty, /*isConstant=*/true,
+ llvm::GlobalValue::PrivateLinkage,
+ llvm::Constant::getNullValue(CGM.Int8Ty),
+ Twine(PrefixName) + "_dtor");
+
+ // We define the parent name of this target region as the name of
+ // the global followed by the suffix _dtor. This suffix is what
+ // distinguishes a initializer from the destructor.
+ SmallString<128> ParentName;
+ ParentName += "__omp_offloading_dtor_";
+ ParentName += D.getName();
+
+ // Register the information for the entry associated with this
+ // target
+ // region.
+ OffloadEntriesInfoManager.registerTargetRegionEntryInfo(
+ DeviceID, FileID, ParentName, Line, ID, ID, OMP_DECLARE_TARGET_DTOR);
+ };
+
+ // If we have a variable, register it and register any ctors/dtors entries.
+ if (auto *D = dyn_cast<VarDecl>(Info.Variable)) {
+ assert(Info.VariableAddr && "No variable address defined??");
+
+ if (auto *Attr = IsDeclareTargetDeclaration(D)) {
+ // If we have a link attribute we need to set the link flag.
+ int64_t Flags = 0;
+ if (Attr->getMapType() == OMPDeclareTargetDeclAttr::MT_Link)
+ Flags |= OMP_DECLARE_TARGET_LINK;
+
+ // Register the variable as declare target.
+ OffloadEntriesInfoManager.registerDeviceGlobalVarEntryInfo(
+ II->first(), Info.VariableAddr, D->getType(), Flags,
+ D->isExternallyVisible());
+
+ // Emit the ctor/dtor launching if required.
+ if (Info.RequiresCtorDtor)
+ RegisterCtorDtor();
+ }
+ continue;
+ }
+
+ const FunctionDecl *FD = cast<FunctionDecl>(Info.Variable);
+ // Only declare target functions are registered.
+ if (!IsDeclareTargetDeclaration(FD))
+ continue;
+
+ OffloadEntriesInfoManager.registerDeviceFunctionEntryInfo(II->first());
+ }
+
// If we have offloading in the current module, we need to emit the entries
// now and register the offloading descriptor.
createOffloadEntriesAndInfoMetadata();
@@ -7950,6 +8445,11 @@
}
}
+
+StringRef CGOpenMPRuntime::RenameStandardFunction(StringRef name) {
+ return name;
+}
+
namespace {
/// Cleanup action for doacross support.
class DoacrossCleanupTy final : public EHScopeStack::Cleanup {
@@ -8078,6 +8578,16 @@
emitCall(CGF, OutlinedFn, Args, Loc);
}
+
+void CGOpenMPRuntime::addTrackedFunction(StringRef MangledName, GlobalDecl GD) {
+ TrackedDecls[MangledName] = GD;
+}
+
+void CGOpenMPRuntime::registerTrackedFunction() {
+ for (auto &GD : TrackedDecls)
+ registerTargetFunctionDefinition(GD.second);
+}
+
Address CGOpenMPRuntime::getParameterAddress(CodeGenFunction &CGF,
const VarDecl *NativeParam,
const VarDecl *TargetParam) const {
@@ -8377,4 +8887,3 @@
const VarDecl *TargetParam) const {
llvm_unreachable("Not supported in SIMD-only mode");
}
-
Index: lib/CodeGen/CGDeclCXX.cpp
===================================================================
--- lib/CodeGen/CGDeclCXX.cpp
+++ lib/CodeGen/CGDeclCXX.cpp
@@ -62,7 +62,7 @@
/// Emit code to cause the destruction of the given variable with
/// static storage duration.
static void EmitDeclDestroy(CodeGenFunction &CGF, const VarDecl &D,
- ConstantAddress addr) {
+ ConstantAddress addr, bool EmitDtorOnly) {
CodeGenModule &CGM = CGF.CGM;
// FIXME: __attribute__((cleanup)) ?
@@ -115,6 +115,13 @@
argument = llvm::Constant::getNullValue(CGF.Int8PtrTy);
}
+ // Only emit the call if that was requested, otherwise register the destructor
+ // following the ABI rules.
+ if (EmitDtorOnly) {
+ CGF.Builder.CreateCall(function, argument);
+ return;
+ }
+
CGM.getCXXABI().registerGlobalDtor(CGF, D, function, argument);
}
@@ -142,7 +149,9 @@
void CodeGenFunction::EmitCXXGlobalVarDeclInit(const VarDecl &D,
llvm::Constant *DeclPtr,
- bool PerformInit) {
+ bool PerformInit,
+ bool EmitInitOnly,
+ bool EmitDtorOnly) {
const Expr *Init = D.getInit();
QualType T = D.getType();
@@ -179,12 +188,17 @@
&D, DeclAddr, D.getAttr<OMPThreadPrivateDeclAttr>()->getLocation(),
PerformInit, this);
}
- if (PerformInit)
+ if (PerformInit && !EmitDtorOnly)
EmitDeclInit(*this, D, DeclAddr);
+
+ // If all we need to emit is the initializer, we are done.
+ if (EmitInitOnly)
+ return;
+
if (CGM.isTypeConstant(D.getType(), true))
EmitDeclInvariant(*this, D, DeclPtr);
else
- EmitDeclDestroy(*this, D, DeclAddr);
+ EmitDeclDestroy(*this, D, DeclAddr, EmitDtorOnly);
return;
}
@@ -250,16 +264,18 @@
void CodeGenFunction::EmitCXXGuardedInit(const VarDecl &D,
llvm::GlobalVariable *DeclPtr,
- bool PerformInit) {
+ bool PerformInit, bool EmitInitOnly,
+ bool EmitDtorOnly) {
// If we've been asked to forbid guard variables, emit an error now.
// This diagnostic is hard-coded for Darwin's use case; we can find
// better phrasing if someone else needs it.
if (CGM.getCodeGenOpts().ForbidGuardVariables)
CGM.Error(D.getLocation(),
"this initialization requires a guard variable, which "
"the kernel does not support");
- CGM.getCXXABI().EmitGuardedInit(*this, D, DeclPtr, PerformInit);
+ CGM.getCXXABI().EmitGuardedInit(*this, D, DeclPtr, PerformInit, EmitInitOnly,
+ EmitDtorOnly);
}
void CodeGenFunction::EmitCXXGuardedInitBranch(llvm::Value *NeedsInit,
@@ -378,6 +394,11 @@
D->hasAttr<CUDASharedAttr>()))
return;
+ // If we are in OpenMP device mode we need to generate an entry point for each
+ // structor instead of the normal initialization.
+ if (OpenMPRuntime && OpenMPRuntime->emitDeviceCtorDtor(*D, Addr, PerformInit))
+ return;
+
// Check if we've already initialized this decl.
auto I = DelayedCXXInitPosition.find(D);
if (I != DelayedCXXInitPosition.end() && I->second == ~0U)
@@ -540,10 +561,9 @@
}
/// Emit the code necessary to initialize the given global variable.
-void CodeGenFunction::GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn,
- const VarDecl *D,
- llvm::GlobalVariable *Addr,
- bool PerformInit) {
+void CodeGenFunction::GenerateCXXGlobalVarDeclInitFunc(
+ llvm::Function *Fn, const VarDecl *D, llvm::GlobalVariable *Addr,
+ bool PerformInit, bool EmitInitOnly, bool EmitDtorOnly) {
// Check if we need to emit debug info for variable initializer.
if (D->hasAttr<NoDebugAttr>())
DebugInfo = nullptr; // disable debug info indefinitely for this function
@@ -559,11 +579,16 @@
// occurs for, e.g., instantiated static data members and
// definitions explicitly marked weak.
if (Addr->hasWeakLinkage() || Addr->hasLinkOnceLinkage()) {
- EmitCXXGuardedInit(*D, Addr, PerformInit);
+ EmitCXXGuardedInit(*D, Addr, PerformInit, EmitInitOnly, EmitDtorOnly);
} else {
- EmitCXXGlobalVarDeclInit(*D, Addr, PerformInit);
+ EmitCXXGlobalVarDeclInit(*D, Addr, PerformInit, EmitInitOnly, EmitDtorOnly);
}
+ // Register initializers and destructors for this variable.
+ if (CGM.getLangOpts().OpenMP)
+ CGM.getOpenMPRuntime().registerDeviceCtorDtorLaunching(*this, *D, Addr,
+ PerformInit);
+
FinishFunction();
}
Index: lib/CodeGen/CGCXXABI.h
===================================================================
--- lib/CodeGen/CGCXXABI.h
+++ lib/CodeGen/CGCXXABI.h
@@ -539,8 +539,8 @@
/// - a static local variable
/// - a static data member of a class template instantiation
virtual void EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D,
- llvm::GlobalVariable *DeclPtr,
- bool PerformInit) = 0;
+ llvm::GlobalVariable *DeclPtr, bool PerformInit,
+ bool EmitInitOnly, bool EmitDtorOnly) = 0;
/// Emit code to force the execution of a destructor during global
/// teardown. The default implementation of this uses atexit.
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
