erichkeane created this revision. GCC's attribute 'target', in addition to being an optimization hint, also allows function multiversioning. We currently have the former implemented, this is the latter's implementation.
Note that it ends up having to permit redefinition of functions so that they can all be emitted. Additionally, all versions of the function must be emitted, so this also manages that. Function templates are NOT supported (not supported in GCC either). Options on how to split this patch up would also be particularly solicited, since this IS a large patch. https://reviews.llvm.org/D38596 Files: include/clang/AST/Decl.h include/clang/Basic/Attr.td include/clang/Basic/DiagnosticSemaKinds.td include/clang/Basic/TargetInfo.h include/clang/Sema/Sema.h lib/Basic/Targets/X86.cpp lib/Basic/Targets/X86.h lib/CodeGen/CodeGenFunction.cpp lib/CodeGen/CodeGenFunction.h lib/CodeGen/CodeGenModule.cpp lib/CodeGen/CodeGenModule.h lib/Sema/SemaDecl.cpp test/CodeGen/attr-target-multiversion.c test/CodeGenCXX/attr-target-multiversion.cpp test/Sema/attr-target-multiversion.c test/SemaCXX/attr-target-multiversion.cpp
Index: test/SemaCXX/attr-target-multiversion.cpp =================================================================== --- /dev/null +++ test/SemaCXX/attr-target-multiversion.cpp @@ -0,0 +1,33 @@ +// RUN: %clang_cc1 -triple x86_64-linux -fsyntax-only -verify -emit-llvm-only %s + +struct S { + __attribute__((target("arch=sandybridge"))) + void mv(){} + __attribute__((target("arch=ivybridge"))) + void mv(){} + __attribute__((target("default"))) + void mv(){} +}; + +// Note: Template attribute 'target' isn't implemented in GCC either, and would +// end up causing some nasty issues attempting it, so ensure that it still gives +// the same errors as without the attribute. +template<typename T> +__attribute__((target("arch=sandybridge"))) +void mv_temp(){} + +template<typename T> +__attribute__((target("arch=ivybridge"))) +//expected-error@+2 {{redefinition of}} +//expected-note@-5{{previous definition is here}} +void mv_temp(){} + +template<typename T> +__attribute__((target("default"))) +void mv_temp(){} + +void foo() { + //expected-error@+2{{no matching function for call to}} + //expected-note@-8{{candidate template ignored}} + mv_temp<int>(); +} Index: test/Sema/attr-target-multiversion.c =================================================================== --- /dev/null +++ test/Sema/attr-target-multiversion.c @@ -0,0 +1,121 @@ +// RUN: %clang_cc1 -triple x86_64-linux -fsyntax-only -verify -emit-llvm-only %s +// RUN: %clang_cc1 -triple x86_64-linux -fsyntax-only -verify -emit-llvm-only -DCHECK_DEFAULT %s + +#if defined(CHECK_DEFAULT) +__attribute__((target("arch=sandybridge"))) +//expected-error@+1 {{function multiversioning with 'target' requires a 'default' implementation}} +void no_default(){} +__attribute__((target("arch=ivybridge"))) +void no_default(){} +#else +// Only multiversioning causes issues with redeclaration changing 'target'. +__attribute__((target("arch=sandybridge"))) +void fine_since_no_mv(); +void fine_since_no_mv(); + +void also_fine_since_no_mv(); +__attribute__((target("arch=sandybridge"))) +void also_fine_since_no_mv(); + +__attribute__((target("arch=sandybridge"))) +void also_fine_since_no_mv2(); +__attribute__((target("arch=sandybridge"))) +void also_fine_since_no_mv2(); +void also_fine_since_no_mv2(); + +__attribute__((target("arch=sandybridge"))) +void mv(){} +__attribute__((target("arch=ivybridge"))) +void mv(){} +__attribute__((target("default"))) +void mv(){} + +void redecl_causes_mv(); +__attribute__((target("arch=sandybridge"))) +void redecl_causes_mv(); +// expected-error@+3 {{function redeclaration causes a multiversioned function, but a previous declaration lacks a 'target' attribute}} +// expected-note@-4 {{previous declaration is here}} +__attribute__((target("arch=ivybridge"))) +void redecl_causes_mv(); + +__attribute__((target("arch=sandybridge"))) +void redecl_causes_mv2(); +void redecl_causes_mv2(); +// expected-error@+3 {{function redeclaration causes a multiversioned function, but a previous declaration lacks a 'target' attribute}} +// expected-note@-2 {{previous declaration is here}} +__attribute__((target("arch=ivybridge"))) +void redecl_causes_mv2(); + +__attribute__((target("arch=sandybridge"))) +void redecl_without_attr(); +__attribute__((target("arch=ivybridge"))) +void redecl_without_attr(); +// expected-error@+2 {{function redeclaration is missing 'target' attribute in a multiversioned function}} +// expected-note@-4 {{previous declaration is here}} +void redecl_without_attr(); + +__attribute__((target("arch=sandybridge"))) +void multiversion_with_predecl(); +__attribute__((target("default"))) +void multiversion_with_predecl(); +__attribute__((target("arch=sandybridge"))) +void multiversion_with_predecl(){} +__attribute__((target("default"))) +void multiversion_with_predecl(){} +//expected-error@+3 {{redefinition of 'multiversion_with_predecl'}} +//expected-note@-2 {{previous definition is here}} +__attribute__((target("arch=sandybridge"))) +void multiversion_with_predecl(){} + +__attribute__((target("arch=sandybridge"))) +void multiversion_with_predecl2(); +__attribute__((target("arch=sandybridge"))) +void multiversion_with_predecl2(){} +__attribute__((target("default"))) +void multiversion_with_predecl2(); +//expected-error@+3 {{redefinition of 'multiversion_with_predecl2'}} +//expected-note@-4 {{previous definition is here}} +__attribute__((target("arch=sandybridge"))) +void multiversion_with_predecl2(){} +__attribute__((target("default"))) +void multiversion_with_predecl2(){} + +// All the following are fine in normal 'target' mode, but not +// in multiversion mode. +// expected-error@+1 {{function multiversioning doesn't support feature 'sgx'}} +__attribute__((target("sgx"))) +void badMVFeature(); +__attribute__((target("sse"))) +void badMVFeature(); + +__attribute__((target("mmx"))) +void badMVFeature2(); +// expected-error@+1 {{function multiversioning doesn't support feature 'lwp'}} +__attribute__((target("lwp"))) +void badMVFeature2(); + +__attribute__((target("arch=ivybridge,mmx"))) +void badMVFeature3(); +// expected-error@+1 {{function multiversioning doesn't support feature 'lwp'}} +__attribute__((target("arch=skylake,lwp"))) +void badMVFeature3(); + +// expected-error@+1 {{function multiversioning doesn't support architecture 'k8'}} +__attribute__((target("arch=k8"))) +void badMVArch(); +__attribute__((target("arch=nehalem"))) +void badMVArch(); + +__attribute__((target("arch=nehalem"))) +void badMVArch2(); +// expected-error@+1 {{function multiversioning doesn't support architecture 'k8'}} +__attribute__((target("arch=k8"))) +void badMVArch2(); + +__attribute__((target("mmx"))) +void badMVNegate(); +// expected-error@+1 {{function multiversioning doesn't support feature 'no-mmx'}} +__attribute__((target("no-mmx"))) +void badMVNegate(); + +#endif Index: test/CodeGenCXX/attr-target-multiversion.cpp =================================================================== --- /dev/null +++ test/CodeGenCXX/attr-target-multiversion.cpp @@ -0,0 +1,28 @@ +// RUN: %clang_cc1 -triple x86_64-linux-gnu -target-cpu x86-64 -emit-llvm %s -o - | FileCheck %s + +struct S { + __attribute__((target("arch=sandybridge"))) + int mv(){return 3;} + __attribute__((target("arch=ivybridge"))) + int mv(){return 2;} + __attribute__((target("default"))) + int mv(){ return 1;} +}; + +// CHECK: @_ZN1S2mvEv.ifunc = ifunc i{{[0-9]+}} (%struct.S*), i{{[0-9]+}} (%struct.S*)* ()* @_ZN1S2mvEv.resolver + +// CHECK: define void @_Z3foov() +// CHECK: call i{{[0-9]+}} @_ZN1S2mvEv.ifunc(%struct.S* % +void foo() { + S s; + s.mv(); +} + +// CHECK: define i{{[0-9]+}} (%struct.S*)* @_ZN1S2mvEv.resolver +// CHECK: ret i{{[0-9]+}} (%struct.S*)* @_ZN1S2mvEv.arch_sandybridge +// CHECK: ret i{{[0-9]+}} (%struct.S*)* @_ZN1S2mvEv.arch_ivybridge +// CHECK: ret i{{[0-9]+}} (%struct.S*)* @_ZN1S2mvEv + +// CHECK-DAG: define linkonce_odr i{{[0-9]+}} @_ZN1S2mvEv.arch_sandybridge(%struct.S* %this) +// CHECK-DAG: define linkonce_odr i{{[0-9]+}} @_ZN1S2mvEv.arch_ivybridge(%struct.S* %this) +// CHECK-DAG: define linkonce_odr i{{[0-9]+}} @_ZN1S2mvEv(%struct.S* %this) Index: test/CodeGen/attr-target-multiversion.c =================================================================== --- /dev/null +++ test/CodeGen/attr-target-multiversion.c @@ -0,0 +1,85 @@ +// RUN: %clang_cc1 -triple x86_64-linux-gnu -target-cpu x86-64 -emit-llvm %s -o - | FileCheck %s + +//CHECK: @foo.ifunc = ifunc i8 (i{{[0-9]+}}), i8 (i{{[0-9]+}})* ()* @foo.resolver +//CHECK: @bar.ifunc = ifunc i8 (...), i8 (...)* ()* @bar.resolver +//CHECK: @baz.ifunc = ifunc i8 (), i8 ()* ()* @baz.resolver + +__attribute__((target("arch=knl"))) char foo(int); + +// This function causes emission of the resolver, check it. +// CHECK: define i8 (i{{[0-9]+}})* @foo.resolver() { +// CHECK: ret i8 (i{{[0-9]+}})* @foo.arch_knl +// AAACHECK: ret i8 (i{{[0-9]+}})* @foo.arch_knl +// AAACHECK: ret i8 (i{{[0-9]+}})* @foo.arch_skylake +// AAACHECK: ret i8 (i{{[0-9]+}})* @foo.arch_sse4.2 +// AAACHECK: ret i8 (i{{[0-9]+}})* @foo.arch_amdfam10 +// AAACHECK: ret i8 (i{{[0-9]+}})* @foo.arch_atom +// AAACHECK: ret i8 (i{{[0-9]+}})* @foo + +// CHECK: define signext i8 @foo( +__attribute__((target("default"))) char foo(int i) { + return 0 + i; +} + +// CHECK: define signext i8 @user1() +// CHECK: call signext i8 @foo.ifunc(i{{[0-9]+}} 1) +char user1(){ + return foo(1); +} + +// CHECK: define signext i8 @foo.sse4.2( +__attribute__((target("sse4.2"))) char foo(int i) { + return 1 + i; +} + +// CHECK: define signext i8 @user2() +// CHECK: call signext i8 @foo.ifunc(i{{[0-9]+}} 2) +char user2(){ + return foo(2); +} + +// CHECK: define signext i8 @foo.arch_atom( +__attribute__((target("arch=atom"))) char foo(int i) { + return 2 + i; +} + +__attribute__((target("arch=amdfam10"))) char foo(int); + +// CHECK: define signext i8 @user3() +// CHECK: call signext i8 @foo.ifunc(i{{[0-9]+}} 3) +char user3() { + return foo(3); +} + +__attribute__((target("arch=skylake"))) char foo(int); + +__attribute__((target("arch=knl"))) char bar(); + +__attribute__((target("default"))) char bar(); + + +// CHECK: define signext i8 @user4() +// CHECK: call signext i8 (...) @bar.ifunc() +char user4() { + return bar(); +} + +// CHECK: declare i8 (...)* @bar.resolver(){{$}} + +__attribute__((target("arch=knl"))) char baz(void); + +__attribute__((target("default"))) char baz(); + + +// CHECK: define signext i8 @user5() +// CHECK: call signext i8 @baz.ifunc() +char user5() { + return baz(); +} +// CHECK: declare i8 ()* @baz.resolver(){{$}} + +// Function Declarations are generated later, so ensure that they +// actually exist. +// CHECK-DAG: declare signext i8 @foo.arch_knl +// CHECK-DAG: declare signext i8 @foo.arch_amdfam10 +// CHECK-DAG: declare signext i8 @foo.arch_skylake Index: lib/Sema/SemaDecl.cpp =================================================================== --- lib/Sema/SemaDecl.cpp +++ lib/Sema/SemaDecl.cpp @@ -3211,7 +3211,7 @@ bool isFriend = NewMethod->getFriendObjectKind(); if (!isFriend && NewMethod->getLexicalDeclContext()->isRecord() && - !IsClassScopeExplicitSpecialization) { + !IsClassScopeExplicitSpecialization ) { // -- Member function declarations with the same name and the // same parameter types cannot be overloaded if any of them // is a static member function declaration. @@ -3221,28 +3221,34 @@ return true; } - // C++ [class.mem]p1: - // [...] A member shall not be declared twice in the - // member-specification, except that a nested class or member - // class template can be declared and then later defined. - if (!inTemplateInstantiation()) { - unsigned NewDiag; - if (isa<CXXConstructorDecl>(OldMethod)) - NewDiag = diag::err_constructor_redeclared; - else if (isa<CXXDestructorDecl>(NewMethod)) - NewDiag = diag::err_destructor_redeclared; - else if (isa<CXXConversionDecl>(NewMethod)) - NewDiag = diag::err_conv_function_redeclared; - else - NewDiag = diag::err_member_redeclared; + if (New->getMultiVersionKind() != + FunctionDecl::MultiVersionKind::MultiVersion) { + // Multiversioning allows multiple decls. + + // C++ [class.mem]p1: + // [...] A member shall not be declared twice in the + // member-specification, except that a nested class or member + // class template can be declared and then later defined. + if (!inTemplateInstantiation()) { + unsigned NewDiag; + if (isa<CXXConstructorDecl>(OldMethod)) + NewDiag = diag::err_constructor_redeclared; + else if (isa<CXXDestructorDecl>(NewMethod)) + NewDiag = diag::err_destructor_redeclared; + else if (isa<CXXConversionDecl>(NewMethod)) + NewDiag = diag::err_conv_function_redeclared; + else + NewDiag = diag::err_member_redeclared; - Diag(New->getLocation(), NewDiag); - } else { - Diag(New->getLocation(), diag::err_member_redeclared_in_instantiation) - << New << New->getType(); + Diag(New->getLocation(), NewDiag); + } else { + Diag(New->getLocation(), + diag::err_member_redeclared_in_instantiation) + << New << New->getType(); + } + Diag(OldLocation, PrevDiag) << Old << Old->getType(); + return true; } - Diag(OldLocation, PrevDiag) << Old << Old->getType(); - return true; // Complain if this is an explicit declaration of a special // member that was initially declared implicitly. @@ -9224,6 +9230,170 @@ D->getFriendObjectKind() != Decl::FOK_None); } +static void setMultiVersionKind(FunctionDecl *OldFD, FunctionDecl *NewFD, + FunctionDecl::MultiVersionKind Kind) { + for (FunctionDecl *F : OldFD->redecls()) + F->setMultiVersionKind(Kind); + NewFD->setMultiVersionKind(Kind); +} + +bool Sema::CheckMultiVersionOption(const FunctionDecl *FD) { + auto *TA = FD->getAttr<TargetAttr>(); + + if (TA->getFeaturesStr() == "default") + return true; + auto ParseInfo = TA->parse(); + + enum BadOption { BOFeature = 0, BOArch = 1 }; + + // Need to check isValidCPUName since it checks X86 vs X86-64 CPUs. From + // there, the subset of valid ones is captured in validateCpuIs. + if (!ParseInfo.Architecture.empty() && + (!Context.getTargetInfo().validateCpuIs(ParseInfo.Architecture) || + !Context.getTargetInfo().isValidCPUName(ParseInfo.Architecture))) { + Diag(TA->getLocation(), diag::err_bad_multiversion_option) + << BOArch << ParseInfo.Architecture; + return false; + } + + for (auto &Feature : ParseInfo.Features) { + auto BareFeat = StringRef{Feature}.substr(1); + // Negative options never allowed. + if (Feature[0] == '-') { + Diag(TA->getLocation(), diag::err_bad_multiversion_option) + << BOFeature << ("no-" + BareFeat).str(); + return false; + } + if (!Context.getTargetInfo().validateCpuSupports(BareFeat) || + !Context.getTargetInfo().isValidFeatureName(BareFeat)) { + Diag(TA->getLocation(), diag::err_bad_multiversion_option) + << BOFeature << BareFeat; + return false; + } + } + return true; +} + +bool Sema::CheckMultiVersionOptions(const FunctionDecl *OldFD, + const FunctionDecl *NewFD) { + // FIXME: Implement for more than X86. Requires CPUIs and CPUSupports + // for each new architecture. + auto Arch = Context.getTargetInfo().getTriple().getArch(); + if (Arch != llvm::Triple::x86_64 && Arch != llvm::Triple::x86) { + Diag(NewFD->getLocation(), diag::err_target_unimplemented_arch); + return false; + } + + bool result = true; + for (const auto *FD : OldFD->redecls()) + result = CheckMultiVersionOption(FD) && result; + + return CheckMultiVersionOption(NewFD) && result; +} + +bool Sema::CheckMultiVersionedDecl(FunctionDecl *NewFD, FunctionDecl *OldFD) { + // The first Decl is easy, it is either the 'All' case, or 'None'. + if (!OldFD) { + if (NewFD->hasAttr<TargetAttr>()) + NewFD->setMultiVersionKind(FunctionDecl::MultiVersionKind::All); + else + NewFD->setMultiVersionKind(FunctionDecl::MultiVersionKind::None); + return true; + } + + auto *TA = NewFD->getAttr<TargetAttr>(); + switch (OldFD->getMultiVersionKind()) { + default: + llvm_unreachable("Invalid State for Multiversioning."); + case FunctionDecl::MultiVersionKind::Error: + // An error case stays an error, mark the new Decl as well. + NewFD->setMultiVersionKind(FunctionDecl::MultiVersionKind::Error); + return false; + case FunctionDecl::MultiVersionKind::None: + // A previously unmarked function can convert to the "Partial" (hint-only) + // case if this one is marked. + if (!TA) { + // No change to the Previous Decls, set current to correct value. + NewFD->setMultiVersionKind(FunctionDecl::MultiVersionKind::None); + return true; + } + setMultiVersionKind(OldFD, NewFD, FunctionDecl::MultiVersionKind::Partial); + return true; + case FunctionDecl::MultiVersionKind::All: + // If all previous Decls have a target attribute, but none are different + // this can become a 'partial' if there is no attribute on this Decl, or it + // can become a multiversion if this is different. + if (!TA) { + setMultiVersionKind(OldFD, NewFD, + FunctionDecl::MultiVersionKind::Partial); + return true; + } + // Could potentially convert to a 'MultiVersion' state, check this + // case. Only have to check the last one, since it is the same as the + // previous ones. + if (TA->parse() != OldFD->getAttr<TargetAttr>()->parse()) { + if (!CheckMultiVersionOptions(OldFD, NewFD)) { + setMultiVersionKind(OldFD, NewFD, + FunctionDecl::MultiVersionKind::Error); + return false; + } + setMultiVersionKind(OldFD, NewFD, + FunctionDecl::MultiVersionKind::MultiVersion); + return true; + } + // If this doesn't convert it to a multi-version, this is clearly a 'all' + // case. + NewFD->setMultiVersionKind(FunctionDecl::MultiVersionKind::All); + return true; + case FunctionDecl::MultiVersionKind::MultiVersion: + // If the existing Decls are already in a MultiVersion case, a 'target' + // attribute is required on this one. Validate that it is legal on this one. + if (!TA) { + Diag(NewFD->getLocation(), diag::err_target_required_in_redecl); + Diag(OldFD->getFirstDecl()->getLocation(), + diag::note_previous_declaration); + setMultiVersionKind(OldFD, NewFD, FunctionDecl::MultiVersionKind::Error); + return false; + } else if (!CheckMultiVersionOption(NewFD)) { + setMultiVersionKind(OldFD, NewFD, FunctionDecl::MultiVersionKind::Error); + return false; + } + + // Otherwise, just continue the multiversion situation. + NewFD->setMultiVersionKind(FunctionDecl::MultiVersionKind::MultiVersion); + return true; + case FunctionDecl::MultiVersionKind::Partial: + // A partial not adding a Target Attribute is fine, everything stays as + // partial. + if (!TA) { + NewFD->setMultiVersionKind(FunctionDecl::MultiVersionKind::Partial); + return true; + } + // A partial situation is not allowed to convert to a MV, so ensure that + // this target doesn't cause that situation. + FunctionDecl *LastFeaturesStr = OldFD; + // Note: Null check shouldn't be necessary. + while (LastFeaturesStr && !LastFeaturesStr->hasAttr<TargetAttr>()) + LastFeaturesStr = LastFeaturesStr->getPreviousDecl(); + + assert(LastFeaturesStr && + "How can we be 'partial' without an existing target attribute?"); + if (TA->parse() != LastFeaturesStr->getAttr<TargetAttr>()->parse()) { + Diag(NewFD->getLocation(), + diag::err_target_causes_illegal_multiversioning); + // Note on all previous decls that lack a target attr. + for (const auto *Cur : OldFD->redecls()) + if (!Cur->hasAttr<TargetAttr>() || + Cur->getAttr<TargetAttr>()->isInherited()) + Diag(Cur->getLocation(), diag::note_previous_declaration); + setMultiVersionKind(OldFD, NewFD, FunctionDecl::MultiVersionKind::Error); + return false; + } + NewFD->setMultiVersionKind(FunctionDecl::MultiVersionKind::Partial); + return true; + } +} + /// \brief Perform semantic checking of a new function declaration. /// /// Performs semantic analysis of the new function declaration @@ -9349,6 +9519,12 @@ } } + // Attribute target requires all declarations to have the 'target' + // attribute. + if (!CheckMultiVersionedDecl(NewFD, OldDecl ? dyn_cast<FunctionDecl>(OldDecl) + : nullptr)) + NewFD->setInvalidDecl(); + if (Redeclaration) { // NewFD and OldDecl represent declarations that need to be // merged. @@ -11997,6 +12173,28 @@ return MissingPrototype; } +// Check to see if this 'redefinition' should be allowed in order to support +// attribute 'target' redefinition. +static bool isValidMultiVersion(FunctionDecl *NewFD, + const FunctionDecl *ExistingFD) { + if (NewFD->getMultiVersionKind() != + FunctionDecl::MultiVersionKind::MultiVersion) + return false; + + TargetAttr *NewTA = NewFD->getAttr<TargetAttr>(); + const auto NewTAParsed = NewTA->parse(); + // Prohibit duplicate definitions. + for (const auto *FD : ExistingFD->redecls()) { + if (FD->isThisDeclarationADefinition()) { + TargetAttr *TA = FD->getAttr<TargetAttr>(); + if (TA && !TA->isInherited() && + TA->parse() == NewTAParsed) + return false; + } + } + return true; +} + void Sema::CheckForFunctionRedefinition(FunctionDecl *FD, const FunctionDecl *EffectiveDefinition, @@ -12009,6 +12207,9 @@ if (canRedefineFunction(Definition, getLangOpts())) return; + if (isValidMultiVersion(FD, Definition)) + return; + // Don't emit an error when this is redefinition of a typo-corrected // definition. if (TypoCorrectedFunctionDefinitions.count(Definition)) Index: lib/CodeGen/CodeGenModule.h =================================================================== --- lib/CodeGen/CodeGenModule.h +++ lib/CodeGen/CodeGenModule.h @@ -323,6 +323,7 @@ /// List of alias we have emitted. Used to make sure that what they point to /// is defined once we get to the end of the of the translation unit. std::vector<GlobalDecl> Aliases; + std::vector<GlobalDecl> MultiVersionFuncs; typedef llvm::StringMap<llvm::TrackingVH<llvm::Constant> > ReplacementsTy; ReplacementsTy Replacements; @@ -743,6 +744,7 @@ llvm::Type *Ty = nullptr, ForDefinition_t IsForDefinition = NotForDefinition); + std::string MakeMultiVersionName(StringRef MangledName, const Decl *D); /// Return the address of the given function. If Ty is non-null, then this /// function will use the specified type if it has to create it. @@ -1221,6 +1223,11 @@ bool DontDefer = false, bool IsThunk = false, llvm::AttributeList ExtraAttrs = llvm::AttributeList(), ForDefinition_t IsForDefinition = NotForDefinition); + llvm::Constant *GetOrCreateLLVMFunctionOrMultiVersion( + StringRef MangledName, llvm::Type *Ty, GlobalDecl D, bool ForVTable, + bool DontDefer = false, bool IsThunk = false, + llvm::AttributeList ExtraAttrs = llvm::AttributeList(), + ForDefinition_t IsForDefinition = NotForDefinition); llvm::Constant *GetOrCreateLLVMGlobal(StringRef MangledName, llvm::PointerType *PTy, @@ -1236,8 +1243,14 @@ ForDefinition_t IsForDefinition); void EmitGlobalDefinition(GlobalDecl D, llvm::GlobalValue *GV = nullptr); + void EmitMultiVersionResolver(StringRef ResolverName, const FunctionDecl *FD); void EmitGlobalFunctionDefinition(GlobalDecl GD, llvm::GlobalValue *GV); + llvm::GlobalValue *GetOrCreateMultiVersionIFunc(GlobalDecl GD, + llvm::Type *DeclTy, + StringRef MangledName, + const FunctionDecl *FD, + llvm::GlobalValue *GV); void EmitGlobalVarDefinition(const VarDecl *D, bool IsTentative = false); void EmitAliasDefinition(GlobalDecl GD); void emitIFuncDefinition(GlobalDecl GD); @@ -1293,6 +1306,7 @@ void applyGlobalValReplacements(); void checkAliases(); + void checkMultiversions(); /// Emit any vtables which we deferred and still have a use for. void EmitDeferredVTables(); Index: lib/CodeGen/CodeGenModule.cpp =================================================================== --- lib/CodeGen/CodeGenModule.cpp +++ lib/CodeGen/CodeGenModule.cpp @@ -285,6 +285,49 @@ } } +void CodeGenModule::EmitMultiVersionResolver(StringRef MangledName, + const FunctionDecl *FD) { + llvm::Function *ResolverFunc = + cast<llvm::Function>(GetGlobalValue((MangledName + ".resolver").str())); + SmallVector<CodeGenFunction::ResolverOption, 8> ResolverOptions; + for (FunctionDecl *CurFD : FD->redecls()) { + std::string FuncName = MakeMultiVersionName(MangledName, CurFD); + llvm::Constant *Func = GetGlobalValue(FuncName); + if (!Func) { + GlobalDecl GD{CurFD}; + if (CurFD->isThisDeclarationADefinition()) { + // If the function is defined, make sure we emit it. + EmitGlobalDefinition(GD); + Func = GetGlobalValue(FuncName); + } else { + // We still need to create declarations, since the resolver needs them. + const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD); + llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); + Func = GetAddrOfFunction(GD, Ty, /*ForVTable=*/false, + /*DontDefer=*/false, ForDefinition); + } + assert(Func && "This should have just been created"); + } + + ResolverOptions.emplace_back(CurFD->getAttr<TargetAttr>()->parse(), + cast<llvm::Function>(Func)); + } + + // Sort resolver options. + auto *TargetInfo = &Context.getTargetInfo(); + auto CmpFunc = [TargetInfo](StringRef LHS, StringRef RHS) { + return TargetInfo->compareCpusAndFeatures(LHS, RHS); + }; + std::sort(std::begin(ResolverOptions), std::end(ResolverOptions), + [CmpFunc](const CodeGenFunction::ResolverOption &LHS, + const CodeGenFunction::ResolverOption &RHS) { + return CmpFunc(LHS.AttrInfo.getHighestPriority(CmpFunc), + RHS.AttrInfo.getHighestPriority(CmpFunc)); + }); + CodeGenFunction CGF(*this); + CGF.EmitMultiVersionResolver(ResolverFunc, ResolverOptions); +} + void CodeGenModule::checkAliases() { // Check if the constructed aliases are well formed. It is really unfortunate // that we have to do this in CodeGen, but we only construct mangled names @@ -362,6 +405,40 @@ } } +void CodeGenModule::checkMultiversions() { + // Cleanup the IFuncs calls for multiversioning. + for (const GlobalDecl &GD : MultiVersionFuncs) { + bool DefaultFound = false; + for (auto *Decl : GD.getDecl()->getMostRecentDecl()->redecls()) + if (Decl->getAttr<TargetAttr>()->getFeaturesStr() == "default") + DefaultFound = true; + + StringRef MangledName = getMangledName(GD); + llvm::GlobalValue *BaseFunc = GetGlobalValue(MangledName); + llvm::GlobalValue *IFunc = GetGlobalValue((MangledName + ".ifunc").str()); + assert(IFunc && "Multiversioning not legal without an IFunc"); + if (!DefaultFound && cast<FunctionDecl>(GD.getDecl())->isDefined()) { + Diags.Report( + cast<FunctionDecl>(GD.getDecl())->getFirstDecl()->getLocation(), + diag::err_target_without_default); + // In the error case, replace all uses with an undef. + if (BaseFunc) { + BaseFunc->replaceAllUsesWith( + llvm::UndefValue::get(BaseFunc->getType())); + BaseFunc->eraseFromParent(); + } + continue; + } + + if (cast<FunctionDecl>(GD.getDecl())->isDefined()) + EmitMultiVersionResolver( + MangledName, cast<FunctionDecl>(GD.getDecl()->getMostRecentDecl())); + if (BaseFunc) + BaseFunc->replaceAllUsesWith( + llvm::ConstantExpr::getBitCast(IFunc, BaseFunc->getType())); + } +} + void CodeGenModule::clear() { DeferredDeclsToEmit.clear(); if (OpenMPRuntime) @@ -391,6 +468,7 @@ applyGlobalValReplacements(); applyReplacements(); checkAliases(); + checkMultiversions(); EmitCXXGlobalInitFunc(); EmitCXXGlobalDtorFunc(); EmitCXXThreadLocalInitFunc(); @@ -2032,6 +2110,73 @@ static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old, llvm::Function *NewFn); +std::string CodeGenModule::MakeMultiVersionName(StringRef MangledName, + const Decl *D) { + assert(D && "Decl cannot be null"); + const auto *TA = D->getAttr<TargetAttr>(); + assert(TA && "Decl must have a Target Attribute"); + auto Append = TA->multiVersionMangleAddition(); + if (Append.empty()) + return MangledName; + return (MangledName + "." + Append).str(); +} + +llvm::GlobalValue *CodeGenModule::GetOrCreateMultiVersionIFunc( + GlobalDecl GD, llvm::Type *DeclTy, StringRef MangledName, + const FunctionDecl *FD, llvm::GlobalValue *GV) { + std::string IFuncName = (MangledName + ".ifunc").str(); + llvm::GlobalValue *IFuncGV = GetGlobalValue(IFuncName); + + if (IFuncGV) + return IFuncGV; + + // Since this is the first time we've created this IFunc, make sure + // that we put this multiversioned function into the list to be + // replaced later. + MultiVersionFuncs.push_back(GD); + + std::string ResolverName = (MangledName + ".resolver").str(); + llvm::Type *ResolverType = llvm::FunctionType::get( + llvm::PointerType::get(DeclTy, + Context.getTargetAddressSpace(FD->getType())), + false); + llvm::Constant *Resolver = + GetOrCreateLLVMFunction(ResolverName, ResolverType, GlobalDecl{}, + /*ForVTable=*/false); + llvm::GlobalIFunc *GIF = llvm::GlobalIFunc::create( + DeclTy, 0, llvm::Function::ExternalLinkage, "", Resolver, &getModule()); + GIF->setName(IFuncName); + SetCommonAttributes(FD, GIF); + + return GIF; +} + +/// GetOrCreateLLVMFunctionOrMultiVersion - Wrapper for GetOrCreateLLVMFunction +/// that will also check to see if this is a FunctionDecl that is +/// multiversioned. In the case where this is NOT for a definition, this will +/// create the llvm GlobalValue, but return the iFunc. If this FOR definition, +/// it will return the newly mangled name. +llvm::Constant *CodeGenModule::GetOrCreateLLVMFunctionOrMultiVersion( + StringRef MangledName, llvm::Type *Ty, GlobalDecl GD, bool ForVTable, + bool DontDefer, bool IsThunk, llvm::AttributeList ExtraAttrs, + ForDefinition_t IsForDefinition) { + const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl()); + std::string MVName; + + if (FD->getMultiVersionKind() == + FunctionDecl::MultiVersionKind::MultiVersion) { + llvm::GlobalValue *ExistingDef = GetGlobalValue(MangledName); + GetOrCreateMultiVersionIFunc(GD, Ty, MangledName, FD, ExistingDef); + if (IsForDefinition == ForDefinition) { + MVName = MakeMultiVersionName(MangledName, FD); + MangledName = MVName; + } + } + + return GetOrCreateLLVMFunction(MangledName, Ty, GD, ForVTable, DontDefer, + IsThunk, ExtraAttrs, IsForDefinition); +} + /// GetOrCreateLLVMFunction - If the specified mangled name is not in the /// module, create and return an llvm Function with the specified type. If there /// is something in the module with the specified name, return it potentially @@ -2209,9 +2354,9 @@ } StringRef MangledName = getMangledName(GD); - return GetOrCreateLLVMFunction(MangledName, Ty, GD, ForVTable, DontDefer, - /*IsThunk=*/false, llvm::AttributeList(), - IsForDefinition); + return GetOrCreateLLVMFunctionOrMultiVersion( + MangledName, Ty, GD, ForVTable, DontDefer, + /*IsThunk=*/false, llvm::AttributeList(), IsForDefinition); } static const FunctionDecl * Index: lib/CodeGen/CodeGenFunction.h =================================================================== --- lib/CodeGen/CodeGenFunction.h +++ lib/CodeGen/CodeGenFunction.h @@ -3701,6 +3701,16 @@ /// point operation, expressed as the maximum relative error in ulp. void SetFPAccuracy(llvm::Value *Val, float Accuracy); + struct ResolverOption { + TargetAttr::ParsedTargetAttr AttrInfo; + llvm::Function *Function; + ResolverOption(TargetAttr::ParsedTargetAttr &&AttrInfo,// can be moved? + llvm::Function *Function) + : AttrInfo(std::move(AttrInfo)), Function(Function) {} + }; + void EmitMultiVersionResolver(llvm::Function *ResolverFunc, + ArrayRef<ResolverOption> ResolverOptions); + private: llvm::MDNode *getRangeForLoadFromType(QualType Ty); void EmitReturnOfRValue(RValue RV, QualType Ty); @@ -3876,6 +3886,7 @@ llvm::Value *EmitX86CpuIs(StringRef CPUStr); llvm::Value *EmitX86CpuSupports(const CallExpr *E); llvm::Value *EmitX86CpuSupports(ArrayRef<StringRef> FeatureStrs); + llvm::Value *FormResolverCondition(const ResolverOption &RO); }; /// Helper class with most of the code for saving a value for a Index: lib/CodeGen/CodeGenFunction.cpp =================================================================== --- lib/CodeGen/CodeGenFunction.cpp +++ lib/CodeGen/CodeGenFunction.cpp @@ -2274,3 +2274,56 @@ return llvm::DebugLoc(); } + +llvm::Value * +CodeGenFunction::FormResolverCondition(const ResolverOption &RO) { + llvm::Value *TrueCondition = nullptr; + if (!RO.AttrInfo.Architecture.empty()) + TrueCondition = EmitX86CpuIs(RO.AttrInfo.Architecture); + + if (!RO.AttrInfo.Features.empty()) { + SmallVector<StringRef, 8> FeatureList; + std::for_each(std::begin(RO.AttrInfo.Features), + std::end(RO.AttrInfo.Features), + [&FeatureList](const std::string &Feature) { + FeatureList.push_back(StringRef{Feature}.substr(1)); + }); + llvm::Value *FeatureCmp = EmitX86CpuSupports(FeatureList); + TrueCondition = TrueCondition + ? Builder.CreateAnd(TrueCondition, FeatureCmp) + : FeatureCmp; + } + return TrueCondition; +} + +void CodeGenFunction::EmitMultiVersionResolver( + llvm::Function *ResolverFunc, ArrayRef<ResolverOption> ResolverOptions) { + assert((getContext().getTargetInfo().getTriple().getArch() == + llvm::Triple::x86 || + getContext().getTargetInfo().getTriple().getArch() == + llvm::Triple::x86_64) && + "Only implemented for x86 targets"); + + // Main function's basic block. + llvm::BasicBlock *CurBlock = createBasicBlock("entry", ResolverFunc); + + llvm::Function *DefaultFunc = nullptr; + for (const ResolverOption &RO : ResolverOptions) { + Builder.SetInsertPoint(CurBlock); + llvm::Value *TrueCondition = FormResolverCondition(RO); + + if (!TrueCondition) { + DefaultFunc = RO.Function; + } else { + llvm::BasicBlock *RetBlock = createBasicBlock("ro_ret", ResolverFunc); + llvm::IRBuilder<> RetBuilder(RetBlock); + RetBuilder.CreateRet(RO.Function); + CurBlock = createBasicBlock("ro_else", ResolverFunc); + Builder.CreateCondBr(TrueCondition, RetBlock, CurBlock); + } + } + assert(DefaultFunc && "No default version?"); + // Emit return from the 'else-ist' block. + Builder.SetInsertPoint(CurBlock); + Builder.CreateRet(DefaultFunc); +} Index: lib/Basic/Targets/X86.h =================================================================== --- lib/Basic/Targets/X86.h +++ lib/Basic/Targets/X86.h @@ -364,6 +364,8 @@ bool validateCpuIs(StringRef Name) const override; + bool compareCpusAndFeatures(StringRef LHS, StringRef RHS) const override; + bool validateAsmConstraint(const char *&Name, TargetInfo::ConstraintInfo &info) const override; Index: lib/Basic/Targets/X86.cpp =================================================================== --- lib/Basic/Targets/X86.cpp +++ lib/Basic/Targets/X86.cpp @@ -1292,7 +1292,9 @@ return llvm::StringSwitch<bool>(FeatureStr) .Case("amd", true) .Case("amdfam10h", true) + .Case("amdfam10", true) .Case("amdfam15h", true) + .Case("amdfam15", true) .Case("atom", true) .Case("barcelona", true) .Case("bdver1", true) @@ -1322,6 +1324,90 @@ .Default(false); } +bool X86TargetInfo::compareCpusAndFeatures(StringRef Lhs, StringRef Rhs) const { + // The following are in order from new to old. These are required for the + // ordering of the GCC Target attribute. + const auto TargetArray = {"avx512vpopcntdq", + "avx5124fmaps", + "avx5124vnniw", + "avx512ifma", + "avx512vbmi", + "avx512pf", + "avx512er", + "avx512cd", + "avx512dq", + "avx512bw", + "avx512vl", + "knl", + "avx512f", + "znver1", + "skylake-avx512", + "skylake", + "haswell", + "broadwell", + "bdver4", + "avx2", + "bmi2", + "bdver3", + "bdver2", + "fma", + "bdver1", + "amdfam15h", + "amdfam15", + "xop", + "fma4", + "btver2", + "bmi", + "sandybridge", + "ivybridge", + "avx", + "pclmul", + "aes", + "popcnt", + "westmere", + "slm", + "silvermont", + "nehalem", + "corei7", + "sse4.2", + "sse4.1", + "btver1", + "istanbul", + "shanghai", + "barcelona", + "amdfam10h", + "amdfam10", + "sse4a", + "core2", + "bonnell", + "atom", + "ssse3", + "sse3", + "sse2", + "sse", + "mmx", + "cmov", + "intel", + "amd"}; + + if (Lhs == Rhs) + return false; + // Default location doesn't matter except that sort should be repeatable, but + // move default to the end. + if (Lhs.empty()) + return false; + if (Rhs.empty()) + return true; + + for (auto &Str : TargetArray) { + if (Lhs == Str) + return true; + else if (Rhs == Str) + return false; + } + assert(false && "Invalid item passed to compareCpusAndFeatures"); +} + bool X86TargetInfo::validateAsmConstraint( const char *&Name, TargetInfo::ConstraintInfo &Info) const { switch (*Name) { @@ -1563,7 +1649,7 @@ .Cases("amdfam10", "barcelona", CK_AMDFAM10) .Case("btver1", CK_BTVER1) .Case("btver2", CK_BTVER2) - .Case("bdver1", CK_BDVER1) + .Cases("amdfam15", "bdver1", CK_BDVER1) .Case("bdver2", CK_BDVER2) .Case("bdver3", CK_BDVER3) .Case("bdver4", CK_BDVER4) Index: include/clang/Sema/Sema.h =================================================================== --- include/clang/Sema/Sema.h +++ include/clang/Sema/Sema.h @@ -1932,6 +1932,15 @@ SmallVectorImpl<CXXMethodDecl*> &OverloadedMethods); void NoteHiddenVirtualMethods(CXXMethodDecl *MD, SmallVectorImpl<CXXMethodDecl*> &OverloadedMethods); + + // Checks to see if attribute 'target' results in a valid situation. + bool CheckMultiVersionedDecl(FunctionDecl *NewFD, FunctionDecl *OldFD); + // Helper function for CheckMultiVersionDecl to ensure that a declaration + // causing a Multi-version case doesn't cause invalid options. + bool CheckMultiVersionOptions(const FunctionDecl *NewFD, + const FunctionDecl *OldFD); + bool CheckMultiVersionOption(const FunctionDecl *FD); + // Returns true if the function declaration is a redeclaration bool CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD, LookupResult &Previous, Index: include/clang/Basic/TargetInfo.h =================================================================== --- include/clang/Basic/TargetInfo.h +++ include/clang/Basic/TargetInfo.h @@ -922,6 +922,12 @@ // argument. virtual bool validateCpuIs(StringRef Name) const { return false; } + // \brief Provide ordering for the CPU Features and Names such that it can + // used for dispatchers. + virtual bool compareCpusAndFeatures(StringRef Lhs, StringRef Rhs) const { + return false; + } + // \brief Returns maximal number of args passed in registers. unsigned getRegParmMax() const { assert(RegParmMax < 7 && "RegParmMax value is larger than AST can handle"); Index: include/clang/Basic/DiagnosticSemaKinds.td =================================================================== --- include/clang/Basic/DiagnosticSemaKinds.td +++ include/clang/Basic/DiagnosticSemaKinds.td @@ -9294,4 +9294,18 @@ InGroup<ShadowField>, DefaultIgnore; def note_shadow_field : Note<"declared here">; +def err_target_required_in_redecl + : Error<"function redeclaration is missing 'target' attribute in a " + "multiversioned function">; +def err_target_causes_illegal_multiversioning + : Error<"function redeclaration causes a multiversioned function, but a " + "previous declaration lacks a 'target' attribute">; +def err_target_unimplemented_arch + : Error<"function multiversioning with 'target' is only supported on " + "X86/x86_64 architectures">; +def err_bad_multiversion_option + : Error<"function multiversioning doesn't support " + "%select{feature|architecture}0 '%1'">; +def err_target_without_default : Error<"function multiversioning with 'target' " + "requires a 'default' implementation">; } // end of sema component. Index: include/clang/Basic/Attr.td =================================================================== --- include/clang/Basic/Attr.td +++ include/clang/Basic/Attr.td @@ -1826,12 +1826,39 @@ std::vector<std::string> Features; StringRef Architecture; bool DuplicateArchitecture = false; + + template <typename Comparer> + StringRef getHighestPriority(Comparer &Cmp) const { + // Default case. + if (Features.empty() && Architecture.empty()) + return {}; + StringRef Highest; + for (const auto &Feat : Features) { + if (Feat[0] == '+' && + (Highest.empty() || Cmp(StringRef{Feat}.substr(1), Highest))) + Highest = StringRef{Feat}.substr(1); + } + + if (Highest.empty() || Cmp(Architecture, Highest)) + return Architecture; + return Highest; + } + + bool operator ==(const ParsedTargetAttr &Other) const { + return std::tie(Features, Architecture) == + std::tie(Other.Features, Other.Architecture); + } + + bool operator !=(const ParsedTargetAttr &Other) const { + return !(*this == Other); + } }; ParsedTargetAttr parse() const { return parse(getFeaturesStr()); } static ParsedTargetAttr parse(StringRef Features) { ParsedTargetAttr Ret; + if (Features == "default") return Ret; SmallVector<StringRef, 1> AttrFeatures; Features.split(AttrFeatures, ","); @@ -1862,6 +1889,27 @@ } return Ret; } + std::string multiVersionMangleAddition() const { + // Simply 'default' is empty. + if (getFeaturesStr() == "default") + return ""; + SmallString<64> Buffer; + llvm::raw_svector_ostream OS(Buffer); + auto Info = parse(); + + // arch= becomes arch_, ',' separator becomes '.'. + // Arch goes first, everything else is alphabetical. + if (!Info.Architecture.empty()) + OS << "arch_" << Info.Architecture; + + std::sort(Info.Features.begin(), Info.Features.end()); + for (auto &&Feat : Info.Features) { + if (!Buffer.empty()) + OS << '_'; + OS << StringRef{Feat}.substr(1); + } + return Buffer.str(); + } }]; } Index: include/clang/AST/Decl.h =================================================================== --- include/clang/AST/Decl.h +++ include/clang/AST/Decl.h @@ -1637,6 +1637,17 @@ TK_FunctionTemplateSpecialization, TK_DependentFunctionTemplateSpecialization }; + enum class MultiVersionKind { + None = 0, // No Decl of this function has a 'target' attribute. + All, // All Decls of this function have a 'target' attribute. None differ + // in contents, so this is the 'hint' case. + MultiVersion, // All Decls of this function have a 'target' attribute, some + // with different strings. This causes a multi-version case. + Partial, // This is NOT a MV case. All Decls with a 'target' attribute + // match, so this is simply the 'hint' case. + Error // A previous decl caused an invalid MultiVersion, so reject + // all further attempts at multiversioning/target. + }; private: /// ParamInfo - new[]'d array of pointers to VarDecls for the formal @@ -1679,6 +1690,7 @@ /// Indicates if the function declaration will have a body, once we're done /// parsing it. unsigned WillHaveBody : 1; + unsigned MultiVersion : 3; /// \brief End part of this FunctionDecl's source range. /// @@ -1764,6 +1776,7 @@ IsLateTemplateParsed(false), IsConstexpr(isConstexprSpecified), InstantiationIsPending(false), UsesSEHTry(false), HasSkippedBody(false), WillHaveBody(false), + MultiVersion(static_cast<unsigned>(MultiVersionKind::None)), EndRangeLoc(NameInfo.getEndLoc()), TemplateOrSpecialization(), DNLoc(NameInfo.getInfo()) {} @@ -2208,6 +2221,14 @@ /// \brief What kind of templated function this is. TemplatedKind getTemplatedKind() const; + /// \brief The multiversion state of this function. + MultiVersionKind getMultiVersionKind() const { + return static_cast<MultiVersionKind>(this->MultiVersion); + } + void setMultiVersionKind(MultiVersionKind MV) { + this->MultiVersion = static_cast<unsigned>(MV); + } + /// \brief If this function is an instantiation of a member function of a /// class template specialization, retrieves the member specialization /// information.
_______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits