Index: include/clang/AST/Decl.h =================================================================== --- include/clang/AST/Decl.h +++ include/clang/AST/Decl.h @@ -1637,6 +1637,17 @@ TK_FunctionTemplateSpecialization, TK_DependentFunctionTemplateSpecialization }; + enum class MultiVersionKind { + None = 0, // No Decl of this function has a 'target' attribute. + All, // All Decls of this function have a 'target' attribute. None differ + // in contents, so this is the 'hint' case. + MultiVersion, // All Decls of this function have a 'target' attribute, some + // with different strings. This causes a multi-version case. + Partial, // This is NOT a MV case. All Decls with a 'target' attribute + // match, so this is simply the 'hint' case. + Error // A previous decl caused an invalid MultiVersion, so reject + // all further attempts at multiversioning/target. + }; private: /// ParamInfo - new[]'d array of pointers to VarDecls for the formal @@ -1679,6 +1690,7 @@ /// Indicates if the function declaration will have a body, once we're done /// parsing it. unsigned WillHaveBody : 1; + unsigned MultiVersion : 3; /// \brief End part of this FunctionDecl's source range. /// @@ -1764,6 +1776,7 @@ IsLateTemplateParsed(false), IsConstexpr(isConstexprSpecified), InstantiationIsPending(false), UsesSEHTry(false), HasSkippedBody(false), WillHaveBody(false), + MultiVersion(static_cast(MultiVersionKind::None)), EndRangeLoc(NameInfo.getEndLoc()), TemplateOrSpecialization(), DNLoc(NameInfo.getInfo()) {} @@ -2208,6 +2221,14 @@ /// \brief What kind of templated function this is. TemplatedKind getTemplatedKind() const; + /// \brief The multiversion state of this function. + MultiVersionKind getMultiVersionKind() const { + return static_cast(MultiVersion); + } + void setMultiVersionKind(MultiVersionKind MV) { + MultiVersion = static_cast(MV); + } + /// \brief If this function is an instantiation of a member function of a /// class template specialization, retrieves the member specialization /// information. Index: include/clang/Basic/Attr.td =================================================================== --- include/clang/Basic/Attr.td +++ include/clang/Basic/Attr.td @@ -1826,12 +1826,39 @@ std::vector Features; StringRef Architecture; bool DuplicateArchitecture = false; + + template + StringRef getHighestPriority(Comparer &Cmp) const { + // Default case. + if (Features.empty() && Architecture.empty()) + return {}; + StringRef Highest; + for (const auto &Feat : Features) { + if (Feat[0] == '+' && + (Highest.empty() || Cmp(StringRef{Feat}.substr(1), Highest))) + Highest = StringRef{Feat}.substr(1); + } + + if (Highest.empty() || Cmp(Architecture, Highest)) + return Architecture; + return Highest; + } + + bool operator==(const ParsedTargetAttr &Other) const { + return std::tie(Features, Architecture) == + std::tie(Other.Features, Other.Architecture); + } + + bool operator!=(const ParsedTargetAttr &Other) const { + return !(*this == Other); + } }; ParsedTargetAttr parse() const { return parse(getFeaturesStr()); } static ParsedTargetAttr parse(StringRef Features) { ParsedTargetAttr Ret; + if (Features == "default") return Ret; SmallVector AttrFeatures; Features.split(AttrFeatures, ","); @@ -1862,6 +1889,27 @@ } return Ret; } + std::string multiVersionMangleAddition() const { + // Simply 'default' is empty. + if (getFeaturesStr() == "default") + return ""; + SmallString<64> Buffer; + llvm::raw_svector_ostream OS(Buffer); + ParsedTargetAttr Info = parse(); + + // arch= becomes arch_, ',' separator becomes '.'. + // Arch goes first, everything else is alphabetical. + if (!Info.Architecture.empty()) + OS << "arch_" << Info.Architecture; + + std::sort(Info.Features.begin(), Info.Features.end()); + for (auto &&Feat : Info.Features) { + if (!Buffer.empty()) + OS << '_'; + OS << StringRef{Feat}.substr(1); + } + return Buffer.str(); + } }]; } Index: include/clang/Basic/DiagnosticSemaKinds.td =================================================================== --- include/clang/Basic/DiagnosticSemaKinds.td +++ include/clang/Basic/DiagnosticSemaKinds.td @@ -9294,4 +9294,20 @@ InGroup, DefaultIgnore; def note_shadow_field : Note<"declared here">; +def err_target_required_in_redecl + : Error<"function redeclaration is missing 'target' attribute in a " + "multiversioned function">; +def err_target_causes_illegal_multiversioning + : Error<"function redeclaration declares a multiversioned function, but a " + "previous declaration lacks a 'target' attribute">; +def err_target_unimplemented_arch + : Error<"function multiversioning with 'target' is not supported on this " + "architecture and platform">; +def err_bad_multiversion_option + : Error<"function multiversioning doesn't support " + "%select{feature|architecture}0 '%1'">; +def err_target_without_default : Error<"function multiversioning with 'target' " + "requires a 'default' implementation">; +def err_multiversion_virtual : Error<"function multiversioning with 'target' " + "doesn't support virtual functions yet">; } // end of sema component. Index: include/clang/Basic/TargetInfo.h =================================================================== --- include/clang/Basic/TargetInfo.h +++ include/clang/Basic/TargetInfo.h @@ -914,6 +914,17 @@ return false; } + // \brief Identify whether this target supports multiversioning of functions, + // which requires support for a cpu_supports and cpu_is functionality. + virtual bool supportsFuncMultiversioning() const { return false; } + + // \brief Validate the architecture for function multiversioning. Some + // will wish to override this, since there might be different rules for + // __builtin_cpu_is vs multiversioning strings. + virtual bool validateMultiversionCpu(StringRef Name) const { + return validateCpuIs(Name); + } + // \brief Validate the contents of the __builtin_cpu_supports(const char*) // argument. virtual bool validateCpuSupports(StringRef Name) const { return false; } @@ -922,6 +933,12 @@ // argument. virtual bool validateCpuIs(StringRef Name) const { return false; } + // \brief Provide ordering for the CPU Features and Names such that it can + // used for dispatchers. + virtual bool compareCpusAndFeatures(StringRef Lhs, StringRef Rhs) const { + return false; + } + // \brief Returns maximal number of args passed in registers. unsigned getRegParmMax() const { assert(RegParmMax < 7 && "RegParmMax value is larger than AST can handle"); Index: include/clang/Sema/Sema.h =================================================================== --- include/clang/Sema/Sema.h +++ include/clang/Sema/Sema.h @@ -1932,6 +1932,15 @@ SmallVectorImpl &OverloadedMethods); void NoteHiddenVirtualMethods(CXXMethodDecl *MD, SmallVectorImpl &OverloadedMethods); + + // Checks to see if attribute 'target' results in a valid situation. + bool TestAndSetDeclForMultiVersion(FunctionDecl *NewFD, FunctionDecl *OldFD); + // Helper function for CheckMultiVersionDecl to ensure that a declaration + // causing a Multi-version case doesn't cause invalid options. + bool CheckMultiVersionOptions(const FunctionDecl *NewFD, + const FunctionDecl *OldFD); + bool CheckMultiVersionOption(const FunctionDecl *FD); + // Returns true if the function declaration is a redeclaration bool CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD, LookupResult &Previous, Index: lib/Basic/Targets/X86.h =================================================================== --- lib/Basic/Targets/X86.h +++ lib/Basic/Targets/X86.h @@ -360,10 +360,22 @@ ArrayRef getGCCAddlRegNames() const override; + bool supportsFuncMultiversioning() const override { return true; } + bool validateMultiversionCpu(StringRef Name) const override { + // 'amdfam10' is a special case where 'march' supports it, but CpuIs + // does not officially (though the Emit function actually does) support + // it, it only supports 'amdfam10h'. + if (Name == "amdfam10") + return true; + + return validateCpuIs(Name); + } bool validateCpuSupports(StringRef Name) const override; bool validateCpuIs(StringRef Name) const override; + bool compareCpusAndFeatures(StringRef LHS, StringRef RHS) const override; + bool validateAsmConstraint(const char *&Name, TargetInfo::ConstraintInfo &info) const override; Index: lib/Basic/Targets/X86.cpp =================================================================== --- lib/Basic/Targets/X86.cpp +++ lib/Basic/Targets/X86.cpp @@ -1322,6 +1322,92 @@ .Default(false); } +bool X86TargetInfo::compareCpusAndFeatures(StringRef Lhs, StringRef Rhs) const { + // The following are in order from new to old. These are required for the + // ordering of the GCC Target attribute. The list has to match the + // implementation in in GCC 7.x in gcc/config/i386/i386.c + // ::get_builtin_code_for_version. This list is simplified from that source. + const auto TargetArray = {"avx512vpopcntdq", + "avx5124fmaps", + "avx5124vnniw", + "avx512ifma", + "avx512vbmi", + "avx512pf", + "avx512er", + "avx512cd", + "avx512dq", + "avx512bw", + "avx512vl", + "knl", + "avx512f", + "znver1", + "skylake-avx512", + "skylake", + "haswell", + "broadwell", + "bdver4", + "avx2", + "bmi2", + "bdver3", + "bdver2", + "fma", + "bdver1", + "amdfam15h", + "amdfam15", + "xop", + "fma4", + "btver2", + "bmi", + "sandybridge", + "ivybridge", + "avx", + "pclmul", + "aes", + "popcnt", + "westmere", + "slm", + "silvermont", + "nehalem", + "corei7", + "sse4.2", + "sse4.1", + "btver1", + "istanbul", + "shanghai", + "barcelona", + "amdfam10h", + "amdfam10", + "sse4a", + "core2", + "bonnell", + "atom", + "ssse3", + "sse3", + "sse2", + "sse", + "mmx", + "cmov", + "intel", + "amd"}; + + if (Lhs == Rhs) + return false; + // Default location doesn't matter except that sort should be repeatable, but + // move default to the end. + if (Lhs.empty()) + return false; + if (Rhs.empty()) + return true; + + for (const auto &Str : TargetArray) { + if (Lhs == Str) + return true; + else if (Rhs == Str) + return false; + } + assert(false && "Invalid item passed to compareCpusAndFeatures"); +} + bool X86TargetInfo::validateAsmConstraint( const char *&Name, TargetInfo::ConstraintInfo &Info) const { switch (*Name) { Index: lib/CodeGen/CGBuiltin.cpp =================================================================== --- lib/CodeGen/CGBuiltin.cpp +++ lib/CodeGen/CGBuiltin.cpp @@ -7708,7 +7708,7 @@ return Builder.CreateICmpNE(Bitset, llvm::ConstantInt::get(Int32Ty, 0)); } -Value *CodeGenFunction::EmitX86CpuInit() { +Value *CodeGenFunction::EmitX86CpuInit(CGBuilderTy &Builder) { llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, /*Variadic*/ false); llvm::Constant *Func = CGM.CreateRuntimeFunction(FTy, "__cpu_indicator_init"); @@ -7722,7 +7722,7 @@ if (BuiltinID == X86::BI__builtin_cpu_supports) return EmitX86CpuSupports(E); if (BuiltinID == X86::BI__builtin_cpu_init) - return EmitX86CpuInit(); + return EmitX86CpuInit(Builder); SmallVector Ops; Index: lib/CodeGen/CodeGenFunction.h =================================================================== --- lib/CodeGen/CodeGenFunction.h +++ lib/CodeGen/CodeGenFunction.h @@ -3726,6 +3726,16 @@ /// point operation, expressed as the maximum relative error in ulp. void SetFPAccuracy(llvm::Value *Val, float Accuracy); + struct ResolverOption { + TargetAttr::ParsedTargetAttr AttrInfo; + llvm::Function *Function; + ResolverOption(TargetAttr::ParsedTargetAttr &&AttrInfo, + llvm::Function *Function) + : AttrInfo(std::move(AttrInfo)), Function(Function) {} + }; + void EmitMultiVersionResolver(llvm::Function *ResolverFunc, + ArrayRef ResolverOptions); + private: llvm::MDNode *getRangeForLoadFromType(QualType Ty); void EmitReturnOfRValue(RValue RV, QualType Ty); @@ -3901,7 +3911,8 @@ llvm::Value *EmitX86CpuIs(StringRef CPUStr); llvm::Value *EmitX86CpuSupports(const CallExpr *E); llvm::Value *EmitX86CpuSupports(ArrayRef FeatureStrs); - llvm::Value *EmitX86CpuInit(); + llvm::Value *EmitX86CpuInit(CGBuilderTy &Builder); + llvm::Value *FormResolverCondition(const ResolverOption &RO); }; /// Helper class with most of the code for saving a value for a Index: lib/CodeGen/CodeGenFunction.cpp =================================================================== --- lib/CodeGen/CodeGenFunction.cpp +++ lib/CodeGen/CodeGenFunction.cpp @@ -2274,3 +2274,58 @@ return llvm::DebugLoc(); } + +llvm::Value * +CodeGenFunction::FormResolverCondition(const ResolverOption &RO) { + llvm::Value *TrueCondition = nullptr; + if (!RO.AttrInfo.Architecture.empty()) + TrueCondition = EmitX86CpuIs(RO.AttrInfo.Architecture); + + if (!RO.AttrInfo.Features.empty()) { + SmallVector FeatureList; + std::for_each(std::begin(RO.AttrInfo.Features), + std::end(RO.AttrInfo.Features), + [&FeatureList](const std::string &Feature) { + FeatureList.push_back(StringRef{Feature}.substr(1)); + }); + llvm::Value *FeatureCmp = EmitX86CpuSupports(FeatureList); + TrueCondition = TrueCondition + ? Builder.CreateAnd(TrueCondition, FeatureCmp) + : FeatureCmp; + } + return TrueCondition; +} + +void CodeGenFunction::EmitMultiVersionResolver( + llvm::Function *ResolverFunc, ArrayRef ResolverOptions) { + assert((getContext().getTargetInfo().getTriple().getArch() == + llvm::Triple::x86 || + getContext().getTargetInfo().getTriple().getArch() == + llvm::Triple::x86_64) && + "Only implemented for x86 targets"); + + // Main function's basic block. + llvm::BasicBlock *CurBlock = createBasicBlock("entry", ResolverFunc); + Builder.SetInsertPoint(CurBlock); + EmitX86CpuInit(Builder); + + llvm::Function *DefaultFunc = nullptr; + for (const ResolverOption &RO : ResolverOptions) { + Builder.SetInsertPoint(CurBlock); + llvm::Value *TrueCondition = FormResolverCondition(RO); + + if (!TrueCondition) { + DefaultFunc = RO.Function; + } else { + llvm::BasicBlock *RetBlock = createBasicBlock("ro_ret", ResolverFunc); + llvm::IRBuilder<> RetBuilder(RetBlock); + RetBuilder.CreateRet(RO.Function); + CurBlock = createBasicBlock("ro_else", ResolverFunc); + Builder.CreateCondBr(TrueCondition, RetBlock, CurBlock); + } + } + assert(DefaultFunc && "No default version?"); + // Emit return from the 'else-ist' block. + Builder.SetInsertPoint(CurBlock); + Builder.CreateRet(DefaultFunc); +} Index: lib/CodeGen/CodeGenModule.h =================================================================== --- lib/CodeGen/CodeGenModule.h +++ lib/CodeGen/CodeGenModule.h @@ -323,6 +323,7 @@ /// List of alias we have emitted. Used to make sure that what they point to /// is defined once we get to the end of the of the translation unit. std::vector Aliases; + std::vector MultiVersionFuncs; typedef llvm::StringMap > ReplacementsTy; ReplacementsTy Replacements; @@ -747,6 +748,7 @@ llvm::Type *Ty = nullptr, ForDefinition_t IsForDefinition = NotForDefinition); + std::string MakeMultiVersionName(StringRef MangledName, const Decl *D); /// Return the address of the given function. If Ty is non-null, then this /// function will use the specified type if it has to create it. @@ -1225,6 +1227,11 @@ bool DontDefer = false, bool IsThunk = false, llvm::AttributeList ExtraAttrs = llvm::AttributeList(), ForDefinition_t IsForDefinition = NotForDefinition); + llvm::Constant *GetOrCreateLLVMFunctionOrMultiVersion( + StringRef MangledName, llvm::Type *Ty, GlobalDecl D, bool ForVTable, + bool DontDefer = false, bool IsThunk = false, + llvm::AttributeList ExtraAttrs = llvm::AttributeList(), + ForDefinition_t IsForDefinition = NotForDefinition); llvm::Constant *GetOrCreateLLVMGlobal(StringRef MangledName, llvm::PointerType *PTy, @@ -1240,8 +1247,14 @@ ForDefinition_t IsForDefinition); void EmitGlobalDefinition(GlobalDecl D, llvm::GlobalValue *GV = nullptr); + void EmitMultiVersionResolver(StringRef ResolverName, const FunctionDecl *FD); void EmitGlobalFunctionDefinition(GlobalDecl GD, llvm::GlobalValue *GV); + llvm::GlobalValue *GetOrCreateMultiVersionIFunc(GlobalDecl GD, + llvm::Type *DeclTy, + StringRef MangledName, + const FunctionDecl *FD, + llvm::GlobalValue *GV); void EmitGlobalVarDefinition(const VarDecl *D, bool IsTentative = false); void EmitAliasDefinition(GlobalDecl GD); void emitIFuncDefinition(GlobalDecl GD); @@ -1297,6 +1310,7 @@ void applyGlobalValReplacements(); void checkAliases(); + void checkMultiversions(); /// Emit any vtables which we deferred and still have a use for. void EmitDeferredVTables(); Index: lib/CodeGen/CodeGenModule.cpp =================================================================== --- lib/CodeGen/CodeGenModule.cpp +++ lib/CodeGen/CodeGenModule.cpp @@ -285,6 +285,49 @@ } } +void CodeGenModule::EmitMultiVersionResolver(StringRef MangledName, + const FunctionDecl *FD) { + llvm::Function *ResolverFunc = + cast(GetGlobalValue((MangledName + ".resolver").str())); + SmallVector ResolverOptions; + for (FunctionDecl *CurFD : FD->redecls()) { + std::string FuncName = MakeMultiVersionName(MangledName, CurFD); + llvm::Constant *Func = GetGlobalValue(FuncName); + if (!Func) { + GlobalDecl GD{CurFD}; + if (CurFD->isThisDeclarationADefinition()) { + // If the function is defined, make sure we emit it. + EmitGlobalDefinition(GD); + Func = GetGlobalValue(FuncName); + } else { + // We still need to create declarations, since the resolver needs them. + const CGFunctionInfo &FI = getTypes().arrangeGlobalDeclaration(GD); + llvm::FunctionType *Ty = getTypes().GetFunctionType(FI); + Func = GetAddrOfFunction(GD, Ty, /*ForVTable=*/false, + /*DontDefer=*/false, ForDefinition); + } + assert(Func && "This should have just been created"); + } + + ResolverOptions.emplace_back(CurFD->getAttr()->parse(), + cast(Func)); + } + + // Sort resolver options. + auto *TargetInfo = &Context.getTargetInfo(); + auto CmpFunc = [TargetInfo](StringRef LHS, StringRef RHS) { + return TargetInfo->compareCpusAndFeatures(LHS, RHS); + }; + std::sort(std::begin(ResolverOptions), std::end(ResolverOptions), + [CmpFunc](const CodeGenFunction::ResolverOption &LHS, + const CodeGenFunction::ResolverOption &RHS) { + return CmpFunc(LHS.AttrInfo.getHighestPriority(CmpFunc), + RHS.AttrInfo.getHighestPriority(CmpFunc)); + }); + CodeGenFunction CGF(*this); + CGF.EmitMultiVersionResolver(ResolverFunc, ResolverOptions); +} + void CodeGenModule::checkAliases() { // Check if the constructed aliases are well formed. It is really unfortunate // that we have to do this in CodeGen, but we only construct mangled names @@ -362,6 +405,40 @@ } } +void CodeGenModule::checkMultiversions() { + // Cleanup the IFuncs calls for multiversioning. + for (const GlobalDecl &GD : MultiVersionFuncs) { + bool DefaultFound = false; + for (auto *Decl : GD.getDecl()->getMostRecentDecl()->redecls()) + if (Decl->getAttr()->getFeaturesStr() == "default") + DefaultFound = true; + + StringRef MangledName = getMangledName(GD); + llvm::GlobalValue *BaseFunc = GetGlobalValue(MangledName); + llvm::GlobalValue *IFunc = GetGlobalValue((MangledName + ".ifunc").str()); + assert(IFunc && "Multiversioning not legal without an IFunc"); + if (!DefaultFound && cast(GD.getDecl())->isDefined()) { + Diags.Report( + cast(GD.getDecl())->getFirstDecl()->getLocation(), + diag::err_target_without_default); + // In the error case, replace all uses with an undef. + if (BaseFunc) { + BaseFunc->replaceAllUsesWith( + llvm::UndefValue::get(BaseFunc->getType())); + BaseFunc->eraseFromParent(); + } + continue; + } + + if (cast(GD.getDecl())->isDefined()) + EmitMultiVersionResolver( + MangledName, cast(GD.getDecl()->getMostRecentDecl())); + if (BaseFunc) + BaseFunc->replaceAllUsesWith( + llvm::ConstantExpr::getBitCast(IFunc, BaseFunc->getType())); + } +} + void CodeGenModule::clear() { DeferredDeclsToEmit.clear(); if (OpenMPRuntime) @@ -391,6 +468,7 @@ applyGlobalValReplacements(); applyReplacements(); checkAliases(); + checkMultiversions(); EmitCXXGlobalInitFunc(); EmitCXXGlobalDtorFunc(); EmitCXXThreadLocalInitFunc(); @@ -2038,6 +2116,73 @@ static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old, llvm::Function *NewFn); +std::string CodeGenModule::MakeMultiVersionName(StringRef MangledName, + const Decl *D) { + assert(D && "Decl cannot be null"); + const auto *TA = D->getAttr(); + assert(TA && "Decl must have a Target Attribute"); + auto Append = TA->multiVersionMangleAddition(); + if (Append.empty()) + return MangledName; + return (MangledName + "." + Append).str(); +} + +llvm::GlobalValue *CodeGenModule::GetOrCreateMultiVersionIFunc( + GlobalDecl GD, llvm::Type *DeclTy, StringRef MangledName, + const FunctionDecl *FD, llvm::GlobalValue *GV) { + std::string IFuncName = (MangledName + ".ifunc").str(); + llvm::GlobalValue *IFuncGV = GetGlobalValue(IFuncName); + + if (IFuncGV) + return IFuncGV; + + // Since this is the first time we've created this IFunc, make sure + // that we put this multiversioned function into the list to be + // replaced later. + MultiVersionFuncs.push_back(GD); + + std::string ResolverName = (MangledName + ".resolver").str(); + llvm::Type *ResolverType = llvm::FunctionType::get( + llvm::PointerType::get(DeclTy, + Context.getTargetAddressSpace(FD->getType())), + false); + llvm::Constant *Resolver = + GetOrCreateLLVMFunction(ResolverName, ResolverType, GlobalDecl{}, + /*ForVTable=*/false); + llvm::GlobalIFunc *GIF = llvm::GlobalIFunc::create( + DeclTy, 0, llvm::Function::ExternalLinkage, "", Resolver, &getModule()); + GIF->setName(IFuncName); + SetCommonAttributes(FD, GIF); + + return GIF; +} + +/// GetOrCreateLLVMFunctionOrMultiVersion - Wrapper for GetOrCreateLLVMFunction +/// that will also check to see if this is a FunctionDecl that is +/// multiversioned. In the case where this is NOT for a definition, this will +/// create the llvm GlobalValue, but return the iFunc. If this FOR definition, +/// it will return the newly mangled name. +llvm::Constant *CodeGenModule::GetOrCreateLLVMFunctionOrMultiVersion( + StringRef MangledName, llvm::Type *Ty, GlobalDecl GD, bool ForVTable, + bool DontDefer, bool IsThunk, llvm::AttributeList ExtraAttrs, + ForDefinition_t IsForDefinition) { + const FunctionDecl *FD = cast(GD.getDecl()); + std::string MVName; + + if (FD->getMultiVersionKind() == + FunctionDecl::MultiVersionKind::MultiVersion) { + llvm::GlobalValue *ExistingDef = GetGlobalValue(MangledName); + GetOrCreateMultiVersionIFunc(GD, Ty, MangledName, FD, ExistingDef); + if (IsForDefinition == ForDefinition) { + MVName = MakeMultiVersionName(MangledName, FD); + MangledName = MVName; + } + } + + return GetOrCreateLLVMFunction(MangledName, Ty, GD, ForVTable, DontDefer, + IsThunk, ExtraAttrs, IsForDefinition); +} + /// GetOrCreateLLVMFunction - If the specified mangled name is not in the /// module, create and return an llvm Function with the specified type. If there /// is something in the module with the specified name, return it potentially @@ -2215,9 +2360,9 @@ } StringRef MangledName = getMangledName(GD); - return GetOrCreateLLVMFunction(MangledName, Ty, GD, ForVTable, DontDefer, - /*IsThunk=*/false, llvm::AttributeList(), - IsForDefinition); + return GetOrCreateLLVMFunctionOrMultiVersion( + MangledName, Ty, GD, ForVTable, DontDefer, + /*IsThunk=*/false, llvm::AttributeList(), IsForDefinition); } static const FunctionDecl * Index: lib/Sema/SemaDecl.cpp =================================================================== --- lib/Sema/SemaDecl.cpp +++ lib/Sema/SemaDecl.cpp @@ -3221,28 +3221,34 @@ return true; } - // C++ [class.mem]p1: - // [...] A member shall not be declared twice in the - // member-specification, except that a nested class or member - // class template can be declared and then later defined. - if (!inTemplateInstantiation()) { - unsigned NewDiag; - if (isa(OldMethod)) - NewDiag = diag::err_constructor_redeclared; - else if (isa(NewMethod)) - NewDiag = diag::err_destructor_redeclared; - else if (isa(NewMethod)) - NewDiag = diag::err_conv_function_redeclared; - else - NewDiag = diag::err_member_redeclared; + if (New->getMultiVersionKind() != + FunctionDecl::MultiVersionKind::MultiVersion) { + // Multiversioning allows multiple decls. + + // C++ [class.mem]p1: + // [...] A member shall not be declared twice in the + // member-specification, except that a nested class or member + // class template can be declared and then later defined. + if (!inTemplateInstantiation()) { + unsigned NewDiag; + if (isa(OldMethod)) + NewDiag = diag::err_constructor_redeclared; + else if (isa(NewMethod)) + NewDiag = diag::err_destructor_redeclared; + else if (isa(NewMethod)) + NewDiag = diag::err_conv_function_redeclared; + else + NewDiag = diag::err_member_redeclared; - Diag(New->getLocation(), NewDiag); - } else { - Diag(New->getLocation(), diag::err_member_redeclared_in_instantiation) - << New << New->getType(); + Diag(New->getLocation(), NewDiag); + } else { + Diag(New->getLocation(), + diag::err_member_redeclared_in_instantiation) + << New << New->getType(); + } + Diag(OldLocation, PrevDiag) << Old << Old->getType(); + return true; } - Diag(OldLocation, PrevDiag) << Old << Old->getType(); - return true; // Complain if this is an explicit declaration of a special // member that was initially declared implicitly. @@ -9224,6 +9230,173 @@ D->getFriendObjectKind() != Decl::FOK_None); } +static void setMultiVersionKind(FunctionDecl *OldFD, FunctionDecl *NewFD, + FunctionDecl::MultiVersionKind Kind) { + for (FunctionDecl *F : OldFD->redecls()) + F->setMultiVersionKind(Kind); + NewFD->setMultiVersionKind(Kind); +} + +bool Sema::CheckMultiVersionOption(const FunctionDecl *FD) { + const auto *TA = FD->getAttr(); + + if (TA->getFeaturesStr() == "default") + return true; + TargetAttr::ParsedTargetAttr ParseInfo = TA->parse(); + + enum BadOption { BOFeature = 0, BOArch = 1 }; + + if (const auto *CMD = dyn_cast(FD)) + if (CMD->isVirtual()) { + Diag(CMD->getLocation(), diag::err_multiversion_virtual); + return false; + } + + // isValidCPU was checked when adding the 'target', so only validating + // that the string is valid for the CpuIs check. + if (!ParseInfo.Architecture.empty() && + !Context.getTargetInfo().validateMultiversionCpu( + ParseInfo.Architecture)) { + Diag(TA->getLocation(), diag::err_bad_multiversion_option) + << BOArch << ParseInfo.Architecture; + return false; + } + + for (const auto &Feature : ParseInfo.Features) { + auto BareFeat = StringRef{Feature}.substr(1); + // Negative options never allowed. + if (Feature[0] == '-') { + Diag(TA->getLocation(), diag::err_bad_multiversion_option) + << BOFeature << ("no-" + BareFeat).str(); + return false; + } + if (!Context.getTargetInfo().validateCpuSupports(BareFeat) || + !Context.getTargetInfo().isValidFeatureName(BareFeat)) { + Diag(TA->getLocation(), diag::err_bad_multiversion_option) + << BOFeature << BareFeat; + return false; + } + } + return true; +} + +bool Sema::CheckMultiVersionOptions(const FunctionDecl *OldFD, + const FunctionDecl *NewFD) { + // FIXME: Implement for more than X86. Requires CPUIs and CPUSupports + // for each new architecture. + if (!Context.getTargetInfo().supportsFuncMultiversioning()) { + Diag(NewFD->getLocation(), diag::err_target_unimplemented_arch); + return false; + } + + bool Result = true; + for (const auto *FD : OldFD->redecls()) + Result = CheckMultiVersionOption(FD) && Result; + + return CheckMultiVersionOption(NewFD) && Result; +} + +bool Sema::TestAndSetDeclForMultiVersion(FunctionDecl *NewFD, + FunctionDecl *OldFD) { + const auto *TA = NewFD->getAttr(); + // The first Decl is easy, it is either the 'All' case or 'None'. + if (!OldFD) { + NewFD->setMultiVersionKind(TA ? FunctionDecl::MultiVersionKind::All + : FunctionDecl::MultiVersionKind::None); + return true; + } + + switch (OldFD->getMultiVersionKind()) { + case FunctionDecl::MultiVersionKind::Error: + // An error case stays an error, mark the new Decl as well. + NewFD->setMultiVersionKind(FunctionDecl::MultiVersionKind::Error); + return false; + case FunctionDecl::MultiVersionKind::None: + // A previously unmarked function can convert to the "Partial" (hint-only) + // case if this one is marked. + if (!TA) { + // No change to the Previous Decls, set current to correct value. + NewFD->setMultiVersionKind(FunctionDecl::MultiVersionKind::None); + return true; + } + setMultiVersionKind(OldFD, NewFD, FunctionDecl::MultiVersionKind::Partial); + return true; + case FunctionDecl::MultiVersionKind::All: + // If all previous Decls have a target attribute, but none are different + // this can become a 'partial' if there is no attribute on this Decl, or it + // can become a multiversion if this is different. + if (!TA) { + setMultiVersionKind(OldFD, NewFD, + FunctionDecl::MultiVersionKind::Partial); + return true; + } + // Could potentially convert to a 'MultiVersion' state, check this + // case. Only have to check the last one, since it is the same as the + // previous ones. + if (TA->parse() != OldFD->getAttr()->parse()) { + if (!CheckMultiVersionOptions(OldFD, NewFD)) { + setMultiVersionKind(OldFD, NewFD, + FunctionDecl::MultiVersionKind::Error); + return false; + } + setMultiVersionKind(OldFD, NewFD, + FunctionDecl::MultiVersionKind::MultiVersion); + return true; + } + // If this doesn't convert it to a multi-version, this is clearly an 'all' + // case. + NewFD->setMultiVersionKind(FunctionDecl::MultiVersionKind::All); + return true; + case FunctionDecl::MultiVersionKind::MultiVersion: + // If the existing Decls are already in a MultiVersion case, a 'target' + // attribute is required on this one. Validate that it is legal on this one. + if (!TA) { + Diag(NewFD->getLocation(), diag::err_target_required_in_redecl); + Diag(OldFD->getFirstDecl()->getLocation(), + diag::note_previous_declaration); + setMultiVersionKind(OldFD, NewFD, FunctionDecl::MultiVersionKind::Error); + return false; + } else if (!CheckMultiVersionOption(NewFD)) { + setMultiVersionKind(OldFD, NewFD, FunctionDecl::MultiVersionKind::Error); + return false; + } + + // Otherwise, just continue the multiversion situation. + NewFD->setMultiVersionKind(FunctionDecl::MultiVersionKind::MultiVersion); + return true; + case FunctionDecl::MultiVersionKind::Partial: + // A partial not adding a Target attribute is fine, everything stays as + // partial. + if (!TA) { + NewFD->setMultiVersionKind(FunctionDecl::MultiVersionKind::Partial); + return true; + } + // A partial situation is not allowed to convert to a MultiVersion, so + // ensure that this target doesn't cause that situation. + FunctionDecl *LastFeaturesStr = OldFD; + // Note: Null check shouldn't be necessary. + while (LastFeaturesStr && !LastFeaturesStr->hasAttr()) + LastFeaturesStr = LastFeaturesStr->getPreviousDecl(); + + assert(LastFeaturesStr && + "How can we be 'partial' without an existing target attribute?"); + if (TA->parse() != LastFeaturesStr->getAttr()->parse()) { + Diag(NewFD->getLocation(), + diag::err_target_causes_illegal_multiversioning); + // Note on all previous decls that lack a target attr. + for (const auto *Cur : OldFD->redecls()) + if (!Cur->hasAttr() || + Cur->getAttr()->isInherited()) + Diag(Cur->getLocation(), diag::note_previous_declaration); + setMultiVersionKind(OldFD, NewFD, FunctionDecl::MultiVersionKind::Error); + return false; + } + NewFD->setMultiVersionKind(FunctionDecl::MultiVersionKind::Partial); + return true; + } + llvm_unreachable("Invalid option for MultiVersionKind"); +} + /// \brief Perform semantic checking of a new function declaration. /// /// Performs semantic analysis of the new function declaration @@ -9349,6 +9522,12 @@ } } + // Attribute target requires all declarations to have the 'target' + // attribute. + if (!TestAndSetDeclForMultiVersion(NewFD, + dyn_cast_or_null(OldDecl))) + NewFD->setInvalidDecl(); + if (Redeclaration) { // NewFD and OldDecl represent declarations that need to be // merged. @@ -11997,6 +12176,28 @@ return MissingPrototype; } +// Check to see if this 'redefinition' should be allowed in order to support +// attribute 'target' redefinition. +static bool isValidMultiVersion(FunctionDecl *NewFD, + const FunctionDecl *ExistingFD) { + if (NewFD->getMultiVersionKind() != + FunctionDecl::MultiVersionKind::MultiVersion) + return false; + + const auto *NewTA = NewFD->getAttr(); + const TargetAttr::ParsedTargetAttr NewTAParsed = NewTA->parse(); + // Prohibit duplicate definitions. + for (const auto *FD : ExistingFD->redecls()) { + if (FD->isThisDeclarationADefinition()) { + const auto *TA = FD->getAttr(); + if (TA && !TA->isInherited() && + TA->parse() == NewTAParsed) + return false; + } + } + return true; +} + void Sema::CheckForFunctionRedefinition(FunctionDecl *FD, const FunctionDecl *EffectiveDefinition, @@ -12009,6 +12210,9 @@ if (canRedefineFunction(Definition, getLangOpts())) return; + if (isValidMultiVersion(FD, Definition)) + return; + // Don't emit an error when this is redefinition of a typo-corrected // definition. if (TypoCorrectedFunctionDefinitions.count(Definition)) Index: test/CodeGen/attr-target-multiversion.c =================================================================== --- /dev/null +++ test/CodeGen/attr-target-multiversion.c @@ -0,0 +1,87 @@ +// RUN: %clang_cc1 -triple x86_64-linux-gnu -target-cpu x86-64 -emit-llvm %s -o - | FileCheck %s + +//CHECK: @foo.ifunc = ifunc i8 (i{{[0-9]+}}), i8 (i{{[0-9]+}})* ()* @foo.resolver +//CHECK: @bar.ifunc = ifunc i8 (...), i8 (...)* ()* @bar.resolver +//CHECK: @baz.ifunc = ifunc i8 (), i8 ()* ()* @baz.resolver + +__attribute__((target("arch=knl"))) char foo(int); + +// This function causes emission of the resolver, check it. +// CHECK: define i8 (i{{[0-9]+}})* @foo.resolver() { +// CHECK: call void @__cpu_indicator_init() +// CHECK: ret i8 (i{{[0-9]+}})* @foo.arch_knl +// CHECK: ret i8 (i{{[0-9]+}})* @foo.arch_skylake +// CHECK: ret i8 (i{{[0-9]+}})* @foo.sse4.2 +// CHECK: ret i8 (i{{[0-9]+}})* @foo.arch_amdfam10 +// CHECK: ret i8 (i{{[0-9]+}})* @foo.arch_atom +// CHECK: ret i8 (i{{[0-9]+}})* @foo + +// CHECK: define signext i8 @foo( +__attribute__((target("default"))) char foo(int i) { + return 0 + i; +} + +// CHECK: define signext i8 @user1() +// CHECK: call signext i8 @foo.ifunc(i{{[0-9]+}} 1) +char user1(){ + return foo(1); +} + +// CHECK: define signext i8 @foo.sse4.2( +__attribute__((target("sse4.2"))) char foo(int i) { + return 1 + i; +} + +// CHECK: define signext i8 @user2() +// CHECK: call signext i8 @foo.ifunc(i{{[0-9]+}} 2) +char user2(){ + return foo(2); +} + +// CHECK: define signext i8 @foo.arch_atom( +__attribute__((target("arch=atom"))) char foo(int i) { + return 2 + i; +} + +__attribute__((target("arch=amdfam10"))) char foo(int); + +// CHECK: define signext i8 @user3() +// CHECK: call signext i8 @foo.ifunc(i{{[0-9]+}} 3) +char user3() { + return foo(3); +} + +__attribute__((target("arch=skylake"))) char foo(int); + +__attribute__((target("arch=knl"))) char bar(); + +__attribute__((target("default"))) char bar(); + + +// CHECK: define signext i8 @user4() +// CHECK: call signext i8 (...) @bar.ifunc() +char user4() { + return bar(); +} + +// CHECK: declare i8 (...)* @bar.resolver(){{$}} + +__attribute__((target("arch=knl"))) char baz(void); + +__attribute__((target("default"))) char baz(); + + +// CHECK: define signext i8 @user5() +// CHECK: store i8 (...)* bitcast (i8 ()* @baz.ifunc to i8 (...)*), i8 (...)** %baz_ptr, align 8 +// CHECK: call signext i8 @baz.ifunc() +char user5() { + char (*baz_ptr)() = &baz; + return baz() + baz_ptr(); +} +// CHECK: declare i8 ()* @baz.resolver(){{$}} + +// Function Declarations are generated later, so ensure that they +// actually exist. +// CHECK-DAG: declare signext i8 @foo.arch_knl +// CHECK-DAG: declare signext i8 @foo.arch_amdfam10 +// CHECK-DAG: declare signext i8 @foo.arch_skylake Index: test/CodeGenCXX/attr-target-multiversion.cpp =================================================================== --- /dev/null +++ test/CodeGenCXX/attr-target-multiversion.cpp @@ -0,0 +1,28 @@ +// RUN: %clang_cc1 -triple x86_64-linux-gnu -target-cpu x86-64 -emit-llvm %s -o - | FileCheck %s + +struct S { + __attribute__((target("arch=sandybridge"))) + int mv(){return 3;} + __attribute__((target("arch=ivybridge"))) + int mv(){return 2;} + __attribute__((target("default"))) + int mv(){ return 1;} +}; + +// CHECK: @_ZN1S2mvEv.ifunc = ifunc i{{[0-9]+}} (%struct.S*), i{{[0-9]+}} (%struct.S*)* ()* @_ZN1S2mvEv.resolver + +// CHECK: define void @_Z3foov() +// CHECK: call i{{[0-9]+}} @_ZN1S2mvEv.ifunc(%struct.S* % +void foo() { + S s; + s.mv(); +} + +// CHECK: define i{{[0-9]+}} (%struct.S*)* @_ZN1S2mvEv.resolver +// CHECK: ret i{{[0-9]+}} (%struct.S*)* @_ZN1S2mvEv.arch_sandybridge +// CHECK: ret i{{[0-9]+}} (%struct.S*)* @_ZN1S2mvEv.arch_ivybridge +// CHECK: ret i{{[0-9]+}} (%struct.S*)* @_ZN1S2mvEv + +// CHECK-DAG: define linkonce_odr i{{[0-9]+}} @_ZN1S2mvEv.arch_sandybridge(%struct.S* %this) +// CHECK-DAG: define linkonce_odr i{{[0-9]+}} @_ZN1S2mvEv.arch_ivybridge(%struct.S* %this) +// CHECK-DAG: define linkonce_odr i{{[0-9]+}} @_ZN1S2mvEv(%struct.S* %this) Index: test/Sema/attr-target-multiversion.c =================================================================== --- /dev/null +++ test/Sema/attr-target-multiversion.c @@ -0,0 +1,136 @@ +// RUN: %clang_cc1 -triple x86_64-linux -fsyntax-only -verify -emit-llvm-only %s +// RUN: %clang_cc1 -triple x86_64-linux -fsyntax-only -verify -emit-llvm-only -DCHECK_DEFAULT %s + +#if defined(CHECK_DEFAULT) +__attribute__((target("arch=sandybridge"))) +//expected-error@+1 {{function multiversioning with 'target' requires a 'default' implementation}} +void no_default(){} +__attribute__((target("arch=ivybridge"))) +void no_default(){} +#else +// Only multiversioning causes issues with redeclaration changing 'target'. +__attribute__((target("arch=sandybridge"))) +void fine_since_no_mv(); +void fine_since_no_mv(); + +void also_fine_since_no_mv(); +__attribute__((target("arch=sandybridge"))) +void also_fine_since_no_mv(); + +__attribute__((target("arch=sandybridge"))) +void also_fine_since_no_mv2(); +__attribute__((target("arch=sandybridge"))) +void also_fine_since_no_mv2(); +void also_fine_since_no_mv2(); + +__attribute__((target("arch=sandybridge"))) +void mv(){} +__attribute__((target("arch=ivybridge"))) +void mv(){} +__attribute__((target("default"))) +void mv(){} + +void redecl_causes_mv(); +__attribute__((target("arch=sandybridge"))) +void redecl_causes_mv(); +// expected-error@+3 {{function redeclaration declares a multiversioned function, but a previous declaration lacks a 'target' attribute}} +// expected-note@-4 {{previous declaration is here}} +__attribute__((target("arch=ivybridge"))) +void redecl_causes_mv(); + +__attribute__((target("arch=sandybridge"))) +void redecl_causes_mv2(); +void redecl_causes_mv2(); +// expected-error@+3 {{function redeclaration declares a multiversioned function, but a previous declaration lacks a 'target' attribute}} +// expected-note@-2 {{previous declaration is here}} +__attribute__((target("arch=ivybridge"))) +void redecl_causes_mv2(); + +__attribute__((target("arch=sandybridge"))) +void redecl_without_attr(); +__attribute__((target("arch=ivybridge"))) +void redecl_without_attr(); +// expected-error@+2 {{function redeclaration is missing 'target' attribute in a multiversioned function}} +// expected-note@-4 {{previous declaration is here}} +void redecl_without_attr(); + +__attribute__((target("arch=sandybridge"))) +void multiversion_with_predecl(); +__attribute__((target("default"))) +void multiversion_with_predecl(); +__attribute__((target("arch=sandybridge"))) +void multiversion_with_predecl(){} +__attribute__((target("default"))) +void multiversion_with_predecl(){} +//expected-error@+3 {{redefinition of 'multiversion_with_predecl'}} +//expected-note@-2 {{previous definition is here}} +__attribute__((target("arch=sandybridge"))) +void multiversion_with_predecl(){} + +__attribute__((target("arch=sandybridge"))) +void multiversion_with_predecl2(); +__attribute__((target("arch=sandybridge"))) +void multiversion_with_predecl2(){} +__attribute__((target("default"))) +void multiversion_with_predecl2(); +//expected-error@+3 {{redefinition of 'multiversion_with_predecl2'}} +//expected-note@-4 {{previous definition is here}} +__attribute__((target("arch=sandybridge"))) +void multiversion_with_predecl2(){} +__attribute__((target("default"))) +void multiversion_with_predecl2(){} + +// All the following are fine in normal 'target' mode, but not +// in multiversion mode. +// expected-error@+1 {{function multiversioning doesn't support feature 'sgx'}} +__attribute__((target("sgx"))) +void badMVFeature(); +__attribute__((target("sse"))) +void badMVFeature(); + +__attribute__((target("mmx"))) +void badMVFeature2(); +// expected-error@+1 {{function multiversioning doesn't support feature 'lwp'}} +__attribute__((target("lwp"))) +void badMVFeature2(); + +__attribute__((target("arch=ivybridge,mmx"))) +void badMVFeature3(); +// expected-error@+1 {{function multiversioning doesn't support feature 'lwp'}} +__attribute__((target("arch=skylake,lwp"))) +void badMVFeature3(); + +// expected-error@+1 {{function multiversioning doesn't support architecture 'k8'}} +__attribute__((target("arch=k8"))) +void badMVArch(); +__attribute__((target("arch=nehalem"))) +void badMVArch(); + +__attribute__((target("arch=nehalem"))) +void badMVArch2(); +// expected-error@+1 {{function multiversioning doesn't support architecture 'k8'}} +__attribute__((target("arch=k8"))) +void badMVArch2(); + +__attribute__((target("mmx"))) +void badMVNegate(); +// expected-error@+1 {{function multiversioning doesn't support feature 'no-mmx'}} +__attribute__((target("no-mmx"))) +void badMVNegate(); + +__attribute__((target("arch=ivybridge"))) +void some_bad_archs(); +__attribute__((target("arch=amdfam10"))) +void some_bad_archs(); +// expected-warning@+2 {{ignoring unsupported architecture 'amdfam15'}} +// expected-error@+1 {{function multiversioning doesn't support architecture 'amdfam15'}} +__attribute__((target("arch=amdfam15"))) +void some_bad_archs(); +// expected-warning@+1 {{ignoring unsupported architecture 'amdfam10h'}} +__attribute__((target("arch=amdfam10h"))) +void some_bad_archs(); +// expected-warning@+1 {{ignoring unsupported architecture 'amdfam15h'}} +__attribute__((target("arch=amdfam15h"))) +void some_bad_archs(); + +#endif Index: test/SemaCXX/attr-target-multiversion.cpp =================================================================== --- /dev/null +++ test/SemaCXX/attr-target-multiversion.cpp @@ -0,0 +1,49 @@ +// RUN: %clang_cc1 -triple x86_64-linux -fsyntax-only -verify -emit-llvm-only %s + +struct S { + __attribute__((target("arch=sandybridge"))) + void mv(){} + __attribute__((target("arch=ivybridge"))) + void mv(){} + __attribute__((target("default"))) + void mv(){} + + // NOTE: Virtual functions aren't implement for multiversioning in GCC either, + // so this is a 'TBD' feature. + // expected-error@+2 {{function multiversioning with 'target' doesn't support virtual functions yet}} + __attribute__((target("arch=sandybridge"))) + virtual void mv_2(){} + // expected-error@+4 {{function multiversioning with 'target' doesn't support virtual functions yet}} + // expected-error@+3 {{class member cannot be redeclared}} + // expected-note@-3 {{previous definition is here}} + __attribute__((target("arch=ivybridge"))) + virtual void mv_2(){} + // expected-error@+3 {{class member cannot be redeclared}} + // expected-note@-7 {{previous definition is here}} + __attribute__((target("default"))) + virtual void mv_2(){} +}; + +// Note: Template attribute 'target' isn't implemented in GCC either, and would +// end up causing some nasty issues attempting it, so ensure that it still gives +// the same errors as without the attribute. + +template +__attribute__((target("arch=sandybridge"))) +void mv_temp(){} + +template +__attribute__((target("arch=ivybridge"))) +//expected-error@+2 {{redefinition of}} +//expected-note@-5{{previous definition is here}} +void mv_temp(){} + +template +__attribute__((target("default"))) +void mv_temp(){} + +void foo() { + //expected-error@+2{{no matching function for call to}} + //expected-note@-8{{candidate template ignored}} + mv_temp(); +}