[NFC][clang] Move multiversion resolver code generation to llvm/ subdirectory

[a.elovikov]

Summary

Some part of the target multiversioning support already resided in
llvm/lib/Support/X86TargetParser.cpp. However, the IR generation could not be
put there because of the component dependencies.

I think Transforms/Utils is a good place to put such kind of utils similar to
AmdGPUEmitPrintf functionality there. The change can allow the use of the
functionality outside clang as it isn't C/C++-specific.

[a.elovikov]

Fix a bug with not saving builder's insertion point.

[a.elovikov]

Rebase + ping

[a.elovikov]

ping.

clang/lib/CodeGen/CGBuiltin.cpp

 #include "llvm/IR/IntrinsicsS390.h"
 #include "llvm/IR/IntrinsicsWebAssembly.h"
 #include "llvm/IR/IntrinsicsX86.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/MatrixBuilder.h"
 #include "llvm/Support/ConvertUTF.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "llvm/Support/X86TargetParser.h"
+#include "llvm/Transforms/Utils/X86EmitMultiVersionResolver.h"
 #include <sstream>
 
 using namespace clang;
 using namespace CodeGen;
 using namespace llvm;
 
 static
 int64_t clamp(int64_t Value, int64_t Low, int64_t High) {
static Value *EmitX86BinaryIntrinsic(CodeGenFunction &CGF,
                                      ArrayRef<Value *> Ops, Intrinsic::ID IID) {
   llvm::Function *F = CGF.CGM.getIntrinsic(IID, Ops[0]->getType());
   return CGF.Builder.CreateCall(F, {Ops[0], Ops[1]});
 }
 
 Value *CodeGenFunction::EmitX86CpuIs(const CallExpr *E) {
   const Expr *CPUExpr = E->getArg(0)->IgnoreParenCasts();
   StringRef CPUStr = cast<clang::StringLiteral>(CPUExpr)->getString();
-  return EmitX86CpuIs(CPUStr);
+  return llvm::X86::emitCpuIs(Builder, CPUStr);
 }
 
 // Convert F16 halfs to floats.
 static Value *EmitX86CvtF16ToFloatExpr(CodeGenFunction &CGF,
                                        ArrayRef<Value *> Ops,
                                        llvm::Type *DstTy) {
   assert((Ops.size() == 1 || Ops.size() == 3 || Ops.size() == 4) &&
          "Unknown cvtph2ps intrinsic");
static Value *EmitX86CvtBF16ToFloatExpr(CodeGenFunction &CGF,
   llvm::Type *Int32Ty = CGF.Builder.getInt32Ty();
   Value *ZeroExt = CGF.Builder.CreateZExt(Ops[0], Int32Ty);
   Value *Shl = CGF.Builder.CreateShl(ZeroExt, 16);
   llvm::Type *ResultType = CGF.ConvertType(E->getType());
   Value *BitCast = CGF.Builder.CreateBitCast(Shl, ResultType);
   return BitCast;
 }
 
-Value *CodeGenFunction::EmitX86CpuIs(StringRef CPUStr) {
-
-  llvm::Type *Int32Ty = Builder.getInt32Ty();
-
-  // Matching the struct layout from the compiler-rt/libgcc structure that is
-  // filled in:
-  // unsigned int __cpu_vendor;
-  // unsigned int __cpu_type;
-  // unsigned int __cpu_subtype;
-  // unsigned int __cpu_features[1];
-  llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, Int32Ty,
-                                          llvm::ArrayType::get(Int32Ty, 1));
-
-  // Grab the global __cpu_model.
-  llvm::Constant *CpuModel = CGM.CreateRuntimeVariable(STy, "__cpu_model");
-  cast<llvm::GlobalValue>(CpuModel)->setDSOLocal(true);
-
-  // Calculate the index needed to access the correct field based on the
-  // range. Also adjust the expected value.
-  unsigned Index;
-  unsigned Value;
-  std::tie(Index, Value) = StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
-#define X86_VENDOR(ENUM, STRING)                                               \
-  .Case(STRING, {0u, static_cast<unsigned>(llvm::X86::ENUM)})
-#define X86_CPU_TYPE_ALIAS(ENUM, ALIAS)                                        \
-  .Case(ALIAS, {1u, static_cast<unsigned>(llvm::X86::ENUM)})
-#define X86_CPU_TYPE(ENUM, STR)                                                \
-  .Case(STR, {1u, static_cast<unsigned>(llvm::X86::ENUM)})
-#define X86_CPU_SUBTYPE(ENUM, STR)                                             \
-  .Case(STR, {2u, static_cast<unsigned>(llvm::X86::ENUM)})
-#include "llvm/Support/X86TargetParser.def"
-                               .Default({0, 0});
-  assert(Value != 0 && "Invalid CPUStr passed to CpuIs");
-
-  // Grab the appropriate field from __cpu_model.
-  llvm::Value *Idxs[] = {ConstantInt::get(Int32Ty, 0),
-                         ConstantInt::get(Int32Ty, Index)};
-  llvm::Value *CpuValue = Builder.CreateGEP(STy, CpuModel, Idxs);
-  CpuValue = Builder.CreateAlignedLoad(Int32Ty, CpuValue,
-                                       CharUnits::fromQuantity(4));
-
-  // Check the value of the field against the requested value.
-  return Builder.CreateICmpEQ(CpuValue,
-                                  llvm::ConstantInt::get(Int32Ty, Value));
-}
-
 Value *CodeGenFunction::EmitX86CpuSupports(const CallExpr *E) {
   const Expr *FeatureExpr = E->getArg(0)->IgnoreParenCasts();
   StringRef FeatureStr = cast<StringLiteral>(FeatureExpr)->getString();
-  return EmitX86CpuSupports(FeatureStr);
-}
-
-Value *CodeGenFunction::EmitX86CpuSupports(ArrayRef<StringRef> FeatureStrs) {
-  return EmitX86CpuSupports(llvm::X86::getCpuSupportsMask(FeatureStrs));
-}
-
-llvm::Value *CodeGenFunction::EmitX86CpuSupports(uint64_t FeaturesMask) {
-  uint32_t Features1 = Lo_32(FeaturesMask);
-  uint32_t Features2 = Hi_32(FeaturesMask);
-
-  Value *Result = Builder.getTrue();
-
-  if (Features1 != 0) {
-    // Matching the struct layout from the compiler-rt/libgcc structure that is
-    // filled in:
-    // unsigned int __cpu_vendor;
-    // unsigned int __cpu_type;
-    // unsigned int __cpu_subtype;
-    // unsigned int __cpu_features[1];
-    llvm::Type *STy = llvm::StructType::get(Int32Ty, Int32Ty, Int32Ty,
-                                            llvm::ArrayType::get(Int32Ty, 1));
-
-    // Grab the global __cpu_model.
-    llvm::Constant *CpuModel = CGM.CreateRuntimeVariable(STy, "__cpu_model");
-    cast<llvm::GlobalValue>(CpuModel)->setDSOLocal(true);
-
-    // Grab the first (0th) element from the field __cpu_features off of the
-    // global in the struct STy.
-    Value *Idxs[] = {Builder.getInt32(0), Builder.getInt32(3),
-                     Builder.getInt32(0)};
-    Value *CpuFeatures = Builder.CreateGEP(STy, CpuModel, Idxs);
-    Value *Features = Builder.CreateAlignedLoad(Int32Ty, CpuFeatures,
-                                                CharUnits::fromQuantity(4));
-
-    // Check the value of the bit corresponding to the feature requested.
-    Value *Mask = Builder.getInt32(Features1);
-    Value *Bitset = Builder.CreateAnd(Features, Mask);
-    Value *Cmp = Builder.CreateICmpEQ(Bitset, Mask);
-    Result = Builder.CreateAnd(Result, Cmp);
-  }
-
-  if (Features2 != 0) {
-    llvm::Constant *CpuFeatures2 = CGM.CreateRuntimeVariable(Int32Ty,
-                                                             "__cpu_features2");
-    cast<llvm::GlobalValue>(CpuFeatures2)->setDSOLocal(true);
-
-    Value *Features = Builder.CreateAlignedLoad(Int32Ty, CpuFeatures2,
-                                                CharUnits::fromQuantity(4));
-
-    // Check the value of the bit corresponding to the feature requested.
-    Value *Mask = Builder.getInt32(Features2);
-    Value *Bitset = Builder.CreateAnd(Features, Mask);
-    Value *Cmp = Builder.CreateICmpEQ(Bitset, Mask);
-    Result = Builder.CreateAnd(Result, Cmp);
-  }
-
-  return Result;
-}
-
-Value *CodeGenFunction::EmitX86CpuInit() {
-  llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy,
-                                                    /*Variadic*/ false);
-  llvm::FunctionCallee Func =
-      CGM.CreateRuntimeFunction(FTy, "__cpu_indicator_init");
-  cast<llvm::GlobalValue>(Func.getCallee())->setDSOLocal(true);
-  cast<llvm::GlobalValue>(Func.getCallee())
-      ->setDLLStorageClass(llvm::GlobalValue::DefaultStorageClass);
-  return Builder.CreateCall(Func);
+  return llvm::X86::emitCpuSupports(Builder, FeatureStr);
 }
 
 Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
                                            const CallExpr *E) {
   if (BuiltinID == X86::BI__builtin_cpu_is)
     return EmitX86CpuIs(E);
   if (BuiltinID == X86::BI__builtin_cpu_supports)
     return EmitX86CpuSupports(E);
   if (BuiltinID == X86::BI__builtin_cpu_init)
-    return EmitX86CpuInit();
+    return llvm::X86::emitCPUInit(Builder);
 
   // Handle MSVC intrinsics before argument evaluation to prevent double
   // evaluation.
   if (Optional<MSVCIntrin> MsvcIntId = translateX86ToMsvcIntrin(BuiltinID))
     return EmitMSVCBuiltinExpr(*MsvcIntId, E);
 
   SmallVector<Value*, 4> Ops;
   bool IsMaskFCmp = false;

clang/lib/CodeGen/CodeGenFunction.h

public:
 
   /// If \p E references a parameter with pass_object_size info or a constant
   /// array size modifier, emit the object size divided by the size of \p EltTy.
   /// Otherwise return null.
   llvm::Value *LoadPassedObjectSize(const Expr *E, QualType EltTy);
 
   void EmitSanitizerStatReport(llvm::SanitizerStatKind SSK);
 
-  struct MultiVersionResolverOption {
-    llvm::Function *Function;
-    struct Conds {
-      StringRef Architecture;
-      llvm::SmallVector<StringRef, 8> Features;
-
-      Conds(StringRef Arch, ArrayRef<StringRef> Feats)
-          : Architecture(Arch), Features(Feats.begin(), Feats.end()) {}
-    } Conditions;
-
-    MultiVersionResolverOption(llvm::Function *F, StringRef Arch,
-                               ArrayRef<StringRef> Feats)
-        : Function(F), Conditions(Arch, Feats) {}
-  };
-
-  // Emits the body of a multiversion function's resolver. Assumes that the
-  // options are already sorted in the proper order, with the 'default' option
-  // last (if it exists).
-  void EmitMultiVersionResolver(llvm::Function *Resolver,
-                                ArrayRef<MultiVersionResolverOption> Options);
-
 private:
   QualType getVarArgType(const Expr *Arg);
 
   void EmitDeclMetadata();
 
   BlockByrefHelpers *buildByrefHelpers(llvm::StructType &byrefType,
                                   const AutoVarEmission &emission);
 
   void AddObjCARCExceptionMetadata(llvm::Instruction *Inst);
 
   llvm::Value *GetValueForARMHint(unsigned BuiltinID);
   llvm::Value *EmitX86CpuIs(const CallExpr *E);
-  llvm::Value *EmitX86CpuIs(StringRef CPUStr);
   llvm::Value *EmitX86CpuSupports(const CallExpr *E);
-  llvm::Value *EmitX86CpuSupports(ArrayRef<StringRef> FeatureStrs);
-  llvm::Value *EmitX86CpuSupports(uint64_t Mask);
-  llvm::Value *EmitX86CpuInit();
-  llvm::Value *FormResolverCondition(const MultiVersionResolverOption &RO);
 };
 
 /// TargetFeatures - This class is used to check whether the builtin function
 /// has the required tagert specific features. It is able to support the
 /// combination of ','(and), '|'(or), and '()'. By default, the priority of
 /// ',' is higher than that of '|' .
 /// E.g:
 /// A,B|C means the builtin function requires both A and B, or C.

clang/lib/CodeGen/CodeGenFunction.cpp

 #include "llvm/IR/FPEnv.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
 #include "llvm/IR/MDBuilder.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/Support/CRC.h"
 #include "llvm/Transforms/Scalar/LowerExpectIntrinsic.h"
 #include "llvm/Transforms/Utils/PromoteMemToReg.h"
+#include "llvm/Transforms/Utils/X86EmitMultiVersionResolver.h"
 using namespace clang;
 using namespace CodeGen;
 
 /// shouldEmitLifetimeMarkers - Decide whether we need emit the life-time
 /// markers.
 static bool shouldEmitLifetimeMarkers(const CodeGenOptions &CGOpts,
                                       const LangOptions &LangOpts) {
   if (CGOpts.DisableLifetimeMarkers)
void CodeGenFunction::EmitSanitizerStatReport(llvm::SanitizerStatKind SSK) {
   if (!CGM.getCodeGenOpts().SanitizeStats)
     return;
 
   llvm::IRBuilder<> IRB(Builder.GetInsertBlock(), Builder.GetInsertPoint());
   IRB.SetCurrentDebugLocation(Builder.getCurrentDebugLocation());
   CGM.getSanStats().create(IRB, SSK);
 }
 
-llvm::Value *
-CodeGenFunction::FormResolverCondition(const MultiVersionResolverOption &RO) {
-  llvm::Value *Condition = nullptr;
-
-  if (!RO.Conditions.Architecture.empty())
-    Condition = EmitX86CpuIs(RO.Conditions.Architecture);
-
-  if (!RO.Conditions.Features.empty()) {
-    llvm::Value *FeatureCond = EmitX86CpuSupports(RO.Conditions.Features);
-    Condition =
-        Condition ? Builder.CreateAnd(Condition, FeatureCond) : FeatureCond;
-  }
-  return Condition;
-}
-
-static void CreateMultiVersionResolverReturn(CodeGenModule &CGM,
-                                             llvm::Function *Resolver,
-                                             CGBuilderTy &Builder,
-                                             llvm::Function *FuncToReturn,
-                                             bool SupportsIFunc) {
-  if (SupportsIFunc) {
-    Builder.CreateRet(FuncToReturn);
-    return;
-  }
-
-  llvm::SmallVector<llvm::Value *, 10> Args;
-  llvm::for_each(Resolver->args(),
-                 [&](llvm::Argument &Arg) { Args.push_back(&Arg); });
-
-  llvm::CallInst *Result = Builder.CreateCall(FuncToReturn, Args);
-  Result->setTailCallKind(llvm::CallInst::TCK_MustTail);
-
-  if (Resolver->getReturnType()->isVoidTy())
-    Builder.CreateRetVoid();
-  else
-    Builder.CreateRet(Result);
-}
-
-void CodeGenFunction::EmitMultiVersionResolver(
-    llvm::Function *Resolver, ArrayRef<MultiVersionResolverOption> Options) {
-  assert(getContext().getTargetInfo().getTriple().isX86() &&
-         "Only implemented for x86 targets");
-
-  bool SupportsIFunc = getContext().getTargetInfo().supportsIFunc();
-
-  // Main function's basic block.
-  llvm::BasicBlock *CurBlock = createBasicBlock("resolver_entry", Resolver);
-  Builder.SetInsertPoint(CurBlock);
-  EmitX86CpuInit();
-
-  for (const MultiVersionResolverOption &RO : Options) {
-    Builder.SetInsertPoint(CurBlock);
-    llvm::Value *Condition = FormResolverCondition(RO);
-
-    // The 'default' or 'generic' case.
-    if (!Condition) {
-      assert(&RO == Options.end() - 1 &&
-             "Default or Generic case must be last");
-      CreateMultiVersionResolverReturn(CGM, Resolver, Builder, RO.Function,
-                                       SupportsIFunc);
-      return;
-    }
-
-    llvm::BasicBlock *RetBlock = createBasicBlock("resolver_return", Resolver);
-    CGBuilderTy RetBuilder(*this, RetBlock);
-    CreateMultiVersionResolverReturn(CGM, Resolver, RetBuilder, RO.Function,
-                                     SupportsIFunc);
-    CurBlock = createBasicBlock("resolver_else", Resolver);
-    Builder.CreateCondBr(Condition, RetBlock, CurBlock);
-  }
-
-  // If no generic/default, emit an unreachable.
-  Builder.SetInsertPoint(CurBlock);
-  llvm::CallInst *TrapCall = EmitTrapCall(llvm::Intrinsic::trap);
-  TrapCall->setDoesNotReturn();
-  TrapCall->setDoesNotThrow();
-  Builder.CreateUnreachable();
-  Builder.ClearInsertionPoint();
-}
-
 // Loc - where the diagnostic will point, where in the source code this
 //  alignment has failed.
 // SecondaryLoc - if present (will be present if sufficiently different from
 //  Loc), the diagnostic will additionally point a "Note:" to this location.
 //  It should be the location where the __attribute__((assume_aligned))
 //  was written e.g.
 void CodeGenFunction::emitAlignmentAssumptionCheck(
     llvm::Value *Ptr, QualType Ty, SourceLocation Loc,

clang/lib/CodeGen/CodeGenModule.cpp

 #include "llvm/ProfileData/InstrProfReader.h"
 #include "llvm/Support/CodeGen.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ConvertUTF.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MD5.h"
 #include "llvm/Support/TimeProfiler.h"
 #include "llvm/Support/X86TargetParser.h"
+#include "llvm/Transforms/Utils/X86EmitMultiVersionResolver.h"
 
 using namespace clang;
 using namespace CodeGen;
 
 static llvm::cl::opt<bool> LimitedCoverage(
     "limited-coverage-experimental", llvm::cl::ZeroOrMore, llvm::cl::Hidden,
     llvm::cl::desc("Emit limited coverage mapping information (experimental)"),
     llvm::cl::init(false));
if (const auto *VD = dyn_cast<VarDecl>(D))
     return EmitGlobalVarDefinition(VD, !VD->hasDefinition());
 
   llvm_unreachable("Invalid argument to EmitGlobalDefinition()");
 }
 
 static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old,
                                                       llvm::Function *NewFn);
 
-static unsigned
-TargetMVPriority(const TargetInfo &TI,
-                 const CodeGenFunction::MultiVersionResolverOption &RO) {
+static unsigned TargetMVPriority(const TargetInfo &TI,
+                                 const llvm::MultiVersionResolverOption &RO) {
   unsigned Priority = 0;
   for (StringRef Feat : RO.Conditions.Features)
     Priority = std::max(Priority, TI.multiVersionSortPriority(Feat));
 
   if (!RO.Conditions.Architecture.empty())
     Priority = std::max(
         Priority, TI.multiVersionSortPriority(RO.Conditions.Architecture));
   return Priority;
 }
 
 void CodeGenModule::emitMultiVersionFunctions() {
   std::vector<GlobalDecl> MVFuncsToEmit;
   MultiVersionFuncs.swap(MVFuncsToEmit);
   for (GlobalDecl GD : MVFuncsToEmit) {
-    SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options;
+    SmallVector<llvm::MultiVersionResolverOption, 10> Options;
     const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
     getContext().forEachMultiversionedFunctionVersion(
         FD, [this, &GD, &Options](const FunctionDecl *CurFD) {
           GlobalDecl CurGD{
               (CurFD->isDefined() ? CurFD->getDefinition() : CurFD)};
           StringRef MangledName = getMangledName(CurGD);
           llvm::Constant *Func = GetGlobalValue(MangledName);
           if (!Func) {
if (TI.supportsIFunc() || FD->isTargetMultiVersion()) {
       ResolverFunc = cast<llvm::Function>(GetGlobalValue(getMangledName(GD)));
     }
 
     if (supportsCOMDAT())
       ResolverFunc->setComdat(
           getModule().getOrInsertComdat(ResolverFunc->getName()));
 
     llvm::stable_sort(
-        Options, [&TI](const CodeGenFunction::MultiVersionResolverOption &LHS,
-                       const CodeGenFunction::MultiVersionResolverOption &RHS) {
+        Options, [&TI](const llvm::MultiVersionResolverOption &LHS,
+                       const llvm::MultiVersionResolverOption &RHS) {
           return TargetMVPriority(TI, LHS) > TargetMVPriority(TI, RHS);
         });
-    CodeGenFunction CGF(*this);
-    CGF.EmitMultiVersionResolver(ResolverFunc, Options);
+    llvm::emitMultiVersionResolver(
+        ResolverFunc, Options, getContext().getTargetInfo().supportsIFunc());
   }
 
   // Ensure that any additions to the deferred decls list caused by emitting a
   // variant are emitted.  This can happen when the variant itself is inline and
   // calls a function without linkage.
   if (!MVFuncsToEmit.empty())
     EmitDeferred();
 
void CodeGenModule::emitCPUDispatchDefinition(GlobalDecl GD) {
 
   auto *ResolverFunc = cast<llvm::Function>(GetOrCreateLLVMFunction(
       ResolverName, ResolverType, ResolverGD, /*ForVTable=*/false));
   ResolverFunc->setLinkage(llvm::Function::WeakODRLinkage);
   if (supportsCOMDAT())
     ResolverFunc->setComdat(
         getModule().getOrInsertComdat(ResolverFunc->getName()));
 
-  SmallVector<CodeGenFunction::MultiVersionResolverOption, 10> Options;
+  SmallVector<llvm::MultiVersionResolverOption, 10> Options;
   const TargetInfo &Target = getTarget();
   unsigned Index = 0;
   for (const IdentifierInfo *II : DD->cpus()) {
     // Get the name of the target function so we can look it up/create it.
     std::string MangledName = getMangledNameImpl(*this, GD, FD, true) +
                               getCPUSpecificMangling(*this, II->getName());
 
     llvm::Constant *Func = GetGlobalValue(MangledName);
for (const IdentifierInfo *II : DD->cpus()) {
     Features.erase(std::remove_if(
         Features.begin(), Features.end(), [&Target](StringRef Feat) {
           return !Target.validateCpuSupports(Feat);
         }), Features.end());
     Options.emplace_back(cast<llvm::Function>(Func), StringRef{}, Features);
     ++Index;
   }
 
-  llvm::stable_sort(
-      Options, [](const CodeGenFunction::MultiVersionResolverOption &LHS,
-                  const CodeGenFunction::MultiVersionResolverOption &RHS) {
-        return llvm::X86::getCpuSupportsMask(LHS.Conditions.Features) >
-               llvm::X86::getCpuSupportsMask(RHS.Conditions.Features);
-      });
+  llvm::stable_sort(Options, [](const llvm::MultiVersionResolverOption &LHS,
+                                const llvm::MultiVersionResolverOption &RHS) {
 
   // If the list contains multiple 'default' versions, such as when it contains
   // 'pentium' and 'generic', don't emit the call to the generic one (since we
   // always run on at least a 'pentium'). We do this by deleting the 'least
   // advanced' (read, lowest mangling letter).
   while (Options.size() > 1 &&
          llvm::X86::getCpuSupportsMask(
              (Options.end() - 2)->Conditions.Features) == 0) {
-    StringRef LHSName = (Options.end() - 2)->Function->getName();
-    StringRef RHSName = (Options.end() - 1)->Function->getName();
+    StringRef LHSName = (Options.end() - 2)->Fn->getName();
+    StringRef RHSName = (Options.end() - 1)->Fn->getName();
     if (LHSName.compare(RHSName) < 0)
       Options.erase(Options.end() - 2);
     else
       Options.erase(Options.end() - 1);
   }
 
-  CodeGenFunction CGF(*this);
-  CGF.EmitMultiVersionResolver(ResolverFunc, Options);
+  llvm::emitMultiVersionResolver(ResolverFunc, Options,
+                                 getContext().getTargetInfo().supportsIFunc());
 
   if (getTarget().supportsIFunc()) {
     std::string AliasName = getMangledNameImpl(
         *this, GD, FD, /*OmitMultiVersionMangling=*/true);
     llvm::Constant *AliasFunc = GetGlobalValue(AliasName);
     if (!AliasFunc) {
       auto *IFunc = cast<llvm::GlobalIFunc>(GetOrCreateLLVMFunction(
           AliasName, DeclTy, GD, /*ForVTable=*/false, /*DontDefer=*/true,

llvm/include/llvm/Transforms/Utils/X86EmitMultiVersionResolver.h

+//===-- X86EmitMultiVersionResolver -----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements utitlities to generate code used for CPU dispatch code.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_UTILS_X86EMITMULTIVERSIONRESOLVER_H
+#define LLVM_TRANSFORMS_UTILS_X86EMITMULTIVERSIONRESOLVER_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/StringRef.h"
+
+namespace llvm {
+class Value;
+class Function;
+class IRBuilderBase;
+
+struct MultiVersionResolverOption {
+  Function *Fn;
+  struct Conds {
+    StringRef Architecture;
+    llvm::SmallVector<StringRef, 8> Features;
+
+    Conds(StringRef Arch, ArrayRef<StringRef> Feats)
+        : Architecture(Arch), Features(Feats.begin(), Feats.end()) {}
+  } Conditions;
+
+  MultiVersionResolverOption(Function *Fn, StringRef Arch,
+                             ArrayRef<StringRef> Feats)
+      : Fn(Fn), Conditions(Arch, Feats) {}
+};
+
+// Emits the body of a multiversion function's resolver. Assumes that the
+// options are already sorted in the proper order, with the 'default' option
+// last (if it exists).
+void emitMultiVersionResolver(Function *Resolver,
+                              ArrayRef<MultiVersionResolverOption> Options,
+                              bool UseIFunc);
+Value *formResolverCondition(IRBuilderBase &Builder,
+                             const MultiVersionResolverOption &RO);
+namespace X86 {
+Value *emitCPUInit(IRBuilderBase &Builder);
+Value *emitCpuIs(IRBuilderBase &Builder, StringRef CPUStr);
+Value *emitCpuSupports(IRBuilderBase &Builder, uint64_t FeaturesMask);
+Value *emitCpuSupports(IRBuilderBase &Builder, ArrayRef<StringRef> FeatureStrs);
+} // namespace X86
+} // namespace llvm
+
+#endif

llvm/lib/Transforms/Utils/CMakeLists.txt

add_llvm_component_library(LLVMTransformUtils
   SplitModule.cpp
   StripNonLineTableDebugInfo.cpp
   SymbolRewriter.cpp
   UnifyFunctionExitNodes.cpp
   UnifyLoopExits.cpp
   Utils.cpp
   ValueMapper.cpp
   VNCoercion.cpp
+  X86EmitMultiVersionResolver.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Transforms
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/Transforms/Utils
 
   DEPENDS
   intrinsics_gen
 
   LINK_COMPONENTS
   Analysis
   Core
   Support
   )

llvm/lib/Transforms/Utils/X86EmitMultiVersionResolver.cpp

+//===-- X86EmitMultiVersionResolver -----------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements utitlities to generate code used for CPU dispatch code.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Utils/X86EmitMultiVersionResolver.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Type.h"
+#include "llvm/Support/X86TargetParser.h"
+
+using namespace llvm;
+using namespace llvm::X86;
+
+Value *llvm::formResolverCondition(IRBuilderBase &Builder,
+                                   const MultiVersionResolverOption &RO) {
+  llvm::Value *Condition = nullptr;
+
+  if (!RO.Conditions.Architecture.empty())
+    Condition = llvm::X86::emitCpuIs(Builder, RO.Conditions.Architecture);
+  if (!RO.Conditions.Features.empty()) {
+    llvm::Value *FeatureCond =
+        llvm::X86::emitCpuSupports(Builder, RO.Conditions.Features);
+    Condition =
+        Condition ? Builder.CreateAnd(Condition, FeatureCond) : FeatureCond;
+  }
+  return Condition;
+}
+
+static void CreateMultiVersionResolverReturn(Function *Resolver,
+                                             IRBuilderBase &Builder,
+                                             Function *FuncToReturn,
+                                             bool UseIFunc) {
+  if (UseIFunc) {
+    Builder.CreateRet(FuncToReturn);
+    return;
+  }
+
+  SmallVector<Value *, 10> Args;
+  for_each(Resolver->args(), [&](Argument &Arg) { Args.push_back(&Arg); });
+
+  CallInst *Result = Builder.CreateCall(FuncToReturn, Args);
+  Result->setTailCallKind(CallInst::TCK_MustTail);
+
+  if (Resolver->getReturnType()->isVoidTy())
+    Builder.CreateRetVoid();
+  else
+    Builder.CreateRet(Result);
+}
+
+void llvm::emitMultiVersionResolver(
+    Function *Resolver, ArrayRef<MultiVersionResolverOption> Options,
+    bool UseIFunc) {
+  assert(Triple(Resolver->getParent()->getTargetTriple()).isX86() &&
+         "Only implemented for x86 targets");
+
+  auto &Ctx = Resolver->getContext();
+  // Main function's basic block.
+  BasicBlock *CurBlock = BasicBlock::Create(Ctx, "resolver_entry", Resolver);
+
+  IRBuilder<> Builder(CurBlock, CurBlock->begin());
+  llvm::X86::emitCPUInit(Builder);
+
+  for (const MultiVersionResolverOption &RO : Options) {
+    Builder.SetInsertPoint(CurBlock);
+    llvm::Value *Condition = formResolverCondition(Builder, RO);
+
+    // The 'default' or 'generic' case.
+    if (!Condition) {
+      assert(&RO == Options.end() - 1 &&
+             "Default or Generic case must be last");
+      CreateMultiVersionResolverReturn(Resolver, Builder, RO.Fn, UseIFunc);
+      return;
+    }
+
+    llvm::BasicBlock *RetBlock =
+        BasicBlock::Create(Ctx, "resolver_return", Resolver);
+    {
+      IRBuilderBase::InsertPointGuard Guard(Builder);
+      Builder.SetInsertPoint(RetBlock);
+      CreateMultiVersionResolverReturn(Resolver, Builder, RO.Fn, UseIFunc);
+    }
+    CurBlock = BasicBlock::Create(Ctx, "resolver_else", Resolver);
+    Builder.CreateCondBr(Condition, RetBlock, CurBlock);
+  }
+
+  // If no generic/default, emit an unreachable.
+  Builder.SetInsertPoint(CurBlock);
+  CallInst *TrapCall = Builder.CreateIntrinsic(Intrinsic::trap, {}, {});
+  TrapCall->setDoesNotReturn();
+  TrapCall->setDoesNotThrow();
+  Builder.CreateUnreachable();
+}
+
+static Type *getCpuModelType(IRBuilderBase &Builder) {
+  Type *Int32Ty = Builder.getInt32Ty();
+
+  // Matching the struct layout from the compiler-rt/libgcc structure that is
+  // filled in:
+  // unsigned int __cpu_vendor;
+  // unsigned int __cpu_type;
+  // unsigned int __cpu_subtype;
+  // unsigned int __cpu_features[1];
+  Type *STy =
+      StructType::get(Int32Ty, Int32Ty, Int32Ty, ArrayType::get(Int32Ty, 1));
+  return STy;
+}
+
+static Value *getOrCreateGlobal(IRBuilderBase &Builder, StringRef Name,
+                                Type *Ty) {
+  Module *M = Builder.GetInsertBlock()->getParent()->getParent();
+  if (GlobalValue *Val = M->getNamedValue(Name)) {
+    return Val;
+  }
+
+  auto *New = new GlobalVariable(
+      *M, Ty, false, GlobalValue::ExternalLinkage, nullptr, Name, nullptr,
+      GlobalVariable::NotThreadLocal, 0 /* AddressSpace */);
+  New->setDSOLocal(true);
+  return New;
+}
+
+Value *llvm::X86::emitCPUInit(IRBuilderBase &Builder) {
+  FunctionType *FTy = FunctionType::get(Builder.getVoidTy(),
+                                        /*Variadic*/ false);
+  Module *M = Builder.GetInsertBlock()->getParent()->getParent();
+  GlobalValue *Entry = M->getNamedValue("__cpu_indicator_init");
+  if (Entry) {
+    // TODO: asserts and possibly return nullptr if something bad.
+    return Builder.CreateCall(cast<Function>(Entry));
+  }
+
+  Function *F = Function::Create(FTy, Function::ExternalLinkage,
+                                 "__cpu_indicator_init", M);
+
+  F->setDSOLocal(true);
+  F->setDLLStorageClass(GlobalValue::DefaultStorageClass);
+  return Builder.CreateCall(F);
+}
+
+Value *llvm::X86::emitCpuIs(IRBuilderBase &Builder, StringRef CPUStr) {
+  // Calculate the index needed to access the correct field based on the
+  // range. Also adjust the expected value.
+  unsigned Index;
+  unsigned Val;
+  std::tie(Index, Val) = StringSwitch<std::pair<unsigned, unsigned>>(CPUStr)
+#define X86_VENDOR(ENUM, STRING)                                               \
+  .Case(STRING, {0u, static_cast<unsigned>(llvm::X86::ENUM)})
+#define X86_CPU_TYPE_ALIAS(ENUM, ALIAS)                                        \
+  .Case(ALIAS, {1u, static_cast<unsigned>(llvm::X86::ENUM)})
+#define X86_CPU_TYPE(ENUM, STR)                                                \
+  .Case(STR, {1u, static_cast<unsigned>(llvm::X86::ENUM)})
+#define X86_CPU_SUBTYPE(ENUM, STR)                                             \
+  .Case(STR, {2u, static_cast<unsigned>(llvm::X86::ENUM)})
+#include "llvm/Support/X86TargetParser.def"
+                             .Default({0, 0});
+  assert(Val != 0 && "Invalid CPUStr passed to CpuIs");
+
+  // Grab the appropriate field from __cpu_model.
+  Value *Idxs[] = {Builder.getInt32(0), Builder.getInt32(Index)};
+  Type *CpuModelType = getCpuModelType(Builder);
+  Value *CpuValue = Builder.CreateGEP(
+      CpuModelType, getOrCreateGlobal(Builder, "__cpu_model", CpuModelType),
+      Idxs);
+  CpuValue =
+      Builder.CreateAlignedLoad(Builder.getInt32Ty(), CpuValue, MaybeAlign(4));
+
+  // Check the value of the field against the requested value.
+  return Builder.CreateICmpEQ(CpuValue, Builder.getInt32(Val));
+}
+
+Value *llvm::X86::emitCpuSupports(IRBuilderBase &Builder,
+                                  uint64_t FeaturesMask) {
+  uint32_t Features1 = Lo_32(FeaturesMask);
+  uint32_t Features2 = Hi_32(FeaturesMask);
+
+  Value *Result = Builder.getTrue();
+  Type *Int32Ty = Builder.getInt32Ty();
+  Type *CpuModelType = getCpuModelType(Builder);
+
+  if (Features1 != 0) {
+    Value *CpuModel = getOrCreateGlobal(Builder, "__cpu_model", CpuModelType);
+
+    // Grab the first (0th) element from the field __cpu_features off of the
+    // global in the struct STy.
+    Value *Idxs[] = {Builder.getInt32(0), Builder.getInt32(3),
+                     Builder.getInt32(0)};
+    Value *CpuFeatures = Builder.CreateGEP(CpuModelType, CpuModel, Idxs);
+    Value *Features =
+        Builder.CreateAlignedLoad(Int32Ty, CpuFeatures, MaybeAlign(4));
+
+    // Check the value of the bit corresponding to the feature requested.
+    Value *Mask = Builder.getInt32(Features1);
+    Value *Bitset = Builder.CreateAnd(Features, Mask);
+    Value *Cmp = Builder.CreateICmpEQ(Bitset, Mask);
+    Result = Builder.CreateAnd(Result, Cmp);
+  }
+
+  if (Features2 != 0) {
+    Value *CpuFeatures2 =
+        getOrCreateGlobal(Builder, "__cpu_features2", Int32Ty);
+
+    Value *Features =
+        Builder.CreateAlignedLoad(Int32Ty, CpuFeatures2, Align(4));
+
+    // Check the value of the bit corresponding to the feature requested.
+    Value *Mask = Builder.getInt32(Features2);
+    Value *Bitset = Builder.CreateAnd(Features, Mask);
+    Value *Cmp = Builder.CreateICmpEQ(Bitset, Mask);
+    Result = Builder.CreateAnd(Result, Cmp);
+  }
+
+  return Result;
+}
+
+Value *llvm::X86::emitCpuSupports(IRBuilderBase &Builder,
+                                  ArrayRef<StringRef> FeatureStrs) {
+  return emitCpuSupports(Builder, getCpuSupportsMask(FeatureStrs));
+}