AMDGPU: Refactor libcall simplify to help with future refined fast math flag usage

[jmmartinez]

llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp

642

For consistency with the rest of the fold_* functions, shouldn't fold_rootn take an FPMathOperator?

[arsenm]

More FPMathOperator

[arsenm]

8f381380905c159bafb8db3b9ce699ec182e892c

llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp

 
 class AMDGPULibCalls {
 private:
 
   typedef llvm::AMDGPULibFunc FuncInfo;
 
   const TargetMachine *TM;
 
+  bool UnsafeFPMath = false;
+
   // -fuse-native.
   bool AllNative = false;
 
   bool useNativeFunc(const StringRef F) const;
 
   // Return a pointer (pointer expr) to the function if function definition with
   // "FuncName" exists. It may create a new function prototype in pre-link mode.
   FunctionCallee getFunction(Module *M, const FuncInfo &fInfo);
 
   bool parseFunctionName(const StringRef &FMangledName, FuncInfo &FInfo);
 
   bool TDOFold(CallInst *CI, const FuncInfo &FInfo);
 
   /* Specialized optimizations */
 
   // pow/powr/pown
-  bool fold_pow(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
+  bool fold_pow(FPMathOperator *FPOp, IRBuilder<> &B,
+                const FuncInfo &FInfo);
 
   // rootn
-  bool fold_rootn(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
+  bool fold_rootn(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo);
 
   // -fuse-native for sincos
   bool sincosUseNative(CallInst *aCI, const FuncInfo &FInfo);
 
   // evaluate calls if calls' arguments are constants.
   bool evaluateScalarMathFunc(const FuncInfo &FInfo, double& Res0,
     double& Res1, Constant *copr0, Constant *copr1, Constant *copr2);
   bool evaluateCall(CallInst *aCI, const FuncInfo &FInfo);
 
   // sqrt
-  bool fold_sqrt(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo);
+  bool fold_sqrt(FPMathOperator *FPOp, IRBuilder<> &B,
+                 const FuncInfo &FInfo);
 
   // sin/cos
-  bool fold_sincos(CallInst *CI, IRBuilder<> &B, const FuncInfo &FInfo,
+  bool fold_sincos(FPMathOperator *FPOp, IRBuilder<> &B, const FuncInfo &FInfo,
                    AliasAnalysis *AA);
 
   // __read_pipe/__write_pipe
   bool fold_read_write_pipe(CallInst *CI, IRBuilder<> &B,
                             const FuncInfo &FInfo);
 
   // llvm.amdgcn.wavefrontsize
   bool fold_wavefrontsize(CallInst *CI, IRBuilder<> &B);
 
   // Get insertion point at entry.
   BasicBlock::iterator getEntryIns(CallInst * UI);
   // Insert an Alloc instruction.
   AllocaInst* insertAlloca(CallInst * UI, IRBuilder<> &B, const char *prefix);
   // Get a scalar native builtin single argument FP function
   FunctionCallee getNativeFunction(Module *M, const FuncInfo &FInfo);
 
 protected:
   CallInst *CI;
 
-  bool isUnsafeMath(const CallInst *CI) const;
+  bool isUnsafeMath(const FPMathOperator *FPOp) const;
+
+  bool canIncreasePrecisionOfConstantFold(const FPMathOperator *FPOp) const;
 
   void replaceCall(Value *With) {
     CI->replaceAllUsesWith(With);
     CI->eraseFromParent();
   }
 
 public:
   AMDGPULibCalls(const TargetMachine *TM_ = nullptr) : TM(TM_) {}
 
   bool fold(CallInst *CI, AliasAnalysis *AA = nullptr);
 
+  void initFunction(const Function &F);
   void initNativeFuncs();
 
   // Replace a normal math function call with that native version
   bool useNative(CallInst *CI);
 };
 
 } // end llvm namespace
 
return EnablePreLink ? AMDGPULibFunc::getOrInsertFunction(M, fInfo)
                        : AMDGPULibFunc::getFunction(M, fInfo);
 }
 
 bool AMDGPULibCalls::parseFunctionName(const StringRef &FMangledName,
                                        FuncInfo &FInfo) {
   return AMDGPULibFunc::parse(FMangledName, FInfo);
 }
 
-bool AMDGPULibCalls::isUnsafeMath(const CallInst *CI) const {
-  if (auto Op = dyn_cast<FPMathOperator>(CI))
-    if (Op->isFast())
-      return true;
-  const Function *F = CI->getParent()->getParent();
-  Attribute Attr = F->getFnAttribute("unsafe-fp-math");
-  return Attr.getValueAsBool();
+bool AMDGPULibCalls::isUnsafeMath(const FPMathOperator *FPOp) const {
+  return UnsafeFPMath || FPOp->isFast();
+}
+
+bool AMDGPULibCalls::canIncreasePrecisionOfConstantFold(
+    const FPMathOperator *FPOp) const {
+  // TODO: Refine to approxFunc or contract
+  return isUnsafeMath(FPOp);
+}
+
+void AMDGPULibCalls::initFunction(const Function &F) {
+  UnsafeFPMath = F.getFnAttribute("unsafe-fp-math").getValueAsBool();
 }
 
 bool AMDGPULibCalls::useNativeFunc(const StringRef F) const {
   return AllNative || llvm::is_contained(UseNative, F);
 }
 
 void AMDGPULibCalls::initNativeFuncs() {
   AllNative = useNativeFunc("all") ||
bool AMDGPULibCalls::fold(CallInst *CI, AliasAnalysis *AA) {
   if (!FInfo.isCompatibleSignature(CI->getFunctionType()))
     return false;
 
   LLVM_DEBUG(dbgs() << "AMDIC: try folding " << *CI << '\n');
 
   if (TDOFold(CI, FInfo))
     return true;
 
+  if (FPMathOperator *FPOp = dyn_cast<FPMathOperator>(CI)) {
     // Under unsafe-math, evaluate calls if possible.
     // According to Brian Sumner, we can do this for all f32 function calls
     // using host's double function calls.
-  if (isUnsafeMath(CI) && evaluateCall(CI, FInfo))
+    if (canIncreasePrecisionOfConstantFold(FPOp) && evaluateCall(CI, FInfo))
       return true;
 
     // Copy fast flags from the original call.
-  if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(CI))
     B.setFastMathFlags(FPOp->getFastMathFlags());
 
     // Specialized optimizations for each function call
     switch (FInfo.getId()) {
     case AMDGPULibFunc::EI_POW:
     case AMDGPULibFunc::EI_POWR:
     case AMDGPULibFunc::EI_POWN:
-    return fold_pow(CI, B, FInfo);
-
+      return fold_pow(FPOp, B, FInfo);
     case AMDGPULibFunc::EI_ROOTN:
-    // skip vector function
-    return (getVecSize(FInfo) != 1) ? false : fold_rootn(CI, B, FInfo);
-
+      return fold_rootn(FPOp, B, FInfo);
     case AMDGPULibFunc::EI_SQRT:
-    return isUnsafeMath(CI) && fold_sqrt(CI, B, FInfo);
+      return fold_sqrt(FPOp, B, FInfo);
     case AMDGPULibFunc::EI_COS:
     case AMDGPULibFunc::EI_SIN:
-    if ((getArgType(FInfo) == AMDGPULibFunc::F32 ||
-         getArgType(FInfo) == AMDGPULibFunc::F64)
-        && (FInfo.getPrefix() == AMDGPULibFunc::NOPFX))
-      return fold_sincos(CI, B, FInfo, AA);
-
+      return fold_sincos(FPOp, B, FInfo, AA);
+    default:
       break;
+    }
+  } else {
+    // Specialized optimizations for each function call
+    switch (FInfo.getId()) {
     case AMDGPULibFunc::EI_READ_PIPE_2:
     case AMDGPULibFunc::EI_READ_PIPE_4:
     case AMDGPULibFunc::EI_WRITE_PIPE_2:
     case AMDGPULibFunc::EI_WRITE_PIPE_4:
       return fold_read_write_pipe(CI, B, FInfo);
-
     default:
       break;
     }
+  }
 
   return false;
 }
 
 bool AMDGPULibCalls::TDOFold(CallInst *CI, const FuncInfo &FInfo) {
   // Table-Driven optimization
   const TableRef tr = getOptTable(FInfo.getId());
   if (tr.empty())

 #if _XOPEN_SOURCE >= 600 || defined(_ISOC99_SOURCE) || _POSIX_C_SOURCE >= 200112L
   return ::log2(V);
 #else
   return log(V) / numbers::ln2;
 #endif
 }
 }
 
-bool AMDGPULibCalls::fold_pow(CallInst *CI, IRBuilder<> &B,
+bool AMDGPULibCalls::fold_pow(FPMathOperator *FPOp, IRBuilder<> &B,
                               const FuncInfo &FInfo) {
   assert((FInfo.getId() == AMDGPULibFunc::EI_POW ||
           FInfo.getId() == AMDGPULibFunc::EI_POWR ||
           FInfo.getId() == AMDGPULibFunc::EI_POWN) &&
          "fold_pow: encounter a wrong function call");
 
   Value *opr0, *opr1;
   ConstantFP *CF;
if (getVecSize(FInfo) == 1) {
     ConstantDataVector *CDV = dyn_cast<ConstantDataVector>(opr1);
 
     // Now, only Handle vector const whose elements have the same value.
     CF = CDV ? dyn_cast_or_null<ConstantFP>(CDV->getSplatValue()) : nullptr;
     CINT = CDV ? dyn_cast_or_null<ConstantInt>(CDV->getSplatValue()) : nullptr;
   }
 
   // No unsafe math , no constant argument, do nothing
-  if (!isUnsafeMath(CI) && !CF && !CINT && !CZero)
+  if (!isUnsafeMath(FPOp) && !CF && !CINT && !CZero)
     return false;
 
   // 0x1111111 means that we don't do anything for this call.
   int ci_opr1 = (CINT ? (int)CINT->getSExtValue() : 0x1111111);
 
   if ((CF && CF->isZero()) || (CINT && ci_opr1 == 0) || CZero) {
     //  pow/powr/pown(x, 0) == 1
     LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> 1\n");
if (FunctionCallee FPExpr =
                         << FInfo.getName().c_str() << "(" << *opr0 << ")\n");
       Value *nval = CreateCallEx(B,FPExpr, opr0, issqrt ? "__pow2sqrt"
                                                         : "__pow2rsqrt");
       replaceCall(nval);
       return true;
     }
   }
 
-  if (!isUnsafeMath(CI))
+  if (!isUnsafeMath(FPOp))
     return false;
 
   // Unsafe Math optimization
 
   // Remember that ci_opr1 is set if opr1 is integral
   if (CF) {
     double dval = (getArgType(FInfo) == AMDGPULibFunc::F32)
                     ? (double)CF->getValueAPF().convertToFloat()
bool AMDGPULibCalls::fold_pow(FPMathOperator *FPOp, IRBuilder<> &B,
 
   LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> "
                     << "exp2(" << *opr1 << " * log2(" << *opr0 << "))\n");
   replaceCall(nval);
 
   return true;
 }
 
-bool AMDGPULibCalls::fold_rootn(CallInst *CI, IRBuilder<> &B,
+bool AMDGPULibCalls::fold_rootn(FPMathOperator *FPOp, IRBuilder<> &B,
                                 const FuncInfo &FInfo) {
-  Value *opr0 = CI->getArgOperand(0);
-  Value *opr1 = CI->getArgOperand(1);
+  // skip vector function
+  if (getVecSize(FInfo) != 1)
+    return false;
+
+  Value *opr0 = FPOp->getOperand(0);
+  Value *opr1 = FPOp->getOperand(1);
 
   ConstantInt *CINT = dyn_cast<ConstantInt>(opr1);
   if (!CINT) {
     return false;
   }
   int ci_opr1 = (int)CINT->getSExtValue();
   if (ci_opr1 == 1) {  // rootn(x, 1) = x
     LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> " << *opr0 << "\n");
FunctionCallee AMDGPULibCalls::getNativeFunction(Module *M,
   if (getArgType(FInfo) == AMDGPULibFunc::F64 || !HasNative(FInfo.getId()))
     return nullptr;
   FuncInfo nf = FInfo;
   nf.setPrefix(AMDGPULibFunc::NATIVE);
   return getFunction(M, nf);
 }
 
 // fold sqrt -> native_sqrt (x)
-bool AMDGPULibCalls::fold_sqrt(CallInst *CI, IRBuilder<> &B,
+bool AMDGPULibCalls::fold_sqrt(FPMathOperator *FPOp, IRBuilder<> &B,
                                const FuncInfo &FInfo) {
+  if (!isUnsafeMath(FPOp))
+    return false;
+
   if (getArgType(FInfo) == AMDGPULibFunc::F32 && (getVecSize(FInfo) == 1) &&
       (FInfo.getPrefix() != AMDGPULibFunc::NATIVE)) {
     if (FunctionCallee FPExpr = getNativeFunction(
             CI->getModule(), AMDGPULibFunc(AMDGPULibFunc::EI_SQRT, FInfo))) {
       Value *opr0 = CI->getArgOperand(0);
       LLVM_DEBUG(errs() << "AMDIC: " << *CI << " ---> "
                         << "sqrt(" << *opr0 << ")\n");
       Value *nval = CreateCallEx(B,FPExpr, opr0, "__sqrt");
       replaceCall(nval);
       return true;
     }
   }
   return false;
 }
 
 // fold sin, cos -> sincos.
-bool AMDGPULibCalls::fold_sincos(CallInst *CI, IRBuilder<> &B,
+bool AMDGPULibCalls::fold_sincos(FPMathOperator *FPOp, IRBuilder<> &B,
                                  const FuncInfo &fInfo, AliasAnalysis *AA) {
   assert(fInfo.getId() == AMDGPULibFunc::EI_SIN ||
          fInfo.getId() == AMDGPULibFunc::EI_COS);
+
+  if ((getArgType(fInfo) != AMDGPULibFunc::F32 &&
+       getArgType(fInfo) != AMDGPULibFunc::F64) ||
+      fInfo.getPrefix() != AMDGPULibFunc::NOPFX)
+    return false;
+
   bool const isSin = fInfo.getId() == AMDGPULibFunc::EI_SIN;
 
   Value *CArgVal = CI->getArgOperand(0);
   BasicBlock * const CBB = CI->getParent();
 
   int const MaxScan = 30;
   bool Changed = false;
 

 FunctionPass *llvm::createAMDGPUUseNativeCallsPass() {
   return new AMDGPUUseNativeCalls();
 }
 
 bool AMDGPUSimplifyLibCalls::runOnFunction(Function &F) {
   if (skipFunction(F))
     return false;
 
+  Simplifier.initFunction(F);
+
   bool Changed = false;
   auto AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
 
   LLVM_DEBUG(dbgs() << "AMDIC: process function ";
              F.printAsOperand(dbgs(), false, F.getParent()); dbgs() << '\n';);
 
   for (auto &BB : F) {
     for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ) {
       // Ignore non-calls.
       CallInst *CI = dyn_cast<CallInst>(I);
       ++I;
       if (CI) {
         if (Simplifier.fold(CI, AA))
           Changed = true;
       }
     }
   }
   return Changed;
 }
 
 PreservedAnalyses AMDGPUSimplifyLibCallsPass::run(Function &F,
                                                   FunctionAnalysisManager &AM) {
   AMDGPULibCalls Simplifier(&TM);
   Simplifier.initNativeFuncs();
+  Simplifier.initFunction(F);
 
   bool Changed = false;
   auto AA = &AM.getResult<AAManager>(F);
 
   LLVM_DEBUG(dbgs() << "AMDIC: process function ";
              F.printAsOperand(dbgs(), false, F.getParent()); dbgs() << '\n';);
 
   for (auto &BB : F) {
PreservedAnalyses AMDGPUSimplifyLibCallsPass::run(Function &F,
   }
   return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
 }
 
 bool AMDGPUUseNativeCalls::runOnFunction(Function &F) {
   if (skipFunction(F) || UseNative.empty())
     return false;
 
+  Simplifier.initFunction(F);
+
   bool Changed = false;
   for (auto &BB : F) {
     for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ) {
       // Ignore non-calls.
       CallInst *CI = dyn_cast<CallInst>(I);
       ++I;
       if (CI && Simplifier.useNative(CI))
         Changed = true;
     }
   }
   return Changed;
 }
 
 PreservedAnalyses AMDGPUUseNativeCallsPass::run(Function &F,
                                                 FunctionAnalysisManager &AM) {
   if (UseNative.empty())
     return PreservedAnalyses::all();
 
   AMDGPULibCalls Simplifier;
   Simplifier.initNativeFuncs();
+  Simplifier.initFunction(F);
 
   bool Changed = false;
   for (auto &BB : F) {
     for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E;) {
       // Ignore non-calls.
       CallInst *CI = dyn_cast<CallInst>(I);
       ++I;
       if (CI && Simplifier.useNative(CI))
         Changed = true;
     }
   }
   return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
 }