[AMDGPU] Set fast-math flags on functions given the options

[rampitec]

Summary

We have a single library build without relaxation options.
When inlined library functions remove fast math attributes
from the functions they are integrated into.

This patch sets relaxation attributes on the functions after
linking provided corresponding relaxation options are given.
Math instructions inside the inlined functions remain to have
no fast flags, but inlining does not prevent fast math
transformations of a surrounding caller code anymore.

[dfukalov]

otherwise LGTM

lib/Target/AMDGPU/AMDGPUTargetMachine.cpp

375 ↗

(On Diff #116851)

It possible would be better to create temporary ref for Options member and capture it instead of whole this pointer

[rampitec]

lib/Target/AMDGPU/AMDGPUTargetMachine.cpp

375 ↗

(On Diff #116851)

It is just a pointer, not the whole content under it.

[rampitec]

lib/Target/AMDGPU/AMDGPUTargetMachine.cpp

375 ↗

(On Diff #116851)

I.e. it will be one line of code more:

const auto &Opt = Options;
Builder.addExtension(
  PassManagerBuilder::EP_EarlyAsPossible,
  [AMDGPUAA, LibCallSimplify, &Opt](const PassManagerBuilder &,
                                    legacy::PassManagerBase &PM) {
    if (AMDGPUAA) {
      PM.add(createAMDGPUAAWrapperPass());
      PM.add(createAMDGPUExternalAAWrapperPass());
    }
    PM.add(llvm::createAMDGPUUseNativeCallsPass());
    if (LibCallSimplify)
      PM.add(llvm::createAMDGPUSimplifyLibCallsPass(Opt));
});

but the capture size is still the same sizeof(void*).

[rampitec]

Created reference to Options to capture as requested.

llvm/trunk/lib/Target/AMDGPU/AMDGPU.h

 
 class AMDGPUTargetMachine;
 class FunctionPass;
 class GCNTargetMachine;
 class ModulePass;
 class Pass;
 class Target;
 class TargetMachine;
+class TargetOptions;
 class PassRegistry;
 class Module;
 
 // R600 Passes
 FunctionPass *createR600VectorRegMerger();
 FunctionPass *createR600ExpandSpecialInstrsPass();
 FunctionPass *createR600EmitClauseMarkers();
 FunctionPass *createR600ClauseMergePass();

 FunctionPass *createSIFixControlFlowLiveIntervalsPass();
 FunctionPass *createSIOptimizeExecMaskingPreRAPass();
 FunctionPass *createSIFixSGPRCopiesPass();
 FunctionPass *createSIMemoryLegalizerPass();
 FunctionPass *createSIDebuggerInsertNopsPass();
 FunctionPass *createSIInsertWaitsPass();
 FunctionPass *createSIInsertWaitcntsPass();
 FunctionPass *createSIFixWWMLivenessPass();
-FunctionPass *createAMDGPUSimplifyLibCallsPass();
+FunctionPass *createAMDGPUSimplifyLibCallsPass(const TargetOptions &);
 FunctionPass *createAMDGPUUseNativeCallsPass();
 FunctionPass *createAMDGPUCodeGenPreparePass();
 FunctionPass *createAMDGPUMachineCFGStructurizerPass();
 FunctionPass *createAMDGPURewriteOutArgumentsPass();
 
 void initializeAMDGPUDAGToDAGISelPass(PassRegistry&);
 
 void initializeAMDGPUMachineCFGStructurizerPass(PassRegistry&);

llvm/trunk/lib/Target/AMDGPU/AMDGPULibCalls.cpp

 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/ValueSymbolTable.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetOptions.h"
 #include <vector>
 #include <cmath>
 
 using namespace llvm;
 
 static cl::opt<bool> EnablePreLink("amdgpu-prelink",
   cl::desc("Enable pre-link mode optimizations"),
   cl::init(false),

 } // end llvm namespace
 
 namespace {
 
   class AMDGPUSimplifyLibCalls : public FunctionPass {
 
   AMDGPULibCalls Simplifier;
 
+  const TargetOptions Options;
+
   public:
     static char ID; // Pass identification
 
-    AMDGPUSimplifyLibCalls() : FunctionPass(ID) {
+    AMDGPUSimplifyLibCalls(const TargetOptions &Opt = TargetOptions())
+      : FunctionPass(ID), Options(Opt) {
       initializeAMDGPUSimplifyLibCallsPass(*PassRegistry::getPassRegistry());
     }
 
     void getAnalysisUsage(AnalysisUsage &AU) const override {
       AU.addRequired<AAResultsWrapperPass>();
     }
 
     bool runOnFunction(Function &M) override;
if (hasTwoResults) {
     new StoreInst(nval1, aCI->getArgOperand(1), aCI);
   }
 
   replaceCall(nval0);
   return true;
 }
 
 // Public interface to the Simplify LibCalls pass.
-FunctionPass *llvm::createAMDGPUSimplifyLibCallsPass() {
-  return new AMDGPUSimplifyLibCalls();
+FunctionPass *llvm::createAMDGPUSimplifyLibCallsPass(const TargetOptions &Opt) {
+  return new AMDGPUSimplifyLibCalls(Opt);
 }
 
 FunctionPass *llvm::createAMDGPUUseNativeCallsPass() {
   return new AMDGPUUseNativeCalls();
 }
 
+static bool setFastFlags(Function &F, const TargetOptions &Options) {
+  AttrBuilder B;
+
+  if (Options.UnsafeFPMath || Options.NoInfsFPMath)
+    B.addAttribute("no-infs-fp-math", "true");
+  if (Options.UnsafeFPMath || Options.NoNaNsFPMath)
+    B.addAttribute("no-nans-fp-math", "true");
+  if (Options.UnsafeFPMath) {
+    B.addAttribute("less-precise-fpmad", "true");
+    B.addAttribute("unsafe-fp-math", "true");
+  }
+
+  if (!B.hasAttributes())
+    return false;
+
+  F.addAttributes(AttributeList::FunctionIndex, B);
+
+  return true;
+}
+
 bool AMDGPUSimplifyLibCalls::runOnFunction(Function &F) {
   if (skipFunction(F))
     return false;
 
   bool Changed = false;
   auto AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
 
   DEBUG(dbgs() << "AMDIC: process function ";
         F.printAsOperand(dbgs(), false, F.getParent());
         dbgs() << '\n';);
 
+  if (!EnablePreLink)
+    Changed |= setFastFlags(F, Options);
+
   for (auto &BB : F) {
     for (BasicBlock::iterator I = BB.begin(), E = BB.end(); I != E; ) {
       // Ignore non-calls.
       CallInst *CI = dyn_cast<CallInst>(I);
       ++I;
       if (!CI) continue;
 
       // Ignore indirect calls.

llvm/trunk/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp

[Internalize, EarlyInline, AMDGPUAA](const PassManagerBuilder &,
       if (Internalize) {
         PM.add(createInternalizePass(mustPreserveGV));
         PM.add(createGlobalDCEPass());
       }
       if (EarlyInline)
         PM.add(createAMDGPUAlwaysInlinePass(false));
   });
 
+  const auto &Opt = Options;
   Builder.addExtension(
     PassManagerBuilder::EP_EarlyAsPossible,
-    [AMDGPUAA, LibCallSimplify](const PassManagerBuilder &,
+    [AMDGPUAA, LibCallSimplify, &Opt](const PassManagerBuilder &,
                                       legacy::PassManagerBase &PM) {
       if (AMDGPUAA) {
         PM.add(createAMDGPUAAWrapperPass());
         PM.add(createAMDGPUExternalAAWrapperPass());
       }
       PM.add(llvm::createAMDGPUUseNativeCallsPass());
       if (LibCallSimplify)
-        PM.add(llvm::createAMDGPUSimplifyLibCallsPass());
+        PM.add(llvm::createAMDGPUSimplifyLibCallsPass(Opt));
   });
 
   Builder.addExtension(
     PassManagerBuilder::EP_CGSCCOptimizerLate,
     [](const PassManagerBuilder &, legacy::PassManagerBase &PM) {
       // Add infer address spaces pass to the opt pipeline after inlining
       // but before SROA to increase SROA opportunities.
       PM.add(createInferAddressSpacesPass());

llvm/trunk/test/CodeGen/AMDGPU/inline-attr.ll

+; RUN: opt -mtriple=amdgcn--amdhsa -S -O3 -enable-unsafe-fp-math %s  | FileCheck -check-prefix=GCN -check-prefix=UNSAFE %s
+; RUN: opt -mtriple=amdgcn--amdhsa -S -O3 -enable-no-nans-fp-math %s | FileCheck -check-prefix=GCN -check-prefix=NONANS %s
+; RUN: opt -mtriple=amdgcn--amdhsa -S -O3 -enable-no-infs-fp-math %s | FileCheck -check-prefix=GCN -check-prefix=NOINFS %s
+
+; GCN: define float @foo(float %x) local_unnamed_addr #0 {
+; GCN: define amdgpu_kernel void @caller(float addrspace(1)* nocapture %p) local_unnamed_addr #1 {
+; GCN: %mul.i = fmul float %load, 1.500000e+01
+
+; UNSAFE: attributes #0 = { norecurse nounwind readnone "less-precise-fpmad"="true" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "unsafe-fp-math"="true" }
+; UNSAFE: attributes #1 = { norecurse nounwind "less-precise-fpmad"="true" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "unsafe-fp-math"="true" }
+
+; NOINFS: attributes #0 = { norecurse nounwind readnone "no-infs-fp-math"="true" }
+; NOINFS: attributes #1 = { norecurse nounwind "less-precise-fpmad"="false" "no-infs-fp-math"="true" "no-nans-fp-math"="false" "unsafe-fp-math"="false" }
+
+; NONANS: attributes #0 = { norecurse nounwind readnone "no-nans-fp-math"="true" }
+; NONANS: attributes #1 = { norecurse nounwind "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="true" "unsafe-fp-math"="false" }
+
+define float @foo(float %x) #0 {
+entry:
+  %mul = fmul float %x, 1.500000e+01
+  ret float %mul
+}
+
+define amdgpu_kernel void @caller(float addrspace(1)* %p) #1 {
+entry:
+  %load = load float, float addrspace(1)* %p, align 4
+  %call = call fast float @foo(float %load) #0
+  store float %call, float addrspace(1)* %p, align 4
+  ret void
+}
+
+attributes #0 = { nounwind }
+attributes #1 = { nounwind "less-precise-fpmad"="true" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "unsafe-fp-math"="true" }