[NVPTX] Only lower sin/cos to approximate instructions if unsafe math is allowed.

[tra]

Summary

Previously we'd always lower @llvm.{sin,cos}.f32 to {sin.cos}.approx.f32 instruction even when unsafe FP math was not allowed.

Clang-generated IR is not affected by this change as it uses precise sin/cos from CUDA's libdevice when unsafe math is disabled.

[jlebar]

Looks great.

Perhaps we should merge the two foo-no-fast-math.ll files into one? I have another few testcases I want to add very shortly.

[jlebar]

In D28619#644775, @jlebar wrote:

Perhaps we should merge the two foo-no-fast-math.ll files into one? I have another few testcases I want to add very shortly.

Oh, right, you're crashing llc, so it has to be in a separate file. Carry on!

lib/Target/NVPTX/NVPTXISelDAGToDAG.h

class LLVM_LIBRARY_VISIBILITY NVPTXDAGToDAGISel : public SelectionDAGISel {
 
   // If true, generate mul.wide from sext and mul
   bool doMulWide;
 
   int getDivF32Level() const;
   bool usePrecSqrtF32() const;
   bool useF32FTZ() const;
   bool allowFMA() const;
+  bool allowUnsafeFPMath() const;
 
 public:
   explicit NVPTXDAGToDAGISel(NVPTXTargetMachine &tm,
                              CodeGenOpt::Level   OptLevel);
 
   // Pass Name
   StringRef getPassName() const override {
     return "NVPTX DAG->DAG Pattern Instruction Selection";

lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp

bool NVPTXDAGToDAGISel::useF32FTZ() const {
   }
 }
 
 bool NVPTXDAGToDAGISel::allowFMA() const {
   const NVPTXTargetLowering *TL = Subtarget->getTargetLowering();
   return TL->allowFMA(*MF, OptLevel);
 }
 
+bool NVPTXDAGToDAGISel::allowUnsafeFPMath() const {
+  const NVPTXTargetLowering *TL = Subtarget->getTargetLowering();
+  return TL->allowUnsafeFPMath(*MF);
+}
+
 /// Select - Select instructions not customized! Used for
 /// expanded, promoted and normal instructions.
 void NVPTXDAGToDAGISel::Select(SDNode *N) {
 
   if (N->isMachineOpcode()) {
     N->setNodeId(-1);
     return; // Already selected.
   }

lib/Target/NVPTX/NVPTXISelLowering.h

public:
   MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override {
     return MVT::i32;
   }
 
   TargetLoweringBase::LegalizeTypeAction
   getPreferredVectorAction(EVT VT) const override;
 
   bool allowFMA(MachineFunction &MF, CodeGenOpt::Level OptLevel) const;
+  bool allowUnsafeFPMath(MachineFunction &MF) const;
 
   bool isFMAFasterThanFMulAndFAdd(EVT) const override { return true; }
 
   bool enableAggressiveFMAFusion(EVT VT) const override { return true; }
 
 private:
   const NVPTXSubtarget &STI; // cache the subtarget here
   SDValue getParamSymbol(SelectionDAG &DAG, int idx, EVT) const;

lib/Target/NVPTX/NVPTXISelLowering.cpp

 }
 
 //===----------------------------------------------------------------------===//
 //                         NVPTX DAG Combining
 //===----------------------------------------------------------------------===//
 
 bool NVPTXTargetLowering::allowFMA(MachineFunction &MF,
                                    CodeGenOpt::Level OptLevel) const {
-  const Function *F = MF.getFunction();
-  const TargetOptions &TO = MF.getTarget().Options;
-
   // Always honor command-line argument
-  if (FMAContractLevelOpt.getNumOccurrences() > 0) {
+  if (FMAContractLevelOpt.getNumOccurrences() > 0)
     return FMAContractLevelOpt > 0;
-  } else if (OptLevel == 0) {
-    // Do not contract if we're not optimizing the code
+
+  // Do not contract if we're not optimizing the code.
+  if (OptLevel == 0)
     return false;
-  } else if (TO.AllowFPOpFusion == FPOpFusion::Fast || TO.UnsafeFPMath) {
-    // Honor TargetOptions flags that explicitly say fusion is okay
+
+  // Honor TargetOptions flags that explicitly say fusion is okay.
+  if (MF.getTarget().Options.AllowFPOpFusion == FPOpFusion::Fast)
+    return true;
+
+  return allowUnsafeFPMath(MF);
+}
+
+bool NVPTXTargetLowering::allowUnsafeFPMath(MachineFunction &MF) const {
+  // Honor TargetOptions flags that explicitly say unsafe math is okay.
+  if (MF.getTarget().Options.UnsafeFPMath)
     return true;
-  } else if (F->hasFnAttribute("unsafe-fp-math")) {
-    // Check for unsafe-fp-math=true coming from Clang
+
+  // Allow unsafe math if unsafe-fp-math attribute explicitly says so.
+  const Function *F = MF.getFunction();
+  if (F->hasFnAttribute("unsafe-fp-math")) {
     Attribute Attr = F->getFnAttribute("unsafe-fp-math");
     StringRef Val = Attr.getValueAsString();
     if (Val == "true")
       return true;
   }
 
-  // We did not have a clear indication that fusion is allowed, so assume not
   return false;
 }
 
 /// PerformADDCombineWithOperands - Try DAG combinations for an ADD with
 /// operands N0 and N1.  This is a helper for PerformADDCombine that is
 /// called with the default operands, and if that fails, with commuted
 /// operands.
 static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,

lib/Target/NVPTX/NVPTXInstrInfo.td

 
 def doF32FTZ : Predicate<"useF32FTZ()">;
 def doNoF32FTZ : Predicate<"!useF32FTZ()">;
 
 def doMulWide      : Predicate<"doMulWide">;
 
 def allowFMA : Predicate<"allowFMA()">;
 def noFMA : Predicate<"!allowFMA()">;
+def allowUnsafeFPMath : Predicate<"allowUnsafeFPMath()">;
 
 def do_DIVF32_APPROX : Predicate<"getDivF32Level()==0">;
 def do_DIVF32_FULL : Predicate<"getDivF32Level()==1">;
 
 def do_SQRTF32_APPROX : Predicate<"!usePrecSqrtF32()">;
 def do_SQRTF32_RN : Predicate<"usePrecSqrtF32()">;
 
 def hasHWROT32 : Predicate<"Subtarget->hasHWROT32()">;

 
 defm FMA32_ftz : FMA<"fma.rn.ftz.f32", Float32Regs, f32imm, doF32FTZ>;
 defm FMA32     : FMA<"fma.rn.f32", Float32Regs, f32imm, true>;
 defm FMA64     : FMA<"fma.rn.f64", Float64Regs, f64imm, true>;
 
 // sin/cos
 def SINF:  NVPTXInst<(outs Float32Regs:$dst), (ins Float32Regs:$src),
                       "sin.approx.f32 \t$dst, $src;",
-                      [(set Float32Regs:$dst, (fsin Float32Regs:$src))]>;
+                      [(set Float32Regs:$dst, (fsin Float32Regs:$src))]>,
+                      Requires<[allowUnsafeFPMath]>;
 def COSF:  NVPTXInst<(outs Float32Regs:$dst), (ins Float32Regs:$src),
                       "cos.approx.f32 \t$dst, $src;",
-                      [(set Float32Regs:$dst, (fcos Float32Regs:$src))]>;
+                      [(set Float32Regs:$dst, (fcos Float32Regs:$src))]>,
+                      Requires<[allowUnsafeFPMath]>;
 
 // Lower (frem x, y) into (sub x, (mul (floor (div x, y)) y)),
 // i.e. "poor man's fmod()"
 
 // frem - f32 FTZ
 def : Pat<(frem Float32Regs:$x, Float32Regs:$y),
           (FSUBf32rr_ftz Float32Regs:$x, (FMULf32rr_ftz (CVT_f32_f32
             (FDIV32rr_prec_ftz Float32Regs:$x, Float32Regs:$y), CvtRMI_FTZ),

test/CodeGen/NVPTX/fast-math.ll

 
 ; CHECK: fadd_ftz
 ; CHECK: add.ftz.f32
 define float @fadd_ftz(float %a, float %b) #1 {
   %t1 = fadd float %a, %b
   ret float %t1
 }
 
+declare float @llvm.sin.f32(float)
+declare float @llvm.cos.f32(float)
 
+; CHECK-LABEL: fsin_approx
+; CHECK:       sin.approx.f32
+define float @fsin_approx(float %a) #0 {
+  %r = tail call float @llvm.sin.f32(float %a)
+  ret float %r
+}
+
+; CHECK-LABEL: fcos_approx
+; CHECK:       cos.approx.f32
+define float @fcos_approx(float %a) #0 {
+  %r = tail call float @llvm.cos.f32(float %a)
+  ret float %r
+}
 
 attributes #0 = { "unsafe-fp-math" = "true" }
 attributes #1 = { "nvptx-f32ftz" = "true" }

test/CodeGen/NVPTX/fcos-no-fast-math.ll

+; RUN: not llc < %s -march=nvptx -mcpu=sm_20 2>&1 | FileCheck %s
+
+; Check that we fail to select fcos without fast-math enabled
+
+declare float @llvm.cos.f32(float)
+
+; CHECK: LLVM ERROR: Cannot select: {{.*}}: f32 = fcos
+; CHECK: In function: test_fcos_safe
+define float @test_fcos_safe(float %a) #0 {
+  %r = tail call float @llvm.cos.f32(float %a)
+  ret float %r
+}
+
+attributes #0 = { "unsafe-fp-math" = "false" }

test/CodeGen/NVPTX/fsin-no-fast-math.ll

+; RUN: not llc < %s -march=nvptx -mcpu=sm_20 2>&1 | FileCheck %s
+
+; Check that we fail to select fsin without fast-math enabled
+
+declare float @llvm.sin.f32(float)
+
+; CHECK: LLVM ERROR: Cannot select: {{.*}}: f32 = fsin
+; CHECK: In function: test_fsin_safe
+define float @test_fsin_safe(float %a) #0 {
+  %r = tail call float @llvm.sin.f32(float %a)
+  ret float %r
+}
+
+attributes #0 = { "unsafe-fp-math" = "false" }