[AMDGPU] Enable output modifiers for double precision instructions

[bcahoon]

Summary

Update SIFoldOperands pass to recognize v_add_f64 and v_mul_f64 instructions for folding output modifiers.

[arsenm]

Should also have some negative tests where the denormal mode, ieee mode, or signed zeros don't match

[bcahoon]

Update with negative test cases.

[arsenm]

llvm/test/CodeGen/AMDGPU/omod.ll

59–60

Should just use the minimum set of fast flags

77–78

Should just use the minimum set of fast flags

95–96

Should just use the minimum set of fast flags

311

You're using the fast math flags, so you don't need the global no-signed-zeros-fp-math

[bcahoon]

Use nsz and nnan flags. Ignore IEEE mode is nnan flag is set.

[arsenm]

llvm/lib/Target/AMDGPU/SIFoldOperands.cpp

1777

This is a separate change

[arsenm]

llvm/lib/Target/AMDGPU/SIFoldOperands.cpp

1777

Also would need a comment explaining that ieee mode only changes snan behavior, nnan lets us ignore it.

[bcahoon]

Remove IEEE mode/nnan change. Will add separately.

llvm/lib/Target/AMDGPU/SIFoldOperands.cpp

bool SIFoldOperands::tryFoldClamp(MachineInstr &MI) {
   DefClamp->setImm(1);
   MRI->replaceRegWith(MI.getOperand(0).getReg(), Def->getOperand(0).getReg());
   MI.eraseFromParent();
   return true;
 }
 
 static int getOModValue(unsigned Opc, int64_t Val) {
   switch (Opc) {
+  case AMDGPU::V_MUL_F64_e64: {
+    switch (Val) {
+    case 0x3fe0000000000000: // 0.5
+      return SIOutMods::DIV2;
+    case 0x4000000000000000: // 2.0
+      return SIOutMods::MUL2;
+    case 0x4010000000000000: // 4.0
+      return SIOutMods::MUL4;
+    default:
+      return SIOutMods::NONE;
+    }
+  }
   case AMDGPU::V_MUL_F32_e64: {
     switch (static_cast<uint32_t>(Val)) {
     case 0x3f000000: // 0.5
       return SIOutMods::DIV2;
     case 0x40000000: // 2.0
       return SIOutMods::MUL2;
     case 0x40800000: // 4.0
       return SIOutMods::MUL4;

 
 // FIXME: Does this really not support denormals with f16?
 // FIXME: Does this need to check IEEE mode bit? SNaNs are generally not
 // handled, so will anything other than that break?
 std::pair<const MachineOperand *, int>
 SIFoldOperands::isOMod(const MachineInstr &MI) const {
   unsigned Op = MI.getOpcode();
   switch (Op) {
+  case AMDGPU::V_MUL_F64_e64:
   case AMDGPU::V_MUL_F32_e64:
   case AMDGPU::V_MUL_F16_e64: {
     // If output denormals are enabled, omod is ignored.
     if ((Op == AMDGPU::V_MUL_F32_e64 && MFI->getMode().FP32OutputDenormals) ||
-        (Op == AMDGPU::V_MUL_F16_e64 && MFI->getMode().FP64FP16OutputDenormals))
+        ((Op == AMDGPU::V_MUL_F64_e64 || Op == AMDGPU::V_MUL_F16_e64) &&
+         MFI->getMode().FP64FP16OutputDenormals))
       return std::make_pair(nullptr, SIOutMods::NONE);
 
     const MachineOperand *RegOp = nullptr;
     const MachineOperand *ImmOp = nullptr;
     const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
     const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
     if (Src0->isImm()) {
       ImmOp = Src0;
if (OMod == SIOutMods::NONE ||
         TII->hasModifiersSet(MI, AMDGPU::OpName::src0_modifiers) ||
         TII->hasModifiersSet(MI, AMDGPU::OpName::src1_modifiers) ||
         TII->hasModifiersSet(MI, AMDGPU::OpName::omod) ||
         TII->hasModifiersSet(MI, AMDGPU::OpName::clamp))
       return std::make_pair(nullptr, SIOutMods::NONE);
 
     return std::make_pair(RegOp, OMod);
   }
+  case AMDGPU::V_ADD_F64_e64:
   case AMDGPU::V_ADD_F32_e64:
   case AMDGPU::V_ADD_F16_e64: {
     // If output denormals are enabled, omod is ignored.
     if ((Op == AMDGPU::V_ADD_F32_e64 && MFI->getMode().FP32OutputDenormals) ||
-        (Op == AMDGPU::V_ADD_F16_e64 && MFI->getMode().FP64FP16OutputDenormals))
+        ((Op == AMDGPU::V_ADD_F64_e64 || Op == AMDGPU::V_ADD_F16_e64) &&
+         MFI->getMode().FP64FP16OutputDenormals))
       return std::make_pair(nullptr, SIOutMods::NONE);
 
     // Look through the DAGCombiner canonicalization fmul x, 2 -> fadd x, x
     const MachineOperand *Src0 = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
     const MachineOperand *Src1 = TII->getNamedOperand(MI, AMDGPU::OpName::src1);
 
     if (Src0->isReg() && Src1->isReg() && Src0->getReg() == Src1->getReg() &&
         Src0->getSubReg() == Src1->getSubReg() &&
for (I = MBB->begin(); I != MBB->end(); I = Next) {
       if (!TII->isFoldableCopy(MI)) {
         // Saw an unknown clobber of m0, so we no longer know what it is.
         if (CurrentKnownM0Val && MI.modifiesRegister(AMDGPU::M0, TRI))
           CurrentKnownM0Val = nullptr;
 
         // TODO: Omod might be OK if there is NSZ only on the source
         // instruction, and not the omod multiply.
         if (IsIEEEMode || (!HasNSZ && !MI.getFlag(MachineInstr::FmNsz)) ||
             !tryFoldOMod(MI))
           tryFoldClamp(MI);
 
         continue;
       }
 
       // Specially track simple redefs of m0 to the same value in a block, so we
       // can erase the later ones.
       if (MI.getOperand(0).getReg() == AMDGPU::M0) {

llvm/test/CodeGen/AMDGPU/omod.ll

 ; GCN: v_add_f32_e64 v{{[0-9]+}}, v0, 1.0 div:2{{$}}
 define amdgpu_ps void @v_omod_div2_f32(float %a) #0 {
   %add = fadd float %a, 1.0
   %div2 = fmul float %add, 0.5
   store float %div2, float addrspace(1)* undef
   ret void
 }
 
+; GCN-LABEL: {{^}}v_omod_div2_f64:
+; GCN: v_add_f64  v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, 1.0 div:2{{$}}
+define amdgpu_ps void @v_omod_div2_f64(double %a) #5 {
+  %add = fadd fast double %a, 1.0
+  %div2 = fmul fast double %add, 0.5
+  store double %div2, double addrspace(1)* undef
+  ret void
+}
+
 ; GCN-LABEL: {{^}}v_omod_mul2_f32:
 ; GCN: v_add_f32_e64 v{{[0-9]+}}, v0, 1.0 mul:2{{$}}
 define amdgpu_ps void @v_omod_mul2_f32(float %a) #0 {
   %add = fadd float %a, 1.0
   %div2 = fmul float %add, 2.0
   store float %div2, float addrspace(1)* undef
   ret void
 }
 
+; GCN-LABEL: {{^}}v_omod_mul2_f64:
+; GCN: v_add_f64  v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, 1.0 mul:2{{$}}
+define amdgpu_ps void @v_omod_mul2_f64(double %a) #5 {
+  %add = fadd fast double %a, 1.0
+  %div2 = fmul fast double %add, 2.0
+  store double %div2, double addrspace(1)* undef
+  ret void
+}
+
 ; GCN-LABEL: {{^}}v_omod_mul4_f32:
 ; GCN: v_add_f32_e64 v{{[0-9]+}}, v0, 1.0 mul:4{{$}}
 define amdgpu_ps void @v_omod_mul4_f32(float %a) #0 {
   %add = fadd float %a, 1.0
   %div2 = fmul float %add, 4.0
   store float %div2, float addrspace(1)* undef
   ret void
 }
 
+; GCN-LABEL: {{^}}v_omod_mul4_f64:
+; GCN: v_add_f64  v{{\[[0-9]+:[0-9]+\]}}, v{{\[[0-9]+:[0-9]+\]}}, 1.0 mul:4{{$}}
+define amdgpu_ps void @v_omod_mul4_f64(double %a) #5 {
+  %add = fadd fast double %a, 1.0
+  %div2 = fmul fast double %add, 4.0
+  store double %div2, double addrspace(1)* undef
+  ret void
+}
+
 ; GCN-LABEL: {{^}}v_omod_mul4_multi_use_f32:
 ; GCN: v_add_f32_e32 [[ADD:v[0-9]+]], 1.0, v0{{$}}
 ; GCN: v_mul_f32_e32 v{{[0-9]+}}, 4.0, [[ADD]]{{$}}
 define amdgpu_ps void @v_omod_mul4_multi_use_f32(float %a) #0 {
   %add = fadd float %a, 1.0
   %div2 = fmul float %add, 4.0
   store float %div2, float addrspace(1)* undef
   store volatile float %add, float addrspace(1)* undef

 declare half @llvm.maxnum.f16(half, half) #1
 declare void @llvm.dbg.value(metadata, i64, metadata, metadata) #1
 
 attributes #0 = { nounwind "denormal-fp-math-f32"="preserve-sign,preserve-sign" "no-signed-zeros-fp-math"="true" }
 attributes #1 = { nounwind readnone }
 attributes #2 = { nounwind "denormal-fp-math-f32"="ieee,ieee" "no-signed-zeros-fp-math"="true" }
 attributes #3 = { nounwind "denormal-fp-math"="preserve-sign,preserve-sign" "no-signed-zeros-fp-math"="true" }
 attributes #4 = { nounwind "no-signed-zeros-fp-math"="false" }
+attributes #5 = { nounwind "denormal-fp-math"="preserve-sign,preserve-sign" "no-signed-zeros-fp-math"="true" }
 
 !llvm.dbg.cu = !{!0}
 !llvm.module.flags = !{!2, !3}
 
 !0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, isOptimized: true, runtimeVersion: 0, emissionKind: NoDebug)
 !1 = !DIFile(filename: "/tmp/foo.cl", directory: "/dev/null")
 !2 = !{i32 2, !"Dwarf Version", i32 4}
 !3 = !{i32 2, !"Debug Info Version", i32 3}
 !4 = !DILocalVariable(name: "add", arg: 1, scope: !5, file: !1, line: 1)
 !5 = distinct !DISubprogram(name: "foo", scope: !1, file: !1, line: 1, type: !6, isLocal: false, isDefinition: true, scopeLine: 2, flags: DIFlagPrototyped, isOptimized: true, unit: !0)
 !6 = !DISubroutineType(types: !7)
 !7 = !{null, !8}
 !8 = !DIBasicType(name: "float", size: 32, align: 32)
 !9 = !DIExpression()
 !10 = !DILocation(line: 1, column: 42, scope: !5)