[AArch64][SVE] Propagate math flags from intrinsics to instructions

[MattDevereau]

Summary

Retain floating-point math flags inside instCombineSVEVectorBinOp

[peterwaller-arm]

Looks reasonable to me but I had a hand in it. Please wait 24h to give other reviewers a chance to chime in.

[david-arm]

llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

713

nit: I'm not sure where the additional cosmetic code changes came from? It looks like you only really need to add the setFastMathFlags line?

[MattDevereau]

llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

713

we had a short discussion about what looks better and might be easier to read, debug and work with. its a simple change if you disagree

[peterwaller-arm]

llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

713

For what it's worth I provoked this by suggesting in private that I wanted to see the parameter to IC.replaceInstUsesWith as its own variable in the original patch, and to generally avoid putting too much into a single line, for readability.

In the case of extracting OpPredicate I agree that it is marginal.

[david-arm]

LGTM!

llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

713

No problem, thanks for the explanation! Happy with the patch

[MattDevereau]

2ac199993764e068494a69a85af098c0ae1ff37e

llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

case Intrinsic::aarch64_sve_fsub:
     return Instruction::BinaryOps::FSub;
   default:
     return Instruction::BinaryOpsEnd;
   }
 }
 
 static Optional<Instruction *> instCombineSVEVectorBinOp(InstCombiner &IC,
                                                          IntrinsicInst &II) {
+  auto *OpPredicate = II.getOperand(0);
   auto BinOpCode = intrinsicIDToBinOpCode(II.getIntrinsicID());
   if (BinOpCode == Instruction::BinaryOpsEnd ||
-      !match(II.getOperand(0),
-             m_Intrinsic<Intrinsic::aarch64_sve_ptrue>(
+      !match(OpPredicate, m_Intrinsic<Intrinsic::aarch64_sve_ptrue>(
                               m_ConstantInt<AArch64SVEPredPattern::all>())))
     return None;
   IRBuilder<> Builder(II.getContext());
   Builder.SetInsertPoint(&II);
-  return IC.replaceInstUsesWith(
-      II, Builder.CreateBinOp(BinOpCode, II.getOperand(1), II.getOperand(2)));
+  Builder.setFastMathFlags(II.getFastMathFlags());
+  auto BinOp =
+      Builder.CreateBinOp(BinOpCode, II.getOperand(1), II.getOperand(2));
+  return IC.replaceInstUsesWith(II, BinOp);
 }
 
 static Optional<Instruction *> instCombineSVEVectorMul(InstCombiner &IC,
                                                        IntrinsicInst &II) {
   auto *OpPredicate = II.getOperand(0);
   auto *OpMultiplicand = II.getOperand(1);
   auto *OpMultiplier = II.getOperand(2);
 

llvm/test/Transforms/InstCombine/AArch64/sve-intrinsic-fma-binops.ll

 ; RUN: opt -S -instcombine < %s | FileCheck %s
 
 target triple = "aarch64-unknown-linux-gnu"
 
 declare <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32)
 declare <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32)
 declare <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32)
 
 ; SVE intrinsics fmul and fadd should be replaced with regular fmul and fadd
 declare <vscale x 8 x half> @llvm.aarch64.sve.fmul.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
 define <vscale x 8 x half> @replace_fmul_intrinsic_half(<vscale x 8 x half> %a, <vscale x 8 x half> %b) #0 {
 ; CHECK-LABEL: @replace_fmul_intrinsic_half
-; CHECK-NEXT:  %1 = fmul <vscale x 8 x half> %a, %b
+; CHECK-NEXT:  %1 = fmul fast <vscale x 8 x half> %a, %b
 ; CHECK-NEXT:  ret <vscale x 8 x half> %1
   %1 = tail call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
   %2 = tail call fast <vscale x 8 x half> @llvm.aarch64.sve.fmul.nxv8f16(<vscale x 8 x i1> %1, <vscale x 8 x half> %a, <vscale x 8 x half> %b)
   ret <vscale x 8 x half> %2
 }
 
 declare <vscale x 4 x float> @llvm.aarch64.sve.fmul.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
 define <vscale x 4 x float> @replace_fmul_intrinsic_float(<vscale x 4 x float> %a, <vscale x 4 x float> %b) #0 {
 ; CHECK-LABEL: @replace_fmul_intrinsic_float
-; CHECK-NEXT:  %1 = fmul <vscale x 4 x float> %a, %b
+; CHECK-NEXT:  %1 = fmul fast <vscale x 4 x float> %a, %b
 ; CHECK-NEXT:  ret <vscale x 4 x float> %1
   %1 = tail call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
   %2 = tail call fast <vscale x 4 x float> @llvm.aarch64.sve.fmul.nxv4f32(<vscale x 4 x i1> %1, <vscale x 4 x float> %a, <vscale x 4 x float> %b)
   ret <vscale x 4 x float> %2
 }
 
 declare <vscale x 2 x double> @llvm.aarch64.sve.fmul.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
 define <vscale x 2 x double> @replace_fmul_intrinsic_double(<vscale x 2 x double> %a, <vscale x 2 x double> %b) #0 {
 ; CHECK-LABEL: @replace_fmul_intrinsic_double
-; CHECK-NEXT:  %1 = fmul <vscale x 2 x double> %a, %b
+; CHECK-NEXT:  %1 = fmul fast <vscale x 2 x double> %a, %b
 ; CHECK-NEXT:  ret <vscale x 2 x double> %1
   %1 = tail call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
   %2 = tail call fast <vscale x 2 x double> @llvm.aarch64.sve.fmul.nxv2f64(<vscale x 2 x i1> %1, <vscale x 2 x double> %a, <vscale x 2 x double> %b)
   ret <vscale x 2 x double> %2
 }
 
 declare <vscale x 8 x half> @llvm.aarch64.sve.fadd.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
 define <vscale x 8 x half> @replace_fadd_intrinsic_half(<vscale x 8 x half> %a, <vscale x 8 x half> %b) #0 {
 ; CHECK-LABEL: @replace_fadd_intrinsic_half
-; CHECK-NEXT:  %1 = fadd <vscale x 8 x half> %a, %b
+; CHECK-NEXT:  %1 = fadd fast <vscale x 8 x half> %a, %b
 ; CHECK-NEXT:  ret <vscale x 8 x half> %1
   %1 = tail call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
   %2 = tail call fast <vscale x 8 x half> @llvm.aarch64.sve.fadd.nxv8f16(<vscale x 8 x i1> %1, <vscale x 8 x half> %a, <vscale x 8 x half> %b)
   ret <vscale x 8 x half> %2
 }
 
 declare <vscale x 4 x float> @llvm.aarch64.sve.fadd.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
 define <vscale x 4 x float> @replace_fadd_intrinsic_float(<vscale x 4 x float> %a, <vscale x 4 x float> %b) #0 {
 ; CHECK-LABEL: @replace_fadd_intrinsic_float
-; CHECK-NEXT:  %1 = fadd <vscale x 4 x float> %a, %b
+; CHECK-NEXT:  %1 = fadd fast <vscale x 4 x float> %a, %b
 ; CHECK-NEXT:  ret <vscale x 4 x float> %1
   %1 = tail call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
   %2 = tail call fast <vscale x 4 x float> @llvm.aarch64.sve.fadd.nxv4f32(<vscale x 4 x i1> %1, <vscale x 4 x float> %a, <vscale x 4 x float> %b)
   ret <vscale x 4 x float> %2
 }
 
 declare <vscale x 2 x double> @llvm.aarch64.sve.fadd.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
 define <vscale x 2 x double> @replace_fadd_intrinsic_double(<vscale x 2 x double> %a, <vscale x 2 x double> %b) #0 {
 ; CHECK-LABEL: @replace_fadd_intrinsic_double
-; CHECK-NEXT:  %1 = fadd <vscale x 2 x double> %a, %b
+; CHECK-NEXT:  %1 = fadd fast <vscale x 2 x double> %a, %b
 ; CHECK-NEXT:  ret <vscale x 2 x double> %1
   %1 = tail call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
   %2 = tail call fast <vscale x 2 x double> @llvm.aarch64.sve.fadd.nxv2f64(<vscale x 2 x i1> %1, <vscale x 2 x double> %a, <vscale x 2 x double> %b)
   ret <vscale x 2 x double> %2
 }
 
 declare <vscale x 8 x half> @llvm.aarch64.sve.fsub.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>, <vscale x 8 x half>)
 define <vscale x 8 x half> @replace_fsub_intrinsic_half(<vscale x 8 x half> %a, <vscale x 8 x half> %b) #0 {
 ; CHECK-LABEL: @replace_fsub_intrinsic_half
-; CHECK-NEXT:  %1 = fsub <vscale x 8 x half> %a, %b
+; CHECK-NEXT:  %1 = fsub fast <vscale x 8 x half> %a, %b
 ; CHECK-NEXT:  ret <vscale x 8 x half> %1
   %1 = tail call <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 31)
   %2 = tail call fast <vscale x 8 x half> @llvm.aarch64.sve.fsub.nxv8f16(<vscale x 8 x i1> %1, <vscale x 8 x half> %a, <vscale x 8 x half> %b)
   ret <vscale x 8 x half> %2
 }
 
 declare <vscale x 4 x float> @llvm.aarch64.sve.fsub.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>, <vscale x 4 x float>)
 define <vscale x 4 x float> @replace_fsub_intrinsic_float(<vscale x 4 x float> %a, <vscale x 4 x float> %b) #0 {
 ; CHECK-LABEL: @replace_fsub_intrinsic_float
-; CHECK-NEXT:  %1 = fsub <vscale x 4 x float> %a, %b
+; CHECK-NEXT:  %1 = fsub fast <vscale x 4 x float> %a, %b
 ; CHECK-NEXT:  ret <vscale x 4 x float> %1
   %1 = tail call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31)
   %2 = tail call fast <vscale x 4 x float> @llvm.aarch64.sve.fsub.nxv4f32(<vscale x 4 x i1> %1, <vscale x 4 x float> %a, <vscale x 4 x float> %b)
   ret <vscale x 4 x float> %2
 }
 
 
 declare <vscale x 2 x double> @llvm.aarch64.sve.fsub.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>, <vscale x 2 x double>)
 define <vscale x 2 x double> @replace_fsub_intrinsic_double(<vscale x 2 x double> %a, <vscale x 2 x double> %b) #0 {
 ; CHECK-LABEL: @replace_fsub_intrinsic_double
-; CHECK-NEXT:  %1 = fsub <vscale x 2 x double> %a, %b
+; CHECK-NEXT:  %1 = fsub fast <vscale x 2 x double> %a, %b
 ; CHECK-NEXT:  ret <vscale x 2 x double> %1
   %1 = tail call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
   %2 = tail call fast <vscale x 2 x double> @llvm.aarch64.sve.fsub.nxv2f64(<vscale x 2 x i1> %1, <vscale x 2 x double> %a, <vscale x 2 x double> %b)
   ret <vscale x 2 x double> %2
 }
 
 define <vscale x 2 x double> @no_replace_on_non_ptrue_all(<vscale x 2 x double> %a, <vscale x 2 x double> %b) #0 {
 ; CHECK-LABEL: @no_replace_on_non_ptrue_all
 ; CHECK-NEXT:  %1 = tail call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 5)
 ; CHECK-NEXT:  %2 = tail call fast <vscale x 2 x double> @llvm.aarch64.sve.fsub.nxv2f64(<vscale x 2 x i1> %1, <vscale x 2 x double> %a, <vscale x 2 x double> %b)
 ; CHECK-NEXT:  ret <vscale x 2 x double> %2
   %1 = tail call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 5)
   %2 = tail call fast <vscale x 2 x double> @llvm.aarch64.sve.fsub.nxv2f64(<vscale x 2 x i1> %1, <vscale x 2 x double> %a, <vscale x 2 x double> %b)
   ret <vscale x 2 x double> %2
 }
 
+define <vscale x 2 x double> @replace_fsub_intrinsic_no_fast_flag(<vscale x 2 x double> %a, <vscale x 2 x double> %b) #0 {
+; CHECK-LABEL: @replace_fsub_intrinsic_no_fast_flag
+; CHECK-NEXT:  %1 = fsub <vscale x 2 x double> %a, %b
+; CHECK-NEXT:  ret <vscale x 2 x double> %1
+  %1 = tail call <vscale x 2 x i1> @llvm.aarch64.sve.ptrue.nxv2i1(i32 31)
+  %2 = tail call <vscale x 2 x double> @llvm.aarch64.sve.fsub.nxv2f64(<vscale x 2 x i1> %1, <vscale x 2 x double> %a, <vscale x 2 x double> %b)
+  ret <vscale x 2 x double> %2
+}
+
 attributes #0 = { "target-features"="+sve" }