[AArch64] Fold vector fmul/fdiv by pow2 into floating-point/fixed-point conversions.

[mcrosier]

Summary

Adds DAG combines for:

1. Folding a floating-point multiply by power of two into floating-point to fixed-point conversion and
2. Folding a floating-point multiply by power of two into fixed-point to floating-point conversion.

This is essentially a port of a combine from the ARM backend (r133813).

Thanks,

Chad

[t.p.northover]

Hi Chad,

I think the bulk of the logic's fine, just one or two cleanups...

lib/Target/AArch64/AArch64ISelLowering.cpp

7538–7539

I think this function could be made simpler (and more generic, e.g. handling undefs too) by calling BuildVectorSDNode::isConstantSplat.

test/CodeGen/AArch64/fcvt_combine.ll

6

This style doesn't check that the fixed-point shift is correctly translated.

[mcrosier]

Updated the helper function, isConstVecPow2, per Tim's comments.

Added support for folding sitofp/uitofp followed by fdiv by power of two.

[t.p.northover]

Thanks for updating the patch, Chad. Still got a few more issues...

lib/Target/AArch64/AArch64ISelLowering.cpp

7538

I've just noticed this started life in ARMISelLowering.cpp; since we're improving it here, can we fix that one too? Or even share the implementation somewhere (BuildVectorSDNode, or perhaps SelectionDAG if you anticipate handling more cases in the future)

7543

Why are we bailing on undef here?

7549

I don't think this can handle 2^64. Really, we might as well convert to an APSInt and use exactLogBase2 or something.

[mcrosier]

Thanks, Tim. I'll post a new revision in a bit.

lib/Target/AArch64/AArch64ISelLowering.cpp

7538

I was planning on updating the version in ARMISelLowering as well, but I agree we can just make this common to BuildVectorSDNode or elsewhere and use the new version in both ARM and AArch64.

7543

No particular reason other than I was trying to be conservative. I'll remove the !UndefElements.none() check.

7549

Thanks for the suggestion, Tim. I'm not overly familiar with all of the APFloat/APInt APIs, hence the less than ideal solution. I'll start digging to see if I can't implement a more robust solution.

[mcrosier]

This should take care of all of your suggested improvements, Tim.

1. Accommodated Tim's request for a generic interface for both ARM and AArch64 to compute const FP splat of power of 2.
2. Allow matching against constant splats with undefs.
   
   Chad

[t.p.northover]

Thanks Chad. I think this looks good now.

Tim.

[mcrosier]

Thanks, Tim! Committed in r249572, r249576, and r249579.

lib/Target/AArch64/AArch64ISelLowering.cpp

   setTargetDAGCombine(ISD::SINT_TO_FP);
   setTargetDAGCombine(ISD::UINT_TO_FP);
 
+  setTargetDAGCombine(ISD::FP_TO_SINT);
+  setTargetDAGCombine(ISD::FP_TO_UINT);
+  setTargetDAGCombine(ISD::FDIV);
+
   setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
 
   setTargetDAGCombine(ISD::ANY_EXTEND);

   return SDValue();
 }
 
+// isConstVecPow2 - Return true if each vector element is a power of 2, all
+// elements are the same constant, C, and Log2(C) ranges from 1 to Range.
+static bool isConstVecPow2(SDValue ConstVec, bool IsSigned, uint64_t &C,
+                           unsigned Range) {
+  if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(ConstVec)) {
+    BitVector UndefElements;
+    ConstantFPSDNode *CN = BV->getConstantFPSplatNode(&UndefElements);
+    if (!CN || !UndefElements.none())
+      return false;
+
+    bool IsExact;
+    integerPart C0;
+    APFloat APF = CN->getValueAPF();
+    if (APF.convertToInteger(&C0, 64, IsSigned, APFloat::rmTowardZero, &IsExact) != APFloat::opOK || !IsExact)
+      return false;
+
+    if (!isPowerOf2_64(C0) || Log2_64(C0) < 1 || Log2_64(C0) > Range)
+      return false;
+
+    C = C0;
+    return true;
+  }
+  return false;
+}
+
+/// Fold a floating-point multiply by power of two into floating-point to
+/// fixed-point conversion.
+static SDValue performFpToIntCombine(SDNode *N, SelectionDAG &DAG,
+                                     const AArch64Subtarget *Subtarget) {
+  if (!Subtarget->hasNEON())
+    return SDValue();
+
+  SDValue Op = N->getOperand(0);
+  if (!Op.getValueType().isVector() || Op.getOpcode() != ISD::FMUL)
+    return SDValue();
+
+  MVT FloatTy = Op.getSimpleValueType().getVectorElementType();
+  uint32_t FloatBits = FloatTy.getSizeInBits();
+  if (FloatBits != 32 && FloatBits != 64)
+    return SDValue();
+
+  MVT IntTy = N->getSimpleValueType(0).getVectorElementType();
+  uint32_t IntBits = IntTy.getSizeInBits();
+  if (IntBits != 16 && IntBits != 32 && IntBits != 64)
+    return SDValue();
+
+  // Avoid conversions where iN is larger than the float (e.g., float -> i64).
+  if (IntBits > FloatBits)
+    return SDValue();
+
+  uint64_t C;
+  SDValue N0 = Op->getOperand(0);
+  SDValue ConstVec = Op->getOperand(1);
+  bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT;
+  if (!isConstVecPow2(ConstVec, IsSigned, C, IntBits == 64 ? 64 : 32))
+    return SDValue();
+
+  MVT ResTy;
+  unsigned NumLanes = Op.getValueType().getVectorNumElements();
+  switch (NumLanes) {
+  default:
+    return SDValue();
+  case 2:
+    ResTy = FloatBits == 32 ? MVT::v2i32 : MVT::v2i64;
+    break;
+  case 4:
+    ResTy = MVT::v4i32;
+    break;
+  }
+
+  SDLoc DL(N);
+  unsigned IntrinsicOpcode = IsSigned ? Intrinsic::aarch64_neon_vcvtfp2fxs
+                                      : Intrinsic::aarch64_neon_vcvtfp2fxu;
+  SDValue FixConv = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ResTy,
+                                DAG.getConstant(IntrinsicOpcode, DL, MVT::i32),
+                                N0, DAG.getConstant(Log2_64(C), DL, MVT::i32));
+  // We can handle smaller integers by generating an extra trunc.
+  if (IntBits < FloatBits)
+    FixConv = DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), FixConv);
+
+  return FixConv;
+}
+
+/// Fold a floating-point divide by power of two into fixed-point to
+/// floating-point conversion.
+static SDValue performFDivCombine(SDNode *N, SelectionDAG &DAG,
+                                  const AArch64Subtarget *Subtarget) {
+  if (!Subtarget->hasNEON())
+    return SDValue();
+
+  SDValue Op = N->getOperand(0);
+  unsigned Opc = Op->getOpcode();
+  if (!Op.getValueType().isVector() ||
+      (Opc != ISD::SINT_TO_FP && Opc != ISD::UINT_TO_FP))
+    return SDValue();
+
+  MVT IntTy = Op.getOperand(0).getSimpleValueType().getVectorElementType();
+  uint32_t IntBits = IntTy.getSizeInBits();
+  if (IntBits != 16 && IntBits != 32 && IntBits != 64)
+    return SDValue();
+
+  MVT FloatTy = N->getSimpleValueType(0).getVectorElementType();
+  uint32_t FloatBits = FloatTy.getSizeInBits();
+  if (FloatBits != 32 && FloatBits != 64)
+    return SDValue();
+
+  // Avoid conversions where iN is larger than the float (e.g., i64 -> float).
+  if (IntBits > FloatBits)
+    return SDValue();
+
+  uint64_t C;
+  SDValue ConstVec = N->getOperand(1);
+  bool IsSigned = Opc == ISD::SINT_TO_FP;
+  if (!isConstVecPow2(ConstVec, IsSigned, C, IntBits == 64 ? 64 : 32))
+    return SDValue();
+
+  MVT ResTy;
+  unsigned NumLanes = Op.getValueType().getVectorNumElements();
+  switch (NumLanes) {
+  default:
+    return SDValue();
+  case 2:
+    ResTy = FloatBits == 32 ? MVT::v2i32 : MVT::v2i64;
+    break;
+  case 4:
+    ResTy = MVT::v4i32;
+    break;
+  }
+
+  SDLoc DL(N);
+  SDValue ConvInput = Op.getOperand(0);
+  if (IntBits < FloatBits)
+    ConvInput = DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL,
+                            ResTy, ConvInput);
+
+  unsigned IntrinsicOpcode = IsSigned ? Intrinsic::aarch64_neon_vcvtfxs2fp
+                                      : Intrinsic::aarch64_neon_vcvtfxu2fp;
+  return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(),
+                     DAG.getConstant(IntrinsicOpcode, DL, MVT::i32), ConvInput,
+                     DAG.getConstant(Log2_64(C), DL, MVT::i32));
+}
+
 /// An EXTR instruction is made up of two shifts, ORed together. This helper
 /// searches for and classifies those shifts.
 static bool findEXTRHalf(SDValue N, SDValue &Src, uint32_t &ShiftAmount,

   case ISD::SINT_TO_FP:
   case ISD::UINT_TO_FP:
     return performIntToFpCombine(N, DAG, Subtarget);
+  case ISD::FP_TO_SINT:
+  case ISD::FP_TO_UINT:
+    return performFpToIntCombine(N, DAG, Subtarget);
+  case ISD::FDIV:
+    return performFDivCombine(N, DAG, Subtarget);
   case ISD::OR:
     return performORCombine(N, DCI, Subtarget);
   case ISD::INTRINSIC_WO_CHAIN:

test/CodeGen/AArch64/fcvt_combine.ll

+; RUN: llc -mtriple=aarch64-linux-gnu -aarch64-neon-syntax=apple -verify-machineinstrs -o - %s | FileCheck %s
+
+; CHECK-LABEL: test1
+; CHECK-NOT: fmul.2s
+; CHECK: fcvtzs.2s v0, v0, #4
+; CHECK: ret
+define <2 x i32> @test1(float %f) {
+entry:
+  %vecinit1.i = insertelement <2 x float> undef, float %f, i32 0
+  %vecinit2.i = insertelement <2 x float> %vecinit1.i, float %f, i32 1
+  %mul.i = fmul <2 x float> %vecinit2.i, <float 16.000000e+00, float 16.000000e+00>
+  %vcvt.i = fptosi <2 x float> %mul.i to <2 x i32>
+  ret <2 x i32> %vcvt.i
+}
+
+; CHECK-LABEL: test2
+; CHECK-NOT: fmul.4s
+; CHECK: fcvtzs.4s v0, v0, #3
+; CHECK: ret
+define <4 x i32> @test2(float %f) {
+entry:
+  %vecinit1.i = insertelement <4 x float> undef, float %f, i32 0
+  %vecinit2.i = insertelement <4 x float> %vecinit1.i, float %f, i32 1
+  %vecinit3.i = insertelement <4 x float> %vecinit2.i, float %f, i32 2
+  %vecinit4.i = insertelement <4 x float> %vecinit3.i, float %f, i32 3
+  %mul.i = fmul <4 x float> %vecinit4.i, <float 8.000000e+00, float 8.000000e+00, float 8.000000e+00, float 8.000000e+00>
+  %vcvt.i = fptosi <4 x float> %mul.i to <4 x i32>
+  ret <4 x i32> %vcvt.i
+}
+
+; CHECK-LABEL: test3
+; CHECK-NOT: fmul.2d
+; CHECK: fcvtzs.2d v0, v0, #5
+; CHECK: ret
+define <2 x i64> @test3(double %d) {
+entry:
+  %vecinit1.i = insertelement <2 x double> undef, double %d, i32 0
+  %vecinit2.i = insertelement <2 x double> %vecinit1.i, double %d, i32 1
+  %mul.i = fmul <2 x double> %vecinit2.i, <double 32.000000e+00, double 32.000000e+00>
+  %vcvt.i = fptosi <2 x double> %mul.i to <2 x i64>
+  ret <2 x i64> %vcvt.i
+}
+
+; Truncate double to i32
+; CHECK-LABEL: test4
+; CHECK-NOT: fmul.2d v0, v0, #4
+; CHECK: fcvtzs.2d v0, v0
+; CHECK: xtn.2s
+; CHECK: ret
+define <2 x i32> @test4(double %d) {
+entry:
+  %vecinit1.i = insertelement <2 x double> undef, double %d, i32 0
+  %vecinit2.i = insertelement <2 x double> %vecinit1.i, double %d, i32 1
+  %mul.i = fmul <2 x double> %vecinit2.i, <double 16.000000e+00, double 16.000000e+00>
+  %vcvt.i = fptosi <2 x double> %mul.i to <2 x i32>
+  ret <2 x i32> %vcvt.i
+}
+
+; Truncate float to i16
+; CHECK-LABEL: test5
+; CHECK-NOT: fmul.2s
+; CHECK: fcvtzs.2s v0, v0, #4
+; CHECK: ret
+define <2 x i16> @test5(float %f) {
+entry:
+  %vecinit1.i = insertelement <2 x float> undef, float %f, i32 0
+  %vecinit2.i = insertelement <2 x float> %vecinit1.i, float %f, i32 1
+  %mul.i = fmul <2 x float> %vecinit2.i, <float 16.000000e+00, float 16.000000e+00>
+  %vcvt.i = fptosi <2 x float> %mul.i to <2 x i16>
+  ret <2 x i16> %vcvt.i
+}
+
+; Don't convert float to i64
+; CHECK-LABEL: test6
+; CHECK: fmov.2s v[[REG1:[0-9]+]], #16.00000000
+; CHECK: fmul.2s v[[REG2:[0-9]+]], v[[REG1]], v0[0]
+; CHECK: fcvtl v[[REG2]].2d, v[[REG2]].2s
+; CHECK: fcvtzs.2d v0, v[[REG2]]
+; CHECK: ret
+define <2 x i64> @test6(float %f) {
+entry:
+  %vecinit1.i = insertelement <2 x float> undef, float %f, i32 0
+  %vecinit2.i = insertelement <2 x float> %vecinit1.i, float %f, i32 1
+  %mul.i = fmul <2 x float> %vecinit2.i, <float 16.000000e+00, float 16.000000e+00>
+  %vcvt.i = fptosi <2 x float> %mul.i to <2 x i64>
+  ret <2 x i64> %vcvt.i
+}
+
+; Check unsigned conversion.
+; CHECK-LABEL: test7
+; CHECK-NOT: fmul.2s
+; CHECK: fcvtzu.2s v0, v0, #4
+; CHECK: ret
+define <2 x i32> @test7(float %f) {
+entry:
+  %vecinit1.i = insertelement <2 x float> undef, float %f, i32 0
+  %vecinit2.i = insertelement <2 x float> %vecinit1.i, float %f, i32 1
+  %mul.i = fmul <2 x float> %vecinit2.i, <float 16.000000e+00, float 16.000000e+00>
+  %vcvt.i = fptoui <2 x float> %mul.i to <2 x i32>
+  ret <2 x i32> %vcvt.i
+}
+
+; Test which should not fold due to non-power of 2.
+; CHECK-LABEL: test8
+; CHECK: fmov.2s v[[REG1:[0-9]+]], #17.00000000
+; CHECK: fmul.2s v[[REG2:[0-9]+]], v[[REG1]], v0[0]
+; CHECK: fcvtzu.2s v0, v[[REG2]]
+; CHECK: ret
+define <2 x i32> @test8(float %f) {
+entry:
+  %vecinit1.i = insertelement <2 x float> undef, float %f, i32 0
+  %vecinit2.i = insertelement <2 x float> %vecinit1.i, float %f, i32 1
+  %mul.i = fmul <2 x float> %vecinit2.i, <float 17.000000e+00, float 17.000000e+00>
+  %vcvt.i = fptoui <2 x float> %mul.i to <2 x i32>
+  ret <2 x i32> %vcvt.i
+}
+
+; Test which should not fold due to non-matching power of 2.
+; CHECK-LABEL: test9
+; CHECK: fmul.2s v[[REG2:[0-9]+]], v{{[0-9]+}}, v0[0]
+; CHECK: fcvtzu.2s v0, v[[REG2]]
+; CHECK: ret
+define <2 x i32> @test9(float %f) {
+entry:
+  %vecinit1.i = insertelement <2 x float> undef, float %f, i32 0
+  %vecinit2.i = insertelement <2 x float> %vecinit1.i, float %f, i32 1
+  %mul.i = fmul <2 x float> %vecinit2.i, <float 16.000000e+00, float 8.000000e+00>
+  %vcvt.i = fptoui <2 x float> %mul.i to <2 x i32>
+  ret <2 x i32> %vcvt.i
+}
+
+; Don't combine undefs.
+; CHECK-LABEL: test10
+; CHECK: fmul.2s v{{[0-9]+}}, v{{[0-9]+}}, v0[0]
+; CHECK: fcvtzu.2s v{{[0-9]+}}, v{{[0-9]+}}
+; CHECK: ret
+define <2 x i32> @test10(float %f) {
+entry:
+  %vecinit1.i = insertelement <2 x float> undef, float %f, i32 0
+  %vecinit2.i = insertelement <2 x float> %vecinit1.i, float %f, i32 1
+  %mul.i = fmul <2 x float> %vecinit2.i, <float undef, float undef>
+  %vcvt.i = fptoui <2 x float> %mul.i to <2 x i32>
+  ret <2 x i32> %vcvt.i
+}

test/CodeGen/AArch64/fdiv_combine.ll

+; RUN: llc -mtriple=aarch64-linux-gnu -aarch64-neon-syntax=apple -verify-machineinstrs -o - %s | FileCheck %s
+
+; Test signed conversion.
+; CHECK-LABEL: @test1
+; CHECK: scvtf.2s v0, v0, #4
+; CHECK: ret
+define <2 x float> @test1(<2 x i32> %in) {
+entry:
+  %vcvt.i = sitofp <2 x i32> %in to <2 x float>
+  %div.i = fdiv <2 x float> %vcvt.i, <float 16.0, float 16.0>
+  ret <2 x float> %div.i
+}
+
+; Test unsigned conversion.
+; CHECK-LABEL: @test2
+; CHECK: ucvtf.2s v0, v0, #3
+; CHECK: ret
+define <2 x float> @test2(<2 x i32> %in) {
+entry:
+  %vcvt.i = uitofp <2 x i32> %in to <2 x float>
+  %div.i = fdiv <2 x float> %vcvt.i, <float 8.0, float 8.0>
+  ret <2 x float> %div.i
+}
+
+; Test which should not fold due to non-power of 2.
+; CHECK-LABEL: @test3
+; CHECK: scvtf.2s v0, v0
+; CHECK: fmov.2s v[[REG1:[0-9]+]], #9.00000000
+; CHECK: fdiv.2s v0, v0, v[[REG1]]
+; CHECK: ret
+define <2 x float> @test3(<2 x i32> %in) {
+entry:
+  %vcvt.i = sitofp <2 x i32> %in to <2 x float>
+  %div.i = fdiv <2 x float> %vcvt.i, <float 9.0, float 9.0>
+  ret <2 x float> %div.i
+}
+
+; Test which should not fold due to power of 2 out of range.
+; CHECK-LABEL: @test4
+; CHECK: scvtf.2s v0, v0
+; CHECK: movi.2s v[[REG1:[0-9]+]], #0x50, lsl #24
+; CHECK: fdiv.2s v0, v0, v[[REG1]]
+; CHECK: ret
+define <2 x float> @test4(<2 x i32> %in) {
+entry:
+  %vcvt.i = sitofp <2 x i32> %in to <2 x float>
+  %div.i = fdiv <2 x float> %vcvt.i, <float 0x4200000000000000, float 0x4200000000000000>
+  ret <2 x float> %div.i
+}
+
+; Test case where const is max power of 2 (i.e., 2^32).
+; CHECK-LABEL: @test5
+; CHECK: scvtf.2s v0, v0, #32
+; CHECK: ret
+define <2 x float> @test5(<2 x i32> %in) {
+entry:
+  %vcvt.i = sitofp <2 x i32> %in to <2 x float>
+  %div.i = fdiv <2 x float> %vcvt.i, <float 0x41F0000000000000, float 0x41F0000000000000>
+  ret <2 x float> %div.i
+}
+
+; Test quadword.
+; CHECK-LABEL: @test6
+; CHECK: scvtf.4s v0, v0, #2
+; CHECK: ret
+define <4 x float> @test6(<4 x i32> %in) {
+entry:
+  %vcvt.i = sitofp <4 x i32> %in to <4 x float>
+  %div.i = fdiv <4 x float> %vcvt.i, <float 4.0, float 4.0, float 4.0, float 4.0>
+  ret <4 x float> %div.i
+}
+
+; Test unsigned i16 to float
+; CHECK-LABEL: @test7
+; CHECK: ushll.4s  v0, v0, #0
+; CHECK: ucvtf.4s  v0, v0, #1
+; CHECK: ret
+define <4 x float> @test7(<4 x i16> %in) {
+  %conv = uitofp <4 x i16> %in to <4 x float>
+  %shift = fdiv <4 x float> %conv, <float 2.0, float 2.0, float 2.0, float 2.0>
+  ret <4 x float> %shift
+}
+
+; Test signed i16 to float
+; CHECK-LABEL: @test8
+; CHECK: sshll.4s v0, v0, #0
+; CHECK: scvtf.4s v0, v0, #2
+; CHECK: ret
+define <4 x float> @test8(<4 x i16> %in) {
+  %conv = sitofp <4 x i16> %in to <4 x float>
+  %shift = fdiv <4 x float> %conv, <float 4.0, float 4.0, float 4.0, float 4.0>
+  ret <4 x float> %shift
+}
+
+; Can't convert i64 to float.
+; CHECK-LABEL: @test9
+; CHECK: ucvtf.2d v[[REG1:[0-9]+]], v0
+; CHECK: fcvtn v[[REG2:[0-9]+]].2s, v[[REG1]].2d
+; CHECK: movi.2s v[[REG3:[0-9]+]], #0x40, lsl #24
+; CHECK: fdiv.2s v0, v[[REG2]], v[[REG3]]
+; CHECK: ret
+define <2 x float> @test9(<2 x i64> %in) {
+  %conv = uitofp <2 x i64> %in to <2 x float>
+  %shift = fdiv <2 x float> %conv, <float 2.0, float 2.0>
+  ret <2 x float> %shift
+}
+
+; CHECK-LABEL: @test10
+; CHECK: ucvtf.2d v0, v0, #1
+; CHECK: ret
+define <2 x double> @test10(<2 x i64> %in) {
+  %conv = uitofp <2 x i64> %in to <2 x double>
+  %shift = fdiv <2 x double> %conv, <double 2.0, double 2.0>
+  ret <2 x double> %shift
+}