[AArch64] Fold vector fmul/fdiv by pow2 into floating-point/fixed-point conversions.

[mcrosier]

Summary

Adds DAG combines for:

1. Folding a floating-point multiply by power of two into floating-point to fixed-point conversion and
2. Folding a floating-point multiply by power of two into fixed-point to floating-point conversion.

This is essentially a port of a combine from the ARM backend (r133813).

Thanks,

Chad

[t.p.northover]

Hi Chad,

I think the bulk of the logic's fine, just one or two cleanups...

lib/Target/AArch64/AArch64ISelLowering.cpp

7537–7538

I think this function could be made simpler (and more generic, e.g. handling undefs too) by calling BuildVectorSDNode::isConstantSplat.

test/CodeGen/AArch64/fcvt_combine.ll

5

This style doesn't check that the fixed-point shift is correctly translated.

[mcrosier]

Updated the helper function, isConstVecPow2, per Tim's comments.

Added support for folding sitofp/uitofp followed by fdiv by power of two.

[t.p.northover]

Thanks for updating the patch, Chad. Still got a few more issues...

lib/Target/AArch64/AArch64ISelLowering.cpp

7537

I've just noticed this started life in ARMISelLowering.cpp; since we're improving it here, can we fix that one too? Or even share the implementation somewhere (BuildVectorSDNode, or perhaps SelectionDAG if you anticipate handling more cases in the future)

7542

Why are we bailing on undef here?

7548

I don't think this can handle 2^64. Really, we might as well convert to an APSInt and use exactLogBase2 or something.

[mcrosier]

Thanks, Tim. I'll post a new revision in a bit.

lib/Target/AArch64/AArch64ISelLowering.cpp

7537

I was planning on updating the version in ARMISelLowering as well, but I agree we can just make this common to BuildVectorSDNode or elsewhere and use the new version in both ARM and AArch64.

7542

No particular reason other than I was trying to be conservative. I'll remove the !UndefElements.none() check.

7548

Thanks for the suggestion, Tim. I'm not overly familiar with all of the APFloat/APInt APIs, hence the less than ideal solution. I'll start digging to see if I can't implement a more robust solution.

[mcrosier]

This should take care of all of your suggested improvements, Tim.

1. Accommodated Tim's request for a generic interface for both ARM and AArch64 to compute const FP splat of power of 2.
2. Allow matching against constant splats with undefs.
   
   Chad

[t.p.northover]

Thanks Chad. I think this looks good now.

Tim.

[mcrosier]

Thanks, Tim! Committed in r249572, r249576, and r249579.

lib/Target/AArch64/AArch64ISelLowering.cpp

   setTargetDAGCombine(ISD::SINT_TO_FP);
   setTargetDAGCombine(ISD::UINT_TO_FP);
 
+  setTargetDAGCombine(ISD::FP_TO_SINT);
+  setTargetDAGCombine(ISD::FP_TO_UINT);
+
   setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
 
   setTargetDAGCombine(ISD::ANY_EXTEND);

   return SDValue();
 }
 
+// isConstVecPow2 - Return true if each vector element is a power of 2, all
+// elements are the same constant, C, and Log2(C) ranges from 1 to Range.
+static bool isConstVecPow2(SDValue ConstVec, bool IsSigned, uint64_t &C,
+                           unsigned Range) {
+  integerPart CN;
+  integerPart C0 = 0;
+  for (unsigned I = 0, E = ConstVec.getValueType().getVectorNumElements();
+       I != E; I++) {
+    ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(ConstVec.getOperand(I));
+    if (!C)
+      return false;
+
+    bool IsExact;
+    APFloat APF = C->getValueAPF();
+    if (APF.convertToInteger(&CN, 64, IsSigned, APFloat::rmTowardZero,
+                             &IsExact) != APFloat::opOK ||
+        !IsExact)
+      return false;
+
+    C0 = (I == 0) ? CN : C0;
+    if (!isPowerOf2_64(CN) || C0 != CN || Log2_64(C0) < 1 ||
+        Log2_64(C0) > Range)
+      return false;
+  }
+  C = C0;
+  return true;
+}
+
+/// FCVTZS (floating-point to fixed-point,Advanced SIMD) can replace
+/// combinations of FMUL and FCVT (floating-point to integer) when the FMUL has
+/// a constant operand that is a power of 2.
+static SDValue performFpToIntCombine(SDNode *N, SelectionDAG &DAG,
+                                     const AArch64Subtarget *Subtarget) {
+  if (!Subtarget->hasNEON())
+    return SDValue();
+
+  SDValue Op = N->getOperand(0);
+  if (!Op.getValueType().isVector() || Op.getOpcode() != ISD::FMUL)
+    return SDValue();
+
+  MVT FloatTy = Op.getSimpleValueType().getVectorElementType();
+  uint32_t FloatBits = FloatTy.getSizeInBits();
+  if (FloatBits != 32 && FloatBits != 64)
+    return SDValue();
+
+  MVT IntTy = N->getSimpleValueType(0).getVectorElementType();
+  uint32_t IntBits = IntTy.getSizeInBits();
+  if (IntBits != 16 && IntBits != 32 && IntBits != 64)
+    return SDValue();
+
+  // Avoid conversions like float -> i64.
+  if (IntBits > FloatBits)
+    return SDValue();
+
+  uint64_t C;
+  SDValue N0 = Op->getOperand(0);
+  SDValue ConstVec = Op->getOperand(1);
+  bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT;
+  if (ConstVec.getOpcode() != ISD::BUILD_VECTOR ||
+      !isConstVecPow2(ConstVec, IsSigned, C, IntBits == 64 ? 64 : 32))
+    return SDValue();
+
+  MVT ResTy;
+  unsigned NumLanes = Op.getValueType().getVectorNumElements();
+  switch (NumLanes) {
+  default:
+    return SDValue();
+  case 2:
+    ResTy = FloatBits == 32 ? MVT::v2i32 : MVT::v2i64;
+    break;
+  case 4:
+    ResTy = MVT::v4i32;
+    break;
+  }
+
+  SDLoc DL(N);
+  unsigned IntrinsicOpcode = IsSigned ? Intrinsic::aarch64_neon_vcvtfp2fxs
+                                      : Intrinsic::aarch64_neon_vcvtfp2fxu;
+  SDValue FixConv = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ResTy,
+                                DAG.getConstant(IntrinsicOpcode, DL, MVT::i32),
+                                N0, DAG.getConstant(Log2_64(C), DL, MVT::i32));
+  // We can handle smaller integers by generating an extra trunc.
+  if (IntBits < FloatBits)
+    FixConv = DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), FixConv);
+
+  return FixConv;
+}
+
 /// An EXTR instruction is made up of two shifts, ORed together. This helper
 /// searches for and classifies those shifts.
 static bool findEXTRHalf(SDValue N, SDValue &Src, uint32_t &ShiftAmount,

   case ISD::SINT_TO_FP:
   case ISD::UINT_TO_FP:
     return performIntToFpCombine(N, DAG, Subtarget);
+  case ISD::FP_TO_SINT:
+  case ISD::FP_TO_UINT:
+    return performFpToIntCombine(N, DAG, Subtarget);
   case ISD::OR:
     return performORCombine(N, DCI, Subtarget);
   case ISD::INTRINSIC_WO_CHAIN:

test/CodeGen/AArch64/fcvt_combine.ll

+; RUN: llc -mtriple=aarch64-linux-gnu -aarch64-neon-syntax=apple -verify-machineinstrs -o - %s | FileCheck %s
+
+; CHECK-LABEL: test1
+; CHECK-NOT: fmul.2s
+; CHECK: fcvtzs.2s
+; CHECK: ret
+define <2 x i32> @test1(float %f) {
+entry:
+  %vecinit1.i = insertelement <2 x float> undef, float %f, i32 0
+  %vecinit2.i = insertelement <2 x float> %vecinit1.i, float %f, i32 1
+  %mul.i = fmul <2 x float> %vecinit2.i, <float 16.000000e+00, float 16.000000e+00>
+  %vcvt.i = fptosi <2 x float> %mul.i to <2 x i32>
+  ret <2 x i32> %vcvt.i
+}
+
+; CHECK-LABEL: test2
+; CHECK-NOT: fmul.4s
+; CHECK: fcvtzs.4s
+; CHECK: ret
+define <4 x i32> @test2(float %f) {
+entry:
+  %vecinit1.i = insertelement <4 x float> undef, float %f, i32 0
+  %vecinit2.i = insertelement <4 x float> %vecinit1.i, float %f, i32 1
+  %vecinit3.i = insertelement <4 x float> %vecinit2.i, float %f, i32 2
+  %vecinit4.i = insertelement <4 x float> %vecinit3.i, float %f, i32 3
+  %mul.i = fmul <4 x float> %vecinit4.i, <float 16.000000e+00, float 16.000000e+00, float 16.000000e+00, float 16.000000e+00>
+  %vcvt.i = fptosi <4 x float> %mul.i to <4 x i32>
+  ret <4 x i32> %vcvt.i
+}
+
+; CHECK-LABEL: test3
+; CHECK-NOT: fmul.2d
+; CHECK: fcvtzs.2d
+; CHECK: ret
+define <2 x i64> @test3(double %d) {
+entry:
+  %vecinit1.i = insertelement <2 x double> undef, double %d, i32 0
+  %vecinit2.i = insertelement <2 x double> %vecinit1.i, double %d, i32 1
+  %mul.i = fmul <2 x double> %vecinit2.i, <double 16.000000e+00, double 16.000000e+00>
+  %vcvt.i = fptosi <2 x double> %mul.i to <2 x i64>
+  ret <2 x i64> %vcvt.i
+}
+
+; Truncate double to i32
+; CHECK-LABEL: test4
+; CHECK-NOT: fmul.2d
+; CHECK: fcvtzs.2d
+; CHECK: xtn.2s
+; CHECK: ret
+define <2 x i32> @test4(double %d) {
+entry:
+  %vecinit1.i = insertelement <2 x double> undef, double %d, i32 0
+  %vecinit2.i = insertelement <2 x double> %vecinit1.i, double %d, i32 1
+  %mul.i = fmul <2 x double> %vecinit2.i, <double 16.000000e+00, double 16.000000e+00>
+  %vcvt.i = fptosi <2 x double> %mul.i to <2 x i32>
+  ret <2 x i32> %vcvt.i
+}
+
+; Truncate float to i16
+; CHECK-LABEL: test5
+; CHECK-NOT: fmul.2s
+; CHECK: fcvtzs.2s
+; CHECK: ret
+define <2 x i16> @test5(float %f) {
+entry:
+  %vecinit1.i = insertelement <2 x float> undef, float %f, i32 0
+  %vecinit2.i = insertelement <2 x float> %vecinit1.i, float %f, i32 1
+  %mul.i = fmul <2 x float> %vecinit2.i, <float 16.000000e+00, float 16.000000e+00>
+  %vcvt.i = fptosi <2 x float> %mul.i to <2 x i16>
+  ret <2 x i16> %vcvt.i
+}
+
+; Don't convert float to i64
+; CHECK-LABEL: test6
+; CHECK: fmul.2s
+; CHECK: fcvtl
+; CHECK: fcvtzs.2d
+; CHECK: ret
+define <2 x i64> @test6(float %f) {
+entry:
+  %vecinit1.i = insertelement <2 x float> undef, float %f, i32 0
+  %vecinit2.i = insertelement <2 x float> %vecinit1.i, float %f, i32 1
+  %mul.i = fmul <2 x float> %vecinit2.i, <float 16.000000e+00, float 16.000000e+00>
+  %vcvt.i = fptosi <2 x float> %mul.i to <2 x i64>
+  ret <2 x i64> %vcvt.i
+}
+
+; Check unsigned conversion.
+; CHECK-LABEL: test7
+; CHECK-NOT: fmul.2s
+; CHECK: fcvtzu.2s
+; CHECK: ret
+define <2 x i32> @test7(float %f) {
+entry:
+  %vecinit1.i = insertelement <2 x float> undef, float %f, i32 0
+  %vecinit2.i = insertelement <2 x float> %vecinit1.i, float %f, i32 1
+  %mul.i = fmul <2 x float> %vecinit2.i, <float 16.000000e+00, float 16.000000e+00>
+  %vcvt.i = fptoui <2 x float> %mul.i to <2 x i32>
+  ret <2 x i32> %vcvt.i
+}
+
+; Test which should not fold due to non-power of 2.
+; CHECK-LABEL: test8
+; CHECK: fmul.2s
+; CHECK: fcvtzu.2s
+; CHECK: ret
+define <2 x i32> @test8(float %f) {
+entry:
+  %vecinit1.i = insertelement <2 x float> undef, float %f, i32 0
+  %vecinit2.i = insertelement <2 x float> %vecinit1.i, float %f, i32 1
+  %mul.i = fmul <2 x float> %vecinit2.i, <float 17.000000e+00, float 17.000000e+00>
+  %vcvt.i = fptoui <2 x float> %mul.i to <2 x i32>
+  ret <2 x i32> %vcvt.i
+}
+
+; Test which should not fold due to non-matching power of 2.
+; CHECK-LABEL: test9
+; CHECK: fmul.2s
+; CHECK: fcvtzu.2s
+; CHECK: ret
+define <2 x i32> @test9(float %f) {
+entry:
+  %vecinit1.i = insertelement <2 x float> undef, float %f, i32 0
+  %vecinit2.i = insertelement <2 x float> %vecinit1.i, float %f, i32 1
+  %mul.i = fmul <2 x float> %vecinit2.i, <float 16.000000e+00, float 8.000000e+00>
+  %vcvt.i = fptoui <2 x float> %mul.i to <2 x i32>
+  ret <2 x i32> %vcvt.i
+}