[AArch64] Improve FP16 instruction selection for vector round and vector convert from half instructions

[az]

Summary

We currently generate inefficient code for vector rounding and vector convert from half due to minor issues in instruction selection lowering. Fix those issues related to checking for the fullfp16 support (for convert instruction) and initialization (for round instruction).

[SjoerdMeijer]

LGTM

llvm/lib/Target/AArch64/AArch64ISelLowering.h

private:
   SDValue LowerShiftRightParts(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerCTPOP(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerF128Call(SDValue Op, SelectionDAG &DAG,
                         RTLIB::Libcall Call) const;
   SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVectorAND(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVectorOR(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerVECREDUCE(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerATOMIC_LOAD_SUB(SDValue Op, SelectionDAG &DAG) const;

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

for (MVT Ty : {MVT::v2f32, MVT::v4f32, MVT::v2f64}) {
       setOperationAction(ISD::FFLOOR, Ty, Legal);
       setOperationAction(ISD::FNEARBYINT, Ty, Legal);
       setOperationAction(ISD::FCEIL, Ty, Legal);
       setOperationAction(ISD::FRINT, Ty, Legal);
       setOperationAction(ISD::FTRUNC, Ty, Legal);
       setOperationAction(ISD::FROUND, Ty, Legal);
     }
 
+    if (Subtarget->hasFullFP16()) {
+      for (MVT Ty : {MVT::v4f16, MVT::v8f16}) {
+        setOperationAction(ISD::FFLOOR, Ty, Legal);
+        setOperationAction(ISD::FNEARBYINT, Ty, Legal);
+        setOperationAction(ISD::FCEIL, Ty, Legal);
+        setOperationAction(ISD::FRINT, Ty, Legal);
+        setOperationAction(ISD::FTRUNC, Ty, Legal);
+        setOperationAction(ISD::FROUND, Ty, Legal);
+      }
+    }
+
     setTruncStoreAction(MVT::v4i16, MVT::v4i8, Custom);
   }
 
   PredictableSelectIsExpensive = Subtarget->predictableSelectIsExpensive();
 }
 
 void AArch64TargetLowering::addTypeForNEON(MVT VT, MVT PromotedBitwiseVT) {
   assert(VT.isVector() && "VT should be a vector type");
SDValue AArch64TargetLowering::LowerFP_ROUND(SDValue Op,
   // FP_ROUND node has a second operand indicating whether it is known to be
   // precise. That doesn't take part in the LibCall so we can't directly use
   // LowerF128Call.
   SDValue SrcVal = Op.getOperand(0);
   return makeLibCall(DAG, LC, Op.getValueType(), SrcVal, /*isSigned*/ false,
                      SDLoc(Op)).first;
 }
 
-static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) {
+SDValue AArch64TargetLowering::LowerVectorFP_TO_INT(SDValue Op,
+                                                    SelectionDAG &DAG) const {
   // Warning: We maintain cost tables in AArch64TargetTransformInfo.cpp.
   // Any additional optimization in this function should be recorded
   // in the cost tables.
   EVT InVT = Op.getOperand(0).getValueType();
   EVT VT = Op.getValueType();
   unsigned NumElts = InVT.getVectorNumElements();
 
-  // f16 vectors are promoted to f32 before a conversion.
-  if (InVT.getVectorElementType() == MVT::f16) {
+  // f16 conversions are promoted to f32 when full fp16 is not supported.
+  if (InVT.getVectorElementType() == MVT::f16 &&
+      !Subtarget->hasFullFP16()) {
     MVT NewVT = MVT::getVectorVT(MVT::f32, NumElts);
     SDLoc dl(Op);
     return DAG.getNode(
         Op.getOpcode(), dl, Op.getValueType(),
         DAG.getNode(ISD::FP_EXTEND, dl, NewVT, Op.getOperand(0)));
   }
 
   if (VT.getSizeInBits() < InVT.getSizeInBits()) {

llvm/test/CodeGen/AArch64/arm64-vfloatintrinsics.ll

define %v4f16 @test_v4f16.nearbyint(%v4f16 %a) {
   ; CHECK-LABEL:          test_v4f16.nearbyint:
   ; CHECK-NOFP16-COUNT-4: frinti s{{[0-9]+}}, s{{[0-9]+}}
   ; CHECK-FP16-NOT:       fcvt
   ; CHECK-FP16:           frinti.4h
   ; CHECK-FP16-NEXT:      ret
   %1 = call %v4f16 @llvm.nearbyint.v4f16(%v4f16 %a)
   ret %v4f16 %1
 }
+define %v4f16 @test_v4f16.round(%v4f16 %a) {
+  ; CHECK-LABEL:          test_v4f16.round:
+  ; CHECK-NOFP16-COUNT-4: frinta s{{[0-9]+}}, s{{[0-9]+}}
+  ; CHECK-FP16-NOT:       fcvt
+  ; CHECK-FP16:           frinta.4h
+  ; CHECK-FP16-NEXT:      ret
+  %1 =  call %v4f16 @llvm.round.v4f16(%v4f16 %a)
+  ret %v4f16 %1
+}
 
 declare %v4f16 @llvm.sqrt.v4f16(%v4f16) #0
 declare %v4f16 @llvm.powi.v4f16(%v4f16, i32) #0
 declare %v4f16 @llvm.sin.v4f16(%v4f16) #0
 declare %v4f16 @llvm.cos.v4f16(%v4f16) #0
 declare %v4f16 @llvm.pow.v4f16(%v4f16, %v4f16) #0
 declare %v4f16 @llvm.exp.v4f16(%v4f16) #0
 declare %v4f16 @llvm.exp2.v4f16(%v4f16) #0
 declare %v4f16 @llvm.log.v4f16(%v4f16) #0
 declare %v4f16 @llvm.log10.v4f16(%v4f16) #0
 declare %v4f16 @llvm.log2.v4f16(%v4f16) #0
 declare %v4f16 @llvm.fma.v4f16(%v4f16, %v4f16, %v4f16) #0
 declare %v4f16 @llvm.fabs.v4f16(%v4f16) #0
 declare %v4f16 @llvm.floor.v4f16(%v4f16) #0
 declare %v4f16 @llvm.ceil.v4f16(%v4f16) #0
 declare %v4f16 @llvm.trunc.v4f16(%v4f16) #0
 declare %v4f16 @llvm.rint.v4f16(%v4f16) #0
 declare %v4f16 @llvm.nearbyint.v4f16(%v4f16) #0
+declare %v4f16 @llvm.round.v4f16(%v4f16) #0
 
 ;;;
 
 %v8f16 = type <8 x half>
 
 define %v8f16 @test_v8f16.sqrt(%v8f16 %a) {
   ; CHECK-LABEL:          test_v8f16.sqrt:
   ; CHECK-NOFP16-COUNT-8: fsqrt s{{[0-9]+}}, s{{[0-9]+}}
define %v8f16 @test_v8f16.nearbyint(%v8f16 %a) {
   ; CHECK-LABEL:          test_v8f16.nearbyint:
   ; CHECK-NOFP16-COUNT-8: frinti s{{[0-9]+}}, s{{[0-9]+}}
   ; CHECK-FP16-NOT:       fcvt
   ; CHECK-FP16:           frinti.8h
   ; CHECK-FP16-NEXT:      ret
   %1 = call %v8f16 @llvm.nearbyint.v8f16(%v8f16 %a)
   ret %v8f16 %1
 }
+define %v8f16 @test_v8f16.round(%v8f16 %a) {
+  ; CHECK-LABEL:          test_v8f16.round:
+  ; CHECK-NOFP16-COUNT-8: frinta s{{[0-9]+}}, s{{[0-9]+}}
+  ; CHECK-FP16-NOT:       fcvt
+  ; CHECK-FP16:           frinta.8h
+  ; CHECK-FP16-NEXT:      ret
+  %1 =  call %v8f16 @llvm.round.v8f16(%v8f16 %a)
+  ret %v8f16 %1
+}
 
 declare %v8f16 @llvm.sqrt.v8f16(%v8f16) #0
 declare %v8f16 @llvm.powi.v8f16(%v8f16, i32) #0
 declare %v8f16 @llvm.sin.v8f16(%v8f16) #0
 declare %v8f16 @llvm.cos.v8f16(%v8f16) #0
 declare %v8f16 @llvm.pow.v8f16(%v8f16, %v8f16) #0
 declare %v8f16 @llvm.exp.v8f16(%v8f16) #0
 declare %v8f16 @llvm.exp2.v8f16(%v8f16) #0
 declare %v8f16 @llvm.log.v8f16(%v8f16) #0
 declare %v8f16 @llvm.log10.v8f16(%v8f16) #0
 declare %v8f16 @llvm.log2.v8f16(%v8f16) #0
 declare %v8f16 @llvm.fma.v8f16(%v8f16, %v8f16, %v8f16) #0
 declare %v8f16 @llvm.fabs.v8f16(%v8f16) #0
 declare %v8f16 @llvm.floor.v8f16(%v8f16) #0
 declare %v8f16 @llvm.ceil.v8f16(%v8f16) #0
 declare %v8f16 @llvm.trunc.v8f16(%v8f16) #0
 declare %v8f16 @llvm.rint.v8f16(%v8f16) #0
 declare %v8f16 @llvm.nearbyint.v8f16(%v8f16) #0
+declare %v8f16 @llvm.round.v8f16(%v8f16) #0
 
 ;;; Float vectors
 
 %v2f32 = type <2 x float>
 
 ; CHECK-LABEL: test_v2f32.sqrt:
 define %v2f32 @test_v2f32.sqrt(%v2f32 %a) {
   ; CHECK: fsqrt.2s

llvm/test/CodeGen/AArch64/fp16-v4-instructions.ll

 ; CHECK-COMMON-NEXT:  ret
   %1 = insertelement <4 x half> undef, half %a, i64 0
   store <4 x half> %1, <4 x half>* %b, align 4
   ret void
 }
 
 define <4 x i8> @fptosi_i8(<4 x half> %a) #0 {
 ; CHECK-COMMON-LABEL: fptosi_i8:
-; CHECK-COMMON-NEXT:  fcvtl  [[REG1:v[0-9]+\.4s]], v0.4h
-; CHECK-COMMON-NEXT:  fcvtzs [[REG2:v[0-9]+\.4s]], [[REG1]]
-; CHECK-COMMON-NEXT:  xtn    v0.4h, [[REG2]]
-; CHECK-COMMON-NEXT:  ret
+; CHECK-FP16:        fcvtzs  v0.4h, v0.4h
+; CHECK-CVT-NEXT:    fcvtl  [[REG1:v[0-9]+\.4s]], v0.4h
+; CHECK-CVT-NEXT:    fcvtzs [[REG2:v[0-9]+\.4s]], [[REG1]]
+; CHECK-CVT-NEXT:    xtn    v0.4h, [[REG2]]
+; CHECK-COMMON-NEXT: ret
   %1 = fptosi<4 x half> %a to <4 x i8>
   ret <4 x i8> %1
 }
 
 define <4 x i16> @fptosi_i16(<4 x half> %a) #0 {
 ; CHECK-COMMON-LABEL: fptosi_i16:
-; CHECK-COMMON-NEXT:  fcvtl  [[REG1:v[0-9]+\.4s]], v0.4h
-; CHECK-COMMON-NEXT:  fcvtzs [[REG2:v[0-9]+\.4s]], [[REG1]]
-; CHECK-COMMON-NEXT:  xtn    v0.4h, [[REG2]]
-; CHECK-COMMON-NEXT:  ret
+; CHECK-FP16:        fcvtzs v0.4h, v0.4h
+; CHECK-CVT-NEXT:    fcvtl  [[REG1:v[0-9]+\.4s]], v0.4h
+; CHECK-CVT-NEXT:    fcvtzs [[REG2:v[0-9]+\.4s]], [[REG1]]
+; CHECK-CVT-NEXT:    xtn    v0.4h, [[REG2]]
+; CHECK-COMMON-NEXT: ret
   %1 = fptosi<4 x half> %a to <4 x i16>
   ret <4 x i16> %1
 }
 
 define <4 x i8> @fptoui_i8(<4 x half> %a) #0 {
 ; CHECK-COMMON-LABEL: fptoui_i8:
-; CHECK-COMMON-NEXT:  fcvtl  [[REG1:v[0-9]+\.4s]], v0.4h
+; CHECK-FP16:        fcvtzs  v0.4h, v0.4h
+; CHECK-CVT-NEXT:    fcvtl  [[REG1:v[0-9]+\.4s]], v0.4h
 ; NOTE: fcvtzs selected here because the xtn shaves the sign bit
-; CHECK-COMMON-NEXT:  fcvtzs [[REG2:v[0-9]+\.4s]], [[REG1]]
-; CHECK-COMMON-NEXT:  xtn    v0.4h, [[REG2]]
-; CHECK-COMMON-NEXT:  ret
+; CHECK-CVT-NEXT:    fcvtzs [[REG2:v[0-9]+\.4s]], [[REG1]]
+; CHECK-CVT-NEXT:    xtn    v0.4h, [[REG2]]
+; CHECK-COMMON-NEXT: ret
   %1 = fptoui<4 x half> %a to <4 x i8>
   ret <4 x i8> %1
 }
 
 define <4 x i16> @fptoui_i16(<4 x half> %a) #0 {
 ; CHECK-COMMON-LABEL: fptoui_i16:
-; CHECK-COMMON-NEXT:  fcvtl  [[REG1:v[0-9]+\.4s]], v0.4h
-; CHECK-COMMON-NEXT:  fcvtzu [[REG2:v[0-9]+\.4s]], [[REG1]]
-; CHECK-COMMON-NEXT:  xtn    v0.4h, [[REG2]]
+; CHECK-FP16:      fcvtzu v0.4h, v0.4h
+; CHECK-CVT-NEXT:  fcvtl  [[REG1:v[0-9]+\.4s]], v0.4h
+; CHECK-CVT-NEXT:  fcvtzu [[REG2:v[0-9]+\.4s]], [[REG1]]
+; CHECK-CVT-NEXT:  xtn    v0.4h, [[REG2]]
 ; CHECK-COMMON-NEXT:  ret
   %1 = fptoui<4 x half> %a to <4 x i16>
   ret <4 x i16> %1
 }
 
 define <4 x i1> @test_fcmp_une(<4 x half> %a, <4 x half> %b) #0 {
 ; CHECK-CVT-LABEL: test_fcmp_une:
 ; CHECK-CVT: fcvtl

llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll

 ; CHECK-NEXT: ret
   %1 = insertelement <8 x half> undef, half %a, i64 0
   store <8 x half> %1, <8 x half>* %b, align 4
   ret void
 }
 
 define <8 x i8> @fptosi_i8(<8 x half> %a) #0 {
 ; CHECK-LABEL: fptosi_i8:
-; CHECK-DAG: fcvtl   [[LO:v[0-9]+\.4s]], v0.4h
-; CHECK-DAG: fcvtl2  [[HI:v[0-9]+\.4s]], v0.8h
-; CHECK-DAG: fcvtzs  [[LOF32:v[0-9]+\.4s]], [[LO]]
-; CHECK-DAG: xtn     [[I16:v[0-9]+]].4h, [[LOF32]]
-; CHECK-DAG: fcvtzs  [[HIF32:v[0-9]+\.4s]], [[HI]]
-; CHECK-DAG: xtn2    [[I16]].8h, [[HIF32]]
-; CHECK-NEXT: xtn     v0.8b, [[I16]].8h
-; CHECK-NEXT: ret
+; CHECK-FP16-NEXT: fcvtzs [[LO:v[0-9]+\.8h]], v0.8h
+; CHECK-CVT-DAG:   fcvtl   [[LO:v[0-9]+\.4s]], v0.4h
+; CHECK-CVT-DAG:   fcvtl2  [[HI:v[0-9]+\.4s]], v0.8h
+; CHECK-CVT-DAG:   fcvtzs  [[LOF32:v[0-9]+\.4s]], [[LO]]
+; CHECK-CVT-DAG:   xtn     [[I16:v[0-9]+]].4h, [[LOF32]]
+; CHECK-CVT-DAG:   fcvtzs  [[HIF32:v[0-9]+\.4s]], [[HI]]
+; CHECK-CVT-DAG:   xtn2    [[I16]].8h, [[HIF32]]
+; CHECK-CVT-DAG:   xtn     v0.8b, [[I16]].8h
+; CHECK-FP16-NEXT: xtn     v0.8b, [[LO]]
+; CHECK-NEXT:      ret
   %1 = fptosi<8 x half> %a to <8 x i8>
   ret <8 x i8> %1
 }
 
 define <8 x i16> @fptosi_i16(<8 x half> %a) #0 {
 ; CHECK-LABEL: fptosi_i16:
-; CHECK-DAG: fcvtl   [[LO:v[0-9]+\.4s]], v0.4h
-; CHECK-DAG: fcvtl2  [[HI:v[0-9]+\.4s]], v0.8h
-; CHECK-DAG: fcvtzs  [[LOF32:v[0-9]+\.4s]], [[LO]]
-; CHECK-DAG: xtn     [[I16:v[0-9]+]].4h, [[LOF32]]
-; CHECK-DAG: fcvtzs  [[HIF32:v[0-9]+\.4s]], [[HI]]
-; CHECK-NEXT: xtn2    [[I16]].8h, [[HIF32]]
-; CHECK-NEXT: ret
+; CHECK-FP16-NEXT: fcvtzs  v0.8h, v0.8h
+; CHECK-CVT_DAG:   fcvtl   [[LO:v[0-9]+\.4s]], v0.4h
+; CHECK-CVT_DAG:   fcvtl2  [[HI:v[0-9]+\.4s]], v0.8h
+; CHECK-CVT_DAG:   fcvtzs  [[LOF32:v[0-9]+\.4s]], [[LO]]
+; CHECK-CVT_DAG:   xtn     [[I16:v[0-9]+]].4h, [[LOF32]]
+; CHECK-CVT_DAG:   fcvtzs  [[HIF32:v[0-9]+\.4s]], [[HI]]
+; CHECK-CVT_DAG:   xtn2    [[I16]].8h, [[HIF32]]
+; CHECK-COMMON_NEXT:      ret
   %1 = fptosi<8 x half> %a to <8 x i16>
   ret <8 x i16> %1
 }
 
 define <8 x i8> @fptoui_i8(<8 x half> %a) #0 {
 ; CHECK-LABEL: fptoui_i8:
-; CHECK-DAG: fcvtl   [[LO:v[0-9]+\.4s]], v0.4h
-; CHECK-DAG: fcvtl2  [[HI:v[0-9]+\.4s]], v0.8h
-; CHECK-DAG: fcvtzu  [[LOF32:v[0-9]+\.4s]], [[LO]]
-; CHECK-DAG: xtn     [[I16:v[0-9]+]].4h, [[LOF32]]
-; CHECK-DAG: fcvtzu  [[HIF32:v[0-9]+\.4s]], [[HI]]
-; CHECK-DAG: xtn2    [[I16]].8h, [[HIF32]]
-; CHECK-NEXT: xtn     v0.8b, [[I16]].8h
+; CHECK-FP16-NEXT: fcvtzu [[LO:v[0-9]+\.8h]], v0.8h
+; CHECK-CVT-DAG:   fcvtl   [[LO:v[0-9]+\.4s]], v0.4h
+; CHECK-CVT-DAG:   fcvtl2  [[HI:v[0-9]+\.4s]], v0.8h
+; CHECK-CVT-DAG:   fcvtzu  [[LOF32:v[0-9]+\.4s]], [[LO]]
+; CHECK-CVT-DAG:   xtn     [[I16:v[0-9]+]].4h, [[LOF32]]
+; CHECK-CVT-DAG:   fcvtzu  [[HIF32:v[0-9]+\.4s]], [[HI]]
+; CHECK-CVT-DAG:   xtn2    [[I16]].8h, [[HIF32]]
+; CHECK-CVT-DAG:   xtn     v0.8b, [[I16]].8h
+; CHECK-FP16-NEXT: xtn     v0.8b, [[LO]]
 ; CHECK-NEXT: ret
   %1 = fptoui<8 x half> %a to <8 x i8>
   ret <8 x i8> %1
 }
 
 define <8 x i16> @fptoui_i16(<8 x half> %a) #0 {
 ; CHECK-LABEL: fptoui_i16:
-; CHECK-DAG: fcvtl   [[LO:v[0-9]+\.4s]], v0.4h
-; CHECK-DAG: fcvtl2  [[HI:v[0-9]+\.4s]], v0.8h
-; CHECK-DAG: fcvtzu  [[LOF32:v[0-9]+\.4s]], [[LO]]
-; CHECK-DAG: xtn     [[I16:v[0-9]+]].4h, [[LOF32]]
-; CHECK-DAG: fcvtzu  [[HIF32:v[0-9]+\.4s]], [[HI]]
-; CHECK-NEXT: xtn2    [[I16]].8h, [[HIF32]]
-; CHECK-NEXT: ret
+; CHECK-FP16-NEXT: fcvtzu  v0.8h, v0.8h
+; CHECK-CVT-DAG:   fcvtl   [[LO:v[0-9]+\.4s]], v0.4h
+; CHECK-CVT-DAG:   fcvtl2  [[HI:v[0-9]+\.4s]], v0.8h
+; CHECK-CVT-DAG:   fcvtzu  [[LOF32:v[0-9]+\.4s]], [[LO]]
+; CHECK-CVT-DAG:   xtn     [[I16:v[0-9]+]].4h, [[LOF32]]
+; CHECK-CVT-DAG:   fcvtzu  [[HIF32:v[0-9]+\.4s]], [[HI]]
+; CHECK-CVT-DAG:   xtn2    [[I16]].8h, [[HIF32]]
+; CHECK-NEXT:      ret
   %1 = fptoui<8 x half> %a to <8 x i16>
   ret <8 x i16> %1
 }
 
 define <8 x i1> @test_fcmp_une(<8 x half> %a, <8 x half> %b) #0 {
 ; FileCheck checks are unwieldy with 16 fcvt and 8 csel tests, so skipped for -fullfp16.
 
 ; CHECK-FP16-LABEL: test_fcmp_une: