[AArch64] match fcvtl2 with bitcasted extract

[spatel]

Summary

This is one of the suggestions for eliminating a regression in D63815.
If we are FP extending the high half extract of a vector, we should be able to peek through a bitcast sitting between the extract and extend.

[efriedma]

llvm/lib/Target/AArch64/AArch64InstrInfo.td

3639

This looks correct.

Can we generalize this to handle the top half of any 128-bit vector, somehow? Extending the set of special cases to include v2i64 in addition to v4f32 doesn't seem like a complete plan.

[spatel]

llvm/lib/Target/AArch64/AArch64InstrInfo.td

3639

I don't know how to hack this in tablegen, but it seems straightforward in DAGToDAGISel.

[spatel]

Patch updated:

1. Added tests for other casts and half->float. (rGfbaf835c5c51)
2. Add a helper to AArch64DAGToDAGISel to deal with all high half extend possibilities.
3. Remove redundant tablegen patterns for fcvtl2.

[efriedma]

LGTM with one minor nit

llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp

3043

Indentation?

[spatel]

llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp

3043

Oops - yes, IDE isn't set to comply with clang's switch bracket setting. Will fix on commit.

llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp

public:
   void SelectStoreLane(SDNode *N, unsigned NumVecs, unsigned Opc);
   void SelectPostStoreLane(SDNode *N, unsigned NumVecs, unsigned Opc);
 
   bool tryBitfieldExtractOp(SDNode *N);
   bool tryBitfieldExtractOpFromSExt(SDNode *N);
   bool tryBitfieldInsertOp(SDNode *N);
   bool tryBitfieldInsertInZeroOp(SDNode *N);
   bool tryShiftAmountMod(SDNode *N);
+  bool tryHighFPExt(SDNode *N);
 
   bool tryReadRegister(SDNode *N);
   bool tryWriteRegister(SDNode *N);
 
 // Include the pieces autogenerated from the target description.
 #include "AArch64GenDAGISel.inc"
 
 private:
bool AArch64DAGToDAGISel::tryBitfieldExtractOpFromSExt(SDNode *N) {
   unsigned Immr = ShiftImm;
   unsigned Imms = NarrowVT.getSizeInBits() - 1;
   SDValue Ops[] = {Opd0, CurDAG->getTargetConstant(Immr, dl, VT),
                    CurDAG->getTargetConstant(Imms, dl, VT)};
   CurDAG->SelectNodeTo(N, AArch64::SBFMXri, VT, Ops);
   return true;
 }
 
+/// Try to form fcvtl2 instructions from a floating-point extend of a high-half
+/// extract of a subvector.
+bool AArch64DAGToDAGISel::tryHighFPExt(SDNode *N) {
+  assert(N->getOpcode() == ISD::FP_EXTEND);
+
+  // There are 2 forms of fcvtl2 - extend to double or extend to float.
+  SDValue Extract = N->getOperand(0);
+  EVT VT = N->getValueType(0);
+  EVT NarrowVT = Extract.getValueType();
+  if ((VT != MVT::v2f64 || NarrowVT != MVT::v2f32) &&
+      (VT != MVT::v4f32 || NarrowVT != MVT::v4f16))
+    return false;
+
+  // Optionally look past a bitcast.
+  Extract = peekThroughBitcasts(Extract);
+  if (Extract.getOpcode() != ISD::EXTRACT_SUBVECTOR)
+    return false;
+
+  // Match extract from start of high half index.
+  // Example: v8i16 -> v4i16 means the extract must begin at index 4.
+  unsigned ExtractIndex = Extract.getConstantOperandVal(1);
+  if (ExtractIndex != Extract.getValueType().getVectorNumElements())
+    return false;
+
+  auto Opcode = VT == MVT::v2f64 ? AArch64::FCVTLv4i32 : AArch64::FCVTLv8i16;
+  CurDAG->SelectNodeTo(N, Opcode, VT, Extract.getOperand(0));
+  return true;
+}
+
 static bool isBitfieldExtractOp(SelectionDAG *CurDAG, SDNode *N, unsigned &Opc,
                                 SDValue &Opd0, unsigned &Immr, unsigned &Imms,
                                 unsigned NumberOfIgnoredLowBits = 0,
                                 bool BiggerPattern = false) {
   if (N->getValueType(0) != MVT::i32 && N->getValueType(0) != MVT::i64)
     return false;
 
   switch (N->getOpcode()) {
if (tryShiftAmountMod(Node))
       return;
     break;
 
   case ISD::SIGN_EXTEND:
     if (tryBitfieldExtractOpFromSExt(Node))
       return;
     break;
 
+    case ISD::FP_EXTEND:
+      if (tryHighFPExt(Node))
+        return;
+      break;
+
   case ISD::OR:
     if (tryBitfieldInsertOp(Node))
       return;
     break;
 
   case ISD::Constant: {
     // Materialize zero constants as copies from WZR/XZR.  This allows
     // the coalescer to propagate these into other instructions.

llvm/lib/Target/AArch64/AArch64InstrInfo.td

 defm FCVTAU : SIMDTwoVectorFPToInt<1,0,0b11100, "fcvtau",int_aarch64_neon_fcvtau>;
 defm FCVTL  : SIMDFPWidenTwoVector<0, 0, 0b10111, "fcvtl">;
 def : Pat<(v4f32 (int_aarch64_neon_vcvthf2fp (v4i16 V64:$Rn))),
           (FCVTLv4i16 V64:$Rn)>;
 def : Pat<(v4f32 (int_aarch64_neon_vcvthf2fp (extract_subvector (v8i16 V128:$Rn),
                                                               (i64 4)))),
           (FCVTLv8i16 V128:$Rn)>;
 def : Pat<(v2f64 (fpextend (v2f32 V64:$Rn))), (FCVTLv2i32 V64:$Rn)>;
-def : Pat<(v2f64 (fpextend (v2f32 (extract_subvector (v4f32 V128:$Rn),
-                                                    (i64 2))))),
-          (FCVTLv4i32 V128:$Rn)>;
 
 def : Pat<(v4f32 (fpextend (v4f16 V64:$Rn))), (FCVTLv4i16 V64:$Rn)>;
-def : Pat<(v4f32 (fpextend (v4f16 (extract_subvector (v8f16 V128:$Rn),
-                                                    (i64 4))))),
-          (FCVTLv8i16 V128:$Rn)>;
 
 defm FCVTMS : SIMDTwoVectorFPToInt<0,0,0b11011, "fcvtms",int_aarch64_neon_fcvtms>;
 defm FCVTMU : SIMDTwoVectorFPToInt<1,0,0b11011, "fcvtmu",int_aarch64_neon_fcvtmu>;
 defm FCVTNS : SIMDTwoVectorFPToInt<0,0,0b11010, "fcvtns",int_aarch64_neon_fcvtns>;
 defm FCVTNU : SIMDTwoVectorFPToInt<1,0,0b11010, "fcvtnu",int_aarch64_neon_fcvtnu>;
 defm FCVTN  : SIMDFPNarrowTwoVector<0, 0, 0b10110, "fcvtn">;
 def : Pat<(v4i16 (int_aarch64_neon_vcvtfp2hf (v4f32 V128:$Rn))),
           (FCVTNv4i16 V128:$Rn)>;
 def : Pat<(concat_vectors V64:$Rd,

llvm/test/CodeGen/AArch64/arm64-vcvt_f.ll

 ; GISEL-NEXT:    fcvtl2 v0.2d, v0.4s
 ; GISEL-NEXT:    ret
   %cvt_in = shufflevector <4 x float> %x, <4 x float> undef, <2 x i32> <i32 2, i32 3>
   %vcvt1.i = fpext <2 x float> %cvt_in to <2 x double>
   ret <2 x double> %vcvt1.i
 }
 
 define <2 x double> @test_vcvt_high_v1f64_f32_bitcast(<4 x float> %x) nounwind readnone ssp {
-; GENERIC-LABEL: test_vcvt_high_v1f64_f32_bitcast:
-; GENERIC:       // %bb.0:
-; GENERIC-NEXT:    ext.16b v0, v0, v0, #8
-; GENERIC-NEXT:    fcvtl v0.2d, v0.2s
-; GENERIC-NEXT:    ret
-;
-; FAST-LABEL: test_vcvt_high_v1f64_f32_bitcast:
-; FAST:       // %bb.0:
-; FAST-NEXT:    fcvtl2 v0.2d, v0.4s
-; FAST-NEXT:    ret
+; CHECK-LABEL: test_vcvt_high_v1f64_f32_bitcast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_vcvt_high_v1f64_f32_bitcast:
 ; GISEL:       // %bb.0:
-; GISEL-NEXT:    ext.16b v0, v0, v0, #8
-; GISEL-NEXT:    fcvtl v0.2d, v0.2s
+; GISEL-NEXT:    fcvtl2 v0.2d, v0.4s
 ; GISEL-NEXT:    ret
   %bc1 = bitcast <4 x float> %x to <2 x double>
   %ext = shufflevector <2 x double> %bc1, <2 x double> undef, <1 x i32> <i32 1>
   %bc2 = bitcast <1 x double> %ext to <2 x float>
   %r = fpext <2 x float> %bc2 to <2 x double>
   ret <2 x double> %r
 }
 
 define <2 x double> @test_vcvt_high_v1i64_f32_bitcast(<2 x i64> %x) nounwind readnone ssp {
-; GENERIC-LABEL: test_vcvt_high_v1i64_f32_bitcast:
-; GENERIC:       // %bb.0:
-; GENERIC-NEXT:    ext.16b v0, v0, v0, #8
-; GENERIC-NEXT:    fcvtl v0.2d, v0.2s
-; GENERIC-NEXT:    ret
-;
-; FAST-LABEL: test_vcvt_high_v1i64_f32_bitcast:
-; FAST:       // %bb.0:
-; FAST-NEXT:    ext.16b v0, v0, v0, #8
-; FAST-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; FAST-NEXT:    fcvtl v0.2d, v0.2s
-; FAST-NEXT:    ret
+; CHECK-LABEL: test_vcvt_high_v1i64_f32_bitcast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_vcvt_high_v1i64_f32_bitcast:
 ; GISEL:       // %bb.0:
-; GISEL-NEXT:    ext.16b v0, v0, v0, #8
-; GISEL-NEXT:    fcvtl v0.2d, v0.2s
+; GISEL-NEXT:    fcvtl2 v0.2d, v0.4s
 ; GISEL-NEXT:    ret
   %ext = shufflevector <2 x i64> %x, <2 x i64> undef, <1 x i32> <i32 1>
   %bc2 = bitcast <1 x i64> %ext to <2 x float>
   %r = fpext <2 x float> %bc2 to <2 x double>
   ret <2 x double> %r
 }
 
 define <2 x double> @test_vcvt_high_v2i32_f32_bitcast(<4 x i32> %x) nounwind readnone ssp {
-; GENERIC-LABEL: test_vcvt_high_v2i32_f32_bitcast:
-; GENERIC:       // %bb.0:
-; GENERIC-NEXT:    ext.16b v0, v0, v0, #8
-; GENERIC-NEXT:    fcvtl v0.2d, v0.2s
-; GENERIC-NEXT:    ret
-;
-; FAST-LABEL: test_vcvt_high_v2i32_f32_bitcast:
-; FAST:       // %bb.0:
-; FAST-NEXT:    ext.16b v0, v0, v0, #8
-; FAST-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; FAST-NEXT:    fcvtl v0.2d, v0.2s
-; FAST-NEXT:    ret
+; CHECK-LABEL: test_vcvt_high_v2i32_f32_bitcast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_vcvt_high_v2i32_f32_bitcast:
 ; GISEL:       // %bb.0:
-; GISEL-NEXT:    ext.16b v0, v0, v0, #8
-; GISEL-NEXT:    fcvtl v0.2d, v0.2s
+; GISEL-NEXT:    fcvtl2 v0.2d, v0.4s
 ; GISEL-NEXT:    ret
   %ext = shufflevector <4 x i32> %x, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
   %bc2 = bitcast <2 x i32> %ext to <2 x float>
   %r = fpext <2 x float> %bc2 to <2 x double>
   ret <2 x double> %r
 }
 
 define <2 x double> @test_vcvt_high_v4i16_f32_bitcast(<8 x i16> %x) nounwind readnone ssp {
-; GENERIC-LABEL: test_vcvt_high_v4i16_f32_bitcast:
-; GENERIC:       // %bb.0:
-; GENERIC-NEXT:    ext.16b v0, v0, v0, #8
-; GENERIC-NEXT:    fcvtl v0.2d, v0.2s
-; GENERIC-NEXT:    ret
-;
-; FAST-LABEL: test_vcvt_high_v4i16_f32_bitcast:
-; FAST:       // %bb.0:
-; FAST-NEXT:    ext.16b v0, v0, v0, #8
-; FAST-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; FAST-NEXT:    fcvtl v0.2d, v0.2s
-; FAST-NEXT:    ret
+; CHECK-LABEL: test_vcvt_high_v4i16_f32_bitcast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_vcvt_high_v4i16_f32_bitcast:
 ; GISEL:       // %bb.0:
-; GISEL-NEXT:    ext.16b v0, v0, v0, #8
-; GISEL-NEXT:    fcvtl v0.2d, v0.2s
+; GISEL-NEXT:    fcvtl2 v0.2d, v0.4s
 ; GISEL-NEXT:    ret
   %ext = shufflevector <8 x i16> %x, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %bc2 = bitcast <4 x i16> %ext to <2 x float>
   %r = fpext <2 x float> %bc2 to <2 x double>
   ret <2 x double> %r
 }
 
 define <2 x double> @test_vcvt_high_v8i8_f32_bitcast(<16 x i8> %x) nounwind readnone ssp {
-; GENERIC-LABEL: test_vcvt_high_v8i8_f32_bitcast:
-; GENERIC:       // %bb.0:
-; GENERIC-NEXT:    ext.16b v0, v0, v0, #8
-; GENERIC-NEXT:    fcvtl v0.2d, v0.2s
-; GENERIC-NEXT:    ret
-;
-; FAST-LABEL: test_vcvt_high_v8i8_f32_bitcast:
-; FAST:       // %bb.0:
-; FAST-NEXT:    ext.16b v0, v0, v0, #8
-; FAST-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; FAST-NEXT:    fcvtl v0.2d, v0.2s
-; FAST-NEXT:    ret
+; CHECK-LABEL: test_vcvt_high_v8i8_f32_bitcast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtl2 v0.2d, v0.4s
+; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_vcvt_high_v8i8_f32_bitcast:
 ; GISEL:       // %bb.0:
-; GISEL-NEXT:    ext.16b v0, v0, v0, #8
-; GISEL-NEXT:    fcvtl v0.2d, v0.2s
+; GISEL-NEXT:    fcvtl2 v0.2d, v0.4s
 ; GISEL-NEXT:    ret
   %ext = shufflevector <16 x i8> %x, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %bc2 = bitcast <8 x i8> %ext to <2 x float>
   %r = fpext <2 x float> %bc2 to <2 x double>
   ret <2 x double> %r
 }
 
 define <4 x float> @test_vcvt_high_v1i64_f16_bitcast(<2 x i64> %x) nounwind readnone ssp {
-; GENERIC-LABEL: test_vcvt_high_v1i64_f16_bitcast:
-; GENERIC:       // %bb.0:
-; GENERIC-NEXT:    ext.16b v0, v0, v0, #8
-; GENERIC-NEXT:    fcvtl v0.4s, v0.4h
-; GENERIC-NEXT:    ret
-;
-; FAST-LABEL: test_vcvt_high_v1i64_f16_bitcast:
-; FAST:       // %bb.0:
-; FAST-NEXT:    ext.16b v0, v0, v0, #8
-; FAST-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; FAST-NEXT:    fcvtl v0.4s, v0.4h
-; FAST-NEXT:    ret
+; CHECK-LABEL: test_vcvt_high_v1i64_f16_bitcast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_vcvt_high_v1i64_f16_bitcast:
 ; GISEL:       // %bb.0:
-; GISEL-NEXT:    ext.16b v0, v0, v0, #8
-; GISEL-NEXT:    fcvtl v0.4s, v0.4h
+; GISEL-NEXT:    fcvtl2 v0.4s, v0.8h
 ; GISEL-NEXT:    ret
   %ext = shufflevector <2 x i64> %x, <2 x i64> undef, <1 x i32> <i32 1>
   %bc2 = bitcast <1 x i64> %ext to <4 x half>
   %r = fpext <4 x half> %bc2 to <4 x float>
   ret <4 x float> %r
 }
 
 define <4 x float> @test_vcvt_high_v2i32_f16_bitcast(<4 x i32> %x) nounwind readnone ssp {
-; GENERIC-LABEL: test_vcvt_high_v2i32_f16_bitcast:
-; GENERIC:       // %bb.0:
-; GENERIC-NEXT:    ext.16b v0, v0, v0, #8
-; GENERIC-NEXT:    fcvtl v0.4s, v0.4h
-; GENERIC-NEXT:    ret
-;
-; FAST-LABEL: test_vcvt_high_v2i32_f16_bitcast:
-; FAST:       // %bb.0:
-; FAST-NEXT:    ext.16b v0, v0, v0, #8
-; FAST-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; FAST-NEXT:    fcvtl v0.4s, v0.4h
-; FAST-NEXT:    ret
+; CHECK-LABEL: test_vcvt_high_v2i32_f16_bitcast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_vcvt_high_v2i32_f16_bitcast:
 ; GISEL:       // %bb.0:
-; GISEL-NEXT:    ext.16b v0, v0, v0, #8
-; GISEL-NEXT:    fcvtl v0.4s, v0.4h
+; GISEL-NEXT:    fcvtl2 v0.4s, v0.8h
 ; GISEL-NEXT:    ret
   %ext = shufflevector <4 x i32> %x, <4 x i32> undef, <2 x i32> <i32 2, i32 3>
   %bc2 = bitcast <2 x i32> %ext to <4 x half>
   %r = fpext <4 x half> %bc2 to <4 x float>
   ret <4 x float> %r
 }
 
 define <4 x float> @test_vcvt_high_v4i16_f16_bitcast(<8 x i16> %x) nounwind readnone ssp {
-; GENERIC-LABEL: test_vcvt_high_v4i16_f16_bitcast:
-; GENERIC:       // %bb.0:
-; GENERIC-NEXT:    ext.16b v0, v0, v0, #8
-; GENERIC-NEXT:    fcvtl v0.4s, v0.4h
-; GENERIC-NEXT:    ret
-;
-; FAST-LABEL: test_vcvt_high_v4i16_f16_bitcast:
-; FAST:       // %bb.0:
-; FAST-NEXT:    ext.16b v0, v0, v0, #8
-; FAST-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; FAST-NEXT:    fcvtl v0.4s, v0.4h
-; FAST-NEXT:    ret
+; CHECK-LABEL: test_vcvt_high_v4i16_f16_bitcast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_vcvt_high_v4i16_f16_bitcast:
 ; GISEL:       // %bb.0:
-; GISEL-NEXT:    ext.16b v0, v0, v0, #8
-; GISEL-NEXT:    fcvtl v0.4s, v0.4h
+; GISEL-NEXT:    fcvtl2 v0.4s, v0.8h
 ; GISEL-NEXT:    ret
   %ext = shufflevector <8 x i16> %x, <8 x i16> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
   %bc2 = bitcast <4 x i16> %ext to <4 x half>
   %r = fpext <4 x half> %bc2 to <4 x float>
   ret <4 x float> %r
 }
 
 define <4 x float> @test_vcvt_high_v8i8_f16_bitcast(<16 x i8> %x) nounwind readnone ssp {
-; GENERIC-LABEL: test_vcvt_high_v8i8_f16_bitcast:
-; GENERIC:       // %bb.0:
-; GENERIC-NEXT:    ext.16b v0, v0, v0, #8
-; GENERIC-NEXT:    fcvtl v0.4s, v0.4h
-; GENERIC-NEXT:    ret
-;
-; FAST-LABEL: test_vcvt_high_v8i8_f16_bitcast:
-; FAST:       // %bb.0:
-; FAST-NEXT:    ext.16b v0, v0, v0, #8
-; FAST-NEXT:    // kill: def $d0 killed $d0 killed $q0
-; FAST-NEXT:    fcvtl v0.4s, v0.4h
-; FAST-NEXT:    ret
+; CHECK-LABEL: test_vcvt_high_v8i8_f16_bitcast:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fcvtl2 v0.4s, v0.8h
+; CHECK-NEXT:    ret
 ;
 ; GISEL-LABEL: test_vcvt_high_v8i8_f16_bitcast:
 ; GISEL:       // %bb.0:
-; GISEL-NEXT:    ext.16b v0, v0, v0, #8
-; GISEL-NEXT:    fcvtl v0.4s, v0.4h
+; GISEL-NEXT:    fcvtl2 v0.4s, v0.8h
 ; GISEL-NEXT:    ret
   %ext = shufflevector <16 x i8> %x, <16 x i8> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
   %bc2 = bitcast <8 x i8> %ext to <4 x half>
   %r = fpext <4 x half> %bc2 to <4 x float>
   ret <4 x float> %r
 }
 
 ; FALLBACK-NOT: remark{{.*}}G_FPEXT{{.*}}(in function: test_vcvt_f32_f64)