[AArch64][SVE] Lower fixed length EXTRACT_VECTOR_ELT

[paulwalker-arm]

llvm/test/CodeGen/AArch64/sve-fixed-length-extract-vector-elt.ll

29

The tests that take 64/128 bit vectors should pass the vector parameter directly rather than by reference.

56

Can the tests use a variable index instead? I ask because for all these tests 2 fits within the NEON range and thus when some future work lands none of these tests will emit SVE instructions.

FYI: This is also generally true as in future I'd even the variable index variants to just emit a scalar load, but this will change once all the tests are updated to use pass by value. This is not something you need to worry about though so updating them to use a variable index is sufficient.

[joechrisellis]

Address review comments.

• @paulwalker-arm:
  • tests that take 64/128 bit vectors now pass the vector parameter directly rather than by reference.
  • as discussed offline, use a higher index for tests.

[paulwalker-arm]

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

10002

Are there any uses of this? If not then you may as well preserve the original EVT VT = Op.getOperand(0).getValueType();

llvm/test/CodeGen/AArch64/sve-fixed-length-extract-vector-elt.ll

94

This falls under my previous "The tests that take 64/128 bit vectors should pass the vector parameter directly rather than by reference." comment.

There should also be a <2 x float> test.

151

You're missing tests for <1 x double> and <2 x double>.

[joechrisellis]

Address comments.

• @paulwalker-arm:
  • do not use pass-by-reference for 64-/128-bit tests.
  • add tests for <2 x float>, <1 x double>, and <2 x double>.

llvm/lib/Target/AArch64/AArch64ISelLowering.h

private:
   SDValue LowerPredReductionToSVE(SDValue ScalarOp, SelectionDAG &DAG) const;
   SDValue LowerReductionToSVE(unsigned Opcode, SDValue ScalarOp,
                               SelectionDAG &DAG) const;
   SDValue LowerFixedLengthVectorSelectToSVE(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFixedLengthVectorSetccToSVE(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFixedLengthVectorStoreToSVE(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerFixedLengthVectorTruncateToSVE(SDValue Op,
                                               SelectionDAG &DAG) const;
+  SDValue LowerFixedLengthExtractVectorElt(SDValue Op, SelectionDAG &DAG) const;
 
   SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
                         SmallVectorImpl<SDNode *> &Created) const override;
   SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
                           int &ExtraSteps, bool &UseOneConst,
                           bool Reciprocal) const override;
   SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
                            int &ExtraSteps) const override;

llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
   setOperationAction(ISD::ANY_EXTEND, VT, Custom);
   setOperationAction(ISD::BITREVERSE, VT, Custom);
   setOperationAction(ISD::BSWAP, VT, Custom);
   setOperationAction(ISD::CTLZ, VT, Custom);
   setOperationAction(ISD::CTPOP, VT, Custom);
   setOperationAction(ISD::CTTZ, VT, Custom);
   setOperationAction(ISD::FABS, VT, Custom);
   setOperationAction(ISD::FADD, VT, Custom);
+  setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
   setOperationAction(ISD::FCEIL, VT, Custom);
   setOperationAction(ISD::FDIV, VT, Custom);
   setOperationAction(ISD::FFLOOR, VT, Custom);
   setOperationAction(ISD::FMA, VT, Custom);
   setOperationAction(ISD::FMAXIMUM, VT, Custom);
   setOperationAction(ISD::FMAXNUM, VT, Custom);
   setOperationAction(ISD::FMINIMUM, VT, Custom);
   setOperationAction(ISD::FMINNUM, VT, Custom);
SDValue AArch64TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
   return NarrowVector(Node, DAG);
 }
 
 SDValue
 AArch64TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
                                                SelectionDAG &DAG) const {
   assert(Op.getOpcode() == ISD::EXTRACT_VECTOR_ELT && "Unknown opcode!");
 
-  // Check for non-constant or out of range lane.
   EVT VT = Op.getOperand(0).getValueType();
+  if (useSVEForFixedLengthVectorVT(VT))
+    return LowerFixedLengthExtractVectorElt(Op, DAG);
+
+  // Check for non-constant or out of range lane.
   ConstantSDNode *CI = dyn_cast<ConstantSDNode>(Op.getOperand(1));
   if (!CI || CI->getZExtValue() >= VT.getVectorNumElements())
     return SDValue();
 
 
   // Insertion/extraction are legal for V128 types.
   if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 ||
       VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64 ||
case MVT::nxv8i16:
     Val = DAG.getNode(AArch64ISD::UZP1, DL, MVT::nxv16i8, Val, Val);
     assert(VT.getVectorElementType() == MVT::i8 && "Unexpected element type!");
     break;
   }
 
   return convertFromScalableVector(DAG, VT, Val);
 }
 
+SDValue AArch64TargetLowering::LowerFixedLengthExtractVectorElt(
+    SDValue Op, SelectionDAG &DAG) const {
+  EVT VT = Op.getValueType();
+  EVT InVT = Op.getOperand(0).getValueType();
+  assert(InVT.isFixedLengthVector() && "Expected fixed length vector type!");
+
+  SDLoc DL(Op);
+  EVT ContainerVT = getContainerForFixedLengthVector(DAG, InVT);
+  SDValue Op0 = convertToScalableVector(DAG, ContainerVT, Op->getOperand(0));
+
+  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op0, Op.getOperand(1));
+}
+
 // Convert vector operation 'Op' to an equivalent predicated operation whereby
 // the original operation's type is used to construct a suitable predicate.
 // NOTE: The results for inactive lanes are undefined.
 SDValue AArch64TargetLowering::LowerToPredicatedOp(SDValue Op,
                                                    SelectionDAG &DAG,
                                                    unsigned NewOp,
                                                    bool OverrideNEON) const {
   EVT VT = Op.getValueType();

llvm/test/CodeGen/AArch64/sve-fixed-length-extract-vector-elt.ll

+; RUN: llc -aarch64-sve-vector-bits-min=128  -asm-verbose=0 < %s | FileCheck %s -check-prefix=NO_SVE
+; RUN: llc -aarch64-sve-vector-bits-min=256  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256
+; RUN: llc -aarch64-sve-vector-bits-min=384  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256
+; RUN: llc -aarch64-sve-vector-bits-min=512  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=640  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=768  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=896  -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256,VBITS_GE_512
+; RUN: llc -aarch64-sve-vector-bits-min=1024 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1152 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1280 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1408 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1536 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1664 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1792 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=1920 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256,VBITS_GE_512,VBITS_GE_1024
+; RUN: llc -aarch64-sve-vector-bits-min=2048 -asm-verbose=0 < %s | FileCheck %s -check-prefixes=CHECK,VBITS_GE_256,VBITS_GE_512,VBITS_GE_1024,VBITS_GE_2048
+
+target triple = "aarch64-unknown-linux-gnu"
+
+; Don't use SVE when its registers are no bigger than NEON.
+; NO_SVE-NOT: ptrue
+
+;
+; extractelement
+;
+
+; Don't use SVE for 64-bit vectors.
+define half @extractelement_v4f16(<4 x half> %op1) #0 {
+; CHECK-LABEL: extractelement_v4f16:
+; CHECK:         mov h0, v0.h[3]
+; CHECK-NEXT:    ret
+    %r = extractelement <4 x half> %op1, i64 3
+    ret half %r
+}
+
+; Don't use SVE for 128-bit vectors.
+define half @extractelement_v8f16(<8 x half> %op1) #0 {
+; CHECK-LABEL: extractelement_v8f16:
+; CHECK:         mov h0, v0.h[7]
+; CHECK-NEXT:    ret
+    %r = extractelement <8 x half> %op1, i64 7
+    ret half %r
+}
+
+define half @extractelement_v16f16(<16 x half>* %a) #0 {
+; CHECK-LABEL: extractelement_v16f16:
+; VBITS_GE_256:         ptrue   p0.h, vl16
+; VBITS_GE_256-NEXT:    ld1h    { z0.h }, p0/z, [x0]
+; VBITS_GE_256-NEXT:    mov z0.h, z0.h[15]
+; VBITS_GE_256-NEXT:    ret
+    %op1 = load <16 x half>, <16 x half>* %a
+    %r = extractelement <16 x half> %op1, i64 15
+    ret half %r
+}
+
+define half @extractelement_v32f16(<32 x half>* %a) #0 {
+; CHECK-LABEL: extractelement_v32f16:
+; VBITS_GE_512:         ptrue   p0.h, vl32
+; VBITS_GE_512-NEXT:    ld1h    { z0.h }, p0/z, [x0]
+; VBITS_GE_512-NEXT:    mov z0.h, z0.h[31]
+; VBITS_GE_512-NEXT:    ret
+    %op1 = load <32 x half>, <32 x half>* %a
+    %r = extractelement <32 x half> %op1, i64 31
+    ret half %r
+}
+
+define half @extractelement_v64f16(<64 x half>* %a) #0 {
+; CHECK-LABEL: extractelement_v64f16:
+; VBITS_GE_1024:         ptrue   p0.h, vl64
+; VBITS_GE_1024-NEXT:    ld1h    { z0.h }, p0/z, [x0]
+; VBITS_GE_1024-NEXT:    mov w8, #63
+; VBITS_GE_1024-NEXT:    whilels p0.h, xzr, x8
+; VBITS_GE_1024-NEXT:    lastb   h0, p0, z0.h
+; VBITS_GE_1024-NEXT:    ret
+    %op1 = load <64 x half>, <64 x half>* %a
+    %r = extractelement <64 x half> %op1, i64 63
+    ret half %r
+}
+
+define half @extractelement_v128f16(<128 x half>* %a) #0 {
+; CHECK-LABEL: extractelement_v128f16:
+; VBITS_GE_2048:      ptrue   p0.h, vl128
+; VBITS_GE_2048-NEXT: ld1h    { z0.h }, p0/z, [x0]
+; VBITS_GE_2048-NEXT: mov w8, #127
+; VBITS_GE_2048-NEXT: whilels p0.h, xzr, x8
+; VBITS_GE_2048-NEXT: lastb   h0, p0, z0.h
+; VBITS_GE_2048-NEXT: ret
+    %op1 = load <128 x half>, <128 x half>* %a
+    %r = extractelement <128 x half> %op1, i64 127
+    ret half %r
+}
+
+; Don't use SVE for 64-bit vectors.
+define float @extractelement_v2f32(<2 x float> %op1) #0 {
+; CHECK-LABEL: extractelement_v2f32:
+; CHECK:         mov s0, v0.s[1]
+; CHECK-NEXT:    ret
+    %r = extractelement <2 x float> %op1, i64 1
+    ret float %r
+}
+
+; Don't use SVE for 128-bit vectors.
+define float @extractelement_v4f32(<4 x float> %op1) #0 {
+; CHECK-LABEL: extractelement_v4f32:
+; CHECK:         mov s0, v0.s[3]
+; CHECK-NEXT:    ret
+    %r = extractelement <4 x float> %op1, i64 3
+    ret float %r
+}
+
+define float @extractelement_v8f32(<8 x float>* %a) #0 {
+; CHECK-LABEL: extractelement_v8f32:
+; VBITS_GE_256:         ptrue   p0.s, vl8
+; VBITS_GE_256-NEXT:    ld1w    { z0.s }, p0/z, [x0]
+; VBITS_GE_256-NEXT:    mov z0.s, z0.s[7]
+; VBITS_GE_256-NEXT:    ret
+    %op1 = load <8 x float>, <8 x float>* %a
+    %r = extractelement <8 x float> %op1, i64 7
+    ret float %r
+}
+
+define float @extractelement_v16f32(<16 x float>* %a) #0 {
+; CHECK-LABEL: extractelement_v16f32:
+; VBITS_GE_512:         ptrue   p0.s, vl16
+; VBITS_GE_512-NEXT:    ld1w    { z0.s }, p0/z, [x0]
+; VBITS_GE_512-NEXT:    mov z0.s, z0.s[15]
+; VBITS_GE_512-NEXT:    ret
+    %op1 = load <16 x float>, <16 x float>* %a
+    %r = extractelement <16 x float> %op1, i64 15
+    ret float %r
+}
+
+define float @extractelement_v32f32(<32 x float>* %a) #0 {
+; CHECK-LABEL: extractelement_v32f32:
+; VBITS_GE_1024:        ptrue   p0.s, vl32
+; VBITS_GE_1024-NEXT:   ld1w    { z0.s }, p0/z, [x0]
+; VBITS_GE_1024-NEXT:   mov w8, #31
+; VBITS_GE_1024-NEXT:   whilels p0.s, xzr, x8
+; VBITS_GE_1024-NEXT:   lastb   s0, p0, z0.s
+; VBITS_GE_1024-NEXT:   ret
+    %op1 = load <32 x float>, <32 x float>* %a
+    %r = extractelement <32 x float> %op1, i64 31
+    ret float %r
+}
+
+define float @extractelement_v64f32(<64 x float>* %a) #0 {
+; CHECK-LABEL: extractelement_v64f32:
+; VBITS_GE_2048:        ptrue   p0.s, vl64
+; VBITS_GE_2048-NEXT:   ld1w    { z0.s }, p0/z, [x0]
+; VBITS_GE_2048-NEXT:   mov w8, #63
+; VBITS_GE_2048-NEXT:   whilels p0.s, xzr, x8
+; VBITS_GE_2048-NEXT:   lastb   s0, p0, z0.s
+; VBITS_GE_2048-NEXT:   ret
+    %op1 = load <64 x float>, <64 x float>* %a
+    %r = extractelement <64 x float> %op1, i64 63
+    ret float %r
+}
+
+; Don't use SVE for 64-bit vectors.
+define double @extractelement_v1f64(<1 x double> %op1) #0 {
+; CHECK-LABEL: extractelement_v1f64:
+; CHECK:         ret
+    %r = extractelement <1 x double> %op1, i64 0
+    ret double %r
+}
+
+; Don't use SVE for 128-bit vectors.
+define double @extractelement_v2f64(<2 x double> %op1) #0 {
+; CHECK-LABEL: extractelement_v2f64:
+; CHECK:         mov d0, v0.d[1]
+; CHECK-NEXT:    ret
+    %r = extractelement <2 x double> %op1, i64 1
+    ret double %r
+}
+
+define double @extractelement_v4f64(<4 x double>* %a) #0 {
+; CHECK-LABEL: extractelement_v4f64:
+; VBITS_GE_256:         ptrue   p0.d, vl4
+; VBITS_GE_256-NEXT:    ld1d    { z0.d }, p0/z, [x0]
+; VBITS_GE_256-NEXT:    mov z0.d, z0.d[3]
+; VBITS_GE_256-NEXT:    ret
+    %op1 = load <4 x double>, <4 x double>* %a
+    %r = extractelement <4 x double> %op1, i64 3
+    ret double %r
+}
+
+define double @extractelement_v8f64(<8 x double>* %a) #0 {
+; CHECK-LABEL: extractelement_v8f64:
+; VBITS_GE_512:         ptrue   p0.d, vl8
+; VBITS_GE_512-NEXT:    ld1d    { z0.d }, p0/z, [x0]
+; VBITS_GE_512-NEXT:    mov z0.d, z0.d[7]
+; VBITS_GE_512-NEXT:    ret
+    %op1 = load <8 x double>, <8 x double>* %a
+    %r = extractelement <8 x double> %op1, i64 7
+    ret double %r
+}
+
+define double @extractelement_v16f64(<16 x double>* %a) #0 {
+; CHECK-LABEL: extractelement_v16f64:
+; VBITS_GE_1024:         ptrue   p0.d, vl16
+; VBITS_GE_1024-NEXT:    ld1d    { z0.d }, p0/z, [x0]
+; VBITS_GE_1024-NEXT:    mov w8, #15
+; VBITS_GE_1024-NEXT:    whilels p0.d, xzr, x8
+; VBITS_GE_1024-NEXT:    lastb   d0, p0, z0.d
+; VBITS_GE_1024-NEXT:    ret
+    %op1 = load <16 x double>, <16 x double>* %a
+    %r = extractelement <16 x double> %op1, i64 15
+    ret double %r
+}
+
+define double @extractelement_v32f64(<32 x double>* %a) #0 {
+; CHECK-LABEL: extractelement_v32f64:
+; VBITS_GE_2048:         ptrue   p0.d, vl32
+; VBITS_GE_2048-NEXT:    ld1d    { z0.d }, p0/z, [x0]
+; VBITS_GE_2048-NEXT:    mov w8, #31
+; VBITS_GE_2048-NEXT:    whilels p0.d, xzr, x8
+; VBITS_GE_2048-NEXT:    lastb   d0, p0, z0.d
+; VBITS_GE_2048-NEXT:    ret
+    %op1 = load <32 x double>, <32 x double>* %a
+    %r = extractelement <32 x double> %op1, i64 31
+    ret double %r
+}
+
+attributes #0 = { "target-features"="+sve" }

llvm/test/CodeGen/AArch64/sve-fixed-length-fp-converts.ll

 ; when lowering it implemented.
 
 ;
 ; fptrunc f32 -> f16
 ;
 
 define <8 x half> @fptrunc_v8f32_v8f16(<8 x float>* %in) #0 {
 ; CHECK-LABEL: fptrunc_v8f32_v8f16:
-; CHECK-COUNT-8: fcvt h{{[0-9]}}, s{{[0-9]}}
+; CHECK-COUNT-8: fcvt h{{[0-9]+}}, s{{[0-9]+}}
 ; CHECK-NOT: fcvt
 ; CHECK: ret
   %a = load <8 x float>, <8 x float>* %in
   %b = fptrunc <8 x float> %a to <8 x half>
   ret <8 x half> %b
 }
 
 define void @fptrunc_v16f32_v16f16(<16 x float>* %in, <16 x half>* %out) #0 {
 ; CHECK-LABEL: fptrunc_v16f32_v16f16:
-; CHECK-COUNT-16: fcvt h{{[0-9]}}, s{{[0-9]}}
+; CHECK-COUNT-16: fcvt h{{[0-9]+}}, s{{[0-9]+}}
 ; CHECK-NOT: fcvt
 ; CHECK: ret
   %a = load <16 x float>, <16 x float>* %in
   %b = fptrunc <16 x float> %a to <16 x half>
   store <16 x half> %b, <16 x half>* %out
   ret void
 }
 
 define void @fptrunc_v32f32_v32f16(<32 x float>* %in, <32 x half>* %out) #0 {
 ; CHECK-LABEL: fptrunc_v32f32_v32f16:
-; CHECK-COUNT-32: fcvt h{{[0-9]}}, s{{[0-9]}}
+; CHECK-COUNT-32: fcvt h{{[0-9]+}}, s{{[0-9]+}}
 ; CHECK-NOT: fcvt
 ; CHECK: ret
   %a = load <32 x float>, <32 x float>* %in
   %b = fptrunc <32 x float> %a to <32 x half>
   store <32 x half> %b, <32 x half>* %out
   ret void
 }
 
 define void @fptrunc_v64f32_v64f16(<64 x float>* %in, <64 x half>* %out) #0 {
 ; CHECK-LABEL: fptrunc_v64f32_v64f16:
-; CHECK-COUNT-64: fcvt h{{[0-9]}}, s{{[0-9]}}
+; CHECK-COUNT-64: fcvt h{{[0-9]+}}, s{{[0-9]+}}
 ; CHECK-NOT: fcvt
 ; CHECK: ret
   %a = load <64 x float>, <64 x float>* %in
   %b = fptrunc <64 x float> %a to <64 x half>
   store <64 x half> %b, <64 x half>* %out
   ret void
 }
 
 ;
 ; fptrunc f64 -> f16
 ;
 
 define <4 x half> @fptrunc_v4f64_v4f16(<4 x double>* %in) #0 {
 ; CHECK-LABEL: fptrunc_v4f64_v4f16:
-; CHECK-COUNT-4: fcvt h{{[0-9]}}, d{{[0-9]}}
+; CHECK-COUNT-4: fcvt h{{[0-9]+}}, d{{[0-9]+}}
 ; CHECK-NOT: fcvt
 ; CHECK: ret
   %a = load <4 x double>, <4 x double>* %in
   %b = fptrunc <4 x double> %a to <4 x half>
   ret <4 x half> %b
 }
 
 define <8 x half> @fptrunc_v8f64_v8f16(<8 x double>* %in) #0 {
 ; CHECK-LABEL: fptrunc_v8f64_v8f16:
-; CHECK-COUNT-8: fcvt h{{[0-9]}}, d{{[0-9]}}
+; CHECK-COUNT-8: fcvt h{{[0-9]+}}, d{{[0-9]+}}
 ; CHECK-NOT: fcvt
 ; CHECK: ret
   %a = load <8 x double>, <8 x double>* %in
   %b = fptrunc <8 x double> %a to <8 x half>
   ret <8 x half> %b
 }
 
 define void @fptrunc_v16f64_v16f16(<16 x double>* %in, <16 x half>* %out) #0 {
 ; CHECK-LABEL: fptrunc_v16f64_v16f16:
-; CHECK-COUNT-16: fcvt h{{[0-9]}}, d{{[0-9]}}
+; CHECK-COUNT-16: fcvt h{{[0-9]+}}, d{{[0-9]+}}
 ; CHECK-NOT: fcvt
 ; CHECK: ret
   %a = load <16 x double>, <16 x double>* %in
   %b = fptrunc <16 x double> %a to <16 x half>
   store <16 x half> %b, <16 x half>* %out
   ret void
 }
 
 define void @fptrunc_v32f64_v32f16(<32 x double>* %in, <32 x half>* %out) #0 {
 ; CHECK-LABEL: fptrunc_v32f64_v32f16:
-; CHECK-COUNT-32: fcvt h{{[0-9]}}, d{{[0-9]}}
+; CHECK-COUNT-32: fcvt h{{[0-9]+}}, d{{[0-9]+}}
 ; CHECK-NOT: fcvt
 ; CHECK: ret
   %a = load <32 x double>, <32 x double>* %in
   %b = fptrunc <32 x double> %a to <32 x half>
   store <32 x half> %b, <32 x half>* %out
   ret void
 }
 
 ;
 ; fptrunc f64 -> f32
 ;
 
 define <4 x float> @fptrunc_v4f64_v4f32(<4 x double>* %in) #0 {
 ; CHECK-LABEL: fptrunc_v4f64_v4f32:
-; CHECK-COUNT-4: fcvt s{{[0-9]}}, d{{[0-9]}}
+; CHECK-COUNT-4: fcvt s{{[0-9]+}}, d{{[0-9]+}}
 ; CHECK-NOT: fcvt
 ; CHECK: ret
   %a = load <4 x double>, <4 x double>* %in
   %b = fptrunc <4 x double> %a to <4 x float>
   ret <4 x float> %b
 }
 
 define void @fptrunc_v8f64_v8f32(<8 x double>* %in, <8 x float>* %out) #0 {
 ; CHECK-LABEL: fptrunc_v8f64_v8f32:
-; CHECK-COUNT-8: fcvt s{{[0-9]}}, d{{[0-9]}}
+; CHECK-COUNT-8: fcvt s{{[0-9]+}}, d{{[0-9]+}}
 ; CHECK-NOT: fcvt
 ; CHECK: ret
   %a = load <8 x double>, <8 x double>* %in
   %b = fptrunc <8 x double> %a to <8 x float>
   store <8 x float> %b, <8 x float>* %out
   ret void
 }
 
 define void @fptrunc_v16f64_v16f32(<16 x double>* %in, <16 x float>* %out) #0 {
 ; CHECK-LABEL: fptrunc_v16f64_v16f32:
-; CHECK-COUNT-16: fcvt s{{[0-9]}}, d{{[0-9]}}
+; CHECK-COUNT-16: fcvt s{{[0-9]+}}, d{{[0-9]+}}
 ; CHECK-NOT: fcvt
 ; CHECK: ret
   %a = load <16 x double>, <16 x double>* %in
   %b = fptrunc <16 x double> %a to <16 x float>
   store <16 x float> %b, <16 x float>* %out
   ret void
 }
 
 define void @fptrunc_v32f64_v32f32(<32 x double>* %in, <32 x float>* %out) #0 {
 ; CHECK-LABEL: fptrunc_v32f64_v32f32:
-; CHECK-COUNT-32: fcvt s{{[0-9]}}, d{{[0-9]}}
+; CHECK-COUNT-32: fcvt s{{[0-9]+}}, d{{[0-9]+}}
 ; CHECK-NOT: fcvt
 ; CHECK: ret
   %a = load <32 x double>, <32 x double>* %in
   %b = fptrunc <32 x double> %a to <32 x float>
   store <32 x float> %b, <32 x float>* %out
   ret void
 }
 
 attributes #0 = { nounwind "target-features"="+sve" }