[SVE][CodeGen] Legalisation of EXTRACT_VECTOR_ELT for scalable vectors

[kmclaughlin]

Summary

This patch changes SplitVecOp_EXTRACT_VECTOR_ELT to work correctly
for scalable vectors and also fixes an a bug in DAGCombiner where
the scalable property is dropped in visitTRUNCATE when attempting
to fold an extract + a truncate.

[efriedma]

LGTM

llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp

SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
   // creates this pattern) and before operation legalization after which
   // we need to be more careful about the vector instructions that we generate.
   if (N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
       LegalTypes && !LegalOperations && N0->hasOneUse() && VT != MVT::i1) {
     EVT VecTy = N0.getOperand(0).getValueType();
     EVT ExTy = N0.getValueType();
     EVT TrTy = N->getValueType(0);
 
-    unsigned NumElem = VecTy.getVectorNumElements();
+    auto EltCnt = VecTy.getVectorElementCount();
     unsigned SizeRatio = ExTy.getSizeInBits()/TrTy.getSizeInBits();
+    auto NewEltCnt = EltCnt * SizeRatio;
 
-    EVT NVT = EVT::getVectorVT(*DAG.getContext(), TrTy, SizeRatio * NumElem);
+    EVT NVT = EVT::getVectorVT(*DAG.getContext(), TrTy, NewEltCnt);
     assert(NVT.getSizeInBits() == VecTy.getSizeInBits() && "Invalid Size");
 
     SDValue EltNo = N0->getOperand(1);
     if (isa<ConstantSDNode>(EltNo) && isTypeLegal(NVT)) {
       int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
       int Index = isLE ? (Elt*SizeRatio) : (Elt*SizeRatio + (SizeRatio-1));
 
       SDLoc DL(N);

llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp

SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
   EVT VecVT = Vec.getValueType();
 
   if (isa<ConstantSDNode>(Idx)) {
     uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
 
     SDValue Lo, Hi;
     GetSplitVector(Vec, Lo, Hi);
 
-    uint64_t LoElts = Lo.getValueType().getVectorNumElements();
+    uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
 
     if (IdxVal < LoElts)
       return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
+    else if (!Vec.getValueType().isScalableVector())
       return SDValue(DAG.UpdateNodeOperands(N, Hi,
                                     DAG.getConstant(IdxVal - LoElts, SDLoc(N),
                                                     Idx.getValueType())), 0);
   }
 
   // See if the target wants to custom expand this node.
   if (CustomLowerNode(N, N->getValueType(0), true))
     return SDValue();
 
   // Make the vector elements byte-addressable if they aren't already.
   SDLoc dl(N);
   EVT EltVT = VecVT.getVectorElementType();
   if (VecVT.getScalarSizeInBits() < 8) {
     EltVT = MVT::i8;
     VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
-                             VecVT.getVectorNumElements());
+                             VecVT.getVectorElementCount());
     Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
   }
 
   // Store the vector to the stack.
   // In cases where the vector is illegal it will be broken down into parts
   // and stored in parts - we should use the alignment for the smallest part.
   Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
   SDValue StackPtr =
if (N->getValueType(0).bitsLT(EltVT)) {
     SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr,
       MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
     return DAG.getZExtOrTrunc(Load, dl, N->getValueType(0));
   }
 
   return DAG.getExtLoad(
       ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
       MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT,
-      commonAlignment(SmallestAlign, EltVT.getSizeInBits() / 8));
+      commonAlignment(SmallestAlign, EltVT.getSizeInBits().getFixedSize() / 8));
 }
 
 SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
   SDValue Lo, Hi;
 
   // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
   // splitting the result has the same effect as splitting the input operand.
   SplitVecRes_ExtVecInRegOp(N, Lo, Hi);

llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll

+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s
+
+; EXTRACT VECTOR ELT
+
+define i32 @promote_extract_2i32_idx(<vscale x 2 x i32> %a, i32 %idx) {
+; CHECK-LABEL: promote_extract_2i32_idx:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sxtw x8, w0
+; CHECK-NEXT:    whilels p0.d, xzr, x8
+; CHECK-NEXT:    lastb x0, p0, z0.d
+; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
+; CHECK-NEXT:    ret
+  %ext = extractelement <vscale x 2 x i32> %a, i32 %idx
+  ret i32 %ext
+}
+
+define i8 @split_extract_32i8_idx(<vscale x 32 x i8> %a, i32 %idx) {
+; CHECK-LABEL: split_extract_32i8_idx:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-2
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    rdvl x10, #2
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sxtw x9, w0
+; CHECK-NEXT:    sub x10, x10, #1 // =1
+; CHECK-NEXT:    ptrue p0.b
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    cmp x9, x10
+; CHECK-NEXT:    st1b { z1.b }, p0, [x8, #1, mul vl]
+; CHECK-NEXT:    st1b { z0.b }, p0, [sp]
+; CHECK-NEXT:    csel x9, x9, x10, lo
+; CHECK-NEXT:    ldrb w0, [x8, x9]
+; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %ext = extractelement <vscale x 32 x i8> %a, i32 %idx
+  ret i8 %ext
+}
+
+define i16 @split_extract_16i16_idx(<vscale x 16 x i16> %a, i32 %idx) {
+; CHECK-LABEL: split_extract_16i16_idx:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-2
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    rdvl x10, #1
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sxtw x9, w0
+; CHECK-NEXT:    sub x10, x10, #1 // =1
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    cmp x9, x10
+; CHECK-NEXT:    st1h { z1.h }, p0, [x8, #1, mul vl]
+; CHECK-NEXT:    st1h { z0.h }, p0, [sp]
+; CHECK-NEXT:    csel x9, x9, x10, lo
+; CHECK-NEXT:    ldrh w0, [x8, x9, lsl #1]
+; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %ext = extractelement <vscale x 16 x i16> %a, i32 %idx
+  ret i16 %ext
+}
+
+define i32 @split_extract_8i32_idx(<vscale x 8 x i32> %a, i32 %idx) {
+; CHECK-LABEL: split_extract_8i32_idx:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-2
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    cnth x10
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sxtw x9, w0
+; CHECK-NEXT:    sub x10, x10, #1 // =1
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    cmp x9, x10
+; CHECK-NEXT:    st1w { z1.s }, p0, [x8, #1, mul vl]
+; CHECK-NEXT:    st1w { z0.s }, p0, [sp]
+; CHECK-NEXT:    csel x9, x9, x10, lo
+; CHECK-NEXT:    ldr w0, [x8, x9, lsl #2]
+; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %ext = extractelement <vscale x 8 x i32> %a, i32 %idx
+  ret i32 %ext
+}
+
+define i64 @split_extract_8i64_idx(<vscale x 8 x i64> %a, i32 %idx) {
+; CHECK-LABEL: split_extract_8i64_idx:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-4
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 32 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    cnth x10
+; CHECK-NEXT:    // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT:    sxtw x9, w0
+; CHECK-NEXT:    sub x10, x10, #1 // =1
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    cmp x9, x10
+; CHECK-NEXT:    st1d { z3.d }, p0, [x8, #3, mul vl]
+; CHECK-NEXT:    st1d { z2.d }, p0, [x8, #2, mul vl]
+; CHECK-NEXT:    st1d { z1.d }, p0, [x8, #1, mul vl]
+; CHECK-NEXT:    st1d { z0.d }, p0, [sp]
+; CHECK-NEXT:    csel x9, x9, x10, lo
+; CHECK-NEXT:    ldr x0, [x8, x9, lsl #3]
+; CHECK-NEXT:    addvl sp, sp, #4
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %ext = extractelement <vscale x 8 x i64> %a, i32 %idx
+  ret i64 %ext
+}
+
+; EXTRACT VECTOR ELT, CONSTANT IDX
+
+define i16 @promote_extract_4i16(<vscale x 4 x i16> %a) {
+; CHECK-LABEL: promote_extract_4i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z0.s, z0.s[1]
+; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ret
+  %ext = extractelement <vscale x 4 x i16> %a, i32 1
+  ret i16 %ext
+}
+
+define i8 @split_extract_32i8(<vscale x 32 x i8> %a) {
+; CHECK-LABEL: split_extract_32i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov z0.b, z0.b[3]
+; CHECK-NEXT:    fmov w0, s0
+; CHECK-NEXT:    ret
+  %ext = extractelement <vscale x 32 x i8> %a, i32 3
+  ret i8 %ext
+}
+
+define i16 @split_extract_16i16(<vscale x 16 x i16> %a) {
+; CHECK-LABEL: split_extract_16i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-2
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    rdvl x10, #1
+; CHECK-NEXT:    sub x10, x10, #1 // =1
+; CHECK-NEXT:    ptrue p0.h
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    mov w9, #128
+; CHECK-NEXT:    cmp x10, #128 // =128
+; CHECK-NEXT:    st1h { z1.h }, p0, [x8, #1, mul vl]
+; CHECK-NEXT:    st1h { z0.h }, p0, [sp]
+; CHECK-NEXT:    csel x9, x10, x9, lo
+; CHECK-NEXT:    ldrh w0, [x8, x9, lsl #1]
+; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %ext = extractelement <vscale x 16 x i16> %a, i32 128
+  ret i16 %ext
+}
+
+define i32 @split_extract_16i32(<vscale x 16 x i32> %a) {
+; CHECK-LABEL: split_extract_16i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-4
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 32 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    mov w9, #34464
+; CHECK-NEXT:    rdvl x10, #1
+; CHECK-NEXT:    movk w9, #1, lsl #16
+; CHECK-NEXT:    sub x10, x10, #1 // =1
+; CHECK-NEXT:    ptrue p0.s
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    cmp x10, x9
+; CHECK-NEXT:    st1w { z3.s }, p0, [x8, #3, mul vl]
+; CHECK-NEXT:    st1w { z2.s }, p0, [x8, #2, mul vl]
+; CHECK-NEXT:    st1w { z1.s }, p0, [x8, #1, mul vl]
+; CHECK-NEXT:    st1w { z0.s }, p0, [sp]
+; CHECK-NEXT:    csel x9, x10, x9, lo
+; CHECK-NEXT:    ldr w0, [x8, x9, lsl #2]
+; CHECK-NEXT:    addvl sp, sp, #4
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %ext = extractelement <vscale x 16 x i32> %a, i32 100000
+  ret i32 %ext
+}
+
+define i64 @split_extract_4i64(<vscale x 4 x i64> %a) {
+; CHECK-LABEL: split_extract_4i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    str x29, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT:    addvl sp, sp, #-2
+; CHECK-NEXT:    .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG
+; CHECK-NEXT:    .cfi_offset w29, -16
+; CHECK-NEXT:    cntw x10
+; CHECK-NEXT:    sub x10, x10, #1 // =1
+; CHECK-NEXT:    ptrue p0.d
+; CHECK-NEXT:    mov x8, sp
+; CHECK-NEXT:    mov w9, #10
+; CHECK-NEXT:    cmp x10, #10 // =10
+; CHECK-NEXT:    st1d { z1.d }, p0, [x8, #1, mul vl]
+; CHECK-NEXT:    st1d { z0.d }, p0, [sp]
+; CHECK-NEXT:    csel x9, x10, x9, lo
+; CHECK-NEXT:    ldr x0, [x8, x9, lsl #3]
+; CHECK-NEXT:    addvl sp, sp, #2
+; CHECK-NEXT:    ldr x29, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT:    ret
+  %ext = extractelement <vscale x 4 x i64> %a, i32 10
+  ret i64 %ext
+}