diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -11249,10 +11249,11 @@ EVT ExTy = N0.getValueType(); EVT TrTy = N->getValueType(0); - unsigned NumElem = VecTy.getVectorNumElements(); + auto EltCnt = VecTy.getVectorElementCount(); unsigned SizeRatio = ExTy.getSizeInBits()/TrTy.getSizeInBits(); + auto NewEltCnt = EltCnt * SizeRatio; - EVT NVT = EVT::getVectorVT(*DAG.getContext(), TrTy, SizeRatio * NumElem); + EVT NVT = EVT::getVectorVT(*DAG.getContext(), TrTy, NewEltCnt); assert(NVT.getSizeInBits() == VecTy.getSizeInBits() && "Invalid Size"); SDValue EltNo = N0->getOperand(1); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -2253,13 +2253,14 @@ SDValue Lo, Hi; GetSplitVector(Vec, Lo, Hi); - uint64_t LoElts = Lo.getValueType().getVectorNumElements(); + uint64_t LoElts = Lo.getValueType().getVectorMinNumElements(); if (IdxVal < LoElts) return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0); - return SDValue(DAG.UpdateNodeOperands(N, Hi, - DAG.getConstant(IdxVal - LoElts, SDLoc(N), - Idx.getValueType())), 0); + else if (!Vec.getValueType().isScalableVector()) + return SDValue(DAG.UpdateNodeOperands(N, Hi, + DAG.getConstant(IdxVal - LoElts, SDLoc(N), + Idx.getValueType())), 0); } // See if the target wants to custom expand this node. @@ -2272,7 +2273,7 @@ if (VecVT.getScalarSizeInBits() < 8) { EltVT = MVT::i8; VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, - VecVT.getVectorNumElements()); + VecVT.getVectorElementCount()); Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec); } @@ -2302,7 +2303,7 @@ return DAG.getExtLoad( ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT, - commonAlignment(SmallestAlign, EltVT.getSizeInBits() / 8)); + commonAlignment(SmallestAlign, EltVT.getSizeInBits().getFixedSize() / 8)); } SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) { diff --git a/llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll b/llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sve-split-extract-elt.ll @@ -0,0 +1,216 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve < %s | FileCheck %s + +; EXTRACT VECTOR ELT + +define i32 @promote_extract_2i32_idx( %a, i32 %idx) { +; CHECK-LABEL: promote_extract_2i32_idx: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 +; CHECK-NEXT: sxtw x8, w0 +; CHECK-NEXT: whilels p0.d, xzr, x8 +; CHECK-NEXT: lastb x0, p0, z0.d +; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0 +; CHECK-NEXT: ret + %ext = extractelement %a, i32 %idx + ret i32 %ext +} + +define i8 @split_extract_32i8_idx( %a, i32 %idx) { +; CHECK-LABEL: split_extract_32i8_idx: +; CHECK: // %bb.0: +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-2 +; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG +; CHECK-NEXT: .cfi_offset w29, -16 +; CHECK-NEXT: rdvl x10, #2 +; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 +; CHECK-NEXT: sxtw x9, w0 +; CHECK-NEXT: sub x10, x10, #1 // =1 +; CHECK-NEXT: ptrue p0.b +; CHECK-NEXT: mov x8, sp +; CHECK-NEXT: cmp x9, x10 +; CHECK-NEXT: st1b { z1.b }, p0, [x8, #1, mul vl] +; CHECK-NEXT: st1b { z0.b }, p0, [sp] +; CHECK-NEXT: csel x9, x9, x10, lo +; CHECK-NEXT: ldrb w0, [x8, x9] +; CHECK-NEXT: addvl sp, sp, #2 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret + %ext = extractelement %a, i32 %idx + ret i8 %ext +} + +define i16 @split_extract_16i16_idx( %a, i32 %idx) { +; CHECK-LABEL: split_extract_16i16_idx: +; CHECK: // %bb.0: +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-2 +; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG +; CHECK-NEXT: .cfi_offset w29, -16 +; CHECK-NEXT: rdvl x10, #1 +; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 +; CHECK-NEXT: sxtw x9, w0 +; CHECK-NEXT: sub x10, x10, #1 // =1 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov x8, sp +; CHECK-NEXT: cmp x9, x10 +; CHECK-NEXT: st1h { z1.h }, p0, [x8, #1, mul vl] +; CHECK-NEXT: st1h { z0.h }, p0, [sp] +; CHECK-NEXT: csel x9, x9, x10, lo +; CHECK-NEXT: ldrh w0, [x8, x9, lsl #1] +; CHECK-NEXT: addvl sp, sp, #2 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret + %ext = extractelement %a, i32 %idx + ret i16 %ext +} + +define i32 @split_extract_8i32_idx( %a, i32 %idx) { +; CHECK-LABEL: split_extract_8i32_idx: +; CHECK: // %bb.0: +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-2 +; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG +; CHECK-NEXT: .cfi_offset w29, -16 +; CHECK-NEXT: cnth x10 +; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 +; CHECK-NEXT: sxtw x9, w0 +; CHECK-NEXT: sub x10, x10, #1 // =1 +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: mov x8, sp +; CHECK-NEXT: cmp x9, x10 +; CHECK-NEXT: st1w { z1.s }, p0, [x8, #1, mul vl] +; CHECK-NEXT: st1w { z0.s }, p0, [sp] +; CHECK-NEXT: csel x9, x9, x10, lo +; CHECK-NEXT: ldr w0, [x8, x9, lsl #2] +; CHECK-NEXT: addvl sp, sp, #2 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret + %ext = extractelement %a, i32 %idx + ret i32 %ext +} + +define i64 @split_extract_8i64_idx( %a, i32 %idx) { +; CHECK-LABEL: split_extract_8i64_idx: +; CHECK: // %bb.0: +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-4 +; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 32 * VG +; CHECK-NEXT: .cfi_offset w29, -16 +; CHECK-NEXT: cnth x10 +; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0 +; CHECK-NEXT: sxtw x9, w0 +; CHECK-NEXT: sub x10, x10, #1 // =1 +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: mov x8, sp +; CHECK-NEXT: cmp x9, x10 +; CHECK-NEXT: st1d { z3.d }, p0, [x8, #3, mul vl] +; CHECK-NEXT: st1d { z2.d }, p0, [x8, #2, mul vl] +; CHECK-NEXT: st1d { z1.d }, p0, [x8, #1, mul vl] +; CHECK-NEXT: st1d { z0.d }, p0, [sp] +; CHECK-NEXT: csel x9, x9, x10, lo +; CHECK-NEXT: ldr x0, [x8, x9, lsl #3] +; CHECK-NEXT: addvl sp, sp, #4 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret + %ext = extractelement %a, i32 %idx + ret i64 %ext +} + +; EXTRACT VECTOR ELT, CONSTANT IDX + +define i16 @promote_extract_4i16( %a) { +; CHECK-LABEL: promote_extract_4i16: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z0.s, z0.s[1] +; CHECK-NEXT: fmov w0, s0 +; CHECK-NEXT: ret + %ext = extractelement %a, i32 1 + ret i16 %ext +} + +define i8 @split_extract_32i8( %a) { +; CHECK-LABEL: split_extract_32i8: +; CHECK: // %bb.0: +; CHECK-NEXT: mov z0.b, z0.b[3] +; CHECK-NEXT: fmov w0, s0 +; CHECK-NEXT: ret + %ext = extractelement %a, i32 3 + ret i8 %ext +} + +define i16 @split_extract_16i16( %a) { +; CHECK-LABEL: split_extract_16i16: +; CHECK: // %bb.0: +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-2 +; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG +; CHECK-NEXT: .cfi_offset w29, -16 +; CHECK-NEXT: rdvl x10, #1 +; CHECK-NEXT: sub x10, x10, #1 // =1 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: mov x8, sp +; CHECK-NEXT: mov w9, #128 +; CHECK-NEXT: cmp x10, #128 // =128 +; CHECK-NEXT: st1h { z1.h }, p0, [x8, #1, mul vl] +; CHECK-NEXT: st1h { z0.h }, p0, [sp] +; CHECK-NEXT: csel x9, x10, x9, lo +; CHECK-NEXT: ldrh w0, [x8, x9, lsl #1] +; CHECK-NEXT: addvl sp, sp, #2 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret + %ext = extractelement %a, i32 128 + ret i16 %ext +} + +define i32 @split_extract_16i32( %a) { +; CHECK-LABEL: split_extract_16i32: +; CHECK: // %bb.0: +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-4 +; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x20, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 32 * VG +; CHECK-NEXT: .cfi_offset w29, -16 +; CHECK-NEXT: mov w9, #34464 +; CHECK-NEXT: rdvl x10, #1 +; CHECK-NEXT: movk w9, #1, lsl #16 +; CHECK-NEXT: sub x10, x10, #1 // =1 +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: mov x8, sp +; CHECK-NEXT: cmp x10, x9 +; CHECK-NEXT: st1w { z3.s }, p0, [x8, #3, mul vl] +; CHECK-NEXT: st1w { z2.s }, p0, [x8, #2, mul vl] +; CHECK-NEXT: st1w { z1.s }, p0, [x8, #1, mul vl] +; CHECK-NEXT: st1w { z0.s }, p0, [sp] +; CHECK-NEXT: csel x9, x10, x9, lo +; CHECK-NEXT: ldr w0, [x8, x9, lsl #2] +; CHECK-NEXT: addvl sp, sp, #4 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret + %ext = extractelement %a, i32 100000 + ret i32 %ext +} + +define i64 @split_extract_4i64( %a) { +; CHECK-LABEL: split_extract_4i64: +; CHECK: // %bb.0: +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-2 +; CHECK-NEXT: .cfi_escape 0x0f, 0x0c, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x10, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 16 * VG +; CHECK-NEXT: .cfi_offset w29, -16 +; CHECK-NEXT: cntw x10 +; CHECK-NEXT: sub x10, x10, #1 // =1 +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: mov x8, sp +; CHECK-NEXT: mov w9, #10 +; CHECK-NEXT: cmp x10, #10 // =10 +; CHECK-NEXT: st1d { z1.d }, p0, [x8, #1, mul vl] +; CHECK-NEXT: st1d { z0.d }, p0, [sp] +; CHECK-NEXT: csel x9, x10, x9, lo +; CHECK-NEXT: ldr x0, [x8, x9, lsl #3] +; CHECK-NEXT: addvl sp, sp, #2 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret + %ext = extractelement %a, i32 10 + ret i64 %ext +}