diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -5428,22 +5428,30 @@ break; } - if (Opcode != ISD::INTRINSIC_VOID) + if (Opcode != ISD::INTRINSIC_VOID && Opcode != ISD::INTRINSIC_W_CHAIN) return EVT(); - const unsigned IntNo = - cast(Root->getOperand(1))->getZExtValue(); - if (IntNo == Intrinsic::aarch64_sme_ldr || - IntNo == Intrinsic::aarch64_sme_str) - return MVT::nxv16i8; - - if (IntNo != Intrinsic::aarch64_sve_prf) + switch (cast(Root->getOperand(1))->getZExtValue()) { + default: return EVT(); - - // We are using an SVE prefetch intrinsic. Type must be inferred - // from the width of the predicate. - return getPackedVectorTypeFromPredicateType( - Ctx, Root->getOperand(2)->getValueType(0), /*NumVec=*/1); + case Intrinsic::aarch64_sme_ldr: + case Intrinsic::aarch64_sme_str: + return MVT::nxv16i8; + case Intrinsic::aarch64_sve_prf: + // We are using an SVE prefetch intrinsic. Type must be inferred from the + // width of the predicate. + return getPackedVectorTypeFromPredicateType( + Ctx, Root->getOperand(2)->getValueType(0), /*NumVec=*/1); + case Intrinsic::aarch64_sve_ld2_sret: + return getPackedVectorTypeFromPredicateType( + Ctx, Root->getOperand(2)->getValueType(0), /*NumVec=*/2); + case Intrinsic::aarch64_sve_ld3_sret: + return getPackedVectorTypeFromPredicateType( + Ctx, Root->getOperand(2)->getValueType(0), /*NumVec=*/3); + case Intrinsic::aarch64_sve_ld4_sret: + return getPackedVectorTypeFromPredicateType( + Ctx, Root->getOperand(2)->getValueType(0), /*NumVec=*/4); + } } /// SelectAddrModeIndexedSVE - Attempt selection of the addressing mode: diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-ldN-sret-reg+imm-addr-mode.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-ldN-sret-reg+imm-addr-mode.ll --- a/llvm/test/CodeGen/AArch64/sve-intrinsics-ldN-sret-reg+imm-addr-mode.ll +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-ldN-sret-reg+imm-addr-mode.ll @@ -12,8 +12,7 @@ define { , } @ld2.nxv32i8( %Pg, *%addr) { ; CHECK-LABEL: ld2.nxv32i8: ; CHECK: // %bb.0: -; CHECK-NEXT: rdvl x8, #2 -; CHECK-NEXT: ld2b { z0.b, z1.b }, p0/z, [x0, x8] +; CHECK-NEXT: ld2b { z0.b, z1.b }, p0/z, [x0, #2, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 2 %base_ptr = bitcast * %base to i8* @@ -24,8 +23,7 @@ define { , } @ld2.nxv32i8_lower_bound( %Pg, *%addr) { ; CHECK-LABEL: ld2.nxv32i8_lower_bound: ; CHECK: // %bb.0: -; CHECK-NEXT: rdvl x8, #-16 -; CHECK-NEXT: ld2b { z0.b, z1.b }, p0/z, [x0, x8] +; CHECK-NEXT: ld2b { z0.b, z1.b }, p0/z, [x0, #-16, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 -16 %base_ptr = bitcast * %base to i8 * @@ -36,8 +34,7 @@ define { , } @ld2.nxv32i8_upper_bound( %Pg, *%addr) { ; CHECK-LABEL: ld2.nxv32i8_upper_bound: ; CHECK: // %bb.0: -; CHECK-NEXT: rdvl x8, #14 -; CHECK-NEXT: ld2b { z0.b, z1.b }, p0/z, [x0, x8] +; CHECK-NEXT: ld2b { z0.b, z1.b }, p0/z, [x0, #14, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 14 %base_ptr = bitcast * %base to i8 * @@ -85,8 +82,7 @@ define { , } @ld2.nxv16i16( %Pg, * %addr) { ; CHECK-LABEL: ld2.nxv16i16: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #14 -; CHECK-NEXT: ld2h { z0.h, z1.h }, p0/z, [x8] +; CHECK-NEXT: ld2h { z0.h, z1.h }, p0/z, [x0, #14, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 14 %base_ptr = bitcast * %base to i16 * @@ -97,8 +93,7 @@ define { , } @ld2.nxv16f16( %Pg, * %addr) { ; CHECK-LABEL: ld2.nxv16f16: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #-16 -; CHECK-NEXT: ld2h { z0.h, z1.h }, p0/z, [x8] +; CHECK-NEXT: ld2h { z0.h, z1.h }, p0/z, [x0, #-16, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 -16 %base_ptr = bitcast * %base to half * @@ -109,8 +104,7 @@ define { , } @ld2.nxv16bf16( %Pg, * %addr) #0 { ; CHECK-LABEL: ld2.nxv16bf16: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #12 -; CHECK-NEXT: ld2h { z0.h, z1.h }, p0/z, [x8] +; CHECK-NEXT: ld2h { z0.h, z1.h }, p0/z, [x0, #12, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 12 %base_ptr = bitcast * %base to bfloat * @@ -122,8 +116,7 @@ define { , } @ld2.nxv8i32( %Pg, * %addr) { ; CHECK-LABEL: ld2.nxv8i32: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #14 -; CHECK-NEXT: ld2w { z0.s, z1.s }, p0/z, [x8] +; CHECK-NEXT: ld2w { z0.s, z1.s }, p0/z, [x0, #14, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 14 %base_ptr = bitcast * %base to i32 * @@ -134,8 +127,7 @@ define { , } @ld2.nxv8f32( %Pg, * %addr) { ; CHECK-LABEL: ld2.nxv8f32: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #-16 -; CHECK-NEXT: ld2w { z0.s, z1.s }, p0/z, [x8] +; CHECK-NEXT: ld2w { z0.s, z1.s }, p0/z, [x0, #-16, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 -16 %base_ptr = bitcast * %base to float * @@ -147,8 +139,7 @@ define { , } @ld2.nxv4i64( %Pg, * %addr) { ; CHECK-LABEL: ld2.nxv4i64: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #14 -; CHECK-NEXT: ld2d { z0.d, z1.d }, p0/z, [x8] +; CHECK-NEXT: ld2d { z0.d, z1.d }, p0/z, [x0, #14, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 14 %base_ptr = bitcast * %base to i64 * @@ -159,8 +150,7 @@ define { , } @ld2.nxv4f64( %Pg, * %addr) { ; CHECK-LABEL: ld2.nxv4f64: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #-16 -; CHECK-NEXT: ld2d { z0.d, z1.d }, p0/z, [x8] +; CHECK-NEXT: ld2d { z0.d, z1.d }, p0/z, [x0, #-16, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 -16 %base_ptr = bitcast * %base to double * @@ -172,8 +162,7 @@ define { , , } @ld3.nxv48i8( %Pg, *%addr) { ; CHECK-LABEL: ld3.nxv48i8: ; CHECK: // %bb.0: -; CHECK-NEXT: rdvl x8, #3 -; CHECK-NEXT: ld3b { z0.b - z2.b }, p0/z, [x0, x8] +; CHECK-NEXT: ld3b { z0.b - z2.b }, p0/z, [x0, #3, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 3 %base_ptr = bitcast * %base to i8 * @@ -184,8 +173,7 @@ define { , , } @ld3.nxv48i8_lower_bound( %Pg, *%addr) { ; CHECK-LABEL: ld3.nxv48i8_lower_bound: ; CHECK: // %bb.0: -; CHECK-NEXT: rdvl x8, #-24 -; CHECK-NEXT: ld3b { z0.b - z2.b }, p0/z, [x0, x8] +; CHECK-NEXT: ld3b { z0.b - z2.b }, p0/z, [x0, #-24, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 -24 %base_ptr = bitcast * %base to i8 * @@ -196,8 +184,7 @@ define { , , } @ld3.nxv48i8_upper_bound( %Pg, *%addr) { ; CHECK-LABEL: ld3.nxv48i8_upper_bound: ; CHECK: // %bb.0: -; CHECK-NEXT: rdvl x8, #21 -; CHECK-NEXT: ld3b { z0.b - z2.b }, p0/z, [x0, x8] +; CHECK-NEXT: ld3b { z0.b - z2.b }, p0/z, [x0, #21, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 21 %base_ptr = bitcast * %base to i8 * @@ -257,8 +244,7 @@ define { , , } @ld3.nxv24i16( %Pg, *%addr) { ; CHECK-LABEL: ld3.nxv24i16: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #21 -; CHECK-NEXT: ld3h { z0.h - z2.h }, p0/z, [x8] +; CHECK-NEXT: ld3h { z0.h - z2.h }, p0/z, [x0, #21, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 21 %base_ptr = bitcast * %base to i16 * @@ -269,8 +255,7 @@ define { , , } @ld3.nxv24f16( %Pg, *%addr) { ; CHECK-LABEL: ld3.nxv24f16: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #21 -; CHECK-NEXT: ld3h { z0.h - z2.h }, p0/z, [x8] +; CHECK-NEXT: ld3h { z0.h - z2.h }, p0/z, [x0, #21, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 21 %base_ptr = bitcast * %base to half * @@ -281,8 +266,7 @@ define { , , } @ld3.nxv24bf16( %Pg, *%addr) #0 { ; CHECK-LABEL: ld3.nxv24bf16: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #-24 -; CHECK-NEXT: ld3h { z0.h - z2.h }, p0/z, [x8] +; CHECK-NEXT: ld3h { z0.h - z2.h }, p0/z, [x0, #-24, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 -24 %base_ptr = bitcast * %base to bfloat * @@ -294,8 +278,7 @@ define { , , } @ld3.nxv12i32( %Pg, *%addr) { ; CHECK-LABEL: ld3.nxv12i32: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #21 -; CHECK-NEXT: ld3w { z0.s - z2.s }, p0/z, [x8] +; CHECK-NEXT: ld3w { z0.s - z2.s }, p0/z, [x0, #21, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 21 %base_ptr = bitcast * %base to i32 * @@ -306,8 +289,7 @@ define { , , } @ld3.nxv12f32( %Pg, *%addr) { ; CHECK-LABEL: ld3.nxv12f32: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #-24 -; CHECK-NEXT: ld3w { z0.s - z2.s }, p0/z, [x8] +; CHECK-NEXT: ld3w { z0.s - z2.s }, p0/z, [x0, #-24, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 -24 %base_ptr = bitcast * %base to float * @@ -319,8 +301,7 @@ define { , , } @ld3.nxv6i64( %Pg, *%addr) { ; CHECK-LABEL: ld3.nxv6i64: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #21 -; CHECK-NEXT: ld3d { z0.d - z2.d }, p0/z, [x8] +; CHECK-NEXT: ld3d { z0.d - z2.d }, p0/z, [x0, #21, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 21 %base_ptr = bitcast * %base to i64 * @@ -331,8 +312,7 @@ define { , , } @ld3.nxv6f64( %Pg, *%addr) { ; CHECK-LABEL: ld3.nxv6f64: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #-24 -; CHECK-NEXT: ld3d { z0.d - z2.d }, p0/z, [x8] +; CHECK-NEXT: ld3d { z0.d - z2.d }, p0/z, [x0, #-24, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 -24 %base_ptr = bitcast * %base to double * @@ -344,8 +324,7 @@ define { , , , } @ld4.nxv64i8( %Pg, *%addr) { ; CHECK-LABEL: ld4.nxv64i8: ; CHECK: // %bb.0: -; CHECK-NEXT: rdvl x8, #4 -; CHECK-NEXT: ld4b { z0.b - z3.b }, p0/z, [x0, x8] +; CHECK-NEXT: ld4b { z0.b - z3.b }, p0/z, [x0, #4, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 4 %base_ptr = bitcast * %base to i8 * @@ -356,8 +335,7 @@ define { , , , } @ld4.nxv64i8_lower_bound( %Pg, *%addr) { ; CHECK-LABEL: ld4.nxv64i8_lower_bound: ; CHECK: // %bb.0: -; CHECK-NEXT: rdvl x8, #-32 -; CHECK-NEXT: ld4b { z0.b - z3.b }, p0/z, [x0, x8] +; CHECK-NEXT: ld4b { z0.b - z3.b }, p0/z, [x0, #-32, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 -32 %base_ptr = bitcast * %base to i8 * @@ -368,8 +346,7 @@ define { , , , } @ld4.nxv64i8_upper_bound( %Pg, *%addr) { ; CHECK-LABEL: ld4.nxv64i8_upper_bound: ; CHECK: // %bb.0: -; CHECK-NEXT: rdvl x8, #28 -; CHECK-NEXT: ld4b { z0.b - z3.b }, p0/z, [x0, x8] +; CHECK-NEXT: ld4b { z0.b - z3.b }, p0/z, [x0, #28, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 28 %base_ptr = bitcast * %base to i8 * @@ -455,8 +432,7 @@ define { , , , } @ld4.nxv32i16( %Pg, *%addr) { ; CHECK-LABEL: ld4.nxv32i16: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #8 -; CHECK-NEXT: ld4h { z0.h - z3.h }, p0/z, [x8] +; CHECK-NEXT: ld4h { z0.h - z3.h }, p0/z, [x0, #8, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 8 %base_ptr = bitcast * %base to i16 * @@ -467,8 +443,7 @@ define { , , , } @ld4.nxv32f16( %Pg, *%addr) { ; CHECK-LABEL: ld4.nxv32f16: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #28 -; CHECK-NEXT: ld4h { z0.h - z3.h }, p0/z, [x8] +; CHECK-NEXT: ld4h { z0.h - z3.h }, p0/z, [x0, #28, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 28 %base_ptr = bitcast * %base to half * @@ -479,8 +454,7 @@ define { , , , } @ld4.nxv32bf16( %Pg, *%addr) #0 { ; CHECK-LABEL: ld4.nxv32bf16: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #-32 -; CHECK-NEXT: ld4h { z0.h - z3.h }, p0/z, [x8] +; CHECK-NEXT: ld4h { z0.h - z3.h }, p0/z, [x0, #-32, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 -32 %base_ptr = bitcast * %base to bfloat * @@ -492,8 +466,7 @@ define { , , , } @ld4.nxv16i32( %Pg, *%addr) { ; CHECK-LABEL: ld4.nxv16i32: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #28 -; CHECK-NEXT: ld4w { z0.s - z3.s }, p0/z, [x8] +; CHECK-NEXT: ld4w { z0.s - z3.s }, p0/z, [x0, #28, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 28 %base_ptr = bitcast * %base to i32 * @@ -504,8 +477,7 @@ define { , , , } @ld4.nxv16f32( %Pg, * %addr) { ; CHECK-LABEL: ld4.nxv16f32: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #-32 -; CHECK-NEXT: ld4w { z0.s - z3.s }, p0/z, [x8] +; CHECK-NEXT: ld4w { z0.s - z3.s }, p0/z, [x0, #-32, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 -32 %base_ptr = bitcast * %base to float * @@ -517,8 +489,7 @@ define { , , , } @ld4.nxv8i64( %Pg, *%addr) { ; CHECK-LABEL: ld4.nxv8i64: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #28 -; CHECK-NEXT: ld4d { z0.d - z3.d }, p0/z, [x8] +; CHECK-NEXT: ld4d { z0.d - z3.d }, p0/z, [x0, #28, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 28 %base_ptr = bitcast * %base to i64 * @@ -529,8 +500,7 @@ define { , , , } @ld4.nxv8f64( %Pg, *%addr) { ; CHECK-LABEL: ld4.nxv8f64: ; CHECK: // %bb.0: -; CHECK-NEXT: addvl x8, x0, #-32 -; CHECK-NEXT: ld4d { z0.d - z3.d }, p0/z, [x8] +; CHECK-NEXT: ld4d { z0.d - z3.d }, p0/z, [x0, #-32, mul vl] ; CHECK-NEXT: ret %base = getelementptr , * %addr, i64 -32 %base_ptr = bitcast * %base to double *