diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -71,6 +71,9 @@ case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break; case ISD::LOAD: Res = PromoteIntRes_LOAD(cast(N)); break; + case ISD::VP_LOAD: + Res = PromoteIntRes_VP_LOAD(cast(N)); + break; case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast(N)); break; case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast(N)); @@ -759,6 +762,23 @@ return Res; } +SDValue DAGTypeLegalizer::PromoteIntRes_VP_LOAD(VPLoadSDNode *N) { + assert(!N->isIndexed() && "Indexed vp_load during type legalization!"); + EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); + ISD::LoadExtType ExtType = (N->getExtensionType() == ISD::NON_EXTLOAD) + ? ISD::EXTLOAD + : N->getExtensionType(); + SDLoc dl(N); + SDValue Res = + DAG.getLoadVP(N->getAddressingMode(), ExtType, NVT, dl, N->getChain(), + N->getBasePtr(), N->getOffset(), N->getMask(), + N->getVectorLength(), N->getMemoryVT(), N->getMemOperand()); + // Legalize the chain result - switch anything that used the old chain to + // use the new one. + ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); + return Res; +} + SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) { EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDValue ExtPassThru = GetPromotedInteger(N->getPassThru()); @@ -1592,8 +1612,14 @@ case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break; case ISD::STORE: Res = PromoteIntOp_STORE(cast(N), OpNo); break; + case ISD::VP_STORE: + Res = PromoteIntOp_VP_STORE(cast(N), OpNo); + break; case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast(N), OpNo); break; + case ISD::VP_LOAD: + Res = PromoteIntOp_VP_LOAD(cast(N), OpNo); + break; case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast(N), OpNo); break; case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast(N), @@ -1938,6 +1964,50 @@ N->getMemoryVT(), N->getMemOperand()); } +SDValue DAGTypeLegalizer::PromoteIntOp_VP_STORE(VPStoreSDNode *N, + unsigned OpNo) { + SDValue DataOp = N->getValue(); + SDValue Operand = N->getOperand(OpNo); + + if (OpNo >= 4) { + // The Mask or EVL. Update in place. + EVT DataVT = DataOp.getValueType(); + SDValue PromotedOperand = OpNo == 4 ? PromoteTargetBoolean(Operand, DataVT) + : ZExtPromotedInteger(Operand); + SmallVector NewOps(N->op_begin(), N->op_end()); + NewOps[OpNo] = PromotedOperand; + return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0); + } + + assert(OpNo == 1 && "Unexpected operand for promotion"); + DataOp = GetPromotedInteger(DataOp); + + assert(!N->isIndexed() && "expecting unindexed vp_store!"); + + return DAG.getTruncStoreVP(N->getChain(), SDLoc(N), DataOp, N->getBasePtr(), + N->getMask(), N->getVectorLength(), + N->getMemoryVT(), N->getMemOperand(), + N->isCompressingStore()); +} + +SDValue DAGTypeLegalizer::PromoteIntOp_VP_LOAD(VPLoadSDNode *N, unsigned OpNo) { + assert(OpNo >= 3 && "Only know how to promote the mask or length!"); + EVT DataVT = N->getValueType(0); + SDValue Operand = N->getOperand(OpNo); + SDValue PromotedOperand = OpNo == 3 ? PromoteTargetBoolean(Operand, DataVT) + : ZExtPromotedInteger(Operand); + SmallVector NewOps(N->op_begin(), N->op_end()); + NewOps[OpNo] = PromotedOperand; + SDNode *Res = DAG.UpdateNodeOperands(N, NewOps); + if (Res == N) + return SDValue(Res, 0); + + // Update triggered CSE, do our own replacement since caller can't. + ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0)); + ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1)); + return SDValue(); +} + SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo) { SDValue DataOp = N->getValue(); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -330,6 +330,7 @@ SDValue PromoteIntRes_FREEZE(SDNode *N); SDValue PromoteIntRes_INT_EXTEND(SDNode *N); SDValue PromoteIntRes_LOAD(LoadSDNode *N); + SDValue PromoteIntRes_VP_LOAD(VPLoadSDNode *N); SDValue PromoteIntRes_MLOAD(MaskedLoadSDNode *N); SDValue PromoteIntRes_MGATHER(MaskedGatherSDNode *N); SDValue PromoteIntRes_Overflow(SDNode *N); @@ -391,7 +392,9 @@ SDValue PromoteIntOp_UINT_TO_FP(SDNode *N); SDValue PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N); SDValue PromoteIntOp_ZERO_EXTEND(SDNode *N); + SDValue PromoteIntOp_VP_STORE(VPStoreSDNode *N, unsigned OpNo); SDValue PromoteIntOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo); + SDValue PromoteIntOp_VP_LOAD(VPLoadSDNode *N, unsigned OpNo); SDValue PromoteIntOp_MLOAD(MaskedLoadSDNode *N, unsigned OpNo); SDValue PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, unsigned OpNo); SDValue PromoteIntOp_MGATHER(MaskedGatherSDNode *N, unsigned OpNo); diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpload.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpload.ll --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpload.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpload.ll @@ -52,6 +52,18 @@ ret <4 x i8> %load } +declare <8 x i7> @llvm.vp.load.v8i7.p0v8i7(<8 x i7>*, <8 x i1>, i32) + +define <8 x i7> @vpload_v8i7(<8 x i7>* %ptr, <8 x i1> %m, i32 zeroext %evl) { +; CHECK-LABEL: vpload_v8i7: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, mu +; CHECK-NEXT: vle8.v v8, (a0), v0.t +; CHECK-NEXT: ret + %load = call <8 x i7> @llvm.vp.load.v8i7.p0v8i7(<8 x i7>* %ptr, <8 x i1> %m, i32 %evl) + ret <8 x i7> %load +} + declare <8 x i8> @llvm.vp.load.v8i8.p0v8i8(<8 x i8>*, <8 x i1>, i32) define <8 x i8> @vpload_v8i8(<8 x i8>* %ptr, <8 x i1> %m, i32 zeroext %evl) { @@ -384,20 +396,20 @@ ; CHECK-NEXT: addi a3, a1, -16 ; CHECK-NEXT: vmv1r.v v8, v0 ; CHECK-NEXT: li a2, 0 -; CHECK-NEXT: bltu a1, a3, .LBB31_2 +; CHECK-NEXT: bltu a1, a3, .LBB32_2 ; CHECK-NEXT: # %bb.1: ; CHECK-NEXT: mv a2, a3 -; CHECK-NEXT: .LBB31_2: +; CHECK-NEXT: .LBB32_2: ; CHECK-NEXT: vsetivli zero, 2, e8, mf4, ta, mu ; CHECK-NEXT: vslidedown.vi v0, v8, 2 ; CHECK-NEXT: addi a3, a0, 128 ; CHECK-NEXT: vsetvli zero, a2, e64, m8, ta, mu ; CHECK-NEXT: vle64.v v16, (a3), v0.t ; CHECK-NEXT: li a2, 16 -; CHECK-NEXT: bltu a1, a2, .LBB31_4 +; CHECK-NEXT: bltu a1, a2, .LBB32_4 ; CHECK-NEXT: # %bb.3: ; CHECK-NEXT: li a1, 16 -; CHECK-NEXT: .LBB31_4: +; CHECK-NEXT: .LBB32_4: ; CHECK-NEXT: vsetvli zero, a1, e64, m8, ta, mu ; CHECK-NEXT: vmv1r.v v0, v8 ; CHECK-NEXT: vle64.v v8, (a0), v0.t @@ -417,39 +429,39 @@ ; CHECK-NEXT: vmv1r.v v8, v0 ; CHECK-NEXT: li a3, 0 ; CHECK-NEXT: li a5, 0 -; CHECK-NEXT: bltu a2, a4, .LBB32_2 +; CHECK-NEXT: bltu a2, a4, .LBB33_2 ; CHECK-NEXT: # %bb.1: ; CHECK-NEXT: mv a5, a4 -; CHECK-NEXT: .LBB32_2: +; CHECK-NEXT: .LBB33_2: ; CHECK-NEXT: li a4, 16 -; CHECK-NEXT: bltu a5, a4, .LBB32_4 +; CHECK-NEXT: bltu a5, a4, .LBB33_4 ; CHECK-NEXT: # %bb.3: ; CHECK-NEXT: li a5, 16 -; CHECK-NEXT: .LBB32_4: +; CHECK-NEXT: .LBB33_4: ; CHECK-NEXT: vsetivli zero, 4, e8, mf2, ta, mu ; CHECK-NEXT: vslidedown.vi v0, v8, 4 ; CHECK-NEXT: addi a6, a1, 256 ; CHECK-NEXT: vsetvli zero, a5, e64, m8, ta, mu ; CHECK-NEXT: vle64.v v16, (a6), v0.t ; CHECK-NEXT: li a5, 32 -; CHECK-NEXT: bltu a2, a5, .LBB32_6 +; CHECK-NEXT: bltu a2, a5, .LBB33_6 ; CHECK-NEXT: # %bb.5: ; CHECK-NEXT: li a2, 32 -; CHECK-NEXT: .LBB32_6: +; CHECK-NEXT: .LBB33_6: ; CHECK-NEXT: addi a5, a2, -16 -; CHECK-NEXT: bltu a2, a5, .LBB32_8 +; CHECK-NEXT: bltu a2, a5, .LBB33_8 ; CHECK-NEXT: # %bb.7: ; CHECK-NEXT: mv a3, a5 -; CHECK-NEXT: .LBB32_8: +; CHECK-NEXT: .LBB33_8: ; CHECK-NEXT: vsetivli zero, 2, e8, mf4, ta, mu ; CHECK-NEXT: vslidedown.vi v0, v8, 2 ; CHECK-NEXT: addi a5, a1, 128 ; CHECK-NEXT: vsetvli zero, a3, e64, m8, ta, mu ; CHECK-NEXT: vle64.v v24, (a5), v0.t -; CHECK-NEXT: bltu a2, a4, .LBB32_10 +; CHECK-NEXT: bltu a2, a4, .LBB33_10 ; CHECK-NEXT: # %bb.9: ; CHECK-NEXT: li a2, 16 -; CHECK-NEXT: .LBB32_10: +; CHECK-NEXT: .LBB33_10: ; CHECK-NEXT: vsetvli zero, a2, e64, m8, ta, mu ; CHECK-NEXT: vmv1r.v v0, v8 ; CHECK-NEXT: vle64.v v8, (a1), v0.t diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpstore.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpstore.ll --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpstore.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-vpstore.ll @@ -28,6 +28,18 @@ ret void } +declare void @llvm.vp.store.v8i7.p0v8i7(<8 x i7>, <8 x i7>*, <8 x i1>, i32) + +define void @vpstore_v8i7(<8 x i7> %val, <8 x i7>* %ptr, <8 x i1> %m, i32 zeroext %evl) { +; CHECK-LABEL: vpstore_v8i7: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, mu +; CHECK-NEXT: vse8.v v8, (a0), v0.t +; CHECK-NEXT: ret + call void @llvm.vp.store.v8i7.p0v8i7(<8 x i7> %val, <8 x i7>* %ptr, <8 x i1> %m, i32 %evl) + ret void +} + declare void @llvm.vp.store.v8i8.p0v8i8(<8 x i8>, <8 x i8>*, <8 x i1>, i32) define void @vpstore_v8i8(<8 x i8> %val, <8 x i8>* %ptr, <8 x i1> %m, i32 zeroext %evl) { @@ -287,18 +299,18 @@ ; CHECK: # %bb.0: ; CHECK-NEXT: li a2, 16 ; CHECK-NEXT: mv a3, a1 -; CHECK-NEXT: bltu a1, a2, .LBB23_2 +; CHECK-NEXT: bltu a1, a2, .LBB24_2 ; CHECK-NEXT: # %bb.1: ; CHECK-NEXT: li a3, 16 -; CHECK-NEXT: .LBB23_2: +; CHECK-NEXT: .LBB24_2: ; CHECK-NEXT: li a2, 0 ; CHECK-NEXT: vsetvli zero, a3, e64, m8, ta, mu ; CHECK-NEXT: addi a3, a1, -16 ; CHECK-NEXT: vse64.v v8, (a0), v0.t -; CHECK-NEXT: bltu a1, a3, .LBB23_4 +; CHECK-NEXT: bltu a1, a3, .LBB24_4 ; CHECK-NEXT: # %bb.3: ; CHECK-NEXT: mv a2, a3 -; CHECK-NEXT: .LBB23_4: +; CHECK-NEXT: .LBB24_4: ; CHECK-NEXT: vsetivli zero, 2, e8, mf4, ta, mu ; CHECK-NEXT: vslidedown.vi v0, v0, 2 ; CHECK-NEXT: addi a0, a0, 128 diff --git a/llvm/test/CodeGen/RISCV/rvv/vpload.ll b/llvm/test/CodeGen/RISCV/rvv/vpload.ll --- a/llvm/test/CodeGen/RISCV/rvv/vpload.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vpload.ll @@ -52,6 +52,18 @@ ret %load } +declare @llvm.vp.load.nxv4i6.p0nxv4i6(*, , i32) + +define @vpload_nxv4i6(* %ptr, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpload_nxv4i6: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e8, mf2, ta, mu +; CHECK-NEXT: vle8.v v8, (a0), v0.t +; CHECK-NEXT: ret + %load = call @llvm.vp.load.nxv4i6.p0nxv4i6(* %ptr, %m, i32 %evl) + ret %load +} + declare @llvm.vp.load.nxv4i8.p0nxv4i8(*, , i32) define @vpload_nxv4i8(* %ptr, %m, i32 zeroext %evl) { @@ -460,18 +472,18 @@ ; CHECK-NEXT: vsetvli a4, zero, e8, mf4, ta, mu ; CHECK-NEXT: sub a4, a1, a2 ; CHECK-NEXT: vslidedown.vx v0, v0, a5 -; CHECK-NEXT: bltu a1, a4, .LBB37_2 +; CHECK-NEXT: bltu a1, a4, .LBB38_2 ; CHECK-NEXT: # %bb.1: ; CHECK-NEXT: mv a3, a4 -; CHECK-NEXT: .LBB37_2: +; CHECK-NEXT: .LBB38_2: ; CHECK-NEXT: slli a4, a2, 3 ; CHECK-NEXT: add a4, a0, a4 ; CHECK-NEXT: vsetvli zero, a3, e64, m8, ta, mu ; CHECK-NEXT: vle64.v v16, (a4), v0.t -; CHECK-NEXT: bltu a1, a2, .LBB37_4 +; CHECK-NEXT: bltu a1, a2, .LBB38_4 ; CHECK-NEXT: # %bb.3: ; CHECK-NEXT: mv a1, a2 -; CHECK-NEXT: .LBB37_4: +; CHECK-NEXT: .LBB38_4: ; CHECK-NEXT: vsetvli zero, a1, e64, m8, ta, mu ; CHECK-NEXT: vmv1r.v v0, v8 ; CHECK-NEXT: vle64.v v8, (a0), v0.t @@ -498,16 +510,16 @@ ; CHECK-NEXT: slli a5, a3, 1 ; CHECK-NEXT: vmv1r.v v8, v0 ; CHECK-NEXT: mv a4, a2 -; CHECK-NEXT: bltu a2, a5, .LBB38_2 +; CHECK-NEXT: bltu a2, a5, .LBB39_2 ; CHECK-NEXT: # %bb.1: ; CHECK-NEXT: mv a4, a5 -; CHECK-NEXT: .LBB38_2: +; CHECK-NEXT: .LBB39_2: ; CHECK-NEXT: sub a7, a4, a3 ; CHECK-NEXT: li a6, 0 -; CHECK-NEXT: bltu a4, a7, .LBB38_4 +; CHECK-NEXT: bltu a4, a7, .LBB39_4 ; CHECK-NEXT: # %bb.3: ; CHECK-NEXT: mv a6, a7 -; CHECK-NEXT: .LBB38_4: +; CHECK-NEXT: .LBB39_4: ; CHECK-NEXT: li a7, 0 ; CHECK-NEXT: srli t0, a3, 3 ; CHECK-NEXT: vsetvli t1, zero, e8, mf4, ta, mu @@ -519,23 +531,23 @@ ; CHECK-NEXT: srli a6, a3, 2 ; CHECK-NEXT: sub t0, a2, a5 ; CHECK-NEXT: slli a5, a3, 4 -; CHECK-NEXT: bltu a2, t0, .LBB38_6 +; CHECK-NEXT: bltu a2, t0, .LBB39_6 ; CHECK-NEXT: # %bb.5: ; CHECK-NEXT: mv a7, t0 -; CHECK-NEXT: .LBB38_6: +; CHECK-NEXT: .LBB39_6: ; CHECK-NEXT: vsetvli a2, zero, e8, mf2, ta, mu ; CHECK-NEXT: vslidedown.vx v0, v8, a6 ; CHECK-NEXT: add a2, a0, a5 -; CHECK-NEXT: bltu a7, a3, .LBB38_8 +; CHECK-NEXT: bltu a7, a3, .LBB39_8 ; CHECK-NEXT: # %bb.7: ; CHECK-NEXT: mv a7, a3 -; CHECK-NEXT: .LBB38_8: +; CHECK-NEXT: .LBB39_8: ; CHECK-NEXT: vsetvli zero, a7, e64, m8, ta, mu ; CHECK-NEXT: vle64.v v24, (a2), v0.t -; CHECK-NEXT: bltu a4, a3, .LBB38_10 +; CHECK-NEXT: bltu a4, a3, .LBB39_10 ; CHECK-NEXT: # %bb.9: ; CHECK-NEXT: mv a4, a3 -; CHECK-NEXT: .LBB38_10: +; CHECK-NEXT: .LBB39_10: ; CHECK-NEXT: vsetvli zero, a4, e64, m8, ta, mu ; CHECK-NEXT: vmv1r.v v0, v8 ; CHECK-NEXT: vle64.v v8, (a0), v0.t diff --git a/llvm/test/CodeGen/RISCV/rvv/vpstore.ll b/llvm/test/CodeGen/RISCV/rvv/vpstore.ll --- a/llvm/test/CodeGen/RISCV/rvv/vpstore.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vpstore.ll @@ -100,6 +100,18 @@ ret void } +declare void @llvm.vp.store.nxv8i12.p0nxv8i12(, *, , i32) + +define void @vpstore_nxv8i12( %val, * %ptr, %m, i32 zeroext %evl) { +; CHECK-LABEL: vpstore_nxv8i12: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a1, e16, m2, ta, mu +; CHECK-NEXT: vse16.v v8, (a0), v0.t +; CHECK-NEXT: ret + call void @llvm.vp.store.nxv8i12.p0nxv8i12( %val, * %ptr, %m, i32 %evl) + ret void +} + declare void @llvm.vp.store.nxv8i16.p0nxv8i16(, *, , i32) define void @vpstore_nxv8i16( %val, * %ptr, %m, i32 zeroext %evl) { @@ -371,10 +383,10 @@ ; CHECK: # %bb.0: ; CHECK-NEXT: csrr a2, vlenb ; CHECK-NEXT: mv a3, a1 -; CHECK-NEXT: bltu a1, a2, .LBB30_2 +; CHECK-NEXT: bltu a1, a2, .LBB31_2 ; CHECK-NEXT: # %bb.1: ; CHECK-NEXT: mv a3, a2 -; CHECK-NEXT: .LBB30_2: +; CHECK-NEXT: .LBB31_2: ; CHECK-NEXT: li a4, 0 ; CHECK-NEXT: vsetvli zero, a3, e64, m8, ta, mu ; CHECK-NEXT: vse64.v v8, (a0), v0.t @@ -382,10 +394,10 @@ ; CHECK-NEXT: vsetvli a3, zero, e8, mf4, ta, mu ; CHECK-NEXT: sub a3, a1, a2 ; CHECK-NEXT: vslidedown.vx v0, v0, a5 -; CHECK-NEXT: bltu a1, a3, .LBB30_4 +; CHECK-NEXT: bltu a1, a3, .LBB31_4 ; CHECK-NEXT: # %bb.3: ; CHECK-NEXT: mv a4, a3 -; CHECK-NEXT: .LBB30_4: +; CHECK-NEXT: .LBB31_4: ; CHECK-NEXT: slli a1, a2, 3 ; CHECK-NEXT: add a0, a0, a1 ; CHECK-NEXT: vsetvli zero, a4, e64, m8, ta, mu @@ -413,25 +425,25 @@ ; CHECK-NEXT: addi a5, sp, 16 ; CHECK-NEXT: vs8r.v v16, (a5) # Unknown-size Folded Spill ; CHECK-NEXT: mv a5, a2 -; CHECK-NEXT: bltu a2, a4, .LBB31_2 +; CHECK-NEXT: bltu a2, a4, .LBB32_2 ; CHECK-NEXT: # %bb.1: ; CHECK-NEXT: mv a5, a4 -; CHECK-NEXT: .LBB31_2: +; CHECK-NEXT: .LBB32_2: ; CHECK-NEXT: mv a7, a5 -; CHECK-NEXT: bltu a5, a3, .LBB31_4 +; CHECK-NEXT: bltu a5, a3, .LBB32_4 ; CHECK-NEXT: # %bb.3: ; CHECK-NEXT: mv a7, a3 -; CHECK-NEXT: .LBB31_4: +; CHECK-NEXT: .LBB32_4: ; CHECK-NEXT: li a6, 0 ; CHECK-NEXT: vl8re64.v v16, (a0) ; CHECK-NEXT: vsetvli zero, a7, e64, m8, ta, mu ; CHECK-NEXT: sub a0, a5, a3 ; CHECK-NEXT: vmv1r.v v0, v24 ; CHECK-NEXT: vse64.v v8, (a1), v0.t -; CHECK-NEXT: bltu a5, a0, .LBB31_6 +; CHECK-NEXT: bltu a5, a0, .LBB32_6 ; CHECK-NEXT: # %bb.5: ; CHECK-NEXT: mv a6, a0 -; CHECK-NEXT: .LBB31_6: +; CHECK-NEXT: .LBB32_6: ; CHECK-NEXT: li a0, 0 ; CHECK-NEXT: srli a5, a3, 3 ; CHECK-NEXT: vsetvli a7, zero, e8, mf4, ta, mu @@ -445,17 +457,17 @@ ; CHECK-NEXT: srli a5, a3, 2 ; CHECK-NEXT: sub a6, a2, a4 ; CHECK-NEXT: slli a4, a3, 4 -; CHECK-NEXT: bltu a2, a6, .LBB31_8 +; CHECK-NEXT: bltu a2, a6, .LBB32_8 ; CHECK-NEXT: # %bb.7: ; CHECK-NEXT: mv a0, a6 -; CHECK-NEXT: .LBB31_8: +; CHECK-NEXT: .LBB32_8: ; CHECK-NEXT: vsetvli a2, zero, e8, mf2, ta, mu ; CHECK-NEXT: vslidedown.vx v0, v24, a5 ; CHECK-NEXT: add a1, a1, a4 -; CHECK-NEXT: bltu a0, a3, .LBB31_10 +; CHECK-NEXT: bltu a0, a3, .LBB32_10 ; CHECK-NEXT: # %bb.9: ; CHECK-NEXT: mv a0, a3 -; CHECK-NEXT: .LBB31_10: +; CHECK-NEXT: .LBB32_10: ; CHECK-NEXT: vsetvli zero, a0, e64, m8, ta, mu ; CHECK-NEXT: vse64.v v16, (a1), v0.t ; CHECK-NEXT: csrr a0, vlenb