diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -130,7 +130,9 @@ Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break; case ISD::SIGN_EXTEND: + case ISD::VP_SIGN_EXTEND: case ISD::ZERO_EXTEND: + case ISD::VP_ZERO_EXTEND: case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break; case ISD::STRICT_FP_TO_SINT: @@ -729,7 +731,7 @@ // If the result and operand types are the same after promotion, simplify // to an in-register extension. - if (NVT == Res.getValueType()) { + if (NVT == Res.getValueType() && N->getNumOperands() == 1) { // The high bits are not guaranteed to be anything. Insert an extend. if (N->getOpcode() == ISD::SIGN_EXTEND) return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res, @@ -739,10 +741,26 @@ assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!"); return Res; } + + unsigned Opcode = N->getOpcode(); + assert(N->getNumOperands() == 3 && "Unexpected number of operands!"); + assert((Opcode == ISD::VP_SIGN_EXTEND || Opcode == ISD::VP_ZERO_EXTEND) && + "Expected VP EXTEND opcode"); + + if (Opcode == ISD::VP_SIGN_EXTEND || Opcode == ISD::VP_ZERO_EXTEND) { + Res = Opcode == ISD::VP_SIGN_EXTEND + ? SExtPromotedInteger(N->getOperand(0)) + : ZExtPromotedInteger(N->getOperand(0)); + return DAG.getNode(Opcode, dl, N->getValueType(0), Res, N->getOperand(1), + N->getOperand(2)); + } } // Otherwise, just extend the original operand all the way to the larger type. - return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); + return Opcode == ISD::VP_SIGN_EXTEND || Opcode == ISD::VP_ZERO_EXTEND + ? DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0), + N->getOperand(1), N->getOperand(2)) + : DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); } SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) { @@ -1613,6 +1631,7 @@ case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break; case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break; case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break; + case ISD::VP_SIGN_EXTEND: case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break; case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break; case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break; @@ -1630,6 +1649,7 @@ case ISD::FP16_TO_FP: case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break; case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break; + case ISD::VP_ZERO_EXTEND: case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break; case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break; case ISD::INSERT_SUBVECTOR: Res = PromoteIntOp_INSERT_SUBVECTOR(N); break; @@ -1945,9 +1965,19 @@ SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) { SDValue Op = GetPromotedInteger(N->getOperand(0)); SDLoc dl(N); - Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); - return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), - Op, DAG.getValueType(N->getOperand(0).getValueType())); + if (N->getNumOperands() == 1) { + Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); + return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), Op, + DAG.getValueType(N->getOperand(0).getValueType())); + } + + unsigned Opcode = N->getOpcode(); + assert(N->getNumOperands() == 3 && "Unexpected number of operands!"); + assert(Opcode == ISD::VP_SIGN_EXTEND && "Expected VP_SIGN_EXTEND opcode"); + + Op = SExtPromotedInteger(N->getOperand(0)); + return DAG.getNode(Opcode, dl, N->getValueType(0), Op, N->getOperand(1), + N->getOperand(2)); } SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) { @@ -2084,8 +2114,18 @@ SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) { SDLoc dl(N); SDValue Op = GetPromotedInteger(N->getOperand(0)); - Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); - return DAG.getZeroExtendInReg(Op, dl, N->getOperand(0).getValueType()); + if (N->getNumOperands() == 1) { + Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op); + return DAG.getZeroExtendInReg(Op, dl, N->getOperand(0).getValueType()); + } + + unsigned Opcode = N->getOpcode(); + assert(N->getNumOperands() == 3 && "Unexpected number of operands!"); + assert(Opcode == ISD::VP_ZERO_EXTEND && "Expected VP_ZERO_EXTEND opcode"); + + Op = ZExtPromotedInteger(N->getOperand(0)); + return DAG.getNode(Opcode, dl, N->getValueType(0), Op, N->getOperand(1), + N->getOperand(2)); } SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) { diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sext-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sext-vp.ll --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sext-vp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-sext-vp.ll @@ -4,6 +4,49 @@ ; RUN: llc -mtriple=riscv64 -mattr=+m,+v -riscv-v-vector-bits-min=128 < %s \ ; RUN: | FileCheck %s +declare <4 x i15> @llvm.vp.sext.v4i15.v4i8(<4 x i8>, <4 x i1>, i32) + +define <4 x i15> @vsext_v4i15_v4i8(<4 x i8> %va, <4 x i1> %m, i32 zeroext %evl) { +; CHECK-LABEL: vsext_v4i15_v4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, mf2, ta, mu +; CHECK-NEXT: vsext.vf2 v9, v8, v0.t +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %v = call <4 x i15> @llvm.vp.sext.v4i15.v4i8(<4 x i8> %va, <4 x i1> %m, i32 %evl) + ret <4 x i15> %v +} + +declare <4 x i16> @llvm.vp.sext.v4i16.v4i7(<4 x i7>, <4 x i1>, i32) + +define <4 x i16> @vsext_v4i16_v4i7(<4 x i7> %va, <4 x i1> %m, i32 zeroext %evl) { +; CHECK-LABEL: vsext_v4i16_v4i7: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, mu +; CHECK-NEXT: vadd.vv v8, v8, v8 +; CHECK-NEXT: vsra.vi v9, v8, 1 +; CHECK-NEXT: vsetvli zero, a0, e16, mf2, ta, mu +; CHECK-NEXT: vsext.vf2 v8, v9, v0.t +; CHECK-NEXT: ret + %v = call <4 x i16> @llvm.vp.sext.v4i16.v4i7(<4 x i7> %va, <4 x i1> %m, i32 %evl) + ret <4 x i16> %v +} + +declare <4 x i15> @llvm.vp.sext.v4i15.v4i7(<4 x i7>, <4 x i1>, i32) + +define <4 x i15> @vsext_v4i15_v4i7(<4 x i7> %va, <4 x i1> %m, i32 zeroext %evl) { +; CHECK-LABEL: vsext_v4i15_v4i7: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, mu +; CHECK-NEXT: vadd.vv v8, v8, v8 +; CHECK-NEXT: vsra.vi v9, v8, 1 +; CHECK-NEXT: vsetvli zero, a0, e16, mf2, ta, mu +; CHECK-NEXT: vsext.vf2 v8, v9, v0.t +; CHECK-NEXT: ret + %v = call <4 x i15> @llvm.vp.sext.v4i15.v4i7(<4 x i7> %va, <4 x i1> %m, i32 %evl) + ret <4 x i15> %v +} + declare <4 x i16> @llvm.vp.sext.v4i16.v4i8(<4 x i8>, <4 x i1>, i32) define <4 x i16> @vsext_v4i16_v4i8(<4 x i8> %va, <4 x i1> %m, i32 zeroext %evl) { diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-zext-vp.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-zext-vp.ll --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-zext-vp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-zext-vp.ll @@ -4,6 +4,49 @@ ; RUN: llc -mtriple=riscv64 -mattr=+m,+v -riscv-v-vector-bits-min=128 < %s \ ; RUN: | FileCheck %s +declare <4 x i15> @llvm.vp.zext.v4i15.v4i8(<4 x i8>, <4 x i1>, i32) + +define <4 x i15> @vzext_v4i15_v4i8(<4 x i8> %va, <4 x i1> %m, i32 zeroext %evl) { +; CHECK-LABEL: vzext_v4i15_v4i8: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, mf2, ta, mu +; CHECK-NEXT: vzext.vf2 v9, v8, v0.t +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %v = call <4 x i15> @llvm.vp.zext.v4i15.v4i8(<4 x i8> %va, <4 x i1> %m, i32 %evl) + ret <4 x i15> %v +} + +declare <4 x i16> @llvm.vp.zext.v4i16.v4i7(<4 x i7>, <4 x i1>, i32) + +define <4 x i16> @vzext_v4i16_v4i7(<4 x i7> %va, <4 x i1> %m, i32 zeroext %evl) { +; CHECK-LABEL: vzext_v4i16_v4i7: +; CHECK: # %bb.0: +; CHECK-NEXT: li a1, 127 +; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, mu +; CHECK-NEXT: vand.vx v9, v8, a1 +; CHECK-NEXT: vsetvli zero, a0, e16, mf2, ta, mu +; CHECK-NEXT: vzext.vf2 v8, v9, v0.t +; CHECK-NEXT: ret + %v = call <4 x i16> @llvm.vp.zext.v4i16.v4i7(<4 x i7> %va, <4 x i1> %m, i32 %evl) + ret <4 x i16> %v +} + +declare <4 x i15> @llvm.vp.zext.v4i15.v4i7(<4 x i7>, <4 x i1>, i32) + +define <4 x i15> @vzext_v4i15_v4i7(<4 x i7> %va, <4 x i1> %m, i32 zeroext %evl) { +; CHECK-LABEL: vzext_v4i15_v4i7: +; CHECK: # %bb.0: +; CHECK-NEXT: li a1, 127 +; CHECK-NEXT: vsetivli zero, 4, e8, mf4, ta, mu +; CHECK-NEXT: vand.vx v9, v8, a1 +; CHECK-NEXT: vsetvli zero, a0, e16, mf2, ta, mu +; CHECK-NEXT: vzext.vf2 v8, v9, v0.t +; CHECK-NEXT: ret + %v = call <4 x i15> @llvm.vp.zext.v4i15.v4i7(<4 x i7> %va, <4 x i1> %m, i32 %evl) + ret <4 x i15> %v +} + declare <4 x i16> @llvm.vp.zext.v4i16.v4i8(<4 x i8>, <4 x i1>, i32) define <4 x i16> @vzext_v4i16_v4i8(<4 x i8> %va, <4 x i1> %m, i32 zeroext %evl) { diff --git a/llvm/test/CodeGen/RISCV/rvv/vsext-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vsext-vp.ll --- a/llvm/test/CodeGen/RISCV/rvv/vsext-vp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vsext-vp.ll @@ -2,6 +2,49 @@ ; RUN: llc -mtriple=riscv32 -mattr=+v < %s | FileCheck %s ; RUN: llc -mtriple=riscv64 -mattr=+v < %s | FileCheck %s +declare @llvm.vp.sext.nxv2i15.nxv2i8(, , i32) + +define @vsext_nxv2i8_nxv2i15( %a, %m, i32 zeroext %vl) { +; CHECK-LABEL: vsext_nxv2i8_nxv2i15: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, mf2, ta, mu +; CHECK-NEXT: vsext.vf2 v9, v8, v0.t +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %v = call @llvm.vp.sext.nxv2i15.nxv2i8( %a, %m, i32 %vl) + ret %v +} + +declare @llvm.vp.sext.nxv2i16.nxv2i7(, , i32) + +define @vsext_nxv2i7_nxv2i16( %a, %m, i32 zeroext %vl) { +; CHECK-LABEL: vsext_nxv2i7_nxv2i16: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, mu +; CHECK-NEXT: vadd.vv v8, v8, v8 +; CHECK-NEXT: vsra.vi v9, v8, 1 +; CHECK-NEXT: vsetvli zero, a0, e16, mf2, ta, mu +; CHECK-NEXT: vsext.vf2 v8, v9, v0.t +; CHECK-NEXT: ret + %v = call @llvm.vp.sext.nxv2i16.nxv2i7( %a, %m, i32 %vl) + ret %v +} + +declare @llvm.vp.sext.nxv2i15.nxv2i7(, , i32) + +define @vsext_nxv2i7_nxv2i15( %a, %m, i32 zeroext %vl) { +; CHECK-LABEL: vsext_nxv2i7_nxv2i15: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli a1, zero, e8, mf4, ta, mu +; CHECK-NEXT: vadd.vv v8, v8, v8 +; CHECK-NEXT: vsra.vi v9, v8, 1 +; CHECK-NEXT: vsetvli zero, a0, e16, mf2, ta, mu +; CHECK-NEXT: vsext.vf2 v8, v9, v0.t +; CHECK-NEXT: ret + %v = call @llvm.vp.sext.nxv2i15.nxv2i7( %a, %m, i32 %vl) + ret %v +} + declare @llvm.vp.sext.nxv2i16.nxv2i8(, , i32) define @vsext_nxv2i8_nxv2i16( %a, %m, i32 zeroext %vl) { diff --git a/llvm/test/CodeGen/RISCV/rvv/vzext-vp.ll b/llvm/test/CodeGen/RISCV/rvv/vzext-vp.ll --- a/llvm/test/CodeGen/RISCV/rvv/vzext-vp.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vzext-vp.ll @@ -2,6 +2,49 @@ ; RUN: llc -mtriple=riscv32 -mattr=+v < %s | FileCheck %s ; RUN: llc -mtriple=riscv64 -mattr=+v < %s | FileCheck %s +declare @llvm.vp.zext.nxv2i15.nxv2i8(, , i32) + +define @vzext_nxv2i8_nxv2i15( %a, %m, i32 zeroext %vl) { +; CHECK-LABEL: vzext_nxv2i8_nxv2i15: +; CHECK: # %bb.0: +; CHECK-NEXT: vsetvli zero, a0, e16, mf2, ta, mu +; CHECK-NEXT: vzext.vf2 v9, v8, v0.t +; CHECK-NEXT: vmv1r.v v8, v9 +; CHECK-NEXT: ret + %v = call @llvm.vp.zext.nxv2i15.nxv2i8( %a, %m, i32 %vl) + ret %v +} + +declare @llvm.vp.zext.nxv2i16.nxv2i7(, , i32) + +define @vzext_nxv2i7_nxv2i16( %a, %m, i32 zeroext %vl) { +; CHECK-LABEL: vzext_nxv2i7_nxv2i16: +; CHECK: # %bb.0: +; CHECK-NEXT: li a1, 127 +; CHECK-NEXT: vsetvli a2, zero, e8, mf4, ta, mu +; CHECK-NEXT: vand.vx v9, v8, a1 +; CHECK-NEXT: vsetvli zero, a0, e16, mf2, ta, mu +; CHECK-NEXT: vzext.vf2 v8, v9, v0.t +; CHECK-NEXT: ret + %v = call @llvm.vp.zext.nxv2i16.nxv2i7( %a, %m, i32 %vl) + ret %v +} + +declare @llvm.vp.zext.nxv2i15.nxv2i7(, , i32) + +define @vzext_nxv2i7_nxv2i15( %a, %m, i32 zeroext %vl) { +; CHECK-LABEL: vzext_nxv2i7_nxv2i15: +; CHECK: # %bb.0: +; CHECK-NEXT: li a1, 127 +; CHECK-NEXT: vsetvli a2, zero, e8, mf4, ta, mu +; CHECK-NEXT: vand.vx v9, v8, a1 +; CHECK-NEXT: vsetvli zero, a0, e16, mf2, ta, mu +; CHECK-NEXT: vzext.vf2 v8, v9, v0.t +; CHECK-NEXT: ret + %v = call @llvm.vp.zext.nxv2i15.nxv2i7( %a, %m, i32 %vl) + ret %v +} + declare @llvm.vp.zext.nxv2i16.nxv2i8(, , i32) define @vzext_nxv2i8_nxv2i16( %a, %m, i32 zeroext %vl) {