Index: llvm/lib/Target/AArch64/AArch64ISelLowering.cpp =================================================================== --- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -988,6 +988,27 @@ // Lower fixed length vector operations to scalable equivalents. setOperationAction(ISD::LOAD, VT, Custom); setOperationAction(ISD::STORE, VT, Custom); + + setOperationAction(ISD::ADD, VT, Custom); + setOperationAction(ISD::AND, VT, Custom); + setOperationAction(ISD::FADD, VT, Custom); + setOperationAction(ISD::FDIV, VT, Custom); + setOperationAction(ISD::FMAXNUM, VT, Custom); + setOperationAction(ISD::FMINNUM, VT, Custom); + setOperationAction(ISD::FMUL, VT, Custom); + setOperationAction(ISD::FSUB, VT, Custom); + setOperationAction(ISD::MUL, VT, Custom); + setOperationAction(ISD::OR, VT, Custom); + setOperationAction(ISD::SETCC, VT, Custom); + setOperationAction(ISD::SHL, VT, Custom); + setOperationAction(ISD::SMAX, VT, Custom); + setOperationAction(ISD::SMIN, VT, Custom); + setOperationAction(ISD::SRA, VT, Custom); + setOperationAction(ISD::SRL, VT, Custom); + setOperationAction(ISD::SUB, VT, Custom); + setOperationAction(ISD::UMAX, VT, Custom); + setOperationAction(ISD::UMIN, VT, Custom); + setOperationAction(ISD::XOR, VT, Custom); } void AArch64TargetLowering::addDRTypeForNEON(MVT VT) { @@ -3365,6 +3386,107 @@ Store->isTruncatingStore()); } +static SDValue LowerFixedVector2OpToSVE(SelectionDAG &DAG, SDValue Op) { + SDLoc DL(Op); + EVT VT = Op.getValueType(); + EVT ContainerVT = getContainerForFixedVector(DAG, VT); + + int IntID; + switch (Op.getOpcode()) { + default: llvm_unreachable("unimplemented opcode"); + case ISD::ADD: IntID = Intrinsic::aarch64_sve_add; break; + case ISD::AND: IntID = Intrinsic::aarch64_sve_and; break; + case ISD::FADD: IntID = Intrinsic::aarch64_sve_fadd; break; + case ISD::FDIV: IntID = Intrinsic::aarch64_sve_fdiv; break; + case ISD::FMAXNUM: IntID = Intrinsic::aarch64_sve_fmaxnm; break; + case ISD::FMINNUM: IntID = Intrinsic::aarch64_sve_fminnm; break; + case ISD::FMUL: IntID = Intrinsic::aarch64_sve_fmul; break; + case ISD::FSUB: IntID = Intrinsic::aarch64_sve_fsub; break; + case ISD::MUL: IntID = Intrinsic::aarch64_sve_mul; break; + case ISD::OR: IntID = Intrinsic::aarch64_sve_orr; break; + case ISD::SHL: IntID = Intrinsic::aarch64_sve_lsl; break; + case ISD::SMAX: IntID = Intrinsic::aarch64_sve_smax; break; + case ISD::SMIN: IntID = Intrinsic::aarch64_sve_smin; break; + case ISD::SRA: IntID = Intrinsic::aarch64_sve_asr; break; + case ISD::SRL: IntID = Intrinsic::aarch64_sve_lsr; break; + case ISD::SUB: IntID = Intrinsic::aarch64_sve_sub; break; + case ISD::UMAX: IntID = Intrinsic::aarch64_sve_umax; break; + case ISD::UMIN: IntID = Intrinsic::aarch64_sve_umin; break; + case ISD::XOR: IntID = Intrinsic::aarch64_sve_eor; break; + } + + auto Op1 = ConvertToScalableVector(DAG, ContainerVT, Op->getOperand(0)); + auto Op2 = ConvertToScalableVector(DAG, ContainerVT, Op->getOperand(1)); + auto Pg = CreatePredicateForFixedVector(DAG, DL, VT); + + auto Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, ContainerVT, + DAG.getConstant(IntID, DL, MVT::i64), Pg, Op1, Op2); + + return ConvertFromScalableVector(DAG, VT, Res); +} + +static SDValue LowerFixedVectorSetccToSVE(SelectionDAG &DAG, SDValue Op) { + SDLoc DL(Op); + EVT InVT = Op.getOperand(0).getValueType(); + EVT ContainerVT = getContainerForFixedVector(DAG, InVT); + + // HACK + if (!InVT.isInteger()) + return SDValue(); + + auto Op1 = ConvertToScalableVector(DAG, ContainerVT, Op.getOperand(0)); + auto Op2 = ConvertToScalableVector(DAG, ContainerVT, Op.getOperand(1)); + auto Pg = CreatePredicateForFixedVector(DAG, DL, InVT); + + int IntID; + switch (cast(Op.getOperand(2))->get()) { + default: + llvm_unreachable("Unknown condition code!"); + case ISD::SETNE: + IntID = Intrinsic::aarch64_sve_cmpne; + break; + + case ISD::SETEQ: + IntID = Intrinsic::aarch64_sve_cmpeq; + break; + + case ISD::SETLT: + std::swap(Op1, Op2); + LLVM_FALLTHROUGH; + case ISD::SETGT: + IntID = Intrinsic::aarch64_sve_cmpgt; + break; + + case ISD::SETLE: + std::swap(Op1, Op2); + LLVM_FALLTHROUGH; + case ISD::SETGE: + IntID = Intrinsic::aarch64_sve_cmpge; + break; + + case ISD::SETULT: + std::swap(Op1, Op2); + LLVM_FALLTHROUGH; + case ISD::SETUGT: + IntID = Intrinsic::aarch64_sve_cmphi; + break; + + case ISD::SETULE: + std::swap(Op1, Op2); + LLVM_FALLTHROUGH; + case ISD::SETUGE: + IntID = Intrinsic::aarch64_sve_cmphs; + break; + } + + auto Cmp = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Pg.getValueType(), + DAG.getConstant(IntID, DL, MVT::i64), Pg, Op1, Op2); + + auto Promote = DAG.getBoolExtOrTrunc(Cmp, DL, ContainerVT, InVT); + auto Extract = ConvertFromScalableVector(DAG, InVT, Promote); + return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Extract); +} + SDValue AArch64TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { if (!useSVEForFixedLengthVectors()) @@ -3431,6 +3553,41 @@ return Op; // Use Neon. return LowerFixedVectorStoreToSVE(DAG, Op); } + + case ISD::ADD: + case ISD::AND: + case ISD::FADD: + case ISD::FDIV: + case ISD::FMAXNUM: + case ISD::FMINNUM: + case ISD::FMUL: + case ISD::FSUB: + case ISD::MUL: + case ISD::OR: + case ISD::SHL: + case ISD::SMAX: + case ISD::SMIN: + case ISD::SRA: + case ISD::SRL: + case ISD::SUB: + case ISD::UMAX: + case ISD::UMIN: + case ISD::XOR: { + EVT VT = Op.getValueType(); + if (!VT.isFixedVector()) + return LowerOperationDefault(Op, DAG); + + if (!UseSVEForAll && VT.getSizeInBits() <= 128) + return Op; // Use Neon. + return LowerFixedVector2OpToSVE(DAG, Op); + } + + case ISD::SETCC: { + if (!Op.getValueType().isFixedVector()) + return LowerOperationDefault(Op, DAG); + + return LowerFixedVectorSetccToSVE(DAG, Op); + } } } @@ -10300,6 +10457,9 @@ if (!VT.isVector() || !DAG.getTargetLoweringInfo().isTypeLegal(VT)) return SDValue(); + if (VT.isFixedVector() && VT.getSizeInBits() > 128) + return SDValue(); + if (VT.isScalableVector()) return performSVEAndCombine(N, DCI); Index: llvm/lib/Target/AArch64/SVEInstrFormats.td =================================================================== --- llvm/lib/Target/AArch64/SVEInstrFormats.td +++ llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -3578,6 +3578,33 @@ (!cast(NAME # _S) ZPR32:$Zd, PPRAny:$Pg, cpy_imm8_opt_lsl_i32:$imm), 1>; def : InstAlias<"mov $Zd, $Pg/z, $imm", (!cast(NAME # _D) ZPR64:$Zd, PPRAny:$Pg, cpy_imm8_opt_lsl_i64:$imm), 1>; + + def : Pat<(nxv16i8 (anyext (nxv16i1 PPR:$Op))), + (!cast(NAME # _B) PPR:$Op, 1, 0)>; + def : Pat<(nxv8i16 (anyext (nxv8i1 PPR:$Op))), + (!cast(NAME # _H) PPR:$Op, 1, 0)>; + def : Pat<(nxv4i32 (anyext (nxv4i1 PPR:$Op))), + (!cast(NAME # _S) PPR:$Op, 1, 0)>; + def : Pat<(nxv2i64 (anyext (nxv2i1 PPR:$Op))), + (!cast(NAME # _D) PPR:$Op, 1, 0)>; + + def : Pat<(nxv16i8 (sext (nxv16i1 PPR:$Op))), + (!cast(NAME # _B) PPR:$Op, -1, 0)>; + def : Pat<(nxv8i16 (sext (nxv8i1 PPR:$Op))), + (!cast(NAME # _H) PPR:$Op, -1, 0)>; + def : Pat<(nxv4i32 (sext (nxv4i1 PPR:$Op))), + (!cast(NAME # _S) PPR:$Op, -1, 0)>; + def : Pat<(nxv2i64 (sext (nxv2i1 PPR:$Op))), + (!cast(NAME # _D) PPR:$Op, -1, 0)>; + + def : Pat<(nxv16i8 (zext (nxv16i1 PPR:$Op))), + (!cast(NAME # _B) PPR:$Op, 1, 0)>; + def : Pat<(nxv8i16 (zext (nxv8i1 PPR:$Op))), + (!cast(NAME # _H) PPR:$Op, 1, 0)>; + def : Pat<(nxv4i32 (zext (nxv4i1 PPR:$Op))), + (!cast(NAME # _S) PPR:$Op, 1, 0)>; + def : Pat<(nxv2i64 (zext (nxv2i1 PPR:$Op))), + (!cast(NAME # _D) PPR:$Op, 1, 0)>; } //===----------------------------------------------------------------------===//