diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -252,8 +252,7 @@ if (Subtarget->hasLS64()) { addRegisterClass(MVT::i64x8, &AArch64::GPR64x8ClassRegClass); - setOperationAction(ISD::LOAD, MVT::i64x8, Custom); - setOperationAction(ISD::STORE, MVT::i64x8, Custom); + setOperationAction({ISD::LOAD, ISD::STORE}, MVT::i64x8, Custom); } if (Subtarget->hasFPARMv8()) { @@ -330,130 +329,70 @@ computeRegisterProperties(Subtarget->getRegisterInfo()); // Provide all sorts of operation actions - setOperationAction(ISD::GlobalAddress, MVT::i64, Custom); - setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); - setOperationAction(ISD::SETCC, MVT::i32, Custom); - setOperationAction(ISD::SETCC, MVT::i64, Custom); - setOperationAction(ISD::SETCC, MVT::f16, Custom); - setOperationAction(ISD::SETCC, MVT::f32, Custom); - setOperationAction(ISD::SETCC, MVT::f64, Custom); - setOperationAction(ISD::STRICT_FSETCC, MVT::f16, Custom); - setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Custom); - setOperationAction(ISD::STRICT_FSETCC, MVT::f64, Custom); - setOperationAction(ISD::STRICT_FSETCCS, MVT::f16, Custom); - setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Custom); - setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Custom); - setOperationAction(ISD::BITREVERSE, MVT::i32, Legal); - setOperationAction(ISD::BITREVERSE, MVT::i64, Legal); + setOperationAction({ISD::GlobalAddress, ISD::GlobalTLSAddress}, MVT::i64, + Custom); + setOperationAction( + ISD::SETCC, {MVT::i32, MVT::i64, MVT::f16, MVT::f32, MVT::f64}, Custom); + setOperationAction({ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS}, + {MVT::f16, MVT::f32, MVT::f64}, Custom); + setOperationAction(ISD::BITREVERSE, {MVT::i32, MVT::i64}, Legal); setOperationAction(ISD::BRCOND, MVT::Other, Custom); - setOperationAction(ISD::BR_CC, MVT::i32, Custom); - setOperationAction(ISD::BR_CC, MVT::i64, Custom); - setOperationAction(ISD::BR_CC, MVT::f16, Custom); - setOperationAction(ISD::BR_CC, MVT::f32, Custom); - setOperationAction(ISD::BR_CC, MVT::f64, Custom); - setOperationAction(ISD::SELECT, MVT::i32, Custom); - setOperationAction(ISD::SELECT, MVT::i64, Custom); - setOperationAction(ISD::SELECT, MVT::f16, Custom); - setOperationAction(ISD::SELECT, MVT::f32, Custom); - setOperationAction(ISD::SELECT, MVT::f64, Custom); - setOperationAction(ISD::SELECT_CC, MVT::i32, Custom); - setOperationAction(ISD::SELECT_CC, MVT::i64, Custom); - setOperationAction(ISD::SELECT_CC, MVT::f16, Custom); - setOperationAction(ISD::SELECT_CC, MVT::f32, Custom); - setOperationAction(ISD::SELECT_CC, MVT::f64, Custom); + setOperationAction({ISD::BR_CC, ISD::SELECT, ISD::SELECT_CC}, + {MVT::i32, MVT::i64, MVT::f16, MVT::f32, MVT::f64}, + Custom); setOperationAction(ISD::BR_JT, MVT::Other, Custom); setOperationAction(ISD::JumpTable, MVT::i64, Custom); - setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom); - setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom); - setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom); + setOperationAction({ISD::SHL_PARTS, ISD::SRA_PARTS, ISD::SRL_PARTS}, MVT::i64, + Custom); - setOperationAction(ISD::FREM, MVT::f32, Expand); - setOperationAction(ISD::FREM, MVT::f64, Expand); - setOperationAction(ISD::FREM, MVT::f80, Expand); + setOperationAction(ISD::FREM, {MVT::f32, MVT::f64, MVT::f80}, Expand); setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand); // Custom lowering hooks are needed for XOR // to fold it into CSINC/CSINV. - setOperationAction(ISD::XOR, MVT::i32, Custom); - setOperationAction(ISD::XOR, MVT::i64, Custom); + setOperationAction(ISD::XOR, {MVT::i32, MVT::i64}, Custom); // Virtually no operation on f128 is legal, but LLVM can't expand them when // there's a valid register class, so we need custom operations in most cases. setOperationAction(ISD::FABS, MVT::f128, Expand); setOperationAction(ISD::FADD, MVT::f128, LibCall); - setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand); - setOperationAction(ISD::FCOS, MVT::f128, Expand); + setOperationAction({ISD::FCOPYSIGN, ISD::FCOS}, MVT::f128, Expand); setOperationAction(ISD::FDIV, MVT::f128, LibCall); setOperationAction(ISD::FMA, MVT::f128, Expand); setOperationAction(ISD::FMUL, MVT::f128, LibCall); - setOperationAction(ISD::FNEG, MVT::f128, Expand); - setOperationAction(ISD::FPOW, MVT::f128, Expand); - setOperationAction(ISD::FREM, MVT::f128, Expand); - setOperationAction(ISD::FRINT, MVT::f128, Expand); - setOperationAction(ISD::FSIN, MVT::f128, Expand); - setOperationAction(ISD::FSINCOS, MVT::f128, Expand); - setOperationAction(ISD::FSQRT, MVT::f128, Expand); + setOperationAction({ISD::FNEG, ISD::FPOW, ISD::FREM, ISD::FRINT, ISD::FSIN, + ISD::FSINCOS, ISD::FSQRT}, + MVT::f128, Expand); setOperationAction(ISD::FSUB, MVT::f128, LibCall); setOperationAction(ISD::FTRUNC, MVT::f128, Expand); - setOperationAction(ISD::SETCC, MVT::f128, Custom); - setOperationAction(ISD::STRICT_FSETCC, MVT::f128, Custom); - setOperationAction(ISD::STRICT_FSETCCS, MVT::f128, Custom); - setOperationAction(ISD::BR_CC, MVT::f128, Custom); - setOperationAction(ISD::SELECT, MVT::f128, Custom); - setOperationAction(ISD::SELECT_CC, MVT::f128, Custom); - setOperationAction(ISD::FP_EXTEND, MVT::f128, Custom); + setOperationAction({ISD::SETCC, ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, + ISD::BR_CC, ISD::SELECT, ISD::SELECT_CC, ISD::FP_EXTEND}, + MVT::f128, Custom); // FIXME: f128 FMINIMUM and FMAXIMUM (including STRICT versions) currently // aren't handled. // Lowering for many of the conversions is actually specified by the non-f128 // type. The LowerXXX function will be trivial when f128 isn't involved. - setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom); - setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom); - setOperationAction(ISD::FP_TO_SINT, MVT::i128, Custom); - setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom); - setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom); - setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i128, Custom); - setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom); - setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom); - setOperationAction(ISD::FP_TO_UINT, MVT::i128, Custom); - setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom); - setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Custom); - setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i128, Custom); - setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom); - setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom); - setOperationAction(ISD::SINT_TO_FP, MVT::i128, Custom); - setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom); - setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom); - setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i128, Custom); - setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom); - setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom); - setOperationAction(ISD::UINT_TO_FP, MVT::i128, Custom); - setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Custom); - setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Custom); - setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i128, Custom); - setOperationAction(ISD::FP_ROUND, MVT::f16, Custom); - setOperationAction(ISD::FP_ROUND, MVT::f32, Custom); - setOperationAction(ISD::FP_ROUND, MVT::f64, Custom); - setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Custom); - setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Custom); - setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Custom); - - setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i32, Custom); - setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i64, Custom); - setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i32, Custom); - setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i64, Custom); + setOperationAction({ISD::FP_TO_SINT, ISD::STRICT_FP_TO_SINT, ISD::FP_TO_UINT, + ISD::STRICT_FP_TO_UINT, ISD::SINT_TO_FP, + ISD::STRICT_SINT_TO_FP, ISD::UINT_TO_FP, + ISD::STRICT_UINT_TO_FP}, + {MVT::i32, MVT::i64, MVT::i128}, Custom); + setOperationAction({ISD::FP_ROUND, ISD::STRICT_FP_ROUND}, + {MVT::f16, MVT::f32, MVT::f64}, Custom); + + setOperationAction({ISD::FP_TO_UINT_SAT, ISD::FP_TO_SINT_SAT}, + {MVT::i32, MVT::i64}, Custom); // Variable arguments. - setOperationAction(ISD::VASTART, MVT::Other, Custom); - setOperationAction(ISD::VAARG, MVT::Other, Custom); - setOperationAction(ISD::VACOPY, MVT::Other, Custom); + setOperationAction({ISD::VASTART, ISD::VAARG, ISD::VACOPY}, MVT::Other, + Custom); setOperationAction(ISD::VAEND, MVT::Other, Expand); // Variable-sized objects. - setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); - setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); + setOperationAction({ISD::STACKSAVE, ISD::STACKRESTORE}, MVT::Other, Expand); if (Subtarget->isTargetWindows()) setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Custom); @@ -467,82 +406,42 @@ setOperationAction(ISD::BlockAddress, MVT::i64, Custom); // Add/Sub overflow ops with MVT::Glues are lowered to NZCV dependences. - setOperationAction(ISD::ADDC, MVT::i32, Custom); - setOperationAction(ISD::ADDE, MVT::i32, Custom); - setOperationAction(ISD::SUBC, MVT::i32, Custom); - setOperationAction(ISD::SUBE, MVT::i32, Custom); - setOperationAction(ISD::ADDC, MVT::i64, Custom); - setOperationAction(ISD::ADDE, MVT::i64, Custom); - setOperationAction(ISD::SUBC, MVT::i64, Custom); - setOperationAction(ISD::SUBE, MVT::i64, Custom); + setOperationAction({ISD::ADDC, ISD::ADDE, ISD::SUBC, ISD::SUBE}, + {MVT::i32, MVT::i64}, Custom); // AArch64 lacks both left-rotate and popcount instructions. - setOperationAction(ISD::ROTL, MVT::i32, Expand); - setOperationAction(ISD::ROTL, MVT::i64, Expand); - for (MVT VT : MVT::fixedlen_vector_valuetypes()) { - setOperationAction(ISD::ROTL, VT, Expand); - setOperationAction(ISD::ROTR, VT, Expand); - } + setOperationAction(ISD::ROTL, {MVT::i32, MVT::i64}, Expand); + for (MVT VT : MVT::fixedlen_vector_valuetypes()) + setOperationAction({ISD::ROTL, ISD::ROTR}, VT, Expand); // AArch64 doesn't have i32 MULH{S|U}. - setOperationAction(ISD::MULHU, MVT::i32, Expand); - setOperationAction(ISD::MULHS, MVT::i32, Expand); + setOperationAction({ISD::MULHU, ISD::MULHS}, MVT::i32, Expand); // AArch64 doesn't have {U|S}MUL_LOHI. - setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand); - setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand); + setOperationAction({ISD::UMUL_LOHI, ISD::SMUL_LOHI}, MVT::i64, Expand); - setOperationAction(ISD::CTPOP, MVT::i32, Custom); - setOperationAction(ISD::CTPOP, MVT::i64, Custom); - setOperationAction(ISD::CTPOP, MVT::i128, Custom); + setOperationAction(ISD::CTPOP, {MVT::i32, MVT::i64, MVT::i128}, Custom); - setOperationAction(ISD::ABS, MVT::i32, Custom); - setOperationAction(ISD::ABS, MVT::i64, Custom); + setOperationAction(ISD::ABS, {MVT::i32, MVT::i64}, Custom); - setOperationAction(ISD::SDIVREM, MVT::i32, Expand); - setOperationAction(ISD::SDIVREM, MVT::i64, Expand); - for (MVT VT : MVT::fixedlen_vector_valuetypes()) { - setOperationAction(ISD::SDIVREM, VT, Expand); - setOperationAction(ISD::UDIVREM, VT, Expand); - } - setOperationAction(ISD::SREM, MVT::i32, Expand); - setOperationAction(ISD::SREM, MVT::i64, Expand); - setOperationAction(ISD::UDIVREM, MVT::i32, Expand); - setOperationAction(ISD::UDIVREM, MVT::i64, Expand); - setOperationAction(ISD::UREM, MVT::i32, Expand); - setOperationAction(ISD::UREM, MVT::i64, Expand); + setOperationAction(ISD::SDIVREM, {MVT::i32, MVT::i64}, Expand); + for (MVT VT : MVT::fixedlen_vector_valuetypes()) + setOperationAction({ISD::SDIVREM, ISD::UDIVREM}, VT, Expand); + setOperationAction({ISD::SREM, ISD::UDIVREM, ISD::UREM}, {MVT::i32, MVT::i64}, + Expand); // Custom lower Add/Sub/Mul with overflow. - setOperationAction(ISD::SADDO, MVT::i32, Custom); - setOperationAction(ISD::SADDO, MVT::i64, Custom); - setOperationAction(ISD::UADDO, MVT::i32, Custom); - setOperationAction(ISD::UADDO, MVT::i64, Custom); - setOperationAction(ISD::SSUBO, MVT::i32, Custom); - setOperationAction(ISD::SSUBO, MVT::i64, Custom); - setOperationAction(ISD::USUBO, MVT::i32, Custom); - setOperationAction(ISD::USUBO, MVT::i64, Custom); - setOperationAction(ISD::SMULO, MVT::i32, Custom); - setOperationAction(ISD::SMULO, MVT::i64, Custom); - setOperationAction(ISD::UMULO, MVT::i32, Custom); - setOperationAction(ISD::UMULO, MVT::i64, Custom); - - setOperationAction(ISD::ADDCARRY, MVT::i32, Custom); - setOperationAction(ISD::ADDCARRY, MVT::i64, Custom); - setOperationAction(ISD::SUBCARRY, MVT::i32, Custom); - setOperationAction(ISD::SUBCARRY, MVT::i64, Custom); - setOperationAction(ISD::SADDO_CARRY, MVT::i32, Custom); - setOperationAction(ISD::SADDO_CARRY, MVT::i64, Custom); - setOperationAction(ISD::SSUBO_CARRY, MVT::i32, Custom); - setOperationAction(ISD::SSUBO_CARRY, MVT::i64, Custom); - - setOperationAction(ISD::FSIN, MVT::f32, Expand); - setOperationAction(ISD::FSIN, MVT::f64, Expand); - setOperationAction(ISD::FCOS, MVT::f32, Expand); - setOperationAction(ISD::FCOS, MVT::f64, Expand); - setOperationAction(ISD::FPOW, MVT::f32, Expand); - setOperationAction(ISD::FPOW, MVT::f64, Expand); - setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); - setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); + setOperationAction( + {ISD::SADDO, ISD::UADDO, ISD::SSUBO, ISD::USUBO, ISD::SMULO, ISD::UMULO}, + {MVT::i32, MVT::i64}, Custom); + + setOperationAction( + {ISD::ADDCARRY, ISD::SUBCARRY, ISD::SADDO_CARRY, ISD::SSUBO_CARRY}, + {MVT::i32, MVT::i64}, Custom); + + setOperationAction({ISD::FSIN, ISD::FCOS, ISD::FPOW}, {MVT::f32, MVT::f64}, + Expand); + setOperationAction(ISD::FCOPYSIGN, {MVT::f64, MVT::f32}, Custom); if (Subtarget->hasFullFP16()) setOperationAction(ISD::FCOPYSIGN, MVT::f16, Custom); else @@ -556,82 +455,56 @@ ISD::STRICT_FSIN, ISD::STRICT_FEXP, ISD::STRICT_FEXP2, ISD::STRICT_FLOG, ISD::STRICT_FLOG2, ISD::STRICT_FLOG10}) { setOperationAction(Op, MVT::f16, Promote); - setOperationAction(Op, MVT::v4f16, Expand); - setOperationAction(Op, MVT::v8f16, Expand); + setOperationAction(Op, {MVT::v4f16, MVT::v8f16}, Expand); } if (!Subtarget->hasFullFP16()) { - for (auto Op : - {ISD::SELECT, ISD::SELECT_CC, ISD::SETCC, - ISD::BR_CC, ISD::FADD, ISD::FSUB, - ISD::FMUL, ISD::FDIV, ISD::FMA, - ISD::FNEG, ISD::FABS, ISD::FCEIL, - ISD::FSQRT, ISD::FFLOOR, ISD::FNEARBYINT, - ISD::FRINT, ISD::FROUND, ISD::FROUNDEVEN, - ISD::FTRUNC, ISD::FMINNUM, ISD::FMAXNUM, - ISD::FMINIMUM, ISD::FMAXIMUM, ISD::STRICT_FADD, - ISD::STRICT_FSUB, ISD::STRICT_FMUL, ISD::STRICT_FDIV, - ISD::STRICT_FMA, ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR, - ISD::STRICT_FSQRT, ISD::STRICT_FRINT, ISD::STRICT_FNEARBYINT, - ISD::STRICT_FROUND, ISD::STRICT_FTRUNC, ISD::STRICT_FROUNDEVEN, - ISD::STRICT_FMINNUM, ISD::STRICT_FMAXNUM, ISD::STRICT_FMINIMUM, - ISD::STRICT_FMAXIMUM}) - setOperationAction(Op, MVT::f16, Promote); + setOperationAction( + {ISD::SELECT, ISD::SELECT_CC, ISD::SETCC, + ISD::BR_CC, ISD::FADD, ISD::FSUB, + ISD::FMUL, ISD::FDIV, ISD::FMA, + ISD::FNEG, ISD::FABS, ISD::FCEIL, + ISD::FSQRT, ISD::FFLOOR, ISD::FNEARBYINT, + ISD::FRINT, ISD::FROUND, ISD::FROUNDEVEN, + ISD::FTRUNC, ISD::FMINNUM, ISD::FMAXNUM, + ISD::FMINIMUM, ISD::FMAXIMUM, ISD::STRICT_FADD, + ISD::STRICT_FSUB, ISD::STRICT_FMUL, ISD::STRICT_FDIV, + ISD::STRICT_FMA, ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR, + ISD::STRICT_FSQRT, ISD::STRICT_FRINT, ISD::STRICT_FNEARBYINT, + ISD::STRICT_FROUND, ISD::STRICT_FTRUNC, ISD::STRICT_FROUNDEVEN, + ISD::STRICT_FMINNUM, ISD::STRICT_FMAXNUM, ISD::STRICT_FMINIMUM, + ISD::STRICT_FMAXIMUM}, + MVT::f16, Promote); // Round-to-integer need custom lowering for fp16, as Promote doesn't work // because the result type is integer. - for (auto Op : {ISD::STRICT_LROUND, ISD::STRICT_LLROUND, ISD::STRICT_LRINT, - ISD::STRICT_LLRINT}) - setOperationAction(Op, MVT::f16, Custom); + setOperationAction({ISD::STRICT_LROUND, ISD::STRICT_LLROUND, + ISD::STRICT_LRINT, ISD::STRICT_LLRINT}, + MVT::f16, Custom); // promote v4f16 to v4f32 when that is known to be safe. - setOperationAction(ISD::FADD, MVT::v4f16, Promote); - setOperationAction(ISD::FSUB, MVT::v4f16, Promote); - setOperationAction(ISD::FMUL, MVT::v4f16, Promote); - setOperationAction(ISD::FDIV, MVT::v4f16, Promote); - AddPromotedToType(ISD::FADD, MVT::v4f16, MVT::v4f32); - AddPromotedToType(ISD::FSUB, MVT::v4f16, MVT::v4f32); - AddPromotedToType(ISD::FMUL, MVT::v4f16, MVT::v4f32); - AddPromotedToType(ISD::FDIV, MVT::v4f16, MVT::v4f32); - - setOperationAction(ISD::FABS, MVT::v4f16, Expand); - setOperationAction(ISD::FNEG, MVT::v4f16, Expand); - setOperationAction(ISD::FROUND, MVT::v4f16, Expand); - setOperationAction(ISD::FROUNDEVEN, MVT::v4f16, Expand); - setOperationAction(ISD::FMA, MVT::v4f16, Expand); - setOperationAction(ISD::SETCC, MVT::v4f16, Expand); - setOperationAction(ISD::BR_CC, MVT::v4f16, Expand); - setOperationAction(ISD::SELECT, MVT::v4f16, Expand); - setOperationAction(ISD::SELECT_CC, MVT::v4f16, Expand); - setOperationAction(ISD::FTRUNC, MVT::v4f16, Expand); - setOperationAction(ISD::FCOPYSIGN, MVT::v4f16, Expand); - setOperationAction(ISD::FFLOOR, MVT::v4f16, Expand); - setOperationAction(ISD::FCEIL, MVT::v4f16, Expand); - setOperationAction(ISD::FRINT, MVT::v4f16, Expand); - setOperationAction(ISD::FNEARBYINT, MVT::v4f16, Expand); - setOperationAction(ISD::FSQRT, MVT::v4f16, Expand); - - setOperationAction(ISD::FABS, MVT::v8f16, Expand); - setOperationAction(ISD::FADD, MVT::v8f16, Expand); - setOperationAction(ISD::FCEIL, MVT::v8f16, Expand); - setOperationAction(ISD::FCOPYSIGN, MVT::v8f16, Expand); - setOperationAction(ISD::FDIV, MVT::v8f16, Expand); - setOperationAction(ISD::FFLOOR, MVT::v8f16, Expand); - setOperationAction(ISD::FMA, MVT::v8f16, Expand); - setOperationAction(ISD::FMUL, MVT::v8f16, Expand); - setOperationAction(ISD::FNEARBYINT, MVT::v8f16, Expand); - setOperationAction(ISD::FNEG, MVT::v8f16, Expand); - setOperationAction(ISD::FROUND, MVT::v8f16, Expand); - setOperationAction(ISD::FROUNDEVEN, MVT::v8f16, Expand); - setOperationAction(ISD::FRINT, MVT::v8f16, Expand); - setOperationAction(ISD::FSQRT, MVT::v8f16, Expand); - setOperationAction(ISD::FSUB, MVT::v8f16, Expand); - setOperationAction(ISD::FTRUNC, MVT::v8f16, Expand); - setOperationAction(ISD::SETCC, MVT::v8f16, Expand); - setOperationAction(ISD::BR_CC, MVT::v8f16, Expand); - setOperationAction(ISD::SELECT, MVT::v8f16, Expand); - setOperationAction(ISD::SELECT_CC, MVT::v8f16, Expand); - setOperationAction(ISD::FP_EXTEND, MVT::v8f16, Expand); + setOperationAction({ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV}, MVT::v4f16, + Promote); + AddPromotedToType(ISD::FADD, MVT::v4f16, MVT::v4f32); + AddPromotedToType(ISD::FSUB, MVT::v4f16, MVT::v4f32); + AddPromotedToType(ISD::FMUL, MVT::v4f16, MVT::v4f32); + AddPromotedToType(ISD::FDIV, MVT::v4f16, MVT::v4f32); + + setOperationAction({ISD::FABS, ISD::FNEG, ISD::FROUND, ISD::FROUNDEVEN, + ISD::FMA, ISD::SETCC, ISD::BR_CC, ISD::SELECT, + ISD::SELECT_CC, ISD::FTRUNC, ISD::FCOPYSIGN, + ISD::FFLOOR, ISD::FCEIL, ISD::FRINT, ISD::FNEARBYINT, + ISD::FSQRT}, + MVT::v4f16, Expand); + + setOperationAction({ISD::FABS, ISD::FADD, ISD::FCEIL, + ISD::FCOPYSIGN, ISD::FDIV, ISD::FFLOOR, + ISD::FMA, ISD::FMUL, ISD::FNEARBYINT, + ISD::FNEG, ISD::FROUND, ISD::FROUNDEVEN, + ISD::FRINT, ISD::FSQRT, ISD::FSUB, + ISD::FTRUNC, ISD::SETCC, ISD::BR_CC, + ISD::SELECT, ISD::SELECT_CC, ISD::FP_EXTEND}, + MVT::v8f16, Expand); } // AArch64 has implementations of a lot of rounding-like FP operations. @@ -646,8 +519,7 @@ ISD::STRICT_FROUND, ISD::STRICT_FMINNUM, ISD::STRICT_FMAXNUM, ISD::STRICT_FMINIMUM, ISD::STRICT_FMAXIMUM, ISD::STRICT_LROUND, ISD::STRICT_LLROUND, ISD::STRICT_LRINT, ISD::STRICT_LLRINT}) { - for (MVT Ty : {MVT::f32, MVT::f64}) - setOperationAction(Op, Ty, Legal); + setOperationAction(Op, {MVT::f32, MVT::f64}, Legal); if (Subtarget->hasFullFP16()) setOperationAction(Op, MVT::f16, Legal); } @@ -655,15 +527,13 @@ // Basic strict FP operations are legal for (auto Op : {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL, ISD::STRICT_FDIV, ISD::STRICT_FMA, ISD::STRICT_FSQRT}) { - for (MVT Ty : {MVT::f32, MVT::f64}) - setOperationAction(Op, Ty, Legal); + setOperationAction(Op, {MVT::f32, MVT::f64}, Legal); if (Subtarget->hasFullFP16()) setOperationAction(Op, MVT::f16, Legal); } // Strict conversion to a larger type is legal - for (auto VT : {MVT::f32, MVT::f64}) - setOperationAction(ISD::STRICT_FP_EXTEND, VT, Legal); + setOperationAction(ISD::STRICT_FP_EXTEND, {MVT::f32, MVT::f64}, Legal); setOperationAction(ISD::PREFETCH, MVT::Other, Custom); @@ -671,39 +541,19 @@ setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom); setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, Custom); - setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom); - setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, Custom); - setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom); - setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i64, Custom); + setOperationAction({ISD::ATOMIC_LOAD_SUB, ISD::ATOMIC_LOAD_AND}, + {MVT::i32, MVT::i64}, Custom); // Generate outline atomics library calls only if LSE was not specified for // subtarget if (Subtarget->outlineAtomics() && !Subtarget->hasLSE()) { - setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i8, LibCall); - setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i16, LibCall); - setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, LibCall); - setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, LibCall); - setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, LibCall); - setOperationAction(ISD::ATOMIC_SWAP, MVT::i8, LibCall); - setOperationAction(ISD::ATOMIC_SWAP, MVT::i16, LibCall); - setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, LibCall); - setOperationAction(ISD::ATOMIC_SWAP, MVT::i64, LibCall); - setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i8, LibCall); - setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i16, LibCall); - setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, LibCall); - setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i64, LibCall); - setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i8, LibCall); - setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i16, LibCall); - setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, LibCall); - setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i64, LibCall); - setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i8, LibCall); - setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i16, LibCall); - setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i32, LibCall); - setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i64, LibCall); - setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i8, LibCall); - setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i16, LibCall); - setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, LibCall); - setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i64, LibCall); + setOperationAction(ISD::ATOMIC_CMP_SWAP, + {MVT::i8, MVT::i16, MVT::i32, MVT::i64, MVT::i128}, + LibCall); + setOperationAction({ISD::ATOMIC_SWAP, ISD::ATOMIC_LOAD_ADD, + ISD::ATOMIC_LOAD_OR, ISD::ATOMIC_LOAD_CLR, + ISD::ATOMIC_LOAD_XOR}, + {MVT::i8, MVT::i16, MVT::i32, MVT::i64}, LibCall); #define LCALLNAMES(A, B, N) \ setLibcallName(A##N##_RELAX, #B #N "_relax"); \ setLibcallName(A##N##_ACQ, #B #N "_acq"); \ @@ -728,26 +578,22 @@ } // 128-bit loads and stores can be done without expanding - setOperationAction(ISD::LOAD, MVT::i128, Custom); - setOperationAction(ISD::STORE, MVT::i128, Custom); + setOperationAction({ISD::LOAD, ISD::STORE}, MVT::i128, Custom); // Aligned 128-bit loads and stores are single-copy atomic according to the // v8.4a spec. if (Subtarget->hasLSE2()) { - setOperationAction(ISD::ATOMIC_LOAD, MVT::i128, Custom); - setOperationAction(ISD::ATOMIC_STORE, MVT::i128, Custom); + setOperationAction({ISD::ATOMIC_LOAD, ISD::ATOMIC_STORE}, MVT::i128, + Custom); } // 256 bit non-temporal stores can be lowered to STNP. Do this as part of the // custom lowering, as there are no un-paired non-temporal stores and // legalization will break up 256 bit inputs. - setOperationAction(ISD::STORE, MVT::v32i8, Custom); - setOperationAction(ISD::STORE, MVT::v16i16, Custom); - setOperationAction(ISD::STORE, MVT::v16f16, Custom); - setOperationAction(ISD::STORE, MVT::v8i32, Custom); - setOperationAction(ISD::STORE, MVT::v8f32, Custom); - setOperationAction(ISD::STORE, MVT::v4f64, Custom); - setOperationAction(ISD::STORE, MVT::v4i64, Custom); + setOperationAction(ISD::STORE, + {MVT::v32i8, MVT::v16i16, MVT::v16f16, MVT::v8i32, + MVT::v8f32, MVT::v4f64, MVT::v4i64}, + Custom); // Lower READCYCLECOUNTER using an mrs from PMCCNTR_EL0. // This requires the Performance Monitors extension. @@ -757,34 +603,26 @@ if (getLibcallName(RTLIB::SINCOS_STRET_F32) != nullptr && getLibcallName(RTLIB::SINCOS_STRET_F64) != nullptr) { // Issue __sincos_stret if available. - setOperationAction(ISD::FSINCOS, MVT::f64, Custom); - setOperationAction(ISD::FSINCOS, MVT::f32, Custom); + setOperationAction(ISD::FSINCOS, {MVT::f64, MVT::f32}, Custom); } else { - setOperationAction(ISD::FSINCOS, MVT::f64, Expand); - setOperationAction(ISD::FSINCOS, MVT::f32, Expand); + setOperationAction(ISD::FSINCOS, {MVT::f64, MVT::f32}, Expand); } - if (Subtarget->getTargetTriple().isOSMSVCRT()) { + if (Subtarget->getTargetTriple().isOSMSVCRT()) // MSVCRT doesn't have powi; fall back to pow - setLibcallName(RTLIB::POWI_F32, nullptr); - setLibcallName(RTLIB::POWI_F64, nullptr); - } + setLibcallName({RTLIB::POWI_F32, RTLIB::POWI_F64}, nullptr); // Make floating-point constants legal for the large code model, so they don't // become loads from the constant pool. - if (Subtarget->isTargetMachO() && TM.getCodeModel() == CodeModel::Large) { - setOperationAction(ISD::ConstantFP, MVT::f32, Legal); - setOperationAction(ISD::ConstantFP, MVT::f64, Legal); - } + if (Subtarget->isTargetMachO() && TM.getCodeModel() == CodeModel::Large) + setOperationAction(ISD::ConstantFP, {MVT::f32, MVT::f64}, Legal); // AArch64 does not have floating-point extending loads, i1 sign-extending // load, floating-point truncating stores, or v2i32->v2i16 truncating store. - for (MVT VT : MVT::fp_valuetypes()) { - setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand); - setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand); - setLoadExtAction(ISD::EXTLOAD, VT, MVT::f64, Expand); - setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand); - } + for (MVT VT : MVT::fp_valuetypes()) + for (auto MemVT : {MVT::f16, MVT::f32, MVT::f64, MVT::f80}) + setLoadExtAction(ISD::EXTLOAD, VT, MemVT, Expand); + for (MVT VT : MVT::integer_valuetypes()) setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Expand); @@ -796,9 +634,7 @@ setTruncStoreAction(MVT::f128, MVT::f32, Expand); setTruncStoreAction(MVT::f128, MVT::f16, Expand); - setOperationAction(ISD::BITCAST, MVT::i16, Custom); - setOperationAction(ISD::BITCAST, MVT::f16, Custom); - setOperationAction(ISD::BITCAST, MVT::bf16, Custom); + setOperationAction(ISD::BITCAST, {MVT::i16, MVT::f16, MVT::bf16}, Custom); // Indexed loads and stores are supported. for (unsigned im = (unsigned)ISD::PRE_INC; @@ -822,9 +658,8 @@ } // Trap. - setOperationAction(ISD::TRAP, MVT::Other, Legal); - setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal); - setOperationAction(ISD::UBSANTRAP, MVT::Other, Legal); + setOperationAction({ISD::TRAP, ISD::DEBUGTRAP, ISD::UBSANTRAP}, MVT::Other, + Legal); // We combine OR nodes for bitfield operations. setTargetDAGCombine(ISD::OR); @@ -908,29 +743,29 @@ if (Subtarget->hasNEON()) { // FIXME: v1f64 shouldn't be legal if we can avoid it, because it leads to // silliness like this: - for (auto Op : - {ISD::SELECT, ISD::SELECT_CC, ISD::SETCC, - ISD::BR_CC, ISD::FADD, ISD::FSUB, - ISD::FMUL, ISD::FDIV, ISD::FMA, - ISD::FNEG, ISD::FABS, ISD::FCEIL, - ISD::FSQRT, ISD::FFLOOR, ISD::FNEARBYINT, - ISD::FRINT, ISD::FROUND, ISD::FROUNDEVEN, - ISD::FTRUNC, ISD::FMINNUM, ISD::FMAXNUM, - ISD::FMINIMUM, ISD::FMAXIMUM, ISD::STRICT_FADD, - ISD::STRICT_FSUB, ISD::STRICT_FMUL, ISD::STRICT_FDIV, - ISD::STRICT_FMA, ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR, - ISD::STRICT_FSQRT, ISD::STRICT_FRINT, ISD::STRICT_FNEARBYINT, - ISD::STRICT_FROUND, ISD::STRICT_FTRUNC, ISD::STRICT_FROUNDEVEN, - ISD::STRICT_FMINNUM, ISD::STRICT_FMAXNUM, ISD::STRICT_FMINIMUM, - ISD::STRICT_FMAXIMUM}) - setOperationAction(Op, MVT::v1f64, Expand); - - for (auto Op : - {ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::SINT_TO_FP, ISD::UINT_TO_FP, - ISD::FP_ROUND, ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT, ISD::MUL, - ISD::STRICT_FP_TO_SINT, ISD::STRICT_FP_TO_UINT, - ISD::STRICT_SINT_TO_FP, ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_ROUND}) - setOperationAction(Op, MVT::v1i64, Expand); + setOperationAction( + {ISD::SELECT, ISD::SELECT_CC, ISD::SETCC, + ISD::BR_CC, ISD::FADD, ISD::FSUB, + ISD::FMUL, ISD::FDIV, ISD::FMA, + ISD::FNEG, ISD::FABS, ISD::FCEIL, + ISD::FSQRT, ISD::FFLOOR, ISD::FNEARBYINT, + ISD::FRINT, ISD::FROUND, ISD::FROUNDEVEN, + ISD::FTRUNC, ISD::FMINNUM, ISD::FMAXNUM, + ISD::FMINIMUM, ISD::FMAXIMUM, ISD::STRICT_FADD, + ISD::STRICT_FSUB, ISD::STRICT_FMUL, ISD::STRICT_FDIV, + ISD::STRICT_FMA, ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR, + ISD::STRICT_FSQRT, ISD::STRICT_FRINT, ISD::STRICT_FNEARBYINT, + ISD::STRICT_FROUND, ISD::STRICT_FTRUNC, ISD::STRICT_FROUNDEVEN, + ISD::STRICT_FMINNUM, ISD::STRICT_FMAXNUM, ISD::STRICT_FMINIMUM, + ISD::STRICT_FMAXIMUM}, + MVT::v1f64, Expand); + + setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::SINT_TO_FP, + ISD::UINT_TO_FP, ISD::FP_ROUND, ISD::FP_TO_SINT_SAT, + ISD::FP_TO_UINT_SAT, ISD::MUL, ISD::STRICT_FP_TO_SINT, + ISD::STRICT_FP_TO_UINT, ISD::STRICT_SINT_TO_FP, + ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_ROUND}, + MVT::v1i64, Expand); // AArch64 doesn't have a direct vector ->f32 conversion instructions for // elements smaller than i32, so promote the input to i32 first. @@ -940,20 +775,14 @@ // Similarly, there is no direct i32 -> f64 vector conversion instruction. // Or, direct i32 -> f16 vector conversion. Set it so custom, so the // conversion happens in two steps: v4i32 -> v4f32 -> v4f16 - for (auto Op : {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::STRICT_SINT_TO_FP, - ISD::STRICT_UINT_TO_FP}) - for (auto VT : {MVT::v2i32, MVT::v2i64, MVT::v4i32}) - setOperationAction(Op, VT, Custom); + setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP, + ISD::STRICT_SINT_TO_FP, ISD::STRICT_UINT_TO_FP}, + {MVT::v2i32, MVT::v2i64, MVT::v4i32}, Custom); if (Subtarget->hasFullFP16()) { - setOperationAction(ISD::SINT_TO_FP, MVT::v8i8, Custom); - setOperationAction(ISD::UINT_TO_FP, MVT::v8i8, Custom); - setOperationAction(ISD::SINT_TO_FP, MVT::v16i8, Custom); - setOperationAction(ISD::UINT_TO_FP, MVT::v16i8, Custom); - setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom); - setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom); - setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Custom); - setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Custom); + setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, + {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16}, + Custom); } else { // when AArch64 doesn't have fullfp16 support, promote the input // to i32 first. @@ -967,65 +796,44 @@ setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v8i16, MVT::v8i32); } - setOperationAction(ISD::CTLZ, MVT::v1i64, Expand); - setOperationAction(ISD::CTLZ, MVT::v2i64, Expand); - setOperationAction(ISD::BITREVERSE, MVT::v8i8, Legal); - setOperationAction(ISD::BITREVERSE, MVT::v16i8, Legal); - setOperationAction(ISD::BITREVERSE, MVT::v2i32, Custom); - setOperationAction(ISD::BITREVERSE, MVT::v4i32, Custom); - setOperationAction(ISD::BITREVERSE, MVT::v1i64, Custom); - setOperationAction(ISD::BITREVERSE, MVT::v2i64, Custom); - for (auto VT : {MVT::v1i64, MVT::v2i64}) { - setOperationAction(ISD::UMAX, VT, Custom); - setOperationAction(ISD::SMAX, VT, Custom); - setOperationAction(ISD::UMIN, VT, Custom); - setOperationAction(ISD::SMIN, VT, Custom); - } + setOperationAction(ISD::CTLZ, {MVT::v1i64, MVT::v2i64}, Expand); + setOperationAction(ISD::BITREVERSE, {MVT::v8i8, MVT::v16i8}, Legal); + setOperationAction(ISD::BITREVERSE, + {MVT::v2i32, MVT::v4i32, MVT::v1i64, MVT::v2i64}, + Custom); + setOperationAction({ISD::UMAX, ISD::SMAX, ISD::UMIN, ISD::SMIN}, + {MVT::v1i64, MVT::v2i64}, Custom); // AArch64 doesn't have MUL.2d: setOperationAction(ISD::MUL, MVT::v2i64, Expand); // Custom handling for some quad-vector types to detect MULL. - setOperationAction(ISD::MUL, MVT::v8i16, Custom); - setOperationAction(ISD::MUL, MVT::v4i32, Custom); - setOperationAction(ISD::MUL, MVT::v2i64, Custom); + setOperationAction(ISD::MUL, {MVT::v8i16, MVT::v4i32, MVT::v2i64}, Custom); // Saturates - for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32, - MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) { - setOperationAction(ISD::SADDSAT, VT, Legal); - setOperationAction(ISD::UADDSAT, VT, Legal); - setOperationAction(ISD::SSUBSAT, VT, Legal); - setOperationAction(ISD::USUBSAT, VT, Legal); - } + setOperationAction({ISD::SADDSAT, ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT}, + {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, + MVT::v8i16, MVT::v4i32, MVT::v2i64}, + Legal); - for (MVT VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, MVT::v8i16, - MVT::v4i32}) { - setOperationAction(ISD::AVGFLOORS, VT, Legal); - setOperationAction(ISD::AVGFLOORU, VT, Legal); - setOperationAction(ISD::AVGCEILS, VT, Legal); - setOperationAction(ISD::AVGCEILU, VT, Legal); - setOperationAction(ISD::ABDS, VT, Legal); - setOperationAction(ISD::ABDU, VT, Legal); - } + setOperationAction( + {ISD::AVGFLOORS, ISD::AVGFLOORU, ISD::AVGCEILS, ISD::AVGCEILU, + ISD::ABDS, ISD::ABDU}, + {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, MVT::v8i16, MVT::v4i32}, + Legal); // Vector reductions - for (MVT VT : { MVT::v4f16, MVT::v2f32, - MVT::v8f16, MVT::v4f32, MVT::v2f64 }) { + for (MVT VT : {MVT::v4f16, MVT::v2f32, MVT::v8f16, MVT::v4f32, MVT::v2f64}) if (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16()) { - setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom); - setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom); + setOperationAction({ISD::VECREDUCE_FMAX, ISD::VECREDUCE_FMIN}, VT, + Custom); setOperationAction(ISD::VECREDUCE_FADD, VT, Legal); } - } - for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32, - MVT::v16i8, MVT::v8i16, MVT::v4i32 }) { - setOperationAction(ISD::VECREDUCE_ADD, VT, Custom); - setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom); - setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom); - setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom); - setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom); - } + setOperationAction( + {ISD::VECREDUCE_ADD, ISD::VECREDUCE_SMAX, ISD::VECREDUCE_SMIN, + ISD::VECREDUCE_UMAX, ISD::VECREDUCE_UMIN}, + {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, MVT::v8i16, MVT::v4i32}, + Custom); setOperationAction(ISD::VECREDUCE_ADD, MVT::v2i64, Custom); setOperationAction(ISD::ANY_EXTEND, MVT::v4i32, Legal); @@ -1035,24 +843,19 @@ for (MVT VT : MVT::fixedlen_vector_valuetypes()) { setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand); - if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32) { - setOperationAction(ISD::MULHS, VT, Legal); - setOperationAction(ISD::MULHU, VT, Legal); - } else { - setOperationAction(ISD::MULHS, VT, Expand); - setOperationAction(ISD::MULHU, VT, Expand); - } - setOperationAction(ISD::SMUL_LOHI, VT, Expand); - setOperationAction(ISD::UMUL_LOHI, VT, Expand); + if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32) + setOperationAction({ISD::MULHS, ISD::MULHU}, VT, Legal); + else + setOperationAction({ISD::MULHS, ISD::MULHU}, VT, Expand); + + setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Expand); - setOperationAction(ISD::BSWAP, VT, Expand); - setOperationAction(ISD::CTTZ, VT, Expand); + setOperationAction({ISD::BSWAP, ISD::CTTZ}, VT, Expand); for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) { setTruncStoreAction(VT, InnerVT, Expand); - setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand); - setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand); - setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand); + setLoadExtAction({ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}, VT, + InnerVT, Expand); } } @@ -1062,138 +865,121 @@ ISD::FROUND, ISD::FROUNDEVEN, ISD::STRICT_FFLOOR, ISD::STRICT_FNEARBYINT, ISD::STRICT_FCEIL, ISD::STRICT_FRINT, ISD::STRICT_FTRUNC, ISD::STRICT_FROUND, ISD::STRICT_FROUNDEVEN}) { - for (MVT Ty : {MVT::v2f32, MVT::v4f32, MVT::v2f64}) - setOperationAction(Op, Ty, Legal); + setOperationAction(Op, {MVT::v2f32, MVT::v4f32, MVT::v2f64}, Legal); if (Subtarget->hasFullFP16()) - for (MVT Ty : {MVT::v4f16, MVT::v8f16}) - setOperationAction(Op, Ty, Legal); + setOperationAction(Op, {MVT::v4f16, MVT::v8f16}, Legal); } setTruncStoreAction(MVT::v4i16, MVT::v4i8, Custom); - setLoadExtAction(ISD::EXTLOAD, MVT::v4i16, MVT::v4i8, Custom); - setLoadExtAction(ISD::SEXTLOAD, MVT::v4i16, MVT::v4i8, Custom); - setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i16, MVT::v4i8, Custom); - setLoadExtAction(ISD::EXTLOAD, MVT::v4i32, MVT::v4i8, Custom); - setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i8, Custom); - setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i8, Custom); + for (auto VT : {MVT::v4i16, MVT::v4i32}) + setLoadExtAction({ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, VT, + MVT::v4i8, Custom); } - if (Subtarget->hasSVE()) { for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64}) { - setOperationAction(ISD::BITREVERSE, VT, Custom); - setOperationAction(ISD::BSWAP, VT, Custom); - setOperationAction(ISD::CTLZ, VT, Custom); - setOperationAction(ISD::CTPOP, VT, Custom); - setOperationAction(ISD::CTTZ, VT, Custom); - setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); - setOperationAction(ISD::UINT_TO_FP, VT, Custom); - setOperationAction(ISD::SINT_TO_FP, VT, Custom); - setOperationAction(ISD::FP_TO_UINT, VT, Custom); - setOperationAction(ISD::FP_TO_SINT, VT, Custom); - setOperationAction(ISD::MGATHER, VT, Custom); - setOperationAction(ISD::MSCATTER, VT, Custom); - setOperationAction(ISD::MLOAD, VT, Custom); - setOperationAction(ISD::MUL, VT, Custom); - setOperationAction(ISD::MULHS, VT, Custom); - setOperationAction(ISD::MULHU, VT, Custom); - setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); - setOperationAction(ISD::VECTOR_SPLICE, VT, Custom); - setOperationAction(ISD::SELECT, VT, Custom); - setOperationAction(ISD::SETCC, VT, Custom); - setOperationAction(ISD::SDIV, VT, Custom); - setOperationAction(ISD::UDIV, VT, Custom); - setOperationAction(ISD::SMIN, VT, Custom); - setOperationAction(ISD::UMIN, VT, Custom); - setOperationAction(ISD::SMAX, VT, Custom); - setOperationAction(ISD::UMAX, VT, Custom); - setOperationAction(ISD::SHL, VT, Custom); - setOperationAction(ISD::SRL, VT, Custom); - setOperationAction(ISD::SRA, VT, Custom); - setOperationAction(ISD::ABS, VT, Custom); - setOperationAction(ISD::ABDS, VT, Custom); - setOperationAction(ISD::ABDU, VT, Custom); - setOperationAction(ISD::VECREDUCE_ADD, VT, Custom); - setOperationAction(ISD::VECREDUCE_AND, VT, Custom); - setOperationAction(ISD::VECREDUCE_OR, VT, Custom); - setOperationAction(ISD::VECREDUCE_XOR, VT, Custom); - setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom); - setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom); - setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom); - setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom); - - setOperationAction(ISD::UMUL_LOHI, VT, Expand); - setOperationAction(ISD::SMUL_LOHI, VT, Expand); - setOperationAction(ISD::SELECT_CC, VT, Expand); - setOperationAction(ISD::ROTL, VT, Expand); - setOperationAction(ISD::ROTR, VT, Expand); - - setOperationAction(ISD::SADDSAT, VT, Legal); - setOperationAction(ISD::UADDSAT, VT, Legal); - setOperationAction(ISD::SSUBSAT, VT, Legal); - setOperationAction(ISD::USUBSAT, VT, Legal); - setOperationAction(ISD::UREM, VT, Expand); - setOperationAction(ISD::SREM, VT, Expand); - setOperationAction(ISD::SDIVREM, VT, Expand); - setOperationAction(ISD::UDIVREM, VT, Expand); + setOperationAction({ISD::BITREVERSE, + ISD::BSWAP, + ISD::CTLZ, + ISD::CTPOP, + ISD::CTTZ, + ISD::INSERT_SUBVECTOR, + ISD::UINT_TO_FP, + ISD::SINT_TO_FP, + ISD::FP_TO_UINT, + ISD::FP_TO_SINT, + ISD::MGATHER, + ISD::MSCATTER, + ISD::MLOAD, + ISD::MUL, + ISD::MULHS, + ISD::MULHU, + ISD::SPLAT_VECTOR, + ISD::VECTOR_SPLICE, + ISD::SELECT, + ISD::SETCC, + ISD::SDIV, + ISD::UDIV, + ISD::SMIN, + ISD::UMIN, + ISD::SMAX, + ISD::UMAX, + ISD::SHL, + ISD::SRL, + ISD::SRA, + ISD::ABS, + ISD::ABDS, + ISD::ABDU, + ISD::VECREDUCE_ADD, + ISD::VECREDUCE_AND, + ISD::VECREDUCE_OR, + ISD::VECREDUCE_XOR, + ISD::VECREDUCE_UMIN, + ISD::VECREDUCE_UMAX, + ISD::VECREDUCE_SMIN, + ISD::VECREDUCE_SMAX}, + VT, Custom); + + setOperationAction({ISD::UMUL_LOHI, ISD::SMUL_LOHI, ISD::SELECT_CC, + ISD::ROTL, ISD::ROTR}, + VT, Expand); + + setOperationAction( + {ISD::SADDSAT, ISD::UADDSAT, ISD::SSUBSAT, ISD::USUBSAT}, VT, Legal); + setOperationAction({ISD::UREM, ISD::SREM, ISD::SDIVREM, ISD::UDIVREM}, VT, + Expand); } // Illegal unpacked integer vector types. - for (auto VT : {MVT::nxv8i8, MVT::nxv4i16, MVT::nxv2i32}) { - setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom); - setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); - } + setOperationAction({ISD::EXTRACT_SUBVECTOR, ISD::INSERT_SUBVECTOR}, + {MVT::nxv8i8, MVT::nxv4i16, MVT::nxv2i32}, Custom); // Legalize unpacked bitcasts to REINTERPRET_CAST. - for (auto VT : {MVT::nxv2i16, MVT::nxv4i16, MVT::nxv2i32, MVT::nxv2bf16, - MVT::nxv2f16, MVT::nxv4f16, MVT::nxv2f32}) - setOperationAction(ISD::BITCAST, VT, Custom); + setOperationAction(ISD::BITCAST, + {MVT::nxv2i16, MVT::nxv4i16, MVT::nxv2i32, MVT::nxv2bf16, + MVT::nxv2f16, MVT::nxv4f16, MVT::nxv2f32}, + Custom); - for (auto VT : - { MVT::nxv2i8, MVT::nxv2i16, MVT::nxv2i32, MVT::nxv2i64, MVT::nxv4i8, - MVT::nxv4i16, MVT::nxv4i32, MVT::nxv8i8, MVT::nxv8i16 }) - setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Legal); + setOperationAction(ISD::SIGN_EXTEND_INREG, + {MVT::nxv2i8, MVT::nxv2i16, MVT::nxv2i32, MVT::nxv2i64, + MVT::nxv4i8, MVT::nxv4i16, MVT::nxv4i32, MVT::nxv8i8, + MVT::nxv8i16}, + Legal); for (auto VT : {MVT::nxv16i1, MVT::nxv8i1, MVT::nxv4i1, MVT::nxv2i1}) { - setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); - setOperationAction(ISD::SELECT, VT, Custom); - setOperationAction(ISD::SETCC, VT, Custom); - setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); - setOperationAction(ISD::TRUNCATE, VT, Custom); - setOperationAction(ISD::VECREDUCE_AND, VT, Custom); - setOperationAction(ISD::VECREDUCE_OR, VT, Custom); - setOperationAction(ISD::VECREDUCE_XOR, VT, Custom); + setOperationAction({ISD::CONCAT_VECTORS, ISD::SELECT, ISD::SETCC, + ISD::SPLAT_VECTOR, ISD::TRUNCATE, ISD::VECREDUCE_AND, + ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, + VT, Custom); setOperationAction(ISD::SELECT_CC, VT, Expand); - setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); - setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); - setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); + setOperationAction({ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT, + ISD::INSERT_SUBVECTOR}, + VT, Custom); // There are no legal MVT::nxv16f## based types. if (VT != MVT::nxv16i1) { - setOperationAction(ISD::SINT_TO_FP, VT, Custom); - setOperationAction(ISD::UINT_TO_FP, VT, Custom); + setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT, Custom); } } // NEON doesn't support masked loads/stores/gathers/scatters, but SVE does - for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v2f32, MVT::v4f32, MVT::v1f64, - MVT::v2f64, MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, - MVT::v2i32, MVT::v4i32, MVT::v1i64, MVT::v2i64}) { - setOperationAction(ISD::MLOAD, VT, Custom); - setOperationAction(ISD::MSTORE, VT, Custom); - setOperationAction(ISD::MGATHER, VT, Custom); - setOperationAction(ISD::MSCATTER, VT, Custom); - } + + setOperationAction({ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER}, + {MVT::v4f16, MVT::v8f16, MVT::v2f32, MVT::v4f32, + MVT::v1f64, MVT::v2f64, MVT::v8i8, MVT::v16i8, + MVT::v4i16, MVT::v8i16, MVT::v2i32, MVT::v4i32, + MVT::v1i64, MVT::v2i64}, + Custom); // Firstly, exclude all scalable vector extending loads/truncating stores, // include both integer and floating scalable vector. for (MVT VT : MVT::scalable_vector_valuetypes()) { for (MVT InnerVT : MVT::scalable_vector_valuetypes()) { setTruncStoreAction(VT, InnerVT, Expand); - setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand); - setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand); - setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand); + // SVE does not have floating-point extending loads. + setLoadExtAction({ISD::SEXTLOAD, ISD::ZEXTLOAD, ISD::EXTLOAD}, VT, + InnerVT, Expand); } } @@ -1222,89 +1008,68 @@ for (auto VT : {MVT::nxv2f16, MVT::nxv4f16, MVT::nxv8f16, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv2f64}) { - setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); - setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); - setOperationAction(ISD::MGATHER, VT, Custom); - setOperationAction(ISD::MSCATTER, VT, Custom); - setOperationAction(ISD::MLOAD, VT, Custom); - setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); - setOperationAction(ISD::SELECT, VT, Custom); - setOperationAction(ISD::FADD, VT, Custom); - setOperationAction(ISD::FCOPYSIGN, VT, Custom); - setOperationAction(ISD::FDIV, VT, Custom); - setOperationAction(ISD::FMA, VT, Custom); - setOperationAction(ISD::FMAXIMUM, VT, Custom); - setOperationAction(ISD::FMAXNUM, VT, Custom); - setOperationAction(ISD::FMINIMUM, VT, Custom); - setOperationAction(ISD::FMINNUM, VT, Custom); - setOperationAction(ISD::FMUL, VT, Custom); - setOperationAction(ISD::FNEG, VT, Custom); - setOperationAction(ISD::FSUB, VT, Custom); - setOperationAction(ISD::FCEIL, VT, Custom); - setOperationAction(ISD::FFLOOR, VT, Custom); - setOperationAction(ISD::FNEARBYINT, VT, Custom); - setOperationAction(ISD::FRINT, VT, Custom); - setOperationAction(ISD::FROUND, VT, Custom); - setOperationAction(ISD::FROUNDEVEN, VT, Custom); - setOperationAction(ISD::FTRUNC, VT, Custom); - setOperationAction(ISD::FSQRT, VT, Custom); - setOperationAction(ISD::FABS, VT, Custom); - setOperationAction(ISD::FP_EXTEND, VT, Custom); - setOperationAction(ISD::FP_ROUND, VT, Custom); - setOperationAction(ISD::VECREDUCE_FADD, VT, Custom); - setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom); - setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom); - setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom); - setOperationAction(ISD::VECTOR_SPLICE, VT, Custom); - - setOperationAction(ISD::SELECT_CC, VT, Expand); - setOperationAction(ISD::FREM, VT, Expand); - setOperationAction(ISD::FPOW, VT, Expand); - setOperationAction(ISD::FPOWI, VT, Expand); - setOperationAction(ISD::FCOS, VT, Expand); - setOperationAction(ISD::FSIN, VT, Expand); - setOperationAction(ISD::FSINCOS, VT, Expand); - setOperationAction(ISD::FEXP, VT, Expand); - setOperationAction(ISD::FEXP2, VT, Expand); - setOperationAction(ISD::FLOG, VT, Expand); - setOperationAction(ISD::FLOG2, VT, Expand); - setOperationAction(ISD::FLOG10, VT, Expand); - - setCondCodeAction(ISD::SETO, VT, Expand); - setCondCodeAction(ISD::SETOLT, VT, Expand); - setCondCodeAction(ISD::SETLT, VT, Expand); - setCondCodeAction(ISD::SETOLE, VT, Expand); - setCondCodeAction(ISD::SETLE, VT, Expand); - setCondCodeAction(ISD::SETULT, VT, Expand); - setCondCodeAction(ISD::SETULE, VT, Expand); - setCondCodeAction(ISD::SETUGE, VT, Expand); - setCondCodeAction(ISD::SETUGT, VT, Expand); - setCondCodeAction(ISD::SETUEQ, VT, Expand); - setCondCodeAction(ISD::SETONE, VT, Expand); + setOperationAction({ISD::CONCAT_VECTORS, + ISD::INSERT_SUBVECTOR, + ISD::MGATHER, + ISD::MSCATTER, + ISD::MLOAD, + ISD::SPLAT_VECTOR, + ISD::SELECT, + ISD::FADD, + ISD::FCOPYSIGN, + ISD::FDIV, + ISD::FMA, + ISD::FMAXIMUM, + ISD::FMAXNUM, + ISD::FMINIMUM, + ISD::FMINNUM, + ISD::FMUL, + ISD::FNEG, + ISD::FSUB, + ISD::FCEIL, + ISD::FFLOOR, + ISD::FNEARBYINT, + ISD::FRINT, + ISD::FROUND, + ISD::FROUNDEVEN, + ISD::FTRUNC, + ISD::FSQRT, + ISD::FABS, + ISD::FP_EXTEND, + ISD::FP_ROUND, + ISD::VECREDUCE_FADD, + ISD::VECREDUCE_FMAX, + ISD::VECREDUCE_FMIN, + ISD::VECREDUCE_SEQ_FADD, + ISD::VECTOR_SPLICE}, + VT, Custom); + + setOperationAction({ISD::SELECT_CC, ISD::FREM, ISD::FPOW, ISD::FPOWI, + ISD::FCOS, ISD::FSIN, ISD::FSINCOS, ISD::FEXP, + ISD::FEXP2, ISD::FLOG, ISD::FLOG2, ISD::FLOG10}, + VT, Expand); + + setCondCodeAction({ISD::SETO, ISD::SETOLT, ISD::SETLT, ISD::SETOLE, + ISD::SETLE, ISD::SETULT, ISD::SETULE, ISD::SETUGE, + ISD::SETUGT, ISD::SETUEQ, ISD::SETONE}, + VT, Expand); } - for (auto VT : {MVT::nxv2bf16, MVT::nxv4bf16, MVT::nxv8bf16}) { - setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); - setOperationAction(ISD::MGATHER, VT, Custom); - setOperationAction(ISD::MSCATTER, VT, Custom); - setOperationAction(ISD::MLOAD, VT, Custom); - setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom); - setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); - } + setOperationAction({ISD::CONCAT_VECTORS, ISD::MGATHER, ISD::MSCATTER, + ISD::MLOAD, ISD::INSERT_SUBVECTOR, ISD::SPLAT_VECTOR}, + {MVT::nxv2bf16, MVT::nxv4bf16, MVT::nxv8bf16}, Custom); - setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i8, Custom); - setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom); + setOperationAction(ISD::INTRINSIC_WO_CHAIN, {MVT::i8, MVT::i16}, Custom); // NEON doesn't support integer divides, but SVE does - for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v2i32, - MVT::v4i32, MVT::v1i64, MVT::v2i64}) { - setOperationAction(ISD::SDIV, VT, Custom); - setOperationAction(ISD::UDIV, VT, Custom); - } + + setOperationAction({ISD::SDIV, ISD::UDIV}, + {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, + MVT::v2i32, MVT::v4i32, MVT::v1i64, MVT::v2i64}, + Custom); // NEON doesn't support 64-bit vector integer muls, but SVE does. - setOperationAction(ISD::MUL, MVT::v1i64, Custom); - setOperationAction(ISD::MUL, MVT::v2i64, Custom); + setOperationAction(ISD::MUL, {MVT::v1i64, MVT::v2i64}, Custom); // NOTE: Currently this has to happen after computeRegisterProperties rather // than the preferred option of combining it with the addRegisterClass call. @@ -1317,56 +1082,39 @@ addTypeForFixedLengthSVE(VT); // 64bit results can mean a bigger than NEON input. - for (auto VT : {MVT::v8i8, MVT::v4i16}) - setOperationAction(ISD::TRUNCATE, VT, Custom); + setOperationAction(ISD::TRUNCATE, {MVT::v8i8, MVT::v4i16}, Custom); setOperationAction(ISD::FP_ROUND, MVT::v4f16, Custom); // 128bit results imply a bigger than NEON input. - for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32}) - setOperationAction(ISD::TRUNCATE, VT, Custom); - for (auto VT : {MVT::v8f16, MVT::v4f32}) - setOperationAction(ISD::FP_ROUND, VT, Custom); + setOperationAction(ISD::TRUNCATE, {MVT::v16i8, MVT::v8i16, MVT::v4i32}, + Custom); + setOperationAction(ISD::FP_ROUND, {MVT::v8f16, MVT::v4f32}, Custom); // These operations are not supported on NEON but SVE can do them. - setOperationAction(ISD::BITREVERSE, MVT::v1i64, Custom); - setOperationAction(ISD::CTLZ, MVT::v1i64, Custom); - setOperationAction(ISD::CTLZ, MVT::v2i64, Custom); - setOperationAction(ISD::CTTZ, MVT::v1i64, Custom); - setOperationAction(ISD::MULHS, MVT::v1i64, Custom); - setOperationAction(ISD::MULHS, MVT::v2i64, Custom); - setOperationAction(ISD::MULHU, MVT::v1i64, Custom); - setOperationAction(ISD::MULHU, MVT::v2i64, Custom); - setOperationAction(ISD::SMAX, MVT::v1i64, Custom); - setOperationAction(ISD::SMAX, MVT::v2i64, Custom); - setOperationAction(ISD::SMIN, MVT::v1i64, Custom); - setOperationAction(ISD::SMIN, MVT::v2i64, Custom); - setOperationAction(ISD::UMAX, MVT::v1i64, Custom); - setOperationAction(ISD::UMAX, MVT::v2i64, Custom); - setOperationAction(ISD::UMIN, MVT::v1i64, Custom); - setOperationAction(ISD::UMIN, MVT::v2i64, Custom); - setOperationAction(ISD::VECREDUCE_SMAX, MVT::v2i64, Custom); - setOperationAction(ISD::VECREDUCE_SMIN, MVT::v2i64, Custom); - setOperationAction(ISD::VECREDUCE_UMAX, MVT::v2i64, Custom); - setOperationAction(ISD::VECREDUCE_UMIN, MVT::v2i64, Custom); + setOperationAction({ISD::BITREVERSE, ISD::CTTZ}, MVT::v1i64, Custom); + setOperationAction({ISD::CTLZ, ISD::MULHS, ISD::MULHU, ISD::SMAX, + ISD::SMIN, ISD::UMAX, ISD::UMIN}, + {MVT::v1i64, MVT::v2i64}, Custom); + setOperationAction({ISD::VECREDUCE_SMAX, ISD::VECREDUCE_SMIN, + ISD::VECREDUCE_UMAX, ISD::VECREDUCE_UMIN}, + MVT::v2i64, Custom); // Int operations with no NEON support. - for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, - MVT::v2i32, MVT::v4i32, MVT::v2i64}) { - setOperationAction(ISD::BITREVERSE, VT, Custom); - setOperationAction(ISD::CTTZ, VT, Custom); - setOperationAction(ISD::VECREDUCE_AND, VT, Custom); - setOperationAction(ISD::VECREDUCE_OR, VT, Custom); - setOperationAction(ISD::VECREDUCE_XOR, VT, Custom); - } + setOperationAction({ISD::BITREVERSE, ISD::CTTZ, ISD::VECREDUCE_AND, + ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, + {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, + MVT::v2i32, MVT::v4i32, MVT::v2i64}, + Custom); // FP operations with no NEON support. - for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v2f32, MVT::v4f32, - MVT::v1f64, MVT::v2f64}) - setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom); + setOperationAction(ISD::VECREDUCE_SEQ_FADD, + {MVT::v4f16, MVT::v8f16, MVT::v2f32, MVT::v4f32, + MVT::v1f64, MVT::v2f64}, + Custom); // Use SVE for vectors with more than 2 elements. - for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v4f32}) - setOperationAction(ISD::VECREDUCE_FADD, VT, Custom); + setOperationAction(ISD::VECREDUCE_FADD, + {MVT::v4f16, MVT::v8f16, MVT::v4f32}, Custom); } setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv2i1, MVT::nxv2i64); @@ -1397,37 +1145,24 @@ } // Mark vector float intrinsics as expand. - if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64) { - setOperationAction(ISD::FSIN, VT, Expand); - setOperationAction(ISD::FCOS, VT, Expand); - setOperationAction(ISD::FPOW, VT, Expand); - setOperationAction(ISD::FLOG, VT, Expand); - setOperationAction(ISD::FLOG2, VT, Expand); - setOperationAction(ISD::FLOG10, VT, Expand); - setOperationAction(ISD::FEXP, VT, Expand); - setOperationAction(ISD::FEXP2, VT, Expand); - } + if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64) + setOperationAction({ISD::FSIN, ISD::FCOS, ISD::FPOW, ISD::FLOG, ISD::FLOG2, + ISD::FLOG10, ISD::FEXP, ISD::FEXP2}, + VT, Expand); // But we do support custom-lowering for FCOPYSIGN. if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64 || ((VT == MVT::v4f16 || VT == MVT::v8f16) && Subtarget->hasFullFP16())) setOperationAction(ISD::FCOPYSIGN, VT, Custom); - setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); - setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); - setOperationAction(ISD::BUILD_VECTOR, VT, Custom); - setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); - setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom); - setOperationAction(ISD::SRA, VT, Custom); - setOperationAction(ISD::SRL, VT, Custom); - setOperationAction(ISD::SHL, VT, Custom); - setOperationAction(ISD::OR, VT, Custom); - setOperationAction(ISD::SETCC, VT, Custom); + setOperationAction({ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT, + ISD::BUILD_VECTOR, ISD::VECTOR_SHUFFLE, + ISD::EXTRACT_SUBVECTOR, ISD::SRA, ISD::SRL, ISD::SHL, + ISD::OR, ISD::SETCC}, + VT, Custom); setOperationAction(ISD::CONCAT_VECTORS, VT, Legal); - setOperationAction(ISD::SELECT, VT, Expand); - setOperationAction(ISD::SELECT_CC, VT, Expand); - setOperationAction(ISD::VSELECT, VT, Expand); + setOperationAction({ISD::SELECT, ISD::SELECT_CC, ISD::VSELECT}, VT, Expand); for (MVT InnerVT : MVT::all_valuetypes()) setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand); @@ -1435,16 +1170,13 @@ if (VT != MVT::v8i8 && VT != MVT::v16i8) setOperationAction(ISD::CTPOP, VT, Custom); - setOperationAction(ISD::UDIV, VT, Expand); - setOperationAction(ISD::SDIV, VT, Expand); - setOperationAction(ISD::UREM, VT, Expand); - setOperationAction(ISD::SREM, VT, Expand); - setOperationAction(ISD::FREM, VT, Expand); + setOperationAction({ISD::UDIV, ISD::SDIV, ISD::UREM, ISD::SREM, ISD::FREM}, + VT, Expand); - for (unsigned Opcode : - {ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::FP_TO_SINT_SAT, - ISD::FP_TO_UINT_SAT, ISD::STRICT_FP_TO_SINT, ISD::STRICT_FP_TO_UINT}) - setOperationAction(Opcode, VT, Custom); + setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::FP_TO_SINT_SAT, + ISD::FP_TO_UINT_SAT, ISD::STRICT_FP_TO_SINT, + ISD::STRICT_FP_TO_UINT}, + VT, Custom); if (!VT.isFloatingPoint()) setOperationAction(ISD::ABS, VT, Legal); @@ -1459,13 +1191,13 @@ if (VT.isFloatingPoint() && VT.getVectorElementType() != MVT::bf16 && (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16())) - for (unsigned Opcode : - {ISD::FMINIMUM, ISD::FMAXIMUM, ISD::FMINNUM, ISD::FMAXNUM, - ISD::STRICT_FMINIMUM, ISD::STRICT_FMAXIMUM, ISD::STRICT_FMINNUM, - ISD::STRICT_FMAXNUM, ISD::STRICT_FADD, ISD::STRICT_FSUB, - ISD::STRICT_FMUL, ISD::STRICT_FDIV, ISD::STRICT_FMA, - ISD::STRICT_FSQRT}) - setOperationAction(Opcode, VT, Legal); + setOperationAction({ISD::FMINIMUM, ISD::FMAXIMUM, ISD::FMINNUM, + ISD::FMAXNUM, ISD::STRICT_FMINIMUM, + ISD::STRICT_FMAXIMUM, ISD::STRICT_FMINNUM, + ISD::STRICT_FMAXNUM, ISD::STRICT_FADD, ISD::STRICT_FSUB, + ISD::STRICT_FMUL, ISD::STRICT_FDIV, ISD::STRICT_FMA, + ISD::STRICT_FSQRT}, + VT, Legal); // Strict fp extend and trunc are legal if (VT.isFloatingPoint() && VT.getScalarSizeInBits() != 16) @@ -1484,8 +1216,7 @@ // * The lowering of the non-strict versions involves target-specific ISD // nodes so we would likely need to add strict versions of all of them and // handle them appropriately. - setOperationAction(ISD::STRICT_FSETCC, VT, Expand); - setOperationAction(ISD::STRICT_FSETCCS, VT, Expand); + setOperationAction({ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS}, VT, Expand); if (Subtarget->isLittleEndian()) { for (unsigned im = (unsigned)ISD::PRE_INC; @@ -1526,27 +1257,18 @@ // We use EXTRACT_SUBVECTOR to "cast" a scalable vector to a fixed length one. setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom); - if (VT.isFloatingPoint()) { - setCondCodeAction(ISD::SETO, VT, Expand); - setCondCodeAction(ISD::SETOLT, VT, Expand); - setCondCodeAction(ISD::SETLT, VT, Expand); - setCondCodeAction(ISD::SETOLE, VT, Expand); - setCondCodeAction(ISD::SETLE, VT, Expand); - setCondCodeAction(ISD::SETULT, VT, Expand); - setCondCodeAction(ISD::SETULE, VT, Expand); - setCondCodeAction(ISD::SETUGE, VT, Expand); - setCondCodeAction(ISD::SETUGT, VT, Expand); - setCondCodeAction(ISD::SETUEQ, VT, Expand); - setCondCodeAction(ISD::SETONE, VT, Expand); - } + if (VT.isFloatingPoint()) + setCondCodeAction({ISD::SETO, ISD::SETOLT, ISD::SETLT, ISD::SETOLE, + ISD::SETLE, ISD::SETULT, ISD::SETULE, ISD::SETUGE, + ISD::SETUGT, ISD::SETUEQ, ISD::SETONE}, + VT, Expand); // Mark integer truncating stores/extending loads as having custom lowering if (VT.isInteger()) { MVT InnerVT = VT.changeVectorElementType(MVT::i8); while (InnerVT != VT) { setTruncStoreAction(VT, InnerVT, Custom); - setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Custom); - setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Custom); + setLoadExtAction({ISD::ZEXTLOAD, ISD::SEXTLOAD}, VT, InnerVT, Custom); InnerVT = InnerVT.changeVectorElementType( MVT::getIntegerVT(2 * InnerVT.getScalarSizeInBits())); } @@ -1565,87 +1287,88 @@ } // Lower fixed length vector operations to scalable equivalents. - setOperationAction(ISD::ABS, VT, Custom); - setOperationAction(ISD::ADD, VT, Custom); - setOperationAction(ISD::AND, VT, Custom); - setOperationAction(ISD::ANY_EXTEND, VT, Custom); - setOperationAction(ISD::BITCAST, VT, Custom); - setOperationAction(ISD::BITREVERSE, VT, Custom); - setOperationAction(ISD::BSWAP, VT, Custom); - setOperationAction(ISD::CONCAT_VECTORS, VT, Custom); - setOperationAction(ISD::CTLZ, VT, Custom); - setOperationAction(ISD::CTPOP, VT, Custom); - setOperationAction(ISD::CTTZ, VT, Custom); - setOperationAction(ISD::FABS, VT, Custom); - setOperationAction(ISD::FADD, VT, Custom); - setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom); - setOperationAction(ISD::FCEIL, VT, Custom); - setOperationAction(ISD::FDIV, VT, Custom); - setOperationAction(ISD::FFLOOR, VT, Custom); - setOperationAction(ISD::FMA, VT, Custom); - setOperationAction(ISD::FMAXIMUM, VT, Custom); - setOperationAction(ISD::FMAXNUM, VT, Custom); - setOperationAction(ISD::FMINIMUM, VT, Custom); - setOperationAction(ISD::FMINNUM, VT, Custom); - setOperationAction(ISD::FMUL, VT, Custom); - setOperationAction(ISD::FNEARBYINT, VT, Custom); - setOperationAction(ISD::FNEG, VT, Custom); - setOperationAction(ISD::FP_EXTEND, VT, Custom); - setOperationAction(ISD::FP_ROUND, VT, Custom); - setOperationAction(ISD::FP_TO_SINT, VT, Custom); - setOperationAction(ISD::FP_TO_UINT, VT, Custom); - setOperationAction(ISD::FRINT, VT, Custom); - setOperationAction(ISD::FROUND, VT, Custom); - setOperationAction(ISD::FROUNDEVEN, VT, Custom); - setOperationAction(ISD::FSQRT, VT, Custom); - setOperationAction(ISD::FSUB, VT, Custom); - setOperationAction(ISD::FTRUNC, VT, Custom); - setOperationAction(ISD::LOAD, VT, Custom); - setOperationAction(ISD::MGATHER, VT, Custom); - setOperationAction(ISD::MLOAD, VT, Custom); - setOperationAction(ISD::MSCATTER, VT, Custom); - setOperationAction(ISD::MSTORE, VT, Custom); - setOperationAction(ISD::MUL, VT, Custom); - setOperationAction(ISD::MULHS, VT, Custom); - setOperationAction(ISD::MULHU, VT, Custom); - setOperationAction(ISD::OR, VT, Custom); - setOperationAction(ISD::SDIV, VT, Custom); - setOperationAction(ISD::SELECT, VT, Custom); - setOperationAction(ISD::SETCC, VT, Custom); - setOperationAction(ISD::SHL, VT, Custom); - setOperationAction(ISD::SIGN_EXTEND, VT, Custom); - setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Custom); - setOperationAction(ISD::SINT_TO_FP, VT, Custom); - setOperationAction(ISD::SMAX, VT, Custom); - setOperationAction(ISD::SMIN, VT, Custom); - setOperationAction(ISD::SPLAT_VECTOR, VT, Custom); - setOperationAction(ISD::VECTOR_SPLICE, VT, Custom); - setOperationAction(ISD::SRA, VT, Custom); - setOperationAction(ISD::SRL, VT, Custom); - setOperationAction(ISD::STORE, VT, Custom); - setOperationAction(ISD::SUB, VT, Custom); - setOperationAction(ISD::TRUNCATE, VT, Custom); - setOperationAction(ISD::UDIV, VT, Custom); - setOperationAction(ISD::UINT_TO_FP, VT, Custom); - setOperationAction(ISD::UMAX, VT, Custom); - setOperationAction(ISD::UMIN, VT, Custom); - setOperationAction(ISD::VECREDUCE_ADD, VT, Custom); - setOperationAction(ISD::VECREDUCE_AND, VT, Custom); - setOperationAction(ISD::VECREDUCE_FADD, VT, Custom); - setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom); - setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom); - setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom); - setOperationAction(ISD::VECREDUCE_OR, VT, Custom); - setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom); - setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom); - setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom); - setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom); - setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom); - setOperationAction(ISD::VECREDUCE_XOR, VT, Custom); - setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom); - setOperationAction(ISD::VSELECT, VT, Custom); - setOperationAction(ISD::XOR, VT, Custom); - setOperationAction(ISD::ZERO_EXTEND, VT, Custom); + setOperationAction({ISD::ABS, + ISD::ADD, + ISD::AND, + ISD::ANY_EXTEND, + ISD::BITCAST, + ISD::BITREVERSE, + ISD::BSWAP, + ISD::CONCAT_VECTORS, + ISD::CTLZ, + ISD::CTPOP, + ISD::CTTZ, + ISD::FABS, + ISD::FADD, + ISD::EXTRACT_VECTOR_ELT, + ISD::FCEIL, + ISD::FDIV, + ISD::FFLOOR, + ISD::FMA, + ISD::FMAXIMUM, + ISD::FMAXNUM, + ISD::FMINIMUM, + ISD::FMINNUM, + ISD::FMUL, + ISD::FNEARBYINT, + ISD::FNEG, + ISD::FP_EXTEND, + ISD::FP_ROUND, + ISD::FP_TO_SINT, + ISD::FP_TO_UINT, + ISD::FRINT, + ISD::FROUND, + ISD::FROUNDEVEN, + ISD::FSQRT, + ISD::FSUB, + ISD::FTRUNC, + ISD::LOAD, + ISD::MGATHER, + ISD::MLOAD, + ISD::MSCATTER, + ISD::MSTORE, + ISD::MUL, + ISD::MULHS, + ISD::MULHU, + ISD::OR, + ISD::SDIV, + ISD::SELECT, + ISD::SETCC, + ISD::SHL, + ISD::SIGN_EXTEND, + ISD::SIGN_EXTEND_INREG, + ISD::SINT_TO_FP, + ISD::SMAX, + ISD::SMIN, + ISD::SPLAT_VECTOR, + ISD::VECTOR_SPLICE, + ISD::SRA, + ISD::SRL, + ISD::STORE, + ISD::SUB, + ISD::TRUNCATE, + ISD::UDIV, + ISD::UINT_TO_FP, + ISD::UMAX, + ISD::UMIN, + ISD::VECREDUCE_ADD, + ISD::VECREDUCE_AND, + ISD::VECREDUCE_FADD, + ISD::VECREDUCE_SEQ_FADD, + ISD::VECREDUCE_FMAX, + ISD::VECREDUCE_FMIN, + ISD::VECREDUCE_OR, + ISD::INSERT_VECTOR_ELT, + ISD::VECREDUCE_SMAX, + ISD::VECREDUCE_SMIN, + ISD::VECREDUCE_UMAX, + ISD::VECREDUCE_UMIN, + ISD::VECREDUCE_XOR, + ISD::VECTOR_SHUFFLE, + ISD::VSELECT, + ISD::XOR, + ISD::ZERO_EXTEND}, + VT, Custom); } void AArch64TargetLowering::addDRTypeForNEON(MVT VT) {