Changeset View
Changeset View
Standalone View
Standalone View
lib/Target/AArch64/AArch64ISelLowering.cpp
- This file is larger than 256 KB, so syntax highlighting is disabled by default.
| Show First 20 Lines • Show All 160 Lines • ▼ Show 20 Lines | AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, | ||||
| // Compute derived properties from the register classes | // Compute derived properties from the register classes | ||||
| computeRegisterProperties(Subtarget->getRegisterInfo()); | computeRegisterProperties(Subtarget->getRegisterInfo()); | ||||
| // Provide all sorts of operation actions | // Provide all sorts of operation actions | ||||
| setOperationAction(ISD::GlobalAddress, MVT::i64, Custom); | setOperationAction(ISD::GlobalAddress, MVT::i64, Custom); | ||||
| setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); | setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom); | ||||
| setOperationAction(ISD::SETCC, MVT::i32, Custom); | setOperationAction(ISD::SETCC, MVT::i32, Custom); | ||||
| setOperationAction(ISD::SETCC, MVT::i64, Custom); | setOperationAction(ISD::SETCC, MVT::i64, Custom); | ||||
| setOperationAction(ISD::SETCC, MVT::f16, Custom); | |||||
| setOperationAction(ISD::SETCC, MVT::f32, Custom); | setOperationAction(ISD::SETCC, MVT::f32, Custom); | ||||
| setOperationAction(ISD::SETCC, MVT::f64, Custom); | setOperationAction(ISD::SETCC, MVT::f64, Custom); | ||||
| setOperationAction(ISD::BITREVERSE, MVT::i32, Legal); | setOperationAction(ISD::BITREVERSE, MVT::i32, Legal); | ||||
| setOperationAction(ISD::BITREVERSE, MVT::i64, Legal); | setOperationAction(ISD::BITREVERSE, MVT::i64, Legal); | ||||
| setOperationAction(ISD::BRCOND, MVT::Other, Expand); | setOperationAction(ISD::BRCOND, MVT::Other, Expand); | ||||
| setOperationAction(ISD::BR_CC, MVT::i32, Custom); | setOperationAction(ISD::BR_CC, MVT::i32, Custom); | ||||
| setOperationAction(ISD::BR_CC, MVT::i64, Custom); | setOperationAction(ISD::BR_CC, MVT::i64, Custom); | ||||
| setOperationAction(ISD::BR_CC, MVT::f16, Custom); | |||||
| setOperationAction(ISD::BR_CC, MVT::f32, Custom); | setOperationAction(ISD::BR_CC, MVT::f32, Custom); | ||||
| setOperationAction(ISD::BR_CC, MVT::f64, Custom); | setOperationAction(ISD::BR_CC, MVT::f64, Custom); | ||||
| setOperationAction(ISD::SELECT, MVT::i32, Custom); | setOperationAction(ISD::SELECT, MVT::i32, Custom); | ||||
| setOperationAction(ISD::SELECT, MVT::i64, Custom); | setOperationAction(ISD::SELECT, MVT::i64, Custom); | ||||
| setOperationAction(ISD::SELECT, MVT::f16, Custom); | |||||
| setOperationAction(ISD::SELECT, MVT::f32, Custom); | setOperationAction(ISD::SELECT, MVT::f32, Custom); | ||||
| setOperationAction(ISD::SELECT, MVT::f64, Custom); | setOperationAction(ISD::SELECT, MVT::f64, Custom); | ||||
| setOperationAction(ISD::SELECT_CC, MVT::i32, Custom); | setOperationAction(ISD::SELECT_CC, MVT::i32, Custom); | ||||
| setOperationAction(ISD::SELECT_CC, MVT::i64, Custom); | setOperationAction(ISD::SELECT_CC, MVT::i64, Custom); | ||||
| setOperationAction(ISD::SELECT_CC, MVT::f16, Custom); | |||||
| setOperationAction(ISD::SELECT_CC, MVT::f32, Custom); | setOperationAction(ISD::SELECT_CC, MVT::f32, Custom); | ||||
| setOperationAction(ISD::SELECT_CC, MVT::f64, Custom); | setOperationAction(ISD::SELECT_CC, MVT::f64, Custom); | ||||
| setOperationAction(ISD::BR_JT, MVT::Other, Expand); | setOperationAction(ISD::BR_JT, MVT::Other, Expand); | ||||
| setOperationAction(ISD::JumpTable, MVT::i64, Custom); | setOperationAction(ISD::JumpTable, MVT::i64, Custom); | ||||
| setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom); | setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom); | ||||
| setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom); | setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom); | ||||
| setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom); | setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom); | ||||
| ▲ Show 20 Lines • Show All 121 Lines • ▼ Show 20 Lines | AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, | ||||
| setOperationAction(ISD::FSIN, MVT::f64, Expand); | setOperationAction(ISD::FSIN, MVT::f64, Expand); | ||||
| setOperationAction(ISD::FCOS, MVT::f32, Expand); | setOperationAction(ISD::FCOS, MVT::f32, Expand); | ||||
| setOperationAction(ISD::FCOS, MVT::f64, Expand); | setOperationAction(ISD::FCOS, MVT::f64, Expand); | ||||
| setOperationAction(ISD::FPOW, MVT::f32, Expand); | setOperationAction(ISD::FPOW, MVT::f32, Expand); | ||||
| setOperationAction(ISD::FPOW, MVT::f64, Expand); | setOperationAction(ISD::FPOW, MVT::f64, Expand); | ||||
| setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); | setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom); | ||||
| setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); | setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom); | ||||
| // f16 is a storage-only type, always promote it to f32. | setOperationAction(ISD::FREM, MVT::f16, Promote); | ||||
| setOperationAction(ISD::FPOW, MVT::f16, Promote); | |||||
| setOperationAction(ISD::FPOWI, MVT::f16, Promote); | |||||
| setOperationAction(ISD::FCOS, MVT::f16, Promote); | |||||
| setOperationAction(ISD::FSIN, MVT::f16, Promote); | |||||
| setOperationAction(ISD::FSINCOS, MVT::f16, Promote); | |||||
| setOperationAction(ISD::FEXP, MVT::f16, Promote); | |||||
| setOperationAction(ISD::FEXP2, MVT::f16, Promote); | |||||
| setOperationAction(ISD::FLOG, MVT::f16, Promote); | |||||
| setOperationAction(ISD::FLOG2, MVT::f16, Promote); | |||||
| setOperationAction(ISD::FLOG10, MVT::f16, Promote); | |||||
| setOperationAction(ISD::FCOPYSIGN, MVT::f16, Promote); | |||||
| if (!Subtarget->hasFullFP16()) { | |||||
| setOperationAction(ISD::SELECT, MVT::f16, Promote); | |||||
| setOperationAction(ISD::SELECT_CC, MVT::f16, Promote); | |||||
| setOperationAction(ISD::SETCC, MVT::f16, Promote); | setOperationAction(ISD::SETCC, MVT::f16, Promote); | ||||
| setOperationAction(ISD::BR_CC, MVT::f16, Promote); | setOperationAction(ISD::BR_CC, MVT::f16, Promote); | ||||
| setOperationAction(ISD::SELECT_CC, MVT::f16, Promote); | |||||
| setOperationAction(ISD::SELECT, MVT::f16, Promote); | |||||
| setOperationAction(ISD::FADD, MVT::f16, Promote); | setOperationAction(ISD::FADD, MVT::f16, Promote); | ||||
| setOperationAction(ISD::FSUB, MVT::f16, Promote); | setOperationAction(ISD::FSUB, MVT::f16, Promote); | ||||
| setOperationAction(ISD::FMUL, MVT::f16, Promote); | setOperationAction(ISD::FMUL, MVT::f16, Promote); | ||||
| setOperationAction(ISD::FDIV, MVT::f16, Promote); | setOperationAction(ISD::FDIV, MVT::f16, Promote); | ||||
| setOperationAction(ISD::FREM, MVT::f16, Promote); | setOperationAction(ISD::FREM, MVT::f16, Promote); | ||||
| setOperationAction(ISD::FMA, MVT::f16, Promote); | setOperationAction(ISD::FMA, MVT::f16, Promote); | ||||
| setOperationAction(ISD::FNEG, MVT::f16, Promote); | setOperationAction(ISD::FNEG, MVT::f16, Promote); | ||||
| setOperationAction(ISD::FABS, MVT::f16, Promote); | setOperationAction(ISD::FABS, MVT::f16, Promote); | ||||
| setOperationAction(ISD::FCEIL, MVT::f16, Promote); | setOperationAction(ISD::FCEIL, MVT::f16, Promote); | ||||
| setOperationAction(ISD::FCOPYSIGN, MVT::f16, Promote); | setOperationAction(ISD::FSQRT, MVT::f16, Promote); | ||||
| setOperationAction(ISD::FCOS, MVT::f16, Promote); | setOperationAction(ISD::FCOS, MVT::f16, Promote); | ||||
| setOperationAction(ISD::FFLOOR, MVT::f16, Promote); | setOperationAction(ISD::FFLOOR, MVT::f16, Promote); | ||||
| setOperationAction(ISD::FNEARBYINT, MVT::f16, Promote); | setOperationAction(ISD::FNEARBYINT, MVT::f16, Promote); | ||||
| setOperationAction(ISD::FPOW, MVT::f16, Promote); | |||||
| setOperationAction(ISD::FPOWI, MVT::f16, Promote); | |||||
| setOperationAction(ISD::FRINT, MVT::f16, Promote); | setOperationAction(ISD::FRINT, MVT::f16, Promote); | ||||
| setOperationAction(ISD::FSIN, MVT::f16, Promote); | |||||
| setOperationAction(ISD::FSINCOS, MVT::f16, Promote); | |||||
| setOperationAction(ISD::FSQRT, MVT::f16, Promote); | |||||
| setOperationAction(ISD::FEXP, MVT::f16, Promote); | |||||
| setOperationAction(ISD::FEXP2, MVT::f16, Promote); | |||||
| setOperationAction(ISD::FLOG, MVT::f16, Promote); | |||||
| setOperationAction(ISD::FLOG2, MVT::f16, Promote); | |||||
| setOperationAction(ISD::FLOG10, MVT::f16, Promote); | |||||
| setOperationAction(ISD::FROUND, MVT::f16, Promote); | setOperationAction(ISD::FROUND, MVT::f16, Promote); | ||||
| setOperationAction(ISD::FTRUNC, MVT::f16, Promote); | setOperationAction(ISD::FTRUNC, MVT::f16, Promote); | ||||
| setOperationAction(ISD::FMINNUM, MVT::f16, Promote); | setOperationAction(ISD::FMINNUM, MVT::f16, Promote); | ||||
| setOperationAction(ISD::FMAXNUM, MVT::f16, Promote); | setOperationAction(ISD::FMAXNUM, MVT::f16, Promote); | ||||
| setOperationAction(ISD::FMINNAN, MVT::f16, Promote); | setOperationAction(ISD::FMINNAN, MVT::f16, Promote); | ||||
| setOperationAction(ISD::FMAXNAN, MVT::f16, Promote); | setOperationAction(ISD::FMAXNAN, MVT::f16, Promote); | ||||
| } | |||||
| // v4f16 is also a storage-only type, so promote it to v4f32 when that is | // v4f16 is also a storage-only type, so promote it to v4f32 when that is | ||||
| // known to be safe. | // known to be safe. | ||||
| setOperationAction(ISD::FADD, MVT::v4f16, Promote); | setOperationAction(ISD::FADD, MVT::v4f16, Promote); | ||||
| setOperationAction(ISD::FSUB, MVT::v4f16, Promote); | setOperationAction(ISD::FSUB, MVT::v4f16, Promote); | ||||
| setOperationAction(ISD::FMUL, MVT::v4f16, Promote); | setOperationAction(ISD::FMUL, MVT::v4f16, Promote); | ||||
| setOperationAction(ISD::FDIV, MVT::v4f16, Promote); | setOperationAction(ISD::FDIV, MVT::v4f16, Promote); | ||||
| setOperationAction(ISD::FP_EXTEND, MVT::v4f16, Promote); | setOperationAction(ISD::FP_EXTEND, MVT::v4f16, Promote); | ||||
| ▲ Show 20 Lines • Show All 76 Lines • ▼ Show 20 Lines | for (MVT Ty : {MVT::f32, MVT::f64}) { | ||||
| setOperationAction(ISD::FTRUNC, Ty, Legal); | setOperationAction(ISD::FTRUNC, Ty, Legal); | ||||
| setOperationAction(ISD::FROUND, Ty, Legal); | setOperationAction(ISD::FROUND, Ty, Legal); | ||||
| setOperationAction(ISD::FMINNUM, Ty, Legal); | setOperationAction(ISD::FMINNUM, Ty, Legal); | ||||
| setOperationAction(ISD::FMAXNUM, Ty, Legal); | setOperationAction(ISD::FMAXNUM, Ty, Legal); | ||||
| setOperationAction(ISD::FMINNAN, Ty, Legal); | setOperationAction(ISD::FMINNAN, Ty, Legal); | ||||
| setOperationAction(ISD::FMAXNAN, Ty, Legal); | setOperationAction(ISD::FMAXNAN, Ty, Legal); | ||||
| } | } | ||||
| if (Subtarget->hasFullFP16()) { | |||||
| setOperationAction(ISD::FNEARBYINT, MVT::f16, Legal); | |||||
| setOperationAction(ISD::FFLOOR, MVT::f16, Legal); | |||||
| setOperationAction(ISD::FCEIL, MVT::f16, Legal); | |||||
| setOperationAction(ISD::FRINT, MVT::f16, Legal); | |||||
| setOperationAction(ISD::FTRUNC, MVT::f16, Legal); | |||||
| setOperationAction(ISD::FROUND, MVT::f16, Legal); | |||||
| setOperationAction(ISD::FMINNUM, MVT::f16, Legal); | |||||
| setOperationAction(ISD::FMAXNUM, MVT::f16, Legal); | |||||
| setOperationAction(ISD::FMINNAN, MVT::f16, Legal); | |||||
| setOperationAction(ISD::FMAXNAN, MVT::f16, Legal); | |||||
| } | |||||
| setOperationAction(ISD::PREFETCH, MVT::Other, Custom); | setOperationAction(ISD::PREFETCH, MVT::Other, Custom); | ||||
| setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, Custom); | setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, Custom); | ||||
| // Lower READCYCLECOUNTER using an mrs from PMCCNTR_EL0. | // Lower READCYCLECOUNTER using an mrs from PMCCNTR_EL0. | ||||
| // This requires the Performance Monitors extension. | // This requires the Performance Monitors extension. | ||||
| if (Subtarget->hasPerfMon()) | if (Subtarget->hasPerfMon()) | ||||
| setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal); | setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal); | ||||
| ▲ Show 20 Lines • Show All 314 Lines • ▼ Show 20 Lines | void AArch64TargetLowering::addTypeForNEON(MVT VT, MVT PromotedBitwiseVT) { | ||||
| if (!VT.isFloatingPoint()) | if (!VT.isFloatingPoint()) | ||||
| setOperationAction(ISD::ABS, VT, Legal); | setOperationAction(ISD::ABS, VT, Legal); | ||||
| // [SU][MIN|MAX] are available for all NEON types apart from i64. | // [SU][MIN|MAX] are available for all NEON types apart from i64. | ||||
| if (!VT.isFloatingPoint() && VT != MVT::v2i64 && VT != MVT::v1i64) | if (!VT.isFloatingPoint() && VT != MVT::v2i64 && VT != MVT::v1i64) | ||||
| for (unsigned Opcode : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}) | for (unsigned Opcode : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}) | ||||
| setOperationAction(Opcode, VT, Legal); | setOperationAction(Opcode, VT, Legal); | ||||
| // F[MIN|MAX][NUM|NAN] are available for all FP NEON types (not f16 though!). | // F[MIN|MAX][NUM|NAN] are available for all FP NEON types. | ||||
| if (VT.isFloatingPoint() && VT.getVectorElementType() != MVT::f16) | if (VT.isFloatingPoint() && | ||||
| (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16())) | |||||
| for (unsigned Opcode : {ISD::FMINNAN, ISD::FMAXNAN, | for (unsigned Opcode : {ISD::FMINNAN, ISD::FMAXNAN, | ||||
| ISD::FMINNUM, ISD::FMAXNUM}) | ISD::FMINNUM, ISD::FMAXNUM}) | ||||
| setOperationAction(Opcode, VT, Legal); | setOperationAction(Opcode, VT, Legal); | ||||
| if (Subtarget->isLittleEndian()) { | if (Subtarget->isLittleEndian()) { | ||||
| for (unsigned im = (unsigned)ISD::PRE_INC; | for (unsigned im = (unsigned)ISD::PRE_INC; | ||||
| im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) { | im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) { | ||||
| setIndexedLoadAction(im, VT, Legal); | setIndexedLoadAction(im, VT, Legal); | ||||
| ▲ Show 20 Lines • Show All 625 Lines • ▼ Show 20 Lines | |||||
| } | } | ||||
| static bool isLegalArithImmed(uint64_t C) { | static bool isLegalArithImmed(uint64_t C) { | ||||
| // Matches AArch64DAGToDAGISel::SelectArithImmed(). | // Matches AArch64DAGToDAGISel::SelectArithImmed(). | ||||
| return (C >> 12 == 0) || ((C & 0xFFFULL) == 0 && C >> 24 == 0); | return (C >> 12 == 0) || ((C & 0xFFFULL) == 0 && C >> 24 == 0); | ||||
| } | } | ||||
| static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC, | static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC, | ||||
| const SDLoc &dl, SelectionDAG &DAG) { | const SDLoc &dl, SelectionDAG &DAG, | ||||
| bool FullFP16 = false) { | |||||
| EVT VT = LHS.getValueType(); | EVT VT = LHS.getValueType(); | ||||
| if (VT.isFloatingPoint()) { | if (VT.isFloatingPoint()) { | ||||
| assert(VT != MVT::f128); | assert(VT != MVT::f128); | ||||
| if (VT == MVT::f16) { | if (VT == MVT::f16 && !FullFP16) { | ||||
| LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS); | LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS); | ||||
| RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS); | RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS); | ||||
| VT = MVT::f32; | VT = MVT::f32; | ||||
| } | } | ||||
| return DAG.getNode(AArch64ISD::FCMP, dl, VT, LHS, RHS); | return DAG.getNode(AArch64ISD::FCMP, dl, VT, LHS, RHS); | ||||
| } | } | ||||
| // The CMP instruction is just an alias for SUBS, and representing it as | // The CMP instruction is just an alias for SUBS, and representing it as | ||||
| ▲ Show 20 Lines • Show All 71 Lines • ▼ Show 20 Lines | |||||
| /// by conditional compare sequences. | /// by conditional compare sequences. | ||||
| /// @{ | /// @{ | ||||
| /// Create a conditional comparison; Use CCMP, CCMN or FCCMP as appropriate. | /// Create a conditional comparison; Use CCMP, CCMN or FCCMP as appropriate. | ||||
| static SDValue emitConditionalComparison(SDValue LHS, SDValue RHS, | static SDValue emitConditionalComparison(SDValue LHS, SDValue RHS, | ||||
| ISD::CondCode CC, SDValue CCOp, | ISD::CondCode CC, SDValue CCOp, | ||||
| AArch64CC::CondCode Predicate, | AArch64CC::CondCode Predicate, | ||||
| AArch64CC::CondCode OutCC, | AArch64CC::CondCode OutCC, | ||||
| const SDLoc &DL, SelectionDAG &DAG) { | const SDLoc &DL, SelectionDAG &DAG, | ||||
| bool FullFP16 = false) { | |||||
| unsigned Opcode = 0; | unsigned Opcode = 0; | ||||
| if (LHS.getValueType().isFloatingPoint()) { | if (LHS.getValueType().isFloatingPoint()) { | ||||
| assert(LHS.getValueType() != MVT::f128); | assert(LHS.getValueType() != MVT::f128); | ||||
| if (LHS.getValueType() == MVT::f16) { | if (LHS.getValueType() == MVT::f16 && !FullFP16) { | ||||
| LHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, LHS); | LHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, LHS); | ||||
| RHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, RHS); | RHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, RHS); | ||||
| } | } | ||||
| Opcode = AArch64ISD::FCCMP; | Opcode = AArch64ISD::FCCMP; | ||||
| } else if (RHS.getOpcode() == ISD::SUB) { | } else if (RHS.getOpcode() == ISD::SUB) { | ||||
| SDValue SubOp0 = RHS.getOperand(0); | SDValue SubOp0 = RHS.getOperand(0); | ||||
| if (isNullConstant(SubOp0) && (CC == ISD::SETEQ || CC == ISD::SETNE)) { | if (isNullConstant(SubOp0) && (CC == ISD::SETEQ || CC == ISD::SETNE)) { | ||||
| // See emitComparison() on why we can only do this for SETEQ and SETNE. | // See emitComparison() on why we can only do this for SETEQ and SETNE. | ||||
| ▲ Show 20 Lines • Show All 72 Lines • ▼ Show 20 Lines | |||||
| /// and sets @p OutCC to the flags that should be tested or returns SDValue() if | /// and sets @p OutCC to the flags that should be tested or returns SDValue() if | ||||
| /// transformation was not possible. | /// transformation was not possible. | ||||
| /// On recursive invocations @p PushNegate may be set to true to have negation | /// On recursive invocations @p PushNegate may be set to true to have negation | ||||
| /// effects pushed to the tree leafs; @p Predicate is an NZCV flag predicate | /// effects pushed to the tree leafs; @p Predicate is an NZCV flag predicate | ||||
| /// for the comparisons in the current subtree; @p Depth limits the search | /// for the comparisons in the current subtree; @p Depth limits the search | ||||
| /// depth to avoid stack overflow. | /// depth to avoid stack overflow. | ||||
| static SDValue emitConjunctionDisjunctionTreeRec(SelectionDAG &DAG, SDValue Val, | static SDValue emitConjunctionDisjunctionTreeRec(SelectionDAG &DAG, SDValue Val, | ||||
| AArch64CC::CondCode &OutCC, bool Negate, SDValue CCOp, | AArch64CC::CondCode &OutCC, bool Negate, SDValue CCOp, | ||||
| AArch64CC::CondCode Predicate) { | AArch64CC::CondCode Predicate, bool FullFP16 = false) { | ||||
| // We're at a tree leaf, produce a conditional comparison operation. | // We're at a tree leaf, produce a conditional comparison operation. | ||||
| unsigned Opcode = Val->getOpcode(); | unsigned Opcode = Val->getOpcode(); | ||||
| if (Opcode == ISD::SETCC) { | if (Opcode == ISD::SETCC) { | ||||
| SDValue LHS = Val->getOperand(0); | SDValue LHS = Val->getOperand(0); | ||||
| SDValue RHS = Val->getOperand(1); | SDValue RHS = Val->getOperand(1); | ||||
| ISD::CondCode CC = cast<CondCodeSDNode>(Val->getOperand(2))->get(); | ISD::CondCode CC = cast<CondCodeSDNode>(Val->getOperand(2))->get(); | ||||
| bool isInteger = LHS.getValueType().isInteger(); | bool isInteger = LHS.getValueType().isInteger(); | ||||
| if (Negate) | if (Negate) | ||||
| Show All 9 Lines | if (isInteger) { | ||||
| // Some floating point conditions can't be tested with a single condition | // Some floating point conditions can't be tested with a single condition | ||||
| // code. Construct an additional comparison in this case. | // code. Construct an additional comparison in this case. | ||||
| if (ExtraCC != AArch64CC::AL) { | if (ExtraCC != AArch64CC::AL) { | ||||
| SDValue ExtraCmp; | SDValue ExtraCmp; | ||||
| if (!CCOp.getNode()) | if (!CCOp.getNode()) | ||||
| ExtraCmp = emitComparison(LHS, RHS, CC, DL, DAG); | ExtraCmp = emitComparison(LHS, RHS, CC, DL, DAG); | ||||
| else | else | ||||
| ExtraCmp = emitConditionalComparison(LHS, RHS, CC, CCOp, Predicate, | ExtraCmp = emitConditionalComparison(LHS, RHS, CC, CCOp, Predicate, | ||||
| ExtraCC, DL, DAG); | ExtraCC, DL, DAG, | ||||
| FullFP16); | |||||
| CCOp = ExtraCmp; | CCOp = ExtraCmp; | ||||
| Predicate = ExtraCC; | Predicate = ExtraCC; | ||||
| } | } | ||||
| } | } | ||||
| // Produce a normal comparison if we are first in the chain | // Produce a normal comparison if we are first in the chain | ||||
| if (!CCOp) | if (!CCOp) | ||||
| return emitComparison(LHS, RHS, CC, DL, DAG); | return emitComparison(LHS, RHS, CC, DL, DAG); | ||||
| // Otherwise produce a ccmp. | // Otherwise produce a ccmp. | ||||
| return emitConditionalComparison(LHS, RHS, CC, CCOp, Predicate, OutCC, DL, | return emitConditionalComparison(LHS, RHS, CC, CCOp, Predicate, OutCC, DL, | ||||
| DAG); | DAG, FullFP16); | ||||
| } | } | ||||
| assert((Opcode == ISD::AND || (Opcode == ISD::OR && Val->hasOneUse())) && | assert((Opcode == ISD::AND || (Opcode == ISD::OR && Val->hasOneUse())) && | ||||
| "Valid conjunction/disjunction tree"); | "Valid conjunction/disjunction tree"); | ||||
| // Check if both sides can be transformed. | // Check if both sides can be transformed. | ||||
| SDValue LHS = Val->getOperand(0); | SDValue LHS = Val->getOperand(0); | ||||
| SDValue RHS = Val->getOperand(1); | SDValue RHS = Val->getOperand(1); | ||||
| Show All 30 Lines | if (NeedsNegOutL) | ||||
| std::swap(LHS, RHS); | std::swap(LHS, RHS); | ||||
| } | } | ||||
| // Emit RHS. If we want to negate the tree we only need to push a negate | // Emit RHS. If we want to negate the tree we only need to push a negate | ||||
| // through if we are already in a PushNegate case, otherwise we can negate | // through if we are already in a PushNegate case, otherwise we can negate | ||||
| // the "flags to test" afterwards. | // the "flags to test" afterwards. | ||||
| AArch64CC::CondCode RHSCC; | AArch64CC::CondCode RHSCC; | ||||
| SDValue CmpR = emitConjunctionDisjunctionTreeRec(DAG, RHS, RHSCC, Negate, | SDValue CmpR = emitConjunctionDisjunctionTreeRec(DAG, RHS, RHSCC, Negate, | ||||
| CCOp, Predicate); | CCOp, Predicate, FullFP16); | ||||
| if (NegateOpsAndResult && !Negate) | if (NegateOpsAndResult && !Negate) | ||||
| RHSCC = AArch64CC::getInvertedCondCode(RHSCC); | RHSCC = AArch64CC::getInvertedCondCode(RHSCC); | ||||
| // Emit LHS. We may need to negate it. | // Emit LHS. We may need to negate it. | ||||
| SDValue CmpL = emitConjunctionDisjunctionTreeRec(DAG, LHS, OutCC, | SDValue CmpL = emitConjunctionDisjunctionTreeRec(DAG, LHS, OutCC, | ||||
| NegateOpsAndResult, CmpR, | NegateOpsAndResult, CmpR, | ||||
| RHSCC); | RHSCC, FullFP16); | ||||
| // If we transformed an OR to and AND then we have to negate the result | // If we transformed an OR to and AND then we have to negate the result | ||||
| // (or absorb the Negate parameter). | // (or absorb the Negate parameter). | ||||
| if (NegateOpsAndResult && !Negate) | if (NegateOpsAndResult && !Negate) | ||||
| OutCC = AArch64CC::getInvertedCondCode(OutCC); | OutCC = AArch64CC::getInvertedCondCode(OutCC); | ||||
| return CmpL; | return CmpL; | ||||
| } | } | ||||
| /// Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain | /// Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain | ||||
| /// of CCMP/CFCMP ops. See @ref AArch64CCMP. | /// of CCMP/CFCMP ops. See @ref AArch64CCMP. | ||||
| /// \see emitConjunctionDisjunctionTreeRec(). | /// \see emitConjunctionDisjunctionTreeRec(). | ||||
| static SDValue emitConjunctionDisjunctionTree(SelectionDAG &DAG, SDValue Val, | static SDValue emitConjunctionDisjunctionTree(SelectionDAG &DAG, SDValue Val, | ||||
| AArch64CC::CondCode &OutCC) { | AArch64CC::CondCode &OutCC, | ||||
| bool FullFP16 = false) { | |||||
| bool CanNegate; | bool CanNegate; | ||||
| if (!isConjunctionDisjunctionTree(Val, CanNegate)) | if (!isConjunctionDisjunctionTree(Val, CanNegate)) | ||||
| return SDValue(); | return SDValue(); | ||||
| return emitConjunctionDisjunctionTreeRec(DAG, Val, OutCC, false, SDValue(), | return emitConjunctionDisjunctionTreeRec(DAG, Val, OutCC, false, SDValue(), | ||||
| AArch64CC::AL); | AArch64CC::AL, FullFP16); | ||||
| } | } | ||||
| /// @} | /// @} | ||||
| static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC, | static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC, | ||||
| SDValue &AArch64cc, SelectionDAG &DAG, | SDValue &AArch64cc, SelectionDAG &DAG, | ||||
| const SDLoc &dl) { | const SDLoc &dl, bool FullFP16 = false) { | ||||
| if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS.getNode())) { | if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS.getNode())) { | ||||
| EVT VT = RHS.getValueType(); | EVT VT = RHS.getValueType(); | ||||
| uint64_t C = RHSC->getZExtValue(); | uint64_t C = RHSC->getZExtValue(); | ||||
| if (!isLegalArithImmed(C)) { | if (!isLegalArithImmed(C)) { | ||||
| // Constant does not fit, try adjusting it by one? | // Constant does not fit, try adjusting it by one? | ||||
| switch (CC) { | switch (CC) { | ||||
| default: | default: | ||||
| break; | break; | ||||
| ▲ Show 20 Lines • Show All 70 Lines • ▼ Show 20 Lines | if ((RHSC->getZExtValue() >> 16 == 0) && isa<LoadSDNode>(LHS) && | ||||
| LHS.getNode()->hasNUsesOfValue(1, 0)) { | LHS.getNode()->hasNUsesOfValue(1, 0)) { | ||||
| int16_t ValueofRHS = cast<ConstantSDNode>(RHS)->getZExtValue(); | int16_t ValueofRHS = cast<ConstantSDNode>(RHS)->getZExtValue(); | ||||
| if (ValueofRHS < 0 && isLegalArithImmed(-ValueofRHS)) { | if (ValueofRHS < 0 && isLegalArithImmed(-ValueofRHS)) { | ||||
| SDValue SExt = | SDValue SExt = | ||||
| DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, LHS.getValueType(), LHS, | DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, LHS.getValueType(), LHS, | ||||
| DAG.getValueType(MVT::i16)); | DAG.getValueType(MVT::i16)); | ||||
| Cmp = emitComparison(SExt, DAG.getConstant(ValueofRHS, dl, | Cmp = emitComparison(SExt, DAG.getConstant(ValueofRHS, dl, | ||||
| RHS.getValueType()), | RHS.getValueType()), | ||||
| CC, dl, DAG); | CC, dl, DAG, FullFP16); | ||||
| AArch64CC = changeIntCCToAArch64CC(CC); | AArch64CC = changeIntCCToAArch64CC(CC); | ||||
| } | } | ||||
| } | } | ||||
| if (!Cmp && (RHSC->isNullValue() || RHSC->isOne())) { | if (!Cmp && (RHSC->isNullValue() || RHSC->isOne())) { | ||||
| if ((Cmp = emitConjunctionDisjunctionTree(DAG, LHS, AArch64CC))) { | if ((Cmp = emitConjunctionDisjunctionTree(DAG, LHS, AArch64CC, FullFP16))) { | ||||
| if ((CC == ISD::SETNE) ^ RHSC->isNullValue()) | if ((CC == ISD::SETNE) ^ RHSC->isNullValue()) | ||||
| AArch64CC = AArch64CC::getInvertedCondCode(AArch64CC); | AArch64CC = AArch64CC::getInvertedCondCode(AArch64CC); | ||||
| } | } | ||||
| } | } | ||||
| } | } | ||||
| if (!Cmp) { | if (!Cmp) { | ||||
| Cmp = emitComparison(LHS, RHS, CC, dl, DAG); | Cmp = emitComparison(LHS, RHS, CC, dl, DAG, FullFP16); | ||||
| AArch64CC = changeIntCCToAArch64CC(CC); | AArch64CC = changeIntCCToAArch64CC(CC); | ||||
| } | } | ||||
| AArch64cc = DAG.getConstant(AArch64CC, dl, MVT_CC); | AArch64cc = DAG.getConstant(AArch64CC, dl, MVT_CC); | ||||
| return Cmp; | return Cmp; | ||||
| } | } | ||||
| static std::pair<SDValue, SDValue> | static std::pair<SDValue, SDValue> | ||||
| getAArch64XALUOOp(AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG) { | getAArch64XALUOOp(AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG) { | ||||
| ▲ Show 20 Lines • Show All 2,175 Lines • ▼ Show 20 Lines | if (RHSC && RHSC->getSExtValue() == -1 && CC == ISD::SETGT && | ||||
| // (a.k.a. TST) and the test in the test bit and branch instruction | // (a.k.a. TST) and the test in the test bit and branch instruction | ||||
| // becomes redundant. This would also increase register pressure. | // becomes redundant. This would also increase register pressure. | ||||
| uint64_t Mask = LHS.getValueSizeInBits() - 1; | uint64_t Mask = LHS.getValueSizeInBits() - 1; | ||||
| return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, LHS, | return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, LHS, | ||||
| DAG.getConstant(Mask, dl, MVT::i64), Dest); | DAG.getConstant(Mask, dl, MVT::i64), Dest); | ||||
| } | } | ||||
| SDValue CCVal; | SDValue CCVal; | ||||
| SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl); | SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl, | ||||
| Subtarget->hasFullFP16()); | |||||
| return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, | return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CCVal, | ||||
| Cmp); | Cmp); | ||||
| } | } | ||||
| assert(LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64); | assert(LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 || | ||||
| LHS.getValueType() == MVT::f64); | |||||
| // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally | // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally | ||||
| // clean. Some of them require two branches to implement. | // clean. Some of them require two branches to implement. | ||||
| SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG); | SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG, Subtarget->hasFullFP16()); | ||||
| AArch64CC::CondCode CC1, CC2; | AArch64CC::CondCode CC1, CC2; | ||||
| changeFPCCToAArch64CC(CC, CC1, CC2); | changeFPCCToAArch64CC(CC, CC1, CC2); | ||||
| SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32); | SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32); | ||||
| SDValue BR1 = | SDValue BR1 = | ||||
| DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CC1Val, Cmp); | DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, Chain, Dest, CC1Val, Cmp); | ||||
| if (CC2 != AArch64CC::AL) { | if (CC2 != AArch64CC::AL) { | ||||
| SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32); | SDValue CC2Val = DAG.getConstant(CC2, dl, MVT::i32); | ||||
| return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, BR1, Dest, CC2Val, | return DAG.getNode(AArch64ISD::BRCOND, dl, MVT::Other, BR1, Dest, CC2Val, | ||||
| ▲ Show 20 Lines • Show All 138 Lines • ▼ Show 20 Lines | if (!RHS.getNode()) { | ||||
| "Unexpected setcc expansion!"); | "Unexpected setcc expansion!"); | ||||
| return LHS; | return LHS; | ||||
| } | } | ||||
| } | } | ||||
| if (LHS.getValueType().isInteger()) { | if (LHS.getValueType().isInteger()) { | ||||
| SDValue CCVal; | SDValue CCVal; | ||||
| SDValue Cmp = | SDValue Cmp = | ||||
| getAArch64Cmp(LHS, RHS, ISD::getSetCCInverse(CC, true), CCVal, DAG, dl); | getAArch64Cmp(LHS, RHS, ISD::getSetCCInverse(CC, true), CCVal, DAG, dl, | ||||
| Subtarget->hasFullFP16()); | |||||
| // Note that we inverted the condition above, so we reverse the order of | // Note that we inverted the condition above, so we reverse the order of | ||||
| // the true and false operands here. This will allow the setcc to be | // the true and false operands here. This will allow the setcc to be | ||||
| // matched to a single CSINC instruction. | // matched to a single CSINC instruction. | ||||
| return DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CCVal, Cmp); | return DAG.getNode(AArch64ISD::CSEL, dl, VT, FVal, TVal, CCVal, Cmp); | ||||
| } | } | ||||
| // Now we know we're dealing with FP values. | // Now we know we're dealing with FP values. | ||||
| assert(LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64); | assert(LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 || | ||||
| LHS.getValueType() == MVT::f64); | |||||
| // If that fails, we'll need to perform an FCMP + CSEL sequence. Go ahead | // If that fails, we'll need to perform an FCMP + CSEL sequence. Go ahead | ||||
| // and do the comparison. | // and do the comparison. | ||||
| SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG); | SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG, Subtarget->hasFullFP16()); | ||||
| AArch64CC::CondCode CC1, CC2; | AArch64CC::CondCode CC1, CC2; | ||||
| changeFPCCToAArch64CC(CC, CC1, CC2); | changeFPCCToAArch64CC(CC, CC1, CC2); | ||||
| if (CC2 == AArch64CC::AL) { | if (CC2 == AArch64CC::AL) { | ||||
| changeFPCCToAArch64CC(ISD::getSetCCInverse(CC, false), CC1, CC2); | changeFPCCToAArch64CC(ISD::getSetCCInverse(CC, false), CC1, CC2); | ||||
| SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32); | SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32); | ||||
| // Note that we inverted the condition above, so we reverse the order of | // Note that we inverted the condition above, so we reverse the order of | ||||
| Show All 29 Lines | if (LHS.getValueType() == MVT::f128) { | ||||
| // against zero to select between true and false values. | // against zero to select between true and false values. | ||||
| if (!RHS.getNode()) { | if (!RHS.getNode()) { | ||||
| RHS = DAG.getConstant(0, dl, LHS.getValueType()); | RHS = DAG.getConstant(0, dl, LHS.getValueType()); | ||||
| CC = ISD::SETNE; | CC = ISD::SETNE; | ||||
| } | } | ||||
| } | } | ||||
| // Also handle f16, for which we need to do a f32 comparison. | // Also handle f16, for which we need to do a f32 comparison. | ||||
| if (LHS.getValueType() == MVT::f16) { | if (LHS.getValueType() == MVT::f16 && !Subtarget->hasFullFP16()) { | ||||
| LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS); | LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS); | ||||
| RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS); | RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS); | ||||
| } | } | ||||
| // Next, handle integers. | // Next, handle integers. | ||||
| if (LHS.getValueType().isInteger()) { | if (LHS.getValueType().isInteger()) { | ||||
| assert((LHS.getValueType() == RHS.getValueType()) && | assert((LHS.getValueType() == RHS.getValueType()) && | ||||
| (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64)); | (LHS.getValueType() == MVT::i32 || LHS.getValueType() == MVT::i64)); | ||||
| ▲ Show 20 Lines • Show All 103 Lines • ▼ Show 20 Lines | if (Opcode == AArch64ISD::CSEL && RHSVal && !RHSVal->isOne() && | ||||
| if (CTVal == RHSVal && AArch64CC == AArch64CC::EQ) { | if (CTVal == RHSVal && AArch64CC == AArch64CC::EQ) { | ||||
| Opcode = AArch64ISD::CSINV; | Opcode = AArch64ISD::CSINV; | ||||
| TVal = LHS; | TVal = LHS; | ||||
| FVal = DAG.getConstant(0, dl, FVal.getValueType()); | FVal = DAG.getConstant(0, dl, FVal.getValueType()); | ||||
| } | } | ||||
| } | } | ||||
| SDValue CCVal; | SDValue CCVal; | ||||
| SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl); | SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl, | ||||
| Subtarget->hasFullFP16()); | |||||
| EVT VT = TVal.getValueType(); | EVT VT = TVal.getValueType(); | ||||
| return DAG.getNode(Opcode, dl, VT, TVal, FVal, CCVal, Cmp); | return DAG.getNode(Opcode, dl, VT, TVal, FVal, CCVal, Cmp); | ||||
| } | } | ||||
| // Now we know we're dealing with FP values. | // Now we know we're dealing with FP values. | ||||
| assert(LHS.getValueType() == MVT::f32 || LHS.getValueType() == MVT::f64); | assert(LHS.getValueType() == MVT::f16 || LHS.getValueType() == MVT::f32 || | ||||
| LHS.getValueType() == MVT::f64); | |||||
| assert(LHS.getValueType() == RHS.getValueType()); | assert(LHS.getValueType() == RHS.getValueType()); | ||||
| EVT VT = TVal.getValueType(); | EVT VT = TVal.getValueType(); | ||||
| SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG); | SDValue Cmp = emitComparison(LHS, RHS, CC, dl, DAG, Subtarget->hasFullFP16()); | ||||
| // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally | // Unfortunately, the mapping of LLVM FP CC's onto AArch64 CC's isn't totally | ||||
| // clean. Some of them require two CSELs to implement. | // clean. Some of them require two CSELs to implement. | ||||
| AArch64CC::CondCode CC1, CC2; | AArch64CC::CondCode CC1, CC2; | ||||
| changeFPCCToAArch64CC(CC, CC1, CC2); | changeFPCCToAArch64CC(CC, CC1, CC2); | ||||
| if (DAG.getTarget().Options.UnsafeFPMath) { | if (DAG.getTarget().Options.UnsafeFPMath) { | ||||
| // Transform "a == 0.0 ? 0.0 : x" to "a == 0.0 ? a : x" and | // Transform "a == 0.0 ? 0.0 : x" to "a == 0.0 ? a : x" and | ||||
| ▲ Show 20 Lines • Show All 481 Lines • ▼ Show 20 Lines | bool AArch64TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { | ||||
| // FIXME: We should be able to handle f128 as well with a clever lowering. | // FIXME: We should be able to handle f128 as well with a clever lowering. | ||||
| if (Imm.isPosZero() && (VT == MVT::f64 || VT == MVT::f32)) | if (Imm.isPosZero() && (VT == MVT::f64 || VT == MVT::f32)) | ||||
| return true; | return true; | ||||
| if (VT == MVT::f64) | if (VT == MVT::f64) | ||||
| return AArch64_AM::getFP64Imm(Imm) != -1; | return AArch64_AM::getFP64Imm(Imm) != -1; | ||||
| else if (VT == MVT::f32) | else if (VT == MVT::f32) | ||||
| return AArch64_AM::getFP32Imm(Imm) != -1; | return AArch64_AM::getFP32Imm(Imm) != -1; | ||||
| else if (VT == MVT::f16 && Subtarget->hasFullFP16()) | |||||
| return AArch64_AM::getFP16Imm(Imm) != -1; | |||||
| return false; | return false; | ||||
| } | } | ||||
| //===----------------------------------------------------------------------===// | //===----------------------------------------------------------------------===// | ||||
| // AArch64 Optimization Hooks | // AArch64 Optimization Hooks | ||||
| //===----------------------------------------------------------------------===// | //===----------------------------------------------------------------------===// | ||||
| static SDValue getEstimate(const AArch64Subtarget *ST, unsigned Opcode, | static SDValue getEstimate(const AArch64Subtarget *ST, unsigned Opcode, | ||||
| ▲ Show 20 Lines • Show All 5,961 Lines • Show Last 20 Lines | |||||