Page MenuHomePhabricator

Please use GitHub pull requests for new patches. Avoid migrating existing patches. Phabricator shutdown timeline

Differential D73863 Diff 242229 llvm/lib/Target/X86/X86ISelLowering.cpp

Changeset View

Standalone View

View Options

llvm/lib/Target/X86/X86ISelLowering.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 20,932 Lines • ▼ Show 20 Linesstatic std::pair<SDValue, SDValue> EmitCmp(SDValue Op0, SDValue Op1,
} }
// Use SUB instead of CMP to enable CSE between SUB and CMP. // Use SUB instead of CMP to enable CSE between SUB and CMP.
SDVTList VTs = DAG.getVTList(CmpVT, MVT::i32); SDVTList VTs = DAG.getVTList(CmpVT, MVT::i32);
SDValue Sub = DAG.getNode(X86ISD::SUB, dl, VTs, Op0, Op1); SDValue Sub = DAG.getNode(X86ISD::SUB, dl, VTs, Op0, Op1);
return std::make_pair(Sub.getValue(1), SDValue()); return std::make_pair(Sub.getValue(1), SDValue());
} }
/// Convert a comparison if required by the subtarget.
SDValue X86TargetLowering::ConvertCmpIfNecessary(SDValue Cmp,
SelectionDAG &DAG) const {
// If the subtarget does not support the FUCOMI instruction, floating-point
// comparisons have to be converted.
bool IsCmp = Cmp.getOpcode() == X86ISD::CMP;
bool IsStrictCmp = Cmp.getOpcode() == X86ISD::STRICT_FCMP ||
Cmp.getOpcode() == X86ISD::STRICT_FCMPS;
if (Subtarget.hasCMov() || (!IsCmp && !IsStrictCmp) ||
!Cmp.getOperand(IsStrictCmp ? 1 : 0).getValueType().isFloatingPoint() ||
!Cmp.getOperand(IsStrictCmp ? 2 : 1).getValueType().isFloatingPoint())
return Cmp;
// The instruction selector will select an FUCOM instruction instead of
// FUCOMI, which writes the comparison result to FPSW instead of EFLAGS. Hence
// build an SDNode sequence that transfers the result from FPSW into EFLAGS:
// (X86sahf (trunc (srl (X86fp_stsw (trunc (X86any_fcmp ...)), 8))))
SDLoc dl(Cmp);
SDValue TruncFPSW = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Cmp);
SDValue FNStSW = DAG.getNode(X86ISD::FNSTSW16r, dl, MVT::i16, TruncFPSW);
SDValue Srl = DAG.getNode(ISD::SRL, dl, MVT::i16, FNStSW,
DAG.getConstant(8, dl, MVT::i8));
SDValue TruncSrl = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Srl);
// Some 64-bit targets lack SAHF support, but they do support FCOMI.
assert(Subtarget.hasLAHFSAHF() && "Target doesn't support SAHF or FCOMI?");
return DAG.getNode(X86ISD::SAHF, dl, MVT::i32, TruncSrl);
}
/// Check if replacement of SQRT with RSQRT should be disabled. /// Check if replacement of SQRT with RSQRT should be disabled.
bool X86TargetLowering::isFsqrtCheap(SDValue Op, SelectionDAG &DAG) const { bool X86TargetLowering::isFsqrtCheap(SDValue Op, SelectionDAG &DAG) const {
EVT VT = Op.getValueType(); EVT VT = Op.getValueType();
// We never want to use both SQRT and RSQRT instructions for the same input. // We never want to use both SQRT and RSQRT instructions for the same input.
if (DAG.getNodeIfExists(X86ISD::FRSQRT, DAG.getVTList(VT), Op)) if (DAG.getNodeIfExists(X86ISD::FRSQRT, DAG.getVTList(VT), Op))
return false; return false;
▲ Show 20 Lines • Show All 947 Lines • ▼ Show 20 LinesSDValue X86TargetLowering::emitFlagsForSetcc(SDValue Op0, SDValue Op1,
if (CondCode == X86::COND_INVALID) if (CondCode == X86::COND_INVALID)
return SDValue(); return SDValue();
std::pair<SDValue, SDValue> Tmp = std::pair<SDValue, SDValue> Tmp =
EmitCmp(Op0, Op1, CondCode, dl, DAG, Subtarget, Chain, IsSignaling); EmitCmp(Op0, Op1, CondCode, dl, DAG, Subtarget, Chain, IsSignaling);
SDValue EFLAGS = Tmp.first; SDValue EFLAGS = Tmp.first;
if (Chain) if (Chain)
Chain = Tmp.second; Chain = Tmp.second;
EFLAGS = ConvertCmpIfNecessary(EFLAGS, DAG);
X86CC = DAG.getTargetConstant(CondCode, dl, MVT::i8); X86CC = DAG.getTargetConstant(CondCode, dl, MVT::i8);
return EFLAGS; return EFLAGS;
} }
SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const { SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
bool IsStrict = Op.getOpcode() == ISD::STRICT_FSETCC || bool IsStrict = Op.getOpcode() == ISD::STRICT_FSETCC ||
Op.getOpcode() == ISD::STRICT_FSETCCS; Op.getOpcode() == ISD::STRICT_FSETCCS;
▲ Show 20 Lines • Show All 123 Lines • ▼ Show 20 Linesstatic SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
SDValue SetCC = getSETCC(Cond, Overflow, DL, DAG); SDValue SetCC = getSETCC(Cond, Overflow, DL, DAG);
assert(Op->getValueType(1) == MVT::i8 && "Unexpected VT!"); assert(Op->getValueType(1) == MVT::i8 && "Unexpected VT!");
return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), Value, SetCC); return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), Value, SetCC);
} }
/// Return true if opcode is a X86 logical comparison. /// Return true if opcode is a X86 logical comparison.
static bool isX86LogicalCmp(SDValue Op) { static bool isX86LogicalCmp(SDValue Op) {
unsigned Opc = Op.getOpcode(); unsigned Opc = Op.getOpcode();
if (Opc == X86ISD::CMP || Opc == X86ISD::COMI || Opc == X86ISD::UCOMI || if (Opc == X86ISD::CMP || Opc == X86ISD::COMI || Opc == X86ISD::UCOMI)
Opc == X86ISD::SAHF)
return true; return true;
if (Op.getResNo() == 1 && if (Op.getResNo() == 1 &&
(Opc == X86ISD::ADD || Opc == X86ISD::SUB || Opc == X86ISD::ADC || (Opc == X86ISD::ADD || Opc == X86ISD::SUB || Opc == X86ISD::ADC ||
Opc == X86ISD::SBB || Opc == X86ISD::SMUL || Opc == X86ISD::UMUL || Opc == X86ISD::SBB || Opc == X86ISD::SMUL || Opc == X86ISD::UMUL ||
Opc == X86ISD::OR || Opc == X86ISD::XOR || Opc == X86ISD::AND)) Opc == X86ISD::OR || Opc == X86ISD::XOR || Opc == X86ISD::AND))
return true; return true;
return false; return false;
▲ Show 20 Lines • Show All 935 Lines • ▼ Show 20 Lines if (Cond.hasOneUse() && isAndOrOfSetCCs(Cond, CondOpc)) {
SDNode *NewBR = SDNode *NewBR =
DAG.UpdateNodeOperands(User, User->getOperand(0), Dest); DAG.UpdateNodeOperands(User, User->getOperand(0), Dest);
assert(NewBR == User); assert(NewBR == User);
(void)NewBR; (void)NewBR;
Dest = FalseBB; Dest = FalseBB;
SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32, SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
Cond.getOperand(0), Cond.getOperand(1)); Cond.getOperand(0), Cond.getOperand(1));
Cmp = ConvertCmpIfNecessary(Cmp, DAG);
CC = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8); CC = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(), Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
Chain, Dest, CC, Cmp); Chain, Dest, CC, Cmp);
CC = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8); CC = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8);
Cond = Cmp; Cond = Cmp;
addTest = false; addTest = false;
} }
} }
} else if (Cond.getOpcode() == ISD::SETCC && } else if (Cond.getOpcode() == ISD::SETCC &&
cast<CondCodeSDNode>(Cond.getOperand(2))->get() == ISD::SETUNE) { cast<CondCodeSDNode>(Cond.getOperand(2))->get() == ISD::SETUNE) {
// For FCMP_UNE, we can emit // For FCMP_UNE, we can emit
// two branches instead of an explicit OR instruction with a // two branches instead of an explicit OR instruction with a
// separate test. // separate test.
SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32, SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
Cond.getOperand(0), Cond.getOperand(1)); Cond.getOperand(0), Cond.getOperand(1));
Cmp = ConvertCmpIfNecessary(Cmp, DAG);
CC = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8); CC = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(), Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
Chain, Dest, CC, Cmp); Chain, Dest, CC, Cmp);
CC = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8); CC = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8);
Cond = Cmp; Cond = Cmp;
addTest = false; addTest = false;
} }
} }
Show All 15 Lines if (addTest) {
} }
} }
if (addTest) { if (addTest) {
X86::CondCode X86Cond = Inverted ? X86::COND_E : X86::COND_NE; X86::CondCode X86Cond = Inverted ? X86::COND_E : X86::COND_NE;
CC = DAG.getTargetConstant(X86Cond, dl, MVT::i8); CC = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
Cond = EmitTest(Cond, X86Cond, dl, DAG, Subtarget); Cond = EmitTest(Cond, X86Cond, dl, DAG, Subtarget);
} }
Cond = ConvertCmpIfNecessary(Cond, DAG);
return DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(), return DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
Chain, Dest, CC, Cond); Chain, Dest, CC, Cond);
} }
// Lower dynamic stack allocation to _alloca call for Cygwin/Mingw targets. // Lower dynamic stack allocation to _alloca call for Cygwin/Mingw targets.
// Calls to _alloca are needed to probe the stack when allocating more than 4k // Calls to _alloca are needed to probe the stack when allocating more than 4k
// bytes in one go. Touching the stack at 4K increments is necessary to ensure // bytes in one go. Touching the stack at 4K increments is necessary to ensure
// that the guard pages used by the OS virtual memory manager are allocated in // that the guard pages used by the OS virtual memory manager are allocated in
▲ Show 20 Lines • Show All 6,630 Lines • ▼ Show 20 Lines#define NODE_NAME_CASE(NODE) case X86ISD::NODE: return "X86ISD::" #NODE;
NODE_NAME_CASE(TLSBASEADDR) NODE_NAME_CASE(TLSBASEADDR)
NODE_NAME_CASE(TLSCALL) NODE_NAME_CASE(TLSCALL)
NODE_NAME_CASE(EH_SJLJ_SETJMP) NODE_NAME_CASE(EH_SJLJ_SETJMP)
NODE_NAME_CASE(EH_SJLJ_LONGJMP) NODE_NAME_CASE(EH_SJLJ_LONGJMP)
NODE_NAME_CASE(EH_SJLJ_SETUP_DISPATCH) NODE_NAME_CASE(EH_SJLJ_SETUP_DISPATCH)
NODE_NAME_CASE(EH_RETURN) NODE_NAME_CASE(EH_RETURN)
NODE_NAME_CASE(TC_RETURN) NODE_NAME_CASE(TC_RETURN)
NODE_NAME_CASE(FNSTCW16m) NODE_NAME_CASE(FNSTCW16m)
NODE_NAME_CASE(FNSTSW16r)
NODE_NAME_CASE(LCMPXCHG_DAG) NODE_NAME_CASE(LCMPXCHG_DAG)
NODE_NAME_CASE(LCMPXCHG8_DAG) NODE_NAME_CASE(LCMPXCHG8_DAG)
NODE_NAME_CASE(LCMPXCHG16_DAG) NODE_NAME_CASE(LCMPXCHG16_DAG)
NODE_NAME_CASE(LCMPXCHG8_SAVE_EBX_DAG) NODE_NAME_CASE(LCMPXCHG8_SAVE_EBX_DAG)
NODE_NAME_CASE(LCMPXCHG16_SAVE_RBX_DAG) NODE_NAME_CASE(LCMPXCHG16_SAVE_RBX_DAG)
NODE_NAME_CASE(LADD) NODE_NAME_CASE(LADD)
NODE_NAME_CASE(LSUB) NODE_NAME_CASE(LSUB)
NODE_NAME_CASE(LOR) NODE_NAME_CASE(LOR)
▲ Show 20 Lines • Show All 108 Lines • ▼ Show 20 Lines#define NODE_NAME_CASE(NODE) case X86ISD::NODE: return "X86ISD::" #NODE;
NODE_NAME_CASE(PSADBW) NODE_NAME_CASE(PSADBW)
NODE_NAME_CASE(DBPSADBW) NODE_NAME_CASE(DBPSADBW)
NODE_NAME_CASE(VASTART_SAVE_XMM_REGS) NODE_NAME_CASE(VASTART_SAVE_XMM_REGS)
NODE_NAME_CASE(VAARG_64) NODE_NAME_CASE(VAARG_64)
NODE_NAME_CASE(WIN_ALLOCA) NODE_NAME_CASE(WIN_ALLOCA)
NODE_NAME_CASE(MEMBARRIER) NODE_NAME_CASE(MEMBARRIER)
NODE_NAME_CASE(MFENCE) NODE_NAME_CASE(MFENCE)
NODE_NAME_CASE(SEG_ALLOCA) NODE_NAME_CASE(SEG_ALLOCA)
NODE_NAME_CASE(SAHF)
NODE_NAME_CASE(RDRAND) NODE_NAME_CASE(RDRAND)
NODE_NAME_CASE(RDSEED) NODE_NAME_CASE(RDSEED)
NODE_NAME_CASE(RDPKRU) NODE_NAME_CASE(RDPKRU)
NODE_NAME_CASE(WRPKRU) NODE_NAME_CASE(WRPKRU)
NODE_NAME_CASE(VPMADDUBSW) NODE_NAME_CASE(VPMADDUBSW)
NODE_NAME_CASE(VPMADDWD) NODE_NAME_CASE(VPMADDWD)
NODE_NAME_CASE(VPSHA) NODE_NAME_CASE(VPSHA)
NODE_NAME_CASE(VPSHL) NODE_NAME_CASE(VPSHL)
▲ Show 20 Lines • Show All 17,729 Lines • Show Last 20 Lines