Index: llvm/trunk/lib/Target/X86/X86ISelLowering.h =================================================================== --- llvm/trunk/lib/Target/X86/X86ISelLowering.h +++ llvm/trunk/lib/Target/X86/X86ISelLowering.h @@ -1282,12 +1282,15 @@ const unsigned char OpFlags = 0) const; SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const; SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerGlobalAddress(const GlobalValue *GV, const SDLoc &dl, - int64_t Offset, SelectionDAG &DAG) const; SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const; SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const; SDValue LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const; + /// Creates target global address or external symbol nodes for calls or + /// other uses. + SDValue LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG, + bool ForCall) const; + SDValue LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) const; SDValue LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG) const; SDValue LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const; Index: llvm/trunk/lib/Target/X86/X86ISelLowering.cpp =================================================================== --- llvm/trunk/lib/Target/X86/X86ISelLowering.cpp +++ llvm/trunk/lib/Target/X86/X86ISelLowering.cpp @@ -3956,46 +3956,13 @@ // through a register, since the call instruction's 32-bit // pc-relative offset may not be large enough to hold the whole // address. - } else if (Callee->getOpcode() == ISD::GlobalAddress) { - // If the callee is a GlobalAddress node (quite common, every direct call - // is) turn it into a TargetGlobalAddress node so that legalize doesn't hack - // it. - GlobalAddressSDNode* G = cast(Callee); - - // We should use extra load for direct calls to dllimported functions in - // non-JIT mode. - const GlobalValue *GV = G->getGlobal(); - if (!GV->hasDLLImportStorageClass()) { - unsigned char OpFlags = Subtarget.classifyGlobalFunctionReference(GV); - - Callee = DAG.getTargetGlobalAddress( - GV, dl, getPointerTy(DAG.getDataLayout()), G->getOffset(), OpFlags); - - if (isGlobalStubReference(OpFlags)) { - // Add a wrapper. - Callee = DAG.getNode(getGlobalWrapperKind(GV, OpFlags), dl, - getPointerTy(DAG.getDataLayout()), Callee); - // Add extra indirection - Callee = DAG.getLoad( - getPointerTy(DAG.getDataLayout()), dl, DAG.getEntryNode(), Callee, - MachinePointerInfo::getGOT(DAG.getMachineFunction())); - } - } - } else if (ExternalSymbolSDNode *S = dyn_cast(Callee)) { - const Module *Mod = DAG.getMachineFunction().getFunction().getParent(); - unsigned char OpFlags = - Subtarget.classifyGlobalFunctionReference(nullptr, *Mod); - - Callee = DAG.getTargetExternalSymbol( - S->getSymbol(), getPointerTy(DAG.getDataLayout()), OpFlags); - - if (isGlobalStubReference(OpFlags)) { - Callee = DAG.getNode(getGlobalWrapperKind(nullptr, OpFlags), dl, - getPointerTy(DAG.getDataLayout()), Callee); - Callee = DAG.getLoad( - getPointerTy(DAG.getDataLayout()), dl, DAG.getEntryNode(), Callee, - MachinePointerInfo::getGOT(DAG.getMachineFunction())); - } + } else if (Callee->getOpcode() == ISD::GlobalAddress || + Callee->getOpcode() == ISD::ExternalSymbol) { + // Lower direct calls to global addresses and external symbols. Setting + // ForCall to true here has the effect of removing WrapperRIP when possible + // to allow direct calls to be selected without first materializing the + // address into a register. + Callee = LowerGlobalOrExternal(Callee, DAG, /*ForCall=*/true); } else if (Subtarget.isTarget64BitILP32() && Callee->getValueType(0) == MVT::i32) { // Zero-extend the 32-bit Callee address into a 64-bit according to x32 ABI @@ -17280,35 +17247,9 @@ return Result; } -SDValue -X86TargetLowering::LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const { - const char *Sym = cast(Op)->getSymbol(); - - // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the - // global base reg. - const Module *Mod = DAG.getMachineFunction().getFunction().getParent(); - unsigned char OpFlag = Subtarget.classifyGlobalReference(nullptr, *Mod); - - auto PtrVT = getPointerTy(DAG.getDataLayout()); - SDValue Result = DAG.getTargetExternalSymbol(Sym, PtrVT, OpFlag); - - SDLoc DL(Op); - Result = DAG.getNode(getGlobalWrapperKind(), DL, PtrVT, Result); - - // With PIC, the address is actually $g + Offset. - if (OpFlag) { - Result = - DAG.getNode(ISD::ADD, DL, PtrVT, - DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), Result); - } - - // For symbols that require a load from a stub to get the address, emit the - // load. - if (isGlobalStubReference(OpFlag)) - Result = DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), Result, - MachinePointerInfo::getGOT(DAG.getMachineFunction())); - - return Result; +SDValue X86TargetLowering::LowerExternalSymbol(SDValue Op, + SelectionDAG &DAG) const { + return LowerGlobalOrExternal(Op, DAG, /*ForCall=*/false); } SDValue @@ -17332,35 +17273,67 @@ return Result; } -SDValue X86TargetLowering::LowerGlobalAddress(const GlobalValue *GV, - const SDLoc &dl, int64_t Offset, - SelectionDAG &DAG) const { - // Create the TargetGlobalAddress node, folding in the constant - // offset if it is legal. - unsigned char OpFlags = Subtarget.classifyGlobalReference(GV); +/// Creates target global address or external symbol nodes for calls or +/// other uses. +SDValue X86TargetLowering::LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG, + bool ForCall) const { + // Unpack the global address or external symbol. + const SDLoc &dl = SDLoc(Op); + const GlobalValue *GV = nullptr; + int64_t Offset = 0; + const char *ExternalSym = nullptr; + if (const auto *G = dyn_cast(Op)) { + GV = G->getGlobal(); + Offset = G->getOffset(); + } else { + const auto *ES = cast(Op); + ExternalSym = ES->getSymbol(); + } + + // Calculate some flags for address lowering. + const Module &Mod = *DAG.getMachineFunction().getFunction().getParent(); + unsigned char OpFlags; + if (ForCall) + OpFlags = Subtarget.classifyGlobalFunctionReference(GV, Mod); + else + OpFlags = Subtarget.classifyGlobalReference(GV, Mod); + bool HasPICReg = isGlobalRelativeToPICBase(OpFlags); + bool NeedsLoad = isGlobalStubReference(OpFlags); + CodeModel::Model M = DAG.getTarget().getCodeModel(); auto PtrVT = getPointerTy(DAG.getDataLayout()); SDValue Result; - if (OpFlags == X86II::MO_NO_FLAG && - X86::isOffsetSuitableForCodeModel(Offset, M)) { - // A direct static reference to a global. - Result = DAG.getTargetGlobalAddress(GV, dl, PtrVT, Offset); - Offset = 0; + + if (GV) { + // Create a target global address if this is a global. If possible, fold the + // offset into the global address reference. Otherwise, ADD it on later. + int64_t GlobalOffset = 0; + if (OpFlags == X86II::MO_NO_FLAG && + X86::isOffsetSuitableForCodeModel(Offset, M)) { + std::swap(GlobalOffset, Offset); + } + Result = DAG.getTargetGlobalAddress(GV, dl, PtrVT, GlobalOffset, OpFlags); } else { - Result = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, OpFlags); + // If this is not a global address, this must be an external symbol. + Result = DAG.getTargetExternalSymbol(ExternalSym, PtrVT, OpFlags); } + // If this is a direct call, avoid the wrapper if we don't need to do any + // loads or adds. This allows SDAG ISel to match direct calls. + if (ForCall && !NeedsLoad && !HasPICReg && Offset == 0) + return Result; + Result = DAG.getNode(getGlobalWrapperKind(GV, OpFlags), dl, PtrVT, Result); // With PIC, the address is actually $g + Offset. - if (isGlobalRelativeToPICBase(OpFlags)) { + if (HasPICReg) { Result = DAG.getNode(ISD::ADD, dl, PtrVT, DAG.getNode(X86ISD::GlobalBaseReg, dl, PtrVT), Result); } // For globals that require a load from a stub to get the address, emit the // load. - if (isGlobalStubReference(OpFlags)) + if (NeedsLoad) Result = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Result, MachinePointerInfo::getGOT(DAG.getMachineFunction())); @@ -17375,9 +17348,7 @@ SDValue X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const { - const GlobalValue *GV = cast(Op)->getGlobal(); - int64_t Offset = cast(Op)->getOffset(); - return LowerGlobalAddress(GV, SDLoc(Op), Offset, DAG); + return LowerGlobalOrExternal(Op, DAG, /*ForCall=*/false); } static SDValue