Index: llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h =================================================================== --- llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -947,7 +947,7 @@ void visitStoreToSwiftError(const StoreInst &I); void visitInlineAsm(ImmutableCallSite CS); - const char *visitIntrinsicCall(const CallInst &I, unsigned Intrinsic); + void visitIntrinsicCall(const CallInst &I, unsigned Intrinsic); void visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic); void visitConstrainedFPIntrinsic(const ConstrainedFPIntrinsic &FPI); @@ -997,6 +997,9 @@ SDDbgValue *getDbgValue(SDValue N, DILocalVariable *Variable, DIExpression *Expr, const DebugLoc &dl, unsigned DbgSDNodeOrder); + + /// Lowers CallInst to an external symbol. + void lowerCallToExternalSymbol(const CallInst &I, const char *FunctionName); }; /// This struct represents the registers (physical or virtual) Index: llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ llvm/trunk/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -5550,11 +5550,18 @@ } } -/// Lower the call to the specified intrinsic function. If we want to emit this -/// as a call to a named external function, return the name. Otherwise, lower it -/// and return null. -const char * -SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { +void SelectionDAGBuilder::lowerCallToExternalSymbol(const CallInst &I, + const char *FunctionName) { + assert(FunctionName && "FunctionName must not be nullptr"); + SDValue Callee = DAG.getExternalSymbol( + FunctionName, + DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout())); + LowerCallTo(&I, Callee, I.isTailCall()); +} + +/// Lower the call to the specified intrinsic function. +void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, + unsigned Intrinsic) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDLoc sdl = getCurSDLoc(); DebugLoc dl = getCurDebugLoc(); @@ -5564,28 +5571,28 @@ default: // By default, turn this into a target intrinsic node. visitTargetIntrinsic(I, Intrinsic); - return nullptr; - case Intrinsic::vastart: visitVAStart(I); return nullptr; - case Intrinsic::vaend: visitVAEnd(I); return nullptr; - case Intrinsic::vacopy: visitVACopy(I); return nullptr; + return; + case Intrinsic::vastart: visitVAStart(I); return; + case Intrinsic::vaend: visitVAEnd(I); return; + case Intrinsic::vacopy: visitVACopy(I); return; case Intrinsic::returnaddress: setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl, TLI.getPointerTy(DAG.getDataLayout()), getValue(I.getArgOperand(0)))); - return nullptr; + return; case Intrinsic::addressofreturnaddress: setValue(&I, DAG.getNode(ISD::ADDROFRETURNADDR, sdl, TLI.getPointerTy(DAG.getDataLayout()))); - return nullptr; + return; case Intrinsic::sponentry: setValue(&I, DAG.getNode(ISD::SPONENTRY, sdl, TLI.getPointerTy(DAG.getDataLayout()))); - return nullptr; + return; case Intrinsic::frameaddress: setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl, TLI.getPointerTy(DAG.getDataLayout()), getValue(I.getArgOperand(0)))); - return nullptr; + return; case Intrinsic::read_register: { Value *Reg = I.getArgOperand(0); SDValue Chain = getRoot(); @@ -5596,7 +5603,7 @@ DAG.getVTList(VT, MVT::Other), Chain, RegName); setValue(&I, Res); DAG.setRoot(Res.getValue(1)); - return nullptr; + return; } case Intrinsic::write_register: { Value *Reg = I.getArgOperand(0); @@ -5606,12 +5613,14 @@ DAG.getMDNode(cast(cast(Reg)->getMetadata())); DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain, RegName, getValue(RegValue))); - return nullptr; + return; } case Intrinsic::setjmp: - return &"_setjmp"[!TLI.usesUnderscoreSetJmp()]; + lowerCallToExternalSymbol(I, &"_setjmp"[!TLI.usesUnderscoreSetJmp()]); + return; case Intrinsic::longjmp: - return &"_longjmp"[!TLI.usesUnderscoreLongJmp()]; + lowerCallToExternalSymbol(I, &"_longjmp"[!TLI.usesUnderscoreLongJmp()]); + return; case Intrinsic::memcpy: { const auto &MCI = cast(I); SDValue Op1 = getValue(I.getArgOperand(0)); @@ -5630,7 +5639,7 @@ MachinePointerInfo(I.getArgOperand(0)), MachinePointerInfo(I.getArgOperand(1))); updateDAGForMaybeTailCall(MC); - return nullptr; + return; } case Intrinsic::memset: { const auto &MSI = cast(I); @@ -5644,7 +5653,7 @@ SDValue MS = DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, isTC, MachinePointerInfo(I.getArgOperand(0))); updateDAGForMaybeTailCall(MS); - return nullptr; + return; } case Intrinsic::memmove: { const auto &MMI = cast(I); @@ -5663,7 +5672,7 @@ isTC, MachinePointerInfo(I.getArgOperand(0)), MachinePointerInfo(I.getArgOperand(1))); updateDAGForMaybeTailCall(MM); - return nullptr; + return; } case Intrinsic::memcpy_element_unordered_atomic: { const AtomicMemCpyInst &MI = cast(I); @@ -5681,7 +5690,7 @@ MachinePointerInfo(MI.getRawDest()), MachinePointerInfo(MI.getRawSource())); updateDAGForMaybeTailCall(MC); - return nullptr; + return; } case Intrinsic::memmove_element_unordered_atomic: { auto &MI = cast(I); @@ -5699,7 +5708,7 @@ MachinePointerInfo(MI.getRawDest()), MachinePointerInfo(MI.getRawSource())); updateDAGForMaybeTailCall(MC); - return nullptr; + return; } case Intrinsic::memset_element_unordered_atomic: { auto &MI = cast(I); @@ -5715,7 +5724,7 @@ LengthTy, ElemSz, isTC, MachinePointerInfo(MI.getRawDest())); updateDAGForMaybeTailCall(MC); - return nullptr; + return; } case Intrinsic::dbg_addr: case Intrinsic::dbg_declare: { @@ -5730,7 +5739,7 @@ if (!Address || isa(Address) || (Address->use_empty() && !isa(Address))) { LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI << "\n"); - return nullptr; + return; } bool isParameter = Variable->isParameter() || isa(Address); @@ -5759,7 +5768,7 @@ Variable, Expression, FI, /*IsIndirect*/ true, dl, SDNodeOrder); DAG.AddDbgValue(SDV, getRoot().getNode(), isParameter); } - return nullptr; + return; } SDValue &N = NodeMap[Address]; @@ -5781,7 +5790,7 @@ // Address is an argument, so try to emit its dbg value using // virtual register info from the FuncInfo.ValueMap. EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, true, N); - return nullptr; + return; } else { SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(), true, dl, SDNodeOrder); @@ -5795,7 +5804,7 @@ LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI << "\n"); } } - return nullptr; + return; } case Intrinsic::dbg_label: { const DbgLabelInst &DI = cast(I); @@ -5805,7 +5814,7 @@ SDDbgLabel *SDV; SDV = DAG.getDbgLabel(Label, dl, SDNodeOrder); DAG.AddDbgLabel(SDV); - return nullptr; + return; } case Intrinsic::dbg_value: { const DbgValueInst &DI = cast(I); @@ -5816,11 +5825,11 @@ dropDanglingDebugInfo(Variable, Expression); const Value *V = DI.getValue(); if (!V) - return nullptr; + return; if (handleDebugValue(V, Variable, Expression, dl, DI.getDebugLoc(), SDNodeOrder)) - return nullptr; + return; // TODO: Dangling debug info will eventually either be resolved or produce // an Undef DBG_VALUE. However in the resolution case, a gap may appear @@ -5828,7 +5837,7 @@ // we should ideally fill with an extra Undef DBG_VALUE. DanglingDebugInfoMap[V].emplace_back(&DI, dl, SDNodeOrder); - return nullptr; + return; } case Intrinsic::eh_typeid_for: { @@ -5837,7 +5846,7 @@ unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV); Res = DAG.getConstant(TypeID, sdl, MVT::i32); setValue(&I, Res); - return nullptr; + return; } case Intrinsic::eh_return_i32: @@ -5848,15 +5857,15 @@ getControlRoot(), getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)))); - return nullptr; + return; case Intrinsic::eh_unwind_init: DAG.getMachineFunction().setCallsUnwindInit(true); - return nullptr; + return; case Intrinsic::eh_dwarf_cfa: setValue(&I, DAG.getNode(ISD::EH_DWARF_CFA, sdl, TLI.getPointerTy(DAG.getDataLayout()), getValue(I.getArgOperand(0)))); - return nullptr; + return; case Intrinsic::eh_sjlj_callsite: { MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); ConstantInt *CI = dyn_cast(I.getArgOperand(0)); @@ -5864,7 +5873,7 @@ assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!"); MMI.setCurrentCallSite(CI->getZExtValue()); - return nullptr; + return; } case Intrinsic::eh_sjlj_functioncontext: { // Get and store the index of the function context. @@ -5873,7 +5882,7 @@ cast(I.getArgOperand(0)->stripPointerCasts()); int FI = FuncInfo.StaticAllocaMap[FnCtx]; MFI.setFunctionContextIndex(FI); - return nullptr; + return; } case Intrinsic::eh_sjlj_setjmp: { SDValue Ops[2]; @@ -5883,34 +5892,34 @@ DAG.getVTList(MVT::i32, MVT::Other), Ops); setValue(&I, Op.getValue(0)); DAG.setRoot(Op.getValue(1)); - return nullptr; + return; } case Intrinsic::eh_sjlj_longjmp: DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other, getRoot(), getValue(I.getArgOperand(0)))); - return nullptr; + return; case Intrinsic::eh_sjlj_setup_dispatch: DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other, getRoot())); - return nullptr; + return; case Intrinsic::masked_gather: visitMaskedGather(I); - return nullptr; + return; case Intrinsic::masked_load: visitMaskedLoad(I); - return nullptr; + return; case Intrinsic::masked_scatter: visitMaskedScatter(I); - return nullptr; + return; case Intrinsic::masked_store: visitMaskedStore(I); - return nullptr; + return; case Intrinsic::masked_expandload: visitMaskedLoad(I, true /* IsExpanding */); - return nullptr; + return; case Intrinsic::masked_compressstore: visitMaskedStore(I, true /* IsCompressing */); - return nullptr; + return; case Intrinsic::x86_mmx_pslli_w: case Intrinsic::x86_mmx_pslli_d: case Intrinsic::x86_mmx_pslli_q: @@ -5922,7 +5931,7 @@ SDValue ShAmt = getValue(I.getArgOperand(1)); if (isa(ShAmt)) { visitTargetIntrinsic(I, Intrinsic); - return nullptr; + return; } unsigned NewIntrinsic = 0; EVT ShAmtVT = MVT::v2i32; @@ -5968,31 +5977,31 @@ DAG.getConstant(NewIntrinsic, sdl, MVT::i32), getValue(I.getArgOperand(0)), ShAmt); setValue(&I, Res); - return nullptr; + return; } case Intrinsic::powi: setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), DAG)); - return nullptr; + return; case Intrinsic::log: setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI)); - return nullptr; + return; case Intrinsic::log2: setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI)); - return nullptr; + return; case Intrinsic::log10: setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI)); - return nullptr; + return; case Intrinsic::exp: setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI)); - return nullptr; + return; case Intrinsic::exp2: setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI)); - return nullptr; + return; case Intrinsic::pow: setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), DAG, TLI)); - return nullptr; + return; case Intrinsic::sqrt: case Intrinsic::fabs: case Intrinsic::sin: @@ -6023,7 +6032,7 @@ setValue(&I, DAG.getNode(Opcode, sdl, getValue(I.getArgOperand(0)).getValueType(), getValue(I.getArgOperand(0)))); - return nullptr; + return; } case Intrinsic::lround_i32: case Intrinsic::lround_i64: @@ -6050,7 +6059,7 @@ setValue(&I, DAG.getNode(Opc, sdl, VT, getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)))); - return nullptr; + return; } case Intrinsic::maxnum: { auto VT = getValue(I.getArgOperand(0)).getValueType(); @@ -6061,33 +6070,33 @@ setValue(&I, DAG.getNode(Opc, sdl, VT, getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)))); - return nullptr; + return; } case Intrinsic::minimum: setValue(&I, DAG.getNode(ISD::FMINIMUM, sdl, getValue(I.getArgOperand(0)).getValueType(), getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)))); - return nullptr; + return; case Intrinsic::maximum: setValue(&I, DAG.getNode(ISD::FMAXIMUM, sdl, getValue(I.getArgOperand(0)).getValueType(), getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)))); - return nullptr; + return; case Intrinsic::copysign: setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl, getValue(I.getArgOperand(0)).getValueType(), getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)))); - return nullptr; + return; case Intrinsic::fma: setValue(&I, DAG.getNode(ISD::FMA, sdl, getValue(I.getArgOperand(0)).getValueType(), getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), getValue(I.getArgOperand(2)))); - return nullptr; + return; case Intrinsic::experimental_constrained_fadd: case Intrinsic::experimental_constrained_fsub: case Intrinsic::experimental_constrained_fmul: @@ -6115,7 +6124,7 @@ case Intrinsic::experimental_constrained_round: case Intrinsic::experimental_constrained_trunc: visitConstrainedFPIntrinsic(cast(I)); - return nullptr; + return; case Intrinsic::fmuladd: { EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict && @@ -6137,7 +6146,7 @@ getValue(I.getArgOperand(2))); setValue(&I, Add); } - return nullptr; + return; } case Intrinsic::convert_to_fp16: setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16, @@ -6145,17 +6154,17 @@ getValue(I.getArgOperand(0)), DAG.getTargetConstant(0, sdl, MVT::i32)))); - return nullptr; + return; case Intrinsic::convert_from_fp16: setValue(&I, DAG.getNode(ISD::FP_EXTEND, sdl, TLI.getValueType(DAG.getDataLayout(), I.getType()), DAG.getNode(ISD::BITCAST, sdl, MVT::f16, getValue(I.getArgOperand(0))))); - return nullptr; + return; case Intrinsic::pcmarker: { SDValue Tmp = getValue(I.getArgOperand(0)); DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp)); - return nullptr; + return; } case Intrinsic::readcyclecounter: { SDValue Op = getRoot(); @@ -6163,25 +6172,25 @@ DAG.getVTList(MVT::i64, MVT::Other), Op); setValue(&I, Res); DAG.setRoot(Res.getValue(1)); - return nullptr; + return; } case Intrinsic::bitreverse: setValue(&I, DAG.getNode(ISD::BITREVERSE, sdl, getValue(I.getArgOperand(0)).getValueType(), getValue(I.getArgOperand(0)))); - return nullptr; + return; case Intrinsic::bswap: setValue(&I, DAG.getNode(ISD::BSWAP, sdl, getValue(I.getArgOperand(0)).getValueType(), getValue(I.getArgOperand(0)))); - return nullptr; + return; case Intrinsic::cttz: { SDValue Arg = getValue(I.getArgOperand(0)); ConstantInt *CI = cast(I.getArgOperand(1)); EVT Ty = Arg.getValueType(); setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF, sdl, Ty, Arg)); - return nullptr; + return; } case Intrinsic::ctlz: { SDValue Arg = getValue(I.getArgOperand(0)); @@ -6189,13 +6198,13 @@ EVT Ty = Arg.getValueType(); setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF, sdl, Ty, Arg)); - return nullptr; + return; } case Intrinsic::ctpop: { SDValue Arg = getValue(I.getArgOperand(0)); EVT Ty = Arg.getValueType(); setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg)); - return nullptr; + return; } case Intrinsic::fshl: case Intrinsic::fshr: { @@ -6211,7 +6220,7 @@ auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR; if (TLI.isOperationLegalOrCustom(FunnelOpcode, VT)) { setValue(&I, DAG.getNode(FunnelOpcode, sdl, VT, X, Y, Z)); - return nullptr; + return; } // When X == Y, this is rotate. If the data type has a power-of-2 size, we @@ -6221,7 +6230,7 @@ auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR; if (TLI.isOperationLegalOrCustom(RotateOpcode, VT)) { setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z)); - return nullptr; + return; } // Some targets only rotate one way. Try the opposite direction. @@ -6230,7 +6239,7 @@ // Negate the shift amount because it is safe to ignore the high bits. SDValue NegShAmt = DAG.getNode(ISD::SUB, sdl, VT, Zero, Z); setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, NegShAmt)); - return nullptr; + return; } // fshl (rotl): (X << (Z % BW)) | (X >> ((0 - Z) % BW)) @@ -6240,7 +6249,7 @@ SDValue ShX = DAG.getNode(ISD::SHL, sdl, VT, X, IsFSHL ? ShAmt : NShAmt); SDValue ShY = DAG.getNode(ISD::SRL, sdl, VT, X, IsFSHL ? NShAmt : ShAmt); setValue(&I, DAG.getNode(ISD::OR, sdl, VT, ShX, ShY)); - return nullptr; + return; } // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW))) @@ -6260,31 +6269,31 @@ // For fshr, 0-shift returns the 2nd arg (Y). SDValue IsZeroShift = DAG.getSetCC(sdl, CCVT, ShAmt, Zero, ISD::SETEQ); setValue(&I, DAG.getSelect(sdl, VT, IsZeroShift, IsFSHL ? X : Y, Or)); - return nullptr; + return; } case Intrinsic::sadd_sat: { SDValue Op1 = getValue(I.getArgOperand(0)); SDValue Op2 = getValue(I.getArgOperand(1)); setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2)); - return nullptr; + return; } case Intrinsic::uadd_sat: { SDValue Op1 = getValue(I.getArgOperand(0)); SDValue Op2 = getValue(I.getArgOperand(1)); setValue(&I, DAG.getNode(ISD::UADDSAT, sdl, Op1.getValueType(), Op1, Op2)); - return nullptr; + return; } case Intrinsic::ssub_sat: { SDValue Op1 = getValue(I.getArgOperand(0)); SDValue Op2 = getValue(I.getArgOperand(1)); setValue(&I, DAG.getNode(ISD::SSUBSAT, sdl, Op1.getValueType(), Op1, Op2)); - return nullptr; + return; } case Intrinsic::usub_sat: { SDValue Op1 = getValue(I.getArgOperand(0)); SDValue Op2 = getValue(I.getArgOperand(1)); setValue(&I, DAG.getNode(ISD::USUBSAT, sdl, Op1.getValueType(), Op1, Op2)); - return nullptr; + return; } case Intrinsic::smul_fix: case Intrinsic::umul_fix: { @@ -6293,7 +6302,7 @@ SDValue Op3 = getValue(I.getArgOperand(2)); setValue(&I, DAG.getNode(FixedPointIntrinsicToOpcode(Intrinsic), sdl, Op1.getValueType(), Op1, Op2, Op3)); - return nullptr; + return; } case Intrinsic::stacksave: { SDValue Op = getRoot(); @@ -6302,12 +6311,12 @@ DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Op); setValue(&I, Res); DAG.setRoot(Res.getValue(1)); - return nullptr; + return; } case Intrinsic::stackrestore: Res = getValue(I.getArgOperand(0)); DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res)); - return nullptr; + return; case Intrinsic::get_dynamic_area_offset: { SDValue Op = getRoot(); EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout()); @@ -6321,7 +6330,7 @@ Op); DAG.setRoot(Op); setValue(&I, Res); - return nullptr; + return; } case Intrinsic::stackguard: { EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout()); @@ -6341,7 +6350,7 @@ Res = TLI.emitStackGuardXorFP(DAG, Res, sdl); DAG.setRoot(Chain); setValue(&I, Res); - return nullptr; + return; } case Intrinsic::stackprotector: { // Emit code into the DAG to store the stack guard onto the stack. @@ -6368,7 +6377,7 @@ /* Alignment = */ 0, MachineMemOperand::MOVolatile); setValue(&I, Res); DAG.setRoot(Res); - return nullptr; + return; } case Intrinsic::objectsize: { // If we don't know by now, we're never going to know. @@ -6385,14 +6394,14 @@ Res = DAG.getConstant(0, sdl, Ty); setValue(&I, Res); - return nullptr; + return; } case Intrinsic::is_constant: // If this wasn't constant-folded away by now, then it's not a // constant. setValue(&I, DAG.getConstant(0, sdl, MVT::i1)); - return nullptr; + return; case Intrinsic::annotation: case Intrinsic::ptr_annotation: @@ -6400,12 +6409,12 @@ case Intrinsic::strip_invariant_group: // Drop the intrinsic, but forward the value setValue(&I, getValue(I.getOperand(0))); - return nullptr; + return; case Intrinsic::assume: case Intrinsic::var_annotation: case Intrinsic::sideeffect: // Discard annotate attributes, assumptions, and artificial side-effects. - return nullptr; + return; case Intrinsic::codeview_annotation: { // Emit a label associated with this metadata. @@ -6416,7 +6425,7 @@ MF.addCodeViewAnnotation(Label, cast(MD)); Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label); DAG.setRoot(Res); - return nullptr; + return; } case Intrinsic::init_trampoline: { @@ -6433,13 +6442,13 @@ Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops); DAG.setRoot(Res); - return nullptr; + return; } case Intrinsic::adjust_trampoline: setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl, TLI.getPointerTy(DAG.getDataLayout()), getValue(I.getArgOperand(0)))); - return nullptr; + return; case Intrinsic::gcroot: { assert(DAG.getMachineFunction().getFunction().hasGC() && "only valid in functions with gc specified, enforced by Verifier"); @@ -6449,19 +6458,19 @@ FrameIndexSDNode *FI = cast(getValue(Alloca).getNode()); GFI->addStackRoot(FI->getIndex(), TypeMap); - return nullptr; + return; } case Intrinsic::gcread: case Intrinsic::gcwrite: llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!"); case Intrinsic::flt_rounds: setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, sdl, MVT::i32)); - return nullptr; + return; case Intrinsic::expect: // Just replace __builtin_expect(exp, c) with EXP. setValue(&I, getValue(I.getArgOperand(0))); - return nullptr; + return; case Intrinsic::debugtrap: case Intrinsic::trap: { @@ -6473,7 +6482,7 @@ ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ? ISD::TRAP : ISD::DEBUGTRAP; DAG.setRoot(DAG.getNode(Op, sdl,MVT::Other, getRoot())); - return nullptr; + return; } TargetLowering::ArgListTy Args; @@ -6486,7 +6495,7 @@ std::pair Result = TLI.LowerCallTo(CLI); DAG.setRoot(Result.second); - return nullptr; + return; } case Intrinsic::uadd_with_overflow: @@ -6516,7 +6525,7 @@ SDVTList VTs = DAG.getVTList(ResultVT, OverflowVT); setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2)); - return nullptr; + return; } case Intrinsic::prefetch: { SDValue Ops[5]; @@ -6539,14 +6548,14 @@ PendingLoads.push_back(Result); Result = getRoot(); DAG.setRoot(Result); - return nullptr; + return; } case Intrinsic::lifetime_start: case Intrinsic::lifetime_end: { bool IsStart = (Intrinsic == Intrinsic::lifetime_start); // Stack coloring is not enabled in O0, discard region information. if (TM.getOptLevel() == CodeGenOpt::None) - return nullptr; + return; const int64_t ObjectSize = cast(I.getArgOperand(0))->getSExtValue(); @@ -6566,7 +6575,7 @@ // valid frame index. auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject); if (SI == FuncInfo.StaticAllocaMap.end()) - return nullptr; + return; const int FrameIndex = SI->second; int64_t Offset; @@ -6577,36 +6586,39 @@ Offset); DAG.setRoot(Res); } - return nullptr; + return; } case Intrinsic::invariant_start: // Discard region information. setValue(&I, DAG.getUNDEF(TLI.getPointerTy(DAG.getDataLayout()))); - return nullptr; + return; case Intrinsic::invariant_end: // Discard region information. - return nullptr; + return; case Intrinsic::clear_cache: - return TLI.getClearCacheBuiltinName(); + /// FunctionName may be null. + if (const char *FunctionName = TLI.getClearCacheBuiltinName()) + lowerCallToExternalSymbol(I, FunctionName); + return; case Intrinsic::donothing: // ignore - return nullptr; + return; case Intrinsic::experimental_stackmap: visitStackmap(I); - return nullptr; + return; case Intrinsic::experimental_patchpoint_void: case Intrinsic::experimental_patchpoint_i64: visitPatchpoint(&I); - return nullptr; + return; case Intrinsic::experimental_gc_statepoint: LowerStatepoint(ImmutableStatepoint(&I)); - return nullptr; + return; case Intrinsic::experimental_gc_result: visitGCResult(cast(I)); - return nullptr; + return; case Intrinsic::experimental_gc_relocate: visitGCRelocate(cast(I)); - return nullptr; + return; case Intrinsic::instrprof_increment: llvm_unreachable("instrprof failed to lower an increment"); case Intrinsic::instrprof_value_profile: @@ -6634,7 +6646,7 @@ .addFrameIndex(FI); } - return nullptr; + return; } case Intrinsic::localrecover: { @@ -6663,7 +6675,7 @@ SDValue Add = DAG.getNode(ISD::ADD, sdl, PtrVT, FPVal, OffsetVal); setValue(&I, Add); - return nullptr; + return; } case Intrinsic::eh_exceptionpointer: @@ -6678,7 +6690,7 @@ if (Intrinsic == Intrinsic::eh_exceptioncode) N = DAG.getZExtOrTrunc(N, getCurSDLoc(), MVT::i32); setValue(&I, N); - return nullptr; + return; } case Intrinsic::xray_customevent: { // Here we want to make sure that the intrinsic behaves as if it has a @@ -6686,7 +6698,7 @@ // FIXME: Support other platforms later. const auto &Triple = DAG.getTarget().getTargetTriple(); if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux()) - return nullptr; + return; SDLoc DL = getCurSDLoc(); SmallVector Ops; @@ -6709,7 +6721,7 @@ SDValue patchableNode = SDValue(MN, 0); DAG.setRoot(patchableNode); setValue(&I, patchableNode); - return nullptr; + return; } case Intrinsic::xray_typedevent: { // Here we want to make sure that the intrinsic behaves as if it has a @@ -6717,7 +6729,7 @@ // FIXME: Support other platforms later. const auto &Triple = DAG.getTarget().getTargetTriple(); if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux()) - return nullptr; + return; SDLoc DL = getCurSDLoc(); SmallVector Ops; @@ -6744,11 +6756,11 @@ SDValue patchableNode = SDValue(MN, 0); DAG.setRoot(patchableNode); setValue(&I, patchableNode); - return nullptr; + return; } case Intrinsic::experimental_deoptimize: LowerDeoptimizeCall(&I); - return nullptr; + return; case Intrinsic::experimental_vector_reduce_fadd: case Intrinsic::experimental_vector_reduce_fmul: @@ -6764,7 +6776,7 @@ case Intrinsic::experimental_vector_reduce_fmax: case Intrinsic::experimental_vector_reduce_fmin: visitVectorReduce(I, Intrinsic); - return nullptr; + return; case Intrinsic::icall_branch_funnel: { SmallVector Ops; @@ -6817,14 +6829,14 @@ DAG.setRoot(N); setValue(&I, N); HasTailCall = true; - return nullptr; + return; } case Intrinsic::wasm_landingpad_index: // Information this intrinsic contained has been transferred to // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely // delete it now. - return nullptr; + return; } } @@ -7458,7 +7470,6 @@ return; } - const char *RenameFn = nullptr; if (Function *F = I.getCalledFunction()) { if (F->isDeclaration()) { // Is this an LLVM intrinsic or a target-specific intrinsic? @@ -7468,9 +7479,8 @@ IID = II->getIntrinsicID(F); if (IID) { - RenameFn = visitIntrinsicCall(I, IID); - if (!RenameFn) - return; + visitIntrinsicCall(I, IID); + return; } } @@ -7619,20 +7629,14 @@ } } - SDValue Callee; - if (!RenameFn) - Callee = getValue(I.getCalledValue()); - else - Callee = DAG.getExternalSymbol( - RenameFn, - DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout())); - // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't // have to do anything here to lower funclet bundles. assert(!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower calls with arbitrary operand bundles!"); + SDValue Callee = getValue(I.getCalledValue()); + if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) LowerCallSiteWithDeoptBundle(&I, Callee, nullptr); else