Index: include/llvm/CodeGen/SelectionDAG.h =================================================================== --- include/llvm/CodeGen/SelectionDAG.h +++ include/llvm/CodeGen/SelectionDAG.h @@ -1128,6 +1128,11 @@ /// Expand the specified \c ISD::VACOPY node as the Legalize pass would. SDValue expandVACopy(SDNode *Node); + /// Returs an GlobalAddress of the function from the current module with + /// name matching the given ExternalSymbol. + /// Panics the function doesn't exists. + SDValue getSymbolFunctionGlobalAddress(SDValue Op); + /// *Mutate* the specified node in-place to have the /// specified operands. If the resultant node already exists in the DAG, /// this does not modify the specified node, instead it returns the node that Index: lib/CodeGen/SelectionDAG/SelectionDAG.cpp =================================================================== --- lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -8409,6 +8409,28 @@ return TokenFactor; } +SDValue SelectionDAG::getSymbolFunctionGlobalAddress(SDValue Op) { + assert(isa(Op) && "Node should be an ExternalSymbol"); + + auto *Symbol = cast(Op)->getSymbol(); + auto *Module = MF->getFunction().getParent(); + auto *Function = Module->getFunction(Symbol); + + if (Function != nullptr) { + auto PtrTy = TLI->getPointerTy(getDataLayout()); + return getGlobalAddress(Function, SDLoc(Op), PtrTy); + } + + std::string ErrorStr; + raw_string_ostream ErrorFormatter(ErrorStr); + + ErrorFormatter << "Undefined external symbol "; + ErrorFormatter << '"' << Symbol << '"'; + ErrorFormatter.flush(); + + report_fatal_error(ErrorStr); +} + //===----------------------------------------------------------------------===// // SDNode Class //===----------------------------------------------------------------------===// Index: lib/Target/NVPTX/CMakeLists.txt =================================================================== --- lib/Target/NVPTX/CMakeLists.txt +++ lib/Target/NVPTX/CMakeLists.txt @@ -32,6 +32,7 @@ NVPTXUtilities.cpp NVVMIntrRange.cpp NVVMReflect.cpp + NVPTXProxyRegErasure.cpp ) add_llvm_target(NVPTXCodeGen ${NVPTXCodeGen_sources}) Index: lib/Target/NVPTX/NVPTX.h =================================================================== --- lib/Target/NVPTX/NVPTX.h +++ lib/Target/NVPTX/NVPTX.h @@ -53,6 +53,7 @@ FunctionPass *createNVPTXLowerArgsPass(const NVPTXTargetMachine *TM); BasicBlockPass *createNVPTXLowerAllocaPass(); MachineFunctionPass *createNVPTXPeephole(); +MachineFunctionPass *createNVPTXProxyRegErasurePass(); Target &getTheNVPTXTarget32(); Target &getTheNVPTXTarget64(); Index: lib/Target/NVPTX/NVPTXISelLowering.h =================================================================== --- lib/Target/NVPTX/NVPTXISelLowering.h +++ lib/Target/NVPTX/NVPTXISelLowering.h @@ -51,6 +51,7 @@ CallSeqBegin, CallSeqEnd, CallPrototype, + ProxyReg, FUN_SHFL_CLAMP, FUN_SHFR_CLAMP, MUL_WIDE_SIGNED, Index: lib/Target/NVPTX/NVPTXISelLowering.cpp =================================================================== --- lib/Target/NVPTX/NVPTXISelLowering.cpp +++ lib/Target/NVPTX/NVPTXISelLowering.cpp @@ -663,6 +663,8 @@ return "NVPTXISD::CallSeqEnd"; case NVPTXISD::CallPrototype: return "NVPTXISD::CallPrototype"; + case NVPTXISD::ProxyReg: + return "NVPTXISD::ProxyReg"; case NVPTXISD::LoadV2: return "NVPTXISD::LoadV2"; case NVPTXISD::LoadV4: @@ -1666,6 +1668,11 @@ // indirect calls but is always null for libcalls. bool isIndirectCall = !Func && CS; + if (isa(Callee)) { + // Try to find the callee in the current module. + Callee = DAG.getSymbolFunctionGlobalAddress(Callee); + } + if (isIndirectCall) { // This is indirect function call case : PTX requires a prototype of the // form @@ -1738,6 +1745,9 @@ InFlag = Chain.getValue(1); } + SmallVector ProxyRegOps; + SmallVector, 16> ProxyRegTruncates; + // Generate loads from param memory/moves from registers for result if (Ins.size() > 0) { SmallVector VTs; @@ -1808,11 +1818,14 @@ MachineMemOperand::MOLoad); for (unsigned j = 0; j < NumElts; ++j) { - SDValue Ret = RetVal.getValue(j); + ProxyRegOps.push_back(RetVal.getValue(j)); + if (needTruncate) - Ret = DAG.getNode(ISD::TRUNCATE, dl, Ins[VecIdx + j].VT, Ret); - InVals.push_back(Ret); + ProxyRegTruncates.push_back(Optional(Ins[VecIdx + j].VT)); + else + ProxyRegTruncates.push_back(Optional()); } + Chain = RetVal.getValue(NumElts); InFlag = RetVal.getValue(NumElts + 1); @@ -1828,8 +1841,29 @@ DAG.getIntPtrConstant(uniqueCallSite + 1, dl, true), InFlag, dl); + InFlag = Chain.getValue(1); uniqueCallSite++; + // Append ProxyReg instructions to the chain to make sure that `callseq_end` + // will not get lost. Otherwise, during libcalls expansion, the nodes can become + // dangling. + for (unsigned i = 0; i < ProxyRegOps.size(); ++i) { + SDValue Ret = DAG.getNode( + NVPTXISD::ProxyReg, dl, + DAG.getVTList(ProxyRegOps[i].getSimpleValueType(), MVT::Other, MVT::Glue), + { Chain, ProxyRegOps[i], InFlag } + ); + + Chain = Ret.getValue(1); + InFlag = Ret.getValue(2); + + if (ProxyRegTruncates[i].hasValue()) { + Ret = DAG.getNode(ISD::TRUNCATE, dl, ProxyRegTruncates[i].getValue(), Ret); + } + + InVals.push_back(Ret); + } + // set isTailCall to false for now, until we figure out how to express // tail call optimization in PTX isTailCall = false; Index: lib/Target/NVPTX/NVPTXInstrInfo.td =================================================================== --- lib/Target/NVPTX/NVPTXInstrInfo.td +++ lib/Target/NVPTX/NVPTXInstrInfo.td @@ -1885,6 +1885,7 @@ def SDTStoreRetvalV2Profile : SDTypeProfile<0, 3, [SDTCisInt<0>]>; def SDTStoreRetvalV4Profile : SDTypeProfile<0, 5, [SDTCisInt<0>]>; def SDTPseudoUseParamProfile : SDTypeProfile<0, 1, []>; +def SDTProxyRegProfile : SDTypeProfile<1, 1, []>; def DeclareParam : SDNode<"NVPTXISD::DeclareParam", SDTDeclareParamProfile, @@ -1972,6 +1973,9 @@ def RETURNNode : SDNode<"NVPTXISD::RETURN", SDTCallArgMarkProfile, [SDNPHasChain, SDNPSideEffect]>; +def ProxyReg : + SDNode<"NVPTXISD::ProxyReg", SDTProxyRegProfile, + [SDNPHasChain, SDNPOutGlue, SDNPInGlue, SDNPSideEffect]>; let mayLoad = 1 in { class LoadParamMemInst : @@ -2249,6 +2253,21 @@ def PseudoUseParamF64 : PseudoUseParamInst; def PseudoUseParamF32 : PseudoUseParamInst; +class ProxyRegInst : + NVPTXInst<(outs regclass:$dst), (ins regclass:$src), + "// Proxy Register pseudo instruction", + [(set regclass:$dst, (ProxyReg regclass:$src))]>; + +let isCodeGenOnly=1, isPseudo=1 in { + def ProxyRegI1 : ProxyRegInst; + def ProxyRegI16 : ProxyRegInst; + def ProxyRegI32 : ProxyRegInst; + def ProxyRegI64 : ProxyRegInst; + def ProxyRegF16 : ProxyRegInst; + def ProxyRegF32 : ProxyRegInst; + def ProxyRegF64 : ProxyRegInst; + def ProxyRegF16x2 : ProxyRegInst; +} // // Load / Store Handling Index: lib/Target/NVPTX/NVPTXProxyRegErasure.cpp =================================================================== --- /dev/null +++ lib/Target/NVPTX/NVPTXProxyRegErasure.cpp @@ -0,0 +1,114 @@ +//===- NVPTXProxyRegErasure.cpp - NVPTX Proxy Register Instruction Erasure -==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// The pass is needed to remove ProxyReg instructions and restore related +// registers. The instructions were needed at instruction selection stage to +// make sure that callseq_end nodes won't be removed as "dead nodes". This can +// happen when we expand instructions into libcalls and the call site doesn't +// care about the libcall chain. Call site cares about data flow only, and the +// latest data flow node happens to be before callseq_end. Therefore the node +// becomes dangling and "dead". The ProxyReg acts like an additional data flow +// node *after* the callseq_end in the chain and ensures that everything will be +// preserved. +// +//===----------------------------------------------------------------------===// + +#include "NVPTX.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/TargetInstrInfo.h" +#include "llvm/CodeGen/TargetRegisterInfo.h" + +using namespace llvm; + +namespace { + +struct NVPTXProxyRegErasure : public MachineFunctionPass { +public: + static char ID; + NVPTXProxyRegErasure() : MachineFunctionPass(ID) {} + + bool runOnMachineFunction(MachineFunction &MF) override; + + StringRef getPassName() const override { + return "NVPTX Proxy Register Instruction Erasure"; + } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + MachineFunctionPass::getAnalysisUsage(AU); + } + +private: + void replaceMachineInstructionUsage(MachineFunction &MF, MachineInstr &MI); + + void replaceRegisterUsage(MachineInstr &Instr, MachineOperand &From, + MachineOperand &To); +}; + +} // namespace + +char NVPTXProxyRegErasure::ID = 0; + +bool NVPTXProxyRegErasure::runOnMachineFunction(MachineFunction &MF) { + SmallVector RemoveList; + + for (auto &BB : MF) { + for (auto &MI : BB) { + switch (MI.getOpcode()) { + case NVPTX::ProxyRegI1: + case NVPTX::ProxyRegI16: + case NVPTX::ProxyRegI32: + case NVPTX::ProxyRegI64: + case NVPTX::ProxyRegF16: + case NVPTX::ProxyRegF16x2: + case NVPTX::ProxyRegF32: + case NVPTX::ProxyRegF64: + replaceMachineInstructionUsage(MF, MI); + RemoveList.push_back(&MI); + break; + } + } + } + + for (auto *MI : RemoveList) { + MI->eraseFromParent(); + } + + return !RemoveList.empty(); +} + +void NVPTXProxyRegErasure::replaceMachineInstructionUsage(MachineFunction &MF, + MachineInstr &MI) { + auto &InOp = *MI.uses().begin(); + auto &OutOp = *MI.defs().begin(); + + assert(InOp.isReg() && "ProxyReg input operand should be a register."); + assert(OutOp.isReg() && "ProxyReg output operand should be a register."); + + for (auto &BB : MF) { + for (auto &I : BB) { + replaceRegisterUsage(I, OutOp, InOp); + } + } +} + +void NVPTXProxyRegErasure::replaceRegisterUsage(MachineInstr &Instr, + MachineOperand &From, + MachineOperand &To) { + for (auto &Op : Instr.uses()) { + if (Op.isReg() && Op.getReg() == From.getReg()) { + Op.setReg(To.getReg()); + } + } +} + +MachineFunctionPass *llvm::createNVPTXProxyRegErasurePass() { + return new NVPTXProxyRegErasure(); +} Index: lib/Target/NVPTX/NVPTXTargetMachine.cpp =================================================================== --- lib/Target/NVPTX/NVPTXTargetMachine.cpp +++ lib/Target/NVPTX/NVPTXTargetMachine.cpp @@ -160,6 +160,7 @@ void addIRPasses() override; bool addInstSelector() override; + void addPreRegAlloc() override; void addPostRegAlloc() override; void addMachineSSAOptimization() override; @@ -301,6 +302,11 @@ return false; } +void NVPTXPassConfig::addPreRegAlloc() { + // Remove Proxy Register pseudo instructions used to keep `callseq_end` alive. + addPass(createNVPTXProxyRegErasurePass()); +} + void NVPTXPassConfig::addPostRegAlloc() { addPass(createNVPTXPrologEpilogPass(), false); if (getOptLevel() != CodeGenOpt::None) { Index: test/CodeGen/NVPTX/calls-with-phi.ll =================================================================== --- /dev/null +++ test/CodeGen/NVPTX/calls-with-phi.ll @@ -0,0 +1,22 @@ +; RUN: llc < %s -march=nvptx 2>&1 | FileCheck %s +; Make sure the example doesn't crash with segfault + +; CHECK: .visible .func ({{.*}}) loop +define i32 @loop(i32, i32) { +entry: + br label %loop + +loop: + %i = phi i32 [ %0, %entry ], [ %res, %loop ] + %res = call i32 @div(i32 %i, i32 %1) + + %exitcond = icmp eq i32 %res, %0 + br i1 %exitcond, label %exit, label %loop + +exit: + ret i32 %res +} + +define i32 @div(i32, i32) { + ret i32 0 +} Index: test/CodeGen/NVPTX/libcall-fulfilled.ll =================================================================== --- /dev/null +++ test/CodeGen/NVPTX/libcall-fulfilled.ll @@ -0,0 +1,27 @@ +; RUN: llc < %s -march=nvptx 2>&1 | FileCheck %s +; Allow to make libcalls that are defined in current module + +define i128 @remainder(i128, i128) { +bb0: + ; CHECK: { // callseq 0, 0 + ; CHECK: call.uni (retval0), + ; CHECK-NEXT: __umodti3, + ; CHECK-NEXT: ( + ; CHECK-NEXT: param0, + ; CHECK-NEXT: param1 + ; CHECK-NEXT: ); + ; CHECK-NEXT: ld.param.v2.b64 {%[[REG0:rd[0-9]+]], %[[REG1:rd[0-9]+]]}, [retval0+0]; + ; CHECK-NEXT: } // callseq 0 + %a = urem i128 %0, %1 + br label %bb1 + +bb1: + ; CHECK-NEXT: st.param.v2.b64 [func_retval0+0], {%[[REG0]], %[[REG1]]}; + ; CHECK-NEXT: ret; + ret i128 %a +} + +; Underlying libcall +define i128 @__umodti3(i128, i128) { + ret i128 0 +} Index: test/CodeGen/NVPTX/libcall-instruction.ll =================================================================== --- test/CodeGen/NVPTX/libcall-instruction.ll +++ test/CodeGen/NVPTX/libcall-instruction.ll @@ -1,7 +1,7 @@ ; RUN: not llc < %s -march=nvptx 2>&1 | FileCheck %s -; used to panic on failed assetion and now fails with a "Cannot select" +; used to panic on failed assetion and now fails with a "Undefined external symbol" -; CHECK: LLVM ERROR: Cannot select: {{t28|0x[0-9a-f]+}}: i32 = ExternalSymbol'__umodti3' +; CHECK: LLVM ERROR: Undefined external symbol "__umodti3" define hidden i128 @remainder(i128, i128) { %3 = urem i128 %0, %1 ret i128 %3 Index: test/CodeGen/NVPTX/libcall-intrinsic.ll =================================================================== --- /dev/null +++ test/CodeGen/NVPTX/libcall-intrinsic.ll @@ -0,0 +1,10 @@ +; RUN: not llc < %s -march=nvptx 2>&1 | FileCheck %s +; used to seqfault and now fails with a "Undefined external symbol" + +; CHECK: LLVM ERROR: Undefined external symbol "__powidf2" +define double @powi(double, i32) { + %a = call double @llvm.powi.f64(double %0, i32 %1) + ret double %a +} + +declare double @llvm.powi.f64(double, i32) nounwind readnone Index: test/CodeGen/NVPTX/zero-cs.ll =================================================================== --- test/CodeGen/NVPTX/zero-cs.ll +++ /dev/null @@ -1,10 +0,0 @@ -; RUN: not llc < %s -march=nvptx 2>&1 | FileCheck %s -; used to seqfault and now fails with a "Cannot select" - -; CHECK: LLVM ERROR: Cannot select: {{t7|0x[0-9a-f]+}}: i32 = ExternalSymbol'__powidf2' -define double @powi() { - %1 = call double @llvm.powi.f64(double 1.000000e+00, i32 undef) - ret double %1 -} - -declare double @llvm.powi.f64(double, i32) nounwind readnone