Index: llvm/trunk/include/llvm/CodeGen/GlobalISel/InstructionSelector.h =================================================================== --- llvm/trunk/include/llvm/CodeGen/GlobalISel/InstructionSelector.h +++ llvm/trunk/include/llvm/CodeGen/GlobalISel/InstructionSelector.h @@ -372,7 +372,16 @@ /// if returns true: /// for I in all mutated/inserted instructions: /// !isPreISelGenericOpcode(I.getOpcode()) - virtual bool select(MachineInstr &I, CodeGenCoverage &CoverageInfo) const = 0; + virtual bool select(MachineInstr &I) = 0; + + CodeGenCoverage *CoverageInfo = nullptr; + MachineFunction *MF = nullptr; + + /// Setup per-MF selector state. + virtual void setupMF(MachineFunction &mf, CodeGenCoverage &covinfo) { + CoverageInfo = &covinfo; + MF = &mf; + } protected: using ComplexRendererFns = Index: llvm/trunk/include/llvm/CodeGen/TargetSubtargetInfo.h =================================================================== --- llvm/trunk/include/llvm/CodeGen/TargetSubtargetInfo.h +++ llvm/trunk/include/llvm/CodeGen/TargetSubtargetInfo.h @@ -106,7 +106,7 @@ // us do things like a dedicated avx512 selector). However, we might want // to also specialize selectors by MachineFunction, which would let us be // aware of optsize/optnone and such. - virtual const InstructionSelector *getInstructionSelector() const { + virtual InstructionSelector *getInstructionSelector() const { return nullptr; } Index: llvm/trunk/lib/CodeGen/GlobalISel/InstructionSelect.cpp =================================================================== --- llvm/trunk/lib/CodeGen/GlobalISel/InstructionSelect.cpp +++ llvm/trunk/lib/CodeGen/GlobalISel/InstructionSelect.cpp @@ -66,9 +66,10 @@ LLVM_DEBUG(dbgs() << "Selecting function: " << MF.getName() << '\n'); const TargetPassConfig &TPC = getAnalysis(); - const InstructionSelector *ISel = MF.getSubtarget().getInstructionSelector(); + InstructionSelector *ISel = MF.getSubtarget().getInstructionSelector(); CodeGenCoverage CoverageInfo; assert(ISel && "Cannot work without InstructionSelector"); + ISel->setupMF(MF, CoverageInfo); // An optimization remark emitter. Used to report failures. MachineOptimizationRemarkEmitter MORE(MF, /*MBFI=*/nullptr); @@ -124,7 +125,7 @@ continue; } - if (!ISel->select(MI, CoverageInfo)) { + if (!ISel->select(MI)) { // FIXME: It would be nice to dump all inserted instructions. It's // not obvious how, esp. considering select() can insert after MI. reportGISelFailure(MF, TPC, MORE, "gisel-select", "cannot select", MI); Index: llvm/trunk/lib/Target/AArch64/AArch64InstructionSelector.cpp =================================================================== --- llvm/trunk/lib/Target/AArch64/AArch64InstructionSelector.cpp +++ llvm/trunk/lib/Target/AArch64/AArch64InstructionSelector.cpp @@ -51,9 +51,18 @@ const AArch64Subtarget &STI, const AArch64RegisterBankInfo &RBI); - bool select(MachineInstr &I, CodeGenCoverage &CoverageInfo) const override; + bool select(MachineInstr &I) override; static const char *getName() { return DEBUG_TYPE; } + void setupMF(MachineFunction &MF, CodeGenCoverage &CoverageInfo) override { + InstructionSelector::setupMF(MF, CoverageInfo); + + // hasFnAttribute() is expensive to call on every BRCOND selection, so + // cache it here for each run of the selector. + ProduceNonFlagSettingCondBr = + !MF.getFunction().hasFnAttribute(Attribute::SpeculativeLoadHardening); + } + private: /// tblgen-erated 'select' implementation, used as the initial selector for /// the patterns that don't require complex C++. @@ -222,6 +231,8 @@ const AArch64RegisterInfo &TRI; const AArch64RegisterBankInfo &RBI; + bool ProduceNonFlagSettingCondBr = false; + #define GET_GLOBALISEL_PREDICATES_DECL #include "AArch64GenGlobalISel.inc" #undef GET_GLOBALISEL_PREDICATES_DECL @@ -1315,8 +1326,7 @@ } } -bool AArch64InstructionSelector::select(MachineInstr &I, - CodeGenCoverage &CoverageInfo) const { +bool AArch64InstructionSelector::select(MachineInstr &I) { assert(I.getParent() && "Instruction should be in a basic block!"); assert(I.getParent()->getParent() && "Instruction should be in a function!"); @@ -1385,7 +1395,7 @@ if (earlySelect(I)) return true; - if (selectImpl(I, CoverageInfo)) + if (selectImpl(I, *CoverageInfo)) return true; LLT Ty = Index: llvm/trunk/lib/Target/AArch64/AArch64Subtarget.h =================================================================== --- llvm/trunk/lib/Target/AArch64/AArch64Subtarget.h +++ llvm/trunk/lib/Target/AArch64/AArch64Subtarget.h @@ -256,7 +256,7 @@ return &getInstrInfo()->getRegisterInfo(); } const CallLowering *getCallLowering() const override; - const InstructionSelector *getInstructionSelector() const override; + InstructionSelector *getInstructionSelector() const override; const LegalizerInfo *getLegalizerInfo() const override; const RegisterBankInfo *getRegBankInfo() const override; const Triple &getTargetTriple() const { return TargetTriple; } Index: llvm/trunk/lib/Target/AArch64/AArch64Subtarget.cpp =================================================================== --- llvm/trunk/lib/Target/AArch64/AArch64Subtarget.cpp +++ llvm/trunk/lib/Target/AArch64/AArch64Subtarget.cpp @@ -196,7 +196,7 @@ return CallLoweringInfo.get(); } -const InstructionSelector *AArch64Subtarget::getInstructionSelector() const { +InstructionSelector *AArch64Subtarget::getInstructionSelector() const { return InstSelector.get(); } Index: llvm/trunk/lib/Target/AMDGPU/AMDGPUInstructionSelector.h =================================================================== --- llvm/trunk/lib/Target/AMDGPU/AMDGPUInstructionSelector.h +++ llvm/trunk/lib/Target/AMDGPU/AMDGPUInstructionSelector.h @@ -47,7 +47,7 @@ const AMDGPURegisterBankInfo &RBI, const AMDGPUTargetMachine &TM); - bool select(MachineInstr &I, CodeGenCoverage &CoverageInfo) const override; + bool select(MachineInstr &I) override; static const char *getName(); private: @@ -81,9 +81,8 @@ bool selectG_GEP(MachineInstr &I) const; bool selectG_IMPLICIT_DEF(MachineInstr &I) const; bool selectG_INSERT(MachineInstr &I) const; - bool selectG_INTRINSIC(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; - bool selectG_INTRINSIC_W_SIDE_EFFECTS(MachineInstr &I, - CodeGenCoverage &CoverageInfo) const; + bool selectG_INTRINSIC(MachineInstr &I) const; + bool selectG_INTRINSIC_W_SIDE_EFFECTS(MachineInstr &I) const; int getS_CMPOpcode(CmpInst::Predicate P, unsigned Size) const; bool selectG_ICMP(MachineInstr &I) const; bool hasVgprParts(ArrayRef AddrInfo) const; @@ -92,8 +91,8 @@ bool selectSMRD(MachineInstr &I, ArrayRef AddrInfo) const; void initM0(MachineInstr &I) const; - bool selectG_LOAD_ATOMICRMW(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; - bool selectG_STORE(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; + bool selectG_LOAD_ATOMICRMW(MachineInstr &I) const; + bool selectG_STORE(MachineInstr &I) const; bool selectG_SELECT(MachineInstr &I) const; bool selectG_BRCOND(MachineInstr &I) const; bool selectG_FRAME_INDEX(MachineInstr &I) const; Index: llvm/trunk/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp =================================================================== --- llvm/trunk/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp +++ llvm/trunk/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp @@ -563,8 +563,7 @@ return true; } -bool AMDGPUInstructionSelector::selectG_INTRINSIC( - MachineInstr &I, CodeGenCoverage &CoverageInfo) const { +bool AMDGPUInstructionSelector::selectG_INTRINSIC(MachineInstr &I) const { unsigned IntrinsicID = I.getOperand(I.getNumExplicitDefs()).getIntrinsicID(); switch (IntrinsicID) { case Intrinsic::amdgcn_if_break: { @@ -593,7 +592,7 @@ return true; } default: - return selectImpl(I, CoverageInfo); + return selectImpl(I, *CoverageInfo); } } @@ -733,7 +732,7 @@ } bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS( - MachineInstr &I, CodeGenCoverage &CoverageInfo) const { + MachineInstr &I) const { MachineBasicBlock *BB = I.getParent(); MachineFunction *MF = BB->getParent(); MachineRegisterInfo &MRI = MF->getRegInfo(); @@ -787,7 +786,7 @@ return true; } default: - return selectImpl(I, CoverageInfo); + return selectImpl(I, *CoverageInfo); } } @@ -840,10 +839,9 @@ return Ret; } -bool AMDGPUInstructionSelector::selectG_STORE( - MachineInstr &I, CodeGenCoverage &CoverageInfo) const { +bool AMDGPUInstructionSelector::selectG_STORE(MachineInstr &I) const { initM0(I); - return selectImpl(I, CoverageInfo); + return selectImpl(I, *CoverageInfo); } static int sizeToSubRegIndex(unsigned Size) { @@ -1215,10 +1213,9 @@ } } -bool AMDGPUInstructionSelector::selectG_LOAD_ATOMICRMW(MachineInstr &I, - CodeGenCoverage &CoverageInfo) const { +bool AMDGPUInstructionSelector::selectG_LOAD_ATOMICRMW(MachineInstr &I) const { initM0(I); - return selectImpl(I, CoverageInfo); + return selectImpl(I, *CoverageInfo); } bool AMDGPUInstructionSelector::selectG_BRCOND(MachineInstr &I) const { @@ -1282,8 +1279,7 @@ DstReg, IsVGPR ? AMDGPU::VGPR_32RegClass : AMDGPU::SReg_32RegClass, MRI); } -bool AMDGPUInstructionSelector::select(MachineInstr &I, - CodeGenCoverage &CoverageInfo) const { +bool AMDGPUInstructionSelector::select(MachineInstr &I) { if (I.isPHI()) return selectPHI(I); @@ -1299,14 +1295,14 @@ case TargetOpcode::G_XOR: if (selectG_AND_OR_XOR(I)) return true; - return selectImpl(I, CoverageInfo); + return selectImpl(I, *CoverageInfo); case TargetOpcode::G_ADD: case TargetOpcode::G_SUB: if (selectG_ADD_SUB(I)) return true; LLVM_FALLTHROUGH; default: - return selectImpl(I, CoverageInfo); + return selectImpl(I, *CoverageInfo); case TargetOpcode::G_INTTOPTR: case TargetOpcode::G_BITCAST: return selectCOPY(I); @@ -1328,13 +1324,13 @@ case TargetOpcode::G_INSERT: return selectG_INSERT(I); case TargetOpcode::G_INTRINSIC: - return selectG_INTRINSIC(I, CoverageInfo); + return selectG_INTRINSIC(I); case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: - return selectG_INTRINSIC_W_SIDE_EFFECTS(I, CoverageInfo); + return selectG_INTRINSIC_W_SIDE_EFFECTS(I); case TargetOpcode::G_ICMP: if (selectG_ICMP(I)) return true; - return selectImpl(I, CoverageInfo); + return selectImpl(I, *CoverageInfo); case TargetOpcode::G_LOAD: case TargetOpcode::G_ATOMIC_CMPXCHG: case TargetOpcode::G_ATOMICRMW_XCHG: @@ -1348,11 +1344,11 @@ case TargetOpcode::G_ATOMICRMW_UMIN: case TargetOpcode::G_ATOMICRMW_UMAX: case TargetOpcode::G_ATOMICRMW_FADD: - return selectG_LOAD_ATOMICRMW(I, CoverageInfo); + return selectG_LOAD_ATOMICRMW(I); case TargetOpcode::G_SELECT: return selectG_SELECT(I); case TargetOpcode::G_STORE: - return selectG_STORE(I, CoverageInfo); + return selectG_STORE(I); case TargetOpcode::G_TRUNC: return selectG_TRUNC(I); case TargetOpcode::G_SEXT: Index: llvm/trunk/lib/Target/AMDGPU/AMDGPUSubtarget.h =================================================================== --- llvm/trunk/lib/Target/AMDGPU/AMDGPUSubtarget.h +++ llvm/trunk/lib/Target/AMDGPU/AMDGPUSubtarget.h @@ -422,7 +422,7 @@ return CallLoweringInfo.get(); } - const InstructionSelector *getInstructionSelector() const override { + InstructionSelector *getInstructionSelector() const override { return InstSelector.get(); } Index: llvm/trunk/lib/Target/ARM/ARMInstructionSelector.cpp =================================================================== --- llvm/trunk/lib/Target/ARM/ARMInstructionSelector.cpp +++ llvm/trunk/lib/Target/ARM/ARMInstructionSelector.cpp @@ -34,7 +34,7 @@ ARMInstructionSelector(const ARMBaseTargetMachine &TM, const ARMSubtarget &STI, const ARMRegisterBankInfo &RBI); - bool select(MachineInstr &I, CodeGenCoverage &CoverageInfo) const override; + bool select(MachineInstr &I) override; static const char *getName() { return DEBUG_TYPE; } private: @@ -833,8 +833,7 @@ NewInstBuilder.addImm(FPImmEncoding); } -bool ARMInstructionSelector::select(MachineInstr &I, - CodeGenCoverage &CoverageInfo) const { +bool ARMInstructionSelector::select(MachineInstr &I) { assert(I.getParent() && "Instruction should be in a basic block!"); assert(I.getParent()->getParent() && "Instruction should be in a function!"); @@ -851,7 +850,7 @@ using namespace TargetOpcode; - if (selectImpl(I, CoverageInfo)) + if (selectImpl(I, *CoverageInfo)) return true; MachineInstrBuilder MIB{MF, I}; Index: llvm/trunk/lib/Target/ARM/ARMSubtarget.h =================================================================== --- llvm/trunk/lib/Target/ARM/ARMSubtarget.h +++ llvm/trunk/lib/Target/ARM/ARMSubtarget.h @@ -536,7 +536,7 @@ } const CallLowering *getCallLowering() const override; - const InstructionSelector *getInstructionSelector() const override; + InstructionSelector *getInstructionSelector() const override; const LegalizerInfo *getLegalizerInfo() const override; const RegisterBankInfo *getRegBankInfo() const override; Index: llvm/trunk/lib/Target/ARM/ARMSubtarget.cpp =================================================================== --- llvm/trunk/lib/Target/ARM/ARMSubtarget.cpp +++ llvm/trunk/lib/Target/ARM/ARMSubtarget.cpp @@ -125,7 +125,7 @@ return CallLoweringInfo.get(); } -const InstructionSelector *ARMSubtarget::getInstructionSelector() const { +InstructionSelector *ARMSubtarget::getInstructionSelector() const { return InstSelector.get(); } Index: llvm/trunk/lib/Target/Mips/MipsInstructionSelector.cpp =================================================================== --- llvm/trunk/lib/Target/Mips/MipsInstructionSelector.cpp +++ llvm/trunk/lib/Target/Mips/MipsInstructionSelector.cpp @@ -34,7 +34,7 @@ MipsInstructionSelector(const MipsTargetMachine &TM, const MipsSubtarget &STI, const MipsRegisterBankInfo &RBI); - bool select(MachineInstr &I, CodeGenCoverage &CoverageInfo) const override; + bool select(MachineInstr &I) override; static const char *getName() { return DEBUG_TYPE; } private: @@ -204,8 +204,7 @@ return Opc; } -bool MipsInstructionSelector::select(MachineInstr &I, - CodeGenCoverage &CoverageInfo) const { +bool MipsInstructionSelector::select(MachineInstr &I) { MachineBasicBlock &MBB = *I.getParent(); MachineFunction &MF = *MBB.getParent(); @@ -232,7 +231,7 @@ return true; } - if (selectImpl(I, CoverageInfo)) + if (selectImpl(I, *CoverageInfo)) return true; MachineInstr *MI = nullptr; Index: llvm/trunk/lib/Target/Mips/MipsSubtarget.h =================================================================== --- llvm/trunk/lib/Target/Mips/MipsSubtarget.h +++ llvm/trunk/lib/Target/Mips/MipsSubtarget.h @@ -391,7 +391,7 @@ const CallLowering *getCallLowering() const override; const LegalizerInfo *getLegalizerInfo() const override; const RegisterBankInfo *getRegBankInfo() const override; - const InstructionSelector *getInstructionSelector() const override; + InstructionSelector *getInstructionSelector() const override; }; } // End llvm namespace Index: llvm/trunk/lib/Target/Mips/MipsSubtarget.cpp =================================================================== --- llvm/trunk/lib/Target/Mips/MipsSubtarget.cpp +++ llvm/trunk/lib/Target/Mips/MipsSubtarget.cpp @@ -286,6 +286,6 @@ return RegBankInfo.get(); } -const InstructionSelector *MipsSubtarget::getInstructionSelector() const { +InstructionSelector *MipsSubtarget::getInstructionSelector() const { return InstSelector.get(); } Index: llvm/trunk/lib/Target/X86/X86InstructionSelector.cpp =================================================================== --- llvm/trunk/lib/Target/X86/X86InstructionSelector.cpp +++ llvm/trunk/lib/Target/X86/X86InstructionSelector.cpp @@ -60,7 +60,7 @@ X86InstructionSelector(const X86TargetMachine &TM, const X86Subtarget &STI, const X86RegisterBankInfo &RBI); - bool select(MachineInstr &I, CodeGenCoverage &CoverageInfo) const override; + bool select(MachineInstr &I) override; static const char *getName() { return DEBUG_TYPE; } private: @@ -94,11 +94,9 @@ MachineFunction &MF) const; bool selectCopy(MachineInstr &I, MachineRegisterInfo &MRI) const; bool selectUnmergeValues(MachineInstr &I, MachineRegisterInfo &MRI, - MachineFunction &MF, - CodeGenCoverage &CoverageInfo) const; + MachineFunction &MF); bool selectMergeValues(MachineInstr &I, MachineRegisterInfo &MRI, - MachineFunction &MF, - CodeGenCoverage &CoverageInfo) const; + MachineFunction &MF); bool selectInsert(MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) const; bool selectExtract(MachineInstr &I, MachineRegisterInfo &MRI, @@ -308,8 +306,7 @@ return true; } -bool X86InstructionSelector::select(MachineInstr &I, - CodeGenCoverage &CoverageInfo) const { +bool X86InstructionSelector::select(MachineInstr &I) { assert(I.getParent() && "Instruction should be in a basic block!"); assert(I.getParent()->getParent() && "Instruction should be in a function!"); @@ -333,7 +330,7 @@ assert(I.getNumOperands() == I.getNumExplicitOperands() && "Generic instruction has unexpected implicit operands\n"); - if (selectImpl(I, CoverageInfo)) + if (selectImpl(I, *CoverageInfo)) return true; LLVM_DEBUG(dbgs() << " C++ instruction selection: "; I.print(dbgs())); @@ -370,10 +367,10 @@ case TargetOpcode::G_UADDE: return selectUadde(I, MRI, MF); case TargetOpcode::G_UNMERGE_VALUES: - return selectUnmergeValues(I, MRI, MF, CoverageInfo); + return selectUnmergeValues(I, MRI, MF); case TargetOpcode::G_MERGE_VALUES: case TargetOpcode::G_CONCAT_VECTORS: - return selectMergeValues(I, MRI, MF, CoverageInfo); + return selectMergeValues(I, MRI, MF); case TargetOpcode::G_EXTRACT: return selectExtract(I, MRI, MF); case TargetOpcode::G_INSERT: @@ -1335,8 +1332,7 @@ } bool X86InstructionSelector::selectUnmergeValues( - MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF, - CodeGenCoverage &CoverageInfo) const { + MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) { assert((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES) && "unexpected instruction"); @@ -1352,7 +1348,7 @@ .addReg(SrcReg) .addImm(Idx * DefSize); - if (!select(ExtrInst, CoverageInfo)) + if (!select(ExtrInst)) return false; } @@ -1361,8 +1357,7 @@ } bool X86InstructionSelector::selectMergeValues( - MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF, - CodeGenCoverage &CoverageInfo) const { + MachineInstr &I, MachineRegisterInfo &MRI, MachineFunction &MF) { assert((I.getOpcode() == TargetOpcode::G_MERGE_VALUES || I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS) && "unexpected instruction"); @@ -1395,7 +1390,7 @@ DefReg = Tmp; - if (!select(InsertInst, CoverageInfo)) + if (!select(InsertInst)) return false; } @@ -1403,7 +1398,7 @@ TII.get(TargetOpcode::COPY), DstReg) .addReg(DefReg); - if (!select(CopyInst, CoverageInfo)) + if (!select(CopyInst)) return false; I.eraseFromParent(); Index: llvm/trunk/lib/Target/X86/X86Subtarget.h =================================================================== --- llvm/trunk/lib/Target/X86/X86Subtarget.h +++ llvm/trunk/lib/Target/X86/X86Subtarget.h @@ -527,7 +527,7 @@ /// Methods used by Global ISel const CallLowering *getCallLowering() const override; - const InstructionSelector *getInstructionSelector() const override; + InstructionSelector *getInstructionSelector() const override; const LegalizerInfo *getLegalizerInfo() const override; const RegisterBankInfo *getRegBankInfo() const override; Index: llvm/trunk/lib/Target/X86/X86Subtarget.cpp =================================================================== --- llvm/trunk/lib/Target/X86/X86Subtarget.cpp +++ llvm/trunk/lib/Target/X86/X86Subtarget.cpp @@ -355,7 +355,7 @@ return CallLoweringInfo.get(); } -const InstructionSelector *X86Subtarget::getInstructionSelector() const { +InstructionSelector *X86Subtarget::getInstructionSelector() const { return InstSelector.get(); }