Index: include/llvm/CodeGen/GlobalISel/CSEInfo.h =================================================================== --- /dev/null +++ include/llvm/CodeGen/GlobalISel/CSEInfo.h @@ -0,0 +1,237 @@ +//===- llvm/CodeGen/GlobalISel/CSEInfo.h ------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +/// Provides analysis for continuously CSEing during GISel passes. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_CODEGEN_GLOBALISEL_CSEINFO_H +#define LLVM_CODEGEN_GLOBALISEL_CSEINFO_H + +#include "llvm/ADT/FoldingSet.h" +#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" +#include "llvm/CodeGen/GlobalISel/GISelWorkList.h" +#include "llvm/CodeGen/GlobalISel/Utils.h" +#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/IR/PassManager.h" +#include "llvm/Pass.h" +#include "llvm/Support/Allocator.h" + +namespace llvm { + +/// A class that wraps MachineInstrs and derives from FoldingSetNode in order to +/// be uniqued in a CSEMap. The tradeoff here is extra memory allocations for +/// UniqueMachineInstr vs making MachineInstr bigger. +class UniqueMachineInstr : public FoldingSetNode { + friend class GISelCSEInfo; + const MachineInstr *MI; + explicit UniqueMachineInstr(const MachineInstr *MI) : MI(MI) {} + +public: + void Profile(FoldingSetNodeID &ID); +}; + +// Class representing some configuration that can be done during CSE analysis. +// Currently it only supports shouldCSE method that each pass can set. +class CSEConfig { +public: + virtual ~CSEConfig() = default; + // Hook for defining which Generic instructions should be CSEd. + // GISelCSEInfo currently only calls this hook when dealing with generic + // opcodes. + virtual bool shouldCSEOpc(unsigned Opc); +}; + +// TODO: Find a better place for this. +// Commonly used for O0 config. +class CSEConfigConstantOnly : public CSEConfig { +public: + virtual ~CSEConfigConstantOnly() = default; + virtual bool shouldCSEOpc(unsigned Opc) override; +}; + +/// The CSE Analysis object. +/// This installs itself as a delegate to the MachineFunction to track +/// new instructions as well as deletions. It however will not be able to +/// track instruction mutations. In such cases, recordNewInstruction should be +/// called (for eg inside MachineIRBuilder::recordInsertion). +/// Also because of how just the instruction can be inserted without adding any +/// operands to the instruction, instructions are uniqued and inserted lazily. +/// CSEInfo should assert when trying to enter an incomplete instruction into +/// the CSEMap. There is Opcode level granularity on which instructions can be +/// CSE'd and for now, only Generic instructions are CSEable. +class GISelCSEInfo : public GISelChangeObserver { + // Make it accessible only to CSEMIRBuilder. + friend class CSEMIRBuilder; + + BumpPtrAllocator UniqueInstrAllocator; + FoldingSet CSEMap; + MachineRegisterInfo *MRI = nullptr; + MachineFunction *MF = nullptr; + std::unique_ptr CSEOpt; + /// Keep a cache of UniqueInstrs for each MachineInstr. In GISel, + /// often instructions are mutated (while their ID has completely changed). + /// Whenever mutation happens, invalidate the UniqueMachineInstr for the + /// MachineInstr + DenseMap InstrMapping; + + /// Store instructions that are not fully formed in TemporaryInsts. + /// Also because CSE insertion happens lazily, we can remove insts from this + /// list and avoid inserting and then removing from the CSEMap. + GISelWorkList<8> TemporaryInsts; + + // Only used in asserts. + DenseMap OpcodeHitTable; + + bool isUniqueMachineInstValid(const UniqueMachineInstr &UMI) const; + + void invalidateUniqueMachineInstr(UniqueMachineInstr *UMI); + + UniqueMachineInstr *getNodeIfExists(FoldingSetNodeID &ID, + MachineBasicBlock *MBB, void *&InsertPos); + + /// Allocate and construct a new UniqueMachineInstr for MI and return. + UniqueMachineInstr *getUniqueInstrForMI(const MachineInstr *MI); + + void insertNode(UniqueMachineInstr *UMI, void *InsertPos = nullptr); + + /// Get the MachineInstr(Unique) if it exists already in the CSEMap and the + /// same MachineBasicBlock. + MachineInstr *getMachineInstrIfExists(FoldingSetNodeID &ID, + MachineBasicBlock *MBB, + void *&InsertPos); + + /// Use this method to allocate a new UniqueMachineInstr for MI and insert it + /// into the CSEMap. MI should return true for shouldCSE(MI->getOpcode()) + void insertInstr(MachineInstr *MI, void *InsertPos = nullptr); + +public: + GISelCSEInfo() = default; + + virtual ~GISelCSEInfo(); + + void setMF(MachineFunction &MF); + + /// Records a newly created inst in a list and lazily insert it to the CSEMap. + /// Sometimes, this method might be called with a partially constructed + /// MachineInstr, + // (right after BuildMI without adding any operands) - and in such cases, + // defer the hashing of the instruction to a later stage. + void recordNewInstruction(MachineInstr *MI); + + /// Use this callback to inform CSE about a newly fully created instruction. + void handleRecordedInst(MachineInstr *MI); + + /// Use this callback to insert all the recorded instructions. At this point, + /// all of these insts need to be fully constructed and should not be missing + /// any operands. + void handleRecordedInsts(); + + /// Remove this inst from the CSE map. If this inst has not been inserted yet, + /// it will be removed from the Tempinsts list if it exists. + void handleRemoveInst(MachineInstr *MI); + + void releaseMemory(); + + void setCSEConfig(std::unique_ptr Opt) { CSEOpt = std::move(Opt); } + + bool shouldCSE(unsigned Opc) const; + + void analyze(MachineFunction &MF); + + void countOpcodeHit(unsigned Opc); + + void print(); + + // Observer API + void erasingInstr(MachineInstr &MI) override; + void createdInstr(MachineInstr &MI) override; + void changingInstr(MachineInstr &MI) override; + void changedInstr(MachineInstr &MI) override; +}; + +class TargetRegisterClass; +class RegisterBank; + +// Simple builder class to easily profile properties about MIs. +class GISelInstProfileBuilder { + FoldingSetNodeID &ID; + const MachineRegisterInfo &MRI; + +public: + GISelInstProfileBuilder(FoldingSetNodeID &ID, const MachineRegisterInfo &MRI) + : ID(ID), MRI(MRI) {} + // Profiling methods. + const GISelInstProfileBuilder &addNodeIDOpcode(unsigned Opc) const; + const GISelInstProfileBuilder &addNodeIDRegType(const LLT &Ty) const; + const GISelInstProfileBuilder &addNodeIDRegType(const unsigned) const; + + const GISelInstProfileBuilder & + addNodeIDRegType(const TargetRegisterClass *RC) const; + const GISelInstProfileBuilder &addNodeIDRegType(const RegisterBank *RB) const; + + const GISelInstProfileBuilder &addNodeIDRegNum(unsigned Reg) const; + + const GISelInstProfileBuilder &addNodeIDImmediate(int64_t Imm) const; + const GISelInstProfileBuilder & + addNodeIDMBB(const MachineBasicBlock *MBB) const; + + const GISelInstProfileBuilder & + addNodeIDMachineOperand(const MachineOperand &MO) const; + + const GISelInstProfileBuilder &addNodeIDFlag(unsigned Flag) const; + const GISelInstProfileBuilder &addNodeID(const MachineInstr *MI) const; +}; + +/// Simple wrapper that does the following. +/// 1) Lazily evaluate the MachineFunction to compute CSEable instructions. +/// 2) Allows configuration of which instructions are CSEd through CSEConfig +/// object. Provides a method called get which takes a CSEConfig object. +class GISelCSEAnalysisWrapper { + GISelCSEInfo Info; + MachineFunction *MF = nullptr; + bool AlreadyComputed = false; + +public: + /// Takes a CSEConfig object that defines what opcodes get CSEd. + /// If CSEConfig is already set, and the CSE Analysis has been preserved, + /// it will not use the new CSEOpt(use Recompute to force using the new + /// CSEOpt). + GISelCSEInfo &get(std::unique_ptr CSEOpt, bool ReCompute = false); + void setMF(MachineFunction &MFunc) { MF = &MFunc; } + void setComputed(bool Computed) { AlreadyComputed = Computed; } + void releaseMemory() { Info.releaseMemory(); } +}; + +/// The actual analysis pass wrapper. +class GISelCSEAnalysisWrapperPass : public MachineFunctionPass { + GISelCSEAnalysisWrapper Wrapper; + +public: + static char ID; + GISelCSEAnalysisWrapperPass() : MachineFunctionPass(ID) { + initializeGISelCSEAnalysisWrapperPassPass(*PassRegistry::getPassRegistry()); + } + + void getAnalysisUsage(AnalysisUsage &AU) const override; + + const GISelCSEAnalysisWrapper &getCSEWrapper() const { return Wrapper; } + GISelCSEAnalysisWrapper &getCSEWrapper() { return Wrapper; } + + bool runOnMachineFunction(MachineFunction &MF) override; + + void releaseMemory() override { + Wrapper.releaseMemory(); + Wrapper.setComputed(false); + } +}; + +} // namespace llvm + +#endif Index: include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h =================================================================== --- /dev/null +++ include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h @@ -0,0 +1,110 @@ +//===-- llvm/CodeGen/GlobalISel/CSEMIRBuilder.h --*- C++ -*-==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// This file implements a version of MachineIRBuilder which CSEs insts within +/// a MachineBasicBlock. +//===----------------------------------------------------------------------===// +#ifndef LLVM_CODEGEN_GLOBALISEL_CSEMIRBUILDER_H +#define LLVM_CODEGEN_GLOBALISEL_CSEMIRBUILDER_H + +#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" +#include "llvm/CodeGen/GlobalISel/Utils.h" + +namespace llvm { + +/// Defines a builder that does CSE of MachineInstructions using GISelCSEInfo. +/// Eg usage. +/// +/// +/// GISelCSEInfo *Info = +/// &getAnalysis().getCSEInfo(); CSEMIRBuilder +/// CB(Builder.getState()); CB.setCSEInfo(Info); auto A = CB.buildConstant(s32, +/// 42); auto B = CB.buildConstant(s32, 42); assert(A == B); unsigned CReg = +/// MRI.createGenericVirtualRegister(s32); auto C = CB.buildConstant(CReg, 42); +/// assert(C->getOpcode() == TargetOpcode::COPY); +/// Explicitly passing in a register would materialize a copy if possible. +/// CSEMIRBuilder also does trivial constant folding for binary ops. +class CSEMIRBuilder : public MachineIRBuilder { + + /// Returns true if A dominates B (within the same basic block). + /// Both iterators must be in the same basic block. + // + // TODO: Another approach for checking dominance is having two iterators and + // making them go towards each other until they meet or reach begin/end. Which + // approach is better? Should this even change dynamically? For G_CONSTANTS + // most of which will be at the top of the BB, the top down approach would be + // a better choice. Does IRTranslator placing constants at the beginning still + // make sense? Should this change based on Opcode? + bool dominates(MachineBasicBlock::const_iterator A, + MachineBasicBlock::const_iterator B) const; + + /// For given ID, find a machineinstr in the CSE Map. If found, check if it + /// dominates the current insertion point and if not, move it just before the + /// current insertion point and return it. If not found, return Null + /// MachineInstrBuilder. + MachineInstrBuilder getDominatingInstrForID(FoldingSetNodeID &ID, + void *&NodeInsertPos); + /// Simple check if we can CSE (we have the CSEInfo) or if this Opcode is + /// safe to CSE. + bool canPerformCSEForOpc(unsigned Opc) const; + + void profileDstOp(const DstOp &Op, GISelInstProfileBuilder &B) const; + + void profileDstOps(ArrayRef Ops, GISelInstProfileBuilder &B) const { + for (const DstOp &Op : Ops) + profileDstOp(Op, B); + } + + void profileSrcOp(const SrcOp &Op, GISelInstProfileBuilder &B) const; + + void profileSrcOps(ArrayRef Ops, GISelInstProfileBuilder &B) const { + for (const SrcOp &Op : Ops) + profileSrcOp(Op, B); + } + + void profileMBBOpcode(GISelInstProfileBuilder &B, unsigned Opc) const; + + void profileEverything(unsigned Opc, ArrayRef DstOps, + ArrayRef SrcOps, Optional Flags, + GISelInstProfileBuilder &B) const; + + // Takes a MachineInstrBuilder and inserts it into the CSEMap using the + // NodeInsertPos. + MachineInstrBuilder memoizeMI(MachineInstrBuilder MIB, void *NodeInsertPos); + + // If we have can CSE an instruction, but still need to materialize to a VReg, + // we emit a copy from the CSE'd inst to the VReg. + MachineInstrBuilder generateCopiesIfRequired(ArrayRef DstOps, + MachineInstrBuilder &MIB); + + // If we have can CSE an instruction, but still need to materialize to a VReg, + // check if we can generate copies. It's not possible to return a single MIB, + // while emitting copies to multiple vregs. + bool checkCopyToDefsPossible(ArrayRef DstOps); + +public: + // Pull in base class constructors. + using MachineIRBuilder::MachineIRBuilder; + // Unhide buildInstr + MachineInstrBuilder buildInstr(unsigned Opc, ArrayRef DstOps, + ArrayRef SrcOps, + Optional Flag = None) override; + // Bring in the other overload from the base class. + using MachineIRBuilder::buildConstant; + + MachineInstrBuilder buildConstant(const DstOp &Res, + const ConstantInt &Val) override; + + // Bring in the other overload from the base class. + using MachineIRBuilder::buildFConstant; + MachineInstrBuilder buildFConstant(const DstOp &Res, + const ConstantFP &Val) override; +}; +} // namespace llvm +#endif Index: include/llvm/CodeGen/GlobalISel/Combiner.h =================================================================== --- include/llvm/CodeGen/GlobalISel/Combiner.h +++ include/llvm/CodeGen/GlobalISel/Combiner.h @@ -21,6 +21,7 @@ namespace llvm { class MachineRegisterInfo; class CombinerInfo; +class GISelCSEInfo; class TargetPassConfig; class MachineFunction; @@ -28,14 +29,17 @@ public: Combiner(CombinerInfo &CombinerInfo, const TargetPassConfig *TPC); - bool combineMachineInstrs(MachineFunction &MF); + /// If CSEInfo is not null, then the Combiner will setup observer for + /// CSEInfo and instantiate a CSEMIRBuilder. Pass nullptr if CSE is not + /// needed. + bool combineMachineInstrs(MachineFunction &MF, GISelCSEInfo *CSEInfo); protected: CombinerInfo &CInfo; MachineRegisterInfo *MRI = nullptr; const TargetPassConfig *TPC; - MachineIRBuilder Builder; + std::unique_ptr Builder; }; } // End namespace llvm. Index: include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h =================================================================== --- include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h +++ include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h @@ -15,57 +15,6 @@ namespace llvm { -static Optional ConstantFoldBinOp(unsigned Opcode, const unsigned Op1, - const unsigned Op2, - const MachineRegisterInfo &MRI) { - auto MaybeOp1Cst = getConstantVRegVal(Op1, MRI); - auto MaybeOp2Cst = getConstantVRegVal(Op2, MRI); - if (MaybeOp1Cst && MaybeOp2Cst) { - LLT Ty = MRI.getType(Op1); - APInt C1(Ty.getSizeInBits(), *MaybeOp1Cst, true); - APInt C2(Ty.getSizeInBits(), *MaybeOp2Cst, true); - switch (Opcode) { - default: - break; - case TargetOpcode::G_ADD: - return C1 + C2; - case TargetOpcode::G_AND: - return C1 & C2; - case TargetOpcode::G_ASHR: - return C1.ashr(C2); - case TargetOpcode::G_LSHR: - return C1.lshr(C2); - case TargetOpcode::G_MUL: - return C1 * C2; - case TargetOpcode::G_OR: - return C1 | C2; - case TargetOpcode::G_SHL: - return C1 << C2; - case TargetOpcode::G_SUB: - return C1 - C2; - case TargetOpcode::G_XOR: - return C1 ^ C2; - case TargetOpcode::G_UDIV: - if (!C2.getBoolValue()) - break; - return C1.udiv(C2); - case TargetOpcode::G_SDIV: - if (!C2.getBoolValue()) - break; - return C1.sdiv(C2); - case TargetOpcode::G_UREM: - if (!C2.getBoolValue()) - break; - return C1.urem(C2); - case TargetOpcode::G_SREM: - if (!C2.getBoolValue()) - break; - return C1.srem(C2); - } - } - return None; -} - /// An MIRBuilder which does trivial constant folding of binary ops. /// Calls to buildInstr will also try to constant fold binary ops. class ConstantFoldingMIRBuilder : public MachineIRBuilder { Index: include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h =================================================================== --- include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h +++ include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h @@ -15,6 +15,7 @@ #define LLVM_CODEGEN_GLOBALISEL_GISELCHANGEOBSERVER_H #include "llvm/ADT/SmallPtrSet.h" +#include "llvm/CodeGen/MachineFunction.h" namespace llvm { class MachineInstr; @@ -32,13 +33,13 @@ virtual ~GISelChangeObserver() {} /// An instruction is about to be erased. - virtual void erasingInstr(const MachineInstr &MI) = 0; + virtual void erasingInstr(MachineInstr &MI) = 0; /// An instruction was created and inserted into the function. - virtual void createdInstr(const MachineInstr &MI) = 0; + virtual void createdInstr(MachineInstr &MI) = 0; /// This instruction is about to be mutated in some way. - virtual void changingInstr(const MachineInstr &MI) = 0; + virtual void changingInstr(MachineInstr &MI) = 0; /// This instruction was mutated in some way. - virtual void changedInstr(const MachineInstr &MI) = 0; + virtual void changedInstr(MachineInstr &MI) = 0; /// All the instructions using the given register are being changed. /// For convenience, finishedChangingAllUsesOfReg() will report the completion @@ -51,5 +52,60 @@ }; +/// Simple wrapper observer that takes several observers, and calls +/// each one for each event. If there are multiple observers (say CSE, +/// Legalizer, Combiner), it's sufficient to register this to the machine +/// function as the delegate. +class GISelObserverWrapper : public MachineFunction::Delegate, + public GISelChangeObserver { + SmallVector Observers; + +public: + GISelObserverWrapper() = default; + GISelObserverWrapper(ArrayRef Obs) + : Observers(Obs.begin(), Obs.end()) {} + // Adds an observer. + void addObserver(GISelChangeObserver *O) { Observers.push_back(O); } + // Removes an observer from the list and does nothing if observer is not + // present. + void removeObserver(GISelChangeObserver *O) { + auto It = std::find(Observers.begin(), Observers.end(), O); + if (It != Observers.end()) + Observers.erase(It); + } + // API for Observer. + void erasingInstr(MachineInstr &MI) override { + for (auto &O : Observers) + O->erasingInstr(MI); + } + void createdInstr(MachineInstr &MI) override { + for (auto &O : Observers) + O->createdInstr(MI); + } + void changingInstr(MachineInstr &MI) override { + for (auto &O : Observers) + O->changingInstr(MI); + } + void changedInstr(MachineInstr &MI) override { + for (auto &O : Observers) + O->changedInstr(MI); + } + // API for MachineFunction::Delegate + void MF_HandleInsertion(MachineInstr &MI) override { createdInstr(MI); } + void MF_HandleRemoval(MachineInstr &MI) override { erasingInstr(MI); } +}; + +/// A simple RAII based CSEInfo installer. +/// Use this in a scope to install a delegate to the MachineFunction and reset +/// it at the end of the scope. +class RAIIDelegateInstaller { + MachineFunction &MF; + MachineFunction::Delegate *Delegate; + +public: + RAIIDelegateInstaller(MachineFunction &MF, MachineFunction::Delegate *Del); + ~RAIIDelegateInstaller(); +}; + } // namespace llvm #endif Index: include/llvm/CodeGen/GlobalISel/GISelWorkList.h =================================================================== --- include/llvm/CodeGen/GlobalISel/GISelWorkList.h +++ include/llvm/CodeGen/GlobalISel/GISelWorkList.h @@ -18,6 +18,7 @@ namespace llvm { +class MachineInstr; class MachineFunction; // Worklist which mostly works similar to InstCombineWorkList, but on @@ -25,23 +26,15 @@ // erasing an element doesn't move all elements over one place - instead just // nulls out the element of the vector. // -// This worklist operates on instructions within a particular function. This is -// important for acquiring the rights to modify/replace instructions a -// GISelChangeObserver reports as the observer doesn't have the right to make -// changes to the instructions it sees so we use our access to the -// MachineFunction to establish that it's ok to add a given instruction to the -// worklist. -// // FIXME: Does it make sense to factor out common code with the // instcombinerWorkList? template class GISelWorkList { - MachineFunction *MF; SmallVector Worklist; DenseMap WorklistMap; public: - GISelWorkList(MachineFunction *MF) : MF(MF) {} + GISelWorkList() {} bool empty() const { return WorklistMap.empty(); } @@ -49,27 +42,8 @@ /// Add the specified instruction to the worklist if it isn't already in it. void insert(MachineInstr *I) { - // It would be safe to add this instruction to the worklist regardless but - // for consistency with the const version, check that the instruction we're - // adding would have been accepted if we were given a const pointer instead. - insert(const_cast(I)); - } - - void insert(const MachineInstr *I) { - // Confirm we'd be able to find the non-const pointer we want to schedule if - // we wanted to. We have the right to schedule work that may modify any - // instruction in MF. - assert(I->getParent() && "Expected parent BB"); - assert(I->getParent()->getParent() && "Expected parent function"); - assert((!MF || I->getParent()->getParent() == MF) && - "Expected parent function to be current function or not given"); - - // But don't actually do the search since we can derive it from the const - // pointer. - MachineInstr *NonConstI = const_cast(I); - if (WorklistMap.try_emplace(NonConstI, Worklist.size()).second) { - Worklist.push_back(NonConstI); - } + if (WorklistMap.try_emplace(I, Worklist.size()).second) + Worklist.push_back(I); } /// Remove I from the worklist if it exists. @@ -83,6 +57,11 @@ WorklistMap.erase(It); } + void clear() { + Worklist.clear(); + WorklistMap.clear(); + } + MachineInstr *pop_back_val() { MachineInstr *I; do { Index: include/llvm/CodeGen/GlobalISel/IRTranslator.h =================================================================== --- include/llvm/CodeGen/GlobalISel/IRTranslator.h +++ include/llvm/CodeGen/GlobalISel/IRTranslator.h @@ -21,11 +21,11 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" +#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h" #include "llvm/CodeGen/GlobalISel/Types.h" #include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/Support/Allocator.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/Support/Allocator.h" #include #include @@ -444,11 +444,13 @@ // I.e., compared to regular MIBuilder, this one also inserts the instruction // in the current block, it can creates block, etc., basically a kind of // IRBuilder, but for Machine IR. - MachineIRBuilder CurBuilder; + // CSEMIRBuilder CurBuilder; + std::unique_ptr CurBuilder; // Builder set to the entry block (just after ABI lowering instructions). Used // as a convenient location for Constants. - MachineIRBuilder EntryBuilder; + // CSEMIRBuilder EntryBuilder; + std::unique_ptr EntryBuilder; // The MachineFunction currently being translated. MachineFunction *MF; Index: include/llvm/CodeGen/GlobalISel/LegalizerHelper.h =================================================================== --- include/llvm/CodeGen/GlobalISel/LegalizerHelper.h +++ include/llvm/CodeGen/GlobalISel/LegalizerHelper.h @@ -49,9 +49,10 @@ UnableToLegalize, }; - LegalizerHelper(MachineFunction &MF, GISelChangeObserver &Observer); + LegalizerHelper(MachineFunction &MF, GISelChangeObserver &Observer, + MachineIRBuilder &B); LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI, - GISelChangeObserver &Observer); + GISelChangeObserver &Observer, MachineIRBuilder &B); /// Replace \p MI by a sequence of legal instructions that can implement the /// same operation. Note that this means \p MI may be deleted, so any iterator @@ -90,7 +91,7 @@ /// Expose MIRBuilder so clients can set their own RecordInsertInstruction /// functions - MachineIRBuilder MIRBuilder; + MachineIRBuilder &MIRBuilder; /// Expose LegalizerInfo so the clients can re-use. const LegalizerInfo &getLegalizerInfo() const { return LI; } Index: include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h =================================================================== --- include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -14,6 +14,7 @@ #ifndef LLVM_CODEGEN_GLOBALISEL_MACHINEIRBUILDER_H #define LLVM_CODEGEN_GLOBALISEL_MACHINEIRBUILDER_H +#include "llvm/CodeGen/GlobalISel/CSEInfo.h" #include "llvm/CodeGen/GlobalISel/Types.h" #include "llvm/CodeGen/LowLevelType.h" @@ -52,6 +53,8 @@ /// @} GISelChangeObserver *Observer; + + GISelCSEInfo *CSEInfo; }; class DstOp { @@ -81,8 +84,6 @@ } } - DstType getType() const { return Ty; } - LLT getLLTTy(const MachineRegisterInfo &MRI) const { switch (Ty) { case DstType::Ty_RC: @@ -95,6 +96,20 @@ llvm_unreachable("Unrecognised DstOp::DstType enum"); } + unsigned getReg() const { + assert(Ty == DstType::Ty_Reg && "Not a register"); + return Reg; + } + + const TargetRegisterClass *getRegClass() const { + switch (Ty) { + case DstType::Ty_RC: + return RC; + default: + llvm_unreachable("Not a RC Operand"); + } + } + DstType getDstOpKind() const { return Ty; } private: @@ -220,16 +235,25 @@ /// Getter for MRI MachineRegisterInfo *getMRI() { return State.MRI; } + const MachineRegisterInfo *getMRI() const { return State.MRI; } /// Getter for the State MachineIRBuilderState &getState() { return State; } /// Getter for the basic block we currently build. - MachineBasicBlock &getMBB() { + const MachineBasicBlock &getMBB() const { assert(State.MBB && "MachineBasicBlock is not set"); return *State.MBB; } + MachineBasicBlock &getMBB() { + return const_cast( + const_cast(this)->getMBB()); + } + + GISelCSEInfo *getCSEInfo() { return State.CSEInfo; } + const GISelCSEInfo *getCSEInfo() const { return State.CSEInfo; } + /// Current insertion point for new instructions. MachineBasicBlock::iterator getInsertPt() { return State.II; } @@ -239,10 +263,12 @@ void setInsertPt(MachineBasicBlock &MBB, MachineBasicBlock::iterator II); /// @} + void setCSEInfo(GISelCSEInfo *Info); + /// \name Setters for the insertion point. /// @{ /// Set the MachineFunction where to build instructions. - void setMF(MachineFunction &); + void setMF(MachineFunction &MF); /// Set the insertion point to the end of \p MBB. /// \pre \p MBB must be contained by getMF(). @@ -534,7 +560,8 @@ /// type. /// /// \return The newly created instruction. - MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val); + virtual MachineInstrBuilder buildConstant(const DstOp &Res, + const ConstantInt &Val); /// Build and insert \p Res = G_CONSTANT \p Val /// @@ -555,7 +582,8 @@ /// \pre \p Res must be a generic virtual register with scalar type. /// /// \return The newly created instruction. - MachineInstrBuilder buildFConstant(const DstOp &Res, const ConstantFP &Val); + virtual MachineInstrBuilder buildFConstant(const DstOp &Res, + const ConstantFP &Val); MachineInstrBuilder buildFConstant(const DstOp &Res, double Val); Index: include/llvm/CodeGen/GlobalISel/Utils.h =================================================================== --- include/llvm/CodeGen/GlobalISel/Utils.h +++ include/llvm/CodeGen/GlobalISel/Utils.h @@ -108,5 +108,8 @@ /// fallback. void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU); +Optional ConstantFoldBinOp(unsigned Opcode, const unsigned Op1, + const unsigned Op2, + const MachineRegisterInfo &MRI); } // End namespace llvm. #endif Index: include/llvm/CodeGen/MachineFunction.h =================================================================== --- include/llvm/CodeGen/MachineFunction.h +++ include/llvm/CodeGen/MachineFunction.h @@ -372,16 +372,18 @@ public: virtual ~Delegate() = default; - virtual void MF_HandleInsertion(const MachineInstr &MI) = 0; - virtual void MF_HandleRemoval(const MachineInstr &MI) = 0; + /// Callback after an insertion. This should not modify the MI directly. + virtual void MF_HandleInsertion(MachineInstr &MI) = 0; + /// Callback before a removal. This should not modify the MI directly. + virtual void MF_HandleRemoval(MachineInstr &MI) = 0; }; private: Delegate *TheDelegate = nullptr; // Callbacks for insertion and removal. - void handleInsertion(const MachineInstr &MI); - void handleRemoval(const MachineInstr &MI); + void handleInsertion(MachineInstr &MI); + void handleRemoval(MachineInstr &MI); friend struct ilist_traits; public: Index: include/llvm/InitializePasses.h =================================================================== --- include/llvm/InitializePasses.h +++ include/llvm/InitializePasses.h @@ -199,6 +199,7 @@ void initializeLegacyLICMPassPass(PassRegistry&); void initializeLegacyLoopSinkPassPass(PassRegistry&); void initializeLegalizerPass(PassRegistry&); +void initializeGISelCSEAnalysisWrapperPassPass(PassRegistry &); void initializeLibCallsShrinkWrapLegacyPassPass(PassRegistry&); void initializeLintPass(PassRegistry&); void initializeLiveDebugValuesPass(PassRegistry&); Index: include/llvm/Support/LowLevelTypeImpl.h =================================================================== --- include/llvm/Support/LowLevelTypeImpl.h +++ include/llvm/Support/LowLevelTypeImpl.h @@ -147,6 +147,7 @@ bool operator!=(const LLT &RHS) const { return !(*this == RHS); } friend struct DenseMapInfo; + friend class GISelInstProfileBuilder; private: /// LLT is packed into 64 bits as follows: @@ -231,6 +232,11 @@ maskAndShift(AddressSpace, PointerVectorAddressSpaceFieldInfo); } } + + uint64_t getUniqueRAWLLTData() const { + return ((uint64_t)RawData) << 2 | ((uint64_t)IsPointer) << 1 | + ((uint64_t)IsVector); + } }; inline raw_ostream& operator<<(raw_ostream &OS, const LLT &Ty) { @@ -250,8 +256,7 @@ return Invalid; } static inline unsigned getHashValue(const LLT &Ty) { - uint64_t Val = ((uint64_t)Ty.RawData) << 2 | ((uint64_t)Ty.IsPointer) << 1 | - ((uint64_t)Ty.IsVector); + uint64_t Val = Ty.getUniqueRAWLLTData(); return DenseMapInfo::getHashValue(Val); } static bool isEqual(const LLT &LHS, const LLT &RHS) { Index: lib/CodeGen/GlobalISel/CMakeLists.txt =================================================================== --- lib/CodeGen/GlobalISel/CMakeLists.txt +++ lib/CodeGen/GlobalISel/CMakeLists.txt @@ -1,4 +1,6 @@ add_llvm_library(LLVMGlobalISel + CSEInfo.cpp + CSEMIRBuilder.cpp CallLowering.cpp GlobalISel.cpp Combiner.cpp Index: lib/CodeGen/GlobalISel/CSEInfo.cpp =================================================================== --- /dev/null +++ lib/CodeGen/GlobalISel/CSEInfo.cpp @@ -0,0 +1,370 @@ +//===- CSEInfo.cpp ------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// +//===----------------------------------------------------------------------===// +#include "llvm/CodeGen/GlobalISel/CSEInfo.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" + +#define DEBUG_TYPE "cseinfo" + +using namespace llvm; +char llvm::GISelCSEAnalysisWrapperPass::ID = 0; +INITIALIZE_PASS_BEGIN(GISelCSEAnalysisWrapperPass, DEBUG_TYPE, + "Analysis containing CSE Info", false, true) +INITIALIZE_PASS_END(GISelCSEAnalysisWrapperPass, DEBUG_TYPE, + "Analysis containing CSE Info", false, true) + +/// -------- UniqueMachineInstr -------------// + +void UniqueMachineInstr::Profile(FoldingSetNodeID &ID) { + GISelInstProfileBuilder(ID, MI->getMF()->getRegInfo()).addNodeID(MI); +} +/// ----------------------------------------- + +/// --------- CSEConfig ---------- /// +bool CSEConfig::shouldCSEOpc(unsigned Opc) { + switch (Opc) { + default: + break; + case TargetOpcode::G_ADD: + case TargetOpcode::G_AND: + case TargetOpcode::G_ASHR: + case TargetOpcode::G_LSHR: + case TargetOpcode::G_MUL: + case TargetOpcode::G_OR: + case TargetOpcode::G_SHL: + case TargetOpcode::G_SUB: + case TargetOpcode::G_XOR: + case TargetOpcode::G_UDIV: + case TargetOpcode::G_SDIV: + case TargetOpcode::G_UREM: + case TargetOpcode::G_SREM: + case TargetOpcode::G_CONSTANT: + case TargetOpcode::G_FCONSTANT: + case TargetOpcode::G_ZEXT: + case TargetOpcode::G_SEXT: + case TargetOpcode::G_ANYEXT: + case TargetOpcode::G_UNMERGE_VALUES: + case TargetOpcode::G_TRUNC: + return true; + } + return false; +} + +bool CSEConfigConstantOnly::shouldCSEOpc(unsigned Opc) { + return Opc == TargetOpcode::G_CONSTANT; +} +/// ----------------------------------------- + +/// -------- GISelCSEInfo -------------// +void GISelCSEInfo::setMF(MachineFunction &MF) { + this->MF = &MF; + this->MRI = &MF.getRegInfo(); +} + +GISelCSEInfo::~GISelCSEInfo() {} + +bool GISelCSEInfo::isUniqueMachineInstValid( + const UniqueMachineInstr &UMI) const { + // Should we check here and assert that the instruction has been fully + // constructed? + // FIXME: Any other checks required to be done here? Remove this method if + // none. + return true; +} + +void GISelCSEInfo::invalidateUniqueMachineInstr(UniqueMachineInstr *UMI) { + bool Removed = CSEMap.RemoveNode(UMI); + (void)Removed; + assert(Removed && "Invalidation called on invalid UMI"); + // FIXME: Should UMI be deallocated/destroyed? +} + +UniqueMachineInstr *GISelCSEInfo::getNodeIfExists(FoldingSetNodeID &ID, + MachineBasicBlock *MBB, + void *&InsertPos) { + auto *Node = CSEMap.FindNodeOrInsertPos(ID, InsertPos); + if (Node) { + if (!isUniqueMachineInstValid(*Node)) { + invalidateUniqueMachineInstr(Node); + return nullptr; + } + + if (Node->MI->getParent() != MBB) + return nullptr; + } + return Node; +} + +void GISelCSEInfo::insertNode(UniqueMachineInstr *UMI, void *InsertPos) { + handleRecordedInsts(); + assert(UMI); + UniqueMachineInstr *MaybeNewNode = UMI; + if (InsertPos) + CSEMap.InsertNode(UMI, InsertPos); + else + MaybeNewNode = CSEMap.GetOrInsertNode(UMI); + if (MaybeNewNode != UMI) { + // A similar node exists in the folding set. Let's ignore this one. + return; + } + assert(InstrMapping.count(UMI->MI) == 0 && + "This instruction should not be in the map"); + InstrMapping[UMI->MI] = MaybeNewNode; +} + +UniqueMachineInstr *GISelCSEInfo::getUniqueInstrForMI(const MachineInstr *MI) { + assert(shouldCSE(MI->getOpcode()) && "Trying to CSE an unsupported Node"); + auto *Node = new (UniqueInstrAllocator) UniqueMachineInstr(MI); + return Node; +} + +void GISelCSEInfo::insertInstr(MachineInstr *MI, void *InsertPos) { + assert(MI); + // If it exists in temporary insts, remove it. + TemporaryInsts.remove(MI); + auto *Node = getUniqueInstrForMI(MI); + insertNode(Node, InsertPos); +} + +MachineInstr *GISelCSEInfo::getMachineInstrIfExists(FoldingSetNodeID &ID, + MachineBasicBlock *MBB, + void *&InsertPos) { + handleRecordedInsts(); + if (auto *Inst = getNodeIfExists(ID, MBB, InsertPos)) { + LLVM_DEBUG(dbgs() << "CSEInfo: Found Instr " << *Inst->MI << "\n";); + return const_cast(Inst->MI); + } + return nullptr; +} + +void GISelCSEInfo::countOpcodeHit(unsigned Opc) { +#ifndef NDEBUG + if (OpcodeHitTable.count(Opc)) + OpcodeHitTable[Opc] += 1; + else + OpcodeHitTable[Opc] = 1; +#endif + // Else do nothing. +} + +void GISelCSEInfo::recordNewInstruction(MachineInstr *MI) { + if (shouldCSE(MI->getOpcode())) { + TemporaryInsts.insert(MI); + LLVM_DEBUG(dbgs() << "CSEInfo: Recording new MI" << *MI << "\n";); + } +} + +void GISelCSEInfo::handleRecordedInst(MachineInstr *MI) { + assert(shouldCSE(MI->getOpcode()) && "Invalid instruction for CSE"); + auto *UMI = InstrMapping.lookup(MI); + LLVM_DEBUG(dbgs() << "CSEInfo: Handling recorded MI" << *MI << "\n";); + if (UMI) { + // Invalidate this MI. + invalidateUniqueMachineInstr(UMI); + InstrMapping.erase(MI); + } + /// Now insert the new instruction. + if (UMI) { + /// We'll reuse the same UniqueMachineInstr to avoid the new + /// allocation. + *UMI = UniqueMachineInstr(MI); + insertNode(UMI, nullptr); + } else { + /// This is a new instruction. Allocate a new UniqueMachineInstr and + /// Insert. + insertInstr(MI); + } +} + +void GISelCSEInfo::handleRemoveInst(MachineInstr *MI) { + if (auto *UMI = InstrMapping.lookup(MI)) { + invalidateUniqueMachineInstr(UMI); + InstrMapping.erase(MI); + } + TemporaryInsts.remove(MI); +} + +void GISelCSEInfo::handleRecordedInsts() { + while (!TemporaryInsts.empty()) { + auto *MI = TemporaryInsts.pop_back_val(); + handleRecordedInst(MI); + } +} + +bool GISelCSEInfo::shouldCSE(unsigned Opc) const { + // Only GISel opcodes are CSEable + if (!isPreISelGenericOpcode(Opc)) + return false; + assert(CSEOpt.get() && "CSEConfig not set"); + return CSEOpt->shouldCSEOpc(Opc); +} + +void GISelCSEInfo::erasingInstr(MachineInstr &MI) { handleRemoveInst(&MI); } +void GISelCSEInfo::createdInstr(MachineInstr &MI) { recordNewInstruction(&MI); } +void GISelCSEInfo::changingInstr(MachineInstr &MI) { + // For now, perform erase, followed by insert. + erasingInstr(MI); + createdInstr(MI); +} +void GISelCSEInfo::changedInstr(MachineInstr &MI) { changingInstr(MI); } + +void GISelCSEInfo::analyze(MachineFunction &MF) { + setMF(MF); + for (auto &MBB : MF) { + if (MBB.empty()) + continue; + for (MachineInstr &MI : MBB) { + if (!shouldCSE(MI.getOpcode())) + continue; + LLVM_DEBUG(dbgs() << "CSEInfo::Add MI: " << MI << "\n";); + insertInstr(&MI); + } + } +} + +void GISelCSEInfo::releaseMemory() { + // print(); + CSEMap.clear(); + InstrMapping.clear(); + UniqueInstrAllocator.Reset(); + TemporaryInsts.clear(); + CSEOpt.reset(); + MRI = nullptr; + MF = nullptr; +#ifndef NDEBUG + OpcodeHitTable.clear(); +#endif +} + +void GISelCSEInfo::print() { +#ifndef NDEBUG + for (auto &It : OpcodeHitTable) { + dbgs() << "CSE Count for Opc " << It.first << " : " << It.second << "\n"; + }; +#endif +} +/// ----------------------------------------- +// ---- Profiling methods for FoldingSetNode --- // +const GISelInstProfileBuilder & +GISelInstProfileBuilder::addNodeID(const MachineInstr *MI) const { + addNodeIDMBB(MI->getParent()); + addNodeIDOpcode(MI->getOpcode()); + for (auto &Op : MI->operands()) + addNodeIDMachineOperand(Op); + addNodeIDFlag(MI->getFlags()); + return *this; +} + +const GISelInstProfileBuilder & +GISelInstProfileBuilder::addNodeIDOpcode(unsigned Opc) const { + ID.AddInteger(Opc); + return *this; +} + +const GISelInstProfileBuilder & +GISelInstProfileBuilder::addNodeIDRegType(const LLT &Ty) const { + uint64_t Val = Ty.getUniqueRAWLLTData(); + ID.AddInteger(Val); + return *this; +} + +const GISelInstProfileBuilder & +GISelInstProfileBuilder::addNodeIDRegType(const TargetRegisterClass *RC) const { + ID.AddPointer(RC); + return *this; +} + +const GISelInstProfileBuilder & +GISelInstProfileBuilder::addNodeIDRegType(const RegisterBank *RB) const { + ID.AddPointer(RB); + return *this; +} + +const GISelInstProfileBuilder & +GISelInstProfileBuilder::addNodeIDImmediate(int64_t Imm) const { + ID.AddInteger(Imm); + return *this; +} + +const GISelInstProfileBuilder & +GISelInstProfileBuilder::addNodeIDRegNum(unsigned Reg) const { + ID.AddInteger(Reg); + return *this; +} + +const GISelInstProfileBuilder & +GISelInstProfileBuilder::addNodeIDRegType(const unsigned Reg) const { + addNodeIDMachineOperand(MachineOperand::CreateReg(Reg, false)); + return *this; +} + +const GISelInstProfileBuilder & +GISelInstProfileBuilder::addNodeIDMBB(const MachineBasicBlock *MBB) const { + ID.AddPointer(MBB); + return *this; +} + +const GISelInstProfileBuilder & +GISelInstProfileBuilder::addNodeIDFlag(unsigned Flag) const { + if (Flag) + ID.AddInteger(Flag); + return *this; +} + +const GISelInstProfileBuilder &GISelInstProfileBuilder::addNodeIDMachineOperand( + const MachineOperand &MO) const { + if (MO.isReg()) { + unsigned Reg = MO.getReg(); + if (!MO.isDef()) + addNodeIDRegNum(Reg); + LLT Ty = MRI.getType(Reg); + if (Ty.isValid()) + addNodeIDRegType(Ty); + auto *RB = MRI.getRegBankOrNull(Reg); + if (RB) + addNodeIDRegType(RB); + auto *RC = MRI.getRegClassOrNull(Reg); + if (RC) + addNodeIDRegType(RC); + assert(!MO.isImplicit() && "Unhandled case"); + } else if (MO.isImm()) + ID.AddInteger(MO.getImm()); + else if (MO.isCImm()) + ID.AddPointer(MO.getCImm()); + else if (MO.isFPImm()) + ID.AddPointer(MO.getFPImm()); + else if (MO.isPredicate()) + ID.AddInteger(MO.getPredicate()); + else + llvm_unreachable("Unhandled operand type"); + // Handle other types + return *this; +} + +GISelCSEInfo &GISelCSEAnalysisWrapper::get(std::unique_ptr CSEOpt, + bool Recompute) { + if (!AlreadyComputed || Recompute) { + Info.setCSEConfig(std::move(CSEOpt)); + Info.analyze(*MF); + AlreadyComputed = true; + } + return Info; +} +void GISelCSEAnalysisWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const { + AU.setPreservesAll(); + MachineFunctionPass::getAnalysisUsage(AU); +} + +bool GISelCSEAnalysisWrapperPass::runOnMachineFunction(MachineFunction &MF) { + releaseMemory(); + Wrapper.setMF(MF); + return false; +} Index: lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp =================================================================== --- /dev/null +++ lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp @@ -0,0 +1,231 @@ +//===-- llvm/CodeGen/GlobalISel/CSEMIRBuilder.cpp - MIBuilder--*- C++ -*-==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// This file implements the CSEMIRBuilder class which CSEs as it builds +/// instructions. +//===----------------------------------------------------------------------===// +// + +#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h" +#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" + +using namespace llvm; + +bool CSEMIRBuilder::dominates(MachineBasicBlock::const_iterator A, + MachineBasicBlock::const_iterator B) const { + auto MBBEnd = getMBB().end(); + if (B == MBBEnd) + return true; + assert(A->getParent() == B->getParent() && + "Iterators should be in same block"); + const MachineBasicBlock *BBA = A->getParent(); + MachineBasicBlock::const_iterator I = BBA->begin(); + for (; &*I != A && &*I != B; ++I) + ; + return &*I == A; +} + +MachineInstrBuilder +CSEMIRBuilder::getDominatingInstrForID(FoldingSetNodeID &ID, + void *&NodeInsertPos) { + GISelCSEInfo *CSEInfo = getCSEInfo(); + assert(CSEInfo && "Can't get here without setting CSEInfo"); + MachineBasicBlock *CurMBB = &getMBB(); + MachineInstr *MI = + CSEInfo->getMachineInstrIfExists(ID, CurMBB, NodeInsertPos); + if (MI) { + auto CurrPos = getInsertPt(); + if (!dominates(MI, CurrPos)) + CurMBB->splice(CurrPos, CurMBB, MI); + return MachineInstrBuilder(getMF(), MI); + } + return MachineInstrBuilder(); +} + +bool CSEMIRBuilder::canPerformCSEForOpc(unsigned Opc) const { + const GISelCSEInfo *CSEInfo = getCSEInfo(); + if (!CSEInfo || !CSEInfo->shouldCSE(Opc)) + return false; + return true; +} + +void CSEMIRBuilder::profileDstOp(const DstOp &Op, + GISelInstProfileBuilder &B) const { + switch (Op.getDstOpKind()) { + case DstOp::DstType::Ty_RC: + B.addNodeIDRegType(Op.getRegClass()); + break; + default: + B.addNodeIDRegType(Op.getLLTTy(*getMRI())); + break; + } +} + +void CSEMIRBuilder::profileSrcOp(const SrcOp &Op, + GISelInstProfileBuilder &B) const { + switch (Op.getSrcOpKind()) { + case SrcOp::SrcType::Ty_Predicate: + B.addNodeIDImmediate(static_cast(Op.getPredicate())); + break; + default: + B.addNodeIDRegType(Op.getReg()); + break; + } +} + +void CSEMIRBuilder::profileMBBOpcode(GISelInstProfileBuilder &B, + unsigned Opc) const { + // First add the MBB (Local CSE). + B.addNodeIDMBB(&getMBB()); + // Then add the opcode. + B.addNodeIDOpcode(Opc); +} + +void CSEMIRBuilder::profileEverything(unsigned Opc, ArrayRef DstOps, + ArrayRef SrcOps, + Optional Flags, + GISelInstProfileBuilder &B) const { + + profileMBBOpcode(B, Opc); + // Then add the DstOps. + profileDstOps(DstOps, B); + // Then add the SrcOps. + profileSrcOps(SrcOps, B); + // Add Flags if passed in. + if (Flags) + B.addNodeIDFlag(*Flags); +} + +MachineInstrBuilder CSEMIRBuilder::memoizeMI(MachineInstrBuilder MIB, + void *NodeInsertPos) { + assert(canPerformCSEForOpc(MIB->getOpcode()) && + "Attempting to CSE illegal op"); + MachineInstr *MIBInstr = MIB; + getCSEInfo()->insertInstr(MIBInstr, NodeInsertPos); + return MIB; +} + +bool CSEMIRBuilder::checkCopyToDefsPossible(ArrayRef DstOps) { + if (DstOps.size() == 1) + return true; // always possible to emit copy to just 1 vreg. + + return std::all_of(DstOps.begin(), DstOps.end(), [](const DstOp &Op) { + DstOp::DstType DT = Op.getDstOpKind(); + return DT == DstOp::DstType::Ty_LLT || DT == DstOp::DstType::Ty_RC; + }); +} + +MachineInstrBuilder +CSEMIRBuilder::generateCopiesIfRequired(ArrayRef DstOps, + MachineInstrBuilder &MIB) { + assert(checkCopyToDefsPossible(DstOps) && + "Impossible return a single MIB with copies to multiple defs"); + if (DstOps.size() == 1) { + const DstOp &Op = DstOps[0]; + if (Op.getDstOpKind() == DstOp::DstType::Ty_Reg) + return buildCopy(Op.getReg(), MIB->getOperand(0).getReg()); + } + return MIB; +} + +MachineInstrBuilder CSEMIRBuilder::buildInstr(unsigned Opc, + ArrayRef DstOps, + ArrayRef SrcOps, + Optional Flag) { + switch (Opc) { + default: + break; + case TargetOpcode::G_ADD: + case TargetOpcode::G_AND: + case TargetOpcode::G_ASHR: + case TargetOpcode::G_LSHR: + case TargetOpcode::G_MUL: + case TargetOpcode::G_OR: + case TargetOpcode::G_SHL: + case TargetOpcode::G_SUB: + case TargetOpcode::G_XOR: + case TargetOpcode::G_UDIV: + case TargetOpcode::G_SDIV: + case TargetOpcode::G_UREM: + case TargetOpcode::G_SREM: { + // Try to constant fold these. + assert(SrcOps.size() == 2 && "Invalid sources"); + assert(DstOps.size() == 1 && "Invalid dsts"); + if (Optional Cst = ConstantFoldBinOp(Opc, SrcOps[0].getReg(), + SrcOps[1].getReg(), *getMRI())) + return buildConstant(DstOps[0], Cst->getSExtValue()); + break; + } + } + bool CanCopy = checkCopyToDefsPossible(DstOps); + if (!canPerformCSEForOpc(Opc)) + return MachineIRBuilder::buildInstr(Opc, DstOps, SrcOps, Flag); + // If we can CSE this instruction, but involves generating copies to multiple + // regs, give up. This frequently happens to UNMERGEs. + if (!CanCopy) { + auto MIB = MachineIRBuilder::buildInstr(Opc, DstOps, SrcOps, Flag); + // CSEInfo would have tracked this instruction. Remove it from the temporary + // insts. + getCSEInfo()->handleRemoveInst(&*MIB); + return MIB; + } + FoldingSetNodeID ID; + GISelInstProfileBuilder ProfBuilder(ID, *getMRI()); + void *InsertPos = nullptr; + profileEverything(Opc, DstOps, SrcOps, Flag, ProfBuilder); + MachineInstrBuilder MIB = getDominatingInstrForID(ID, InsertPos); + if (MIB) { + // Handle generating copies here. + return generateCopiesIfRequired(DstOps, MIB); + } + // This instruction does not exist in the CSEInfo. Build it and CSE it. + MachineInstrBuilder NewMIB = + MachineIRBuilder::buildInstr(Opc, DstOps, SrcOps, Flag); + return memoizeMI(NewMIB, InsertPos); +} + +MachineInstrBuilder CSEMIRBuilder::buildConstant(const DstOp &Res, + const ConstantInt &Val) { + constexpr unsigned Opc = TargetOpcode::G_CONSTANT; + if (!canPerformCSEForOpc(Opc)) + return MachineIRBuilder::buildConstant(Res, Val); + FoldingSetNodeID ID; + GISelInstProfileBuilder ProfBuilder(ID, *getMRI()); + void *InsertPos = nullptr; + profileMBBOpcode(ProfBuilder, Opc); + profileDstOp(Res, ProfBuilder); + ProfBuilder.addNodeIDMachineOperand(MachineOperand::CreateCImm(&Val)); + MachineInstrBuilder MIB = getDominatingInstrForID(ID, InsertPos); + if (MIB) { + // Handle generating copies here. + return generateCopiesIfRequired({Res}, MIB); + } + MachineInstrBuilder NewMIB = MachineIRBuilder::buildConstant(Res, Val); + return memoizeMI(NewMIB, InsertPos); +} + +MachineInstrBuilder CSEMIRBuilder::buildFConstant(const DstOp &Res, + const ConstantFP &Val) { + constexpr unsigned Opc = TargetOpcode::G_FCONSTANT; + if (!canPerformCSEForOpc(Opc)) + return MachineIRBuilder::buildFConstant(Res, Val); + FoldingSetNodeID ID; + GISelInstProfileBuilder ProfBuilder(ID, *getMRI()); + void *InsertPos = nullptr; + profileMBBOpcode(ProfBuilder, Opc); + profileDstOp(Res, ProfBuilder); + ProfBuilder.addNodeIDMachineOperand(MachineOperand::CreateFPImm(&Val)); + MachineInstrBuilder MIB = getDominatingInstrForID(ID, InsertPos); + if (MIB) { + // Handle generating copies here. + return generateCopiesIfRequired({Res}, MIB); + } + MachineInstrBuilder NewMIB = MachineIRBuilder::buildFConstant(Res, Val); + return memoizeMI(NewMIB, InsertPos); +} Index: lib/CodeGen/GlobalISel/Combiner.cpp =================================================================== --- lib/CodeGen/GlobalISel/Combiner.cpp +++ lib/CodeGen/GlobalISel/Combiner.cpp @@ -13,7 +13,9 @@ #include "llvm/CodeGen/GlobalISel/Combiner.h" #include "llvm/ADT/PostOrderIterator.h" +#include "llvm/CodeGen/GlobalISel/CSEInfo.h" #include "llvm/CodeGen/GlobalISel/CombinerInfo.h" +#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h" #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" #include "llvm/CodeGen/GlobalISel/GISelWorkList.h" #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" @@ -35,38 +37,33 @@ /// instruction creation will schedule that instruction for a future visit. /// Other Combiner implementations may require more complex behaviour from /// their GISelChangeObserver subclass. -class WorkListMaintainer : public GISelChangeObserver, - public MachineFunction::Delegate { +class WorkListMaintainer : public GISelChangeObserver { using WorkListTy = GISelWorkList<512>; - MachineFunction &MF; WorkListTy &WorkList; /// The instructions that have been created but we want to report once they /// have their operands. This is only maintained if debug output is requested. SmallPtrSet CreatedInstrs; public: - WorkListMaintainer(MachineFunction &MF, WorkListTy &WorkList) - : GISelChangeObserver(), MF(MF), WorkList(WorkList) { - MF.setDelegate(this); - } + WorkListMaintainer(WorkListTy &WorkList) + : GISelChangeObserver(), WorkList(WorkList) {} virtual ~WorkListMaintainer() { - MF.resetDelegate(this); } - void erasingInstr(const MachineInstr &MI) override { + void erasingInstr(MachineInstr &MI) override { LLVM_DEBUG(dbgs() << "Erased: " << MI << "\n"); WorkList.remove(&MI); } - void createdInstr(const MachineInstr &MI) override { + void createdInstr(MachineInstr &MI) override { LLVM_DEBUG(dbgs() << "Creating: " << MI << "\n"); WorkList.insert(&MI); LLVM_DEBUG(CreatedInstrs.insert(&MI)); } - void changingInstr(const MachineInstr &MI) override { + void changingInstr(MachineInstr &MI) override { LLVM_DEBUG(dbgs() << "Changing: " << MI << "\n"); WorkList.insert(&MI); } - void changedInstr(const MachineInstr &MI) override { + void changedInstr(MachineInstr &MI) override { LLVM_DEBUG(dbgs() << "Changed: " << MI << "\n"); WorkList.insert(&MI); } @@ -79,13 +76,6 @@ }); LLVM_DEBUG(CreatedInstrs.clear()); } - - void MF_HandleInsertion(const MachineInstr &MI) override { - createdInstr(MI); - } - void MF_HandleRemoval(const MachineInstr &MI) override { - erasingInstr(MI); - } }; } @@ -94,15 +84,20 @@ (void)this->TPC; // FIXME: Remove when used. } -bool Combiner::combineMachineInstrs(MachineFunction &MF) { +bool Combiner::combineMachineInstrs(MachineFunction &MF, + GISelCSEInfo *CSEInfo) { // If the ISel pipeline failed, do not bother running this pass. // FIXME: Should this be here or in individual combiner passes. if (MF.getProperties().hasProperty( MachineFunctionProperties::Property::FailedISel)) return false; + Builder = + CSEInfo ? make_unique() : make_unique(); MRI = &MF.getRegInfo(); - Builder.setMF(MF); + Builder->setMF(MF); + if (CSEInfo) + Builder->setCSEInfo(CSEInfo); LLVM_DEBUG(dbgs() << "Generic MI Combiner for: " << MF.getName() << '\n'); @@ -110,14 +105,19 @@ bool MFChanged = false; bool Changed; + MachineIRBuilder &B = *Builder.get(); do { // Collect all instructions. Do a post order traversal for basic blocks and // insert with list bottom up, so while we pop_back_val, we'll traverse top // down RPOT. Changed = false; - GISelWorkList<512> WorkList(&MF); - WorkListMaintainer Observer(MF, WorkList); + GISelWorkList<512> WorkList; + WorkListMaintainer Observer(WorkList); + GISelObserverWrapper WrapperObserver(&Observer); + if (CSEInfo) + WrapperObserver.addObserver(CSEInfo); + RAIIDelegateInstaller DelInstall(MF, &WrapperObserver); for (MachineBasicBlock *MBB : post_order(&MF)) { if (MBB->empty()) continue; @@ -137,7 +137,7 @@ while (!WorkList.empty()) { MachineInstr *CurrInst = WorkList.pop_back_val(); LLVM_DEBUG(dbgs() << "\nTry combining " << *CurrInst;); - Changed |= CInfo.combine(Observer, *CurrInst, Builder); + Changed |= CInfo.combine(WrapperObserver, *CurrInst, B); Observer.reportFullyCreatedInstrs(); } MFChanged |= Changed; Index: lib/CodeGen/GlobalISel/GISelChangeObserver.cpp =================================================================== --- lib/CodeGen/GlobalISel/GISelChangeObserver.cpp +++ lib/CodeGen/GlobalISel/GISelChangeObserver.cpp @@ -29,3 +29,12 @@ changedInstr(*ChangedMI); } +RAIIDelegateInstaller::RAIIDelegateInstaller(MachineFunction &MF, + MachineFunction::Delegate *Del) + : MF(MF), Delegate(Del) { + // Register this as the delegate for handling insertions and deletions of + // instructions. + MF.setDelegate(Del); +} + +RAIIDelegateInstaller::~RAIIDelegateInstaller() { MF.resetDelegate(Delegate); } Index: lib/CodeGen/GlobalISel/IRTranslator.cpp =================================================================== --- lib/CodeGen/GlobalISel/IRTranslator.cpp +++ lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -19,6 +19,7 @@ #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/GlobalISel/CallLowering.h" +#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" #include "llvm/CodeGen/LowLevelType.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFrameInfo.h" @@ -75,11 +76,16 @@ using namespace llvm; +static cl::opt + EnableCSEInIRTranslator("enable-cse-in-irtranslator", + cl::desc("Should enable CSE in irtranslator"), + cl::Optional, cl::init(false)); char IRTranslator::ID = 0; INITIALIZE_PASS_BEGIN(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI", false, false) INITIALIZE_PASS_DEPENDENCY(TargetPassConfig) +INITIALIZE_PASS_DEPENDENCY(GISelCSEAnalysisWrapperPass) INITIALIZE_PASS_END(IRTranslator, DEBUG_TYPE, "IRTranslator LLVM IR -> MI", false, false) @@ -108,18 +114,21 @@ namespace { /// Verify that every instruction created has the same DILocation as the /// instruction being translated. -class DILocationVerifier : MachineFunction::Delegate { - MachineFunction &MF; +class DILocationVerifier : public GISelChangeObserver { const Instruction *CurrInst = nullptr; public: - DILocationVerifier(MachineFunction &MF) : MF(MF) { MF.setDelegate(this); } - ~DILocationVerifier() { MF.resetDelegate(this); } + DILocationVerifier() = default; + ~DILocationVerifier() = default; const Instruction *getCurrentInst() const { return CurrInst; } void setCurrentInst(const Instruction *Inst) { CurrInst = Inst; } - void MF_HandleInsertion(const MachineInstr &MI) override { + void erasingInstr(MachineInstr &MI) override {} + void changingInstr(MachineInstr &MI) override {} + void changedInstr(MachineInstr &MI) override {} + + void createdInstr(MachineInstr &MI) override { assert(getCurrentInst() && "Inserted instruction without a current MI"); // Only print the check message if we're actually checking it. @@ -130,7 +139,6 @@ assert(CurrInst->getDebugLoc() == MI.getDebugLoc() && "Line info was not transferred to all instructions"); } - void MF_HandleRemoval(const MachineInstr &MI) override {} }; } // namespace #endif // ifndef NDEBUG @@ -139,6 +147,7 @@ void IRTranslator::getAnalysisUsage(AnalysisUsage &AU) const { AU.addRequired(); AU.addRequired(); + AU.addRequired(); getSelectionDAGFallbackAnalysisUsage(AU); MachineFunctionPass::getAnalysisUsage(AU); } @@ -1553,12 +1562,14 @@ void IRTranslator::finishPendingPhis() { #ifndef NDEBUG - DILocationVerifier Verifier(*MF); + DILocationVerifier Verifier; + GISelObserverWrapper WrapperObserver(&Verifier); + RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver); #endif // ifndef NDEBUG for (auto &Phi : PendingPHIs) { const PHINode *PI = Phi.first; ArrayRef ComponentPHIs = Phi.second; - EntryBuilder.setDebugLoc(PI->getDebugLoc()); + EntryBuilder->setDebugLoc(PI->getDebugLoc()); #ifndef NDEBUG Verifier.setCurrentInst(PI); #endif // ifndef NDEBUG @@ -1599,11 +1610,12 @@ } bool IRTranslator::translate(const Instruction &Inst) { - CurBuilder.setDebugLoc(Inst.getDebugLoc()); - EntryBuilder.setDebugLoc(Inst.getDebugLoc()); + CurBuilder->setDebugLoc(Inst.getDebugLoc()); + EntryBuilder->setDebugLoc(Inst.getDebugLoc()); switch(Inst.getOpcode()) { -#define HANDLE_INST(NUM, OPCODE, CLASS) \ - case Instruction::OPCODE: return translate##OPCODE(Inst, CurBuilder); +#define HANDLE_INST(NUM, OPCODE, CLASS) \ + case Instruction::OPCODE: \ + return translate##OPCODE(Inst, *CurBuilder.get()); #include "llvm/IR/Instruction.def" default: return false; @@ -1612,11 +1624,11 @@ bool IRTranslator::translate(const Constant &C, unsigned Reg) { if (auto CI = dyn_cast(&C)) - EntryBuilder.buildConstant(Reg, *CI); + EntryBuilder->buildConstant(Reg, *CI); else if (auto CF = dyn_cast(&C)) - EntryBuilder.buildFConstant(Reg, *CF); + EntryBuilder->buildFConstant(Reg, *CF); else if (isa(C)) - EntryBuilder.buildUndef(Reg); + EntryBuilder->buildUndef(Reg); else if (isa(C)) { // As we are trying to build a constant val of 0 into a pointer, // insert a cast to make them correct with respect to types. @@ -1624,9 +1636,9 @@ auto *ZeroTy = Type::getIntNTy(C.getContext(), NullSize); auto *ZeroVal = ConstantInt::get(ZeroTy, 0); unsigned ZeroReg = getOrCreateVReg(*ZeroVal); - EntryBuilder.buildCast(Reg, ZeroReg); + EntryBuilder->buildCast(Reg, ZeroReg); } else if (auto GV = dyn_cast(&C)) - EntryBuilder.buildGlobalValue(Reg, GV); + EntryBuilder->buildGlobalValue(Reg, GV); else if (auto CAZ = dyn_cast(&C)) { if (!CAZ->getType()->isVectorTy()) return false; @@ -1638,7 +1650,7 @@ Constant &Elt = *CAZ->getElementValue(i); Ops.push_back(getOrCreateVReg(Elt)); } - EntryBuilder.buildBuildVector(Reg, Ops); + EntryBuilder->buildBuildVector(Reg, Ops); } else if (auto CV = dyn_cast(&C)) { // Return the scalar if it is a <1 x Ty> vector. if (CV->getNumElements() == 1) @@ -1648,11 +1660,12 @@ Constant &Elt = *CV->getElementAsConstant(i); Ops.push_back(getOrCreateVReg(Elt)); } - EntryBuilder.buildBuildVector(Reg, Ops); + EntryBuilder->buildBuildVector(Reg, Ops); } else if (auto CE = dyn_cast(&C)) { switch(CE->getOpcode()) { -#define HANDLE_INST(NUM, OPCODE, CLASS) \ - case Instruction::OPCODE: return translate##OPCODE(*CE, EntryBuilder); +#define HANDLE_INST(NUM, OPCODE, CLASS) \ + case Instruction::OPCODE: \ + return translate##OPCODE(*CE, *EntryBuilder.get()); #include "llvm/IR/Instruction.def" default: return false; @@ -1664,9 +1677,9 @@ for (unsigned i = 0; i < CV->getNumOperands(); ++i) { Ops.push_back(getOrCreateVReg(*CV->getOperand(i))); } - EntryBuilder.buildBuildVector(Reg, Ops); + EntryBuilder->buildBuildVector(Reg, Ops); } else if (auto *BA = dyn_cast(&C)) { - EntryBuilder.buildBlockAddress(Reg, BA); + EntryBuilder->buildBlockAddress(Reg, BA); } else return false; @@ -1683,8 +1696,8 @@ // MachineIRBuilder::DebugLoc can outlive the DILocation it holds. Clear it // to avoid accessing free’d memory (in runOnMachineFunction) and to avoid // destroying it twice (in ~IRTranslator() and ~LLVMContext()) - EntryBuilder = MachineIRBuilder(); - CurBuilder = MachineIRBuilder(); + EntryBuilder.reset(); + CurBuilder.reset(); } bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) { @@ -1692,12 +1705,30 @@ const Function &F = MF->getFunction(); if (F.empty()) return false; + GISelCSEAnalysisWrapper &Wrapper = + getAnalysis().getCSEWrapper(); + // Set the CSEConfig and run the analysis. + GISelCSEInfo *CSEInfo = nullptr; + TPC = &getAnalysis(); + bool IsO0 = TPC->getOptLevel() == CodeGenOpt::Level::None; + // Disable CSE for O0. + bool EnableCSE = !IsO0 && EnableCSEInIRTranslator; + if (EnableCSE) { + EntryBuilder = make_unique(CurMF); + std::unique_ptr Config = make_unique(); + CSEInfo = &Wrapper.get(std::move(Config)); + EntryBuilder->setCSEInfo(CSEInfo); + CurBuilder = make_unique(CurMF); + CurBuilder->setCSEInfo(CSEInfo); + } else { + EntryBuilder = make_unique(); + CurBuilder = make_unique(); + } CLI = MF->getSubtarget().getCallLowering(); - CurBuilder.setMF(*MF); - EntryBuilder.setMF(*MF); + CurBuilder->setMF(*MF); + EntryBuilder->setMF(*MF); MRI = &MF->getRegInfo(); DL = &F.getParent()->getDataLayout(); - TPC = &getAnalysis(); ORE = llvm::make_unique(&F); assert(PendingPHIs.empty() && "stale PHIs"); @@ -1716,7 +1747,7 @@ // Setup a separate basic-block for the arguments and constants MachineBasicBlock *EntryBB = MF->CreateMachineBasicBlock(); MF->push_back(EntryBB); - EntryBuilder.setMBB(*EntryBB); + EntryBuilder->setMBB(*EntryBB); // Create all blocks, in IR order, to preserve the layout. for (const BasicBlock &BB: F) { @@ -1753,7 +1784,7 @@ } } - if (!CLI->lowerFormalArguments(EntryBuilder, F, VRegArgs)) { + if (!CLI->lowerFormalArguments(*EntryBuilder.get(), F, VRegArgs)) { OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure", F.getSubprogram(), &F.getEntryBlock()); R << "unable to lower arguments: " << ore::NV("Prototype", F.getType()); @@ -1770,22 +1801,27 @@ assert(VRegs.empty() && "VRegs already populated?"); VRegs.push_back(VArg); } else { - unpackRegs(*ArgIt, VArg, EntryBuilder); + unpackRegs(*ArgIt, VArg, *EntryBuilder.get()); } ArgIt++; } // Need to visit defs before uses when translating instructions. + GISelObserverWrapper WrapperObserver; + if (EnableCSE && CSEInfo) + WrapperObserver.addObserver(CSEInfo); { ReversePostOrderTraversal RPOT(&F); #ifndef NDEBUG - DILocationVerifier Verifier(*MF); + DILocationVerifier Verifier; + WrapperObserver.addObserver(&Verifier); #endif // ifndef NDEBUG + RAIIDelegateInstaller DelInstall(*MF, &WrapperObserver); for (const BasicBlock *BB : RPOT) { MachineBasicBlock &MBB = getMBB(*BB); // Set the insertion point of all the following translations to // the end of this basic block. - CurBuilder.setMBB(MBB); + CurBuilder->setMBB(MBB); for (const Instruction &Inst : *BB) { #ifndef NDEBUG @@ -1810,6 +1846,9 @@ return false; } } +#ifndef NDEBUG + WrapperObserver.removeObserver(&Verifier); +#endif } finishPendingPhis(); Index: lib/CodeGen/GlobalISel/Legalizer.cpp =================================================================== --- lib/CodeGen/GlobalISel/Legalizer.cpp +++ lib/CodeGen/GlobalISel/Legalizer.cpp @@ -16,6 +16,8 @@ #include "llvm/CodeGen/GlobalISel/Legalizer.h" #include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/SetVector.h" +#include "llvm/CodeGen/GlobalISel/CSEInfo.h" +#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h" #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" #include "llvm/CodeGen/GlobalISel/GISelWorkList.h" #include "llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h" @@ -33,11 +35,17 @@ using namespace llvm; +static cl::opt + EnableCSEInLegalizer("enable-cse-in-legalizer", + cl::desc("Should enable CSE in Legalizer"), + cl::Optional, cl::init(false)); + char Legalizer::ID = 0; INITIALIZE_PASS_BEGIN(Legalizer, DEBUG_TYPE, "Legalize the Machine IR a function's Machine IR", false, false) INITIALIZE_PASS_DEPENDENCY(TargetPassConfig) +INITIALIZE_PASS_DEPENDENCY(GISelCSEAnalysisWrapperPass) INITIALIZE_PASS_END(Legalizer, DEBUG_TYPE, "Legalize the Machine IR a function's Machine IR", false, false) @@ -48,6 +56,8 @@ void Legalizer::getAnalysisUsage(AnalysisUsage &AU) const { AU.addRequired(); + AU.addRequired(); + AU.addPreserved(); getSelectionDAGFallbackAnalysisUsage(AU); MachineFunctionPass::getAnalysisUsage(AU); } @@ -82,7 +92,7 @@ LegalizerWorkListManager(InstListTy &Insts, ArtifactListTy &Arts) : InstList(Insts), ArtifactList(Arts) {} - void createdInstr(const MachineInstr &MI) override { + void createdInstr(MachineInstr &MI) override { // Only legalize pre-isel generic instructions. // Legalization process could generate Target specific pseudo // instructions with generic types. Don't record them @@ -95,17 +105,17 @@ LLVM_DEBUG(dbgs() << ".. .. New MI: " << MI); } - void erasingInstr(const MachineInstr &MI) override { + void erasingInstr(MachineInstr &MI) override { LLVM_DEBUG(dbgs() << ".. .. Erasing: " << MI); InstList.remove(&MI); ArtifactList.remove(&MI); } - void changingInstr(const MachineInstr &MI) override { + void changingInstr(MachineInstr &MI) override { LLVM_DEBUG(dbgs() << ".. .. Changing MI: " << MI); } - void changedInstr(const MachineInstr &MI) override { + void changedInstr(MachineInstr &MI) override { // When insts change, we want to revisit them to legalize them again. // We'll consider them the same as created. LLVM_DEBUG(dbgs() << ".. .. Changed MI: " << MI); @@ -122,14 +132,16 @@ LLVM_DEBUG(dbgs() << "Legalize Machine IR for: " << MF.getName() << '\n'); init(MF); const TargetPassConfig &TPC = getAnalysis(); + GISelCSEAnalysisWrapper &Wrapper = + getAnalysis().getCSEWrapper(); MachineOptimizationRemarkEmitter MORE(MF, /*MBFI=*/nullptr); const size_t NumBlocks = MF.size(); MachineRegisterInfo &MRI = MF.getRegInfo(); // Populate Insts - InstListTy InstList(&MF); - ArtifactListTy ArtifactList(&MF); + InstListTy InstList; + ArtifactListTy ArtifactList; ReversePostOrderTraversal RPOT(&MF); // Perform legalization bottom up so we can DCE as we legalize. // Traverse BB in RPOT and within each basic block, add insts top down, @@ -148,12 +160,34 @@ InstList.insert(&MI); } } + std::unique_ptr MIRBuilder; + GISelCSEInfo *CSEInfo = nullptr; + bool IsO0 = TPC.getOptLevel() == CodeGenOpt::Level::None; + // Disable CSE for O0. + bool EnableCSE = !IsO0 && EnableCSEInLegalizer; + if (EnableCSE) { + MIRBuilder = make_unique(); + std::unique_ptr Config = make_unique(); + CSEInfo = &Wrapper.get(std::move(Config)); + MIRBuilder->setCSEInfo(CSEInfo); + } else + MIRBuilder = make_unique(); + // This observer keeps the worklist updated. LegalizerWorkListManager WorkListObserver(InstList, ArtifactList); - LegalizerHelper Helper(MF, WorkListObserver); + // We want both WorkListObserver as well as CSEInfo to observe all changes. + // Use the wrapper observer. + GISelObserverWrapper WrapperObserver(&WorkListObserver); + if (EnableCSE && CSEInfo) + WrapperObserver.addObserver(CSEInfo); + // Now install the observer as the delegate to MF. + // This will keep all the observers notified about new insertions/deletions. + RAIIDelegateInstaller DelInstall(MF, &WrapperObserver); + LegalizerHelper Helper(MF, WrapperObserver, *MIRBuilder.get()); const LegalizerInfo &LInfo(Helper.getLegalizerInfo()); - LegalizationArtifactCombiner ArtCombiner(Helper.MIRBuilder, MF.getRegInfo(), LInfo); - auto RemoveDeadInstFromLists = [&WorkListObserver](MachineInstr *DeadMI) { - WorkListObserver.erasingInstr(*DeadMI); + LegalizationArtifactCombiner ArtCombiner(*MIRBuilder.get(), MF.getRegInfo(), + LInfo); + auto RemoveDeadInstFromLists = [&WrapperObserver](MachineInstr *DeadMI) { + WrapperObserver.erasingInstr(*DeadMI); }; bool Changed = false; do { Index: lib/CodeGen/GlobalISel/LegalizerHelper.cpp =================================================================== --- lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -31,16 +31,18 @@ using namespace LegalizeActions; LegalizerHelper::LegalizerHelper(MachineFunction &MF, - GISelChangeObserver &Observer) - : MRI(MF.getRegInfo()), LI(*MF.getSubtarget().getLegalizerInfo()), - Observer(Observer) { + GISelChangeObserver &Observer, + MachineIRBuilder &Builder) + : MIRBuilder(Builder), MRI(MF.getRegInfo()), + LI(*MF.getSubtarget().getLegalizerInfo()), Observer(Observer) { MIRBuilder.setMF(MF); MIRBuilder.setChangeObserver(Observer); } LegalizerHelper::LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI, - GISelChangeObserver &Observer) - : MRI(MF.getRegInfo()), LI(LI), Observer(Observer) { + GISelChangeObserver &Observer, + MachineIRBuilder &B) + : MIRBuilder(B), MRI(MF.getRegInfo()), LI(LI), Observer(Observer) { MIRBuilder.setMF(MF); MIRBuilder.setChangeObserver(Observer); } Index: lib/CodeGen/GlobalISel/MachineIRBuilder.cpp =================================================================== --- lib/CodeGen/GlobalISel/MachineIRBuilder.cpp +++ lib/CodeGen/GlobalISel/MachineIRBuilder.cpp @@ -46,6 +46,8 @@ State.II = MI.getIterator(); } +void MachineIRBuilder::setCSEInfo(GISelCSEInfo *Info) { State.CSEInfo = Info; } + void MachineIRBuilder::setInsertPt(MachineBasicBlock &MBB, MachineBasicBlock::iterator II) { assert(MBB.getParent() == &getMF() && Index: lib/CodeGen/GlobalISel/Utils.cpp =================================================================== --- lib/CodeGen/GlobalISel/Utils.cpp +++ lib/CodeGen/GlobalISel/Utils.cpp @@ -235,6 +235,57 @@ return APF; } +Optional llvm::ConstantFoldBinOp(unsigned Opcode, const unsigned Op1, + const unsigned Op2, + const MachineRegisterInfo &MRI) { + auto MaybeOp1Cst = getConstantVRegVal(Op1, MRI); + auto MaybeOp2Cst = getConstantVRegVal(Op2, MRI); + if (MaybeOp1Cst && MaybeOp2Cst) { + LLT Ty = MRI.getType(Op1); + APInt C1(Ty.getSizeInBits(), *MaybeOp1Cst, true); + APInt C2(Ty.getSizeInBits(), *MaybeOp2Cst, true); + switch (Opcode) { + default: + break; + case TargetOpcode::G_ADD: + return C1 + C2; + case TargetOpcode::G_AND: + return C1 & C2; + case TargetOpcode::G_ASHR: + return C1.ashr(C2); + case TargetOpcode::G_LSHR: + return C1.lshr(C2); + case TargetOpcode::G_MUL: + return C1 * C2; + case TargetOpcode::G_OR: + return C1 | C2; + case TargetOpcode::G_SHL: + return C1 << C2; + case TargetOpcode::G_SUB: + return C1 - C2; + case TargetOpcode::G_XOR: + return C1 ^ C2; + case TargetOpcode::G_UDIV: + if (!C2.getBoolValue()) + break; + return C1.udiv(C2); + case TargetOpcode::G_SDIV: + if (!C2.getBoolValue()) + break; + return C1.sdiv(C2); + case TargetOpcode::G_UREM: + if (!C2.getBoolValue()) + break; + return C1.urem(C2); + case TargetOpcode::G_SREM: + if (!C2.getBoolValue()) + break; + return C1.srem(C2); + } + } + return None; +} + void llvm::getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU) { AU.addPreserved(); } Index: lib/CodeGen/MachineFunction.cpp =================================================================== --- lib/CodeGen/MachineFunction.cpp +++ lib/CodeGen/MachineFunction.cpp @@ -139,12 +139,12 @@ init(); } -void MachineFunction::handleInsertion(const MachineInstr &MI) { +void MachineFunction::handleInsertion(MachineInstr &MI) { if (TheDelegate) TheDelegate->MF_HandleInsertion(MI); } -void MachineFunction::handleRemoval(const MachineInstr &MI) { +void MachineFunction::handleRemoval(MachineInstr &MI) { if (TheDelegate) TheDelegate->MF_HandleRemoval(MI); } Index: lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp =================================================================== --- lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp +++ lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp @@ -88,7 +88,7 @@ auto *TPC = &getAnalysis(); AArch64PreLegalizerCombinerInfo PCInfo; Combiner C(PCInfo, TPC); - return C.combineMachineInstrs(MF); + return C.combineMachineInstrs(MF, /*CSEInfo*/ nullptr); } char AArch64PreLegalizerCombiner::ID = 0; Index: lib/Target/Mips/MipsPreLegalizerCombiner.cpp =================================================================== --- lib/Target/Mips/MipsPreLegalizerCombiner.cpp +++ lib/Target/Mips/MipsPreLegalizerCombiner.cpp @@ -73,7 +73,7 @@ auto *TPC = &getAnalysis(); MipsPreLegalizerCombinerInfo PCInfo; Combiner C(PCInfo, TPC); - return C.combineMachineInstrs(MF); + return C.combineMachineInstrs(MF, nullptr); } char MipsPreLegalizerCombiner::ID = 0; Index: test/CodeGen/AArch64/GlobalISel/call-translator-cse.ll =================================================================== --- /dev/null +++ test/CodeGen/AArch64/GlobalISel/call-translator-cse.ll @@ -0,0 +1,34 @@ +; RUN: llc -mtriple=aarch64-linux-gnu -O1 -stop-after=irtranslator -enable-cse-in-irtranslator=1 -global-isel -verify-machineinstrs %s -o - 2>&1 | FileCheck %s + +; CHECK-LABEL: name: test_split_struct +; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 +; CHECK: [[LO:%[0-9]+]]:_(s64) = G_LOAD %0(p0) :: (load 8 from %ir.ptr) +; CHECK: [[CST:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 +; CHECK: [[GEP:%[0-9]+]]:_(p0) = G_GEP [[ADDR]], [[CST]](s64) +; CHECK: [[HI:%[0-9]+]]:_(s64) = G_LOAD [[GEP]](p0) :: (load 8 from %ir.ptr + 8) + +; CHECK: [[IMPDEF:%[0-9]+]]:_(s128) = G_IMPLICIT_DEF +; CHECK: [[INS1:%[0-9]+]]:_(s128) = G_INSERT [[IMPDEF]], [[LO]](s64), 0 +; CHECK: [[INS2:%[0-9]+]]:_(s128) = G_INSERT [[INS1]], [[HI]](s64), 64 +; CHECK: [[EXTLO:%[0-9]+]]:_(s64) = G_EXTRACT [[INS2]](s128), 0 +; CHECK: [[EXTHI:%[0-9]+]]:_(s64) = G_EXTRACT [[INS2]](s128), 64 + +; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp +; CHECK: [[CST2:%[0-9]+]]:_(s64) = G_CONSTANT i64 0 +; CHECK: [[GEP2:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[CST2]](s64) +; CHECK: G_STORE [[EXTLO]](s64), [[GEP2]](p0) :: (store 8 into stack, align 0) +; CHECK: [[SP:%[0-9]+]]:_(p0) = COPY $sp +; CHECK: [[CST3:%[0-9]+]]:_(s64) = COPY [[CST]] +; CHECK: [[GEP3:%[0-9]+]]:_(p0) = G_GEP [[SP]], [[CST3]](s64) +; CHECK: G_STORE [[EXTHI]](s64), [[GEP3]](p0) :: (store 8 into stack + 8, align 0) +define void @test_split_struct([2 x i64]* %ptr) { + %struct = load [2 x i64], [2 x i64]* %ptr + call void @take_split_struct([2 x i64]* null, i64 1, i64 2, i64 3, + i64 4, i64 5, i64 6, + [2 x i64] %struct) + ret void +} + +declare void @take_split_struct([2 x i64]* %ptr, i64, i64, i64, + i64, i64, i64, + [2 x i64] %in) ; Index: test/CodeGen/AArch64/GlobalISel/gisel-commandline-option.ll =================================================================== --- test/CodeGen/AArch64/GlobalISel/gisel-commandline-option.ll +++ test/CodeGen/AArch64/GlobalISel/gisel-commandline-option.ll @@ -45,6 +45,7 @@ ; VERIFY-NEXT: Verify generated machine code ; ENABLED-NEXT: PreLegalizerCombiner ; VERIFY-NEXT: Verify generated machine code +; ENABLED-NEXT: Analysis containing CSE Info ; ENABLED-NEXT: Legalizer ; VERIFY-NEXT: Verify generated machine code ; ENABLED-NEXT: RegBankSelect Index: test/CodeGen/AArch64/GlobalISel/legalize-ext-cse.mir =================================================================== --- /dev/null +++ test/CodeGen/AArch64/GlobalISel/legalize-ext-cse.mir @@ -0,0 +1,21 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -march=aarch64 -run-pass=legalizer %s -o - -enable-cse-in-legalizer=1 -O1 | FileCheck %s +--- +name: test_cse_in_legalizer +body: | + bb.0.entry: + ; CHECK-LABEL: name: test_cse_in_legalizer + ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY $x0 + ; CHECK: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 255 + ; CHECK: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64) + ; CHECK: [[AND:%[0-9]+]]:_(s32) = G_AND [[TRUNC]], [[C]] + ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY [[AND]](s32) + ; CHECK: $w0 = COPY [[COPY1]](s32) + ; CHECK: $w0 = COPY [[AND]](s32) + %0:_(s64) = COPY $x0 + %1:_(s8) = G_TRUNC %0(s64) + %19:_(s32) = G_ZEXT %1(s8) + $w0 = COPY %19(s32) + %2:_(s8) = G_TRUNC %0(s64) + %20:_(s32) = G_ZEXT %2(s8) + $w0 = COPY %20(s32) Index: test/CodeGen/AArch64/O0-pipeline.ll =================================================================== --- test/CodeGen/AArch64/O0-pipeline.ll +++ test/CodeGen/AArch64/O0-pipeline.ll @@ -32,8 +32,10 @@ ; CHECK-NEXT: Safe Stack instrumentation pass ; CHECK-NEXT: Insert stack protectors ; CHECK-NEXT: Module Verifier +; CHECK-NEXT: Analysis containing CSE Info ; CHECK-NEXT: IRTranslator ; CHECK-NEXT: AArch64PreLegalizerCombiner +; CHECK-NEXT: Analysis containing CSE Info ; CHECK-NEXT: Legalizer ; CHECK-NEXT: RegBankSelect ; CHECK-NEXT: Localizer Index: unittests/CodeGen/GlobalISel/CMakeLists.txt =================================================================== --- unittests/CodeGen/GlobalISel/CMakeLists.txt +++ unittests/CodeGen/GlobalISel/CMakeLists.txt @@ -13,4 +13,5 @@ LegalizerInfoTest.cpp PatternMatchTest.cpp LegalizerHelperTest.cpp + CSETest.cpp ) Index: unittests/CodeGen/GlobalISel/CSETest.cpp =================================================================== --- /dev/null +++ unittests/CodeGen/GlobalISel/CSETest.cpp @@ -0,0 +1,87 @@ +//===- CSETest.cpp -----------------------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "GISelMITest.h" +#include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h" + +namespace { + +TEST_F(GISelMITest, TestCSE) { + if (!TM) + return; + + LLT s16{LLT::scalar(16)}; + LLT s32{LLT::scalar(32)}; + auto MIBInput = B.buildInstr(TargetOpcode::G_TRUNC, {s16}, {Copies[0]}); + auto MIBInput1 = B.buildInstr(TargetOpcode::G_TRUNC, {s16}, {Copies[1]}); + auto MIBAdd = B.buildInstr(TargetOpcode::G_ADD, {s16}, {MIBInput, MIBInput}); + GISelCSEInfo CSEInfo; + CSEInfo.setCSEConfig(make_unique()); + CSEInfo.analyze(*MF); + B.setCSEInfo(&CSEInfo); + CSEMIRBuilder CSEB(B.getState()); + CSEB.setInsertPt(*EntryMBB, EntryMBB->begin()); + unsigned AddReg = MRI->createGenericVirtualRegister(s16); + auto MIBAddCopy = + CSEB.buildInstr(TargetOpcode::G_ADD, {AddReg}, {MIBInput, MIBInput}); + ASSERT_EQ(MIBAddCopy->getOpcode(), TargetOpcode::COPY); + auto MIBAdd2 = + CSEB.buildInstr(TargetOpcode::G_ADD, {s16}, {MIBInput, MIBInput}); + ASSERT_TRUE(&*MIBAdd == &*MIBAdd2); + auto MIBAdd4 = + CSEB.buildInstr(TargetOpcode::G_ADD, {s16}, {MIBInput, MIBInput}); + ASSERT_TRUE(&*MIBAdd == &*MIBAdd4); + auto MIBAdd5 = + CSEB.buildInstr(TargetOpcode::G_ADD, {s16}, {MIBInput, MIBInput1}); + ASSERT_TRUE(&*MIBAdd != &*MIBAdd5); + + // Try building G_CONSTANTS. + auto MIBCst = CSEB.buildConstant(s32, 0); + auto MIBCst1 = CSEB.buildConstant(s32, 0); + ASSERT_TRUE(&*MIBCst == &*MIBCst1); + // Try the CFing of BinaryOps. + auto MIBCF1 = CSEB.buildInstr(TargetOpcode::G_ADD, {s32}, {MIBCst, MIBCst}); + ASSERT_TRUE(&*MIBCF1 == &*MIBCst); + + // Try out building FCONSTANTs. + auto MIBFP0 = CSEB.buildFConstant(s32, 1.0); + auto MIBFP0_1 = CSEB.buildFConstant(s32, 1.0); + ASSERT_TRUE(&*MIBFP0 == &*MIBFP0_1); + CSEInfo.print(); + + // Check G_UNMERGE_VALUES + auto MIBUnmerge = CSEB.buildUnmerge({s32, s32}, Copies[0]); + auto MIBUnmerge2 = CSEB.buildUnmerge({s32, s32}, Copies[0]); + ASSERT_TRUE(&*MIBUnmerge == &*MIBUnmerge2); +} + +TEST_F(GISelMITest, TestCSEConstantConfig) { + if (!TM) + return; + + LLT s16{LLT::scalar(16)}; + auto MIBInput = B.buildInstr(TargetOpcode::G_TRUNC, {s16}, {Copies[0]}); + auto MIBAdd = B.buildInstr(TargetOpcode::G_ADD, {s16}, {MIBInput, MIBInput}); + auto MIBZero = B.buildConstant(s16, 0); + GISelCSEInfo CSEInfo; + CSEInfo.setCSEConfig(make_unique()); + CSEInfo.analyze(*MF); + B.setCSEInfo(&CSEInfo); + CSEMIRBuilder CSEB(B.getState()); + CSEB.setInsertPt(*EntryMBB, EntryMBB->begin()); + auto MIBAdd1 = + CSEB.buildInstr(TargetOpcode::G_ADD, {s16}, {MIBInput, MIBInput}); + // We should CSE constants only. Adds should not be CSEd. + ASSERT_TRUE(MIBAdd1->getOpcode() != TargetOpcode::COPY); + ASSERT_TRUE(&*MIBAdd1 != &*MIBAdd); + // We should CSE constant. + auto MIBZeroTmp = CSEB.buildConstant(s16, 0); + ASSERT_TRUE(&*MIBZero == &*MIBZeroTmp); +} +} // namespace Index: unittests/CodeGen/GlobalISel/GISelMITest.h =================================================================== --- unittests/CodeGen/GlobalISel/GISelMITest.h +++ unittests/CodeGen/GlobalISel/GISelMITest.h @@ -1,4 +1,4 @@ -//===- LegalizerHelperTest.h +//===- GISelMITest.h //-----------------------------------------------===// // // The LLVM Compiler Infrastructure @@ -7,6 +7,8 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// +#ifndef LLVM_UNITTEST_CODEGEN_GLOBALISEL_GISELMI_H +#define LLVM_UNITTEST_CODEGEN_GLOBALISEL_GISELMI_H #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h" @@ -32,7 +34,7 @@ using namespace llvm; using namespace MIPatternMatch; -void initLLVM() { +static inline void initLLVM() { InitializeAllTargets(); InitializeAllTargetMCs(); InitializeAllAsmPrinters(); @@ -45,7 +47,7 @@ /// Create a TargetMachine. As we lack a dedicated always available target for /// unittests, we go for "AArch64". -std::unique_ptr createTargetMachine() { +static std::unique_ptr createTargetMachine() { Triple TargetTriple("aarch64--"); std::string Error; const Target *T = TargetRegistry::lookupTarget("", TargetTriple, Error); @@ -53,15 +55,16 @@ return nullptr; TargetOptions Options; - return std::unique_ptr(static_cast( - T->createTargetMachine("AArch64", "", "", Options, None, None, - CodeGenOpt::Aggressive))); + return std::unique_ptr( + static_cast(T->createTargetMachine( + "AArch64", "", "", Options, None, None, CodeGenOpt::Aggressive))); } -std::unique_ptr parseMIR(LLVMContext &Context, - std::unique_ptr &MIR, - const TargetMachine &TM, StringRef MIRCode, - const char *FuncName, MachineModuleInfo &MMI) { +static std::unique_ptr parseMIR(LLVMContext &Context, + std::unique_ptr &MIR, + const TargetMachine &TM, + StringRef MIRCode, const char *FuncName, + MachineModuleInfo &MMI) { SMDiagnostic Diagnostic; std::unique_ptr MBuffer = MemoryBuffer::getMemBuffer(MIRCode); MIR = createMIRParser(std::move(MBuffer), Context); @@ -80,7 +83,7 @@ return M; } -std::pair, std::unique_ptr> +static std::pair, std::unique_ptr> createDummyModule(LLVMContext &Context, const LLVMTargetMachine &TM, StringRef MIRFunc) { SmallString<512> S; @@ -123,9 +126,9 @@ } } -class LegalizerHelperTest : public ::testing::Test { +class GISelMITest : public ::testing::Test { protected: - LegalizerHelperTest() : ::testing::Test() { + GISelMITest() : ::testing::Test() { TM = createTargetMachine(); if (!TM) return; @@ -168,8 +171,8 @@ } \ }; -static bool CheckMachineFunction(const MachineFunction &MF, - StringRef CheckStr) { +static inline bool CheckMachineFunction(const MachineFunction &MF, + StringRef CheckStr) { SmallString<512> Msg; raw_svector_ostream OS(Msg); MF.print(OS); @@ -190,3 +193,4 @@ SM.AddNewSourceBuffer(std::move(OutputBuf), SMLoc()); return FC.CheckInput(SM, OutBuffer, CheckStrings); } +#endif Index: unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp =================================================================== --- unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp +++ unittests/CodeGen/GlobalISel/LegalizerHelperTest.cpp @@ -1,4 +1,5 @@ -//===- PatternMatchTest.cpp -----------------------------------------------===// +//===- LegalizerHelperTest.cpp +//-----------------------------------------------===// // // The LLVM Compiler Infrastructure // @@ -7,21 +8,21 @@ // //===----------------------------------------------------------------------===// -#include "LegalizerHelperTest.h" +#include "GISelMITest.h" namespace { class DummyGISelObserver : public GISelChangeObserver { public: - void changingInstr(const MachineInstr &MI) override {} - void changedInstr(const MachineInstr &MI) override {} - void createdInstr(const MachineInstr &MI) override {} - void erasingInstr(const MachineInstr &MI) override {} + void changingInstr(MachineInstr &MI) override {} + void changedInstr(MachineInstr &MI) override {} + void createdInstr(MachineInstr &MI) override {} + void erasingInstr(MachineInstr &MI) override {} }; // Test CTTZ expansion when CTTZ_ZERO_UNDEF is legal or custom, // in which case it becomes CTTZ_ZERO_UNDEF with select. -TEST_F(LegalizerHelperTest, LowerBitCountingCTTZ0) { +TEST_F(GISelMITest, LowerBitCountingCTTZ0) { if (!TM) return; @@ -33,7 +34,7 @@ B.buildInstr(TargetOpcode::G_CTTZ, {LLT::scalar(64)}, {Copies[0]}); AInfo Info(MF->getSubtarget()); DummyGISelObserver Observer; - LegalizerHelper Helper(*MF, Info, Observer); + LegalizerHelper Helper(*MF, Info, Observer, B); // Perform Legalization ASSERT_TRUE(Helper.lower(*MIBCTTZ, 0, LLT::scalar(64)) == LegalizerHelper::LegalizeResult::Legalized); @@ -51,7 +52,7 @@ } // CTTZ expansion in terms of CTLZ -TEST_F(LegalizerHelperTest, LowerBitCountingCTTZ1) { +TEST_F(GISelMITest, LowerBitCountingCTTZ1) { if (!TM) return; @@ -63,7 +64,7 @@ B.buildInstr(TargetOpcode::G_CTTZ, {LLT::scalar(64)}, {Copies[0]}); AInfo Info(MF->getSubtarget()); DummyGISelObserver Observer; - LegalizerHelper Helper(*MF, Info, Observer); + LegalizerHelper Helper(*MF, Info, Observer, B); // Perform Legalization ASSERT_TRUE(Helper.lower(*MIBCTTZ, 0, LLT::scalar(64)) == LegalizerHelper::LegalizeResult::Legalized); @@ -83,7 +84,7 @@ } // CTTZ expansion in terms of CTPOP -TEST_F(LegalizerHelperTest, LowerBitCountingCTTZ2) { +TEST_F(GISelMITest, LowerBitCountingCTTZ2) { if (!TM) return; @@ -95,7 +96,7 @@ B.buildInstr(TargetOpcode::G_CTTZ, {LLT::scalar(64)}, {Copies[0]}); AInfo Info(MF->getSubtarget()); DummyGISelObserver Observer; - LegalizerHelper Helper(*MF, Info, Observer); + LegalizerHelper Helper(*MF, Info, Observer, B); ASSERT_TRUE(Helper.lower(*MIBCTTZ, 0, LLT::scalar(64)) == LegalizerHelper::LegalizeResult::Legalized); @@ -112,7 +113,7 @@ } // CTTZ_ZERO_UNDEF expansion in terms of CTTZ -TEST_F(LegalizerHelperTest, LowerBitCountingCTTZ3) { +TEST_F(GISelMITest, LowerBitCountingCTTZ3) { if (!TM) return; @@ -124,7 +125,7 @@ {LLT::scalar(64)}, {Copies[0]}); AInfo Info(MF->getSubtarget()); DummyGISelObserver Observer; - LegalizerHelper Helper(*MF, Info, Observer); + LegalizerHelper Helper(*MF, Info, Observer, B); ASSERT_TRUE(Helper.lower(*MIBCTTZ, 0, LLT::scalar(64)) == LegalizerHelper::LegalizeResult::Legalized); @@ -137,7 +138,7 @@ } // CTLZ expansion in terms of CTLZ_ZERO_UNDEF -TEST_F(LegalizerHelperTest, LowerBitCountingCTLZ0) { +TEST_F(GISelMITest, LowerBitCountingCTLZ0) { if (!TM) return; @@ -149,7 +150,7 @@ B.buildInstr(TargetOpcode::G_CTLZ, {LLT::scalar(64)}, {Copies[0]}); AInfo Info(MF->getSubtarget()); DummyGISelObserver Observer; - LegalizerHelper Helper(*MF, Info, Observer); + LegalizerHelper Helper(*MF, Info, Observer, B); ASSERT_TRUE(Helper.lower(*MIBCTLZ, 0, LLT::scalar(64)) == LegalizerHelper::LegalizeResult::Legalized); @@ -166,7 +167,7 @@ } // CTLZ expansion in terms of CTLZ_ZERO_UNDEF if the latter is a libcall -TEST_F(LegalizerHelperTest, LowerBitCountingCTLZLibcall) { +TEST_F(GISelMITest, LowerBitCountingCTLZLibcall) { if (!TM) return; @@ -178,7 +179,7 @@ B.buildInstr(TargetOpcode::G_CTLZ, {LLT::scalar(64)}, {Copies[0]}); AInfo Info(MF->getSubtarget()); DummyGISelObserver Observer; - LegalizerHelper Helper(*MF, Info, Observer); + LegalizerHelper Helper(*MF, Info, Observer, B); ASSERT_TRUE(Helper.lower(*MIBCTLZ, 0, LLT::scalar(64)) == LegalizerHelper::LegalizeResult::Legalized); @@ -195,7 +196,7 @@ } // CTLZ expansion -TEST_F(LegalizerHelperTest, LowerBitCountingCTLZ1) { +TEST_F(GISelMITest, LowerBitCountingCTLZ1) { if (!TM) return; @@ -209,7 +210,7 @@ auto MIBCTLZ = B.buildInstr(TargetOpcode::G_CTLZ, {s8}, {MIBTrunc}); AInfo Info(MF->getSubtarget()); DummyGISelObserver Observer; - LegalizerHelper Helper(*MF, Info, Observer); + LegalizerHelper Helper(*MF, Info, Observer, B); ASSERT_TRUE(Helper.lower(*MIBCTLZ, 0, s8) == LegalizerHelper::LegalizeResult::Legalized); @@ -234,7 +235,7 @@ } // CTLZ widening. -TEST_F(LegalizerHelperTest, WidenBitCountingCTLZ) { +TEST_F(GISelMITest, WidenBitCountingCTLZ) { if (!TM) return; @@ -249,7 +250,7 @@ auto MIBCTLZ = B.buildInstr(TargetOpcode::G_CTLZ, {s8}, {MIBTrunc}); AInfo Info(MF->getSubtarget()); DummyGISelObserver Observer; - LegalizerHelper Helper(*MF, Info, Observer); + LegalizerHelper Helper(*MF, Info, Observer, B); ASSERT_TRUE(Helper.widenScalar(*MIBCTLZ, 0, s16) == LegalizerHelper::LegalizeResult::Legalized); @@ -267,7 +268,7 @@ } // CTLZ_ZERO_UNDEF widening. -TEST_F(LegalizerHelperTest, WidenBitCountingCTLZZeroUndef) { +TEST_F(GISelMITest, WidenBitCountingCTLZZeroUndef) { if (!TM) return; @@ -283,7 +284,7 @@ B.buildInstr(TargetOpcode::G_CTLZ_ZERO_UNDEF, {s8}, {MIBTrunc}); AInfo Info(MF->getSubtarget()); DummyGISelObserver Observer; - LegalizerHelper Helper(*MF, Info, Observer); + LegalizerHelper Helper(*MF, Info, Observer, B); ASSERT_TRUE(Helper.widenScalar(*MIBCTLZ_ZU, 0, s16) == LegalizerHelper::LegalizeResult::Legalized); @@ -301,7 +302,7 @@ } // CTPOP widening. -TEST_F(LegalizerHelperTest, WidenBitCountingCTPOP) { +TEST_F(GISelMITest, WidenBitCountingCTPOP) { if (!TM) return; @@ -316,7 +317,7 @@ auto MIBCTPOP = B.buildInstr(TargetOpcode::G_CTPOP, {s8}, {MIBTrunc}); AInfo Info(MF->getSubtarget()); DummyGISelObserver Observer; - LegalizerHelper Helper(*MF, Info, Observer); + LegalizerHelper Helper(*MF, Info, Observer, B); ASSERT_TRUE(Helper.widenScalar(*MIBCTPOP, 0, s16) == LegalizerHelper::LegalizeResult::Legalized); @@ -332,7 +333,7 @@ } // CTTZ_ZERO_UNDEF widening. -TEST_F(LegalizerHelperTest, WidenBitCountingCTTZ_ZERO_UNDEF) { +TEST_F(GISelMITest, WidenBitCountingCTTZ_ZERO_UNDEF) { if (!TM) return; @@ -348,7 +349,7 @@ B.buildInstr(TargetOpcode::G_CTTZ_ZERO_UNDEF, {s8}, {MIBTrunc}); AInfo Info(MF->getSubtarget()); DummyGISelObserver Observer; - LegalizerHelper Helper(*MF, Info, Observer); + LegalizerHelper Helper(*MF, Info, Observer, B); ASSERT_TRUE(Helper.widenScalar(*MIBCTTZ_ZERO_UNDEF, 0, s16) == LegalizerHelper::LegalizeResult::Legalized); @@ -364,7 +365,7 @@ } // CTTZ widening. -TEST_F(LegalizerHelperTest, WidenBitCountingCTTZ) { +TEST_F(GISelMITest, WidenBitCountingCTTZ) { if (!TM) return; @@ -379,7 +380,7 @@ auto MIBCTTZ = B.buildInstr(TargetOpcode::G_CTTZ, {s8}, {MIBTrunc}); AInfo Info(MF->getSubtarget()); DummyGISelObserver Observer; - LegalizerHelper Helper(*MF, Info, Observer); + LegalizerHelper Helper(*MF, Info, Observer, B); ASSERT_TRUE(Helper.widenScalar(*MIBCTTZ, 0, s16) == LegalizerHelper::LegalizeResult::Legalized); @@ -396,7 +397,7 @@ ASSERT_TRUE(CheckMachineFunction(*MF, CheckStr)); } // UADDO widening. -TEST_F(LegalizerHelperTest, WidenUADDO) { +TEST_F(GISelMITest, WidenUADDO) { if (!TM) return; @@ -413,7 +414,7 @@ B.buildInstr(TargetOpcode::G_UADDO, {s8, CarryReg}, {MIBTrunc, MIBTrunc}); AInfo Info(MF->getSubtarget()); DummyGISelObserver Observer; - LegalizerHelper Helper(*MF, Info, Observer); + LegalizerHelper Helper(*MF, Info, Observer, B); ASSERT_TRUE(Helper.widenScalar(*MIBUAddO, 0, s16) == LegalizerHelper::LegalizeResult::Legalized); @@ -433,7 +434,7 @@ } // USUBO widening. -TEST_F(LegalizerHelperTest, WidenUSUBO) { +TEST_F(GISelMITest, WidenUSUBO) { if (!TM) return; @@ -450,7 +451,7 @@ B.buildInstr(TargetOpcode::G_USUBO, {s8, CarryReg}, {MIBTrunc, MIBTrunc}); AInfo Info(MF->getSubtarget()); DummyGISelObserver Observer; - LegalizerHelper Helper(*MF, Info, Observer); + LegalizerHelper Helper(*MF, Info, Observer, B); ASSERT_TRUE(Helper.widenScalar(*MIBUSUBO, 0, s16) == LegalizerHelper::LegalizeResult::Legalized);