Index: lib/Target/AArch64/AArch64.h =================================================================== --- lib/Target/AArch64/AArch64.h +++ lib/Target/AArch64/AArch64.h @@ -38,7 +38,7 @@ ModulePass *createAArch64PromoteConstantPass(); FunctionPass *createAArch64ConditionOptimizerPass(); FunctionPass *createAArch64AddressTypePromotionPass(); -FunctionPass *createAArch64A57FPLoadBalancing(); +FunctionPass *createAArch64FPLoadBalancing(); FunctionPass *createAArch64A53Fix835769(); /// \brief Creates an ARM-specific Target Transformation Info pass. ImmutablePass * Index: lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp =================================================================== --- lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp +++ /dev/null @@ -1,700 +0,0 @@ -//===-- AArch64A57FPLoadBalancing.cpp - Balance FP ops statically on A57---===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// For best-case performance on Cortex-A57, we should try to use a balanced -// mix of odd and even D-registers when performing a critical sequence of -// independent, non-quadword FP/ASIMD floating-point multiply or -// multiply-accumulate operations. -// -// This pass attempts to detect situations where the register allocation may -// adversely affect this load balancing and to change the registers used so as -// to better utilize the CPU. -// -// Ideally we'd just take each multiply or multiply-accumulate in turn and -// allocate it alternating even or odd registers. However, multiply-accumulates -// are most efficiently performed in the same functional unit as their -// accumulation operand. Therefore this pass tries to find maximal sequences -// ("Chains") of multiply-accumulates linked via their accumulation operand, -// and assign them all the same "color" (oddness/evenness). -// -// This optimization affects S-register and D-register floating point -// multiplies and FMADD/FMAs, as well as vector (floating point only) muls and -// FMADD/FMA. Q register instructions (and 128-bit vector instructions) are -// not affected. -//===----------------------------------------------------------------------===// - -#include "AArch64.h" -#include "AArch64InstrInfo.h" -#include "AArch64Subtarget.h" -#include "llvm/ADT/BitVector.h" -#include "llvm/ADT/EquivalenceClasses.h" -#include "llvm/CodeGen/MachineFunction.h" -#include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/MachineInstr.h" -#include "llvm/CodeGen/MachineInstrBuilder.h" -#include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/RegisterScavenging.h" -#include "llvm/CodeGen/RegisterClassInfo.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/raw_ostream.h" -#include -using namespace llvm; - -#define DEBUG_TYPE "aarch64-a57-fp-load-balancing" - -// Enforce the algorithm to use the scavenged register even when the original -// destination register is the correct color. Used for testing. -static cl::opt -TransformAll("aarch64-a57-fp-load-balancing-force-all", - cl::desc("Always modify dest registers regardless of color"), - cl::init(false), cl::Hidden); - -// Never use the balance information obtained from chains - return a specific -// color always. Used for testing. -static cl::opt -OverrideBalance("aarch64-a57-fp-load-balancing-override", - cl::desc("Ignore balance information, always return " - "(1: Even, 2: Odd)."), - cl::init(0), cl::Hidden); - -//===----------------------------------------------------------------------===// -// Helper functions - -// Is the instruction a type of multiply on 64-bit (or 32-bit) FPRs? -static bool isMul(MachineInstr *MI) { - switch (MI->getOpcode()) { - case AArch64::FMULSrr: - case AArch64::FNMULSrr: - case AArch64::FMULDrr: - case AArch64::FNMULDrr: - return true; - default: - return false; - } -} - -// Is the instruction a type of FP multiply-accumulate on 64-bit (or 32-bit) FPRs? -static bool isMla(MachineInstr *MI) { - switch (MI->getOpcode()) { - case AArch64::FMSUBSrrr: - case AArch64::FMADDSrrr: - case AArch64::FNMSUBSrrr: - case AArch64::FNMADDSrrr: - case AArch64::FMSUBDrrr: - case AArch64::FMADDDrrr: - case AArch64::FNMSUBDrrr: - case AArch64::FNMADDDrrr: - return true; - default: - return false; - } -} - -//===----------------------------------------------------------------------===// - -namespace { -/// A "color", which is either even or odd. Yes, these aren't really colors -/// but the algorithm is conceptually doing two-color graph coloring. -enum class Color { Even, Odd }; -#ifndef NDEBUG -static const char *ColorNames[2] = { "Even", "Odd" }; -#endif - -class Chain; - -class AArch64A57FPLoadBalancing : public MachineFunctionPass { - const AArch64InstrInfo *TII; - MachineRegisterInfo *MRI; - const TargetRegisterInfo *TRI; - RegisterClassInfo RCI; - -public: - static char ID; - explicit AArch64A57FPLoadBalancing() : MachineFunctionPass(ID) {} - - bool runOnMachineFunction(MachineFunction &F) override; - - const char *getPassName() const override { - return "A57 FP Anti-dependency breaker"; - } - - void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.setPreservesCFG(); - MachineFunctionPass::getAnalysisUsage(AU); - } - -private: - bool runOnBasicBlock(MachineBasicBlock &MBB); - bool colorChainSet(std::vector GV, MachineBasicBlock &MBB, - int &Balance); - bool colorChain(Chain *G, Color C, MachineBasicBlock &MBB); - int scavengeRegister(Chain *G, Color C, MachineBasicBlock &MBB); - void scanInstruction(MachineInstr *MI, unsigned Idx, - std::map &Active, - std::set> &AllChains); - void maybeKillChain(MachineOperand &MO, unsigned Idx, - std::map &RegChains); - Color getColor(unsigned Register); - Chain *getAndEraseNext(Color PreferredColor, std::vector &L); -}; -char AArch64A57FPLoadBalancing::ID = 0; - -/// A Chain is a sequence of instructions that are linked together by -/// an accumulation operand. For example: -/// -/// fmul d0, ? -/// fmla d1, ?, ?, d0 -/// fmla d2, ?, ?, d1 -/// -/// There may be other instructions interleaved in the sequence that -/// do not belong to the chain. These other instructions must not use -/// the "chain" register at any point. -/// -/// We currently only support chains where the "chain" operand is killed -/// at each link in the chain for simplicity. -/// A chain has three important instructions - Start, Last and Kill. -/// * The start instruction is the first instruction in the chain. -/// * Last is the final instruction in the chain. -/// * Kill may or may not be defined. If defined, Kill is the instruction -/// where the outgoing value of the Last instruction is killed. -/// This information is important as if we know the outgoing value is -/// killed with no intervening uses, we can safely change its register. -/// -/// Without a kill instruction, we must assume the outgoing value escapes -/// beyond our model and either must not change its register or must -/// create a fixup FMOV to keep the old register value consistent. -/// -class Chain { -public: - /// The important (marker) instructions. - MachineInstr *StartInst, *LastInst, *KillInst; - /// The index, from the start of the basic block, that each marker - /// appears. These are stored so we can do quick interval tests. - unsigned StartInstIdx, LastInstIdx, KillInstIdx; - /// All instructions in the chain. - std::set Insts; - /// True if KillInst cannot be modified. If this is true, - /// we cannot change LastInst's outgoing register. - /// This will be true for tied values and regmasks. - bool KillIsImmutable; - /// The "color" of LastInst. This will be the preferred chain color, - /// as changing intermediate nodes is easy but changing the last - /// instruction can be more tricky. - Color LastColor; - - Chain(MachineInstr *MI, unsigned Idx, Color C) - : StartInst(MI), LastInst(MI), KillInst(nullptr), - StartInstIdx(Idx), LastInstIdx(Idx), KillInstIdx(0), - LastColor(C) { - Insts.insert(MI); - } - - /// Add a new instruction into the chain. The instruction's dest operand - /// has the given color. - void add(MachineInstr *MI, unsigned Idx, Color C) { - LastInst = MI; - LastInstIdx = Idx; - LastColor = C; - assert((KillInstIdx == 0 || LastInstIdx < KillInstIdx) && - "Chain: broken invariant. A Chain can only be killed after its last " - "def"); - - Insts.insert(MI); - } - - /// Return true if MI is a member of the chain. - bool contains(MachineInstr *MI) { return Insts.count(MI) > 0; } - - /// Return the number of instructions in the chain. - unsigned size() const { - return Insts.size(); - } - - /// Inform the chain that its last active register (the dest register of - /// LastInst) is killed by MI with no intervening uses or defs. - void setKill(MachineInstr *MI, unsigned Idx, bool Immutable) { - KillInst = MI; - KillInstIdx = Idx; - KillIsImmutable = Immutable; - assert((KillInstIdx == 0 || LastInstIdx < KillInstIdx) && - "Chain: broken invariant. A Chain can only be killed after its last " - "def"); - } - - /// Return the first instruction in the chain. - MachineInstr *getStart() const { return StartInst; } - /// Return the last instruction in the chain. - MachineInstr *getLast() const { return LastInst; } - /// Return the "kill" instruction (as set with setKill()) or NULL. - MachineInstr *getKill() const { return KillInst; } - /// Return an instruction that can be used as an iterator for the end - /// of the chain. This is the maximum of KillInst (if set) and LastInst. - MachineBasicBlock::iterator getEnd() const { - return ++MachineBasicBlock::iterator(KillInst ? KillInst : LastInst); - } - - /// Can the Kill instruction (assuming one exists) be modified? - bool isKillImmutable() const { return KillIsImmutable; } - - /// Return the preferred color of this chain. - Color getPreferredColor() { - if (OverrideBalance != 0) - return OverrideBalance == 1 ? Color::Even : Color::Odd; - return LastColor; - } - - /// Return true if this chain (StartInst..KillInst) overlaps with Other. - bool rangeOverlapsWith(const Chain &Other) const { - unsigned End = KillInst ? KillInstIdx : LastInstIdx; - unsigned OtherEnd = Other.KillInst ? - Other.KillInstIdx : Other.LastInstIdx; - - return StartInstIdx <= OtherEnd && Other.StartInstIdx <= End; - } - - /// Return true if this chain starts before Other. - bool startsBefore(Chain *Other) { - return StartInstIdx < Other->StartInstIdx; - } - - /// Return true if the group will require a fixup MOV at the end. - bool requiresFixup() const { - return (getKill() && isKillImmutable()) || !getKill(); - } - - /// Return a simple string representation of the chain. - std::string str() const { - std::string S; - raw_string_ostream OS(S); - - OS << "{"; - StartInst->print(OS, NULL, true); - OS << " -> "; - LastInst->print(OS, NULL, true); - if (KillInst) { - OS << " (kill @ "; - KillInst->print(OS, NULL, true); - OS << ")"; - } - OS << "}"; - - return OS.str(); - } - -}; - -} // end anonymous namespace - -//===----------------------------------------------------------------------===// - -bool AArch64A57FPLoadBalancing::runOnMachineFunction(MachineFunction &F) { - bool Changed = false; - DEBUG(dbgs() << "***** AArch64A57FPLoadBalancing *****\n"); - - const TargetMachine &TM = F.getTarget(); - MRI = &F.getRegInfo(); - TRI = F.getRegInfo().getTargetRegisterInfo(); - TII = TM.getSubtarget().getInstrInfo(); - RCI.runOnMachineFunction(F); - - for (auto &MBB : F) { - Changed |= runOnBasicBlock(MBB); - } - - return Changed; -} - -bool AArch64A57FPLoadBalancing::runOnBasicBlock(MachineBasicBlock &MBB) { - bool Changed = false; - DEBUG(dbgs() << "Running on MBB: " << MBB << " - scanning instructions...\n"); - - // First, scan the basic block producing a set of chains. - - // The currently "active" chains - chains that can be added to and haven't - // been killed yet. This is keyed by register - all chains can only have one - // "link" register between each inst in the chain. - std::map ActiveChains; - std::set> AllChains; - unsigned Idx = 0; - for (auto &MI : MBB) - scanInstruction(&MI, Idx++, ActiveChains, AllChains); - - DEBUG(dbgs() << "Scan complete, "<< AllChains.size() << " chains created.\n"); - - // Group the chains into disjoint sets based on their liveness range. This is - // a poor-man's version of graph coloring. Ideally we'd create an interference - // graph and perform full-on graph coloring on that, but; - // (a) That's rather heavyweight for only two colors. - // (b) We expect multiple disjoint interference regions - in practice the live - // range of chains is quite small and they are clustered between loads - // and stores. - EquivalenceClasses EC; - for (auto &I : AllChains) - EC.insert(I.get()); - - for (auto &I : AllChains) - for (auto &J : AllChains) - if (I != J && I->rangeOverlapsWith(*J)) - EC.unionSets(I.get(), J.get()); - DEBUG(dbgs() << "Created " << EC.getNumClasses() << " disjoint sets.\n"); - - // Now we assume that every member of an equivalence class interferes - // with every other member of that class, and with no members of other classes. - - // Convert the EquivalenceClasses to a simpler set of sets. - std::vector > V; - for (auto I = EC.begin(), E = EC.end(); I != E; ++I) { - std::vector Cs(EC.member_begin(I), EC.member_end()); - if (Cs.empty()) continue; - V.push_back(std::move(Cs)); - } - - // Now we have a set of sets, order them by start address so - // we can iterate over them sequentially. - std::sort(V.begin(), V.end(), - [](const std::vector &A, - const std::vector &B) { - return A.front()->startsBefore(B.front()); - }); - - // As we only have two colors, we can track the global (BB-level) balance of - // odds versus evens. We aim to keep this near zero to keep both execution - // units fed. - // Positive means we're even-heavy, negative we're odd-heavy. - // - // FIXME: If chains have interdependencies, for example: - // mul r0, r1, r2 - // mul r3, r0, r1 - // We do not model this and may color each one differently, assuming we'll - // get ILP when we obviously can't. This hasn't been seen to be a problem - // in practice so far, so we simplify the algorithm by ignoring it. - int Parity = 0; - - for (auto &I : V) - Changed |= colorChainSet(std::move(I), MBB, Parity); - - return Changed; -} - -Chain *AArch64A57FPLoadBalancing::getAndEraseNext(Color PreferredColor, - std::vector &L) { - if (L.empty()) - return nullptr; - - // We try and get the best candidate from L to color next, given that our - // preferred color is "PreferredColor". L is ordered from larger to smaller - // chains. It is beneficial to color the large chains before the small chains, - // but if we can't find a chain of the maximum length with the preferred color, - // we fuzz the size and look for slightly smaller chains before giving up and - // returning a chain that must be recolored. - - // FIXME: Does this need to be configurable? - const unsigned SizeFuzz = 1; - unsigned MinSize = L.front()->size() - SizeFuzz; - for (auto I = L.begin(), E = L.end(); I != E; ++I) { - if ((*I)->size() <= MinSize) { - // We've gone past the size limit. Return the previous item. - Chain *Ch = *--I; - L.erase(I); - return Ch; - } - - if ((*I)->getPreferredColor() == PreferredColor) { - Chain *Ch = *I; - L.erase(I); - return Ch; - } - } - - // Bailout case - just return the first item. - Chain *Ch = L.front(); - L.erase(L.begin()); - return Ch; -} - -bool AArch64A57FPLoadBalancing::colorChainSet(std::vector GV, - MachineBasicBlock &MBB, - int &Parity) { - bool Changed = false; - DEBUG(dbgs() << "colorChainSet(): #sets=" << GV.size() << "\n"); - - // Sort by descending size order so that we allocate the most important - // sets first. - // Tie-break equivalent sizes by sorting chains requiring fixups before - // those without fixups. The logic here is that we should look at the - // chains that we cannot change before we look at those we can, - // so the parity counter is updated and we know what color we should - // change them to! - std::sort(GV.begin(), GV.end(), [](const Chain *G1, const Chain *G2) { - if (G1->size() != G2->size()) - return G1->size() > G2->size(); - return G1->requiresFixup() > G2->requiresFixup(); - }); - - Color PreferredColor = Parity < 0 ? Color::Even : Color::Odd; - while (Chain *G = getAndEraseNext(PreferredColor, GV)) { - // Start off by assuming we'll color to our own preferred color. - Color C = PreferredColor; - if (Parity == 0) - // But if we really don't care, use the chain's preferred color. - C = G->getPreferredColor(); - - DEBUG(dbgs() << " - Parity=" << Parity << ", Color=" - << ColorNames[(int)C] << "\n"); - - // If we'll need a fixup FMOV, don't bother. Testing has shown that this - // happens infrequently and when it does it has at least a 50% chance of - // slowing code down instead of speeding it up. - if (G->requiresFixup() && C != G->getPreferredColor()) { - C = G->getPreferredColor(); - DEBUG(dbgs() << " - " << G->str() << " - not worthwhile changing; " - "color remains " << ColorNames[(int)C] << "\n"); - } - - Changed |= colorChain(G, C, MBB); - - Parity += (C == Color::Even) ? G->size() : -G->size(); - PreferredColor = Parity < 0 ? Color::Even : Color::Odd; - } - - return Changed; -} - -int AArch64A57FPLoadBalancing::scavengeRegister(Chain *G, Color C, - MachineBasicBlock &MBB) { - RegScavenger RS; - RS.enterBasicBlock(&MBB); - RS.forward(MachineBasicBlock::iterator(G->getStart())); - - // Can we find an appropriate register that is available throughout the life - // of the chain? - unsigned RegClassID = G->getStart()->getDesc().OpInfo[0].RegClass; - BitVector AvailableRegs = RS.getRegsAvailable(TRI->getRegClass(RegClassID)); - for (MachineBasicBlock::iterator I = G->getStart(), E = G->getEnd(); - I != E; ++I) { - RS.forward(I); - AvailableRegs &= RS.getRegsAvailable(TRI->getRegClass(RegClassID)); - - // Remove any registers clobbered by a regmask. - for (auto J : I->operands()) { - if (J.isRegMask()) - AvailableRegs.clearBitsNotInMask(J.getRegMask()); - } - } - - // Make sure we allocate in-order, to get the cheapest registers first. - auto Ord = RCI.getOrder(TRI->getRegClass(RegClassID)); - for (auto Reg : Ord) { - if (!AvailableRegs[Reg]) - continue; - if ((C == Color::Even && (Reg % 2) == 0) || - (C == Color::Odd && (Reg % 2) == 1)) - return Reg; - } - - return -1; -} - -bool AArch64A57FPLoadBalancing::colorChain(Chain *G, Color C, - MachineBasicBlock &MBB) { - bool Changed = false; - DEBUG(dbgs() << " - colorChain(" << G->str() << ", " - << ColorNames[(int)C] << ")\n"); - - // Try and obtain a free register of the right class. Without a register - // to play with we cannot continue. - int Reg = scavengeRegister(G, C, MBB); - if (Reg == -1) { - DEBUG(dbgs() << "Scavenging (thus coloring) failed!\n"); - return false; - } - DEBUG(dbgs() << " - Scavenged register: " << TRI->getName(Reg) << "\n"); - - std::map Substs; - for (MachineBasicBlock::iterator I = G->getStart(), E = G->getEnd(); - I != E; ++I) { - if (!G->contains(I) && - (&*I != G->getKill() || G->isKillImmutable())) - continue; - - // I is a member of G, or I is a mutable instruction that kills G. - - std::vector ToErase; - for (auto &U : I->operands()) { - if (U.isReg() && U.isUse() && Substs.find(U.getReg()) != Substs.end()) { - unsigned OrigReg = U.getReg(); - U.setReg(Substs[OrigReg]); - if (U.isKill()) - // Don't erase straight away, because there may be other operands - // that also reference this substitution! - ToErase.push_back(OrigReg); - } else if (U.isRegMask()) { - for (auto J : Substs) { - if (U.clobbersPhysReg(J.first)) - ToErase.push_back(J.first); - } - } - } - // Now it's safe to remove the substs identified earlier. - for (auto J : ToErase) - Substs.erase(J); - - // Only change the def if this isn't the last instruction. - if (&*I != G->getKill()) { - MachineOperand &MO = I->getOperand(0); - - bool Change = TransformAll || getColor(MO.getReg()) != C; - if (G->requiresFixup() && &*I == G->getLast()) - Change = false; - - if (Change) { - Substs[MO.getReg()] = Reg; - MO.setReg(Reg); - MRI->setPhysRegUsed(Reg); - - Changed = true; - } - } - } - assert(Substs.size() == 0 && "No substitutions should be left active!"); - - if (G->getKill()) { - DEBUG(dbgs() << " - Kill instruction seen.\n"); - } else { - // We didn't have a kill instruction, but we didn't seem to need to change - // the destination register anyway. - DEBUG(dbgs() << " - Destination register not changed.\n"); - } - return Changed; -} - -void AArch64A57FPLoadBalancing:: -scanInstruction(MachineInstr *MI, unsigned Idx, - std::map &ActiveChains, - std::set> &AllChains) { - // Inspect "MI", updating ActiveChains and AllChains. - - if (isMul(MI)) { - - for (auto &I : MI->uses()) - maybeKillChain(I, Idx, ActiveChains); - for (auto &I : MI->defs()) - maybeKillChain(I, Idx, ActiveChains); - - // Create a new chain. Multiplies don't require forwarding so can go on any - // unit. - unsigned DestReg = MI->getOperand(0).getReg(); - - DEBUG(dbgs() << "New chain started for register " - << TRI->getName(DestReg) << " at " << *MI); - - auto G = llvm::make_unique(MI, Idx, getColor(DestReg)); - ActiveChains[DestReg] = G.get(); - AllChains.insert(std::move(G)); - - } else if (isMla(MI)) { - - // It is beneficial to keep MLAs on the same functional unit as their - // accumulator operand. - unsigned DestReg = MI->getOperand(0).getReg(); - unsigned AccumReg = MI->getOperand(3).getReg(); - - maybeKillChain(MI->getOperand(1), Idx, ActiveChains); - maybeKillChain(MI->getOperand(2), Idx, ActiveChains); - if (DestReg != AccumReg) - maybeKillChain(MI->getOperand(0), Idx, ActiveChains); - - if (ActiveChains.find(AccumReg) != ActiveChains.end()) { - DEBUG(dbgs() << "Chain found for accumulator register " - << TRI->getName(AccumReg) << " in MI " << *MI); - - // For simplicity we only chain together sequences of MULs/MLAs where the - // accumulator register is killed on each instruction. This means we don't - // need to track other uses of the registers we want to rewrite. - // - // FIXME: We could extend to handle the non-kill cases for more coverage. - if (MI->getOperand(3).isKill()) { - // Add to chain. - DEBUG(dbgs() << "Instruction was successfully added to chain.\n"); - ActiveChains[AccumReg]->add(MI, Idx, getColor(DestReg)); - // Handle cases where the destination is not the same as the accumulator. - if (DestReg != AccumReg) { - ActiveChains[DestReg] = ActiveChains[AccumReg]; - ActiveChains.erase(AccumReg); - } - return; - } - - DEBUG(dbgs() << "Cannot add to chain because accumulator operand wasn't " - << "marked !\n"); - maybeKillChain(MI->getOperand(3), Idx, ActiveChains); - } - - DEBUG(dbgs() << "Creating new chain for dest register " - << TRI->getName(DestReg) << "\n"); - auto G = llvm::make_unique(MI, Idx, getColor(DestReg)); - ActiveChains[DestReg] = G.get(); - AllChains.insert(std::move(G)); - - } else { - - // Non-MUL or MLA instruction. Invalidate any chain in the uses or defs - // lists. - for (auto &I : MI->uses()) - maybeKillChain(I, Idx, ActiveChains); - for (auto &I : MI->defs()) - maybeKillChain(I, Idx, ActiveChains); - - } -} - -void AArch64A57FPLoadBalancing:: -maybeKillChain(MachineOperand &MO, unsigned Idx, - std::map &ActiveChains) { - // Given an operand and the set of active chains (keyed by register), - // determine if a chain should be ended and remove from ActiveChains. - MachineInstr *MI = MO.getParent(); - - if (MO.isReg()) { - - // If this is a KILL of a current chain, record it. - if (MO.isKill() && ActiveChains.find(MO.getReg()) != ActiveChains.end()) { - DEBUG(dbgs() << "Kill seen for chain " << TRI->getName(MO.getReg()) - << "\n"); - ActiveChains[MO.getReg()]->setKill(MI, Idx, /*Immutable=*/MO.isTied()); - } - ActiveChains.erase(MO.getReg()); - - } else if (MO.isRegMask()) { - - for (auto I = ActiveChains.begin(), E = ActiveChains.end(); - I != E;) { - if (MO.clobbersPhysReg(I->first)) { - DEBUG(dbgs() << "Kill (regmask) seen for chain " - << TRI->getName(I->first) << "\n"); - I->second->setKill(MI, Idx, /*Immutable=*/true); - ActiveChains.erase(I++); - } else - ++I; - } - - } -} - -Color AArch64A57FPLoadBalancing::getColor(unsigned Reg) { - if ((TRI->getEncodingValue(Reg) % 2) == 0) - return Color::Even; - else - return Color::Odd; -} - -// Factory function used by AArch64TargetMachine to add the pass to the passmanager. -FunctionPass *llvm::createAArch64A57FPLoadBalancing() { - return new AArch64A57FPLoadBalancing(); -} Index: lib/Target/AArch64/AArch64FPLoadBalancing.cpp =================================================================== --- /dev/null +++ lib/Target/AArch64/AArch64FPLoadBalancing.cpp @@ -0,0 +1,700 @@ +//===-- AArch64FPLoadBalancing.cpp - Balance FP ops statically on A57---===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// For best-case performance on Cortex-A57, we should try to use a balanced +// mix of odd and even D-registers when performing a critical sequence of +// independent, non-quadword FP/ASIMD floating-point multiply or +// multiply-accumulate operations. +// +// This pass attempts to detect situations where the register allocation may +// adversely affect this load balancing and to change the registers used so as +// to better utilize the CPU. +// +// Ideally we'd just take each multiply or multiply-accumulate in turn and +// allocate it alternating even or odd registers. However, multiply-accumulates +// are most efficiently performed in the same functional unit as their +// accumulation operand. Therefore this pass tries to find maximal sequences +// ("Chains") of multiply-accumulates linked via their accumulation operand, +// and assign them all the same "color" (oddness/evenness). +// +// This optimization affects S-register and D-register floating point +// multiplies and FMADD/FMAs, as well as vector (floating point only) muls and +// FMADD/FMA. Q register instructions (and 128-bit vector instructions) are +// not affected. +//===----------------------------------------------------------------------===// + +#include "AArch64.h" +#include "AArch64InstrInfo.h" +#include "AArch64Subtarget.h" +#include "llvm/ADT/BitVector.h" +#include "llvm/ADT/EquivalenceClasses.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/RegisterScavenging.h" +#include "llvm/CodeGen/RegisterClassInfo.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +#include +using namespace llvm; + +#define DEBUG_TYPE "aarch64-a57-fp-load-balancing" + +// Enforce the algorithm to use the scavenged register even when the original +// destination register is the correct color. Used for testing. +static cl::opt +TransformAll("aarch64-a57-fp-load-balancing-force-all", + cl::desc("Always modify dest registers regardless of color"), + cl::init(false), cl::Hidden); + +// Never use the balance information obtained from chains - return a specific +// color always. Used for testing. +static cl::opt +OverrideBalance("aarch64-a57-fp-load-balancing-override", + cl::desc("Ignore balance information, always return " + "(1: Even, 2: Odd)."), + cl::init(0), cl::Hidden); + +//===----------------------------------------------------------------------===// +// Helper functions + +// Is the instruction a type of multiply on 64-bit (or 32-bit) FPRs? +static bool isMul(MachineInstr *MI) { + switch (MI->getOpcode()) { + case AArch64::FMULSrr: + case AArch64::FNMULSrr: + case AArch64::FMULDrr: + case AArch64::FNMULDrr: + return true; + default: + return false; + } +} + +// Is the instruction a type of FP multiply-accumulate on 64-bit (or 32-bit) FPRs? +static bool isMla(MachineInstr *MI) { + switch (MI->getOpcode()) { + case AArch64::FMSUBSrrr: + case AArch64::FMADDSrrr: + case AArch64::FNMSUBSrrr: + case AArch64::FNMADDSrrr: + case AArch64::FMSUBDrrr: + case AArch64::FMADDDrrr: + case AArch64::FNMSUBDrrr: + case AArch64::FNMADDDrrr: + return true; + default: + return false; + } +} + +//===----------------------------------------------------------------------===// + +namespace { +/// A "color", which is either even or odd. Yes, these aren't really colors +/// but the algorithm is conceptually doing two-color graph coloring. +enum class Color { Even, Odd }; +#ifndef NDEBUG +static const char *ColorNames[2] = { "Even", "Odd" }; +#endif + +class Chain; + +class AArch64FPLoadBalancing : public MachineFunctionPass { + const AArch64InstrInfo *TII; + MachineRegisterInfo *MRI; + const TargetRegisterInfo *TRI; + RegisterClassInfo RCI; + +public: + static char ID; + explicit AArch64FPLoadBalancing() : MachineFunctionPass(ID) {} + + bool runOnMachineFunction(MachineFunction &F) override; + + const char *getPassName() const override { + return "A57 FP Anti-dependency breaker"; + } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesCFG(); + MachineFunctionPass::getAnalysisUsage(AU); + } + +private: + bool runOnBasicBlock(MachineBasicBlock &MBB); + bool colorChainSet(std::vector GV, MachineBasicBlock &MBB, + int &Balance); + bool colorChain(Chain *G, Color C, MachineBasicBlock &MBB); + int scavengeRegister(Chain *G, Color C, MachineBasicBlock &MBB); + void scanInstruction(MachineInstr *MI, unsigned Idx, + std::map &Active, + std::set> &AllChains); + void maybeKillChain(MachineOperand &MO, unsigned Idx, + std::map &RegChains); + Color getColor(unsigned Register); + Chain *getAndEraseNext(Color PreferredColor, std::vector &L); +}; +char AArch64FPLoadBalancing::ID = 0; + +/// A Chain is a sequence of instructions that are linked together by +/// an accumulation operand. For example: +/// +/// fmul d0, ? +/// fmla d1, ?, ?, d0 +/// fmla d2, ?, ?, d1 +/// +/// There may be other instructions interleaved in the sequence that +/// do not belong to the chain. These other instructions must not use +/// the "chain" register at any point. +/// +/// We currently only support chains where the "chain" operand is killed +/// at each link in the chain for simplicity. +/// A chain has three important instructions - Start, Last and Kill. +/// * The start instruction is the first instruction in the chain. +/// * Last is the final instruction in the chain. +/// * Kill may or may not be defined. If defined, Kill is the instruction +/// where the outgoing value of the Last instruction is killed. +/// This information is important as if we know the outgoing value is +/// killed with no intervening uses, we can safely change its register. +/// +/// Without a kill instruction, we must assume the outgoing value escapes +/// beyond our model and either must not change its register or must +/// create a fixup FMOV to keep the old register value consistent. +/// +class Chain { +public: + /// The important (marker) instructions. + MachineInstr *StartInst, *LastInst, *KillInst; + /// The index, from the start of the basic block, that each marker + /// appears. These are stored so we can do quick interval tests. + unsigned StartInstIdx, LastInstIdx, KillInstIdx; + /// All instructions in the chain. + std::set Insts; + /// True if KillInst cannot be modified. If this is true, + /// we cannot change LastInst's outgoing register. + /// This will be true for tied values and regmasks. + bool KillIsImmutable; + /// The "color" of LastInst. This will be the preferred chain color, + /// as changing intermediate nodes is easy but changing the last + /// instruction can be more tricky. + Color LastColor; + + Chain(MachineInstr *MI, unsigned Idx, Color C) + : StartInst(MI), LastInst(MI), KillInst(nullptr), + StartInstIdx(Idx), LastInstIdx(Idx), KillInstIdx(0), + LastColor(C) { + Insts.insert(MI); + } + + /// Add a new instruction into the chain. The instruction's dest operand + /// has the given color. + void add(MachineInstr *MI, unsigned Idx, Color C) { + LastInst = MI; + LastInstIdx = Idx; + LastColor = C; + assert((KillInstIdx == 0 || LastInstIdx < KillInstIdx) && + "Chain: broken invariant. A Chain can only be killed after its last " + "def"); + + Insts.insert(MI); + } + + /// Return true if MI is a member of the chain. + bool contains(MachineInstr *MI) { return Insts.count(MI) > 0; } + + /// Return the number of instructions in the chain. + unsigned size() const { + return Insts.size(); + } + + /// Inform the chain that its last active register (the dest register of + /// LastInst) is killed by MI with no intervening uses or defs. + void setKill(MachineInstr *MI, unsigned Idx, bool Immutable) { + KillInst = MI; + KillInstIdx = Idx; + KillIsImmutable = Immutable; + assert((KillInstIdx == 0 || LastInstIdx < KillInstIdx) && + "Chain: broken invariant. A Chain can only be killed after its last " + "def"); + } + + /// Return the first instruction in the chain. + MachineInstr *getStart() const { return StartInst; } + /// Return the last instruction in the chain. + MachineInstr *getLast() const { return LastInst; } + /// Return the "kill" instruction (as set with setKill()) or NULL. + MachineInstr *getKill() const { return KillInst; } + /// Return an instruction that can be used as an iterator for the end + /// of the chain. This is the maximum of KillInst (if set) and LastInst. + MachineBasicBlock::iterator getEnd() const { + return ++MachineBasicBlock::iterator(KillInst ? KillInst : LastInst); + } + + /// Can the Kill instruction (assuming one exists) be modified? + bool isKillImmutable() const { return KillIsImmutable; } + + /// Return the preferred color of this chain. + Color getPreferredColor() { + if (OverrideBalance != 0) + return OverrideBalance == 1 ? Color::Even : Color::Odd; + return LastColor; + } + + /// Return true if this chain (StartInst..KillInst) overlaps with Other. + bool rangeOverlapsWith(const Chain &Other) const { + unsigned End = KillInst ? KillInstIdx : LastInstIdx; + unsigned OtherEnd = Other.KillInst ? + Other.KillInstIdx : Other.LastInstIdx; + + return StartInstIdx <= OtherEnd && Other.StartInstIdx <= End; + } + + /// Return true if this chain starts before Other. + bool startsBefore(Chain *Other) { + return StartInstIdx < Other->StartInstIdx; + } + + /// Return true if the group will require a fixup MOV at the end. + bool requiresFixup() const { + return (getKill() && isKillImmutable()) || !getKill(); + } + + /// Return a simple string representation of the chain. + std::string str() const { + std::string S; + raw_string_ostream OS(S); + + OS << "{"; + StartInst->print(OS, NULL, true); + OS << " -> "; + LastInst->print(OS, NULL, true); + if (KillInst) { + OS << " (kill @ "; + KillInst->print(OS, NULL, true); + OS << ")"; + } + OS << "}"; + + return OS.str(); + } + +}; + +} // end anonymous namespace + +//===----------------------------------------------------------------------===// + +bool AArch64FPLoadBalancing::runOnMachineFunction(MachineFunction &F) { + bool Changed = false; + DEBUG(dbgs() << "***** AArch64FPLoadBalancing *****\n"); + + const TargetMachine &TM = F.getTarget(); + MRI = &F.getRegInfo(); + TRI = F.getRegInfo().getTargetRegisterInfo(); + TII = TM.getSubtarget().getInstrInfo(); + RCI.runOnMachineFunction(F); + + for (auto &MBB : F) { + Changed |= runOnBasicBlock(MBB); + } + + return Changed; +} + +bool AArch64FPLoadBalancing::runOnBasicBlock(MachineBasicBlock &MBB) { + bool Changed = false; + DEBUG(dbgs() << "Running on MBB: " << MBB << " - scanning instructions...\n"); + + // First, scan the basic block producing a set of chains. + + // The currently "active" chains - chains that can be added to and haven't + // been killed yet. This is keyed by register - all chains can only have one + // "link" register between each inst in the chain. + std::map ActiveChains; + std::set> AllChains; + unsigned Idx = 0; + for (auto &MI : MBB) + scanInstruction(&MI, Idx++, ActiveChains, AllChains); + + DEBUG(dbgs() << "Scan complete, "<< AllChains.size() << " chains created.\n"); + + // Group the chains into disjoint sets based on their liveness range. This is + // a poor-man's version of graph coloring. Ideally we'd create an interference + // graph and perform full-on graph coloring on that, but; + // (a) That's rather heavyweight for only two colors. + // (b) We expect multiple disjoint interference regions - in practice the live + // range of chains is quite small and they are clustered between loads + // and stores. + EquivalenceClasses EC; + for (auto &I : AllChains) + EC.insert(I.get()); + + for (auto &I : AllChains) + for (auto &J : AllChains) + if (I != J && I->rangeOverlapsWith(*J)) + EC.unionSets(I.get(), J.get()); + DEBUG(dbgs() << "Created " << EC.getNumClasses() << " disjoint sets.\n"); + + // Now we assume that every member of an equivalence class interferes + // with every other member of that class, and with no members of other classes. + + // Convert the EquivalenceClasses to a simpler set of sets. + std::vector > V; + for (auto I = EC.begin(), E = EC.end(); I != E; ++I) { + std::vector Cs(EC.member_begin(I), EC.member_end()); + if (Cs.empty()) continue; + V.push_back(std::move(Cs)); + } + + // Now we have a set of sets, order them by start address so + // we can iterate over them sequentially. + std::sort(V.begin(), V.end(), + [](const std::vector &A, + const std::vector &B) { + return A.front()->startsBefore(B.front()); + }); + + // As we only have two colors, we can track the global (BB-level) balance of + // odds versus evens. We aim to keep this near zero to keep both execution + // units fed. + // Positive means we're even-heavy, negative we're odd-heavy. + // + // FIXME: If chains have interdependencies, for example: + // mul r0, r1, r2 + // mul r3, r0, r1 + // We do not model this and may color each one differently, assuming we'll + // get ILP when we obviously can't. This hasn't been seen to be a problem + // in practice so far, so we simplify the algorithm by ignoring it. + int Parity = 0; + + for (auto &I : V) + Changed |= colorChainSet(std::move(I), MBB, Parity); + + return Changed; +} + +Chain *AArch64FPLoadBalancing::getAndEraseNext(Color PreferredColor, + std::vector &L) { + if (L.empty()) + return nullptr; + + // We try and get the best candidate from L to color next, given that our + // preferred color is "PreferredColor". L is ordered from larger to smaller + // chains. It is beneficial to color the large chains before the small chains, + // but if we can't find a chain of the maximum length with the preferred color, + // we fuzz the size and look for slightly smaller chains before giving up and + // returning a chain that must be recolored. + + // FIXME: Does this need to be configurable? + const unsigned SizeFuzz = 1; + unsigned MinSize = L.front()->size() - SizeFuzz; + for (auto I = L.begin(), E = L.end(); I != E; ++I) { + if ((*I)->size() <= MinSize) { + // We've gone past the size limit. Return the previous item. + Chain *Ch = *--I; + L.erase(I); + return Ch; + } + + if ((*I)->getPreferredColor() == PreferredColor) { + Chain *Ch = *I; + L.erase(I); + return Ch; + } + } + + // Bailout case - just return the first item. + Chain *Ch = L.front(); + L.erase(L.begin()); + return Ch; +} + +bool AArch64FPLoadBalancing::colorChainSet(std::vector GV, + MachineBasicBlock &MBB, + int &Parity) { + bool Changed = false; + DEBUG(dbgs() << "colorChainSet(): #sets=" << GV.size() << "\n"); + + // Sort by descending size order so that we allocate the most important + // sets first. + // Tie-break equivalent sizes by sorting chains requiring fixups before + // those without fixups. The logic here is that we should look at the + // chains that we cannot change before we look at those we can, + // so the parity counter is updated and we know what color we should + // change them to! + std::sort(GV.begin(), GV.end(), [](const Chain *G1, const Chain *G2) { + if (G1->size() != G2->size()) + return G1->size() > G2->size(); + return G1->requiresFixup() > G2->requiresFixup(); + }); + + Color PreferredColor = Parity < 0 ? Color::Even : Color::Odd; + while (Chain *G = getAndEraseNext(PreferredColor, GV)) { + // Start off by assuming we'll color to our own preferred color. + Color C = PreferredColor; + if (Parity == 0) + // But if we really don't care, use the chain's preferred color. + C = G->getPreferredColor(); + + DEBUG(dbgs() << " - Parity=" << Parity << ", Color=" + << ColorNames[(int)C] << "\n"); + + // If we'll need a fixup FMOV, don't bother. Testing has shown that this + // happens infrequently and when it does it has at least a 50% chance of + // slowing code down instead of speeding it up. + if (G->requiresFixup() && C != G->getPreferredColor()) { + C = G->getPreferredColor(); + DEBUG(dbgs() << " - " << G->str() << " - not worthwhile changing; " + "color remains " << ColorNames[(int)C] << "\n"); + } + + Changed |= colorChain(G, C, MBB); + + Parity += (C == Color::Even) ? G->size() : -G->size(); + PreferredColor = Parity < 0 ? Color::Even : Color::Odd; + } + + return Changed; +} + +int AArch64FPLoadBalancing::scavengeRegister(Chain *G, Color C, + MachineBasicBlock &MBB) { + RegScavenger RS; + RS.enterBasicBlock(&MBB); + RS.forward(MachineBasicBlock::iterator(G->getStart())); + + // Can we find an appropriate register that is available throughout the life + // of the chain? + unsigned RegClassID = G->getStart()->getDesc().OpInfo[0].RegClass; + BitVector AvailableRegs = RS.getRegsAvailable(TRI->getRegClass(RegClassID)); + for (MachineBasicBlock::iterator I = G->getStart(), E = G->getEnd(); + I != E; ++I) { + RS.forward(I); + AvailableRegs &= RS.getRegsAvailable(TRI->getRegClass(RegClassID)); + + // Remove any registers clobbered by a regmask. + for (auto J : I->operands()) { + if (J.isRegMask()) + AvailableRegs.clearBitsNotInMask(J.getRegMask()); + } + } + + // Make sure we allocate in-order, to get the cheapest registers first. + auto Ord = RCI.getOrder(TRI->getRegClass(RegClassID)); + for (auto Reg : Ord) { + if (!AvailableRegs[Reg]) + continue; + if ((C == Color::Even && (Reg % 2) == 0) || + (C == Color::Odd && (Reg % 2) == 1)) + return Reg; + } + + return -1; +} + +bool AArch64FPLoadBalancing::colorChain(Chain *G, Color C, + MachineBasicBlock &MBB) { + bool Changed = false; + DEBUG(dbgs() << " - colorChain(" << G->str() << ", " + << ColorNames[(int)C] << ")\n"); + + // Try and obtain a free register of the right class. Without a register + // to play with we cannot continue. + int Reg = scavengeRegister(G, C, MBB); + if (Reg == -1) { + DEBUG(dbgs() << "Scavenging (thus coloring) failed!\n"); + return false; + } + DEBUG(dbgs() << " - Scavenged register: " << TRI->getName(Reg) << "\n"); + + std::map Substs; + for (MachineBasicBlock::iterator I = G->getStart(), E = G->getEnd(); + I != E; ++I) { + if (!G->contains(I) && + (&*I != G->getKill() || G->isKillImmutable())) + continue; + + // I is a member of G, or I is a mutable instruction that kills G. + + std::vector ToErase; + for (auto &U : I->operands()) { + if (U.isReg() && U.isUse() && Substs.find(U.getReg()) != Substs.end()) { + unsigned OrigReg = U.getReg(); + U.setReg(Substs[OrigReg]); + if (U.isKill()) + // Don't erase straight away, because there may be other operands + // that also reference this substitution! + ToErase.push_back(OrigReg); + } else if (U.isRegMask()) { + for (auto J : Substs) { + if (U.clobbersPhysReg(J.first)) + ToErase.push_back(J.first); + } + } + } + // Now it's safe to remove the substs identified earlier. + for (auto J : ToErase) + Substs.erase(J); + + // Only change the def if this isn't the last instruction. + if (&*I != G->getKill()) { + MachineOperand &MO = I->getOperand(0); + + bool Change = TransformAll || getColor(MO.getReg()) != C; + if (G->requiresFixup() && &*I == G->getLast()) + Change = false; + + if (Change) { + Substs[MO.getReg()] = Reg; + MO.setReg(Reg); + MRI->setPhysRegUsed(Reg); + + Changed = true; + } + } + } + assert(Substs.size() == 0 && "No substitutions should be left active!"); + + if (G->getKill()) { + DEBUG(dbgs() << " - Kill instruction seen.\n"); + } else { + // We didn't have a kill instruction, but we didn't seem to need to change + // the destination register anyway. + DEBUG(dbgs() << " - Destination register not changed.\n"); + } + return Changed; +} + +void AArch64FPLoadBalancing:: +scanInstruction(MachineInstr *MI, unsigned Idx, + std::map &ActiveChains, + std::set> &AllChains) { + // Inspect "MI", updating ActiveChains and AllChains. + + if (isMul(MI)) { + + for (auto &I : MI->uses()) + maybeKillChain(I, Idx, ActiveChains); + for (auto &I : MI->defs()) + maybeKillChain(I, Idx, ActiveChains); + + // Create a new chain. Multiplies don't require forwarding so can go on any + // unit. + unsigned DestReg = MI->getOperand(0).getReg(); + + DEBUG(dbgs() << "New chain started for register " + << TRI->getName(DestReg) << " at " << *MI); + + auto G = llvm::make_unique(MI, Idx, getColor(DestReg)); + ActiveChains[DestReg] = G.get(); + AllChains.insert(std::move(G)); + + } else if (isMla(MI)) { + + // It is beneficial to keep MLAs on the same functional unit as their + // accumulator operand. + unsigned DestReg = MI->getOperand(0).getReg(); + unsigned AccumReg = MI->getOperand(3).getReg(); + + maybeKillChain(MI->getOperand(1), Idx, ActiveChains); + maybeKillChain(MI->getOperand(2), Idx, ActiveChains); + if (DestReg != AccumReg) + maybeKillChain(MI->getOperand(0), Idx, ActiveChains); + + if (ActiveChains.find(AccumReg) != ActiveChains.end()) { + DEBUG(dbgs() << "Chain found for accumulator register " + << TRI->getName(AccumReg) << " in MI " << *MI); + + // For simplicity we only chain together sequences of MULs/MLAs where the + // accumulator register is killed on each instruction. This means we don't + // need to track other uses of the registers we want to rewrite. + // + // FIXME: We could extend to handle the non-kill cases for more coverage. + if (MI->getOperand(3).isKill()) { + // Add to chain. + DEBUG(dbgs() << "Instruction was successfully added to chain.\n"); + ActiveChains[AccumReg]->add(MI, Idx, getColor(DestReg)); + // Handle cases where the destination is not the same as the accumulator. + if (DestReg != AccumReg) { + ActiveChains[DestReg] = ActiveChains[AccumReg]; + ActiveChains.erase(AccumReg); + } + return; + } + + DEBUG(dbgs() << "Cannot add to chain because accumulator operand wasn't " + << "marked !\n"); + maybeKillChain(MI->getOperand(3), Idx, ActiveChains); + } + + DEBUG(dbgs() << "Creating new chain for dest register " + << TRI->getName(DestReg) << "\n"); + auto G = llvm::make_unique(MI, Idx, getColor(DestReg)); + ActiveChains[DestReg] = G.get(); + AllChains.insert(std::move(G)); + + } else { + + // Non-MUL or MLA instruction. Invalidate any chain in the uses or defs + // lists. + for (auto &I : MI->uses()) + maybeKillChain(I, Idx, ActiveChains); + for (auto &I : MI->defs()) + maybeKillChain(I, Idx, ActiveChains); + + } +} + +void AArch64FPLoadBalancing:: +maybeKillChain(MachineOperand &MO, unsigned Idx, + std::map &ActiveChains) { + // Given an operand and the set of active chains (keyed by register), + // determine if a chain should be ended and remove from ActiveChains. + MachineInstr *MI = MO.getParent(); + + if (MO.isReg()) { + + // If this is a KILL of a current chain, record it. + if (MO.isKill() && ActiveChains.find(MO.getReg()) != ActiveChains.end()) { + DEBUG(dbgs() << "Kill seen for chain " << TRI->getName(MO.getReg()) + << "\n"); + ActiveChains[MO.getReg()]->setKill(MI, Idx, /*Immutable=*/MO.isTied()); + } + ActiveChains.erase(MO.getReg()); + + } else if (MO.isRegMask()) { + + for (auto I = ActiveChains.begin(), E = ActiveChains.end(); + I != E;) { + if (MO.clobbersPhysReg(I->first)) { + DEBUG(dbgs() << "Kill (regmask) seen for chain " + << TRI->getName(I->first) << "\n"); + I->second->setKill(MI, Idx, /*Immutable=*/true); + ActiveChains.erase(I++); + } else + ++I; + } + + } +} + +Color AArch64FPLoadBalancing::getColor(unsigned Reg) { + if ((TRI->getEncodingValue(Reg) % 2) == 0) + return Color::Even; + else + return Color::Odd; +} + +// Factory function used by AArch64TargetMachine to add the pass to the passmanager. +FunctionPass *llvm::createAArch64FPLoadBalancing() { + return new AArch64FPLoadBalancing(); +} Index: lib/Target/AArch64/AArch64PBQPRegAlloc.cpp =================================================================== --- lib/Target/AArch64/AArch64PBQPRegAlloc.cpp +++ lib/Target/AArch64/AArch64PBQPRegAlloc.cpp @@ -10,7 +10,7 @@ // constraints for use by the PBQP register allocator. // // It is essentially a transcription of what is contained in -// AArch64A57FPLoadBalancing, which tries to use a balanced +// AArch64FPLoadBalancing, which tries to use a balanced // mix of odd and even D-registers when performing a critical sequence of // independent, non-quadword FP/ASIMD floating-point multiply-accumulates. //===----------------------------------------------------------------------===// Index: lib/Target/AArch64/AArch64TargetMachine.cpp =================================================================== --- lib/Target/AArch64/AArch64TargetMachine.cpp +++ lib/Target/AArch64/AArch64TargetMachine.cpp @@ -255,7 +255,7 @@ TM->getSubtarget().isCortexA57()) && usingDefaultRegAlloc()) // Improve performance for some FP/SIMD code for A57. - addPass(createAArch64A57FPLoadBalancing()); + addPass(createAArch64FPLoadBalancing()); return true; } Index: lib/Target/AArch64/CMakeLists.txt =================================================================== --- lib/Target/AArch64/CMakeLists.txt +++ lib/Target/AArch64/CMakeLists.txt @@ -15,7 +15,7 @@ add_public_tablegen_target(AArch64CommonTableGen) add_llvm_target(AArch64CodeGen - AArch64A57FPLoadBalancing.cpp + AArch64FPLoadBalancing.cpp AArch64AddressTypePromotion.cpp AArch64AdvSIMDScalarPass.cpp AArch64AsmPrinter.cpp Index: test/CodeGen/AArch64/aarch64-a57-fp-load-balancing.ll =================================================================== --- test/CodeGen/AArch64/aarch64-a57-fp-load-balancing.ll +++ /dev/null @@ -1,329 +0,0 @@ -; RUN: llc < %s -mcpu=cortex-a57 -aarch64-a57-fp-load-balancing-override=1 -aarch64-a57-fp-load-balancing-force-all | FileCheck %s --check-prefix CHECK --check-prefix CHECK-A57 --check-prefix CHECK-EVEN -; RUN: llc < %s -mcpu=cortex-a57 -aarch64-a57-fp-load-balancing-override=2 -aarch64-a57-fp-load-balancing-force-all | FileCheck %s --check-prefix CHECK --check-prefix CHECK-A57 --check-prefix CHECK-ODD -; RUN: llc < %s -mcpu=cortex-a53 -aarch64-a57-fp-load-balancing-override=1 -aarch64-a57-fp-load-balancing-force-all | FileCheck %s --check-prefix CHECK --check-prefix CHECK-A53 --check-prefix CHECK-EVEN -; RUN: llc < %s -mcpu=cortex-a53 -aarch64-a57-fp-load-balancing-override=2 -aarch64-a57-fp-load-balancing-force-all | FileCheck %s --check-prefix CHECK --check-prefix CHECK-A53 --check-prefix CHECK-ODD - -; Test the AArch64A57FPLoadBalancing pass. This pass relies heavily on register allocation, so -; our test strategy is to: -; * Force the pass to always perform register swapping even if the dest register is of the -; correct color already (-force-all) -; * Force the pass to ignore all hints it obtained from regalloc (-deterministic-balance), -; and run it twice, once where it always hints odd, and once where it always hints even. -; -; We then use regex magic to check that in the two cases the register allocation is -; different; this is what gives us the testing coverage and distinguishes cases where -; the pass has done some work versus accidental regalloc. - -target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" -target triple = "aarch64" - -; Non-overlapping groups - shouldn't need any changing at all. - -; CHECK-LABEL: f1: -; CHECK-EVEN: fmadd [[x:d[0-9]*[02468]]] -; CHECK-ODD: fmadd [[x:d[0-9]*[13579]]] -; CHECK: fmadd [[x]] -; CHECK: fmsub [[x]] -; CHECK: fmadd [[x]] -; CHECK: str [[x]] - -define void @f1(double* nocapture readonly %p, double* nocapture %q) #0 { -entry: - %0 = load double* %p, align 8 - %arrayidx1 = getelementptr inbounds double* %p, i64 1 - %1 = load double* %arrayidx1, align 8 - %arrayidx2 = getelementptr inbounds double* %p, i64 2 - %2 = load double* %arrayidx2, align 8 - %arrayidx3 = getelementptr inbounds double* %p, i64 3 - %3 = load double* %arrayidx3, align 8 - %arrayidx4 = getelementptr inbounds double* %p, i64 4 - %4 = load double* %arrayidx4, align 8 - %mul = fmul fast double %0, %1 - %add = fadd fast double %mul, %4 - %mul5 = fmul fast double %1, %2 - %add6 = fadd fast double %mul5, %add - %mul7 = fmul fast double %1, %3 - %sub = fsub fast double %add6, %mul7 - %mul8 = fmul fast double %2, %3 - %add9 = fadd fast double %mul8, %sub - store double %add9, double* %q, align 8 - %arrayidx11 = getelementptr inbounds double* %p, i64 5 - %5 = load double* %arrayidx11, align 8 - %arrayidx12 = getelementptr inbounds double* %p, i64 6 - %6 = load double* %arrayidx12, align 8 - %arrayidx13 = getelementptr inbounds double* %p, i64 7 - %7 = load double* %arrayidx13, align 8 - %mul15 = fmul fast double %6, %7 - %mul16 = fmul fast double %0, %5 - %add17 = fadd fast double %mul16, %mul15 - %mul18 = fmul fast double %5, %6 - %add19 = fadd fast double %mul18, %add17 - %arrayidx20 = getelementptr inbounds double* %q, i64 1 - store double %add19, double* %arrayidx20, align 8 - ret void -} - -; Overlapping groups - coloring needed. - -; CHECK-LABEL: f2: -; CHECK-EVEN: fmadd [[x:d[0-9]*[02468]]] -; CHECK-EVEN: fmul [[y:d[0-9]*[13579]]] -; CHECK-ODD: fmadd [[x:d[0-9]*[13579]]] -; CHECK-ODD: fmul [[y:d[0-9]*[02468]]] -; CHECK: fmadd [[x]] -; CHECK: fmadd [[y]] -; CHECK: fmsub [[x]] -; CHECK: fmadd [[y]] -; CHECK: fmadd [[x]] -; CHECK-A57: stp [[x]], [[y]] -; CHECK-A53-DAG: str [[x]] -; CHECK-A53-DAG: str [[y]] - -define void @f2(double* nocapture readonly %p, double* nocapture %q) #0 { -entry: - %0 = load double* %p, align 8 - %arrayidx1 = getelementptr inbounds double* %p, i64 1 - %1 = load double* %arrayidx1, align 8 - %arrayidx2 = getelementptr inbounds double* %p, i64 2 - %2 = load double* %arrayidx2, align 8 - %arrayidx3 = getelementptr inbounds double* %p, i64 3 - %3 = load double* %arrayidx3, align 8 - %arrayidx4 = getelementptr inbounds double* %p, i64 4 - %4 = load double* %arrayidx4, align 8 - %arrayidx5 = getelementptr inbounds double* %p, i64 5 - %5 = load double* %arrayidx5, align 8 - %arrayidx6 = getelementptr inbounds double* %p, i64 6 - %6 = load double* %arrayidx6, align 8 - %arrayidx7 = getelementptr inbounds double* %p, i64 7 - %7 = load double* %arrayidx7, align 8 - %mul = fmul fast double %0, %1 - %add = fadd fast double %mul, %7 - %mul8 = fmul fast double %5, %6 - %mul9 = fmul fast double %1, %2 - %add10 = fadd fast double %mul9, %add - %mul11 = fmul fast double %3, %4 - %add12 = fadd fast double %mul11, %mul8 - %mul13 = fmul fast double %1, %3 - %sub = fsub fast double %add10, %mul13 - %mul14 = fmul fast double %4, %5 - %add15 = fadd fast double %mul14, %add12 - %mul16 = fmul fast double %2, %3 - %add17 = fadd fast double %mul16, %sub - store double %add17, double* %q, align 8 - %arrayidx19 = getelementptr inbounds double* %q, i64 1 - store double %add15, double* %arrayidx19, align 8 - ret void -} - -; Dest register is live on block exit - fixup needed. - -; CHECK-LABEL: f3: -; CHECK-EVEN: fmadd [[x:d[0-9]*[02468]]] -; CHECK-ODD: fmadd [[x:d[0-9]*[13579]]] -; CHECK: fmadd [[x]] -; CHECK: fmsub [[x]] -; CHECK: fmadd [[y:d[0-9]+]], {{.*}}, [[x]] -; CHECK: str [[y]] - -define void @f3(double* nocapture readonly %p, double* nocapture %q) #0 { -entry: - %0 = load double* %p, align 8 - %arrayidx1 = getelementptr inbounds double* %p, i64 1 - %1 = load double* %arrayidx1, align 8 - %arrayidx2 = getelementptr inbounds double* %p, i64 2 - %2 = load double* %arrayidx2, align 8 - %arrayidx3 = getelementptr inbounds double* %p, i64 3 - %3 = load double* %arrayidx3, align 8 - %arrayidx4 = getelementptr inbounds double* %p, i64 4 - %4 = load double* %arrayidx4, align 8 - %mul = fmul fast double %0, %1 - %add = fadd fast double %mul, %4 - %mul5 = fmul fast double %1, %2 - %add6 = fadd fast double %mul5, %add - %mul7 = fmul fast double %1, %3 - %sub = fsub fast double %add6, %mul7 - %mul8 = fmul fast double %2, %3 - %add9 = fadd fast double %mul8, %sub - %cmp = fcmp oeq double %3, 0.000000e+00 - br i1 %cmp, label %if.then, label %if.end - -if.then: ; preds = %entry - tail call void bitcast (void (...)* @g to void ()*)() #2 - br label %if.end - -if.end: ; preds = %if.then, %entry - store double %add9, double* %q, align 8 - ret void -} - -declare void @g(...) #1 - -; Single precision version of f2. - -; CHECK-LABEL: f4: -; CHECK-EVEN: fmadd [[x:s[0-9]*[02468]]] -; CHECK-EVEN: fmul [[y:s[0-9]*[13579]]] -; CHECK-ODD: fmadd [[x:s[0-9]*[13579]]] -; CHECK-ODD: fmul [[y:s[0-9]*[02468]]] -; CHECK: fmadd [[x]] -; CHECK: fmadd [[y]] -; CHECK: fmsub [[x]] -; CHECK: fmadd [[y]] -; CHECK: fmadd [[x]] -; CHECK-A57: stp [[x]], [[y]] -; CHECK-A53-DAG: str [[x]] -; CHECK-A53-DAG: str [[y]] - -define void @f4(float* nocapture readonly %p, float* nocapture %q) #0 { -entry: - %0 = load float* %p, align 4 - %arrayidx1 = getelementptr inbounds float* %p, i64 1 - %1 = load float* %arrayidx1, align 4 - %arrayidx2 = getelementptr inbounds float* %p, i64 2 - %2 = load float* %arrayidx2, align 4 - %arrayidx3 = getelementptr inbounds float* %p, i64 3 - %3 = load float* %arrayidx3, align 4 - %arrayidx4 = getelementptr inbounds float* %p, i64 4 - %4 = load float* %arrayidx4, align 4 - %arrayidx5 = getelementptr inbounds float* %p, i64 5 - %5 = load float* %arrayidx5, align 4 - %arrayidx6 = getelementptr inbounds float* %p, i64 6 - %6 = load float* %arrayidx6, align 4 - %arrayidx7 = getelementptr inbounds float* %p, i64 7 - %7 = load float* %arrayidx7, align 4 - %mul = fmul fast float %0, %1 - %add = fadd fast float %mul, %7 - %mul8 = fmul fast float %5, %6 - %mul9 = fmul fast float %1, %2 - %add10 = fadd fast float %mul9, %add - %mul11 = fmul fast float %3, %4 - %add12 = fadd fast float %mul11, %mul8 - %mul13 = fmul fast float %1, %3 - %sub = fsub fast float %add10, %mul13 - %mul14 = fmul fast float %4, %5 - %add15 = fadd fast float %mul14, %add12 - %mul16 = fmul fast float %2, %3 - %add17 = fadd fast float %mul16, %sub - store float %add17, float* %q, align 4 - %arrayidx19 = getelementptr inbounds float* %q, i64 1 - store float %add15, float* %arrayidx19, align 4 - ret void -} - -; Single precision version of f3 - -; CHECK-LABEL: f5: -; CHECK-EVEN: fmadd [[x:s[0-9]*[02468]]] -; CHECK-ODD: fmadd [[x:s[0-9]*[13579]]] -; CHECK: fmadd [[x]] -; CHECK: fmsub [[x]] -; CHECK: fmadd [[y:s[0-9]+]], {{.*}}, [[x]] -; CHECK: str [[y]] - -define void @f5(float* nocapture readonly %p, float* nocapture %q) #0 { -entry: - %0 = load float* %p, align 4 - %arrayidx1 = getelementptr inbounds float* %p, i64 1 - %1 = load float* %arrayidx1, align 4 - %arrayidx2 = getelementptr inbounds float* %p, i64 2 - %2 = load float* %arrayidx2, align 4 - %arrayidx3 = getelementptr inbounds float* %p, i64 3 - %3 = load float* %arrayidx3, align 4 - %arrayidx4 = getelementptr inbounds float* %p, i64 4 - %4 = load float* %arrayidx4, align 4 - %mul = fmul fast float %0, %1 - %add = fadd fast float %mul, %4 - %mul5 = fmul fast float %1, %2 - %add6 = fadd fast float %mul5, %add - %mul7 = fmul fast float %1, %3 - %sub = fsub fast float %add6, %mul7 - %mul8 = fmul fast float %2, %3 - %add9 = fadd fast float %mul8, %sub - %cmp = fcmp oeq float %3, 0.000000e+00 - br i1 %cmp, label %if.then, label %if.end - -if.then: ; preds = %entry - tail call void bitcast (void (...)* @g to void ()*)() #2 - br label %if.end - -if.end: ; preds = %if.then, %entry - store float %add9, float* %q, align 4 - ret void -} - -; Test that regmask clobbering stops a chain sequence. - -; CHECK-LABEL: f6: -; CHECK-EVEN: fmadd [[x:d[0-9]*[02468]]] -; CHECK-ODD: fmadd [[x:d[0-9]*[13579]]] -; CHECK: fmadd [[x]] -; CHECK: fmsub [[x]] -; CHECK: fmadd d0, {{.*}}, [[x]] -; CHECK: bl hh -; CHECK: str d0 - -define void @f6(double* nocapture readonly %p, double* nocapture %q) #0 { -entry: - %0 = load double* %p, align 8 - %arrayidx1 = getelementptr inbounds double* %p, i64 1 - %1 = load double* %arrayidx1, align 8 - %arrayidx2 = getelementptr inbounds double* %p, i64 2 - %2 = load double* %arrayidx2, align 8 - %arrayidx3 = getelementptr inbounds double* %p, i64 3 - %3 = load double* %arrayidx3, align 8 - %arrayidx4 = getelementptr inbounds double* %p, i64 4 - %4 = load double* %arrayidx4, align 8 - %mul = fmul fast double %0, %1 - %add = fadd fast double %mul, %4 - %mul5 = fmul fast double %1, %2 - %add6 = fadd fast double %mul5, %add - %mul7 = fmul fast double %1, %3 - %sub = fsub fast double %add6, %mul7 - %mul8 = fmul fast double %2, %3 - %add9 = fadd fast double %mul8, %sub - %call = tail call double @hh(double %add9) #2 - store double %call, double* %q, align 8 - ret void -} - -declare double @hh(double) #1 - -; Check that we correctly deal with repeated operands. -; The following testcase creates: -; %D1 = FADDDrr %D0, %D0 -; We'll get a crash if we naively look at the first operand, remove it -; from the substitution list then look at the second operand. - -; CHECK: fmadd [[x:d[0-9]+]] -; CHECK: fadd d1, [[x]], [[x]] - -define void @f7(double* nocapture readonly %p, double* nocapture %q) #0 { -entry: - %0 = load double* %p, align 8 - %arrayidx1 = getelementptr inbounds double* %p, i64 1 - %1 = load double* %arrayidx1, align 8 - %arrayidx2 = getelementptr inbounds double* %p, i64 2 - %2 = load double* %arrayidx2, align 8 - %arrayidx3 = getelementptr inbounds double* %p, i64 3 - %3 = load double* %arrayidx3, align 8 - %arrayidx4 = getelementptr inbounds double* %p, i64 4 - %4 = load double* %arrayidx4, align 8 - %mul = fmul fast double %0, %1 - %add = fadd fast double %mul, %4 - %mul5 = fmul fast double %1, %2 - %add6 = fadd fast double %mul5, %add - %mul7 = fmul fast double %1, %3 - %sub = fsub fast double %add6, %mul7 - %mul8 = fmul fast double %2, %3 - %add9 = fadd fast double %mul8, %sub - %add10 = fadd fast double %add9, %add9 - call void @hhh(double 0.0, double %add10) - ret void -} - -declare void @hhh(double, double) - -attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="true" "use-soft-float"="false" } -attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="true" "use-soft-float"="false" } -attributes #2 = { nounwind } - Index: test/CodeGen/AArch64/aarch64-fp-load-balancing.ll =================================================================== --- /dev/null +++ test/CodeGen/AArch64/aarch64-fp-load-balancing.ll @@ -0,0 +1,329 @@ +; RUN: llc < %s -mcpu=cortex-a57 -aarch64-a57-fp-load-balancing-override=1 -aarch64-a57-fp-load-balancing-force-all | FileCheck %s --check-prefix CHECK --check-prefix CHECK-A57 --check-prefix CHECK-EVEN +; RUN: llc < %s -mcpu=cortex-a57 -aarch64-a57-fp-load-balancing-override=2 -aarch64-a57-fp-load-balancing-force-all | FileCheck %s --check-prefix CHECK --check-prefix CHECK-A57 --check-prefix CHECK-ODD +; RUN: llc < %s -mcpu=cortex-a53 -aarch64-a57-fp-load-balancing-override=1 -aarch64-a57-fp-load-balancing-force-all | FileCheck %s --check-prefix CHECK --check-prefix CHECK-A53 --check-prefix CHECK-EVEN +; RUN: llc < %s -mcpu=cortex-a53 -aarch64-a57-fp-load-balancing-override=2 -aarch64-a57-fp-load-balancing-force-all | FileCheck %s --check-prefix CHECK --check-prefix CHECK-A53 --check-prefix CHECK-ODD + +; Test the AArch64A57FPLoadBalancing pass. This pass relies heavily on register allocation, so +; our test strategy is to: +; * Force the pass to always perform register swapping even if the dest register is of the +; correct color already (-force-all) +; * Force the pass to ignore all hints it obtained from regalloc (-deterministic-balance), +; and run it twice, once where it always hints odd, and once where it always hints even. +; +; We then use regex magic to check that in the two cases the register allocation is +; different; this is what gives us the testing coverage and distinguishes cases where +; the pass has done some work versus accidental regalloc. + +target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" +target triple = "aarch64" + +; Non-overlapping groups - shouldn't need any changing at all. + +; CHECK-LABEL: f1: +; CHECK-EVEN: fmadd [[x:d[0-9]*[02468]]] +; CHECK-ODD: fmadd [[x:d[0-9]*[13579]]] +; CHECK: fmadd [[x]] +; CHECK: fmsub [[x]] +; CHECK: fmadd [[x]] +; CHECK: str [[x]] + +define void @f1(double* nocapture readonly %p, double* nocapture %q) #0 { +entry: + %0 = load double* %p, align 8 + %arrayidx1 = getelementptr inbounds double* %p, i64 1 + %1 = load double* %arrayidx1, align 8 + %arrayidx2 = getelementptr inbounds double* %p, i64 2 + %2 = load double* %arrayidx2, align 8 + %arrayidx3 = getelementptr inbounds double* %p, i64 3 + %3 = load double* %arrayidx3, align 8 + %arrayidx4 = getelementptr inbounds double* %p, i64 4 + %4 = load double* %arrayidx4, align 8 + %mul = fmul fast double %0, %1 + %add = fadd fast double %mul, %4 + %mul5 = fmul fast double %1, %2 + %add6 = fadd fast double %mul5, %add + %mul7 = fmul fast double %1, %3 + %sub = fsub fast double %add6, %mul7 + %mul8 = fmul fast double %2, %3 + %add9 = fadd fast double %mul8, %sub + store double %add9, double* %q, align 8 + %arrayidx11 = getelementptr inbounds double* %p, i64 5 + %5 = load double* %arrayidx11, align 8 + %arrayidx12 = getelementptr inbounds double* %p, i64 6 + %6 = load double* %arrayidx12, align 8 + %arrayidx13 = getelementptr inbounds double* %p, i64 7 + %7 = load double* %arrayidx13, align 8 + %mul15 = fmul fast double %6, %7 + %mul16 = fmul fast double %0, %5 + %add17 = fadd fast double %mul16, %mul15 + %mul18 = fmul fast double %5, %6 + %add19 = fadd fast double %mul18, %add17 + %arrayidx20 = getelementptr inbounds double* %q, i64 1 + store double %add19, double* %arrayidx20, align 8 + ret void +} + +; Overlapping groups - coloring needed. + +; CHECK-LABEL: f2: +; CHECK-EVEN: fmadd [[x:d[0-9]*[02468]]] +; CHECK-EVEN: fmul [[y:d[0-9]*[13579]]] +; CHECK-ODD: fmadd [[x:d[0-9]*[13579]]] +; CHECK-ODD: fmul [[y:d[0-9]*[02468]]] +; CHECK: fmadd [[x]] +; CHECK: fmadd [[y]] +; CHECK: fmsub [[x]] +; CHECK: fmadd [[y]] +; CHECK: fmadd [[x]] +; CHECK-A57: stp [[x]], [[y]] +; CHECK-A53-DAG: str [[x]] +; CHECK-A53-DAG: str [[y]] + +define void @f2(double* nocapture readonly %p, double* nocapture %q) #0 { +entry: + %0 = load double* %p, align 8 + %arrayidx1 = getelementptr inbounds double* %p, i64 1 + %1 = load double* %arrayidx1, align 8 + %arrayidx2 = getelementptr inbounds double* %p, i64 2 + %2 = load double* %arrayidx2, align 8 + %arrayidx3 = getelementptr inbounds double* %p, i64 3 + %3 = load double* %arrayidx3, align 8 + %arrayidx4 = getelementptr inbounds double* %p, i64 4 + %4 = load double* %arrayidx4, align 8 + %arrayidx5 = getelementptr inbounds double* %p, i64 5 + %5 = load double* %arrayidx5, align 8 + %arrayidx6 = getelementptr inbounds double* %p, i64 6 + %6 = load double* %arrayidx6, align 8 + %arrayidx7 = getelementptr inbounds double* %p, i64 7 + %7 = load double* %arrayidx7, align 8 + %mul = fmul fast double %0, %1 + %add = fadd fast double %mul, %7 + %mul8 = fmul fast double %5, %6 + %mul9 = fmul fast double %1, %2 + %add10 = fadd fast double %mul9, %add + %mul11 = fmul fast double %3, %4 + %add12 = fadd fast double %mul11, %mul8 + %mul13 = fmul fast double %1, %3 + %sub = fsub fast double %add10, %mul13 + %mul14 = fmul fast double %4, %5 + %add15 = fadd fast double %mul14, %add12 + %mul16 = fmul fast double %2, %3 + %add17 = fadd fast double %mul16, %sub + store double %add17, double* %q, align 8 + %arrayidx19 = getelementptr inbounds double* %q, i64 1 + store double %add15, double* %arrayidx19, align 8 + ret void +} + +; Dest register is live on block exit - fixup needed. + +; CHECK-LABEL: f3: +; CHECK-EVEN: fmadd [[x:d[0-9]*[02468]]] +; CHECK-ODD: fmadd [[x:d[0-9]*[13579]]] +; CHECK: fmadd [[x]] +; CHECK: fmsub [[x]] +; CHECK: fmadd [[y:d[0-9]+]], {{.*}}, [[x]] +; CHECK: str [[y]] + +define void @f3(double* nocapture readonly %p, double* nocapture %q) #0 { +entry: + %0 = load double* %p, align 8 + %arrayidx1 = getelementptr inbounds double* %p, i64 1 + %1 = load double* %arrayidx1, align 8 + %arrayidx2 = getelementptr inbounds double* %p, i64 2 + %2 = load double* %arrayidx2, align 8 + %arrayidx3 = getelementptr inbounds double* %p, i64 3 + %3 = load double* %arrayidx3, align 8 + %arrayidx4 = getelementptr inbounds double* %p, i64 4 + %4 = load double* %arrayidx4, align 8 + %mul = fmul fast double %0, %1 + %add = fadd fast double %mul, %4 + %mul5 = fmul fast double %1, %2 + %add6 = fadd fast double %mul5, %add + %mul7 = fmul fast double %1, %3 + %sub = fsub fast double %add6, %mul7 + %mul8 = fmul fast double %2, %3 + %add9 = fadd fast double %mul8, %sub + %cmp = fcmp oeq double %3, 0.000000e+00 + br i1 %cmp, label %if.then, label %if.end + +if.then: ; preds = %entry + tail call void bitcast (void (...)* @g to void ()*)() #2 + br label %if.end + +if.end: ; preds = %if.then, %entry + store double %add9, double* %q, align 8 + ret void +} + +declare void @g(...) #1 + +; Single precision version of f2. + +; CHECK-LABEL: f4: +; CHECK-EVEN: fmadd [[x:s[0-9]*[02468]]] +; CHECK-EVEN: fmul [[y:s[0-9]*[13579]]] +; CHECK-ODD: fmadd [[x:s[0-9]*[13579]]] +; CHECK-ODD: fmul [[y:s[0-9]*[02468]]] +; CHECK: fmadd [[x]] +; CHECK: fmadd [[y]] +; CHECK: fmsub [[x]] +; CHECK: fmadd [[y]] +; CHECK: fmadd [[x]] +; CHECK-A57: stp [[x]], [[y]] +; CHECK-A53-DAG: str [[x]] +; CHECK-A53-DAG: str [[y]] + +define void @f4(float* nocapture readonly %p, float* nocapture %q) #0 { +entry: + %0 = load float* %p, align 4 + %arrayidx1 = getelementptr inbounds float* %p, i64 1 + %1 = load float* %arrayidx1, align 4 + %arrayidx2 = getelementptr inbounds float* %p, i64 2 + %2 = load float* %arrayidx2, align 4 + %arrayidx3 = getelementptr inbounds float* %p, i64 3 + %3 = load float* %arrayidx3, align 4 + %arrayidx4 = getelementptr inbounds float* %p, i64 4 + %4 = load float* %arrayidx4, align 4 + %arrayidx5 = getelementptr inbounds float* %p, i64 5 + %5 = load float* %arrayidx5, align 4 + %arrayidx6 = getelementptr inbounds float* %p, i64 6 + %6 = load float* %arrayidx6, align 4 + %arrayidx7 = getelementptr inbounds float* %p, i64 7 + %7 = load float* %arrayidx7, align 4 + %mul = fmul fast float %0, %1 + %add = fadd fast float %mul, %7 + %mul8 = fmul fast float %5, %6 + %mul9 = fmul fast float %1, %2 + %add10 = fadd fast float %mul9, %add + %mul11 = fmul fast float %3, %4 + %add12 = fadd fast float %mul11, %mul8 + %mul13 = fmul fast float %1, %3 + %sub = fsub fast float %add10, %mul13 + %mul14 = fmul fast float %4, %5 + %add15 = fadd fast float %mul14, %add12 + %mul16 = fmul fast float %2, %3 + %add17 = fadd fast float %mul16, %sub + store float %add17, float* %q, align 4 + %arrayidx19 = getelementptr inbounds float* %q, i64 1 + store float %add15, float* %arrayidx19, align 4 + ret void +} + +; Single precision version of f3 + +; CHECK-LABEL: f5: +; CHECK-EVEN: fmadd [[x:s[0-9]*[02468]]] +; CHECK-ODD: fmadd [[x:s[0-9]*[13579]]] +; CHECK: fmadd [[x]] +; CHECK: fmsub [[x]] +; CHECK: fmadd [[y:s[0-9]+]], {{.*}}, [[x]] +; CHECK: str [[y]] + +define void @f5(float* nocapture readonly %p, float* nocapture %q) #0 { +entry: + %0 = load float* %p, align 4 + %arrayidx1 = getelementptr inbounds float* %p, i64 1 + %1 = load float* %arrayidx1, align 4 + %arrayidx2 = getelementptr inbounds float* %p, i64 2 + %2 = load float* %arrayidx2, align 4 + %arrayidx3 = getelementptr inbounds float* %p, i64 3 + %3 = load float* %arrayidx3, align 4 + %arrayidx4 = getelementptr inbounds float* %p, i64 4 + %4 = load float* %arrayidx4, align 4 + %mul = fmul fast float %0, %1 + %add = fadd fast float %mul, %4 + %mul5 = fmul fast float %1, %2 + %add6 = fadd fast float %mul5, %add + %mul7 = fmul fast float %1, %3 + %sub = fsub fast float %add6, %mul7 + %mul8 = fmul fast float %2, %3 + %add9 = fadd fast float %mul8, %sub + %cmp = fcmp oeq float %3, 0.000000e+00 + br i1 %cmp, label %if.then, label %if.end + +if.then: ; preds = %entry + tail call void bitcast (void (...)* @g to void ()*)() #2 + br label %if.end + +if.end: ; preds = %if.then, %entry + store float %add9, float* %q, align 4 + ret void +} + +; Test that regmask clobbering stops a chain sequence. + +; CHECK-LABEL: f6: +; CHECK-EVEN: fmadd [[x:d[0-9]*[02468]]] +; CHECK-ODD: fmadd [[x:d[0-9]*[13579]]] +; CHECK: fmadd [[x]] +; CHECK: fmsub [[x]] +; CHECK: fmadd d0, {{.*}}, [[x]] +; CHECK: bl hh +; CHECK: str d0 + +define void @f6(double* nocapture readonly %p, double* nocapture %q) #0 { +entry: + %0 = load double* %p, align 8 + %arrayidx1 = getelementptr inbounds double* %p, i64 1 + %1 = load double* %arrayidx1, align 8 + %arrayidx2 = getelementptr inbounds double* %p, i64 2 + %2 = load double* %arrayidx2, align 8 + %arrayidx3 = getelementptr inbounds double* %p, i64 3 + %3 = load double* %arrayidx3, align 8 + %arrayidx4 = getelementptr inbounds double* %p, i64 4 + %4 = load double* %arrayidx4, align 8 + %mul = fmul fast double %0, %1 + %add = fadd fast double %mul, %4 + %mul5 = fmul fast double %1, %2 + %add6 = fadd fast double %mul5, %add + %mul7 = fmul fast double %1, %3 + %sub = fsub fast double %add6, %mul7 + %mul8 = fmul fast double %2, %3 + %add9 = fadd fast double %mul8, %sub + %call = tail call double @hh(double %add9) #2 + store double %call, double* %q, align 8 + ret void +} + +declare double @hh(double) #1 + +; Check that we correctly deal with repeated operands. +; The following testcase creates: +; %D1 = FADDDrr %D0, %D0 +; We'll get a crash if we naively look at the first operand, remove it +; from the substitution list then look at the second operand. + +; CHECK: fmadd [[x:d[0-9]+]] +; CHECK: fadd d1, [[x]], [[x]] + +define void @f7(double* nocapture readonly %p, double* nocapture %q) #0 { +entry: + %0 = load double* %p, align 8 + %arrayidx1 = getelementptr inbounds double* %p, i64 1 + %1 = load double* %arrayidx1, align 8 + %arrayidx2 = getelementptr inbounds double* %p, i64 2 + %2 = load double* %arrayidx2, align 8 + %arrayidx3 = getelementptr inbounds double* %p, i64 3 + %3 = load double* %arrayidx3, align 8 + %arrayidx4 = getelementptr inbounds double* %p, i64 4 + %4 = load double* %arrayidx4, align 8 + %mul = fmul fast double %0, %1 + %add = fadd fast double %mul, %4 + %mul5 = fmul fast double %1, %2 + %add6 = fadd fast double %mul5, %add + %mul7 = fmul fast double %1, %3 + %sub = fsub fast double %add6, %mul7 + %mul8 = fmul fast double %2, %3 + %add9 = fadd fast double %mul8, %sub + %add10 = fadd fast double %add9, %add9 + call void @hhh(double 0.0, double %add10) + ret void +} + +declare void @hhh(double, double) + +attributes #0 = { nounwind "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="true" "use-soft-float"="false" } +attributes #1 = { "less-precise-fpmad"="false" "no-frame-pointer-elim"="true" "no-frame-pointer-elim-non-leaf" "no-infs-fp-math"="true" "no-nans-fp-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="true" "use-soft-float"="false" } +attributes #2 = { nounwind } +