Index: llvm/include/llvm/CodeGen/FunctionLoweringInfo.h =================================================================== --- llvm/include/llvm/CodeGen/FunctionLoweringInfo.h +++ llvm/include/llvm/CodeGen/FunctionLoweringInfo.h @@ -71,48 +71,6 @@ /// MBBMap - A mapping from LLVM basic blocks to their machine code entry. DenseMap MBBMap; - /// A map from swifterror value in a basic block to the virtual register it is - /// currently represented by. - DenseMap, unsigned> - SwiftErrorVRegDefMap; - - /// A list of upward exposed vreg uses that need to be satisfied by either a - /// copy def or a phi node at the beginning of the basic block representing - /// the predecessor(s) swifterror value. - DenseMap, unsigned> - SwiftErrorVRegUpwardsUse; - - /// A map from instructions that define/use a swifterror value to the virtual - /// register that represents that def/use. - llvm::DenseMap, unsigned> - SwiftErrorVRegDefUses; - - /// The swifterror argument of the current function. - const Value *SwiftErrorArg; - - using SwiftErrorValues = SmallVector; - /// A function can only have a single swifterror argument. And if it does - /// have a swifterror argument, it must be the first entry in - /// SwiftErrorVals. - SwiftErrorValues SwiftErrorVals; - - /// Get or create the swifterror value virtual register in - /// SwiftErrorVRegDefMap for this basic block. - unsigned getOrCreateSwiftErrorVReg(const MachineBasicBlock *, - const Value *); - - /// Set the swifterror virtual register in the SwiftErrorVRegDefMap for this - /// basic block. - void setCurrentSwiftErrorVReg(const MachineBasicBlock *MBB, const Value *, - unsigned); - - /// Get or create the swifterror value virtual register for a def of a - /// swifterror by an instruction. - std::pair getOrCreateSwiftErrorVRegDefAt(const Instruction *); - std::pair - getOrCreateSwiftErrorVRegUseAt(const Instruction *, const MachineBasicBlock *, - const Value *); - /// ValueMap - Since we emit code for the function a basic block at a time, /// we must remember which virtual registers hold the values for /// cross-basic-block values. Index: llvm/include/llvm/CodeGen/SelectionDAGISel.h =================================================================== --- llvm/include/llvm/CodeGen/SelectionDAGISel.h +++ llvm/include/llvm/CodeGen/SelectionDAGISel.h @@ -34,6 +34,7 @@ class TargetLibraryInfo; class FunctionLoweringInfo; class ScheduleHazardRecognizer; + class SwiftErrorValueTracking; class GCFunctionInfo; class ScheduleDAGSDNodes; class LoadInst; @@ -45,6 +46,7 @@ TargetMachine &TM; const TargetLibraryInfo *LibInfo; FunctionLoweringInfo *FuncInfo; + SwiftErrorValueTracking *SwiftError; MachineFunction *MF; MachineRegisterInfo *RegInfo; SelectionDAG *CurDAG; Index: llvm/include/llvm/CodeGen/SwiftErrorValueTracking.h =================================================================== --- /dev/null +++ llvm/include/llvm/CodeGen/SwiftErrorValueTracking.h @@ -0,0 +1,109 @@ +//===- SwiftErrorValueTracking.h - Track swifterror VReg vals --*- C++ -*--===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This implements a limited mem2reg-like analysis to promote uses of function +// arguments and allocas marked with swiftalloc from memory into virtual +// registers tracked by this class. +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFTERRORVALUETRACKING_H +#define SWIFTERRORVALUETRACKING_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/DebugLoc.h" +#include +#include +#include + + +namespace llvm { + class Function; + class MachineBasicBlock; + class MachineFunction; + class MachineInstr; + class TargetInstrInfo; + class TargetLowering; + +class SwiftErrorValueTracking { + // Some useful objects to reduce the number of function arguments needed. + MachineFunction *MF; + const Function *Fn; + const TargetLowering *TLI; + const TargetInstrInfo *TII; + + /// A map from swifterror value in a basic block to the virtual register it is + /// currently represented by. + DenseMap, unsigned> + VRegDefMap; + + /// A list of upward exposed vreg uses that need to be satisfied by either a + /// copy def or a phi node at the beginning of the basic block representing + /// the predecessor(s) swifterror value. + DenseMap, unsigned> + VRegUpwardsUse; + + /// A map from instructions that define/use a swifterror value to the virtual + /// register that represents that def/use. + llvm::DenseMap, unsigned> + VRegDefUses; + + /// The swifterror argument of the current function. + const Value *SwiftErrorArg; + + using SwiftErrorValues = SmallVector; + /// A function can only have a single swifterror argument. And if it does + /// have a swifterror argument, it must be the first entry in + /// SwiftErrorVals. + SwiftErrorValues SwiftErrorVals; + +public: + /// Initialize data structures for specified new function. + void setFunction(MachineFunction &MF); + + /// Get the (unique) function argument that was marked swifterror, or nullptr + /// if this function has no swifterror args. + const Value *getFunctionArg() const { + return SwiftErrorArg; + } + + /// Get or create the swifterror value virtual register in + /// VRegDefMap for this basic block. + unsigned getOrCreateVReg(const MachineBasicBlock *, const Value *); + + /// Set the swifterror virtual register in the VRegDefMap for this + /// basic block. + void setCurrentVReg(const MachineBasicBlock *MBB, const Value *, unsigned); + + /// Get or create the swifterror value virtual register for a def of a + /// swifterror by an instruction. + unsigned getOrCreateVRegDefAt(const Instruction *, const MachineBasicBlock *, + const Value *); + + /// Get or create the swifterror value virtual register for a use of a + /// swifterror by an instruction. + unsigned getOrCreateVRegUseAt(const Instruction *, const MachineBasicBlock *, + const Value *); + + /// Create initial definitions of swifterror values in the entry block of the + /// current function. + bool createEntriesInEntryBlock(DebugLoc DbgLoc); + + /// Propagate assigned swifterror vregs through a function, synthesizing PHI + /// nodes when needed to maintain consistency. + void propagateVRegs(); + + void preassignVRegs(MachineBasicBlock *MBB, BasicBlock::const_iterator Begin, + BasicBlock::const_iterator End); +}; + +} + +#endif Index: llvm/lib/CodeGen/CMakeLists.txt =================================================================== --- llvm/lib/CodeGen/CMakeLists.txt +++ llvm/lib/CodeGen/CMakeLists.txt @@ -143,6 +143,7 @@ StackMaps.cpp StackProtector.cpp StackSlotColoring.cpp + SwiftErrorValueTracking.cpp TailDuplication.cpp TailDuplicator.cpp TargetFrameLoweringImpl.cpp Index: llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp +++ llvm/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp @@ -519,56 +519,6 @@ return VReg; } -unsigned -FunctionLoweringInfo::getOrCreateSwiftErrorVReg(const MachineBasicBlock *MBB, - const Value *Val) { - auto Key = std::make_pair(MBB, Val); - auto It = SwiftErrorVRegDefMap.find(Key); - // If this is the first use of this swifterror value in this basic block, - // create a new virtual register. - // After we processed all basic blocks we will satisfy this "upwards exposed - // use" by inserting a copy or phi at the beginning of this block. - if (It == SwiftErrorVRegDefMap.end()) { - auto &DL = MF->getDataLayout(); - const TargetRegisterClass *RC = TLI->getRegClassFor(TLI->getPointerTy(DL)); - auto VReg = MF->getRegInfo().createVirtualRegister(RC); - SwiftErrorVRegDefMap[Key] = VReg; - SwiftErrorVRegUpwardsUse[Key] = VReg; - return VReg; - } else return It->second; -} - -void FunctionLoweringInfo::setCurrentSwiftErrorVReg( - const MachineBasicBlock *MBB, const Value *Val, unsigned VReg) { - SwiftErrorVRegDefMap[std::make_pair(MBB, Val)] = VReg; -} - -std::pair -FunctionLoweringInfo::getOrCreateSwiftErrorVRegDefAt(const Instruction *I) { - auto Key = PointerIntPair(I, true); - auto It = SwiftErrorVRegDefUses.find(Key); - if (It == SwiftErrorVRegDefUses.end()) { - auto &DL = MF->getDataLayout(); - const TargetRegisterClass *RC = TLI->getRegClassFor(TLI->getPointerTy(DL)); - unsigned VReg = MF->getRegInfo().createVirtualRegister(RC); - SwiftErrorVRegDefUses[Key] = VReg; - return std::make_pair(VReg, true); - } - return std::make_pair(It->second, false); -} - -std::pair -FunctionLoweringInfo::getOrCreateSwiftErrorVRegUseAt(const Instruction *I, const MachineBasicBlock *MBB, const Value *Val) { - auto Key = PointerIntPair(I, false); - auto It = SwiftErrorVRegDefUses.find(Key); - if (It == SwiftErrorVRegDefUses.end()) { - unsigned VReg = getOrCreateSwiftErrorVReg(MBB, Val); - SwiftErrorVRegDefUses[Key] = VReg; - return std::make_pair(VReg, true); - } - return std::make_pair(It->second, false); -} - const Value * FunctionLoweringInfo::getValueFromVirtualReg(unsigned Vreg) { if (VirtReg2Value.empty()) { Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h =================================================================== --- llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -77,6 +77,7 @@ class ReturnInst; class SDDbgValue; class StoreInst; +class SwiftErrorValueTracking; class SwitchInst; class TargetLibraryInfo; class TargetMachine; @@ -613,6 +614,9 @@ /// Information about the function as a whole. FunctionLoweringInfo &FuncInfo; + /// Information about the swifterror values used throughout the function. + SwiftErrorValueTracking &SwiftError; + /// Garbage collection metadata for the function. GCFunctionInfo *GFI; @@ -626,9 +630,9 @@ LLVMContext *Context; SelectionDAGBuilder(SelectionDAG &dag, FunctionLoweringInfo &funcinfo, - CodeGenOpt::Level ol) - : SDNodeOrder(LowestSDNodeOrder), TM(dag.getTarget()), DAG(dag), - FuncInfo(funcinfo) {} + SwiftErrorValueTracking &swifterror, CodeGenOpt::Level ol) + : SDNodeOrder(LowestSDNodeOrder), TM(dag.getTarget()), DAG(dag), + FuncInfo(funcinfo), SwiftError(swifterror) {} void init(GCFunctionInfo *gfi, AliasAnalysis *AA, const TargetLibraryInfo *li); Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -54,6 +54,7 @@ #include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/CodeGen/SelectionDAGTargetInfo.h" #include "llvm/CodeGen/StackMaps.h" +#include "llvm/CodeGen/SwiftErrorValueTracking.h" #include "llvm/CodeGen/TargetFrameLowering.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetLowering.h" @@ -1895,7 +1896,7 @@ const Function *F = I.getParent()->getParent(); if (TLI.supportSwiftError() && F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) { - assert(FuncInfo.SwiftErrorArg && "Need a swift error argument"); + assert(SwiftError.getFunctionArg() && "Need a swift error argument"); ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); Flags.setSwiftError(); Outs.push_back(ISD::OutputArg(Flags, EVT(TLI.getPointerTy(DL)) /*vt*/, @@ -1904,8 +1905,8 @@ 0 /*partOffs*/)); // Create SDNode for the swifterror virtual register. OutVals.push_back( - DAG.getRegister(FuncInfo.getOrCreateSwiftErrorVRegUseAt( - &I, FuncInfo.MBB, FuncInfo.SwiftErrorArg).first, + DAG.getRegister(SwiftError.getOrCreateVRegUseAt( + &I, FuncInfo.MBB, SwiftError.getFunctionArg()), EVT(TLI.getPointerTy(DL)))); } @@ -4150,15 +4151,13 @@ SDValue Src = getValue(SrcV); // Create a virtual register, then update the virtual register. - unsigned VReg; bool CreatedVReg; - std::tie(VReg, CreatedVReg) = FuncInfo.getOrCreateSwiftErrorVRegDefAt(&I); + unsigned VReg = + SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand()); // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue // Chain can be getRoot or getControlRoot. SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg, SDValue(Src.getNode(), Src.getResNo())); DAG.setRoot(CopyNode); - if (CreatedVReg) - FuncInfo.setCurrentSwiftErrorVReg(FuncInfo.MBB, I.getOperand(1), VReg); } void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) { @@ -4191,8 +4190,7 @@ // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT SDValue L = DAG.getCopyFromReg( getRoot(), getCurSDLoc(), - FuncInfo.getOrCreateSwiftErrorVRegUseAt(&I, FuncInfo.MBB, SV).first, - ValueVTs[0]); + SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]); setValue(&I, L); } @@ -7059,11 +7057,9 @@ SwiftErrorVal = V; // We find the virtual register for the actual swifterror argument. // Instead of using the Value, we use the virtual register instead. - Entry.Node = DAG.getRegister(FuncInfo - .getOrCreateSwiftErrorVRegUseAt( - CS.getInstruction(), FuncInfo.MBB, V) - .first, - EVT(TLI.getPointerTy(DL))); + Entry.Node = DAG.getRegister( + SwiftError.getOrCreateVRegUseAt(CS.getInstruction(), FuncInfo.MBB, V), + EVT(TLI.getPointerTy(DL))); } Args.push_back(Entry); @@ -7104,13 +7100,9 @@ if (SwiftErrorVal && TLI.supportSwiftError()) { // Get the last element of InVals. SDValue Src = CLI.InVals.back(); - unsigned VReg; bool CreatedVReg; - std::tie(VReg, CreatedVReg) = - FuncInfo.getOrCreateSwiftErrorVRegDefAt(CS.getInstruction()); + unsigned VReg = SwiftError.getOrCreateVRegDefAt( + CS.getInstruction(), FuncInfo.MBB, SwiftErrorVal); SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src); - // We update the virtual register for the actual swifterror argument. - if (CreatedVReg) - FuncInfo.setCurrentSwiftErrorVReg(FuncInfo.MBB, SwiftErrorVal, VReg); DAG.setRoot(CopyNode); } } @@ -9747,8 +9739,8 @@ if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) { unsigned Reg = cast(Res.getOperand(1))->getReg(); if (TargetRegisterInfo::isVirtualRegister(Reg)) - FuncInfo->setCurrentSwiftErrorVReg(FuncInfo->MBB, - FuncInfo->SwiftErrorArg, Reg); + SwiftError->setCurrentVReg(FuncInfo->MBB, SwiftError->getFunctionArg(), + Reg); } // If this argument is live outside of the entry block, insert a copy from Index: llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp =================================================================== --- llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -49,6 +49,7 @@ #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/CodeGen/StackProtector.h" +#include "llvm/CodeGen/SwiftErrorValueTracking.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/TargetRegisterInfo.h" @@ -307,8 +308,9 @@ CodeGenOpt::Level OL) : MachineFunctionPass(ID), TM(tm), FuncInfo(new FunctionLoweringInfo()), + SwiftError(new SwiftErrorValueTracking()), CurDAG(new SelectionDAG(tm, OL)), - SDB(new SelectionDAGBuilder(*CurDAG, *FuncInfo, OL)), + SDB(new SelectionDAGBuilder(*CurDAG, *FuncInfo, *SwiftError, OL)), AA(), GFI(), OptLevel(OL), DAGSize(0) { @@ -324,6 +326,7 @@ delete SDB; delete CurDAG; delete FuncInfo; + delete SwiftError; } void SelectionDAGISel::getAnalysisUsage(AnalysisUsage &AU) const { @@ -446,6 +449,7 @@ CurDAG->init(*MF, *ORE, this, LibInfo, getAnalysisIfAvailable()); FuncInfo->set(Fn, *MF, CurDAG); + SwiftError->setFunction(*MF); // Now get the optional analyzes if we want to. // This is based on the possibly changed OptLevel (after optnone is taken @@ -1254,77 +1258,6 @@ !FuncInfo->isExportedInst(I); // Exported instrs must be computed. } -/// Set up SwiftErrorVals by going through the function. If the function has -/// swifterror argument, it will be the first entry. -static void setupSwiftErrorVals(const Function &Fn, const TargetLowering *TLI, - FunctionLoweringInfo *FuncInfo) { - if (!TLI->supportSwiftError()) - return; - - FuncInfo->SwiftErrorVals.clear(); - FuncInfo->SwiftErrorVRegDefMap.clear(); - FuncInfo->SwiftErrorVRegUpwardsUse.clear(); - FuncInfo->SwiftErrorVRegDefUses.clear(); - FuncInfo->SwiftErrorArg = nullptr; - - // Check if function has a swifterror argument. - bool HaveSeenSwiftErrorArg = false; - for (Function::const_arg_iterator AI = Fn.arg_begin(), AE = Fn.arg_end(); - AI != AE; ++AI) - if (AI->hasSwiftErrorAttr()) { - assert(!HaveSeenSwiftErrorArg && - "Must have only one swifterror parameter"); - (void)HaveSeenSwiftErrorArg; // silence warning. - HaveSeenSwiftErrorArg = true; - FuncInfo->SwiftErrorArg = &*AI; - FuncInfo->SwiftErrorVals.push_back(&*AI); - } - - for (const auto &LLVMBB : Fn) - for (const auto &Inst : LLVMBB) { - if (const AllocaInst *Alloca = dyn_cast(&Inst)) - if (Alloca->isSwiftError()) - FuncInfo->SwiftErrorVals.push_back(Alloca); - } -} - -static void createSwiftErrorEntriesInEntryBlock(FunctionLoweringInfo *FuncInfo, - FastISel *FastIS, - const TargetLowering *TLI, - const TargetInstrInfo *TII, - SelectionDAGBuilder *SDB) { - if (!TLI->supportSwiftError()) - return; - - // We only need to do this when we have swifterror parameter or swifterror - // alloc. - if (FuncInfo->SwiftErrorVals.empty()) - return; - - assert(FuncInfo->MBB == &*FuncInfo->MF->begin() && - "expected to insert into entry block"); - auto &DL = FuncInfo->MF->getDataLayout(); - auto const *RC = TLI->getRegClassFor(TLI->getPointerTy(DL)); - for (const auto *SwiftErrorVal : FuncInfo->SwiftErrorVals) { - // We will always generate a copy from the argument. It is always used at - // least by the 'return' of the swifterror. - if (FuncInfo->SwiftErrorArg && FuncInfo->SwiftErrorArg == SwiftErrorVal) - continue; - unsigned VReg = FuncInfo->MF->getRegInfo().createVirtualRegister(RC); - // Assign Undef to Vreg. We construct MI directly to make sure it works - // with FastISel. - BuildMI(*FuncInfo->MBB, FuncInfo->MBB->getFirstNonPHI(), - SDB->getCurDebugLoc(), TII->get(TargetOpcode::IMPLICIT_DEF), - VReg); - - // Keep FastIS informed about the value we just inserted. - if (FastIS) - FastIS->setLastLocalValue(&*std::prev(FuncInfo->InsertPt)); - - FuncInfo->setCurrentSwiftErrorVReg(FuncInfo->MBB, SwiftErrorVal, VReg); - } -} - /// Collect llvm.dbg.declare information. This is done after argument lowering /// in case the declarations refer to arguments. static void processDbgDeclares(FunctionLoweringInfo *FuncInfo) { @@ -1370,195 +1303,6 @@ } } -/// Propagate swifterror values through the machine function CFG. -static void propagateSwiftErrorVRegs(FunctionLoweringInfo *FuncInfo) { - auto *TLI = FuncInfo->TLI; - if (!TLI->supportSwiftError()) - return; - - // We only need to do this when we have swifterror parameter or swifterror - // alloc. - if (FuncInfo->SwiftErrorVals.empty()) - return; - - // For each machine basic block in reverse post order. - ReversePostOrderTraversal RPOT(FuncInfo->MF); - for (MachineBasicBlock *MBB : RPOT) { - // For each swifterror value in the function. - for(const auto *SwiftErrorVal : FuncInfo->SwiftErrorVals) { - auto Key = std::make_pair(MBB, SwiftErrorVal); - auto UUseIt = FuncInfo->SwiftErrorVRegUpwardsUse.find(Key); - auto VRegDefIt = FuncInfo->SwiftErrorVRegDefMap.find(Key); - bool UpwardsUse = UUseIt != FuncInfo->SwiftErrorVRegUpwardsUse.end(); - unsigned UUseVReg = UpwardsUse ? UUseIt->second : 0; - bool DownwardDef = VRegDefIt != FuncInfo->SwiftErrorVRegDefMap.end(); - assert(!(UpwardsUse && !DownwardDef) && - "We can't have an upwards use but no downwards def"); - - // If there is no upwards exposed use and an entry for the swifterror in - // the def map for this value we don't need to do anything: We already - // have a downward def for this basic block. - if (!UpwardsUse && DownwardDef) - continue; - - // Otherwise we either have an upwards exposed use vreg that we need to - // materialize or need to forward the downward def from predecessors. - - // Check whether we have a single vreg def from all predecessors. - // Otherwise we need a phi. - SmallVector, 4> VRegs; - SmallSet Visited; - for (auto *Pred : MBB->predecessors()) { - if (!Visited.insert(Pred).second) - continue; - VRegs.push_back(std::make_pair( - Pred, FuncInfo->getOrCreateSwiftErrorVReg(Pred, SwiftErrorVal))); - if (Pred != MBB) - continue; - // We have a self-edge. - // If there was no upwards use in this basic block there is now one: the - // phi needs to use it self. - if (!UpwardsUse) { - UpwardsUse = true; - UUseIt = FuncInfo->SwiftErrorVRegUpwardsUse.find(Key); - assert(UUseIt != FuncInfo->SwiftErrorVRegUpwardsUse.end()); - UUseVReg = UUseIt->second; - } - } - - // We need a phi node if we have more than one predecessor with different - // downward defs. - bool needPHI = - VRegs.size() >= 1 && - std::find_if( - VRegs.begin(), VRegs.end(), - [&](const std::pair &V) - -> bool { return V.second != VRegs[0].second; }) != - VRegs.end(); - - // If there is no upwards exposed used and we don't need a phi just - // forward the swifterror vreg from the predecessor(s). - if (!UpwardsUse && !needPHI) { - assert(!VRegs.empty() && - "No predecessors? The entry block should bail out earlier"); - // Just forward the swifterror vreg from the predecessor(s). - FuncInfo->setCurrentSwiftErrorVReg(MBB, SwiftErrorVal, VRegs[0].second); - continue; - } - - auto DLoc = isa(SwiftErrorVal) - ? cast(SwiftErrorVal)->getDebugLoc() - : DebugLoc(); - const auto *TII = FuncInfo->MF->getSubtarget().getInstrInfo(); - - // If we don't need a phi create a copy to the upward exposed vreg. - if (!needPHI) { - assert(UpwardsUse); - assert(!VRegs.empty() && - "No predecessors? Is the Calling Convention correct?"); - unsigned DestReg = UUseVReg; - BuildMI(*MBB, MBB->getFirstNonPHI(), DLoc, TII->get(TargetOpcode::COPY), - DestReg) - .addReg(VRegs[0].second); - continue; - } - - // We need a phi: if there is an upwards exposed use we already have a - // destination virtual register number otherwise we generate a new one. - auto &DL = FuncInfo->MF->getDataLayout(); - auto const *RC = TLI->getRegClassFor(TLI->getPointerTy(DL)); - unsigned PHIVReg = - UpwardsUse ? UUseVReg - : FuncInfo->MF->getRegInfo().createVirtualRegister(RC); - MachineInstrBuilder SwiftErrorPHI = - BuildMI(*MBB, MBB->getFirstNonPHI(), DLoc, - TII->get(TargetOpcode::PHI), PHIVReg); - for (auto BBRegPair : VRegs) { - SwiftErrorPHI.addReg(BBRegPair.second).addMBB(BBRegPair.first); - } - - // We did not have a definition in this block before: store the phi's vreg - // as this block downward exposed def. - if (!UpwardsUse) - FuncInfo->setCurrentSwiftErrorVReg(MBB, SwiftErrorVal, PHIVReg); - } - } -} - -static void preassignSwiftErrorRegs(const TargetLowering *TLI, - FunctionLoweringInfo *FuncInfo, - BasicBlock::const_iterator Begin, - BasicBlock::const_iterator End) { - if (!TLI->supportSwiftError() || FuncInfo->SwiftErrorVals.empty()) - return; - - // Iterator over instructions and assign vregs to swifterror defs and uses. - for (auto It = Begin; It != End; ++It) { - ImmutableCallSite CS(&*It); - if (CS) { - // A call-site with a swifterror argument is both use and def. - const Value *SwiftErrorAddr = nullptr; - for (auto &Arg : CS.args()) { - if (!Arg->isSwiftError()) - continue; - // Use of swifterror. - assert(!SwiftErrorAddr && "Cannot have multiple swifterror arguments"); - SwiftErrorAddr = &*Arg; - assert(SwiftErrorAddr->isSwiftError() && - "Must have a swifterror value argument"); - unsigned VReg; bool CreatedReg; - std::tie(VReg, CreatedReg) = FuncInfo->getOrCreateSwiftErrorVRegUseAt( - &*It, FuncInfo->MBB, SwiftErrorAddr); - assert(CreatedReg); - } - if (!SwiftErrorAddr) - continue; - - // Def of swifterror. - unsigned VReg; bool CreatedReg; - std::tie(VReg, CreatedReg) = - FuncInfo->getOrCreateSwiftErrorVRegDefAt(&*It); - assert(CreatedReg); - FuncInfo->setCurrentSwiftErrorVReg(FuncInfo->MBB, SwiftErrorAddr, VReg); - - // A load is a use. - } else if (const LoadInst *LI = dyn_cast(&*It)) { - const Value *V = LI->getOperand(0); - if (!V->isSwiftError()) - continue; - - unsigned VReg; bool CreatedReg; - std::tie(VReg, CreatedReg) = - FuncInfo->getOrCreateSwiftErrorVRegUseAt(LI, FuncInfo->MBB, V); - assert(CreatedReg); - - // A store is a def. - } else if (const StoreInst *SI = dyn_cast(&*It)) { - const Value *SwiftErrorAddr = SI->getOperand(1); - if (!SwiftErrorAddr->isSwiftError()) - continue; - - // Def of swifterror. - unsigned VReg; bool CreatedReg; - std::tie(VReg, CreatedReg) = - FuncInfo->getOrCreateSwiftErrorVRegDefAt(&*It); - assert(CreatedReg); - FuncInfo->setCurrentSwiftErrorVReg(FuncInfo->MBB, SwiftErrorAddr, VReg); - - // A return in a swiferror returning function is a use. - } else if (const ReturnInst *R = dyn_cast(&*It)) { - const Function *F = R->getParent()->getParent(); - if(!F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) - continue; - - unsigned VReg; bool CreatedReg; - std::tie(VReg, CreatedReg) = FuncInfo->getOrCreateSwiftErrorVRegUseAt( - R, FuncInfo->MBB, FuncInfo->SwiftErrorArg); - assert(CreatedReg); - } - } -} - void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { FastISelFailed = false; // Initialize the Fast-ISel state, if needed. @@ -1568,8 +1312,6 @@ FastIS = TLI->createFastISel(*FuncInfo, LibInfo); } - setupSwiftErrorVals(Fn, TLI, FuncInfo); - ReversePostOrderTraversal RPOT(&Fn); // Lower arguments up front. An RPO iteration always visits the entry block @@ -1615,7 +1357,11 @@ else FastIS->setLastLocalValue(nullptr); } - createSwiftErrorEntriesInEntryBlock(FuncInfo, FastIS, TLI, TII, SDB); + + bool Inserted = SwiftError->createEntriesInEntryBlock(SDB->getCurDebugLoc()); + + if (FastIS && Inserted) + FastIS->setLastLocalValue(&*std::prev(FuncInfo->InsertPt)); processDbgDeclares(FuncInfo); @@ -1670,7 +1416,7 @@ unsigned NumFastIselRemaining = std::distance(Begin, End); // Pre-assign swifterror vregs. - preassignSwiftErrorRegs(TLI, FuncInfo, Begin, End); + SwiftError->preassignVRegs(FuncInfo->MBB, Begin, End); // Do FastISel on as many instructions as possible. for (; BI != Begin; --BI) { @@ -1826,7 +1572,7 @@ SP.copyToMachineFrameInfo(MF->getFrameInfo()); - propagateSwiftErrorVRegs(FuncInfo); + SwiftError->propagateVRegs(); delete FastIS; SDB->clearDanglingDebugInfo(); Index: llvm/lib/CodeGen/SwiftErrorValueTracking.cpp =================================================================== --- /dev/null +++ llvm/lib/CodeGen/SwiftErrorValueTracking.cpp @@ -0,0 +1,312 @@ +//===-- SwiftErrorValueTracking.cpp --------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This implements a limited mem2reg-like analysis to promote uses of function +// arguments and allocas marked with swiftalloc from memory into virtual +// registers tracked by this class. +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/SwiftErrorValueTracking.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/TargetInstrInfo.h" +#include "llvm/CodeGen/TargetLowering.h" +#include "llvm/IR/Value.h" + +using namespace llvm; + +unsigned SwiftErrorValueTracking::getOrCreateVReg(const MachineBasicBlock *MBB, + const Value *Val) { + auto Key = std::make_pair(MBB, Val); + auto It = VRegDefMap.find(Key); + // If this is the first use of this swifterror value in this basic block, + // create a new virtual register. + // After we processed all basic blocks we will satisfy this "upwards exposed + // use" by inserting a copy or phi at the beginning of this block. + if (It == VRegDefMap.end()) { + auto &DL = MF->getDataLayout(); + const TargetRegisterClass *RC = TLI->getRegClassFor(TLI->getPointerTy(DL)); + auto VReg = MF->getRegInfo().createVirtualRegister(RC); + VRegDefMap[Key] = VReg; + VRegUpwardsUse[Key] = VReg; + return VReg; + } else + return It->second; +} + +void SwiftErrorValueTracking::setCurrentVReg(const MachineBasicBlock *MBB, + const Value *Val, unsigned VReg) { + VRegDefMap[std::make_pair(MBB, Val)] = VReg; +} + +unsigned SwiftErrorValueTracking::getOrCreateVRegDefAt( + const Instruction *I, const MachineBasicBlock *MBB, const Value *Val) { + auto Key = PointerIntPair(I, true); + auto It = VRegDefUses.find(Key); + if (It != VRegDefUses.end()) + return It->second; + + auto &DL = MF->getDataLayout(); + const TargetRegisterClass *RC = TLI->getRegClassFor(TLI->getPointerTy(DL)); + unsigned VReg = MF->getRegInfo().createVirtualRegister(RC); + VRegDefUses[Key] = VReg; + setCurrentVReg(MBB, Val, VReg); + return VReg; +} + +unsigned SwiftErrorValueTracking::getOrCreateVRegUseAt( + const Instruction *I, const MachineBasicBlock *MBB, const Value *Val) { + auto Key = PointerIntPair(I, false); + auto It = VRegDefUses.find(Key); + if (It != VRegDefUses.end()) + return It->second; + + unsigned VReg = getOrCreateVReg(MBB, Val); + VRegDefUses[Key] = VReg; + return VReg; +} + +/// Set up SwiftErrorVals by going through the function. If the function has +/// swifterror argument, it will be the first entry. +void SwiftErrorValueTracking::setFunction(MachineFunction &mf) { + MF = &mf; + Fn = &MF->getFunction(); + TLI = MF->getSubtarget().getTargetLowering(); + TII = MF->getSubtarget().getInstrInfo(); + + if (!TLI->supportSwiftError()) + return; + + SwiftErrorVals.clear(); + VRegDefMap.clear(); + VRegUpwardsUse.clear(); + VRegDefUses.clear(); + SwiftErrorArg = nullptr; + + // Check if function has a swifterror argument. + bool HaveSeenSwiftErrorArg = false; + for (Function::const_arg_iterator AI = Fn->arg_begin(), AE = Fn->arg_end(); + AI != AE; ++AI) + if (AI->hasSwiftErrorAttr()) { + assert(!HaveSeenSwiftErrorArg && + "Must have only one swifterror parameter"); + (void)HaveSeenSwiftErrorArg; // silence warning. + HaveSeenSwiftErrorArg = true; + SwiftErrorArg = &*AI; + SwiftErrorVals.push_back(&*AI); + } + + for (const auto &LLVMBB : *Fn) + for (const auto &Inst : LLVMBB) { + if (const AllocaInst *Alloca = dyn_cast(&Inst)) + if (Alloca->isSwiftError()) + SwiftErrorVals.push_back(Alloca); + } +} + +bool SwiftErrorValueTracking::createEntriesInEntryBlock(DebugLoc DbgLoc) { + if (!TLI->supportSwiftError()) + return false; + + // We only need to do this when we have swifterror parameter or swifterror + // alloc. + if (SwiftErrorVals.empty()) + return false; + + MachineBasicBlock *MBB = &*MF->begin(); + auto &DL = MF->getDataLayout(); + auto const *RC = TLI->getRegClassFor(TLI->getPointerTy(DL)); + bool Inserted = false; + for (const auto *SwiftErrorVal : SwiftErrorVals) { + // We will always generate a copy from the argument. It is always used at + // least by the 'return' of the swifterror. + if (SwiftErrorArg && SwiftErrorArg == SwiftErrorVal) + continue; + unsigned VReg = MF->getRegInfo().createVirtualRegister(RC); + // Assign Undef to Vreg. We construct MI directly to make sure it works + // with FastISel. + BuildMI(*MBB, MBB->getFirstNonPHI(), DbgLoc, + TII->get(TargetOpcode::IMPLICIT_DEF), VReg); + + setCurrentVReg(MBB, SwiftErrorVal, VReg); + Inserted = true; + } + + return Inserted; +} + +/// Propagate swifterror values through the machine function CFG. +void SwiftErrorValueTracking::propagateVRegs() { + if (!TLI->supportSwiftError()) + return; + + // We only need to do this when we have swifterror parameter or swifterror + // alloc. + if (SwiftErrorVals.empty()) + return; + + // For each machine basic block in reverse post order. + ReversePostOrderTraversal RPOT(MF); + for (MachineBasicBlock *MBB : RPOT) { + // For each swifterror value in the function. + for (const auto *SwiftErrorVal : SwiftErrorVals) { + auto Key = std::make_pair(MBB, SwiftErrorVal); + auto UUseIt = VRegUpwardsUse.find(Key); + auto VRegDefIt = VRegDefMap.find(Key); + bool UpwardsUse = UUseIt != VRegUpwardsUse.end(); + unsigned UUseVReg = UpwardsUse ? UUseIt->second : 0; + bool DownwardDef = VRegDefIt != VRegDefMap.end(); + assert(!(UpwardsUse && !DownwardDef) && + "We can't have an upwards use but no downwards def"); + + // If there is no upwards exposed use and an entry for the swifterror in + // the def map for this value we don't need to do anything: We already + // have a downward def for this basic block. + if (!UpwardsUse && DownwardDef) + continue; + + // Otherwise we either have an upwards exposed use vreg that we need to + // materialize or need to forward the downward def from predecessors. + + // Check whether we have a single vreg def from all predecessors. + // Otherwise we need a phi. + SmallVector, 4> VRegs; + SmallSet Visited; + for (auto *Pred : MBB->predecessors()) { + if (!Visited.insert(Pred).second) + continue; + VRegs.push_back(std::make_pair( + Pred, getOrCreateVReg(Pred, SwiftErrorVal))); + if (Pred != MBB) + continue; + // We have a self-edge. + // If there was no upwards use in this basic block there is now one: the + // phi needs to use it self. + if (!UpwardsUse) { + UpwardsUse = true; + UUseIt = VRegUpwardsUse.find(Key); + assert(UUseIt != VRegUpwardsUse.end()); + UUseVReg = UUseIt->second; + } + } + + // We need a phi node if we have more than one predecessor with different + // downward defs. + bool needPHI = + VRegs.size() >= 1 && + std::find_if( + VRegs.begin(), VRegs.end(), + [&](const std::pair &V) + -> bool { return V.second != VRegs[0].second; }) != + VRegs.end(); + + // If there is no upwards exposed used and we don't need a phi just + // forward the swifterror vreg from the predecessor(s). + if (!UpwardsUse && !needPHI) { + assert(!VRegs.empty() && + "No predecessors? The entry block should bail out earlier"); + // Just forward the swifterror vreg from the predecessor(s). + setCurrentVReg(MBB, SwiftErrorVal, VRegs[0].second); + continue; + } + + auto DLoc = isa(SwiftErrorVal) + ? cast(SwiftErrorVal)->getDebugLoc() + : DebugLoc(); + const auto *TII = MF->getSubtarget().getInstrInfo(); + + // If we don't need a phi create a copy to the upward exposed vreg. + if (!needPHI) { + assert(UpwardsUse); + assert(!VRegs.empty() && + "No predecessors? Is the Calling Convention correct?"); + unsigned DestReg = UUseVReg; + BuildMI(*MBB, MBB->getFirstNonPHI(), DLoc, TII->get(TargetOpcode::COPY), + DestReg) + .addReg(VRegs[0].second); + continue; + } + + // We need a phi: if there is an upwards exposed use we already have a + // destination virtual register number otherwise we generate a new one. + auto &DL = MF->getDataLayout(); + auto const *RC = TLI->getRegClassFor(TLI->getPointerTy(DL)); + unsigned PHIVReg = + UpwardsUse ? UUseVReg : MF->getRegInfo().createVirtualRegister(RC); + MachineInstrBuilder PHI = + BuildMI(*MBB, MBB->getFirstNonPHI(), DLoc, + TII->get(TargetOpcode::PHI), PHIVReg); + for (auto BBRegPair : VRegs) { + PHI.addReg(BBRegPair.second).addMBB(BBRegPair.first); + } + + // We did not have a definition in this block before: store the phi's vreg + // as this block downward exposed def. + if (!UpwardsUse) + setCurrentVReg(MBB, SwiftErrorVal, PHIVReg); + } + } +} + +void SwiftErrorValueTracking::preassignVRegs( + MachineBasicBlock *MBB, BasicBlock::const_iterator Begin, + BasicBlock::const_iterator End) { + if (!TLI->supportSwiftError() || SwiftErrorVals.empty()) + return; + + // Iterator over instructions and assign vregs to swifterror defs and uses. + for (auto It = Begin; It != End; ++It) { + ImmutableCallSite CS(&*It); + if (CS) { + // A call-site with a swifterror argument is both use and def. + const Value *SwiftErrorAddr = nullptr; + for (auto &Arg : CS.args()) { + if (!Arg->isSwiftError()) + continue; + // Use of swifterror. + assert(!SwiftErrorAddr && "Cannot have multiple swifterror arguments"); + SwiftErrorAddr = &*Arg; + assert(SwiftErrorAddr->isSwiftError() && + "Must have a swifterror value argument"); + getOrCreateVRegUseAt(&*It, MBB, SwiftErrorAddr); + } + if (!SwiftErrorAddr) + continue; + + // Def of swifterror. + getOrCreateVRegDefAt(&*It, MBB, SwiftErrorAddr); + + // A load is a use. + } else if (const LoadInst *LI = dyn_cast(&*It)) { + const Value *V = LI->getOperand(0); + if (!V->isSwiftError()) + continue; + + getOrCreateVRegUseAt(LI, MBB, V); + + // A store is a def. + } else if (const StoreInst *SI = dyn_cast(&*It)) { + const Value *SwiftErrorAddr = SI->getOperand(1); + if (!SwiftErrorAddr->isSwiftError()) + continue; + + // Def of swifterror. + getOrCreateVRegDefAt(&*It, MBB, SwiftErrorAddr); + + // A return in a swiferror returning function is a use. + } else if (const ReturnInst *R = dyn_cast(&*It)) { + const Function *F = R->getParent()->getParent(); + if (!F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) + continue; + + getOrCreateVRegUseAt(R, MBB, SwiftErrorArg); + } + } +}