Index: llvm/trunk/include/llvm/Transforms/Utils/CodeExtractor.h =================================================================== --- llvm/trunk/include/llvm/Transforms/Utils/CodeExtractor.h +++ llvm/trunk/include/llvm/Transforms/Utils/CodeExtractor.h @@ -151,6 +151,16 @@ BasicBlock *findOrCreateBlockForHoisting(BasicBlock *CommonExitBlock); private: + struct LifetimeMarkerInfo { + bool SinkLifeStart = false; + bool HoistLifeEnd = false; + Instruction *LifeStart = nullptr; + Instruction *LifeEnd = nullptr; + }; + + LifetimeMarkerInfo getLifetimeMarkers(Instruction *Addr, + BasicBlock *ExitBlock) const; + void severSplitPHINodesOfEntry(BasicBlock *&Header); void severSplitPHINodesOfExits(const SmallPtrSetImpl &Exits); void splitReturnBlocks(); Index: llvm/trunk/lib/Transforms/Utils/CodeExtractor.cpp =================================================================== --- llvm/trunk/lib/Transforms/Utils/CodeExtractor.cpp +++ llvm/trunk/lib/Transforms/Utils/CodeExtractor.cpp @@ -410,11 +410,74 @@ return CommonExitBlock; } +// Find the pair of life time markers for address 'Addr' that are either +// defined inside the outline region or can legally be shrinkwrapped into the +// outline region. If there are not other untracked uses of the address, return +// the pair of markers if found; otherwise return a pair of nullptr. +CodeExtractor::LifetimeMarkerInfo +CodeExtractor::getLifetimeMarkers(Instruction *Addr, + BasicBlock *ExitBlock) const { + LifetimeMarkerInfo Info; + + for (User *U : Addr->users()) { + IntrinsicInst *IntrInst = dyn_cast(U); + if (IntrInst) { + if (IntrInst->getIntrinsicID() == Intrinsic::lifetime_start) { + // Do not handle the case where Addr has multiple start markers. + if (Info.LifeStart) + return {}; + Info.LifeStart = IntrInst; + } + if (IntrInst->getIntrinsicID() == Intrinsic::lifetime_end) { + if (Info.LifeEnd) + return {}; + Info.LifeEnd = IntrInst; + } + continue; + } + // Find untracked uses of the address, bail. + if (!definedInRegion(Blocks, U)) + return {}; + } + + if (!Info.LifeStart || !Info.LifeEnd) + return {}; + + Info.SinkLifeStart = !definedInRegion(Blocks, Info.LifeStart); + Info.HoistLifeEnd = !definedInRegion(Blocks, Info.LifeEnd); + // Do legality check. + if ((Info.SinkLifeStart || Info.HoistLifeEnd) && + !isLegalToShrinkwrapLifetimeMarkers(Addr)) + return {}; + + // Check to see if we have a place to do hoisting, if not, bail. + if (Info.HoistLifeEnd && !ExitBlock) + return {}; + + return Info; +} + void CodeExtractor::findAllocas(ValueSet &SinkCands, ValueSet &HoistCands, BasicBlock *&ExitBlock) const { Function *Func = (*Blocks.begin())->getParent(); ExitBlock = getCommonExitBlock(Blocks); + auto moveOrIgnoreLifetimeMarkers = + [&](const LifetimeMarkerInfo &LMI) -> bool { + if (!LMI.LifeStart) + return false; + if (LMI.SinkLifeStart) { + LLVM_DEBUG(dbgs() << "Sinking lifetime.start: " << *LMI.LifeStart + << "\n"); + SinkCands.insert(LMI.LifeStart); + } + if (LMI.HoistLifeEnd) { + LLVM_DEBUG(dbgs() << "Hoisting lifetime.end: " << *LMI.LifeEnd << "\n"); + HoistCands.insert(LMI.LifeEnd); + } + return true; + }; + for (BasicBlock &BB : *Func) { if (Blocks.count(&BB)) continue; @@ -423,96 +486,51 @@ if (!AI) continue; - // Find the pair of life time markers for address 'Addr' that are either - // defined inside the outline region or can legally be shrinkwrapped into - // the outline region. If there are not other untracked uses of the - // address, return the pair of markers if found; otherwise return a pair - // of nullptr. - auto GetLifeTimeMarkers = - [&](Instruction *Addr, bool &SinkLifeStart, - bool &HoistLifeEnd) -> std::pair { - Instruction *LifeStart = nullptr, *LifeEnd = nullptr; - - for (User *U : Addr->users()) { - IntrinsicInst *IntrInst = dyn_cast(U); - if (IntrInst) { - if (IntrInst->getIntrinsicID() == Intrinsic::lifetime_start) { - // Do not handle the case where AI has multiple start markers. - if (LifeStart) - return std::make_pair(nullptr, nullptr); - LifeStart = IntrInst; - } - if (IntrInst->getIntrinsicID() == Intrinsic::lifetime_end) { - if (LifeEnd) - return std::make_pair(nullptr, nullptr); - LifeEnd = IntrInst; - } - continue; - } - // Find untracked uses of the address, bail. - if (!definedInRegion(Blocks, U)) - return std::make_pair(nullptr, nullptr); - } - - if (!LifeStart || !LifeEnd) - return std::make_pair(nullptr, nullptr); - - SinkLifeStart = !definedInRegion(Blocks, LifeStart); - HoistLifeEnd = !definedInRegion(Blocks, LifeEnd); - // Do legality Check. - if ((SinkLifeStart || HoistLifeEnd) && - !isLegalToShrinkwrapLifetimeMarkers(Addr)) - return std::make_pair(nullptr, nullptr); - - // Check to see if we have a place to do hoisting, if not, bail. - if (HoistLifeEnd && !ExitBlock) - return std::make_pair(nullptr, nullptr); - - return std::make_pair(LifeStart, LifeEnd); - }; - - bool SinkLifeStart = false, HoistLifeEnd = false; - auto Markers = GetLifeTimeMarkers(AI, SinkLifeStart, HoistLifeEnd); - - if (Markers.first) { - if (SinkLifeStart) - SinkCands.insert(Markers.first); + LifetimeMarkerInfo MarkerInfo = getLifetimeMarkers(AI, ExitBlock); + bool Moved = moveOrIgnoreLifetimeMarkers(MarkerInfo); + if (Moved) { + LLVM_DEBUG(dbgs() << "Sinking alloca: " << *AI << "\n"); SinkCands.insert(AI); - if (HoistLifeEnd) - HoistCands.insert(Markers.second); continue; } - // Follow the bitcast. - Instruction *MarkerAddr = nullptr; + // Follow any bitcasts. + SmallVector Bitcasts; + SmallVector BitcastLifetimeInfo; for (User *U : AI->users()) { if (U->stripInBoundsConstantOffsets() == AI) { - SinkLifeStart = false; - HoistLifeEnd = false; Instruction *Bitcast = cast(U); - Markers = GetLifeTimeMarkers(Bitcast, SinkLifeStart, HoistLifeEnd); - if (Markers.first) { - MarkerAddr = Bitcast; + LifetimeMarkerInfo LMI = getLifetimeMarkers(Bitcast, ExitBlock); + if (LMI.LifeStart) { + Bitcasts.push_back(Bitcast); + BitcastLifetimeInfo.push_back(LMI); continue; } } // Found unknown use of AI. if (!definedInRegion(Blocks, U)) { - MarkerAddr = nullptr; + Bitcasts.clear(); break; } } - if (MarkerAddr) { - if (SinkLifeStart) - SinkCands.insert(Markers.first); - if (!definedInRegion(Blocks, MarkerAddr)) - SinkCands.insert(MarkerAddr); - SinkCands.insert(AI); - if (HoistLifeEnd) - HoistCands.insert(Markers.second); + // Either no bitcasts reference the alloca or there are unknown uses. + if (Bitcasts.empty()) + continue; + + Instruction *BitcastAddr = Bitcasts.back(); + const LifetimeMarkerInfo &LMI = BitcastLifetimeInfo.back(); + assert(LMI.LifeStart && + "Unsafe to sink bitcast without lifetime markers"); + moveOrIgnoreLifetimeMarkers(LMI); + if (!definedInRegion(Blocks, BitcastAddr)) { + LLVM_DEBUG(dbgs() << "Sinking bitcast-of-alloca: " << *BitcastAddr + << "\n"); + SinkCands.insert(BitcastAddr); } + LLVM_DEBUG(dbgs() << "Sinking alloca (via bitcast): " << *AI << "\n"); + SinkCands.insert(AI); } } } @@ -1412,7 +1430,7 @@ // Find inputs to, outputs from the code region. findInputsOutputs(inputs, outputs, SinkingCands); - // Now sink all instructions which only have non-phi uses inside the region + // Now sink all instructions which only have non-phi uses inside the region. for (auto *II : SinkingCands) cast(II)->moveBefore(*newFuncRoot, newFuncRoot->getFirstInsertionPt());