diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -1155,6 +1155,8 @@ }; using ElementCountSet = SmallSet; +using InstructionVFPair = std::pair; + /// LoopVectorizationCostModel - estimates the expected speedups due to /// vectorization. /// In many cases vectorization is not profitable. This can happen because of @@ -1657,7 +1659,6 @@ /// the factor width. If \p Invalid is not nullptr, this function /// will add a pair(Instruction*, ElementCount) to \p Invalid for /// each instruction that has an Invalid cost for the given VF. - using InstructionVFPair = std::pair; VectorizationCostTy expectedCost(ElementCount VF, SmallVectorImpl *Invalid = nullptr); @@ -5362,6 +5363,70 @@ return (CostA * EstimatedWidthB) < (CostB * EstimatedWidthA); } +static void emitInvalidCostRemarks(SmallVector InvalidCosts, + OptimizationRemarkEmitter *ORE, + Loop *TheLoop) { + if (InvalidCosts.empty()) + return; + + // Emit a report of VFs with invalid costs in the loop. + + // Group the remarks per instruction, keeping the instruction order from + // InvalidCosts. + std::map Numbering; + unsigned I = 0; + for (auto &Pair : InvalidCosts) + if (!Numbering.count(Pair.first)) + Numbering[Pair.first] = I++; + + // Sort the list, first on instruction(number) then on VF. + sort(InvalidCosts, [&Numbering](InstructionVFPair &A, InstructionVFPair &B) { + if (Numbering[A.first] != Numbering[B.first]) + return Numbering[A.first] < Numbering[B.first]; + ElementCountComparator ECC; + return ECC(A.second, B.second); + }); + + // For a list of ordered instruction-vf pairs: + // [(load, vf1), (load, vf2), (store, vf1)] + // Group the instructions together to emit separate remarks for: + // load (vf1, vf2) + // store (vf1) + auto Tail = ArrayRef(InvalidCosts); + auto Subset = ArrayRef(); + do { + if (Subset.empty()) + Subset = Tail.take_front(1); + + Instruction *I = Subset.front().first; + + // If the next instruction is different, or if there are no other pairs, + // emit a remark for the collated subset. e.g. + // [(load, vf1), (load, vf2))] + // to emit: + // remark: invalid costs for 'load' at VF=(vf, vf2) + if (Subset == Tail || Tail[Subset.size()].first != I) { + std::string OutString; + raw_string_ostream OS(OutString); + assert(!Subset.empty() && "Unexpected empty range"); + OS << "Instruction with invalid costs prevented vectorization at VF=("; + for (const auto &Pair : Subset) + OS << (Pair.second == Subset.front().second ? "" : ", ") << Pair.second; + OS << "):"; + if (auto *CI = dyn_cast(I)) + OS << " call to " << CI->getCalledFunction()->getName(); + else + OS << " " << I->getOpcodeName(); + OS.flush(); + reportVectorizationInfo(OutString, "InvalidCost", ORE, TheLoop, I); + Tail = Tail.drop_front(Subset.size()); + Subset = {}; + } else + // Grow the subset by one element + Subset = Tail.take_front(Subset.size() + 1); + } while (!Tail.empty()); +} + VectorizationFactor LoopVectorizationCostModel::selectVectorizationFactor( const ElementCountSet &VFCandidates) { InstructionCost ExpectedCost = expectedCost(ElementCount::getFixed(1)).first; @@ -5422,65 +5487,7 @@ ChosenFactor = Candidate; } - // Emit a report of VFs with invalid costs in the loop. - if (!InvalidCosts.empty()) { - // Group the remarks per instruction, keeping the instruction order from - // InvalidCosts. - std::map Numbering; - unsigned I = 0; - for (auto &Pair : InvalidCosts) - if (!Numbering.count(Pair.first)) - Numbering[Pair.first] = I++; - - // Sort the list, first on instruction(number) then on VF. - llvm::sort(InvalidCosts, - [&Numbering](InstructionVFPair &A, InstructionVFPair &B) { - if (Numbering[A.first] != Numbering[B.first]) - return Numbering[A.first] < Numbering[B.first]; - ElementCountComparator ECC; - return ECC(A.second, B.second); - }); - - // For a list of ordered instruction-vf pairs: - // [(load, vf1), (load, vf2), (store, vf1)] - // Group the instructions together to emit separate remarks for: - // load (vf1, vf2) - // store (vf1) - auto Tail = ArrayRef(InvalidCosts); - auto Subset = ArrayRef(); - do { - if (Subset.empty()) - Subset = Tail.take_front(1); - - Instruction *I = Subset.front().first; - - // If the next instruction is different, or if there are no other pairs, - // emit a remark for the collated subset. e.g. - // [(load, vf1), (load, vf2))] - // to emit: - // remark: invalid costs for 'load' at VF=(vf, vf2) - if (Subset == Tail || Tail[Subset.size()].first != I) { - std::string OutString; - raw_string_ostream OS(OutString); - assert(!Subset.empty() && "Unexpected empty range"); - OS << "Instruction with invalid costs prevented vectorization at VF=("; - for (const auto &Pair : Subset) - OS << (Pair.second == Subset.front().second ? "" : ", ") - << Pair.second; - OS << "):"; - if (auto *CI = dyn_cast(I)) - OS << " call to " << CI->getCalledFunction()->getName(); - else - OS << " " << I->getOpcodeName(); - OS.flush(); - reportVectorizationInfo(OutString, "InvalidCost", ORE, TheLoop, I); - Tail = Tail.drop_front(Subset.size()); - Subset = {}; - } else - // Grow the subset by one element - Subset = Tail.take_front(Subset.size() + 1); - } while (!Tail.empty()); - } + emitInvalidCostRemarks(InvalidCosts, ORE, TheLoop); if (!EnableCondStoresVectorization && NumPredStores) { reportVectorizationFailure("There are conditional stores.",