Index: llvm/include/llvm/Analysis/TargetTransformInfo.h =================================================================== --- llvm/include/llvm/Analysis/TargetTransformInfo.h +++ llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -30,6 +30,7 @@ #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/ScalarEvolution.h" #include "llvm/IR/Dominators.h" +#include "llvm/Analysis/AssumptionCache.h" #include namespace llvm { @@ -525,6 +526,12 @@ /// calculation for the instructions in a loop. bool canMacroFuseCmp() const; + /// Return true if the target can save a compare for loop count, for example + /// hardware loop saves a compare. + bool canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE, LoopInfo *LI, + DominatorTree *DT, AssumptionCache *AC, + TargetLibraryInfo *LibInfo) const; + /// \return True is LSR should make efforts to create/preserve post-inc /// addressing mode expressions. bool shouldFavorPostInc() const; @@ -1149,6 +1156,9 @@ virtual bool isLSRCostLess(TargetTransformInfo::LSRCost &C1, TargetTransformInfo::LSRCost &C2) = 0; virtual bool canMacroFuseCmp() = 0; + virtual bool canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE, + LoopInfo *LI, DominatorTree *DT, AssumptionCache *AC, + TargetLibraryInfo *LibInfo) = 0; virtual bool shouldFavorPostInc() const = 0; virtual bool shouldFavorBackedgeIndex(const Loop *L) const = 0; virtual bool isLegalMaskedStore(Type *DataType) = 0; @@ -1399,6 +1409,13 @@ bool canMacroFuseCmp() override { return Impl.canMacroFuseCmp(); } + + bool canSaveCmp(Loop *L, BranchInst **BI, + ScalarEvolution *SE, + LoopInfo *LI, DominatorTree *DT, AssumptionCache *AC, + TargetLibraryInfo *LibInfo) override { + return Impl.canSaveCmp(L, BI, SE, LI, DT, AC, LibInfo); + } bool shouldFavorPostInc() const override { return Impl.shouldFavorPostInc(); } Index: llvm/include/llvm/Analysis/TargetTransformInfoImpl.h =================================================================== --- llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -221,6 +221,12 @@ bool canMacroFuseCmp() { return false; } + bool canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE, LoopInfo *LI, + DominatorTree *DT, AssumptionCache *AC, + TargetLibraryInfo *LibInfo) { + return false; + } + bool shouldFavorPostInc() const { return false; } bool shouldFavorBackedgeIndex(const Loop *L) const { return false; } Index: llvm/lib/Analysis/TargetTransformInfo.cpp =================================================================== --- llvm/lib/Analysis/TargetTransformInfo.cpp +++ llvm/lib/Analysis/TargetTransformInfo.cpp @@ -264,6 +264,13 @@ return TTIImpl->canMacroFuseCmp(); } +bool TargetTransformInfo::canSaveCmp(Loop *L, BranchInst **BI, + ScalarEvolution *SE, LoopInfo *LI, + DominatorTree *DT, AssumptionCache *AC, + TargetLibraryInfo *LibInfo) const { + return TTIImpl->canSaveCmp(L, BI, SE, LI, DT, AC, LibInfo); +} + bool TargetTransformInfo::shouldFavorPostInc() const { return TTIImpl->shouldFavorPostInc(); } Index: llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h =================================================================== --- llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h +++ llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h @@ -57,6 +57,9 @@ AssumptionCache &AC, TargetLibraryInfo *LibInfo, HardwareLoopInfo &HWLoopInfo); + bool canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE, LoopInfo *LI, + DominatorTree *DT, AssumptionCache *AC, + TargetLibraryInfo *LibInfo); void getUnrollingPreferences(Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP); Index: llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp =================================================================== --- llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp +++ llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp @@ -875,3 +875,25 @@ return Cost; } +bool PPCTTIImpl::canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE, + LoopInfo *LI, DominatorTree *DT, + AssumptionCache *AC, TargetLibraryInfo *LibInfo) { + // Process nested loops first. + for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I) + if (canSaveCmp(*I, BI, SE, LI, DT, AC, LibInfo)) + return false; // Stop search. + + HardwareLoopInfo HWLoopInfo(L); + + if (!HWLoopInfo.canAnalyze(*LI)) + return false; + + if (!isHardwareLoopProfitable(L, *SE, *AC, LibInfo, HWLoopInfo)) + return false; + + if (!HWLoopInfo.isHardwareLoopCandidate(*SE, *LI, *DT)) + return false; + + *BI = HWLoopInfo.ExitBranch; + return true; +} Index: llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp =================================================================== --- llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp +++ llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp @@ -1909,6 +1909,8 @@ ScalarEvolution &SE; DominatorTree &DT; LoopInfo &LI; + AssumptionCache &AC; + TargetLibraryInfo &LibInfo; const TargetTransformInfo &TTI; Loop *const L; bool FavorBackedgeIndex = false; @@ -2047,7 +2049,8 @@ public: LSRInstance(Loop *L, IVUsers &IU, ScalarEvolution &SE, DominatorTree &DT, - LoopInfo &LI, const TargetTransformInfo &TTI); + LoopInfo &LI, const TargetTransformInfo &TTI, AssumptionCache &AC, + TargetLibraryInfo &LibInfo); bool getChanged() const { return Changed; } @@ -3232,6 +3235,9 @@ } void LSRInstance::CollectFixupsAndInitialFormulae() { + BranchInst *ExitBranch = nullptr; + bool SaveCmp = TTI.canSaveCmp(L, &ExitBranch, &SE, &LI, &DT, &AC, &LibInfo); + for (const IVStrideUse &U : IU) { Instruction *UserInst = U.getUser(); // Skip IV users that are part of profitable IV Chains. @@ -3261,6 +3267,10 @@ // equality icmps, thanks to IndVarSimplify. if (ICmpInst *CI = dyn_cast(UserInst)) if (CI->isEquality()) { + // If CI can be saved in some target, like replaced inside hardware loop + // in PowerPC, no need to generate initial formulae for it. + if (SaveCmp && CI == cast(ExitBranch->getCondition())) + continue; // Swap the operands if needed to put the OperandValToReplace on the // left, for consistency. Value *NV = CI->getOperand(1); @@ -5479,8 +5489,9 @@ LSRInstance::LSRInstance(Loop *L, IVUsers &IU, ScalarEvolution &SE, DominatorTree &DT, LoopInfo &LI, - const TargetTransformInfo &TTI) - : IU(IU), SE(SE), DT(DT), LI(LI), TTI(TTI), L(L), + const TargetTransformInfo &TTI, AssumptionCache &AC, + TargetLibraryInfo &LibInfo) + : IU(IU), SE(SE), DT(DT), LI(LI), AC(AC), LibInfo(LibInfo), TTI(TTI), L(L), FavorBackedgeIndex(EnableBackedgeIndexing && TTI.shouldFavorBackedgeIndex(L)) { // If LoopSimplify form is not available, stay out of trouble. @@ -5677,6 +5688,8 @@ AU.addPreserved(); AU.addRequired(); AU.addPreserved(); + AU.addRequired(); + AU.addRequired(); // Requiring LoopSimplify a second time here prevents IVUsers from running // twice, since LoopSimplify was invalidated by running ScalarEvolution. AU.addRequiredID(LoopSimplifyID); @@ -5687,11 +5700,14 @@ static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE, DominatorTree &DT, LoopInfo &LI, - const TargetTransformInfo &TTI) { + const TargetTransformInfo &TTI, + AssumptionCache &AC, + TargetLibraryInfo &LibInfo) { + bool Changed = false; // Run the main LSR transformation. - Changed |= LSRInstance(L, IU, SE, DT, LI, TTI).getChanged(); + Changed |= LSRInstance(L, IU, SE, DT, LI, TTI, AC, LibInfo).getChanged(); // Remove any extra phis created by processing inner loops. Changed |= DeleteDeadPHIs(L->getHeader()); @@ -5722,14 +5738,17 @@ auto &LI = getAnalysis().getLoopInfo(); const auto &TTI = getAnalysis().getTTI( *L->getHeader()->getParent()); - return ReduceLoopStrength(L, IU, SE, DT, LI, TTI); + auto &AC = getAnalysis().getAssumptionCache( + *L->getHeader()->getParent()); + auto &LibInfo = getAnalysis().getTLI(); + return ReduceLoopStrength(L, IU, SE, DT, LI, TTI, AC, LibInfo); } PreservedAnalyses LoopStrengthReducePass::run(Loop &L, LoopAnalysisManager &AM, LoopStandardAnalysisResults &AR, LPMUpdater &) { if (!ReduceLoopStrength(&L, AM.getResult(L, AR), AR.SE, - AR.DT, AR.LI, AR.TTI)) + AR.DT, AR.LI, AR.TTI, AR.AC, AR.TLI)) return PreservedAnalyses::all(); return getLoopPassPreservedAnalyses(); Index: llvm/test/CodeGen/PowerPC/addi-licm.ll =================================================================== --- llvm/test/CodeGen/PowerPC/addi-licm.ll +++ llvm/test/CodeGen/PowerPC/addi-licm.ll @@ -18,8 +18,8 @@ ; CHECK: addi [[REG1:[0-9]+]], 1, ; CHECK: addi [[REG2:[0-9]+]], 1, ; CHECK: %for.body.i -; CHECK-DAG: lfsx {{[0-9]+}}, [[REG1]], -; CHECK-DAG: lfsx {{[0-9]+}}, [[REG2]], +; CHECK-DAG: lfs {{[0-9]+}}, 0([[REG1]]) +; CHECK-DAG: lfs {{[0-9]+}}, 0([[REG2]]) ; CHECK: blr ; PIP-LABEL: @foo Index: llvm/test/CodeGen/PowerPC/ctrloop-ne.ll =================================================================== --- llvm/test/CodeGen/PowerPC/ctrloop-ne.ll +++ llvm/test/CodeGen/PowerPC/ctrloop-ne.ll @@ -32,8 +32,7 @@ ; CHECK: test_pos2_ir_ne -; FIXME: Support this loop! -; CHECK-NOT: bdnz +; CHECK: bdnz ; a < b define void @test_pos2_ir_ne(i8* nocapture %p, i32 %a, i32 %b) nounwind { entry: @@ -62,8 +61,7 @@ ; CHECK: test_pos4_ir_ne -; FIXME: Support this loop! -; CHECK-NOT: bdnz +; CHECK: bdnz ; a < b define void @test_pos4_ir_ne(i8* nocapture %p, i32 %a, i32 %b) nounwind { entry: @@ -92,8 +90,7 @@ ; CHECK: test_pos8_ir_ne -; FIXME: Support this loop! -; CHECK-NOT: bdnz +; CHECK: bdnz ; a < b define void @test_pos8_ir_ne(i8* nocapture %p, i32 %a, i32 %b) nounwind { entry: @@ -122,8 +119,7 @@ ; CHECK: test_pos16_ir_ne -; FIXME: Support this loop! -; CHECK-NOT: bdnz +; CHECK: bdnz ; a < b define void @test_pos16_ir_ne(i8* nocapture %p, i32 %a, i32 %b) nounwind { entry: @@ -181,8 +177,7 @@ ; CHECK: test_pos2_ri_ne -; FIXME: Support this loop! -; CHECK-NOT: bdnz +; CHECK: bdnz ; a < b define void @test_pos2_ri_ne(i8* nocapture %p, i32 %a, i32 %b) nounwind { entry: @@ -211,8 +206,7 @@ ; CHECK: test_pos4_ri_ne -; FIXME: Support this loop! -; CHECK-NOT: bdnz +; CHECK: bdnz ; a < b define void @test_pos4_ri_ne(i8* nocapture %p, i32 %a, i32 %b) nounwind { entry: @@ -241,8 +235,7 @@ ; CHECK: test_pos8_ri_ne -; FIXME: Support this loop! -; CHECK-NOT: bdnz +; CHECK: bdnz ; a < b define void @test_pos8_ri_ne(i8* nocapture %p, i32 %a, i32 %b) nounwind { entry: @@ -271,8 +264,7 @@ ; CHECK: test_pos16_ri_ne -; FIXME: Support this loop! -; CHECK-NOT: bdnz +; CHECK: bdnz ; a < b define void @test_pos16_ri_ne(i8* nocapture %p, i32 %a, i32 %b) nounwind { entry: @@ -330,8 +322,7 @@ ; CHECK: test_pos2_rr_ne -; FIXME: Support this loop! -; CHECK-NOT: bdnz +; CHECK: bdnz ; a < b define void @test_pos2_rr_ne(i8* nocapture %p, i32 %a, i32 %b) nounwind { entry: @@ -360,8 +351,7 @@ ; CHECK: test_pos4_rr_ne -; FIXME: Support this loop! -; CHECK-NOT: bdnz +; CHECK: bdnz ; a < b define void @test_pos4_rr_ne(i8* nocapture %p, i32 %a, i32 %b) nounwind { entry: @@ -390,8 +380,7 @@ ; CHECK: test_pos8_rr_ne -; FIXME: Support this loop! -; CHECK-NOT: bdnz +; CHECK: bdnz ; a < b define void @test_pos8_rr_ne(i8* nocapture %p, i32 %a, i32 %b) nounwind { entry: @@ -420,8 +409,7 @@ ; CHECK: test_pos16_rr_ne -; FIXME: Support this loop! -; CHECK-NOT: bdnz +; CHECK: bdnz ; a < b define void @test_pos16_rr_ne(i8* nocapture %p, i32 %a, i32 %b) nounwind { entry: Index: llvm/test/CodeGen/PowerPC/ctrloop-shortLoops.ll =================================================================== --- llvm/test/CodeGen/PowerPC/ctrloop-shortLoops.ll +++ llvm/test/CodeGen/PowerPC/ctrloop-shortLoops.ll @@ -86,10 +86,12 @@ } ; Function Attrs: norecurse nounwind +; On core a2q, IssueWidth is 1. On core pwr8, IssueWidth is 8. +; a2q should use mtctr, but pwr8 should not use mtctr. define signext i32 @testTripCount2NonSmallLoop() { ; CHECK-LABEL: testTripCount2NonSmallLoop: -; CHECK: blt -; CHECK: beq +; CHECK-A2Q: mtctr +; CHECK-PWR8-NOT: mtctr ; CHECK: blr entry: Index: llvm/test/CodeGen/PowerPC/lsr-ctrloop.ll =================================================================== --- llvm/test/CodeGen/PowerPC/lsr-ctrloop.ll +++ llvm/test/CodeGen/PowerPC/lsr-ctrloop.ll @@ -14,32 +14,31 @@ define void @foo(float* nocapture %data, float %d) { ; CHECK-LABEL: foo: ; CHECK: .LBB0_1: # %vector.body -; CHECK: add 5, 3, 4 -; CHECK-NEXT: stxvx 0, 3, 4 +; CHECK: stxv 0, -192(4) +; CHECK-NEXT: stxv 0, -176(4) +; CHECK-NEXT: stxv 0, -160(4) +; CHECK-NEXT: stxv 0, -144(4) +; CHECK-NEXT: stxv 0, -128(4) +; CHECK-NEXT: stxv 0, -112(4) +; CHECK-NEXT: stxv 0, -96(4) +; CHECK-NEXT: stxv 0, -80(4) +; CHECK-NEXT: stxv 0, -64(4) +; CHECK-NEXT: stxv 0, -48(4) +; CHECK-NEXT: stxv 0, -32(4) +; CHECK-NEXT: stxv 0, -16(4) +; CHECK-NEXT: stxv 0, 0(4) +; CHECK-NEXT: stxv 0, 16(4) +; CHECK-NEXT: stxv 0, 32(4) +; CHECK-NEXT: stxv 0, 48(4) +; CHECK-NEXT: stxv 0, 64(4) +; CHECK-NEXT: stxv 0, 80(4) +; CHECK-NEXT: stxv 0, 96(4) +; CHECK-NEXT: stxv 0, 112(4) +; CHECK-NEXT: stxv 0, 128(4) +; CHECK-NEXT: stxv 0, 144(4) +; CHECK-NEXT: stxv 0, 160(4) +; CHECK-NEXT: stxv 0, 176(4) ; CHECK-NEXT: addi 4, 4, 384 -; CHECK-NEXT: stxv 0, 16(5) -; CHECK-NEXT: stxv 0, 32(5) -; CHECK-NEXT: stxv 0, 48(5) -; CHECK-NEXT: stxv 0, 64(5) -; CHECK-NEXT: stxv 0, 80(5) -; CHECK-NEXT: stxv 0, 96(5) -; CHECK-NEXT: stxv 0, 112(5) -; CHECK-NEXT: stxv 0, 128(5) -; CHECK-NEXT: stxv 0, 144(5) -; CHECK-NEXT: stxv 0, 160(5) -; CHECK-NEXT: stxv 0, 176(5) -; CHECK-NEXT: stxv 0, 192(5) -; CHECK-NEXT: stxv 0, 208(5) -; CHECK-NEXT: stxv 0, 224(5) -; CHECK-NEXT: stxv 0, 240(5) -; CHECK-NEXT: stxv 0, 256(5) -; CHECK-NEXT: stxv 0, 272(5) -; CHECK-NEXT: stxv 0, 288(5) -; CHECK-NEXT: stxv 0, 304(5) -; CHECK-NEXT: stxv 0, 320(5) -; CHECK-NEXT: stxv 0, 336(5) -; CHECK-NEXT: stxv 0, 352(5) -; CHECK-NEXT: stxv 0, 368(5) ; CHECK-NEXT: bdnz .LBB0_1 entry: Index: llvm/test/CodeGen/PowerPC/negctr.ll =================================================================== --- llvm/test/CodeGen/PowerPC/negctr.ll +++ llvm/test/CodeGen/PowerPC/negctr.ll @@ -35,10 +35,14 @@ %exitcond = icmp eq i64 %indvars.iv.next, 0 br i1 %exitcond, label %for.end, label %for.body +; FIXME: This should be a hardware loop. +; cmp is optimized to uadd intrinsic in CGP pass which can not be recognized in +; later HardwareLoops Pass. ; CHECK: @main1 -; CHECK: li [[REG:[0-9]+]], -1 -; CHECK: mtctr [[REG]] -; CHECK: bdnz +; CHECK: li [[REG:[0-9]+]], 1 +; CHECK: addi [[REG2:[0-9]+]], [[REG]], 1 +; CHECK: cmpld +; CHECK: bge for.end: ; preds = %for.body, %entry ret void Index: llvm/test/CodeGen/PowerPC/stwu-sched.ll =================================================================== --- llvm/test/CodeGen/PowerPC/stwu-sched.ll +++ llvm/test/CodeGen/PowerPC/stwu-sched.ll @@ -1,9 +1,9 @@ ; RUN: llc -mcpu=pwr9 -mtriple=powerpc64-unknown-linux-gnu < %s -verify-machineinstrs | FileCheck %s ; RUN: llc -mcpu=pwr9 -mtriple=powerpc64le-unknown-linux-gnu < %s -verify-machineinstrs | FileCheck %s -; RUN: llc -mcpu=pwr8 -mtriple=powerpc64-unknown-linux-gnu < %s -verify-machineinstrs | FileCheck %s \ -; RUN: --check-prefix=CHECK-ITIN -; RUN: llc -mcpu=pwr8 -mtriple=powerpc64le-unknown-linux-gnu < %s -verify-machineinstrs | FileCheck %s \ -; RUN: --check-prefix=CHECK-ITIN +; RUN: llc -mcpu=pwr8 -mtriple=powerpc64-unknown-linux-gnu -disable-ppc-ctrloops < %s -verify-machineinstrs \ +; RUN: | FileCheck %s --check-prefix=CHECK-ITIN +; RUN: llc -mcpu=pwr8 -mtriple=powerpc64le-unknown-linux-gnu -disable-ppc-ctrloops < %s -verify-machineinstrs \ +; RUN: | FileCheck %s --check-prefix=CHECK-ITIN %0 = type { i32, i32 } @@ -12,11 +12,11 @@ define void @initCombList(%0* nocapture, i32 signext) local_unnamed_addr #0 { ; CHECK-LABEL: initCombList: ; CHECK: addi 4, 4, -8 -; CHECK: stwu 5, 64(3) +; CHECK: stwu [[REG:[0-9]+]], 64(3) ; CHECK-ITIN-LABEL: initCombList: -; CHECK-ITIN: stwu 5, 64(4) -; CHECK-ITIN-NEXT: addi 3, 3, -8 +; CHECK-ITIN: stwu [[REG:[0-9]+]], 64(3) +; CHECK-ITIN-NEXT: addi [[REG2:[0-9]+]], [[REG2]], 8 %3 = zext i32 %1 to i64 Index: llvm/test/CodeGen/PowerPC/unal-altivec.ll =================================================================== --- llvm/test/CodeGen/PowerPC/unal-altivec.ll +++ llvm/test/CodeGen/PowerPC/unal-altivec.ll @@ -29,15 +29,14 @@ br i1 %10, label %for.end, label %vector.body ; CHECK: @foo -; CHECK-DAG: li [[C0:[0-9]+]], 0 +; CHECK-DAG: li [[C16:[0-9]+]], 16 ; CHECK-DAG: lvx [[CNST:[0-9]+]], ; CHECK: .LBB0_1: -; CHECK-DAG: lvsl [[MASK1:[0-9]+]], [[B1:[0-9]+]], [[C0]] -; CHECK-DAG: add [[B3:[0-9]+]], [[B1]], [[C0]] -; CHECK-DAG: lvx [[LD1:[0-9]+]], [[B1]], [[C0]] -; CHECK-DAG: lvx [[LD2:[0-9]+]], [[B3]], -; CHECK-DAG: vperm [[R1:[0-9]+]], [[LD1]], [[LD2]], [[MASK1]] -; CHECK-DAG: vaddfp {{[0-9]+}}, [[R1]], [[CNST]] +; CHECK-DAG: lvx [[LD1:[0-9]+]], 0, [[C0:[0-9]+]] +; CHECK-DAG: lvx [[LD2:[0-9]+]], [[C0]], [[C16]] +; CHECK-DAG: lvsl [[MASK1:[0-9]+]], 0, [[C0]] +; CHECK-DAG: vperm [[VR1:[0-9]+]], [[LD1]], [[LD2]], [[MASK1]] +; CHECK-DAG: vaddfp {{[0-9]+}}, [[VR1]], [[CNST]] ; CHECK: blr for.end: ; preds = %vector.body