Index: lib/Target/PowerPC/PPCCTRLoops.cpp =================================================================== --- lib/Target/PowerPC/PPCCTRLoops.cpp +++ lib/Target/PowerPC/PPCCTRLoops.cpp @@ -26,12 +26,17 @@ #include "PPC.h" #include "PPCSubtarget.h" #include "PPCTargetMachine.h" +#include "PPCTargetTransformInfo.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/AssumptionCache.h" +#include "llvm/Analysis/CodeMetrics.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/ScalarEvolutionExpander.h" #include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/CodeGen/TargetSchedule.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Dominators.h" @@ -64,6 +69,13 @@ static cl::opt CTRLoopLimit("ppc-max-ctrloop", cl::Hidden, cl::init(-1)); #endif +// The latency of mtctr is only justified if there are more than 4 +// comparisons that will be removed as a result. +static cl::opt +SmallCTRLoopThreshold("min-ctr-loop-threshold", cl::init(4), cl::Hidden, + cl::desc("Loops with a constant trip count smaller than " + "this value will not use the count register.")); + STATISTIC(NumCTRLoops, "Number of loops converted to CTR loops"); namespace llvm { @@ -95,6 +107,8 @@ AU.addRequired(); AU.addPreserved(); AU.addRequired(); + AU.addRequired(); + AU.addRequired(); } private: @@ -107,10 +121,12 @@ const PPCTargetLowering *TLI; const DataLayout *DL; const TargetLibraryInfo *LibInfo; + const TargetTransformInfo *TTI=nullptr; LoopInfo *LI; ScalarEvolution *SE; DominatorTree *DT; bool PreserveLCSSA; + TargetSchedModel SchedModel; }; char PPCCTRLoops::ID = 0; @@ -179,6 +195,7 @@ LI = &getAnalysis().getLoopInfo(); SE = &getAnalysis().getSE(); DT = &getAnalysis().getDomTree(); + TTI = &getAnalysis().getTTI(F); DL = &F.getParent()->getDataLayout(); auto *TLIP = getAnalysisIfAvailable(); LibInfo = TLIP ? &TLIP->getTLI() : nullptr; @@ -462,10 +479,23 @@ return false; } - bool PPCCTRLoops::convertToCTRLoop(Loop *L) { bool MadeChange = false; + // Do not convert small short loops to CTR loop. + unsigned ConstTripCount = SE->getSmallConstantTripCount(L); + if (ConstTripCount && ConstTripCount < SmallCTRLoopThreshold) { + SmallPtrSet EphValues; + auto AC = getAnalysis().getAssumptionCache( + *L->getHeader()->getParent()); + CodeMetrics::collectEphemeralValues(L, &AC, EphValues); + CodeMetrics Metrics; + for (BasicBlock *BB : L->blocks()) + Metrics.analyzeBasicBlock(BB, *TTI, EphValues); + if (Metrics.NumInsts < (6 * SchedModel.getIssueWidth())) + return false; + } + // Process nested loops first. for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I) { MadeChange |= convertToCTRLoop(*I); Index: test/CodeGen/PowerPC/ctr-minmaxnum.ll =================================================================== --- test/CodeGen/PowerPC/ctr-minmaxnum.ll +++ test/CodeGen/PowerPC/ctr-minmaxnum.ll @@ -1,6 +1,5 @@ -; RUN: llc -verify-machineinstrs -mcpu=pwr7 < %s | FileCheck %s -; RUN: llc -verify-machineinstrs -mcpu=a2q < %s | FileCheck %s --check-prefix=QPX -target triple = "powerpc64-unknown-linux-gnu" +; RUN: llc -mtriple=powerpc64-unknown-linux-gnu -verify-machineinstrs -mcpu=pwr7 < %s | FileCheck %s -implicit-check-not=mtctr +; RUN: llc -mtriple=powerpc64-unknown-linux-gnu -verify-machineinstrs -mcpu=a2q < %s | FileCheck %s --check-prefix=QPX -implicit-check-not=mtctr declare float @fabsf(float) @@ -36,8 +35,9 @@ } ; CHECK-LABEL: test1: -; CHECK-NOT: mtctr ; CHECK: bl fminf +; CHECK-NOT: bl fminf +; CHECK: blr define void @test1v(<4 x float> %f, <4 x float>* %fp) { entry: @@ -56,11 +56,14 @@ } ; CHECK-LABEL: test1v: -; CHECK-NOT: mtctr ; CHECK: bl fminf +; CHECK: bl fminf +; CHECK: bl fminf +; CHECK: bl fminf +; CHECK-NOT: bl fminf +; CHECK: blr ; QPX-LABEL: test1v: -; QPX: mtctr ; QPX-NOT: bl fminf ; QPX: blr @@ -81,8 +84,9 @@ } ; CHECK-LABEL: test1a: -; CHECK-NOT: mtctr ; CHECK: bl fminf +; CHECK-NOT: bl fminf +; CHECK: blr define void @test2(float %f, float* %fp) { entry: @@ -101,8 +105,9 @@ } ; CHECK-LABEL: test2: -; CHECK-NOT: mtctr ; CHECK: bl fmaxf +; CHECK-NOT: bl fmaxf +; CHECK: blr define void @test2v(<4 x double> %f, <4 x double>* %fp) { entry: @@ -121,11 +126,14 @@ } ; CHECK-LABEL: test2v: -; CHECK-NOT: mtctr ; CHECK: bl fmax +; CHECK: bl fmax +; CHECK: bl fmax +; CHECK: bl fmax +; CHECK-NOT: bl fmax +; CHECK: blr ; QPX-LABEL: test2v: -; QPX: mtctr ; QPX-NOT: bl fmax ; QPX: blr @@ -146,8 +154,9 @@ } ; CHECK-LABEL: test2a: -; CHECK-NOT: mtctr ; CHECK: bl fmaxf +; CHECK-NOT: bl fmaxf +; CHECK: blr define void @test3(double %f, double* %fp) { entry: @@ -166,8 +175,9 @@ } ; CHECK-LABEL: test3: -; CHECK-NOT: mtctr ; CHECK: bl fmin +; CHECK-NOT: bl fmin +; CHECK: blr define void @test3a(double %f, double* %fp) { entry: @@ -186,8 +196,9 @@ } ; CHECK-LABEL: test3a: -; CHECK-NOT: mtctr ; CHECK: bl fmin +; CHECK-NOT: bl fmin +; CHECK: blr define void @test4(double %f, double* %fp) { entry: @@ -206,8 +217,9 @@ } ; CHECK-LABEL: test4: -; CHECK-NOT: mtctr ; CHECK: bl fmax +; CHECK-NOT: bl fmax +; CHECK: blr define void @test4a(double %f, double* %fp) { entry: @@ -226,6 +238,7 @@ } ; CHECK-LABEL: test4a: -; CHECK-NOT: mtctr ; CHECK: bl fmax +; CHECK-NOT: bl fmax +; CHECK: blr Index: test/CodeGen/PowerPC/ctrloop-shortLoops.ll =================================================================== --- /dev/null +++ test/CodeGen/PowerPC/ctrloop-shortLoops.ll @@ -0,0 +1,116 @@ +; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -verify-machineinstrs -mcpu=pwr8 | FileCheck %s +; RUN: llc < %s -mtriple=powerpc64le-unknown-linux-gnu -verify-machineinstrs -mcpu=a2q | FileCheck %s + +; Verify that we do NOT generate the mtctr instruction for loop trip counts < 4 +; The latency of the mtctr is only justified if there are more than 4 comparisons that are removed as a result. + +@a = common local_unnamed_addr global i32 0, align 4 +@arr = common local_unnamed_addr global [5 x i32] zeroinitializer, align 4 + +; Function Attrs: norecurse nounwind readonly +define signext i32 @testTripCount2(i32 signext %a) { + +; CHECK-LABEL: testTripCount2: +; CHECK-NOT: mtctr +; CHECK: blr + +entry: + br label %for.body + +for.cond.cleanup: ; preds = %for.body + ret i32 %add + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ 1, %entry ], [ %indvars.iv.next, %for.body ] + %Sum.05 = phi i32 [ 0, %entry ], [ %add, %for.body ] + %arrayidx = getelementptr inbounds [5 x i32], [5 x i32]* @arr, i64 0, i64 %indvars.iv + %0 = load i32, i32* %arrayidx, align 4 + %add = add nsw i32 %0, %Sum.05 + %indvars.iv.next = add nsw i64 %indvars.iv, -1 + %tobool = icmp eq i64 %indvars.iv, 0 + br i1 %tobool, label %for.cond.cleanup, label %for.body +} + +; Function Attrs: norecurse nounwind readonly +define signext i32 @testTripCount3(i32 signext %a) { + +; CHECK-LABEL: testTripCount3: +; CHECK-NOT: mtctr +; CHECK: blr + +entry: + br label %for.body + +for.cond.cleanup: ; preds = %for.body + ret i32 %add + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ 2, %entry ], [ %indvars.iv.next, %for.body ] + %Sum.05 = phi i32 [ 0, %entry ], [ %add, %for.body ] + %arrayidx = getelementptr inbounds [5 x i32], [5 x i32]* @arr, i64 0, i64 %indvars.iv + %0 = load i32, i32* %arrayidx, align 4 + %add = add nsw i32 %0, %Sum.05 + %indvars.iv.next = add nsw i64 %indvars.iv, -1 + %tobool = icmp eq i64 %indvars.iv, 0 + br i1 %tobool, label %for.cond.cleanup, label %for.body +} + +; Function Attrs: norecurse nounwind readonly + +define signext i32 @testTripCount4(i32 signext %a) { + +; CHECK-LABEL: testTripCount4: +; CHECK: mtctr +; CHECK: bdnz + +entry: + br label %for.body + +for.cond.cleanup: ; preds = %for.body + ret i32 %add + +for.body: ; preds = %entry, %for.body + %indvars.iv = phi i64 [ 3, %entry ], [ %indvars.iv.next, %for.body ] + %Sum.05 = phi i32 [ 0, %entry ], [ %add, %for.body ] + %arrayidx = getelementptr inbounds [5 x i32], [5 x i32]* @arr, i64 0, i64 %indvars.iv + %0 = load i32, i32* %arrayidx, align 4 + %add = add nsw i32 %0, %Sum.05 + %indvars.iv.next = add nsw i64 %indvars.iv, -1 + %tobool = icmp eq i64 %indvars.iv, 0 + br i1 %tobool, label %for.cond.cleanup, label %for.body +} + +; Function Attrs: norecurse nounwind +define signext i32 @testTripCount2NonSmallLoop() { + +; CHECK-LABEL: testTripCount2NonSmallLoop: +; CHECK: mtctr +; CHECK: blr + +entry: + %.pre = load i32, i32* @a, align 4 + br label %for.body + +for.body: ; preds = %entry, %if.end + %0 = phi i32 [ %.pre, %entry ], [ %1, %if.end ] + %dec4 = phi i32 [ 1, %entry ], [ %dec, %if.end ] + %b.03 = phi i8 [ 0, %entry ], [ %b.1, %if.end ] + %tobool1 = icmp eq i32 %0, 0 + br i1 %tobool1, label %if.end, label %if.then + +if.then: ; preds = %for.body + store i32 2, i32* @a, align 4 + br label %if.end + +if.end: ; preds = %for.body, %if.then + %1 = phi i32 [ 2, %if.then ], [ 0, %for.body ] + %b.1 = phi i8 [ 2, %if.then ], [ %b.03, %for.body ] + %dec = add nsw i32 %dec4, -1 + %tobool = icmp eq i32 %dec4, 0 + br i1 %tobool, label %for.end, label %for.body + +for.end: ; preds = %if.end + %conv = zext i8 %b.1 to i32 + ret i32 %conv +} +