Index: lib/Transforms/Scalar/LoopStrengthReduce.cpp =================================================================== --- lib/Transforms/Scalar/LoopStrengthReduce.cpp +++ lib/Transforms/Scalar/LoopStrengthReduce.cpp @@ -115,6 +115,7 @@ #include #include #include +#include #include #include @@ -163,6 +164,10 @@ cl::init(std::numeric_limits::max()), cl::desc("LSR search space complexity limit")); +static cl::opt EnableRecursiveSetupCost( + "lsr-recursive-setupcost", cl::Hidden, cl::init(true), + cl::desc("Enable more thourough lsr setup cost")); + #ifndef NDEBUG // Stress test IV chain generation. static cl::opt StressIVChain( @@ -1211,6 +1216,25 @@ bool HasBaseReg, int64_t Scale, Instruction *Fixup = nullptr); +static unsigned getSetupCost(const SCEV *Reg) { + if (isa(Reg) || isa(Reg)) + return 1; + if (const auto *S = dyn_cast(Reg)) + return getSetupCost(S->getStart()); + if (!EnableRecursiveSetupCost) + return 0; + if (auto S = dyn_cast(Reg)) + return getSetupCost(S->getOperand()); + if (auto S = dyn_cast(Reg)) + return std::accumulate(S->op_begin(), S->op_end(), 0, + [](unsigned i, const SCEV *Reg) { + return i + getSetupCost(Reg); + }); + if (auto S = dyn_cast(Reg)) + return getSetupCost(S->getLHS()) + getSetupCost(S->getRHS()); + return 0; +} + /// Tally up interesting quantities from the given register. void Cost::RateRegister(const Formula &F, const SCEV *Reg, SmallPtrSetImpl &Regs, @@ -1276,12 +1300,7 @@ // Rough heuristic; favor registers which don't require extra setup // instructions in the preheader. - if (!isa(Reg) && - !isa(Reg) && - !(isa(Reg) && - (isa(cast(Reg)->getStart()) || - isa(cast(Reg)->getStart())))) - ++C.SetupCost; + C.SetupCost += getSetupCost(Reg); C.NumIVMuls += isa(Reg) && SE.hasComputableLoopEvolution(Reg, L); Index: test/CodeGen/ARM/lsr-setupcost.ll =================================================================== --- test/CodeGen/ARM/lsr-setupcost.ll +++ test/CodeGen/ARM/lsr-setupcost.ll @@ -23,11 +23,10 @@ ; CHECK-NEXT: [[I3:%.*]] = load i16*, i16** [[PDATA1]], align 4 ; CHECK-NEXT: [[PDATA:%.*]] = getelementptr inbounds [[STRUCT_ARM_MATRIX_INSTANCE_Q15]], %struct.arm_matrix_instance_q15* [[PSRCA]], i32 0, i32 2 ; CHECK-NEXT: [[I4:%.*]] = load i16*, i16** [[PDATA]], align 4 -; CHECK-NEXT: [[TMP0:%.*]] = sub i32 0, [[CONV5]] ; CHECK-NEXT: br label [[WHILE_BODY:%.*]] ; CHECK: while.body: -; CHECK-NEXT: [[LSR_IV:%.*]] = phi i32 [ [[LSR_IV_NEXT:%.*]], [[WHILE_BODY]] ], [ [[TMP0]], [[WHILE_BODY_PREHEADER]] ] ; CHECK-NEXT: [[PINA_026:%.*]] = phi i16* [ [[INCDEC_PTR:%.*]], [[WHILE_BODY]] ], [ [[I4]], [[WHILE_BODY_PREHEADER]] ] +; CHECK-NEXT: [[BLKCNT_025:%.*]] = phi i32 [ [[DEC:%.*]], [[WHILE_BODY]] ], [ [[CONV5]], [[WHILE_BODY_PREHEADER]] ] ; CHECK-NEXT: [[PINB_024:%.*]] = phi i16* [ [[INCDEC_PTR8:%.*]], [[WHILE_BODY]] ], [ [[I3]], [[WHILE_BODY_PREHEADER]] ] ; CHECK-NEXT: [[POUT_023:%.*]] = phi i16* [ [[INCDEC_PTR11:%.*]], [[WHILE_BODY]] ], [ [[I2]], [[WHILE_BODY_PREHEADER]] ] ; CHECK-NEXT: [[INCDEC_PTR]] = getelementptr inbounds i16, i16* [[PINA_026]], i32 1 @@ -44,8 +43,8 @@ ; CHECK-NEXT: [[CONV10:%.*]] = trunc i32 [[CALL21]] to i16 ; CHECK-NEXT: [[INCDEC_PTR11]] = getelementptr inbounds i16, i16* [[POUT_023]], i32 1 ; CHECK-NEXT: store i16 [[CONV10]], i16* [[POUT_023]], align 2 -; CHECK-NEXT: [[LSR_IV_NEXT]] = add nsw i32 [[LSR_IV]], 1 -; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[LSR_IV_NEXT]], 0 +; CHECK-NEXT: [[DEC]] = add nsw i32 [[BLKCNT_025]], -1 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[DEC]], 0 ; CHECK-NEXT: br i1 [[CMP]], label [[WHILE_END_LOOPEXIT:%.*]], label [[WHILE_BODY]] ; CHECK: while.end.loopexit: ; CHECK-NEXT: br label [[WHILE_END]] Index: test/CodeGen/Hexagon/swp-carried-1.ll =================================================================== --- test/CodeGen/Hexagon/swp-carried-1.ll +++ test/CodeGen/Hexagon/swp-carried-1.ll @@ -1,4 +1,4 @@ -; RUN: llc -march=hexagon -rdf-opt=0 -disable-hexagon-misched -hexagon-initial-cfg-cleanup=0 < %s | FileCheck %s +; RUN: llc -march=hexagon -rdf-opt=0 -disable-hexagon-misched -hexagon-initial-cfg-cleanup=0 -lsr-recursive-setupcost=0 < %s | FileCheck %s ; Test that we generate the correct code when a loop carried value ; is scheduled one stage earlier than it's use. The code in Index: test/CodeGen/Hexagon/swp-epilog-phi5.ll =================================================================== --- test/CodeGen/Hexagon/swp-epilog-phi5.ll +++ test/CodeGen/Hexagon/swp-epilog-phi5.ll @@ -7,10 +7,10 @@ ; In this test case, the second loop is pipelined, block b5. -; CHECK: loop0 +; CHECK: loop1 ; CHECK: [[REG0:r([0-9]+)]] += mpyi ; CHECK: [[REG2:r([0-9]+)]] = add([[REG1:r([0-9]+)]],add([[REG0]],#8 -; CHECK: endloop0 +; CHECK: endloop1 %s.0 = type { %s.1*, %s.4*, %s.7*, i8*, i8, i32, %s.8*, i32, i32, i32, i8, i8, i32, i32, double, i8, i8, i8, i8, i8, i8, i8, i8, i32, i8, i8, i8, i32, i32, i32, i32, i32, i32, i8**, i32, i32, i32, i32, i32, [64 x i32]*, [4 x %s.9*], [4 x %s.10*], [4 x %s.10*], i32, %s.23*, i8, i8, [16 x i8], [16 x i8], [16 x i8], i32, i8, i8, i8, i8, i16, i16, i8, i8, i8, %s.11*, i32, i32, i32, i32, i8*, i32, [4 x %s.23*], i32, i32, i32, [10 x i32], i32, i32, i32, i32, i32, %s.12*, %s.13*, %s.14*, %s.15*, %s.16*, %s.17*, %s.18*, %s.19*, %s.20*, %s.21*, %s.22* } %s.1 = type { void (%s.2*)*, void (%s.2*, i32)*, void (%s.2*)*, void (%s.2*, i8*)*, void (%s.2*)*, i32, %s.3, i32, i32, i8**, i32, i8**, i32, i32 } Index: test/Transforms/LoopStrengthReduce/two-combinations-bug.ll =================================================================== --- test/Transforms/LoopStrengthReduce/two-combinations-bug.ll +++ test/Transforms/LoopStrengthReduce/two-combinations-bug.ll @@ -1,4 +1,4 @@ -; RUN: opt < %s -loop-reduce -S | FileCheck %s +; RUN: opt < %s -loop-reduce -lsr-recursive-setupcost=0 -S | FileCheck %s ; This test is adapted from the n-body test of the LLVM test-suite: A bug in ; r345114 caused LSR to generate incorrect code. The test verifies that the