Index: llvm/include/llvm/Analysis/LoopInfo.h =================================================================== --- llvm/include/llvm/Analysis/LoopInfo.h +++ llvm/include/llvm/Analysis/LoopInfo.h @@ -567,6 +567,11 @@ Instruction *InsertPt = nullptr, MemorySSAUpdater *MSSAU = nullptr) const; + /// If this loop has an induction variable with a negative constant induction + /// step value 1, then reverse the induction variable and return this, thus + /// transforming a counting-down loop to a counting-up loop. + PHINode *reverseInductionVariable(ScalarEvolution &SE); + /// Check to see if the loop has a canonical induction variable: an integer /// recurrence that starts at 0 and increments by one each time through the /// loop. If so, return the phi node that corresponds to it. Index: llvm/lib/Analysis/LoopInfo.cpp =================================================================== --- llvm/lib/Analysis/LoopInfo.cpp +++ llvm/lib/Analysis/LoopInfo.cpp @@ -29,6 +29,7 @@ #include "llvm/IR/Constants.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/Dominators.h" +#include "llvm/IR/IRBuilder.h" #include "llvm/IR/IRPrintingPasses.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/LLVMContext.h" @@ -196,6 +197,104 @@ return nullptr; } +// Transform a counting-down loop: +// +// %N.addr.09 = phi i32 [ %dec, %loop.body ], [ %N, %preheader ] +// %dec = add nsw i32 %N.addr.09, -1 +// %cmp = icmp sgt i32 %N.addr.09, 1 +// br i1 %cmp, label %loop.body, label %loopexit +// +// to a counting-up loop: +// +// %i.011 = phi i32 [ %inc, %loop.body ], [ 1, %preheader ] +// %inc = add nuw nsw i32 %i.011, 1 +// %exitcond = icmp eq i32 %inc, %N +// br i1 %exitcond, label %loopexit, label %loop.body +// +// by creating new increment, compare, and branch instructions, and return +// the modified induction variable. +// +// For now, we only rewrite loops that have a step value of -1 and the +// backedge is of the form: %iter > 1 +// +PHINode *Loop::reverseInductionVariable(ScalarEvolution &SE) { + BasicBlock *Preheader = getLoopPreheader(); + if (!Preheader) + return nullptr; + + PHINode *IndVar = getInductionVariable(SE); + if (!IndVar) + return nullptr; + + InductionDescriptor ID; + if (!InductionDescriptor::isInductionPHI(IndVar, this, &SE, ID)) + return nullptr; + + // TODO: for now only support a -1 step-value. This could be relaxed, + // but need more work to rewrite the loop. + ConstantInt *Step = ID.getConstIntStepValue(); + if (!Step || !Step->isMinusOne()) + return nullptr; + + ICmpInst *Latch = getLatchCmpInst(*this); + if (!Latch || !Latch->hasOneUse() || + Latch->getSignedPredicate() != ICmpInst::ICMP_SGT) + return nullptr; + + User *Br = *Latch->user_begin(); + if (!dyn_cast(Br)) + return nullptr; + + BasicBlock *ExitBlock = getExitBlock(); + if (!ExitBlock) + return nullptr; + + ConstantInt *LatchRHS = dyn_cast(Latch->getOperand(1)); + if (Latch->getOperand(0) != IndVar || !LatchRHS || !LatchRHS->isOne()) + return nullptr; + + Value *StartValue = ID.getStartValue(); + if (isa(StartValue)) + return nullptr; + + // If the start value is a runtime value, we need a loop-guard that + // ensures the loop executes at least one iteration, otherwise this + // rewrite isn't valid. If the start value is a constant, we don't + // necessarily need a guard as we can check that ourselves. + ConstantInt *ConstStartValue = dyn_cast(StartValue); + if (ConstStartValue && ConstStartValue->getValue().slt(1)) + return nullptr; + else if (!isGuarded()) + return nullptr; + + Value *IndOp = ID.getInductionBinOp(); + IRBuilder<> IRB(IndVar->getParent()); + + APInt NewStepValue = Step->getValue(); + NewStepValue.negate(); + + // Create new increment, compare and branch instructions. + Value *NewIndOp = IRB.CreateAdd( + IndVar, + llvm::ConstantInt::get(IndOp->getType(), NewStepValue.getSExtValue())); + IndOp->replaceAllUsesWith(NewIndOp); + IRB.CreateCondBr(IRB.CreateICmpEQ(NewIndOp, StartValue), ExitBlock, + IndVar->getParent()); + + // Remove all the old instructions. + dyn_cast(Br)->eraseFromParent(); + Latch->eraseFromParent(); + ID.getInductionBinOp()->eraseFromParent(); + + // Modify the Phi node with the new start value, i.e. 1. + IndVar->removeIncomingValue(Preheader); + IndVar->addIncoming(llvm::ConstantInt::get(IndOp->getType(), 1), + Preheader); + + // Return the modified Phi node, the induction variable. + return IndVar; +} + Optional Loop::LoopBounds::getBounds(const Loop &L, PHINode &IndVar, ScalarEvolution &SE) { Index: llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp +++ llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp @@ -1184,6 +1184,18 @@ return false; } + // If this is a counting-down loop, reverse the induction variable and create + // a counting-up loop. This will probably lead to more efficient + // vectorisation, as this will enable earlier discovery of a primary + // induction variable which is e.g. required for tail-folding of the scalar + // epilogue loop. + if (PHINode *IndVar = TheLoop->reverseInductionVariable(*PSE.getSE())) { + PrimaryInduction = IndVar; + LLVM_DEBUG(dbgs() << "LV: Loop after indvar reversal:\n"; + TheLoop->dumpVerbose();); + PSE.getSE()->forgetLoop(TheLoop); + } + // Check if we can vectorize the instructions and CFG in this loop. if (!canVectorizeInstrs()) { LLVM_DEBUG(dbgs() << "LV: Can't vectorize the instructions or CFG\n"); Index: llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll =================================================================== --- llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll +++ llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll @@ -1,19 +1,265 @@ ; RUN: opt < %s -loop-vectorize -S | FileCheck %s -; RUN: opt < %s -loop-vectorize -prefer-predicate-over-epilog -S | FileCheck %s -; RUN: opt < %s -loop-vectorize -disable-mve-tail-predication=false -S | FileCheck %s - -; Check that when we can't predicate this loop that it is still vectorised (with -; an epilogue). -; TODO: the reason this can't be predicated is because a primary induction -; variable can't be found (not yet) for this counting down loop. But with that -; fixed, this should be able to be predicated. - -; CHECK-LABEL: vector.body: +; RUN: opt < %s -loop-vectorize -prefer-predicate-over-epilog -S | FileCheck %s --check-prefixes=CHECK,CHECK-TF,CHECK-PREFER +; RUN: opt < %s -loop-vectorize -disable-mve-tail-predication=false -S | FileCheck %s --check-prefixes=CHECK,CHECK-TF,CHECK-DISABLE-TP target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64" target triple = "thumbv8.1m.main-arm-unknown-eabihf" -define dso_local void @foo(i8* noalias nocapture readonly %A, i8* noalias nocapture readonly %B, i8* noalias nocapture %C, i32 %N) #0 { +; This IR corresponds to this type of C-code: +; +; void f(char *a, char *b, char *c, int N) { +; while (N-- > 0) +; *c++ = *a++ + *b++; +; } +; +define dso_local void @sgt_loopguard(i8* noalias nocapture readonly %a, i8* noalias nocapture readonly %b, i8* noalias nocapture %c, i32 %N) local_unnamed_addr #0 { +; CHECK-LABEL: @sgt_loopguard( +; CHECK: vector.body: +; CHECK-TF: masked.load +; CHECK-TF: masked.load +; CHECK-TF: masked.store +entry: + %cmp5 = icmp sgt i32 %N, 0 + br i1 %cmp5, label %while.body.preheader, label %while.end + +while.body.preheader: + br label %while.body + +while.body: + %N.addr.09 = phi i32 [ %dec, %while.body ], [ %N, %while.body.preheader ] + %c.addr.08 = phi i8* [ %incdec.ptr4, %while.body ], [ %c, %while.body.preheader ] + %b.addr.07 = phi i8* [ %incdec.ptr1, %while.body ], [ %b, %while.body.preheader ] + %a.addr.06 = phi i8* [ %incdec.ptr, %while.body ], [ %a, %while.body.preheader ] + %dec = add nsw i32 %N.addr.09, -1 + %incdec.ptr = getelementptr inbounds i8, i8* %a.addr.06, i32 1 + %0 = load i8, i8* %a.addr.06, align 1 + %incdec.ptr1 = getelementptr inbounds i8, i8* %b.addr.07, i32 1 + %1 = load i8, i8* %b.addr.07, align 1 + %add = add i8 %1, %0 + %incdec.ptr4 = getelementptr inbounds i8, i8* %c.addr.08, i32 1 + store i8 %add, i8* %c.addr.08, align 1 + %cmp = icmp sgt i32 %N.addr.09, 1 + br i1 %cmp, label %while.body, label %while.end.loopexit + +while.end.loopexit: + br label %while.end + +while.end: + ret void +} + +; Without a loop-guard, check that we don't reverse the induction variable +; and thus that we don't tail-fold here. +; +define dso_local void @sgt_no_loopguard(i8* noalias nocapture readonly %a, i8* noalias nocapture readonly %b, i8* noalias nocapture %c, i32 %N) local_unnamed_addr #0 { +; CHECK-LABEL: @sgt_no_loopguard( +; CHECK: vector.body: +; CHECK-TF-NOT: masked.load +; CHECK-TF-NOT: masked.load +; CHECK-TF-NOT: masked.store +entry: + br label %while.body + +while.body: + %N.addr.09 = phi i32 [ %dec, %while.body ], [ %N, %entry ] + %c.addr.08 = phi i8* [ %incdec.ptr4, %while.body ], [ %c, %entry ] + %b.addr.07 = phi i8* [ %incdec.ptr1, %while.body ], [ %b, %entry ] + %a.addr.06 = phi i8* [ %incdec.ptr, %while.body ], [ %a, %entry ] + %dec = add nsw i32 %N.addr.09, -1 + %incdec.ptr = getelementptr inbounds i8, i8* %a.addr.06, i32 1 + %0 = load i8, i8* %a.addr.06, align 1 + %incdec.ptr1 = getelementptr inbounds i8, i8* %b.addr.07, i32 1 + %1 = load i8, i8* %b.addr.07, align 1 + %add = add i8 %1, %0 + %incdec.ptr4 = getelementptr inbounds i8, i8* %c.addr.08, i32 1 + store i8 %add, i8* %c.addr.08, align 1 + %cmp = icmp sgt i32 %N.addr.09, 1 + br i1 %cmp, label %while.body, label %while.end.loopexit + +while.end.loopexit: + br label %while.end + +while.end: + ret void +} + +define dso_local void @sgt_extra_use_cmp(i8* noalias nocapture readonly %a, i8* noalias nocapture readonly %b, i8* noalias nocapture %c, i32 %N) local_unnamed_addr #0 { +; CHECK-LABEL: @sgt_extra_use_cmp( +; CHECK: vector.body: +; CHECK-TF-NOT: masked.load +; CHECK-TF-NOT: masked.load +; CHECK-TF-NOT: masked.store +entry: + br label %while.body + +while.body: + %N.addr.09 = phi i32 [ %dec, %while.body ], [ %N, %entry ] + %c.addr.08 = phi i8* [ %incdec.ptr4, %while.body ], [ %c, %entry ] + %b.addr.07 = phi i8* [ %incdec.ptr1, %while.body ], [ %b, %entry ] + %a.addr.06 = phi i8* [ %incdec.ptr, %while.body ], [ %a, %entry ] + %dec = add nsw i32 %N.addr.09, -1 + %incdec.ptr = getelementptr inbounds i8, i8* %a.addr.06, i32 1 + %0 = load i8, i8* %a.addr.06, align 1 + %incdec.ptr1 = getelementptr inbounds i8, i8* %b.addr.07, i32 1 + %1 = load i8, i8* %b.addr.07, align 1 + %add = add i8 %1, %0 + %incdec.ptr4 = getelementptr inbounds i8, i8* %c.addr.08, i32 1 + store i8 %add, i8* %c.addr.08, align 1 + %cmp = icmp sgt i32 %N.addr.09, 1 + %select = select i1 %cmp, i8 %0, i8 %1 + br i1 %cmp, label %while.body, label %while.end.loopexit + +while.end.loopexit: + br label %while.end + +while.end: + ret void +} + +define dso_local void @sgt_const_tripcount(i8* noalias nocapture readonly %a, i8* noalias nocapture readonly %b, i8* noalias nocapture %c, i32 %N) local_unnamed_addr #0 { +; CHECK-LABEL: @sgt_const_tripcount( +; CHECK: vector.body: +; CHECK-TF: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ] +; CHECK-TF: masked.load +; CHECK-TF: masked.load +; CHECK-TF: masked.store +; CHECK-TF: %index.next = add i32 %index, 16 +; CHECK-TF: %[[CMP:.*]] = icmp eq i32 %index.next, 2048 +; CHECK-TF: br i1 %[[CMP]], label %middle.block, label %vector.body +entry: + %cmp5 = icmp sgt i32 %N, 0 + br i1 %cmp5, label %while.body.preheader, label %while.end + +while.body.preheader: + br label %while.body + +while.body: + %N.addr.09 = phi i32 [ %dec, %while.body ], [ 2048, %while.body.preheader ] + %c.addr.08 = phi i8* [ %incdec.ptr4, %while.body ], [ %c, %while.body.preheader ] + %b.addr.07 = phi i8* [ %incdec.ptr1, %while.body ], [ %b, %while.body.preheader ] + %a.addr.06 = phi i8* [ %incdec.ptr, %while.body ], [ %a, %while.body.preheader ] + %dec = add nsw i32 %N.addr.09, -1 + %incdec.ptr = getelementptr inbounds i8, i8* %a.addr.06, i32 1 + %0 = load i8, i8* %a.addr.06, align 1 + %incdec.ptr1 = getelementptr inbounds i8, i8* %b.addr.07, i32 1 + %1 = load i8, i8* %b.addr.07, align 1 + %add = add i8 %1, %0 + %incdec.ptr4 = getelementptr inbounds i8, i8* %c.addr.08, i32 1 + store i8 %add, i8* %c.addr.08, align 1 + %cmp = icmp sgt i32 %N.addr.09, 1 + br i1 %cmp, label %while.body, label %while.end.loopexit + +while.end.loopexit: + br label %while.end + +while.end: + ret void +} + +define dso_local void @sgt_no_guard_0_startval(i8* noalias nocapture readonly %a, i8* noalias nocapture readonly %b, i8* noalias nocapture %c, i32 %N) local_unnamed_addr #0 { +; CHECK-LABEL: @sgt_no_guard_0_startval( +; CHECK-NOT: vector.body: +entry: + br label %while.body + +while.body: + %N.addr.09 = phi i32 [ %dec, %while.body ], [ 0, %entry ] + %c.addr.08 = phi i8* [ %incdec.ptr4, %while.body ], [ %c, %entry ] + %b.addr.07 = phi i8* [ %incdec.ptr1, %while.body ], [ %b, %entry ] + %a.addr.06 = phi i8* [ %incdec.ptr, %while.body ], [ %a, %entry] + %dec = add nsw i32 %N.addr.09, -1 + %incdec.ptr = getelementptr inbounds i8, i8* %a.addr.06, i32 1 + %0 = load i8, i8* %a.addr.06, align 1 + %incdec.ptr1 = getelementptr inbounds i8, i8* %b.addr.07, i32 1 + %1 = load i8, i8* %b.addr.07, align 1 + %add = add i8 %1, %0 + %incdec.ptr4 = getelementptr inbounds i8, i8* %c.addr.08, i32 1 + store i8 %add, i8* %c.addr.08, align 1 + %cmp = icmp sgt i32 %N.addr.09, 1 + br i1 %cmp, label %while.body, label %while.end.loopexit + +while.end.loopexit: + br label %while.end + +while.end: + ret void +} + +; Step values other than -1 are not yet supported. +; +define dso_local void @sgt_step_minus_two(i8* noalias nocapture readonly %a, i8* noalias nocapture readonly %b, i8* noalias nocapture %c, i32 %N) local_unnamed_addr #0 { +; CHECK-LABEL: @sgt_step_minus_two( +; CHECK: vector.body: +; CHECK-TF-NOT: masked.load +; CHECK-TF-NOT: masked.load +; CHECK-TF-NOT: masked.store +entry: + %cmp5 = icmp sgt i32 %N, 0 + br i1 %cmp5, label %while.body.preheader, label %while.end + +while.body.preheader: + br label %while.body + +while.body: + %N.addr.09 = phi i32 [ %dec, %while.body ], [ %N, %while.body.preheader ] + %c.addr.08 = phi i8* [ %incdec.ptr4, %while.body ], [ %c, %while.body.preheader ] + %b.addr.07 = phi i8* [ %incdec.ptr1, %while.body ], [ %b, %while.body.preheader ] + %a.addr.06 = phi i8* [ %incdec.ptr, %while.body ], [ %a, %while.body.preheader ] + %dec = add nsw i32 %N.addr.09, -2 + %incdec.ptr = getelementptr inbounds i8, i8* %a.addr.06, i32 1 + %0 = load i8, i8* %a.addr.06, align 1 + %incdec.ptr1 = getelementptr inbounds i8, i8* %b.addr.07, i32 1 + %1 = load i8, i8* %b.addr.07, align 1 + %add = add i8 %1, %0 + %incdec.ptr4 = getelementptr inbounds i8, i8* %c.addr.08, i32 1 + store i8 %add, i8* %c.addr.08, align 1 + %cmp = icmp sgt i32 %N.addr.09, 1 + br i1 %cmp, label %while.body, label %while.end.loopexit + +while.end.loopexit: + br label %while.end + +while.end: + ret void +} + +define dso_local void @sgt_step_not_constant(i8* noalias nocapture readonly %a, i8* noalias nocapture readonly %b, i8* noalias nocapture %c, i32 %N, i32 %S) local_unnamed_addr #0 { +; CHECK-LABEL: @sgt_step_not_constant( +; CHECK-NOT: vector.body: +entry: + %cmp5 = icmp sgt i32 %N, 0 + br i1 %cmp5, label %while.body.preheader, label %while.end + +while.body.preheader: + br label %while.body + +while.body: + %N.addr.09 = phi i32 [ %dec, %while.body ], [ %N, %while.body.preheader ] + %c.addr.08 = phi i8* [ %incdec.ptr4, %while.body ], [ %c, %while.body.preheader ] + %b.addr.07 = phi i8* [ %incdec.ptr1, %while.body ], [ %b, %while.body.preheader ] + %a.addr.06 = phi i8* [ %incdec.ptr, %while.body ], [ %a, %while.body.preheader ] + %dec = add nsw i32 %N.addr.09, %S + %incdec.ptr = getelementptr inbounds i8, i8* %a.addr.06, i32 1 + %0 = load i8, i8* %a.addr.06, align 1 + %incdec.ptr1 = getelementptr inbounds i8, i8* %b.addr.07, i32 1 + %1 = load i8, i8* %b.addr.07, align 1 + %add = add i8 %1, %0 + %incdec.ptr4 = getelementptr inbounds i8, i8* %c.addr.08, i32 1 + store i8 %add, i8* %c.addr.08, align 1 + %cmp = icmp sgt i32 %N.addr.09, 1 + br i1 %cmp, label %while.body, label %while.end.loopexit + +while.end.loopexit: + br label %while.end + +while.end: + ret void +} + +define dso_local void @icmp_eq(i8* noalias nocapture readonly %A, i8* noalias nocapture readonly %B, i8* noalias nocapture %C, i32 %N) #0 { +; CHECK-LABEL: @icmp_eq +; CHECK: vector.body: +; TODO entry: %cmp6 = icmp eq i32 %N, 0 br i1 %cmp6, label %while.end, label %while.body.preheader @@ -44,4 +290,143 @@ ret void } +; This IR corresponds to this type of C-code: +; +; void f(char *a, char *b, char * __restrict c, int N) { +; for (int i = N; i>0; i--) +; c[i] = a[i] + b[i]; +; } +; +define dso_local void @sgt_for_loop(i8* noalias nocapture readonly %a, i8* noalias nocapture readonly %b, i8* noalias nocapture %c, i32 %N) local_unnamed_addr #0 { +; CHECK-LABEL: @sgt_for_loop( +; CHECK: vector.body: +; CHECK-TF: masked.load +; CHECK-TF: masked.load +; CHECK-TF: masked.store +entry: + %cmp5 = icmp sgt i32 %N, 0 + br i1 %cmp5, label %for.body.preheader, label %for.end + +for.body.preheader: + br label %for.body + +for.body: + %i.011 = phi i32 [ %dec, %for.body ], [ %N, %for.body.preheader ] + %arrayidx = getelementptr inbounds i8, i8* %a, i32 %i.011 + %0 = load i8, i8* %arrayidx, align 1 + %arrayidx1 = getelementptr inbounds i8, i8* %b, i32 %i.011 + %1 = load i8, i8* %arrayidx1, align 1 + %add = add i8 %1, %0 + %arrayidx4 = getelementptr inbounds i8, i8* %c, i32 %i.011 + store i8 %add, i8* %arrayidx4, align 1 + %dec = add nsw i32 %i.011, -1 + %cmp = icmp sgt i32 %i.011, 1 + br i1 %cmp, label %for.body, label %for.end + +for.end: + ret void +} + +define dso_local void @sgt_for_loop_i64(i8* noalias nocapture readonly %a, i8* noalias nocapture readonly %b, i8* noalias nocapture %c, i32 %N) local_unnamed_addr #0 { +; CHECK-LABEL: @sgt_for_loop_i64( +; CHECK: vector.body: +; CHECK-TF: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] +; CHECK-PREFER: masked.load +; CHECK-PREFER: masked.load +; CHECK-PREFER: masked.store +; +; With -disable-mve-tail-predication=false, the cost-model returns that +; creating a hardwareloop is not profitable/possible, so here we don't +; expect the tail-folding: +; +; CHECK-DISABLE-TP-NOT: masked.load +; CHECK-DISABLE-TP-NOT: masked.load +; CHECK-DISABLE-TP-NOT: masked.store +; +; CHECK-TF: %index.next = add i64 %index, 16 +; CHECK-TF: %[[CMP:.*]] = icmp eq i64 %index.next, %n.vec +; CHECK-TF: br i1 %[[CMP]], label %middle.block, label %vector.body +entry: + %cmp14 = icmp sgt i32 %N, 0 + br i1 %cmp14, label %for.body.preheader, label %for.cond.cleanup + +for.body.preheader: + %conv16 = zext i32 %N to i64 + br label %for.body + +for.cond.cleanup.loopexit: + br label %for.cond.cleanup + +for.cond.cleanup: + ret void + +for.body: + %i.015 = phi i64 [ %dec, %for.body ], [ %conv16, %for.body.preheader ] + %idxprom = trunc i64 %i.015 to i32 + %arrayidx = getelementptr inbounds i8, i8* %a, i32 %idxprom + %0 = load i8, i8* %arrayidx, align 1 + %arrayidx4 = getelementptr inbounds i8, i8* %b, i32 %idxprom + %1 = load i8, i8* %arrayidx4, align 1 + %add = add i8 %1, %0 + %arrayidx8 = getelementptr inbounds i8, i8* %c, i32 %idxprom + store i8 %add, i8* %arrayidx8, align 1 + %dec = add nsw i64 %i.015, -1 + %cmp = icmp sgt i64 %i.015, 1 + br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit +} + +; This IR corresponds to this nested-loop: +; +; for (int i = 0; i0; j--) +; c[j] = a[j] + b[j]; +; +; while the inner-loop looks similar to previous examples, we can't +; transform this because the inner loop because isGuarded returns +; false for the inner-loop. +; +define dso_local void @sgt_nested_loop(i8* noalias nocapture readonly %a, i8* noalias nocapture readonly %b, i8* noalias nocapture %c, i32 %N) local_unnamed_addr #0 { +; CHECK-LABEL: @sgt_nested_loop( +; CHECK-NOT: vector.body: +; CHECK-TF-NOT: masked.load +; CHECK-TF-NOT: masked.load +; CHECK-TF-NOT: masked.store +; CHECK: } +; +entry: + %cmp21 = icmp sgt i32 %N, 0 + br i1 %cmp21, label %for.body.preheader, label %for.cond.cleanup + +for.body.preheader: + br label %for.body + +for.cond.loopexit: + %exitcond = icmp eq i32 %add, %N + br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body + +for.cond.cleanup.loopexit: + br label %for.cond.cleanup + +for.cond.cleanup: + ret void + +for.body: + %i.022 = phi i32 [ %add, %for.cond.loopexit ], [ 0, %for.body.preheader ] + %add = add nuw nsw i32 %i.022, 1 + br label %for.body4 + +for.body4: ; preds = %for.body, %for.body4 + %j.020 = phi i32 [ %add, %for.body ], [ %dec, %for.body4 ] + %arrayidx = getelementptr inbounds i8, i8* %a, i32 %j.020 + %0 = load i8, i8* %arrayidx, align 1 + %arrayidx5 = getelementptr inbounds i8, i8* %b, i32 %j.020 + %1 = load i8, i8* %arrayidx5, align 1 + %add7 = add i8 %1, %0 + %arrayidx9 = getelementptr inbounds i8, i8* %c, i32 %j.020 + store i8 %add7, i8* %arrayidx9, align 1 + %dec = add nsw i32 %j.020, -1 + %cmp2 = icmp sgt i32 %j.020, 1 + br i1 %cmp2, label %for.body4, label %for.cond.loopexit +} + attributes #0 = { nofree norecurse nounwind "target-features"="+armv8.1-m.main,+mve.fp" }