[LV] fold-tail flag

[dorit]

Summary

This is the compiler-flag equivalent of the Predicate pragma (https://reviews.llvm.org/D65197), to direct the vectorizer to fold the remainer-loop into the main-loop using predication.

[SjoerdMeijer]

test/Transforms/LoopVectorize/X86/tail_loop_folding.ll

2

Because these test cases have the vectorize.predicate.enable metadata set, I am not sure we are actually testing this new option. I don't think so, I think we need a separate function without the predicate metadata.

17

I expect the output to be the same whether a pragma was used or this new options, so can we just use the CHECK tag?

[dorit]

Thanks for taking a look! Please see responses below.

test/Transforms/LoopVectorize/X86/tail_loop_folding.ll

2

"Because these test cases have the vectorize.predicate.enable metadata set"

not really, the second function has the vectorize.predicate.enable metadata *disabled*. This is why in the second function *without* the new flag the check for no masked loads/stores passes (see CHECK part), and *with* the new flag the check expects to find masked loads/stores (see PREDFLAG part).

17

I'm not sure I understand what you are suggesting... I'm trying to distinguish between two runs - one without the flag, and one with the flag. In the first function the output is indeed the same for both runs; but this is not the case in the second function: if I'll have the second run (with the flag) check all the CHECK tags I will fail in the second function where the output differs.

[SjoerdMeijer]

Ah sorry, ignore me! I messed that up.

This looks like a good and straightforward change to me.

lib/Transforms/Vectorize/LoopVectorize.cpp

 /// Loops with a known constant trip count below this number are vectorized only
 /// if no scalar iteration overheads are incurred.
 static cl::opt<unsigned> TinyTripCountVectorThreshold(
     "vectorizer-min-trip-count", cl::init(16), cl::Hidden,
     cl::desc("Loops with a constant trip count that is smaller than this "
              "value are vectorized only if no scalar iteration overheads "
              "are incurred."));
 
+// Indicates that an epilogue is undesired, predication is preferred.
+// This means that the vectorizer will try to fold the loop-tail (epilogue)
+// into the loop and predicate the loop body accordingly.
+static cl::opt<bool> PreferPredicateOverEpilog(
+    "prefer-predicate-over-epilog", cl::init(false), cl::Hidden,
+    cl::desc("Indicate that an epilogue is undesired, predication should be "
+             "used instead."));
+
 static cl::opt<bool> MaximizeBandwidth(
     "vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden,
     cl::desc("Maximize bandwidth when selecting vectorization factor which "
              "will be determined by the smallest type in loop."));
 
 static cl::opt<bool> EnableInterleavedMemAccesses(
     "enable-interleaved-mem-accesses", cl::init(false), cl::Hidden,
     cl::desc("Enable vectorization on interleaved memory accesses in a loop"));
enum ScalarEpilogueLowering {
 
   // A special case of vectorisation with OptForSize: loops with a very small
   // trip count are considered for vectorization under OptForSize, thereby
   // making sure the cost of their loop body is dominant, free of runtime
   // guards and scalar iteration overheads.
   CM_ScalarEpilogueNotAllowedLowTripLoop,
 
   // Loop hint predicate indicating an epilogue is undesired.
-  CM_ScalarEpilogueNotNeededPredicatePragma
+  CM_ScalarEpilogueNotNeededUsePredicate
 };
 
 /// LoopVectorizationCostModel - estimates the expected speedups due to
 /// vectorization.
 /// In many cases vectorization is not profitable. This can happen because of
 /// a number of reasons. In this class we mainly attempt to predict the
 /// expected speedup/slowdowns due to the supported instruction set. We use the
 /// TargetTransformInfo to query the different backends for the cost of
reportVectorizationFailure("Single iteration (non) loop",
         "loop trip count is one, irrelevant for vectorization",
         "SingleIterationLoop", ORE, TheLoop);
     return None;
   }
 
   switch (ScalarEpilogueStatus) {
   case CM_ScalarEpilogueAllowed:
     return computeFeasibleMaxVF(TC);
-  case CM_ScalarEpilogueNotNeededPredicatePragma:
+  case CM_ScalarEpilogueNotNeededUsePredicate:
     LLVM_DEBUG(
-        dbgs() << "LV: vector predicate hint found.\n"
+        dbgs() << "LV: vector predicate hint/switch found.\n"
                << "LV: Not allowing scalar epilogue, creating predicated "
                << "vector loop.\n");
     break;
   case CM_ScalarEpilogueNotAllowedLowTripLoop:
     // fallthrough as a special case of OptForSize
   case CM_ScalarEpilogueNotAllowedOptSize:
     if (ScalarEpilogueStatus == CM_ScalarEpilogueNotAllowedOptSize)
       LLVM_DEBUG(

 static ScalarEpilogueLowering
 getScalarEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
                           ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI) {
   ScalarEpilogueLowering SEL = CM_ScalarEpilogueAllowed;
   if (Hints.getForce() != LoopVectorizeHints::FK_Enabled &&
       (F->hasOptSize() ||
        llvm::shouldOptimizeForSize(L->getHeader(), PSI, BFI)))
     SEL = CM_ScalarEpilogueNotAllowedOptSize;
-  else if (Hints.getPredicate())
-    SEL = CM_ScalarEpilogueNotNeededPredicatePragma;
+  else if (PreferPredicateOverEpilog || Hints.getPredicate()) 
+    SEL = CM_ScalarEpilogueNotNeededUsePredicate;
 
   return SEL;
 }
 
 // Process the loop in the VPlan-native vectorization path. This path builds
 // VPlan upfront in the vectorization pipeline, which allows to apply
 // VPlan-to-VPlan transformations from the very beginning without modifying the
 // input LLVM IR.

test/Transforms/LoopVectorize/X86/tail_loop_folding.ll

 ; RUN: opt < %s -loop-vectorize -S | FileCheck %s
+; RUN: opt < %s -loop-vectorize -prefer-predicate-over-epilog -S | FileCheck -check-prefix=PREDFLAG %s
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 define dso_local void @tail_folding_enabled(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) local_unnamed_addr #0 {
 ; CHECK-LABEL: tail_folding_enabled(
 ; CHECK:  vector.body:
 ; CHECK:  %wide.masked.load = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(
 ; CHECK:  %wide.masked.load1 = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(
 ; CHECK:  %8 = add nsw <8 x i32> %wide.masked.load1, %wide.masked.load
 ; CHECK:  call void @llvm.masked.store.v8i32.p0v8i32(
 ; CHECK:  %index.next = add i64 %index, 8
 ; CHECK:  %12 = icmp eq i64 %index.next, 432
 ; CHECK:  br i1 %12, label %middle.block, label %vector.body, !llvm.loop !0
-
+; PREDFLAG-LABEL: tail_folding_enabled(
+; PREDFLAG:  vector.body:
+; PREDFLAG:  %wide.masked.load = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(
+; PREDFLAG:  %wide.masked.load1 = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(
+; PREDFLAG:  %8 = add nsw <8 x i32> %wide.masked.load1, %wide.masked.load
+; PREDFLAG:  call void @llvm.masked.store.v8i32.p0v8i32(
+; PREDFLAG:  %index.next = add i64 %index, 8
+; PREDFLAG:  %12 = icmp eq i64 %index.next, 432
+; PREDFLAG:  br i1 %12, label %middle.block, label %vector.body, !llvm.loop !0
 entry:
   br label %for.body
 
 for.cond.cleanup:
   ret void
 
 for.body:
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]

 }
 
 define dso_local void @tail_folding_disabled(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) local_unnamed_addr #0 {
 ; CHECK-LABEL: tail_folding_disabled(
 ; CHECK:      vector.body:
 ; CHECK-NOT:  @llvm.masked.load.v8i32.p0v8i32(
 ; CHECK-NOT:  @llvm.masked.store.v8i32.p0v8i32(
 ; CHECK:      br i1 %44, label {{.*}}, label %vector.body
+; PREDFLAG-LABEL: tail_folding_disabled(
+; PREDFLAG:  vector.body:
+; PREDFLAG:  %wide.masked.load = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(
+; PREDFLAG:  %wide.masked.load1 = call <8 x i32> @llvm.masked.load.v8i32.p0v8i32(
+; PREDFLAG:  %8 = add nsw <8 x i32> %wide.masked.load1, %wide.masked.load
+; PREDFLAG:  call void @llvm.masked.store.v8i32.p0v8i32(
+; PREDFLAG:  %index.next = add i64 %index, 8
+; PREDFLAG:  %12 = icmp eq i64 %index.next, 432
+; PREDFLAG:  br i1 %12, label %middle.block, label %vector.body, !llvm.loop !4
 entry:
   br label %for.body
 
 for.cond.cleanup:
   ret void
 
 for.body:
   %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]