[LV] PreferPredicateOverEpilog respecting predicate loop hint

[SjoerdMeijer]

Summary

The vectoriser queries TTI->preferPredicateOverEpilogue to determine whether tail-folding is preferred for a loop, but it was not respecting loop hint 'predicate' that can disable this, which has now been added. This showed that we were incorrectly initialising loop hint 'vectorize.predicate.enable' with 0 (i.e. FK_Disabled) but this should have been FK_Undefined, which has been fixed.

[samparker]

Looks sensbile.

[SjoerdMeijer]

Thanks for reviewing!
I now see that I would have to follow up on this to in addition also respect the option when that disables predication. But I will do that separately, one functional change at a time, since this essentially is a bug fix for the incorrectly initialised loop hint.

llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp

 
 LoopVectorizeHints::LoopVectorizeHints(const Loop *L,
                                        bool InterleaveOnlyWhenForced,
                                        OptimizationRemarkEmitter &ORE)
     : Width("vectorize.width", VectorizerParams::VectorizationFactor, HK_WIDTH),
       Interleave("interleave.count", InterleaveOnlyWhenForced, HK_UNROLL),
       Force("vectorize.enable", FK_Undefined, HK_FORCE),
       IsVectorized("isvectorized", 0, HK_ISVECTORIZED),
-      Predicate("vectorize.predicate.enable", 0, HK_PREDICATE), TheLoop(L),
+      Predicate("vectorize.predicate.enable", FK_Undefined, HK_PREDICATE), TheLoop(L),
       ORE(ORE) {
   // Populate values with existing loop metadata.
   getHintsFromMetadata();
 
   // force-vector-interleave overrides DisableInterleaving.
   if (VectorizerParams::isInterleaveForced())
     Interleave.Value = VectorizerParams::VectorizationInterleave;
 

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

getScalarEpilogueLowering(Function *F, Loop *L, LoopVectorizeHints &Hints,
                           AssumptionCache *AC, LoopInfo *LI,
                           ScalarEvolution *SE, DominatorTree *DT,
                           const LoopAccessInfo *LAI) {
   ScalarEpilogueLowering SEL = CM_ScalarEpilogueAllowed;
   if (Hints.getForce() != LoopVectorizeHints::FK_Enabled &&
       (F->hasOptSize() ||
        llvm::shouldOptimizeForSize(L->getHeader(), PSI, BFI)))
     SEL = CM_ScalarEpilogueNotAllowedOptSize;
-  else if (PreferPredicateOverEpilog || Hints.getPredicate() ||
-           TTI->preferPredicateOverEpilogue(L, LI, *SE, *AC, TLI, DT, LAI))
+  else if (PreferPredicateOverEpilog ||
+           Hints.getPredicate() == LoopVectorizeHints::FK_Enabled ||
+           (TTI->preferPredicateOverEpilogue(L, LI, *SE, *AC, TLI, DT, LAI) &&
+            Hints.getPredicate() != LoopVectorizeHints::FK_Disabled))
     SEL = CM_ScalarEpilogueNotNeededUsePredicate;
 
   return SEL;
 }
 
 // Process the loop in the VPlan-native vectorization path. This path builds
 // VPlan upfront in the vectorization pipeline, which allows to apply
 // VPlan-to-VPlan transformations from the very beginning without modifying the

llvm/test/Transforms/LoopVectorize/ARM/prefer-tail-loop-folding.ll

for.body:
   %exitcond = icmp slt i32 %inc, 1000
   br i1 %exitcond, label %for.body, label %for.end
 
 for.end:
   ret i32 0
 }
 
 define void @pragma_vect_predicate_disable(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) #0 {
-; CHECK-LABEL: pragma_vect_predicate_disable(
-;
-; FIXME:
-; respect loop hint predicate.enable = false, and don't tail-fold here:
-;
-; PREFER-FOLDING: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32
-; PREFER-FOLDING: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32
-; PREFER-FOLDING: call void @llvm.masked.store.v4i32.p0v4i32
-; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body
+; CHECK-LABEL:        pragma_vect_predicate_disable(
+; PREFER-FOLDING:     vector.body:
+; PREFER-FOLDING-NOT: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32
+; PREFER-FOLDING-NOT: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32
+; PREFER-FOLDING-NOT: call void @llvm.masked.store.v4i32.p0v4i32
+; PREFER-FOLDING:     br i1 %{{.*}}, label %{{.*}}, label %vector.body
 entry:
   br label %for.body
 
 for.cond.cleanup:
   ret void
 
 for.body:
   %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]