use riscv vsetvl to tailfolding

[lizhengxian.123]

Summary

we found that llvm now doesn't support riscv vsetvl instruction to tail folding. we create a new cmdline option:

--prefer-predicate-over-epilogue=vlen-else-scalar-epilogue

which can do tali folding with riscv vsetvl instruction.

Co-authored-by: lidawei [e-mail]

[• pcwang-thead]

You don't have to create a new differential, you could just upload your new patch in D136583. For now, let's discuss in this differential.

I have got what you want to do, but I think you can refer to Vector Predication and works in llvm-epi.

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

115

It's target-dependent.
I would suggest that we add target hooks in TargetTransformInfo.

9813

Ditto.

9850

Ditto.

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

238

Ditto.

[reames]

FYI, this isn't going to be accepted in the current form. We don't allow the use of target specific intrinsics directly from the generic IR optimizer.

If you're interested in this approach, I recommend you take a look at the work that BSC has done using the VP intrinsics. I think that's the closest we have to something which is likely to eventually make it upstream. If you wanted to spend time working on that, you should talk to @simoll, and come to the next RISC-V sync up call or VP sync up. (The VP syncs have been getting canceled, so the general RISC-V sync is probably a better idea.) I'd also be happy to jump on a call and talk through this topic.

[kito-cheng]

D99750 should be better approach for that

llvm/include/llvm/Analysis/TargetTransformInfo.h

public:
 
   bool isTypeBasedOnly() const {
     return Arguments.empty();
   }
 
   bool skipScalarizationCost() const { return ScalarizationCost.isValid(); }
 };
 
-enum class PredicationStyle { None, Data, DataAndControlFlow };
+enum class PredicationStyle { None, Data, DataAndControlFlow, ImplictData };
 
 class TargetTransformInfo;
 typedef TargetTransformInfo TTI;
 
 /// This pass provides access to the codegen interfaces that are needed
 /// for IR-level transformations.
 class TargetTransformInfo {
 public:

llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h

InstructionCost getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
                                       TTI::TargetCostKind CostKind);
 
   TargetTransformInfo::PopcntSupportKind getPopcntSupport(unsigned TyWidth);
 
   bool shouldExpandReduction(const IntrinsicInst *II) const;
   bool supportsScalableVectors() const { return ST->hasVInstructions(); }
   bool enableScalableVectorization() const { return ST->hasVInstructions(); }
   PredicationStyle emitGetActiveLaneMask() const {
-    return ST->hasVInstructions() ? PredicationStyle::Data
+    return ST->hasVInstructions() ? PredicationStyle::ImplictData
                                   : PredicationStyle::None;
   }
   Optional<unsigned> getMaxVScale() const;
   Optional<unsigned> getVScaleForTuning() const;
 
   TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const;
 
   unsigned getRegUsageForType(Type *Ty);

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InstrTypes.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/IntrinsicsRISCV.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Operator.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Use.h"
 #include "llvm/IR/User.h"
 #include "llvm/IR/Value.h"
static cl::opt<unsigned> VectorizeMemoryCheckThreshold(
     cl::desc("The maximum allowed number of runtime memory checks"));
 
 // Option prefer-predicate-over-epilogue indicates that an epilogue is undesired,
 // that predication is preferred, and this lists all options. I.e., the
 // vectorizer will try to fold the tail-loop (epilogue) into the vector body
 // and predicate the instructions accordingly. If tail-folding fails, there are
 // different fallback strategies depending on these values:
 namespace PreferPredicateTy {
-  enum Option {
-    ScalarEpilogue = 0,
-    PredicateElseScalarEpilogue,
-    PredicateOrDontVectorize
-  };
+  VlElseScalarEpilogue,
 } // namespace PreferPredicateTy
 
 static cl::opt<PreferPredicateTy::Option> PreferPredicateOverEpilogue(
     "prefer-predicate-over-epilogue",
-    cl::init(PreferPredicateTy::ScalarEpilogue),
-    cl::Hidden,
+    cl::init(PreferPredicateTy::ScalarEpilogue), cl::Hidden,
     cl::desc("Tail-folding and predication preferences over creating a scalar "
              "epilogue loop."),
-    cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue,
-                         "scalar-epilogue",
-                         "Don't tail-predicate loops, create scalar epilogue"),
-              clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue,
-                         "predicate-else-scalar-epilogue",
-                         "prefer tail-folding, create scalar epilogue if tail "
-                         "folding fails."),
-              clEnumValN(PreferPredicateTy::PredicateOrDontVectorize,
-                         "predicate-dont-vectorize",
-                         "prefers tail-folding, don't attempt vectorization if "
-                         "tail-folding fails.")));
+    cl::values(
+        clEnumValN(PreferPredicateTy::ScalarEpilogue, "scalar-epilogue",
+        clEnumValN(PreferPredicateTy::VlElseScalarEpilogue,
+                   "vlen-else-scalar-epilogue",
+                   "prefer tail-folding, we use dynamic vlen to support it. "
+                   "Only works for rvv, create scalar epilogue if tail "
+                   "folding fails."),
 
 static cl::opt<bool> MaximizeBandwidth(
     "vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden,
     cl::desc("Maximize bandwidth when selecting vectorization factor which "
              "will be determined by the smallest type in loop."));
 
 static cl::opt<bool> EnableInterleavedMemAccesses(
     "enable-interleaved-mem-accesses", cl::init(false), cl::Hidden,
enum ScalarEpilogueLowering {
   // trip count are considered for vectorization under OptForSize, thereby
   // making sure the cost of their loop body is dominant, free of runtime
   // guards and scalar iteration overheads.
   CM_ScalarEpilogueNotAllowedLowTripLoop,
 
   // Loop hint predicate indicating an epilogue is undesired.
   CM_ScalarEpilogueNotNeededUsePredicate,
 
+  CM_ScalarEpilogueNotNeededUseVl,
+
   // Directive indicating we must either tail fold or not vectorize
   CM_ScalarEpilogueNotAllowedUsePredicate
 };
 
 /// ElementCountComparator creates a total ordering for ElementCount
 /// for the purposes of using it in a set structure.
 struct ElementCountComparator {
   bool operator()(const ElementCount &LHS, const ElementCount &RHS) const {
public:
   /// loop hint annotation.
   bool isScalarEpilogueAllowed() const {
     return ScalarEpilogueStatus == CM_ScalarEpilogueAllowed;
   }
 
   /// Returns true if all loop blocks should be masked to fold tail loop.
   bool foldTailByMasking() const { return FoldTailByMasking; }
 
+  bool foldTailByImplictMasking() const { return FoldTailByImplictMasking; }
+
   /// Returns true if were tail-folding and want to use the active lane mask
   /// for vector loop control flow.
   bool useActiveLaneMaskForControlFlow() const {
     return FoldTailByMasking &&
            TTI.emitGetActiveLaneMask() == PredicationStyle::DataAndControlFlow;
   }
 
   /// Returns true if the instructions in this block requires predication
private:
   /// or as a peel-loop to handle gaps in interleave-groups.
   /// Under optsize and when the trip count is very small we don't allow any
   /// iterations to execute in the scalar loop.
   ScalarEpilogueLowering ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
 
   /// All blocks of loop are to be masked to fold tail of scalar iterations.
   bool FoldTailByMasking = false;
 
+  /// All blocks of loop are to be masked to fold tail of scalar iterations by
+  /// vl.
+  bool FoldTailByImplictMasking = false;
+
   /// A map holding scalar costs for different vectorization factors. The
   /// presence of a cost for an instruction in the mapping indicates that the
   /// instruction will be scalarized when vectorizing with the associated
   /// vectorization factor. The entries are VF-ScalarCostTy pairs.
   DenseMap<ElementCount, ScalarCostsTy> InstsToScalarize;
 
   /// Holds the instructions known to be uniform after vectorization.
   /// The data is collected per VF.
LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
   case CM_ScalarEpilogueNotAllowedUsePredicate:
     [[fallthrough]];
   case CM_ScalarEpilogueNotNeededUsePredicate:
     LLVM_DEBUG(
         dbgs() << "LV: vector predicate hint/switch found.\n"
                << "LV: Not allowing scalar epilogue, creating predicated "
                << "vector loop.\n");
     break;
+  case CM_ScalarEpilogueNotNeededUseVl:
+    LLVM_DEBUG(errs() << "LV: vector Vl hint/switch found.\n"
+                      << "LV: Not allowing scalar epilogue, creating vlenset "
+                      << "vector loop.\n";);
+    break;
   case CM_ScalarEpilogueNotAllowedLowTripLoop:
     // fallthrough as a special case of OptForSize
   case CM_ScalarEpilogueNotAllowedOptSize:
     if (ScalarEpilogueStatus == CM_ScalarEpilogueNotAllowedOptSize)
       LLVM_DEBUG(
           dbgs() << "LV: Not allowing scalar epilogue due to -Os/-Oz.\n");
     else
       LLVM_DEBUG(dbgs() << "LV: Not allowing scalar epilogue due to low trip "
LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
 
   // The only loops we can vectorize without a scalar epilogue, are loops with
   // a bottom-test and a single exiting block. We'd have to handle the fact
   // that not every instruction executes on the last iteration.  This will
   // require a lane mask which varies through the vector loop body.  (TODO)
   if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch()) {
     // If there was a tail-folding hint/switch, but we can't fold the tail by
     // masking, fallback to a vectorization with a scalar epilogue.
-    if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate) {
-      LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with a "
-                           "scalar epilogue instead.\n");
+    if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate ||
+        ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUseVl) {
+      LLVM_DEBUG(
+          dbgs() << "LV: Cannot fold tail by masking or vl: vectorize with a "
       ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
       return computeFeasibleMaxVF(TC, UserVF, false);
     }
     return FixedScalableVFPair::getNone();
   }
 
   // Now try the tail folding
 
LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
   }
 
   // If we don't know the precise trip count, or if the trip count that we
   // found modulo the vectorization factor is not zero, try to fold the tail
   // by masking.
   // FIXME: look for a smaller MaxVF that does divide TC rather than masking.
   if (Legal->prepareToFoldTailByMasking()) {
     FoldTailByMasking = true;
+    if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUseVl)
+      FoldTailByImplictMasking = true;
     return MaxFactors;
   }
 
   // If there was a tail-folding hint/switch, but we can't fold the tail by
   // masking, fallback to a vectorization with a scalar epilogue.
-  if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate) {
+  if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate ||
+      ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUseVl) {
     LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with a "
                          "scalar epilogue instead.\n");
     ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
     return MaxFactors;
   }
 
   if (ScalarEpilogueStatus == CM_ScalarEpilogueNotAllowedUsePredicate) {
     LLVM_DEBUG(dbgs() << "LV: Can't fold tail by masking: don't vectorize\n");
void LoopVectorizationPlanner::executePlan(ElementCount BestVF, unsigned BestUF,
                                            bool IsEpilogueVectorization) {
   LLVM_DEBUG(dbgs() << "Executing best plan with VF=" << BestVF << ", UF=" << BestUF
                     << '\n');
 
   // Perform the actual loop transformation.
 
   // 1. Set up the skeleton for vectorization, including vector pre-header and
   // middle block. The vector loop is created during VPlan execution.
-  VPTransformState State{BestVF, BestUF, LI, DT, ILV.Builder, &ILV, &BestVPlan};
+  unsigned WidestType;
+
+  std::tie(std::ignore, WidestType) = CM.getSmallestAndWidestTypes();
+  VPTransformState State{BestVF,      BestUF, LI,         DT,
+                         ILV.Builder, &ILV,   &BestVPlan, WidestType};
   Value *CanonicalIVStartValue;
   std::tie(State.CFG.PrevBB, CanonicalIVStartValue) =
       ILV.createVectorizedLoopSkeleton();
 
   // Only use noalias metadata when using memory checks guaranteeing no overlap
   // across all iterations.
   const LoopAccessInfo *LAI = ILV.Legal->getLAI();
   if (LAI && !LAI->getRuntimePointerChecking()->getChecks().empty() &&
if (OrigLoop->getHeader() == BB) {
     auto NewInsertionPoint = HeaderVPBB->getFirstNonPhi();
     auto *IV = new VPWidenCanonicalIVRecipe(Plan->getCanonicalIV());
     HeaderVPBB->insert(IV, HeaderVPBB->getFirstNonPhi());
 
     VPBuilder::InsertPointGuard Guard(Builder);
     Builder.setInsertPoint(HeaderVPBB, NewInsertionPoint);
     if (EmitGetActiveLaneMask != PredicationStyle::None) {
       VPValue *TC = Plan->getOrCreateTripCount();
-      BlockMask = Builder.createNaryOp(VPInstruction::ActiveLaneMask, {IV, TC},
-                                       nullptr, "active.lane.mask");
+      if (EmitGetActiveLaneMask == PredicationStyle::Data) {
+        assert((!CM.foldTailByImplictMasking()) &&
+               "target doesn't support dynamic vl");
+        BlockMask = Builder.createNaryOp(VPInstruction::ActiveLaneMask,
+                                         {IV, TC}, nullptr, "active.lane.mask");
+      }
+      // riscv vsetvl is implicit predicate ImplictData
+      else {
+        if (CM.foldTailByImplictMasking())
+          BlockMask = Builder.createNaryOp(VPInstruction::GetDynamicVl,
+                                           {IV, TC}, nullptr, "get.dynamic.vl");
+        else
+          BlockMask =
+              Builder.createNaryOp(VPInstruction::ActiveLaneMask, {IV, TC},
+      }
     } else {
       VPValue *BTC = Plan->getOrCreateBackedgeTakenCount();
       BlockMask = Builder.createNaryOp(VPInstruction::ICmpULE, {IV, BTC});
     }
     return BlockMaskCache[BB] = BlockMask;
   }
 
   // This is the block mask. We OR all incoming edges.
void VPWidenMemoryInstructionRecipe::execute(VPTransformState &State) {
   if (SI) {
     State.setDebugLocFromInst(SI);
 
     for (unsigned Part = 0; Part < State.UF; ++Part) {
       Instruction *NewSI = nullptr;
       Value *StoredVal = State.get(StoredValue, Part);
       if (CreateGatherScatter) {
         Value *MaskPart = isMaskRequired ? BlockInMaskParts[Part] : nullptr;
+        assert(
+            (MaskPart == nullptr || !MaskPart->getType()->isIntegerTy()) &&
+            " -prefer-predicate-over-epilogue=vlen-else-scalar-epilogue not "
+            "support gather scatter, you maybe prefer "
+            "-prefer-predicate-over-epilogue=predicate-else-scalar-epilogue");
         Value *VectorGep = State.get(getAddr(), Part);
         NewSI = Builder.CreateMaskedScatter(StoredVal, VectorGep, Alignment,
                                             MaskPart);
       } else {
         if (Reverse) {
           // If we store to reverse consecutive memory locations, then we need
           // to reverse the order of elements in the stored value.
           StoredVal = Builder.CreateVectorReverse(StoredVal, "reverse");
           // We don't want to update the value in the map as it might be used in
           // another expression. So don't call resetVectorValue(StoredVal).
         }
         auto *VecPtr =
             CreateVecPtr(Part, State.get(getAddr(), VPIteration(0, 0)));
-        if (isMaskRequired)
-          NewSI = Builder.CreateMaskedStore(StoredVal, VecPtr, Alignment,
-                                            BlockInMaskParts[Part]);
+        if (isMaskRequired) {
+          Value *Mask = BlockInMaskParts[Part];
+          Type *MaskTy = Mask->getType();
+          if (MaskTy->isIntegerTy())
+            NewSI = Builder.CreateIntrinsic(Intrinsic::riscv_vse,
+                                            {StoredVal->getType(), MaskTy},
+                                            {StoredVal, VecPtr, Mask}, nullptr);
           else
+            NewSI =
+                Builder.CreateMaskedStore(StoredVal, VecPtr, Alignment, Mask);
+        } else
           NewSI = Builder.CreateAlignedStore(StoredVal, VecPtr, Alignment);
       }
       State.addMetadata(NewSI, SI);
     }
     return;
   }
 
   // Handle loads.
   assert(LI && "Must have a load instruction");
   State.setDebugLocFromInst(LI);
   for (unsigned Part = 0; Part < State.UF; ++Part) {
     Value *NewLI;
     if (CreateGatherScatter) {
       Value *MaskPart = isMaskRequired ? BlockInMaskParts[Part] : nullptr;
+      assert((MaskPart == nullptr || !MaskPart->getType()->isIntegerTy()) &&
+             " -prefer-predicate-over-epilogue=vlen-else-scalar-epilogue not "
+             "support gather scatter, you maybe prefer "
+             "-prefer-predicate-over-epilogue=predicate-else-scalar-epilogue");
       Value *VectorGep = State.get(getAddr(), Part);
       NewLI = Builder.CreateMaskedGather(DataTy, VectorGep, Alignment, MaskPart,
                                          nullptr, "wide.masked.gather");
       State.addMetadata(NewLI, LI);
     } else {
       auto *VecPtr =
           CreateVecPtr(Part, State.get(getAddr(), VPIteration(0, 0)));
-      if (isMaskRequired)
+      if (isMaskRequired) {
+        Value *Mask = BlockInMaskParts[Part]; // State.get(getMask(), Part);
+        Type *MaskTy = Mask->getType();
+        if (MaskTy->isIntegerTy())
+          NewLI = Builder.CreateIntrinsic(
+              Intrinsic::riscv_vle, {DataTy, MaskTy},
+              {PoisonValue::get(DataTy), VecPtr, Mask}, nullptr);
+        else
           NewLI = Builder.CreateMaskedLoad(
               DataTy, VecPtr, Alignment, BlockInMaskParts[Part],
               PoisonValue::get(DataTy), "wide.masked.load");
-      else
+      } else
         NewLI =
             Builder.CreateAlignedLoad(DataTy, VecPtr, Alignment, "wide.load");
 
       // Add metadata to the load, but setVectorValue to the reverse shuffle.
       State.addMetadata(NewLI, LI);
       if (Reverse)
         NewLI = Builder.CreateVectorReverse(NewLI, "reverse");
     }
if (F->hasOptSize() || (llvm::shouldOptimizeForSize(L->getHeader(), PSI, BFI,
                           Hints.getForce() != LoopVectorizeHints::FK_Enabled))
     return CM_ScalarEpilogueNotAllowedOptSize;
 
   // 2) If set, obey the directives
   if (PreferPredicateOverEpilogue.getNumOccurrences()) {
     switch (PreferPredicateOverEpilogue) {
     case PreferPredicateTy::ScalarEpilogue:
       return CM_ScalarEpilogueAllowed;
+    case PreferPredicateTy::VlElseScalarEpilogue:
+      return CM_ScalarEpilogueNotNeededUseVl;
     case PreferPredicateTy::PredicateElseScalarEpilogue:
       return CM_ScalarEpilogueNotNeededUsePredicate;
     case PreferPredicateTy::PredicateOrDontVectorize:
       return CM_ScalarEpilogueNotAllowedUsePredicate;
     };
   }
 
   // 3) If set, obey the hints

llvm/lib/Transforms/Vectorize/VPlan.h

struct VPIteration {
   bool isFirstIteration() const { return Part == 0 && Lane.isFirstLane(); }
 };
 
 /// VPTransformState holds information passed down when "executing" a VPlan,
 /// needed for generating the output IR.
 struct VPTransformState {
   VPTransformState(ElementCount VF, unsigned UF, LoopInfo *LI,
                    DominatorTree *DT, IRBuilderBase &Builder,
-                   InnerLoopVectorizer *ILV, VPlan *Plan)
-      : VF(VF), UF(UF), LI(LI), DT(DT), Builder(Builder), ILV(ILV), Plan(Plan),
-        LVer(nullptr) {}
+                   InnerLoopVectorizer *ILV, VPlan *Plan, unsigned WidestTy)
+      : VF(VF), UF(UF), WidestTy(WidestTy), LI(LI), DT(DT), Builder(Builder),
+        ILV(ILV), Plan(Plan), LVer(nullptr) {}
 
   /// The chosen Vectorization and Unroll Factors of the loop being vectorized.
   ElementCount VF;
   unsigned UF;
+  unsigned WidestTy;
 
   /// Hold the indices to generate specific scalar instructions. Null indicates
   /// that all instances are to be generated, using either scalar or vector
   /// instructions.
   Optional<VPIteration> Instance;
 
   struct DataState {
     /// A type for vectorized values in the new loop. Each value from the
enum {
     FirstOrderRecurrenceSplice =
         Instruction::OtherOpsEnd + 1, // Combines the incoming and previous
                                       // values of a first-order recurrence.
     Not,
     ICmpULE,
     SLPLoad,
     SLPStore,
     ActiveLaneMask,
+    GetDynamicVl,
     CanonicalIVIncrement,
     CanonicalIVIncrementNUW,
     // The next two are similar to the above, but instead increment the
     // canonical IV separately for each unrolled part.
     CanonicalIVIncrementForPart,
     CanonicalIVIncrementForPartNUW,
     BranchOnCount,
     BranchOnCond
bool onlyFirstLaneUsed(const VPValue *Op) const override {
     assert(is_contained(operands(), Op) &&
            "Op must be an operand of the recipe");
     if (getOperand(0) != Op)
       return false;
     switch (getOpcode()) {
     default:
       return false;
     case VPInstruction::ActiveLaneMask:
+    case VPInstruction::GetDynamicVl:
     case VPInstruction::CanonicalIVIncrement:
     case VPInstruction::CanonicalIVIncrementNUW:
     case VPInstruction::CanonicalIVIncrementForPart:
     case VPInstruction::CanonicalIVIncrementForPartNUW:
     case VPInstruction::BranchOnCount:
       return true;
     };
     llvm_unreachable("switch should return");

llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

 //===----------------------------------------------------------------------===//
 
 #include "VPlan.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Analysis/IVDescriptors.h"
 #include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/Constants.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicsRISCV.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Value.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
 #include <cassert>
 
 using namespace llvm;
 
 using VectorParts = SmallVector<Value *, 2>;
void VPInstruction::generateInstruction(VPTransformState &State,
   case Instruction::Select: {
     Value *Cond = State.get(getOperand(0), Part);
     Value *Op1 = State.get(getOperand(1), Part);
     Value *Op2 = State.get(getOperand(2), Part);
     Value *V = Builder.CreateSelect(Cond, Op1, Op2, Name);
     State.set(this, V, Part);
     break;
   }
+  case VPInstruction::GetDynamicVl: {
+    // Get first lane of vector induction variable.
+    Value *VIVElem0 = State.get(getOperand(0), VPIteration(Part, 0));
+    // Get the original loop tripcount.
+    Value *ScalarTC = State.get(getOperand(1), Part);
+    ScalarTC = Builder.CreateIntCast(ScalarTC, VIVElem0->getType(), true);
+    Value *AvLen = Builder.CreateSub(ScalarTC, VIVElem0, "avl_length");
+    auto *IntTy = VIVElem0->getType();
+    // auto *PredTy = VectorType::get(Int1Ty, State.VF);
+    Instruction *Call = Builder.CreateIntrinsic(
+        Intrinsic::riscv_vsetvli, IntTy,
+        {AvLen, ConstantInt::get(IntTy, Log2_32(State.WidestTy / 8)),
+         ConstantInt::get(IntTy, Log2_32(State.VF.getKnownMinValue() *
+                                         State.WidestTy / 64))},
+        nullptr, Name);
+    State.set(this, Call, Part);
+    break;
+  }
   case VPInstruction::ActiveLaneMask: {
     // Get first lane of vector induction variable.
     Value *VIVElem0 = State.get(getOperand(0), VPIteration(Part, 0));
     // Get the original loop tripcount.
     Value *ScalarTC = State.get(getOperand(1), Part);
 
     auto *Int1Ty = Type::getInt1Ty(Builder.getContext());
     auto *PredTy = VectorType::get(Int1Ty, State.VF);
case VPInstruction::SLPLoad:
     O << "combined load";
     break;
   case VPInstruction::SLPStore:
     O << "combined store";
     break;
   case VPInstruction::ActiveLaneMask:
     O << "active lane mask";
     break;
+  case VPInstruction::GetDynamicVl:
+    O << "get dynamic vl";
+    break;
   case VPInstruction::FirstOrderRecurrenceSplice:
     O << "first-order splice";
     break;
   case VPInstruction::CanonicalIVIncrement:
     O << "VF * UF + ";
     break;
   case VPInstruction::CanonicalIVIncrementNUW:
     O << "VF * UF +(nuw) ";

llvm/test/Transforms/LoopVectorize/RISCV/prefer_tail_folding_with_vsetvl.ll

+; RUN: opt -mtriple riscv64-linux-gnu -mattr=+v,+f   \
+; RUN:   -loop-vectorize -scalable-vectorization=on  \
+; RUN: -prefer-predicate-over-epilogue=vlen-else-scalar-epilogue  -S < %s | \
+; RUN:   FileCheck %s -check-prefixes=CHECK,PREFER-FOLDING
+
+target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+; /data00/home/lizhengxian.123/company_code/llvm-dev/llvm_community/bytedance-riscv/llvm-project/build/bin/opt < tail_folding.ll -loop-vectorize -scalable-vectorization=on -prefer-predicate-over-epilogue=vlen-else-scalar-epilogue -mtriple riscv64-linux-gnu -mattr=+v,+f -S
+define void @prefer_folding(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) #0 {
+; CHECK-LABEL:    prefer_folding(
+; PREFER-FOLDING: vector.body:
+; PREFER-FOLDING: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; PREFER-FOLDING: %[[VIVELEM0:.*]] = add i32 %index, 0
+; PREFER-FOLDING: %avl_length = sub i32 431, %[[VIVELEM0]]
+; PREFER-FOLDING: %get.dynamic.vl = call i32 @llvm.riscv.vsetvli.i32(i32 %avl_length, i32 2, i32 0)
+; PREFER-FOLDING: call <vscale x 2 x i32> @llvm.riscv.vle.nxv2i32.i32({{.*}}, {{.*}}, i32 %get.dynamic.vl)
+; PREFER-FOLDING: call <vscale x 2 x i32> @llvm.riscv.vle.nxv2i32.i32({{.*}}, {{.*}}, i32 %get.dynamic.vl)
+; PREFER-FOLDING: call void @llvm.riscv.vse.nxv2i32.i32({{.*}}, {{.*}}, i32 %get.dynamic.vl)
+; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body
+;
+; NO-FOLDING-NOT: call <4 x i32> @llvm.masked.load.v4i32.p0v4i32(
+; NO-FOLDING-NOT: call void @llvm.masked.store.v4i32.p0v4i32(
+; NO-FOLDING:     br i1 %{{.*}}, label %{{.*}}, label %for.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %i.09
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %C, i32 %i.09
+  %1 = load i32, i32* %arrayidx1, align 4
+  %add = add nsw i32 %1, %0
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i32 %i.09
+  store i32 %add, i32* %arrayidx2, align 4
+  %add3 = add nuw nsw i32 %i.09, 1
+  %exitcond = icmp eq i32 %add3, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+
+define void @mixed_types(i16* noalias nocapture %A, i16* noalias nocapture readonly %B, i16* noalias nocapture readonly %C, i32* noalias nocapture %D, i32* noalias nocapture readonly %E, i32* noalias nocapture readonly %F) #0 {
+; CHECK-LABEL:        mixed_types(
+; PREFER-FOLDING:     vector.body:
+; PREFER-FOLDING: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; PREFER-FOLDING: %[[VIVELEM0:.*]] = add i32 %index, 0
+; PREFER-FOLDING: %avl_length = sub i32 431, %[[VIVELEM0]]
+; PREFER-FOLDING: %get.dynamic.vl = call i32 @llvm.riscv.vsetvli.i32(i32 %avl_length, i32 2, i32 0)
+; PREFER-FOLDING:     call <vscale x 2 x i16> @llvm.riscv.vle.nxv2i16.i32
+; PREFER-FOLDING:     call <vscale x 2 x i16> @llvm.riscv.vle.nxv2i16.i32
+; PREFER-FOLDING:     call void @llvm.riscv.vse.nxv2i16.i32
+; PREFER-FOLDING:     call <vscale x 2 x i32> @llvm.riscv.vle.nxv2i32.i32
+; PREFER-FOLDING:     call <vscale x 2 x i32> @llvm.riscv.vle.nxv2i32.i32
+; PREFER-FOLDING:     call void @llvm.riscv.vse.nxv2i32.i32
+; PREFER-FOLDING:     br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.018 = phi i32 [ 0, %entry ], [ %add9, %for.body ]
+  %arrayidx = getelementptr inbounds i16, i16* %B, i32 %i.018
+  %0 = load i16, i16* %arrayidx, align 2
+  %arrayidx1 = getelementptr inbounds i16, i16* %C, i32 %i.018
+  %1 = load i16, i16* %arrayidx1, align 2
+  %add = add i16 %1, %0
+  %arrayidx4 = getelementptr inbounds i16, i16* %A, i32 %i.018
+  store i16 %add, i16* %arrayidx4, align 2
+  %arrayidx5 = getelementptr inbounds i32, i32* %E, i32 %i.018
+  %2 = load i32, i32* %arrayidx5, align 4
+  %arrayidx6 = getelementptr inbounds i32, i32* %F, i32 %i.018
+  %3 = load i32, i32* %arrayidx6, align 4
+  %add7 = add nsw i32 %3, %2
+  %arrayidx8 = getelementptr inbounds i32, i32* %D, i32 %i.018
+  store i32 %add7, i32* %arrayidx8, align 4
+  %add9 = add nuw nsw i32 %i.018, 1
+  %exitcond = icmp eq i32 %add9, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define void @zero_extending_load_allowed(i32* noalias nocapture %A, i8* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) #0 {
+; CHECK-LABEL:    zero_extending_load_allowed(
+; PREFER-FOLDING: vector.body:
+; PREFER-FOLDING: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; PREFER-FOLDING: %[[VIVELEM0:.*]] = add i32 %index, 0
+; PREFER-FOLDING: %avl_length = sub i32 431, %[[VIVELEM0]]
+; PREFER-FOLDING: %get.dynamic.vl = call i32 @llvm.riscv.vsetvli.i32(i32 %avl_length, i32 2, i32 0)
+; PREFER-FOLDING: call <vscale x 2 x i8> @llvm.riscv.vle.nxv2i8.i32
+; PREFER-FOLDING: call <vscale x 2 x i32> @llvm.riscv.vle.nxv2i32.i32
+; PREFER-FOLDING: call void @llvm.riscv.vse.nxv2i32.i32
+; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds i8, i8* %B, i32 %i.09
+  %0 = load i8, i8* %arrayidx, align 1
+  %conv = zext i8 %0 to i32
+  %arrayidx1 = getelementptr inbounds i32, i32* %C, i32 %i.09
+  %1 = load i32, i32* %arrayidx1, align 4
+  %add = add nsw i32 %1, %conv
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i32 %i.09
+  store i32 %add, i32* %arrayidx2, align 4
+  %add3 = add nuw nsw i32 %i.09, 1
+  %exitcond = icmp eq i32 %add3, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define void @sign_extending_load_allowed(i32* noalias nocapture %A, i8* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) #0 {
+; CHECK-LABEL:    sign_extending_load_allowed(
+; PREFER-FOLDING: vector.body:
+; PREFER-FOLDING: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; PREFER-FOLDING: %[[VIVELEM0:.*]] = add i32 %index, 0
+; PREFER-FOLDING: %avl_length = sub i32 431, %[[VIVELEM0]]
+; PREFER-FOLDING: %get.dynamic.vl = call i32 @llvm.riscv.vsetvli.i32(i32 %avl_length, i32 2, i32 0)
+; PREFER-FOLDING: call <vscale x 2 x i8> @llvm.riscv.vle.nxv2i8.i32
+; PREFER-FOLDING: call <vscale x 2 x i32> @llvm.riscv.vle.nxv2i32.i32
+; PREFER-FOLDING: call void @llvm.riscv.vse.nxv2i32.i32
+; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds i8, i8* %B, i32 %i.09
+  %0 = load i8, i8* %arrayidx, align 1
+  %conv = sext i8 %0 to i32
+  %arrayidx1 = getelementptr inbounds i32, i32* %C, i32 %i.09
+  %1 = load i32, i32* %arrayidx1, align 4
+  %add = add nsw i32 %1, %conv
+  %arrayidx2 = getelementptr inbounds i32, i32* %A, i32 %i.09
+  store i32 %add, i32* %arrayidx2, align 4
+  %add3 = add nuw nsw i32 %i.09, 1
+  %exitcond = icmp eq i32 %add3, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define void @narrowing_store_allowed(i8* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) #0 {
+; CHECK-LABEL:    narrowing_store_allowed(
+; PREFER-FOLDING: call void @llvm.riscv.vse.nxv2i8.i32
+; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds i32, i32* %B, i32 %i.09
+  %0 = load i32, i32* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds i32, i32* %C, i32 %i.09
+  %1 = load i32, i32* %arrayidx1, align 4
+  %add = add nsw i32 %1, %0
+  %conv = trunc i32 %add to i8
+  %arrayidx2 = getelementptr inbounds i8, i8* %A, i32 %i.09
+  store i8 %conv, i8* %arrayidx2, align 1
+  %add3 = add nuw nsw i32 %i.09, 1
+  %exitcond = icmp eq i32 %add3, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define dso_local void @half(half* noalias nocapture %A, half* noalias nocapture readonly %B, half* noalias nocapture readonly %C) #0 {
+; CHECK-LABEL:    half(
+; PREFER-FOLDING: vector.body:
+; PREFER-FOLDING: %[[VIVELEM0:.*]] = add i32 %index, 0
+; PREFER-FOLDING: %avl_length = sub i32 431, %[[VIVELEM0]]
+; PREFER-FOLDING: %get.dynamic.vl = call i32 @llvm.riscv.vsetvli.i32(i32 %avl_length, i32 1, i32 0)
+; PREFER-FOLDING: call <vscale x 4 x half> @llvm.riscv.vle.nxv4f16.i32
+; PREFER-FOLDING: call <vscale x 4 x half> @llvm.riscv.vle.nxv4f16.i32
+; PREFER-FOLDING: call void @llvm.riscv.vse.nxv4f16.i32
+; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds half, half* %B, i32 %i.09
+  %0 = load half, half* %arrayidx, align 2
+  %arrayidx1 = getelementptr inbounds half, half* %C, i32 %i.09
+  %1 = load half, half* %arrayidx1, align 2
+  %add = fadd fast half %1, %0
+  %arrayidx2 = getelementptr inbounds half, half* %A, i32 %i.09
+  store half %add, half* %arrayidx2, align 2
+  %add3 = add nuw nsw i32 %i.09, 1
+  %exitcond = icmp eq i32 %add3, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define void @float(float* noalias nocapture %A, float* noalias nocapture readonly %B, float* noalias nocapture readonly %C) #0 {
+; CHECK-LABEL:    float(
+; PREFER-FOLDING: vector.body:
+; PREFER-FOLDING: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; PREFER-FOLDING: %[[VIVELEM0:.*]] = add i32 %index, 0
+; PREFER-FOLDING: %avl_length = sub i32 431, %[[VIVELEM0]]
+; PREFER-FOLDING: %get.dynamic.vl = call i32 @llvm.riscv.vsetvli.i32(i32 %avl_length, i32 2, i32 1)
+; PREFER-FOLDING: call <vscale x 4 x float> @llvm.riscv.vle.nxv4f32.i32
+; PREFER-FOLDING: call <vscale x 4 x float> @llvm.riscv.vle.nxv4f32.i32
+; PREFER-FOLDING: call void @llvm.riscv.vse.nxv4f32.i32
+; PREFER-FOLDING: %[[VSCALE:.*]] = call i32 @llvm.vscale.i32()
+; PREFER-FOLDING: %[[STEP:.*]] = mul i32 %[[VSCALE]], 4
+; PREFER-FOLDING: %index.next = add i32 %index, %[[STEP]]
+; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %B, i32 %i.09
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %C, i32 %i.09
+  %1 = load float, float* %arrayidx1, align 4
+  %add = fadd fast float %1, %0
+  %arrayidx2 = getelementptr inbounds float, float* %A, i32 %i.09
+  store float %add, float* %arrayidx2, align 4
+  %add3 = add nuw nsw i32 %i.09, 1
+  %exitcond = icmp eq i32 %add3, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !10
+}
+
+define void @fpext_allowed(float* noalias nocapture %A, half* noalias nocapture readonly %B, float* noalias nocapture readonly %C) #0 {
+; CHECK-LABEL:        fpext_allowed(
+; PREFER-FOLDING:     vector.body:
+; PREFER-FOLDING: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; PREFER-FOLDING: %[[VIVELEM0:.*]] = add i32 %index, 0
+; PREFER-FOLDING: %avl_length = sub i32 431, %[[VIVELEM0]]
+; PREFER-FOLDING: %get.dynamic.vl = call i32 @llvm.riscv.vsetvli.i32(i32 %avl_length, i32 2, i32 0)
+; PREFER-FOLDING: call <vscale x 2 x half> @llvm.riscv.vle.nxv2f16.i32
+; PREFER-FOLDING: call <vscale x 2 x float> @llvm.riscv.vle.nxv2f32.i32
+; PREFER-FOLDING: call void @llvm.riscv.vse.nxv2f32.i32
+; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds half, half* %B, i32 %i.09
+  %0 = load half, half* %arrayidx, align 2
+  %conv = fpext half %0 to float
+  %arrayidx1 = getelementptr inbounds float, float* %C, i32 %i.09
+  %1 = load float, float* %arrayidx1, align 4
+  %add = fadd fast float %1, %conv
+  %arrayidx2 = getelementptr inbounds float, float* %A, i32 %i.09
+  store float %add, float* %arrayidx2, align 4
+  %add3 = add nuw nsw i32 %i.09, 1
+  %exitcond = icmp eq i32 %add3, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+define void @fptrunc_allowed(half* noalias nocapture %A, float* noalias nocapture readonly %B, float* noalias nocapture readonly %C) #0 {
+; CHECK-LABEL:        fptrunc_allowed(
+; PREFER-FOLDING:     vector.body:
+; PREFER-FOLDING: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
+; PREFER-FOLDING: %[[VIVELEM0:.*]] = add i32 %index, 0
+; PREFER-FOLDING: %avl_length = sub i32 431, %[[VIVELEM0]]
+; PREFER-FOLDING: %get.dynamic.vl = call i32 @llvm.riscv.vsetvli.i32(i32 %avl_length, i32 2, i32 0)
+; PREFER-FOLDING: call <vscale x 2 x float> @llvm.riscv.vle.nxv2f32.i32
+; PREFER-FOLDING: call <vscale x 2 x float> @llvm.riscv.vle.nxv2f32.i32
+; PREFER-FOLDING: call void @llvm.riscv.vse.nxv2f16.i32
+; PREFER-FOLDING: br i1 %{{.*}}, label %{{.*}}, label %vector.body
+entry:
+  br label %for.body
+
+for.cond.cleanup:
+  ret void
+
+for.body:
+  %i.09 = phi i32 [ 0, %entry ], [ %add3, %for.body ]
+  %arrayidx = getelementptr inbounds float, float* %B, i32 %i.09
+  %0 = load float, float* %arrayidx, align 4
+  %arrayidx1 = getelementptr inbounds float, float* %C, i32 %i.09
+  %1 = load float, float* %arrayidx1, align 4
+  %add = fadd fast float %1, %0
+  %conv = fptrunc float %add to half
+  %arrayidx2 = getelementptr inbounds half, half* %A, i32 %i.09
+  store half %conv, half* %arrayidx2, align 2
+  %add3 = add nuw nsw i32 %i.09, 1
+  %exitcond = icmp eq i32 %add3, 431
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}
+
+attributes #0 = { "target-features"="+v,+f,+experimental-zvfh,+zfh" }
+!5 = distinct !{!5, !6}
+!6 = !{!"llvm.loop.vectorize.enable", i1 true}
+
+!7 = distinct !{!7, !8}
+!8 = !{!"llvm.loop.vectorize.predicate.enable", i1 false}
+
+!10 = distinct !{!10, !11}
+!11 = !{!"llvm.loop.vectorize.width", i32 4}