diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -208,35 +208,35 @@ // and predicate the instructions accordingly. If tail-folding fails, there are // different fallback strategies depending on these values: namespace PreferPredicateTy { - enum Option { - ScalarEpilogue = 0, - PredicateElseScalarEpilogue, - VlenElseScalarEpilogue, - PredicateOrDontVectorize - }; +enum Option { + ScalarEpilogue = 0, + PredicateElseScalarEpilogue, + VlElseScalarEpilogue, + PredicateOrDontVectorize +}; } // namespace PreferPredicateTy static cl::opt PreferPredicateOverEpilogue( "prefer-predicate-over-epilogue", - cl::init(PreferPredicateTy::ScalarEpilogue), - cl::Hidden, + cl::init(PreferPredicateTy::ScalarEpilogue), cl::Hidden, cl::desc("Tail-folding and predication preferences over creating a scalar " "epilogue loop."), - cl::values(clEnumValN(PreferPredicateTy::ScalarEpilogue, - "scalar-epilogue", - "Don't tail-predicate loops, create scalar epilogue"), - clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue, - "predicate-else-scalar-epilogue", - "prefer tail-folding, create scalar epilogue if tail " - "folding fails."), - clEnumValN(PreferPredicateTy::VlenElseScalarEpilogue, - "vlen-else-scalar-epilogue", - "prefer tail-folding, we use dynamic vlen to support it. Only works for rvv, create scalar epilogue if tail " - "folding fails."), - clEnumValN(PreferPredicateTy::PredicateOrDontVectorize, - "predicate-dont-vectorize", - "prefers tail-folding, don't attempt vectorization if " - "tail-folding fails."))); + cl::values( + clEnumValN(PreferPredicateTy::ScalarEpilogue, "scalar-epilogue", + "Don't tail-predicate loops, create scalar epilogue"), + clEnumValN(PreferPredicateTy::PredicateElseScalarEpilogue, + "predicate-else-scalar-epilogue", + "prefer tail-folding, create scalar epilogue if tail " + "folding fails."), + clEnumValN(PreferPredicateTy::VlElseScalarEpilogue, + "vlen-else-scalar-epilogue", + "prefer tail-folding, we use dynamic vlen to support it. " + "Only works for rvv, create scalar epilogue if tail " + "folding fails."), + clEnumValN(PreferPredicateTy::PredicateOrDontVectorize, + "predicate-dont-vectorize", + "prefers tail-folding, don't attempt vectorization if " + "tail-folding fails."))); static cl::opt MaximizeBandwidth( "vectorizer-maximize-bandwidth", cl::init(false), cl::Hidden, @@ -1115,7 +1115,7 @@ // Loop hint predicate indicating an epilogue is undesired. CM_ScalarEpilogueNotNeededUsePredicate, - CM_ScalarEpilogueNotNeededUseVlen, + CM_ScalarEpilogueNotNeededUseVl, // Directive indicating we must either tail fold or not vectorize CM_ScalarEpilogueNotAllowedUsePredicate @@ -5005,11 +5005,11 @@ return computeFeasibleMaxVF(TC, UserVF, false); case CM_ScalarEpilogueNotAllowedUsePredicate: LLVM_FALLTHROUGH; - case CM_ScalarEpilogueNotNeededUseVlen: - LLVM_DEBUG(errs() << "LV: vector Vlen hint/switch found.\n" - << "LV: Not allowing scalar epilogue, creating vlenset " - << "vector loop.\n";); - break; + case CM_ScalarEpilogueNotNeededUseVl: + LLVM_DEBUG(errs() << "LV: vector Vlen hint/switch found.\n" + << "LV: Not allowing scalar epilogue, creating vlenset " + << "vector loop.\n";); + break; case CM_ScalarEpilogueNotNeededUsePredicate: LLVM_DEBUG( dbgs() << "LV: vector predicate hint/switch found.\n" @@ -5041,7 +5041,8 @@ if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch()) { // If there was a tail-folding hint/switch, but we can't fold the tail by // masking, fallback to a vectorization with a scalar epilogue. - if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate|| ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUseVlen) { + if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate || + ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUseVl) { LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking or vlen: vectorize with a " "scalar epilogue instead.\n"); ScalarEpilogueStatus = CM_ScalarEpilogueAllowed; @@ -5093,14 +5094,14 @@ // FIXME: look for a smaller MaxVF that does divide TC rather than masking. if (Legal->prepareToFoldTailByMasking()) { FoldTailByMasking = true; - if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUseVlen) + if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUseVl) FoldTailByImplictMasking = true; return MaxFactors; } // If there was a tail-folding hint/switch, but we can't fold the tail by // masking, fallback to a vectorization with a scalar epilogue. - if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate|| - ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUseVlen) { + if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate || + ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUseVl) { LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking or vlen: vectorize with a " "scalar epilogue instead.\n"); ScalarEpilogueStatus = CM_ScalarEpilogueAllowed; @@ -9779,8 +9780,8 @@ switch (PreferPredicateOverEpilogue) { case PreferPredicateTy::ScalarEpilogue: return CM_ScalarEpilogueAllowed; - case PreferPredicateTy::VlenElseScalarEpilogue: - return CM_ScalarEpilogueNotNeededUseVlen; + case PreferPredicateTy::VlElseScalarEpilogue: + return CM_ScalarEpilogueNotNeededUseVl; case PreferPredicateTy::PredicateElseScalarEpilogue: return CM_ScalarEpilogueNotNeededUsePredicate; case PreferPredicateTy::PredicateOrDontVectorize: diff --git a/vectorizationREADME.md b/vectorizationREADME.md --- a/vectorizationREADME.md +++ b/vectorizationREADME.md @@ -1,7 +1,9 @@ # 向量化tail folding功能实现 ## 注意事项 添加新选项: -```--prefer-predicate-over-epilogue=vlen-else-scalar-epilogue ``` +```--prefer-predicate-over-epilogue=vlen-else-scalar-epilogue ``` ,可以在riscv 后端使用vsetvl指令进行tail folding +在test/Transform/LoopVectorize/RISCV添加测试prefer_tail_folding_with_vsetvl.ll 测试,是由arm下的文件修改而来,满足了关于各种类型,各种类型混合,指定向量宽度后下tail_folding的正确性 +【WIP】:vector scatter gather 基于目前基础设施无法实现tail folding ,直接用assert表示 ## 环境 类似于如下构建 ```