Index: llvm/docs/LangRef.rst =================================================================== --- llvm/docs/LangRef.rst +++ llvm/docs/LangRef.rst @@ -5440,7 +5440,7 @@ !1 = !{!"llvm.loop.vectorize.enable", i1 1} '``llvm.loop.vectorize.predicate.enable``' Metadata -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ This metadata selectively enables or disables creating predicated instructions for the loop, which can enable folding of the scalar epilogue loop into the @@ -5454,6 +5454,25 @@ !0 = !{!"llvm.loop.vectorize.predicate.enable", i1 0} !1 = !{!"llvm.loop.vectorize.predicate.enable", i1 1} +'``llvm.loop.aligned.enable``' Metadata +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This metadata selectively enables or disables creating aligned +instructions +for the loop, the aligned store instruction can be generated for +load/store into the +main loop. The first operand is the string +``llvm.loop.aligned.enable`` and the second operand is a bit. If the +bit operand value is 1 aligned store instruction is enabled. A value +of 0 disables aligned load/store: + +.. code-block::llvm + + !0 = !{!"llvm.loop.aligned.enable", i1 1} + !1 = !{!"llvm.loop.aligned.disable", i1 0} + + + '``llvm.loop.vectorize.width``' Metadata ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Index: llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h =================================================================== --- llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h +++ llvm/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h @@ -44,7 +44,7 @@ /// careful NOT to add them if the user hasn't specifically asked so. class LoopVectorizeHints { enum HintKind { HK_WIDTH, HK_UNROLL, HK_FORCE, HK_ISVECTORIZED, - HK_PREDICATE }; + HK_PREDICATE, HK_ALIGNED }; /// Hint - associates name and validation with the hint value. struct Hint { @@ -73,6 +73,9 @@ /// Vector Predicate Hint Predicate; + /// references aligned + Hint IsAligned; + /// Return the loop metadata prefix. static StringRef Prefix() { return "llvm.loop."; } @@ -102,6 +105,7 @@ unsigned getInterleave() const { return Interleave.Value; } unsigned getIsVectorized() const { return IsVectorized.Value; } unsigned getPredicate() const { return Predicate.Value; } + unsigned getAligned() const { return IsAligned.Value; } enum ForceKind getForce() const { if ((ForceKind)Force.Value == FK_Undefined && hasDisableAllTransformsHint(TheLoop)) Index: llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp +++ llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp @@ -203,6 +203,9 @@ /// Check if this load/store access is misaligned accesses. bool accessIsMisaligned(unsigned SzInBytes, unsigned AddressSpace, unsigned Alignment); + + // /// Align all the array references + // bool alignArrayAccess(ArrayRef Instrs, const DataLayout &DL); }; class LoadStoreVectorizerLegacyPass : public FunctionPass { Index: llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp +++ llvm/lib/Transforms/Vectorize/LoopVectorizationLegality.cpp @@ -58,6 +58,7 @@ return (Val <= 1); case HK_ISVECTORIZED: case HK_PREDICATE: + case HK_ALIGNED: return (Val == 0 || Val == 1); } return false; @@ -70,7 +71,8 @@ Interleave("interleave.count", InterleaveOnlyWhenForced, HK_UNROLL), Force("vectorize.enable", FK_Undefined, HK_FORCE), IsVectorized("isvectorized", 0, HK_ISVECTORIZED), - Predicate("vectorize.predicate.enable", 0, HK_PREDICATE), TheLoop(L), + Predicate("vectorize.predicate.enable", 0, HK_PREDICATE), + IsAligned("aligned.enable", 0, HK_ALIGNED), TheLoop(L), ORE(ORE) { // Populate values with existing loop metadata. getHintsFromMetadata(); @@ -222,7 +224,8 @@ return; unsigned Val = C->getZExtValue(); - Hint *Hints[] = {&Width, &Interleave, &Force, &IsVectorized, &Predicate}; + Hint *Hints[] = {&Width, &Interleave, &Force, &IsVectorized, &Predicate, + &IsAligned }; for (auto H : Hints) { if (Name == H->Name) { if (H->validate(Val)) Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -803,6 +803,7 @@ B.SetCurrentDebugLocation(DebugLoc()); } + /// Write a record \p DebugMsg about vectorization failure to the debug /// output stream. If \p I is passed, it is an instruction that prevents /// vectorization. @@ -7312,6 +7313,37 @@ return SEL; } +void alignAccess(Loop *L, + LoopVectorizeHints &Hints){ + + if(!(Hints.getAligned())){ + return; + } + for(auto *BB : L->getBlocks()){ + for(auto &I : *BB){ + LoadInst *LI = dyn_cast(&I); + StoreInst *SI = dyn_cast(&I); + //Handle stores + if(SI){ + // LLVM_DEBUG(dbgs()<<"Store "<<++st<<"\n"); + // CreateAlignedLoad(SI->getType, SI->getPointerElementType(), 32, "aligned.load"); + SI->setAlignment(32); + // unsigned Alignment = getLoadStoreAlignment(SI); + // LLVM_DEBUG(dbgs()<<" alignment of store after" << Alignment<<"\n"); + } + //Handle load/store + if(LI){ + // LLVM_DEBUG(dbgs()<<"load "<<++ld<<"\n"); + LI->setAlignment(32); + // unsigned Alignment = getLoadStoreAlignment(LI); + // LLVM_DEBUG(dbgs()<<" alignment of load after setting" << Alignment<<"\n"); + } + + } + } +} + + // Process the loop in the VPlan-native vectorization path. This path builds // VPlan upfront in the vectorization pipeline, which allows to apply // VPlan-to-VPlan transformations from the very beginning without modifying the @@ -7386,7 +7418,8 @@ ? "enabled" : "?")) << " width=" << Hints.getWidth() - << " unroll=" << Hints.getInterleave() << "\n"); + << " unroll=" << Hints.getInterleave() + << " aligned=" << Hints.getAligned()<<"\n"); // Function containing loop Function *F = L->getHeader()->getParent(); @@ -7647,6 +7680,7 @@ // If we decided that it is *legal* to vectorize the loop, then do it. InnerLoopVectorizer LB(L, PSE, LI, DT, TLI, TTI, AC, ORE, VF.Width, IC, &LVL, &CM); + alignAccess(L, Hints); LVP.executePlan(LB, DT); ++LoopsVectorized;