Index: lib/Transforms/Vectorize/LoopVectorize.cpp =================================================================== --- lib/Transforms/Vectorize/LoopVectorize.cpp +++ lib/Transforms/Vectorize/LoopVectorize.cpp @@ -580,9 +580,10 @@ LoopVectorizationLegality(Loop *L, ScalarEvolution *SE, const DataLayout *DL, DominatorTree *DT, TargetLibraryInfo *TLI, - AliasAnalysis *AA, Function *F) + AliasAnalysis *AA, Function *F, + const TargetTransformInfo *TTI) : NumLoads(0), NumStores(0), NumPredStores(0), TheLoop(L), SE(SE), DL(DL), - DT(DT), TLI(TLI), AA(AA), TheFunction(F), Induction(nullptr), + DT(DT), TLI(TLI), AA(AA), TheFunction(F), TTI(TTI), Induction(nullptr), WidestIndTy(nullptr), HasFunNoNaNAttr(false), MaxSafeDepDistBytes(-1U) { } @@ -768,6 +769,15 @@ } SmallPtrSet::iterator strides_end() { return StrideSet.end(); } + bool canPredicateStore(Type *DataType, Value *Ptr) { + return TTI->isLegalPredicatedStore(DataType, isConsecutivePtr(Ptr)); + } + bool canPredicateLoad(Type *DataType, Value *Ptr) { + return TTI->isLegalPredicatedLoad(DataType, isConsecutivePtr(Ptr)); + } + bool setMaskedOp(const Instruction* I) { + return (MaskedOp.find(I) != MaskedOp.end()); + } private: /// Check if a single basic block loop is vectorizable. /// At this point we know that this is a loop with a constant trip count @@ -840,6 +850,8 @@ AliasAnalysis *AA; /// Parent function Function *TheFunction; + /// Target Transform Info + const TargetTransformInfo *TTI; // --- vectorization state --- // @@ -871,6 +883,10 @@ ValueToValueMap Strides; SmallPtrSet StrideSet; + + /// While vectorizing these instructions we have to generate a + /// call to an appropriate masked intrinsic + std::set MaskedOp; }; /// LoopVectorizationCostModel - estimates the expected speedups due to @@ -1373,7 +1389,7 @@ } // Check if it is legal to vectorize the loop. - LoopVectorizationLegality LVL(L, SE, DL, DT, TLI, AA, F); + LoopVectorizationLegality LVL(L, SE, DL, DT, TLI, AA, F, TTI); if (!LVL.canVectorize()) { DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n"); emitMissedWarning(F, L, Hints); @@ -1761,7 +1777,8 @@ unsigned ScalarAllocatedSize = DL->getTypeAllocSize(ScalarDataTy); unsigned VectorElementSize = DL->getTypeStoreSize(DataTy)/VF; - if (SI && Legal->blockNeedsPredication(SI->getParent())) + if (SI && Legal->blockNeedsPredication(SI->getParent()) && + !Legal->setMaskedOp(SI)) return scalarizeInstruction(Instr, true); if (ScalarAllocatedSize != VectorElementSize) @@ -1855,8 +1872,24 @@ Value *VecPtr = Builder.CreateBitCast(PartPtr, DataTy->getPointerTo(AddressSpace)); - StoreInst *NewSI = - Builder.CreateAlignedStore(StoredVal[Part], VecPtr, Alignment); + + Instruction *NewSI; + if (Legal->setMaskedOp(SI)) { + Type *I8PtrTy = + Builder.getInt8PtrTy(PartPtr->getType()->getPointerAddressSpace()); + + Value *I8Ptr = Builder.CreateBitCast(PartPtr, I8PtrTy); + + VectorParts Cond = createBlockInMask(SI->getParent()); + SmallVector Ops; + Ops.push_back(I8Ptr); + Ops.push_back(StoredVal[Part]); + Ops.push_back(Builder.getInt32(Alignment)); + Ops.push_back(Cond[Part]); + NewSI = Builder.CreateMaskedStore(Ops); + } + else + NewSI = Builder.CreateAlignedStore(StoredVal[Part], VecPtr, Alignment); propagateMetadata(NewSI, SI); } return; @@ -1871,14 +1904,31 @@ if (Reverse) { // If the address is consecutive but reversed, then the - // wide store needs to start at the last vector element. + // wide load needs to start at the last vector element. PartPtr = Builder.CreateGEP(Ptr, Builder.getInt32(-Part * VF)); PartPtr = Builder.CreateGEP(PartPtr, Builder.getInt32(1 - VF)); } - Value *VecPtr = Builder.CreateBitCast(PartPtr, - DataTy->getPointerTo(AddressSpace)); - LoadInst *NewLI = Builder.CreateAlignedLoad(VecPtr, Alignment, "wide.load"); + Instruction* NewLI; + if (Legal->setMaskedOp(LI)) { + Type *I8PtrTy = + Builder.getInt8PtrTy(PartPtr->getType()->getPointerAddressSpace()); + + Value *I8Ptr = Builder.CreateBitCast(PartPtr, I8PtrTy); + + VectorParts SrcMask = createBlockInMask(LI->getParent()); + SmallVector Ops; + Ops.push_back(I8Ptr); + Ops.push_back(UndefValue::get(DataTy)); + Ops.push_back(Builder.getInt32(Alignment)); + Ops.push_back(SrcMask[Part]); + NewLI = Builder.CreateMaskedLoad(Ops); + } + else { + Value *VecPtr = Builder.CreateBitCast(PartPtr, + DataTy->getPointerTo(AddressSpace)); + NewLI = Builder.CreateAlignedLoad(VecPtr, Alignment, "wide.load"); + } propagateMetadata(NewLI, LI); Entry[Part] = Reverse ? reverseVector(NewLI) : NewLI; } @@ -5311,8 +5361,15 @@ // We might be able to hoist the load. if (it->mayReadFromMemory()) { LoadInst *LI = dyn_cast(it); - if (!LI || !SafePtrs.count(LI->getPointerOperand())) + if (!LI) return false; + if (!SafePtrs.count(LI->getPointerOperand())) { + if (canPredicateLoad(LI->getType(), LI->getPointerOperand())) { + MaskedOp.insert(LI); + continue; + } + return false; + } } // We don't predicate stores at the moment. @@ -5320,10 +5377,20 @@ StoreInst *SI = dyn_cast(it); // We only support predication of stores in basic blocks with one // predecessor. - if (!SI || ++NumPredStores > NumberOfStoresToPredicate || + if (!SI) + return false; + + if (++NumPredStores > NumberOfStoresToPredicate || !SafePtrs.count(SI->getPointerOperand()) || - !SI->getParent()->getSinglePredecessor()) + !SI->getParent()->getSinglePredecessor()) { + if (canPredicateStore(SI->getValueOperand()->getType(), + SI->getPointerOperand())) { + MaskedOp.insert(SI); + --NumPredStores; + continue; + } return false; + } } if (it->mayThrow()) return false; @@ -5387,7 +5454,7 @@ MaxVectorSize = 1; } - assert(MaxVectorSize <= 32 && "Did not expect to pack so many elements" + assert(MaxVectorSize <= 64 && "Did not expect to pack so many elements" " into one vector!"); unsigned VF = MaxVectorSize; Index: test/Transforms/LoopVectorize/X86/mask1.ll =================================================================== --- test/Transforms/LoopVectorize/X86/mask1.ll +++ test/Transforms/LoopVectorize/X86/mask1.ll @@ -0,0 +1,83 @@ +; RUN: opt < %s -O3 -mcpu=corei7-avx -S | FileCheck %s -check-prefix=AVX1 +; RUN: opt < %s -O3 -mcpu=core-avx2 -S | FileCheck %s -check-prefix=AVX2 +; RUN: opt < %s -O3 -mcpu=knl -S | FileCheck %s -check-prefix=AVX512 + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc_linux" + +; The source code: +; +;void foo(int *A, int *B, int *trigger) { +; +; for (int i=0; i<10000; i++) { +; if (trigger[i] < 100) { +; A[i] = B[i] + trigger[i]; +; } +; } +;} + + +;AVX2: llvm.masked.load.v8i32 +;AVX2: llvm.masked.store.v8i32 +;AVX512: llvm.masked.load.v16i32 +;AVX512: llvm.masked.store.v16i32 +;AVX1-NOT: llvm.masked + +; Function Attrs: nounwind uwtable +define void @foo(i32* %A, i32* %B, i32* %trigger) { +entry: + %A.addr = alloca i32*, align 8 + %B.addr = alloca i32*, align 8 + %trigger.addr = alloca i32*, align 8 + %i = alloca i32, align 4 + store i32* %A, i32** %A.addr, align 8 + store i32* %B, i32** %B.addr, align 8 + store i32* %trigger, i32** %trigger.addr, align 8 + store i32 0, i32* %i, align 4 + br label %for.cond + +for.cond: ; preds = %for.inc, %entry + %0 = load i32* %i, align 4 + %cmp = icmp slt i32 %0, 10000 + br i1 %cmp, label %for.body, label %for.end + +for.body: ; preds = %for.cond + %1 = load i32* %i, align 4 + %idxprom = sext i32 %1 to i64 + %2 = load i32** %trigger.addr, align 8 + %arrayidx = getelementptr inbounds i32* %2, i64 %idxprom + %3 = load i32* %arrayidx, align 4 + %cmp1 = icmp slt i32 %3, 100 + br i1 %cmp1, label %if.then, label %if.end + +if.then: ; preds = %for.body + %4 = load i32* %i, align 4 + %idxprom2 = sext i32 %4 to i64 + %5 = load i32** %B.addr, align 8 + %arrayidx3 = getelementptr inbounds i32* %5, i64 %idxprom2 + %6 = load i32* %arrayidx3, align 4 + %7 = load i32* %i, align 4 + %idxprom4 = sext i32 %7 to i64 + %8 = load i32** %trigger.addr, align 8 + %arrayidx5 = getelementptr inbounds i32* %8, i64 %idxprom4 + %9 = load i32* %arrayidx5, align 4 + %add = add nsw i32 %6, %9 + %10 = load i32* %i, align 4 + %idxprom6 = sext i32 %10 to i64 + %11 = load i32** %A.addr, align 8 + %arrayidx7 = getelementptr inbounds i32* %11, i64 %idxprom6 + store i32 %add, i32* %arrayidx7, align 4 + br label %if.end + +if.end: ; preds = %if.then, %for.body + br label %for.inc + +for.inc: ; preds = %if.end + %12 = load i32* %i, align 4 + %inc = add nsw i32 %12, 1 + store i32 %inc, i32* %i, align 4 + br label %for.cond + +for.end: ; preds = %for.cond + ret void +} Index: test/Transforms/LoopVectorize/X86/mask2.ll =================================================================== --- test/Transforms/LoopVectorize/X86/mask2.ll +++ test/Transforms/LoopVectorize/X86/mask2.ll @@ -0,0 +1,84 @@ +; RUN: opt < %s -O3 -mcpu=corei7-avx -S | FileCheck %s -check-prefix=AVX1 +; RUN: opt < %s -O3 -mcpu=core-avx2 -S | FileCheck %s -check-prefix=AVX2 +; RUN: opt < %s -O3 -mcpu=knl -S | FileCheck %s -check-prefix=AVX512 + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc_linux" + +; The source code: +; +;void foo(float *A, float *B, int *trigger) { +; +; for (int i=0; i<10000; i++) { +; if (trigger[i] < 100) { +; A[i] = B[i] + trigger[i]; +; } +; } +;} + + +;AVX2: llvm.masked.load.v8f32 +;AVX2: llvm.masked.store.v8f32 +;AVX512: llvm.masked.load.v16f32 +;AVX512: llvm.masked.store.v16f32 +;AVX1-NOT: llvm.masked + +; Function Attrs: nounwind uwtable +define void @foo(float* %A, float* %B, i32* %trigger) { +entry: + %A.addr = alloca float*, align 8 + %B.addr = alloca float*, align 8 + %trigger.addr = alloca i32*, align 8 + %i = alloca i32, align 4 + store float* %A, float** %A.addr, align 8 + store float* %B, float** %B.addr, align 8 + store i32* %trigger, i32** %trigger.addr, align 8 + store i32 0, i32* %i, align 4 + br label %for.cond + +for.cond: ; preds = %for.inc, %entry + %0 = load i32* %i, align 4 + %cmp = icmp slt i32 %0, 10000 + br i1 %cmp, label %for.body, label %for.end + +for.body: ; preds = %for.cond + %1 = load i32* %i, align 4 + %idxprom = sext i32 %1 to i64 + %2 = load i32** %trigger.addr, align 8 + %arrayidx = getelementptr inbounds i32* %2, i64 %idxprom + %3 = load i32* %arrayidx, align 4 + %cmp1 = icmp slt i32 %3, 100 + br i1 %cmp1, label %if.then, label %if.end + +if.then: ; preds = %for.body + %4 = load i32* %i, align 4 + %idxprom2 = sext i32 %4 to i64 + %5 = load float** %B.addr, align 8 + %arrayidx3 = getelementptr inbounds float* %5, i64 %idxprom2 + %6 = load float* %arrayidx3, align 4 + %7 = load i32* %i, align 4 + %idxprom4 = sext i32 %7 to i64 + %8 = load i32** %trigger.addr, align 8 + %arrayidx5 = getelementptr inbounds i32* %8, i64 %idxprom4 + %9 = load i32* %arrayidx5, align 4 + %conv = sitofp i32 %9 to float + %add = fadd float %6, %conv + %10 = load i32* %i, align 4 + %idxprom6 = sext i32 %10 to i64 + %11 = load float** %A.addr, align 8 + %arrayidx7 = getelementptr inbounds float* %11, i64 %idxprom6 + store float %add, float* %arrayidx7, align 4 + br label %if.end + +if.end: ; preds = %if.then, %for.body + br label %for.inc + +for.inc: ; preds = %if.end + %12 = load i32* %i, align 4 + %inc = add nsw i32 %12, 1 + store i32 %inc, i32* %i, align 4 + br label %for.cond + +for.end: ; preds = %for.cond + ret void +} Index: test/Transforms/LoopVectorize/X86/mask3.ll =================================================================== --- test/Transforms/LoopVectorize/X86/mask3.ll +++ test/Transforms/LoopVectorize/X86/mask3.ll @@ -0,0 +1,84 @@ +; RUN: opt < %s -O3 -mcpu=corei7-avx -S | FileCheck %s -check-prefix=AVX1 +; RUN: opt < %s -O3 -mcpu=core-avx2 -S | FileCheck %s -check-prefix=AVX2 +; RUN: opt < %s -O3 -mcpu=knl -S | FileCheck %s -check-prefix=AVX512 + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc_linux" + +; The source code: +; +;void foo(double *A, double *B, int *trigger) { +; +; for (int i=0; i<10000; i++) { +; if (trigger[i] < 100) { +; A[i] = B[i] + trigger[i]; +; } +; } +;} + + +;AVX2: llvm.masked.load.v4f64 +;AVX2: llvm.masked.store.v4f64 +;AVX512: llvm.masked.load.v8f64 +;AVX512: llvm.masked.store.v8f64 +;AVX1-NOT: llvm.masked + +; Function Attrs: nounwind uwtable +define void @foo(double* %A, double* %B, i32* %trigger) #0 { +entry: + %A.addr = alloca double*, align 8 + %B.addr = alloca double*, align 8 + %trigger.addr = alloca i32*, align 8 + %i = alloca i32, align 4 + store double* %A, double** %A.addr, align 8 + store double* %B, double** %B.addr, align 8 + store i32* %trigger, i32** %trigger.addr, align 8 + store i32 0, i32* %i, align 4 + br label %for.cond + +for.cond: ; preds = %for.inc, %entry + %0 = load i32* %i, align 4 + %cmp = icmp slt i32 %0, 10000 + br i1 %cmp, label %for.body, label %for.end + +for.body: ; preds = %for.cond + %1 = load i32* %i, align 4 + %idxprom = sext i32 %1 to i64 + %2 = load i32** %trigger.addr, align 8 + %arrayidx = getelementptr inbounds i32* %2, i64 %idxprom + %3 = load i32* %arrayidx, align 4 + %cmp1 = icmp slt i32 %3, 100 + br i1 %cmp1, label %if.then, label %if.end + +if.then: ; preds = %for.body + %4 = load i32* %i, align 4 + %idxprom2 = sext i32 %4 to i64 + %5 = load double** %B.addr, align 8 + %arrayidx3 = getelementptr inbounds double* %5, i64 %idxprom2 + %6 = load double* %arrayidx3, align 8 + %7 = load i32* %i, align 4 + %idxprom4 = sext i32 %7 to i64 + %8 = load i32** %trigger.addr, align 8 + %arrayidx5 = getelementptr inbounds i32* %8, i64 %idxprom4 + %9 = load i32* %arrayidx5, align 4 + %conv = sitofp i32 %9 to double + %add = fadd double %6, %conv + %10 = load i32* %i, align 4 + %idxprom6 = sext i32 %10 to i64 + %11 = load double** %A.addr, align 8 + %arrayidx7 = getelementptr inbounds double* %11, i64 %idxprom6 + store double %add, double* %arrayidx7, align 8 + br label %if.end + +if.end: ; preds = %if.then, %for.body + br label %for.inc + +for.inc: ; preds = %if.end + %12 = load i32* %i, align 4 + %inc = add nsw i32 %12, 1 + store i32 %inc, i32* %i, align 4 + br label %for.cond + +for.end: ; preds = %for.cond + ret void +} Index: test/Transforms/LoopVectorize/X86/mask4.ll =================================================================== --- test/Transforms/LoopVectorize/X86/mask4.ll +++ test/Transforms/LoopVectorize/X86/mask4.ll @@ -0,0 +1,83 @@ +; RUN: opt < %s -O3 -mcpu=corei7-avx -S | FileCheck %s -check-prefix=AVX1 +; RUN: opt < %s -O3 -mcpu=core-avx2 -S | FileCheck %s -check-prefix=AVX2 +; RUN: opt < %s -O3 -mcpu=knl -S | FileCheck %s -check-prefix=AVX512 + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-pc_linux" + +; The source code: +; +;void foo(double *A, double *B, int *trigger) { +; +; for (int i=0; i<10000; i++) { +; if (trigger[i] < 100) { +; A[i] = B[i*2] + trigger[i]; << non-cosecutive access +; } +; } +;} + + +;AVX2-NOT: llvm.masked +;AVX512-NOT: llvm.masked +;AVX1-NOT: llvm.masked + +; Function Attrs: nounwind uwtable +define void @foo(double* %A, double* %B, i32* %trigger) { +entry: + %A.addr = alloca double*, align 8 + %B.addr = alloca double*, align 8 + %trigger.addr = alloca i32*, align 8 + %i = alloca i32, align 4 + store double* %A, double** %A.addr, align 8 + store double* %B, double** %B.addr, align 8 + store i32* %trigger, i32** %trigger.addr, align 8 + store i32 0, i32* %i, align 4 + br label %for.cond + +for.cond: ; preds = %for.inc, %entry + %0 = load i32* %i, align 4 + %cmp = icmp slt i32 %0, 10000 + br i1 %cmp, label %for.body, label %for.end + +for.body: ; preds = %for.cond + %1 = load i32* %i, align 4 + %idxprom = sext i32 %1 to i64 + %2 = load i32** %trigger.addr, align 8 + %arrayidx = getelementptr inbounds i32* %2, i64 %idxprom + %3 = load i32* %arrayidx, align 4 + %cmp1 = icmp slt i32 %3, 100 + br i1 %cmp1, label %if.then, label %if.end + +if.then: ; preds = %for.body + %4 = load i32* %i, align 4 + %mul = mul nsw i32 %4, 2 + %idxprom2 = sext i32 %mul to i64 + %5 = load double** %B.addr, align 8 + %arrayidx3 = getelementptr inbounds double* %5, i64 %idxprom2 + %6 = load double* %arrayidx3, align 8 + %7 = load i32* %i, align 4 + %idxprom4 = sext i32 %7 to i64 + %8 = load i32** %trigger.addr, align 8 + %arrayidx5 = getelementptr inbounds i32* %8, i64 %idxprom4 + %9 = load i32* %arrayidx5, align 4 + %conv = sitofp i32 %9 to double + %add = fadd double %6, %conv + %10 = load i32* %i, align 4 + %idxprom6 = sext i32 %10 to i64 + %11 = load double** %A.addr, align 8 + %arrayidx7 = getelementptr inbounds double* %11, i64 %idxprom6 + store double %add, double* %arrayidx7, align 8 + br label %if.end + +if.end: ; preds = %if.then, %for.body + br label %for.inc + +for.inc: ; preds = %if.end + %12 = load i32* %i, align 4 + %inc = add nsw i32 %12, 1 + store i32 %inc, i32* %i, align 4 + br label %for.cond + +for.end: ; preds = %for.cond + ret void +}