Index: lib/Transforms/Vectorize/SLPVectorizer.cpp =================================================================== --- lib/Transforms/Vectorize/SLPVectorizer.cpp +++ lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -219,6 +219,25 @@ return Opcode; } +/// Get the intersection (logical and) of all of the potential IR flags +/// of each scalar operation (VL) that will be converted into a vector (I) +/// only in case of scalar sub-operation from (VL) is equal to (OpValue). Flag set: +/// NSW, NUW, exact, and all of fast-math. +static void propagateIRFlagsWithOp(Value *I, ArrayRef VL, Value *OpValue) { + if (auto *VecOp = dyn_cast(I)) { + auto *Intersection = cast(OpValue); + const unsigned Opcode = Intersection->getOpcode(); + for (auto *V : VL) { + auto *Instr = dyn_cast(V); + if (!Instr) + continue; + if (Opcode == Instr->getOpcode()) + Intersection->andIRFlags(V); + } + VecOp->copyIRFlags(Intersection); + } +} + /// \returns true if all of the values in \p VL have the same type or false /// otherwise. static bool allSameType(ArrayRef VL) { @@ -2586,7 +2605,7 @@ V = Builder.CreateICmp(P0, L, R); E->VectorizedValue = V; - propagateIRFlags(E->VectorizedValue, E->Scalars); + propagateIRFlagsWithOp(E->VectorizedValue, E->Scalars, VL0); ++NumVectorInstructions; return V; } @@ -2650,7 +2669,7 @@ BinaryOperator *BinOp = cast(VL0); Value *V = Builder.CreateBinOp(BinOp->getOpcode(), LHS, RHS); E->VectorizedValue = V; - propagateIRFlags(E->VectorizedValue, E->Scalars); + propagateIRFlagsWithOp(E->VectorizedValue, E->Scalars, VL0); ++NumVectorInstructions; if (Instruction *I = dyn_cast(V)) @@ -2793,7 +2812,7 @@ ExternalUses.push_back(ExternalUser(ScalarArg, cast(V), 0)); E->VectorizedValue = V; - propagateIRFlags(E->VectorizedValue, E->Scalars); + propagateIRFlagsWithOp(E->VectorizedValue, E->Scalars, VL0); ++NumVectorInstructions; return V; } @@ -2835,8 +2854,8 @@ } Value *ShuffleMask = ConstantVector::get(Mask); - propagateIRFlags(V0, EvenScalars); - propagateIRFlags(V1, OddScalars); + propagateIRFlagsWithOp(V0, EvenScalars, EvenScalars[0]); + propagateIRFlagsWithOp(V1, OddScalars, OddScalars[0]); Value *V = Builder.CreateShuffleVector(V0, V1, ShuffleMask); E->VectorizedValue = V; @@ -4539,7 +4558,7 @@ Builder.SetCurrentDebugLocation(Loc); VectorizedTree = Builder.CreateBinOp(ReductionOpcode, VectorizedTree, ReducedSubTree, "bin.rdx"); - propagateIRFlags(VectorizedTree, ReductionOps); + propagateIRFlagsWithOp(VectorizedTree, ReductionOps, ReductionOps[0]); } else VectorizedTree = ReducedSubTree; i += ReduxWidth; @@ -4553,7 +4572,7 @@ Builder.SetCurrentDebugLocation(I->getDebugLoc()); VectorizedTree = Builder.CreateBinOp(ReductionOpcode, VectorizedTree, I); - propagateIRFlags(VectorizedTree, ReductionOps); + propagateIRFlagsWithOp(VectorizedTree, ReductionOps, ReductionOps[0]); } for (auto &Pair : ExternallyUsedValues) { assert(!Pair.second.empty() && @@ -4563,7 +4582,7 @@ Builder.SetCurrentDebugLocation(I->getDebugLoc()); VectorizedTree = Builder.CreateBinOp(ReductionOpcode, VectorizedTree, Pair.first, "bin.extra"); - propagateIRFlags(VectorizedTree, I); + propagateIRFlagsWithOp(VectorizedTree, I, I); } } // Update users. @@ -4629,7 +4648,7 @@ "rdx.shuf.r"); TmpVec = Builder.CreateBinOp(ReductionOpcode, LeftShuf, RightShuf, "bin.rdx"); - propagateIRFlags(TmpVec, RedOps); + propagateIRFlagsWithOp(TmpVec, RedOps, RedOps[0]); } // The result is in the first element of the vector.