Index: lib/Transforms/Vectorize/SLPVectorizer.cpp =================================================================== --- lib/Transforms/Vectorize/SLPVectorizer.cpp +++ lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -347,15 +347,16 @@ /// The very first instruction in the list with the main opcode. Value *OpValue = nullptr; - /// The main opcode for the list of instructions. + /// The main/alternate opcodes for the list of instructions. unsigned Opcode = 0; + unsigned AltOpcode = 0; /// Some of the instructions in the list have alternate opcodes. - bool IsAltShuffle = false; + bool IsAltShuffle() const { return Opcode != AltOpcode; } InstructionsState() = default; - InstructionsState(Value *OpValue, unsigned Opcode, bool IsAltShuffle) - : OpValue(OpValue), Opcode(Opcode), IsAltShuffle(IsAltShuffle) {} + InstructionsState(Value *OpValue, unsigned Opcode, unsigned AltOpcode) + : OpValue(OpValue), Opcode(Opcode), AltOpcode(AltOpcode) {} }; } // end anonymous namespace @@ -366,24 +367,25 @@ static InstructionsState getSameOpcode(ArrayRef VL) { // Make sure these are all Instructions. if (llvm::any_of(VL, [](Value *V) { return !isa(V); })) - return InstructionsState(VL[0], 0, false); + return InstructionsState(VL[0], 0, 0); unsigned Opcode = cast(VL[0])->getOpcode(); + unsigned AltOpcode = Opcode; bool HasAltOpcodes = llvm::any_of(VL, [Opcode](Value *V) { return Opcode != cast(V)->getOpcode(); }); // Check for an alternate opcode pattern. if (HasAltOpcodes) { - unsigned AltOpcode = getAltOpcode(Opcode); + AltOpcode = getAltOpcode(Opcode); for (int Cnt = 0, E = VL.size(); Cnt < E; Cnt++) { unsigned InstOpcode = cast(VL[Cnt])->getOpcode(); if (InstOpcode != (isOdd(Cnt) ? AltOpcode : Opcode)) - return InstructionsState(VL[0], 0, false); + return InstructionsState(VL[0], 0, 0); } } - return InstructionsState(VL[0], Opcode, HasAltOpcodes); + return InstructionsState(VL[0], Opcode, AltOpcode); } /// \returns true if all of the values in \p VL have the same type or false @@ -1519,7 +1521,7 @@ } LLVM_DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n"); - unsigned ShuffleOrOp = S.IsAltShuffle ? + unsigned ShuffleOrOp = S.IsAltShuffle() ? (unsigned) Instruction::ShuffleVector : S.Opcode; switch (ShuffleOrOp) { case Instruction::PHI: { @@ -1906,7 +1908,7 @@ case Instruction::ShuffleVector: // If this is not an alternate sequence of opcode like add-sub // then do not vectorize this instruction. - if (!S.IsAltShuffle) { + if (!S.IsAltShuffle()) { BS.cancelScheduling(VL, VL0); newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies); LLVM_DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n"); @@ -1918,8 +1920,7 @@ // Reorder operands if reordering would enable vectorization. if (isa(VL0)) { ValueList Left, Right; - unsigned AltOpcode = getAltOpcode(S.Opcode); - reorderAltShuffleOperands(S.Opcode, AltOpcode, VL, Left, Right); + reorderAltShuffleOperands(S.Opcode, S.AltOpcode, VL, Left, Right); buildTree_rec(Left, Depth + 1, UserTreeIdx); buildTree_rec(Right, Depth + 1, UserTreeIdx); return; @@ -2098,7 +2099,7 @@ InstructionsState S = getSameOpcode(VL); assert(S.Opcode && allSameType(VL) && allSameBlock(VL) && "Invalid VL"); Instruction *VL0 = cast(S.OpValue); - unsigned ShuffleOrOp = S.IsAltShuffle ? + unsigned ShuffleOrOp = S.IsAltShuffle() ? (unsigned) Instruction::ShuffleVector : S.Opcode; switch (ShuffleOrOp) { case Instruction::PHI: @@ -3055,7 +3056,7 @@ return V; } - unsigned ShuffleOrOp = S.IsAltShuffle ? + unsigned ShuffleOrOp = S.IsAltShuffle() ? (unsigned) Instruction::ShuffleVector : S.Opcode; switch (ShuffleOrOp) { case Instruction::PHI: { @@ -3487,8 +3488,8 @@ ValueList LHSVL, RHSVL; assert(Instruction::isBinaryOp(S.Opcode) && "Invalid Shuffle Vector Operand"); - unsigned AltOpcode = getAltOpcode(S.Opcode); - reorderAltShuffleOperands(S.Opcode, AltOpcode, E->Scalars, LHSVL, RHSVL); + reorderAltShuffleOperands(S.Opcode, S.AltOpcode, E->Scalars, LHSVL, + RHSVL); setInsertPointAfterBundle(E->Scalars, VL0); Value *LHS = vectorizeTree(LHSVL); @@ -3505,7 +3506,7 @@ // Create a vector of LHS op2 RHS Value *V1 = Builder.CreateBinOp( - static_cast(AltOpcode), LHS, RHS); + static_cast(S.AltOpcode), LHS, RHS); // Create shuffle to take alternate operations from the vector. // Also, gather up odd and even scalar ops to propagate IR flags to