Index: llvm/trunk/lib/Transforms/Vectorize/SLPVectorizer.cpp =================================================================== --- llvm/trunk/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ llvm/trunk/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -340,21 +340,22 @@ /// Main data required for vectorization of instructions. struct InstructionsState { - /// The very first instruction in the list with the main opcode. - Value *OpValue = nullptr; - - /// The main opcode for the list of instructions. - unsigned Opcode = 0; - - /// Some of the instructions in the list have alternate opcodes. - bool IsAltShuffle = false; - - InstructionsState() = default; - InstructionsState(Value *OpValue, unsigned Opcode, bool IsAltShuffle) - : OpValue(OpValue), Opcode(Opcode), IsAltShuffle(IsAltShuffle) {} -}; - -} // end anonymous namespace + /// The very first instruction in the list with the main opcode. + Value *OpValue = nullptr; + + /// The main/alternate opcodes for the list of instructions. + unsigned Opcode = 0; + unsigned AltOpcode = 0; + + /// Some of the instructions in the list have alternate opcodes. + bool isAltShuffle() const { return Opcode != AltOpcode; } + + InstructionsState() = default; + InstructionsState(Value *OpValue, unsigned Opcode, unsigned AltOpcode) + : OpValue(OpValue), Opcode(Opcode), AltOpcode(AltOpcode) {} +}; + +} // end anonymous namespace /// \returns analysis of the Instructions in \p VL described in /// InstructionsState, the Opcode that we suppose the whole list @@ -362,24 +363,25 @@ static InstructionsState getSameOpcode(ArrayRef VL) { // Make sure these are all Instructions. if (llvm::any_of(VL, [](Value *V) { return !isa(V); })) - return InstructionsState(VL[0], 0, false); + return InstructionsState(VL[0], 0, 0); unsigned Opcode = cast(VL[0])->getOpcode(); + unsigned AltOpcode = Opcode; bool HasAltOpcodes = llvm::any_of(VL, [Opcode](Value *V) { return Opcode != cast(V)->getOpcode(); }); // Check for an alternate opcode pattern. if (HasAltOpcodes) { - unsigned AltOpcode = getAltOpcode(Opcode); + AltOpcode = getAltOpcode(Opcode); for (int Cnt = 0, E = VL.size(); Cnt < E; Cnt++) { unsigned InstOpcode = cast(VL[Cnt])->getOpcode(); if (!sameOpcodeOrAlt(Opcode, AltOpcode, InstOpcode)) - return InstructionsState(VL[0], 0, false); + return InstructionsState(VL[0], 0, 0); } } - return InstructionsState(VL[0], Opcode, HasAltOpcodes); + return InstructionsState(VL[0], Opcode, AltOpcode); } /// \returns true if all of the values in \p VL have the same type or false @@ -1512,13 +1514,13 @@ "tryScheduleBundle should cancelScheduling on failure"); newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies); return; - } - LLVM_DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n"); - - unsigned ShuffleOrOp = S.IsAltShuffle ? - (unsigned) Instruction::ShuffleVector : S.Opcode; - switch (ShuffleOrOp) { - case Instruction::PHI: { + } + LLVM_DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n"); + + unsigned ShuffleOrOp = S.isAltShuffle() ? + (unsigned) Instruction::ShuffleVector : S.Opcode; + switch (ShuffleOrOp) { + case Instruction::PHI: { PHINode *PH = dyn_cast(VL0); // Check for terminator values (e.g. invoke). @@ -1899,26 +1901,25 @@ } return; } - case Instruction::ShuffleVector: - // If this is not an alternate sequence of opcode like add-sub - // then do not vectorize this instruction. - if (!S.IsAltShuffle) { - BS.cancelScheduling(VL, VL0); - newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies); - LLVM_DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n"); + case Instruction::ShuffleVector: + // If this is not an alternate sequence of opcode like add-sub + // then do not vectorize this instruction. + if (!S.isAltShuffle()) { + BS.cancelScheduling(VL, VL0); + newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies); + LLVM_DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n"); return; } newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies); LLVM_DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n"); - // Reorder operands if reordering would enable vectorization. - if (isa(VL0)) { - ValueList Left, Right; - unsigned AltOpcode = getAltOpcode(S.Opcode); - reorderAltShuffleOperands(S.Opcode, AltOpcode, VL, Left, Right); - buildTree_rec(Left, Depth + 1, UserTreeIdx); - buildTree_rec(Right, Depth + 1, UserTreeIdx); - return; + // Reorder operands if reordering would enable vectorization. + if (isa(VL0)) { + ValueList Left, Right; + reorderAltShuffleOperands(S.Opcode, S.AltOpcode, VL, Left, Right); + buildTree_rec(Left, Depth + 1, UserTreeIdx); + buildTree_rec(Right, Depth + 1, UserTreeIdx); + return; } for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) { @@ -2091,13 +2092,13 @@ } return ReuseShuffleCost + getGatherCost(VL); } - InstructionsState S = getSameOpcode(VL); - assert(S.Opcode && allSameType(VL) && allSameBlock(VL) && "Invalid VL"); - Instruction *VL0 = cast(S.OpValue); - unsigned ShuffleOrOp = S.IsAltShuffle ? - (unsigned) Instruction::ShuffleVector : S.Opcode; - switch (ShuffleOrOp) { - case Instruction::PHI: + InstructionsState S = getSameOpcode(VL); + assert(S.Opcode && allSameType(VL) && allSameBlock(VL) && "Invalid VL"); + Instruction *VL0 = cast(S.OpValue); + unsigned ShuffleOrOp = S.isAltShuffle() ? + (unsigned) Instruction::ShuffleVector : S.Opcode; + switch (ShuffleOrOp) { + case Instruction::PHI: return 0; case Instruction::ExtractValue: @@ -3048,13 +3049,13 @@ } } E->VectorizedValue = V; - return V; - } - - unsigned ShuffleOrOp = S.IsAltShuffle ? - (unsigned) Instruction::ShuffleVector : S.Opcode; - switch (ShuffleOrOp) { - case Instruction::PHI: { + return V; + } + + unsigned ShuffleOrOp = S.isAltShuffle() ? + (unsigned) Instruction::ShuffleVector : S.Opcode; + switch (ShuffleOrOp) { + case Instruction::PHI: { PHINode *PH = dyn_cast(VL0); Builder.SetInsertPoint(PH->getParent()->getFirstNonPHI()); Builder.SetCurrentDebugLocation(PH->getDebugLoc()); @@ -3480,14 +3481,14 @@ return V; } case Instruction::ShuffleVector: { - ValueList LHSVL, RHSVL; - assert(Instruction::isBinaryOp(S.Opcode) && - "Invalid Shuffle Vector Operand"); - unsigned AltOpcode = getAltOpcode(S.Opcode); - reorderAltShuffleOperands(S.Opcode, AltOpcode, E->Scalars, LHSVL, RHSVL); - setInsertPointAfterBundle(E->Scalars, VL0); - - Value *LHS = vectorizeTree(LHSVL); + ValueList LHSVL, RHSVL; + assert(Instruction::isBinaryOp(S.Opcode) && + "Invalid Shuffle Vector Operand"); + reorderAltShuffleOperands(S.Opcode, S.AltOpcode, E->Scalars, LHSVL, + RHSVL); + setInsertPointAfterBundle(E->Scalars, VL0); + + Value *LHS = vectorizeTree(LHSVL); Value *RHS = vectorizeTree(RHSVL); if (E->VectorizedValue) { @@ -3498,13 +3499,13 @@ // Create a vector of LHS op1 RHS Value *V0 = Builder.CreateBinOp( static_cast(S.Opcode), LHS, RHS); - - // Create a vector of LHS op2 RHS - Value *V1 = Builder.CreateBinOp( - static_cast(AltOpcode), LHS, RHS); - - // Create shuffle to take alternate operations from the vector. - // Also, gather up odd and even scalar ops to propagate IR flags to + + // Create a vector of LHS op2 RHS + Value *V1 = Builder.CreateBinOp( + static_cast(S.AltOpcode), LHS, RHS); + + // Create shuffle to take alternate operations from the vector. + // Also, gather up odd and even scalar ops to propagate IR flags to // each vector operation. ValueList OpScalars, AltScalars; unsigned e = E->Scalars.size(); @@ -3512,9 +3513,9 @@ for (unsigned i = 0; i < e; ++i) { auto *OpInst = cast(E->Scalars[i]); unsigned InstOpcode = OpInst->getOpcode(); - assert(sameOpcodeOrAlt(S.Opcode, AltOpcode, InstOpcode) && + assert(sameOpcodeOrAlt(S.Opcode, S.AltOpcode, InstOpcode) && "Unexpected main/alternate opcode"); - if (InstOpcode == AltOpcode) { + if (InstOpcode == S.AltOpcode) { Mask[i] = Builder.getInt32(e + i); AltScalars.push_back(E->Scalars[i]); } else {