diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8199,11 +8199,10 @@ } void VPWidenMemoryInstructionRecipe::execute(VPTransformState &State) { - Instruction *Instr = getUnderlyingInstr(); - VPValue *StoredValue = isa(Instr) ? getStoredValue() : nullptr; - State.ILV->vectorizeMemoryInstruction(Instr, State, - StoredValue ? nullptr : this, getAddr(), - StoredValue, getMask()); + VPValue *StoredValue = isStore() ? getStoredValue() : nullptr; + State.ILV->vectorizeMemoryInstruction(getUnderlyingInstr(), State, + StoredValue ? nullptr : toVPValue(), + getAddr(), StoredValue, getMask()); } // Determine how to lower the scalar epilogue, which depends on 1) optimising diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1248,9 +1248,9 @@ /// TODO: We currently execute only per-part unless a specific instance is /// provided. class VPWidenMemoryInstructionRecipe : public VPRecipeBase, - public VPValue, - public VPUser { - + public VPDef, + public VPUser, + public VPValue { void setMask(VPValue *Mask) { if (!Mask) return; @@ -1258,22 +1258,20 @@ } bool isMasked() const { - return (isa(getUnderlyingInstr()) && getNumOperands() == 2) || - (isa(getUnderlyingInstr()) && getNumOperands() == 3); + return isStore() ? getNumOperands() == 3 : getNumOperands() == 2; } public: VPWidenMemoryInstructionRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask) - : VPRecipeBase(VPWidenMemoryInstructionSC), - VPValue(VPValue::VPMemoryInstructionSC, &Load), VPUser({Addr}) { + : VPRecipeBase(VPWidenMemoryInstructionSC), VPUser({Addr}), + VPValue(VPValue::VPMemoryInstructionSC, &Load, this) { setMask(Mask); } VPWidenMemoryInstructionRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredValue, VPValue *Mask) - : VPRecipeBase(VPWidenMemoryInstructionSC), - VPValue(VPValue::VPMemoryInstructionSC, &Store), - VPUser({Addr, StoredValue}) { + : VPRecipeBase(VPWidenMemoryInstructionSC), VPUser({Addr, StoredValue}), + VPValue(VPValue::VPMemoryInstructionSC, &Store, this) { setMask(Mask); } @@ -1294,10 +1292,11 @@ return isMasked() ? getOperand(getNumOperands() - 1) : nullptr; } + bool isStore() const { return isa(getUnderlyingInstr()); } + /// Return the address accessed by this recipe. VPValue *getStoredValue() const { - assert(isa(getUnderlyingInstr()) && - "Stored value only available for store instructions"); + assert(isStore() && "Stored value only available for store instructions"); return getOperand(1); // Stored value is the 2nd, mandatory operand. }