diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8189,11 +8189,10 @@ } void VPWidenMemoryInstructionRecipe::execute(VPTransformState &State) { - Instruction *Instr = getUnderlyingInstr(); - VPValue *StoredValue = isa(Instr) ? getStoredValue() : nullptr; - State.ILV->vectorizeMemoryInstruction(Instr, State, - StoredValue ? nullptr : this, getAddr(), - StoredValue, getMask()); + VPValue *StoredValue = isStore() ? getStoredValue() : nullptr; + State.ILV->vectorizeMemoryInstruction(&Instr, State, + StoredValue ? nullptr : getVPValue(), + getAddr(), StoredValue, getMask()); } // Determine how to lower the scalar epilogue, which depends on 1) optimising diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1230,10 +1230,9 @@ /// - For store: Address, stored value, optional mask /// TODO: We currently execute only per-part unless a specific instance is /// provided. -class VPWidenMemoryInstructionRecipe : public VPRecipeBase, - public VPValue, - public VPUser { +class VPWidenMemoryInstructionRecipe : public VPRecipeBase, public VPDef { + Instruction &Instr; void setMask(VPValue *Mask) { if (!Mask) return; @@ -1241,22 +1240,20 @@ } bool isMasked() const { - return (isa(getUnderlyingInstr()) && getNumOperands() == 2) || - (isa(getUnderlyingInstr()) && getNumOperands() == 3); + return isStore() ? getNumOperands() == 3 : getNumOperands() == 2; } public: VPWidenMemoryInstructionRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask) - : VPRecipeBase(VPWidenMemoryInstructionSC), - VPValue(VPValue::VPMemoryInstructionSC, &Load), VPUser({Addr}) { + : VPRecipeBase(VPWidenMemoryInstructionSC), VPDef({Addr}), Instr(Load) { + new VPValue(&Load, this); setMask(Mask); } VPWidenMemoryInstructionRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredValue, VPValue *Mask) - : VPRecipeBase(VPWidenMemoryInstructionSC), - VPValue(VPValue::VPMemoryInstructionSC, &Store), - VPUser({Addr, StoredValue}) { + : VPRecipeBase(VPWidenMemoryInstructionSC), VPDef({Addr, StoredValue}), + Instr(Store) { setMask(Mask); } @@ -1277,10 +1274,11 @@ return isMasked() ? getOperand(getNumOperands() - 1) : nullptr; } + bool isStore() const { return getNumDefinedValues() == 0; } + /// Return the address accessed by this recipe. VPValue *getStoredValue() const { - assert(isa(getUnderlyingInstr()) && - "Stored value only available for store instructions"); + assert(isStore() && "Stored value only available for store instructions"); return getOperand(1); // Stored value is the 2nd, mandatory operand. } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -111,7 +111,8 @@ if (auto *V = dyn_cast(this)) return V; if (auto *V = dyn_cast(this)) - return V; + if (V->getNumDefinedValues() == 1) + return V->getVPValue(); if (auto *V = dyn_cast(this)) return V->getVPValue(); if (auto *V = dyn_cast(this)) @@ -125,7 +126,8 @@ if (auto *V = dyn_cast(this)) return V; if (auto *V = dyn_cast(this)) - return V; + if (V->getNumDefinedValues() == 1) + return V->getVPValue(); if (auto *V = dyn_cast(this)) return V->getVPValue(); if (auto *V = dyn_cast(this)) @@ -941,8 +943,7 @@ void VPWidenMemoryInstructionRecipe::print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const { - O << "\"WIDEN " - << Instruction::getOpcodeName(getUnderlyingInstr()->getOpcode()) << " "; + O << "\"WIDEN " << Instruction::getOpcodeName(Instr.getOpcode()) << " "; bool First = true; for (VPValue *Op : operands()) {