Index: llvm/include/llvm/IR/InstVisitor.h =================================================================== --- llvm/include/llvm/IR/InstVisitor.h +++ llvm/include/llvm/IR/InstVisitor.h @@ -167,7 +167,7 @@ RetTy visitICmpInst(ICmpInst &I) { DELEGATE(CmpInst);} RetTy visitFCmpInst(FCmpInst &I) { DELEGATE(CmpInst);} RetTy visitAllocaInst(AllocaInst &I) { DELEGATE(UnaryInstruction);} - RetTy visitLoadInst(LoadInst &I) { DELEGATE(UnaryInstruction);} + RetTy visitLoadInst(LoadInst &I) { DELEGATE(Instruction); } RetTy visitStoreInst(StoreInst &I) { DELEGATE(Instruction);} RetTy visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) { DELEGATE(Instruction);} RetTy visitAtomicRMWInst(AtomicRMWInst &I) { DELEGATE(Instruction);} Index: llvm/include/llvm/IR/Instructions.h =================================================================== --- llvm/include/llvm/IR/Instructions.h +++ llvm/include/llvm/IR/Instructions.h @@ -164,7 +164,7 @@ /// An instruction for reading from memory. This uses the SubclassData field in /// Value to store whether or not the load is volatile. -class LoadInst : public UnaryInstruction { +class LoadInst : public Instruction { void AssertOK(); protected: @@ -193,6 +193,16 @@ Align Align, AtomicOrdering Order, SyncScope::ID SSID, BasicBlock *InsertAtEnd); + ~LoadInst() { + // FIXME: needed by operator delete + setLoadInstNumOperands(2); + } + // allocate space for exactly two operands + void *operator new(size_t s) { return User::operator new(s, 2); } + + /// Transparently provide more efficient getOperand methods. + DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); + /// Return true if this is a load from a volatile memory location. bool isVolatile() const { return getSubclassDataFromInstruction() & 1; } @@ -262,6 +272,23 @@ return getPointerOperandType()->getPointerAddressSpace(); } + bool hasNoaliasProvenanceOperand() const { return getNumOperands() == 2; } + Value *getNoaliasProvenanceOperand() const { + assert(hasNoaliasProvenanceOperand() && "we need a ptr_provenance"); + return getOperand(1); + } + static unsigned getNoaliasProvenanceOperandIndex() { return 1U; } + void setNoaliasProvenanceOperand(Value *Provenance) { + assert(Provenance && "Needs a provenance"); + setLoadInstNumOperands(2); + setOperand(1, Provenance); + } + void removeNoaliasProvenanceOperand() { + assert(hasNoaliasProvenanceOperand() && "nothing to remove"); + // make sure 'uses' are updated + setOperand(getNoaliasProvenanceOperandIndex(), nullptr); + setLoadInstNumOperands(1); + } // Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const Instruction *I) { return I->getOpcode() == Instruction::Load; @@ -283,6 +310,11 @@ SyncScope::ID SSID; }; +template <> +struct OperandTraits : public OptionalOperandTraits {}; + +DEFINE_TRANSPARENT_OPERAND_ACCESSORS(LoadInst, Value) + //===----------------------------------------------------------------------===// // StoreInst Class //===----------------------------------------------------------------------===// @@ -312,10 +344,11 @@ StoreInst(Value *Val, Value *Ptr, bool isVolatile, Align Align, AtomicOrdering Order, SyncScope::ID SSID, BasicBlock *InsertAtEnd); - // allocate space for exactly two operands - void *operator new(size_t s) { - return User::operator new(s, 2); + ~StoreInst() { + // FIXME: needed by operator delete + setStoreInstNumOperands(3); } + void *operator new(size_t s) { return User::operator new(s, 3); } /// Return true if this is a store to a volatile memory location. bool isVolatile() const { return getSubclassDataFromInstruction() & 1; } @@ -391,6 +424,23 @@ return getPointerOperandType()->getPointerAddressSpace(); } + bool hasNoaliasProvenanceOperand() const { return getNumOperands() == 3; } + Value *getNoaliasProvenanceOperand() const { + assert(hasNoaliasProvenanceOperand() && "we need a ptr_provenance"); + return getOperand(2); + } + static unsigned getNoaliasProvenanceOperandIndex() { return 2U; } + void setNoaliasProvenanceOperand(Value *Provenance) { + assert(Provenance && "Needs a provenance"); + setStoreInstNumOperands(3); + setOperand(2, Provenance); + } + void removeNoaliasProvenanceOperand() { + assert(hasNoaliasProvenanceOperand() && "nothing to remove"); + // make sure 'uses' are updated + setOperand(getNoaliasProvenanceOperandIndex(), nullptr); + setStoreInstNumOperands(2); + } // Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const Instruction *I) { return I->getOpcode() == Instruction::Store; @@ -413,8 +463,7 @@ }; template <> -struct OperandTraits : public FixedNumOperandTraits { -}; +struct OperandTraits : public OptionalOperandTraits {}; DEFINE_TRANSPARENT_OPERAND_ACCESSORS(StoreInst, Value) Index: llvm/include/llvm/IR/User.h =================================================================== --- llvm/include/llvm/IR/User.h +++ llvm/include/llvm/IR/User.h @@ -167,12 +167,12 @@ } Value *getOperand(unsigned i) const { - assert(i < NumUserOperands && "getOperand() out of range!"); + assert(i < getNumOperands() && "getOperand() out of range!"); return getOperandList()[i]; } void setOperand(unsigned i, Value *Val) { - assert(i < NumUserOperands && "setOperand() out of range!"); + assert(i < getNumOperands() && "setOperand() out of range!"); assert((!isa((const Value*)this) || isa((const Value*)this)) && "Cannot mutate a constant with setOperand!"); @@ -180,15 +180,17 @@ } const Use &getOperandUse(unsigned i) const { - assert(i < NumUserOperands && "getOperandUse() out of range!"); + assert(i < getNumOperands() && "getOperandUse() out of range!"); return getOperandList()[i]; } Use &getOperandUse(unsigned i) { - assert(i < NumUserOperands && "getOperandUse() out of range!"); + assert(i < getNumOperands() && "getOperandUse() out of range!"); return getOperandList()[i]; } - unsigned getNumOperands() const { return NumUserOperands; } + unsigned getNumOperands() const { + return NumUserOperands - NumUserOperandsDelta; + } /// Returns the descriptor co-allocated with this User instance. ArrayRef getDescriptor() const; @@ -209,6 +211,24 @@ NumUserOperands = NumOps; } + /// FIXME: As that the number of operands is used to find the start of + /// the allocated memory in operator delete, we need to always think we have + /// 3 operand before delete. + void setStoreInstNumOperands(unsigned NumOps) { + assert((2 <= NumOps) && (NumOps <= 3) && + "StoreInst can only have 2 or 3 operands"); + NumUserOperandsDelta = 3 - NumOps; + } + + /// FIXME: As that the number of operands is used to find the start of + /// the allocated memory in operator delete, we need to always think we have + /// 2 operand before delete. + void setLoadInstNumOperands(unsigned NumOps) { + assert((1 <= NumOps) && (NumOps <= 2) && + "LoadInst can only have 1 or 2 operands"); + NumUserOperandsDelta = 2 - NumOps; + } + /// Subclasses with hung off uses need to manage the operand count /// themselves. In these instances, the operand count isn't used to find the /// OperandList, so there's no issue in having the operand count change. @@ -234,10 +254,10 @@ op_iterator op_begin() { return getOperandList(); } const_op_iterator op_begin() const { return getOperandList(); } op_iterator op_end() { - return getOperandList() + NumUserOperands; + return getOperandList() + NumUserOperands - NumUserOperandsDelta; } const_op_iterator op_end() const { - return getOperandList() + NumUserOperands; + return getOperandList() + NumUserOperands - NumUserOperandsDelta; } op_range operands() { return op_range(op_begin(), op_end()); Index: llvm/include/llvm/IR/Value.h =================================================================== --- llvm/include/llvm/IR/Value.h +++ llvm/include/llvm/IR/Value.h @@ -110,7 +110,7 @@ /// /// Note, this should *NOT* be used directly by any class other than User. /// User uses this value to find the Use list. - enum : unsigned { NumUserOperandsBits = 28 }; + enum : unsigned { NumUserOperandsBits = 27 }; unsigned NumUserOperands : NumUserOperandsBits; // Use the same type as the bitfield above so that MSVC will pack them. @@ -118,6 +118,7 @@ unsigned HasName : 1; unsigned HasHungOffUses : 1; unsigned HasDescriptor : 1; + unsigned NumUserOperandsDelta : 1; private: template // UseT == 'Use' or 'const Use' Index: llvm/lib/IR/AsmWriter.cpp =================================================================== --- llvm/lib/IR/AsmWriter.cpp +++ llvm/lib/IR/AsmWriter.cpp @@ -4138,15 +4138,30 @@ } Out << ", "; TypePrinter.print(I.getType(), Out); - } else if (Operand) { // Print the normal way. + } else if (const auto *LI = dyn_cast(&I)) { + Out << ' '; + TypePrinter.print(LI->getType(), Out); + Out << ", "; + writeOperand(I.getOperand(0), true); + if (LI->hasNoaliasProvenanceOperand()) { + Out << ", ptr_provenance "; + writeOperand(LI->getNoaliasProvenanceOperand(), true); + } + } else if (const auto *SI = dyn_cast(&I)) { + Out << ' '; + writeOperand(I.getOperand(0), true); + Out << ", "; + writeOperand(I.getOperand(1), true); + + if (SI->hasNoaliasProvenanceOperand()) { + Out << ", ptr_provenance "; + writeOperand(SI->getNoaliasProvenanceOperand(), true); + } + } else if (Operand) { // Print the normal way. if (const auto *GEP = dyn_cast(&I)) { Out << ' '; TypePrinter.print(GEP->getSourceElementType(), Out); Out << ','; - } else if (const auto *LI = dyn_cast(&I)) { - Out << ' '; - TypePrinter.print(LI->getType(), Out); - Out << ','; } // PrintAllTypes - Instructions who have operands of all the same type @@ -4156,8 +4171,7 @@ Type *TheType = Operand->getType(); // Select, Store and ShuffleVector always print all types. - if (isa(I) || isa(I) || isa(I) - || isa(I)) { + if (isa(I) || isa(I) || isa(I)) { PrintAllTypes = true; } else { for (unsigned i = 1, E = I.getNumOperands(); i != E; ++i) { Index: llvm/lib/IR/Instructions.cpp =================================================================== --- llvm/lib/IR/Instructions.cpp +++ llvm/lib/IR/Instructions.cpp @@ -1373,8 +1373,11 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile, Align Align, AtomicOrdering Order, SyncScope::ID SSID, Instruction *InsertBef) - : UnaryInstruction(Ty, Load, Ptr, InsertBef) { + : Instruction(Ty, Load, OperandTraits::op_begin(this), 2, + InsertBef) { assert(Ty == cast(Ptr->getType())->getElementType()); + Op<0>() = Ptr; + setLoadInstNumOperands(1); setVolatile(isVolatile); setAlignment(Align); setAtomic(Order, SSID); @@ -1385,7 +1388,10 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile, Align Align, AtomicOrdering Order, SyncScope::ID SSID, BasicBlock *InsertAE) - : UnaryInstruction(Ty, Load, Ptr, InsertAE) { + : Instruction(Ty, Load, OperandTraits::op_begin(this), 2, + InsertAE) { + Op<0>() = Ptr; + setLoadInstNumOperands(1); assert(Ty == cast(Ptr->getType())->getElementType()); setVolatile(isVolatile); setAlignment(Align); @@ -1449,10 +1455,10 @@ AtomicOrdering Order, SyncScope::ID SSID, Instruction *InsertBefore) : Instruction(Type::getVoidTy(val->getContext()), Store, - OperandTraits::op_begin(this), - OperandTraits::operands(this), InsertBefore) { + OperandTraits::op_begin(this), 3, InsertBefore) { Op<0>() = val; Op<1>() = addr; + setStoreInstNumOperands(2); setVolatile(isVolatile); setAlignment(Align); setAtomic(Order, SSID); @@ -1463,10 +1469,10 @@ AtomicOrdering Order, SyncScope::ID SSID, BasicBlock *InsertAtEnd) : Instruction(Type::getVoidTy(val->getContext()), Store, - OperandTraits::op_begin(this), - OperandTraits::operands(this), InsertAtEnd) { + OperandTraits::op_begin(this), 3, InsertAtEnd) { Op<0>() = val; Op<1>() = addr; + setStoreInstNumOperands(2); setVolatile(isVolatile); setAlignment(Align); setAtomic(Order, SSID); @@ -4259,13 +4265,32 @@ } LoadInst *LoadInst::cloneImpl() const { - return new LoadInst(getType(), getOperand(0), Twine(), isVolatile(), - getAlign(), getOrdering(), getSyncScopeID()); + LoadInst *Result = + new LoadInst(getType(), getOperand(0), Twine(), isVolatile(), getAlign(), + getOrdering(), getSyncScopeID()); + // - we must keep the same number of arguments (for vector optimizations) + // - if we duplicate the provenance, we can get into problems with passes + // that don't know how to handle it (Like MergeLoadStoreMotion shows) + // - safe alternative: keep the argument, but map it to undef. + if (hasNoaliasProvenanceOperand()) + Result->setNoaliasProvenanceOperand( + UndefValue::get(getNoaliasProvenanceOperand()->getType())); + return Result; } StoreInst *StoreInst::cloneImpl() const { - return new StoreInst(getOperand(0), getOperand(1), isVolatile(), getAlign(), - getOrdering(), getSyncScopeID()); + StoreInst *Result = + new StoreInst(getOperand(0), getOperand(1), isVolatile(), getAlign(), + getOrdering(), getSyncScopeID()); + + // we must keep the same number of arguments (for vector optimizations) + // - if we duplicate the provenance, we can get into problems with passes + // that don't know how to handle it (Like MergeLoadStoreMotion shows) + // - safe alternative: keep the argument, but map it to undef. + if (hasNoaliasProvenanceOperand()) + Result->setNoaliasProvenanceOperand( + UndefValue::get(getNoaliasProvenanceOperand()->getType())); + return Result; } AtomicCmpXchgInst *AtomicCmpXchgInst::cloneImpl() const { Index: llvm/lib/IR/User.cpp =================================================================== --- llvm/lib/IR/User.cpp +++ llvm/lib/IR/User.cpp @@ -177,20 +177,19 @@ Use **HungOffOperandList = static_cast(Usr) - 1; // drop the hung off uses. - Use::zap(*HungOffOperandList, *HungOffOperandList + Obj->NumUserOperands, + Use::zap(*HungOffOperandList, *HungOffOperandList + Obj->getNumOperands(), /* Delete */ true); ::operator delete(HungOffOperandList); } else if (Obj->HasDescriptor) { Use *UseBegin = static_cast(Usr) - Obj->NumUserOperands; - Use::zap(UseBegin, UseBegin + Obj->NumUserOperands, /* Delete */ false); + Use::zap(UseBegin, UseBegin + Obj->getNumOperands(), /* Delete */ false); auto *DI = reinterpret_cast(UseBegin) - 1; uint8_t *Storage = reinterpret_cast(DI) - DI->SizeInBytes; ::operator delete(Storage); } else { Use *Storage = static_cast(Usr) - Obj->NumUserOperands; - Use::zap(Storage, Storage + Obj->NumUserOperands, - /* Delete */ false); + Use::zap(Storage, Storage + Obj->getNumOperands(), /* Delete */ false); ::operator delete(Storage); } } Index: llvm/lib/IR/Value.cpp =================================================================== --- llvm/lib/IR/Value.cpp +++ llvm/lib/IR/Value.cpp @@ -51,9 +51,9 @@ } Value::Value(Type *ty, unsigned scid) - : VTy(checkType(ty)), UseList(nullptr), SubclassID(scid), - HasValueHandle(0), SubclassOptionalData(0), SubclassData(0), - NumUserOperands(0), IsUsedByMD(false), HasName(false) { + : VTy(checkType(ty)), UseList(nullptr), SubclassID(scid), HasValueHandle(0), + SubclassOptionalData(0), SubclassData(0), NumUserOperands(0), + IsUsedByMD(false), HasName(false), NumUserOperandsDelta(0) { static_assert(ConstantFirstVal == 0, "!(SubclassID < ConstantFirstVal)"); // FIXME: Why isn't this in the subclass gunk?? // Note, we cannot call isa before the CallInst has been Index: llvm/unittests/IR/IRBuilderTest.cpp =================================================================== --- llvm/unittests/IR/IRBuilderTest.cpp +++ llvm/unittests/IR/IRBuilderTest.cpp @@ -332,6 +332,31 @@ EXPECT_EQ(fp::ebStrict, CII->getExceptionBehavior()); } +TEST_F(IRBuilderTest, Provenance) { + IRBuilder<> Builder(BB); + + auto *L = Builder.CreateLoad(GV->getValueType(), GV); + EXPECT_TRUE(!L->hasNoaliasProvenanceOperand()); + + auto *S = Builder.CreateStore(L, GV); + EXPECT_TRUE(!S->hasNoaliasProvenanceOperand()); + + L->setNoaliasProvenanceOperand(GV); + EXPECT_TRUE(L->hasNoaliasProvenanceOperand()); + + S->setNoaliasProvenanceOperand(GV); + EXPECT_TRUE(S->hasNoaliasProvenanceOperand()); + + EXPECT_EQ(L->getNoaliasProvenanceOperand(), GV); + EXPECT_EQ(S->getNoaliasProvenanceOperand(), GV); + + L->removeNoaliasProvenanceOperand(); + EXPECT_TRUE(!L->hasNoaliasProvenanceOperand()); + + S->removeNoaliasProvenanceOperand(); + EXPECT_TRUE(!S->hasNoaliasProvenanceOperand()); +} + TEST_F(IRBuilderTest, Lifetime) { IRBuilder<> Builder(BB); AllocaInst *Var1 = Builder.CreateAlloca(Builder.getInt8Ty());