Index: llvm/include/llvm/IR/InstVisitor.h =================================================================== --- llvm/include/llvm/IR/InstVisitor.h +++ llvm/include/llvm/IR/InstVisitor.h @@ -168,7 +168,7 @@ RetTy visitICmpInst(ICmpInst &I) { DELEGATE(CmpInst);} RetTy visitFCmpInst(FCmpInst &I) { DELEGATE(CmpInst);} RetTy visitAllocaInst(AllocaInst &I) { DELEGATE(UnaryInstruction);} - RetTy visitLoadInst(LoadInst &I) { DELEGATE(UnaryInstruction);} + RetTy visitLoadInst(LoadInst &I) { DELEGATE(Instruction); } RetTy visitStoreInst(StoreInst &I) { DELEGATE(Instruction);} RetTy visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) { DELEGATE(Instruction);} RetTy visitAtomicRMWInst(AtomicRMWInst &I) { DELEGATE(Instruction);} Index: llvm/include/llvm/IR/Instructions.h =================================================================== --- llvm/include/llvm/IR/Instructions.h +++ llvm/include/llvm/IR/Instructions.h @@ -166,7 +166,7 @@ /// An instruction for reading from memory. This uses the SubclassData field in /// Value to store whether or not the load is volatile. -class LoadInst : public UnaryInstruction { +class LoadInst : public Instruction { void AssertOK(); protected: @@ -229,6 +229,16 @@ : LoadInst(Ptr->getType()->getPointerElementType(), Ptr, NameStr, isVolatile, Align, Order, SSID, InsertAtEnd) {} + ~LoadInst() { + // FIXME: needed by operator delete + setLoadInstNumOperands(2); + } + // allocate space for exactly two operands + void *operator new(size_t s) { return User::operator new(s, 2); } + + /// Transparently provide more efficient getOperand methods. + DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); + /// Return true if this is a load from a volatile memory location. bool isVolatile() const { return getSubclassDataFromInstruction() & 1; } @@ -296,6 +306,23 @@ return getPointerOperandType()->getPointerAddressSpace(); } + bool hasNoaliasSideChannelOperand() const { return getNumOperands() == 2; } + Value *getNoaliasSideChannelOperand() const { + assert(hasNoaliasSideChannelOperand() && "we need a noalias_sidechannel"); + return getOperand(1); + } + static unsigned getNoaliasSideChannelOperandIndex() { return 1U; } + void setNoaliasSideChannelOperand(Value *SideChannel) { + assert(SideChannel && "Needs a side channel"); + setLoadInstNumOperands(2); + setOperand(1, SideChannel); + } + void removeNoaliasSideChannelOperand() { + assert(hasNoaliasSideChannelOperand() && "nothing to remove"); + // make sure 'uses' are updated + setOperand(getNoaliasSideChannelOperandIndex(), nullptr); + setLoadInstNumOperands(1); + } // Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const Instruction *I) { return I->getOpcode() == Instruction::Load; @@ -317,6 +344,11 @@ SyncScope::ID SSID; }; +template <> +struct OperandTraits : public OptionalOperandTraits {}; + +DEFINE_TRANSPARENT_OPERAND_ACCESSORS(LoadInst, Value) + //===----------------------------------------------------------------------===// // StoreInst Class //===----------------------------------------------------------------------===// @@ -347,10 +379,11 @@ StoreInst(Value *Val, Value *Ptr, bool isVolatile, MaybeAlign Align, AtomicOrdering Order, SyncScope::ID SSID, BasicBlock *InsertAtEnd); - // allocate space for exactly two operands - void *operator new(size_t s) { - return User::operator new(s, 2); + ~StoreInst() { + // FIXME: needed by operator delete + setStoreInstNumOperands(3); } + void *operator new(size_t s) { return User::operator new(s, 3); } /// Return true if this is a store to a volatile memory location. bool isVolatile() const { return getSubclassDataFromInstruction() & 1; } @@ -425,6 +458,23 @@ return getPointerOperandType()->getPointerAddressSpace(); } + bool hasNoaliasSideChannelOperand() const { return getNumOperands() == 3; } + Value *getNoaliasSideChannelOperand() const { + assert(hasNoaliasSideChannelOperand() && "we need a noalias_sidechannel"); + return getOperand(2); + } + static unsigned getNoaliasSideChannelOperandIndex() { return 2U; } + void setNoaliasSideChannelOperand(Value *SideChannel) { + assert(SideChannel && "Needs a side channel"); + setStoreInstNumOperands(3); + setOperand(2, SideChannel); + } + void removeNoaliasSideChannelOperand() { + assert(hasNoaliasSideChannelOperand() && "nothing to remove"); + // make sure 'uses' are updated + setOperand(getNoaliasSideChannelOperandIndex(), nullptr); + setStoreInstNumOperands(2); + } // Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const Instruction *I) { return I->getOpcode() == Instruction::Store; @@ -447,8 +497,7 @@ }; template <> -struct OperandTraits : public FixedNumOperandTraits { -}; +struct OperandTraits : public OptionalOperandTraits {}; DEFINE_TRANSPARENT_OPERAND_ACCESSORS(StoreInst, Value) Index: llvm/include/llvm/IR/User.h =================================================================== --- llvm/include/llvm/IR/User.h +++ llvm/include/llvm/IR/User.h @@ -167,12 +167,12 @@ } Value *getOperand(unsigned i) const { - assert(i < NumUserOperands && "getOperand() out of range!"); + assert(i < getNumOperands() && "getOperand() out of range!"); return getOperandList()[i]; } void setOperand(unsigned i, Value *Val) { - assert(i < NumUserOperands && "setOperand() out of range!"); + assert(i < getNumOperands() && "setOperand() out of range!"); assert((!isa((const Value*)this) || isa((const Value*)this)) && "Cannot mutate a constant with setOperand!"); @@ -180,15 +180,17 @@ } const Use &getOperandUse(unsigned i) const { - assert(i < NumUserOperands && "getOperandUse() out of range!"); + assert(i < getNumOperands() && "getOperandUse() out of range!"); return getOperandList()[i]; } Use &getOperandUse(unsigned i) { - assert(i < NumUserOperands && "getOperandUse() out of range!"); + assert(i < getNumOperands() && "getOperandUse() out of range!"); return getOperandList()[i]; } - unsigned getNumOperands() const { return NumUserOperands; } + unsigned getNumOperands() const { + return NumUserOperands - NumUserOperandsDelta; + } /// Returns the descriptor co-allocated with this User instance. ArrayRef getDescriptor() const; @@ -209,6 +211,24 @@ NumUserOperands = NumOps; } + /// FIXME: As that the number of operands is used to find the start of + /// the allocated memory in operator delete, we need to always think we have + /// 3 operand before delete. + void setStoreInstNumOperands(unsigned NumOps) { + assert((2 <= NumOps) && (NumOps <= 3) && + "StoreInst can only have 2 or 3 operands"); + NumUserOperandsDelta = 3 - NumOps; + } + + /// FIXME: As that the number of operands is used to find the start of + /// the allocated memory in operator delete, we need to always think we have + /// 2 operand before delete. + void setLoadInstNumOperands(unsigned NumOps) { + assert((1 <= NumOps) && (NumOps <= 2) && + "LoadInst can only have 1 or 2 operands"); + NumUserOperandsDelta = 2 - NumOps; + } + /// Subclasses with hung off uses need to manage the operand count /// themselves. In these instances, the operand count isn't used to find the /// OperandList, so there's no issue in having the operand count change. @@ -229,10 +249,10 @@ op_iterator op_begin() { return getOperandList(); } const_op_iterator op_begin() const { return getOperandList(); } op_iterator op_end() { - return getOperandList() + NumUserOperands; + return getOperandList() + NumUserOperands - NumUserOperandsDelta; } const_op_iterator op_end() const { - return getOperandList() + NumUserOperands; + return getOperandList() + NumUserOperands - NumUserOperandsDelta; } op_range operands() { return op_range(op_begin(), op_end()); Index: llvm/include/llvm/IR/Value.h =================================================================== --- llvm/include/llvm/IR/Value.h +++ llvm/include/llvm/IR/Value.h @@ -112,7 +112,7 @@ /// /// Note, this should *NOT* be used directly by any class other than User. /// User uses this value to find the Use list. - enum : unsigned { NumUserOperandsBits = 28 }; + enum : unsigned { NumUserOperandsBits = 27 }; unsigned NumUserOperands : NumUserOperandsBits; // Use the same type as the bitfield above so that MSVC will pack them. @@ -120,6 +120,7 @@ unsigned HasName : 1; unsigned HasHungOffUses : 1; unsigned HasDescriptor : 1; + unsigned NumUserOperandsDelta : 1; private: template // UseT == 'Use' or 'const Use' Index: llvm/lib/IR/AsmWriter.cpp =================================================================== --- llvm/lib/IR/AsmWriter.cpp +++ llvm/lib/IR/AsmWriter.cpp @@ -4008,15 +4008,30 @@ } Out << ", "; TypePrinter.print(I.getType(), Out); - } else if (Operand) { // Print the normal way. + } else if (const auto *LI = dyn_cast(&I)) { + Out << ' '; + TypePrinter.print(LI->getType(), Out); + Out << ", "; + writeOperand(I.getOperand(0), true); + if (LI->hasNoaliasSideChannelOperand()) { + Out << ", noalias_sidechannel "; + writeOperand(LI->getNoaliasSideChannelOperand(), true); + } + } else if (const auto *SI = dyn_cast(&I)) { + Out << ' '; + writeOperand(I.getOperand(0), true); + Out << ", "; + writeOperand(I.getOperand(1), true); + + if (SI->hasNoaliasSideChannelOperand()) { + Out << ", noalias_sidechannel "; + writeOperand(SI->getNoaliasSideChannelOperand(), true); + } + } else if (Operand) { // Print the normal way. if (const auto *GEP = dyn_cast(&I)) { Out << ' '; TypePrinter.print(GEP->getSourceElementType(), Out); Out << ','; - } else if (const auto *LI = dyn_cast(&I)) { - Out << ' '; - TypePrinter.print(LI->getType(), Out); - Out << ','; } // PrintAllTypes - Instructions who have operands of all the same type @@ -4026,8 +4041,7 @@ Type *TheType = Operand->getType(); // Select, Store and ShuffleVector always print all types. - if (isa(I) || isa(I) || isa(I) - || isa(I)) { + if (isa(I) || isa(I) || isa(I)) { PrintAllTypes = true; } else { for (unsigned i = 1, E = I.getNumOperands(); i != E; ++i) { Index: llvm/lib/IR/Instructions.cpp =================================================================== --- llvm/lib/IR/Instructions.cpp +++ llvm/lib/IR/Instructions.cpp @@ -1316,8 +1316,11 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile, MaybeAlign Align, AtomicOrdering Order, SyncScope::ID SSID, Instruction *InsertBef) - : UnaryInstruction(Ty, Load, Ptr, InsertBef) { + : Instruction(Ty, Load, OperandTraits::op_begin(this), 2, + InsertBef) { assert(Ty == cast(Ptr->getType())->getElementType()); + Op<0>() = Ptr; + setLoadInstNumOperands(1); setVolatile(isVolatile); setAlignment(MaybeAlign(Align)); setAtomic(Order, SSID); @@ -1328,7 +1331,10 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile, MaybeAlign Align, AtomicOrdering Order, SyncScope::ID SSID, BasicBlock *InsertAE) - : UnaryInstruction(Ty, Load, Ptr, InsertAE) { + : Instruction(Ty, Load, OperandTraits::op_begin(this), 2, + InsertAE) { + Op<0>() = Ptr; + setLoadInstNumOperands(1); assert(Ty == cast(Ptr->getType())->getElementType()); setVolatile(isVolatile); setAlignment(Align); @@ -1392,10 +1398,10 @@ AtomicOrdering Order, SyncScope::ID SSID, Instruction *InsertBefore) : Instruction(Type::getVoidTy(val->getContext()), Store, - OperandTraits::op_begin(this), - OperandTraits::operands(this), InsertBefore) { + OperandTraits::op_begin(this), 3, InsertBefore) { Op<0>() = val; Op<1>() = addr; + setStoreInstNumOperands(2); setVolatile(isVolatile); setAlignment(Align); setAtomic(Order, SSID); @@ -1406,10 +1412,10 @@ AtomicOrdering Order, SyncScope::ID SSID, BasicBlock *InsertAtEnd) : Instruction(Type::getVoidTy(val->getContext()), Store, - OperandTraits::op_begin(this), - OperandTraits::operands(this), InsertAtEnd) { + OperandTraits::op_begin(this), 3, InsertAtEnd) { Op<0>() = val; Op<1>() = addr; + setStoreInstNumOperands(2); setVolatile(isVolatile); setAlignment(Align); setAtomic(Order, SSID); @@ -4136,15 +4142,32 @@ } LoadInst *LoadInst::cloneImpl() const { - return new LoadInst(getType(), getOperand(0), Twine(), isVolatile(), - MaybeAlign(getAlignment()), getOrdering(), - getSyncScopeID()); + LoadInst *Result = + new LoadInst(getType(), getOperand(0), Twine(), isVolatile(), + MaybeAlign(getAlignment()), getOrdering(), getSyncScopeID()); + // - we must keep the same number of arguments (for vector optimizations) + // - if we duplicate the side channel, we can get into problems with passes + // that don't know how to handle it (Like MergeLoadStoreMotion shows) + // - safe alternative: keep the argument, but map it to undef. + if (hasNoaliasSideChannelOperand()) + Result->setNoaliasSideChannelOperand( + UndefValue::get(getNoaliasSideChannelOperand()->getType())); + return Result; } StoreInst *StoreInst::cloneImpl() const { - return new StoreInst(getOperand(0), getOperand(1), isVolatile(), - MaybeAlign(getAlignment()), getOrdering(), - getSyncScopeID()); + StoreInst *Result = new StoreInst(getOperand(0), getOperand(1), isVolatile(), + MaybeAlign(getAlignment()), getOrdering(), + getSyncScopeID()); + + // we must keep the same number of arguments (for vector optimizations) + // - if we duplicate the side channel, we can get into problems with passes + // that don't know how to handle it (Like MergeLoadStoreMotion shows) + // - safe alternative: keep the argument, but map it to undef. + if (hasNoaliasSideChannelOperand()) + Result->setNoaliasSideChannelOperand( + UndefValue::get(getNoaliasSideChannelOperand()->getType())); + return Result; } AtomicCmpXchgInst *AtomicCmpXchgInst::cloneImpl() const { Index: llvm/lib/IR/User.cpp =================================================================== --- llvm/lib/IR/User.cpp +++ llvm/lib/IR/User.cpp @@ -171,20 +171,19 @@ Use **HungOffOperandList = static_cast(Usr) - 1; // drop the hung off uses. - Use::zap(*HungOffOperandList, *HungOffOperandList + Obj->NumUserOperands, + Use::zap(*HungOffOperandList, *HungOffOperandList + Obj->getNumOperands(), /* Delete */ true); ::operator delete(HungOffOperandList); } else if (Obj->HasDescriptor) { Use *UseBegin = static_cast(Usr) - Obj->NumUserOperands; - Use::zap(UseBegin, UseBegin + Obj->NumUserOperands, /* Delete */ false); + Use::zap(UseBegin, UseBegin + Obj->getNumOperands(), /* Delete */ false); auto *DI = reinterpret_cast(UseBegin) - 1; uint8_t *Storage = reinterpret_cast(DI) - DI->SizeInBytes; ::operator delete(Storage); } else { Use *Storage = static_cast(Usr) - Obj->NumUserOperands; - Use::zap(Storage, Storage + Obj->NumUserOperands, - /* Delete */ false); + Use::zap(Storage, Storage + Obj->getNumOperands(), /* Delete */ false); ::operator delete(Storage); } } Index: llvm/lib/IR/Value.cpp =================================================================== --- llvm/lib/IR/Value.cpp +++ llvm/lib/IR/Value.cpp @@ -50,9 +50,9 @@ } Value::Value(Type *ty, unsigned scid) - : VTy(checkType(ty)), UseList(nullptr), SubclassID(scid), - HasValueHandle(0), SubclassOptionalData(0), SubclassData(0), - NumUserOperands(0), IsUsedByMD(false), HasName(false) { + : VTy(checkType(ty)), UseList(nullptr), SubclassID(scid), HasValueHandle(0), + SubclassOptionalData(0), SubclassData(0), NumUserOperands(0), + IsUsedByMD(false), HasName(false), NumUserOperandsDelta(0) { static_assert(ConstantFirstVal == 0, "!(SubclassID < ConstantFirstVal)"); // FIXME: Why isn't this in the subclass gunk?? // Note, we cannot call isa before the CallInst has been Index: llvm/unittests/IR/IRBuilderTest.cpp =================================================================== --- llvm/unittests/IR/IRBuilderTest.cpp +++ llvm/unittests/IR/IRBuilderTest.cpp @@ -293,6 +293,31 @@ EXPECT_FALSE(verifyModule(*M)); } +TEST_F(IRBuilderTest, SideChannel) { + IRBuilder<> Builder(BB); + + auto *L = Builder.CreateLoad(GV->getValueType(), GV); + EXPECT_TRUE(!L->hasNoaliasSideChannelOperand()); + + auto *S = Builder.CreateStore(L, GV); + EXPECT_TRUE(!S->hasNoaliasSideChannelOperand()); + + L->setNoaliasSideChannelOperand(GV); + EXPECT_TRUE(L->hasNoaliasSideChannelOperand()); + + S->setNoaliasSideChannelOperand(GV); + EXPECT_TRUE(S->hasNoaliasSideChannelOperand()); + + EXPECT_EQ(L->getNoaliasSideChannelOperand(), GV); + EXPECT_EQ(S->getNoaliasSideChannelOperand(), GV); + + L->removeNoaliasSideChannelOperand(); + EXPECT_TRUE(!L->hasNoaliasSideChannelOperand()); + + S->removeNoaliasSideChannelOperand(); + EXPECT_TRUE(!S->hasNoaliasSideChannelOperand()); +} + TEST_F(IRBuilderTest, Lifetime) { IRBuilder<> Builder(BB); AllocaInst *Var1 = Builder.CreateAlloca(Builder.getInt8Ty());