Index: llvm/include/llvm/IR/InstVisitor.h =================================================================== --- llvm/include/llvm/IR/InstVisitor.h +++ llvm/include/llvm/IR/InstVisitor.h @@ -168,7 +168,7 @@ RetTy visitICmpInst(ICmpInst &I) { DELEGATE(CmpInst);} RetTy visitFCmpInst(FCmpInst &I) { DELEGATE(CmpInst);} RetTy visitAllocaInst(AllocaInst &I) { DELEGATE(UnaryInstruction);} - RetTy visitLoadInst(LoadInst &I) { DELEGATE(UnaryInstruction);} + RetTy visitLoadInst(LoadInst &I) { DELEGATE(Instruction); } RetTy visitStoreInst(StoreInst &I) { DELEGATE(Instruction);} RetTy visitAtomicCmpXchgInst(AtomicCmpXchgInst &I) { DELEGATE(Instruction);} RetTy visitAtomicRMWInst(AtomicRMWInst &I) { DELEGATE(Instruction);} Index: llvm/include/llvm/IR/Instructions.h =================================================================== --- llvm/include/llvm/IR/Instructions.h +++ llvm/include/llvm/IR/Instructions.h @@ -166,7 +166,7 @@ /// An instruction for reading from memory. This uses the SubclassData field in /// Value to store whether or not the load is volatile. -class LoadInst : public UnaryInstruction { +class LoadInst : public Instruction { void AssertOK(); protected: @@ -229,6 +229,16 @@ : LoadInst(Ptr->getType()->getPointerElementType(), Ptr, NameStr, isVolatile, Align, Order, SSID, InsertAtEnd) {} + ~LoadInst() { + // FIXME: needed by operator delete + setLoadInstNumOperands(2); + } + // allocate space for exactly two operands + void *operator new(size_t s) { return User::operator new(s, 2); } + + /// Transparently provide more efficient getOperand methods. + DECLARE_TRANSPARENT_OPERAND_ACCESSORS(Value); + /// Return true if this is a load from a volatile memory location. bool isVolatile() const { return getSubclassDataFromInstruction() & 1; } @@ -296,6 +306,23 @@ return getPointerOperandType()->getPointerAddressSpace(); } + bool hasNoaliasSideChannelOperand() const { return getNumOperands() == 2; } + Value *getNoaliasSideChannelOperand() const { + assert(hasNoaliasSideChannelOperand() && "we need a noalias_sidechannel"); + return getOperand(1); + } + static unsigned getNoaliasSideChannelOperandIndex() { return 1U; } + void setNoaliasSideChannelOperand(Value *SideChannel) { + assert(SideChannel && "Needs a side channel"); + setLoadInstNumOperands(2); + setOperand(1, SideChannel); + } + void removeNoaliasSideChannelOperand() { + assert(hasNoaliasSideChannelOperand() && "nothing to remove"); + // make sure 'uses' are updated + setOperand(getNoaliasSideChannelOperandIndex(), nullptr); + setLoadInstNumOperands(1); + } // Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const Instruction *I) { return I->getOpcode() == Instruction::Load; @@ -317,6 +344,11 @@ SyncScope::ID SSID; }; +template <> +struct OperandTraits : public OptionalOperandTraits {}; + +DEFINE_TRANSPARENT_OPERAND_ACCESSORS(LoadInst, Value) + //===----------------------------------------------------------------------===// // StoreInst Class //===----------------------------------------------------------------------===// @@ -349,10 +381,11 @@ unsigned Align, AtomicOrdering Order, SyncScope::ID SSID, BasicBlock *InsertAtEnd); - // allocate space for exactly two operands - void *operator new(size_t s) { - return User::operator new(s, 2); + ~StoreInst() { + // FIXME: needed by operator delete + setStoreInstNumOperands(3); } + void *operator new(size_t s) { return User::operator new(s, 3); } /// Return true if this is a store to a volatile memory location. bool isVolatile() const { return getSubclassDataFromInstruction() & 1; } @@ -427,6 +460,23 @@ return getPointerOperandType()->getPointerAddressSpace(); } + bool hasNoaliasSideChannelOperand() const { return getNumOperands() == 3; } + Value *getNoaliasSideChannelOperand() const { + assert(hasNoaliasSideChannelOperand() && "we need a noalias_sidechannel"); + return getOperand(2); + } + static unsigned getNoaliasSideChannelOperandIndex() { return 2U; } + void setNoaliasSideChannelOperand(Value *SideChannel) { + assert(SideChannel && "Needs a side channel"); + setStoreInstNumOperands(3); + setOperand(2, SideChannel); + } + void removeNoaliasSideChannelOperand() { + assert(hasNoaliasSideChannelOperand() && "nothing to remove"); + // make sure 'uses' are updated + setOperand(getNoaliasSideChannelOperandIndex(), nullptr); + setStoreInstNumOperands(2); + } // Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const Instruction *I) { return I->getOpcode() == Instruction::Store; @@ -449,8 +499,7 @@ }; template <> -struct OperandTraits : public FixedNumOperandTraits { -}; +struct OperandTraits : public OptionalOperandTraits {}; DEFINE_TRANSPARENT_OPERAND_ACCESSORS(StoreInst, Value) Index: llvm/include/llvm/IR/User.h =================================================================== --- llvm/include/llvm/IR/User.h +++ llvm/include/llvm/IR/User.h @@ -167,12 +167,12 @@ } Value *getOperand(unsigned i) const { - assert(i < NumUserOperands && "getOperand() out of range!"); + assert(i < getNumOperands() && "getOperand() out of range!"); return getOperandList()[i]; } void setOperand(unsigned i, Value *Val) { - assert(i < NumUserOperands && "setOperand() out of range!"); + assert(i < getNumOperands() && "setOperand() out of range!"); assert((!isa((const Value*)this) || isa((const Value*)this)) && "Cannot mutate a constant with setOperand!"); @@ -180,15 +180,17 @@ } const Use &getOperandUse(unsigned i) const { - assert(i < NumUserOperands && "getOperandUse() out of range!"); + assert(i < getNumOperands() && "getOperandUse() out of range!"); return getOperandList()[i]; } Use &getOperandUse(unsigned i) { - assert(i < NumUserOperands && "getOperandUse() out of range!"); + assert(i < getNumOperands() && "getOperandUse() out of range!"); return getOperandList()[i]; } - unsigned getNumOperands() const { return NumUserOperands; } + unsigned getNumOperands() const { + return NumUserOperands - NumUserOperandsDelta; + } /// Returns the descriptor co-allocated with this User instance. ArrayRef getDescriptor() const; @@ -209,6 +211,24 @@ NumUserOperands = NumOps; } + /// FIXME: As that the number of operands is used to find the start of + /// the allocated memory in operator delete, we need to always think we have + /// 3 operand before delete. + void setStoreInstNumOperands(unsigned NumOps) { + assert((2 <= NumOps) && (NumOps <= 3) && + "StoreInst can only have 2 or 3 operands"); + NumUserOperandsDelta = 3 - NumOps; + } + + /// FIXME: As that the number of operands is used to find the start of + /// the allocated memory in operator delete, we need to always think we have + /// 2 operand before delete. + void setLoadInstNumOperands(unsigned NumOps) { + assert((1 <= NumOps) && (NumOps <= 2) && + "LoadInst can only have 1 or 2 operands"); + NumUserOperandsDelta = 2 - NumOps; + } + /// Subclasses with hung off uses need to manage the operand count /// themselves. In these instances, the operand count isn't used to find the /// OperandList, so there's no issue in having the operand count change. @@ -229,10 +249,10 @@ op_iterator op_begin() { return getOperandList(); } const_op_iterator op_begin() const { return getOperandList(); } op_iterator op_end() { - return getOperandList() + NumUserOperands; + return getOperandList() + NumUserOperands - NumUserOperandsDelta; } const_op_iterator op_end() const { - return getOperandList() + NumUserOperands; + return getOperandList() + NumUserOperands - NumUserOperandsDelta; } op_range operands() { return op_range(op_begin(), op_end()); Index: llvm/include/llvm/IR/Value.h =================================================================== --- llvm/include/llvm/IR/Value.h +++ llvm/include/llvm/IR/Value.h @@ -111,7 +111,7 @@ /// /// Note, this should *NOT* be used directly by any class other than User. /// User uses this value to find the Use list. - enum : unsigned { NumUserOperandsBits = 28 }; + enum : unsigned { NumUserOperandsBits = 27 }; unsigned NumUserOperands : NumUserOperandsBits; // Use the same type as the bitfield above so that MSVC will pack them. @@ -119,6 +119,7 @@ unsigned HasName : 1; unsigned HasHungOffUses : 1; unsigned HasDescriptor : 1; + unsigned NumUserOperandsDelta : 1; private: template // UseT == 'Use' or 'const Use' Index: llvm/lib/IR/AsmWriter.cpp =================================================================== --- llvm/lib/IR/AsmWriter.cpp +++ llvm/lib/IR/AsmWriter.cpp @@ -4006,15 +4006,30 @@ } Out << ", "; TypePrinter.print(I.getType(), Out); - } else if (Operand) { // Print the normal way. + } else if (const auto *LI = dyn_cast(&I)) { + Out << ' '; + TypePrinter.print(LI->getType(), Out); + Out << ", "; + writeOperand(I.getOperand(0), true); + if (LI->hasNoaliasSideChannelOperand()) { + Out << ", noalias_sidechannel "; + writeOperand(LI->getNoaliasSideChannelOperand(), true); + } + } else if (const auto *SI = dyn_cast(&I)) { + Out << ' '; + writeOperand(I.getOperand(0), true); + Out << ", "; + writeOperand(I.getOperand(1), true); + + if (SI->hasNoaliasSideChannelOperand()) { + Out << ", noalias_sidechannel "; + writeOperand(SI->getNoaliasSideChannelOperand(), true); + } + } else if (Operand) { // Print the normal way. if (const auto *GEP = dyn_cast(&I)) { Out << ' '; TypePrinter.print(GEP->getSourceElementType(), Out); Out << ','; - } else if (const auto *LI = dyn_cast(&I)) { - Out << ' '; - TypePrinter.print(LI->getType(), Out); - Out << ','; } // PrintAllTypes - Instructions who have operands of all the same type @@ -4024,8 +4039,7 @@ Type *TheType = Operand->getType(); // Select, Store and ShuffleVector always print all types. - if (isa(I) || isa(I) || isa(I) - || isa(I)) { + if (isa(I) || isa(I) || isa(I)) { PrintAllTypes = true; } else { for (unsigned i = 1, E = I.getNumOperands(); i != E; ++i) { Index: llvm/lib/IR/Instructions.cpp =================================================================== --- llvm/lib/IR/Instructions.cpp +++ llvm/lib/IR/Instructions.cpp @@ -1312,10 +1312,13 @@ SyncScope::System, InsertAE) {} LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile, - unsigned Align, AtomicOrdering Order, - SyncScope::ID SSID, Instruction *InsertBef) - : UnaryInstruction(Ty, Load, Ptr, InsertBef) { + unsigned Align, AtomicOrdering Order, SyncScope::ID SSID, + Instruction *InsertBef) + : Instruction(Ty, Load, OperandTraits::op_begin(this), 2, + InsertBef) { assert(Ty == cast(Ptr->getType())->getElementType()); + Op<0>() = Ptr; + setLoadInstNumOperands(1); setVolatile(isVolatile); setAlignment(MaybeAlign(Align)); setAtomic(Order, SSID); @@ -1326,7 +1329,10 @@ LoadInst::LoadInst(Type *Ty, Value *Ptr, const Twine &Name, bool isVolatile, unsigned Align, AtomicOrdering Order, SyncScope::ID SSID, BasicBlock *InsertAE) - : UnaryInstruction(Ty, Load, Ptr, InsertAE) { + : Instruction(Ty, Load, OperandTraits::op_begin(this), 2, + InsertAE) { + Op<0>() = Ptr; + setLoadInstNumOperands(1); assert(Ty == cast(Ptr->getType())->getElementType()); setVolatile(isVolatile); setAlignment(MaybeAlign(Align)); @@ -1386,32 +1392,28 @@ : StoreInst(val, addr, isVolatile, Align, AtomicOrdering::NotAtomic, SyncScope::System, InsertAtEnd) {} -StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, - unsigned Align, AtomicOrdering Order, - SyncScope::ID SSID, +StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, unsigned Align, + AtomicOrdering Order, SyncScope::ID SSID, Instruction *InsertBefore) - : Instruction(Type::getVoidTy(val->getContext()), Store, - OperandTraits::op_begin(this), - OperandTraits::operands(this), - InsertBefore) { + : Instruction(Type::getVoidTy(val->getContext()), Store, + OperandTraits::op_begin(this), 3, InsertBefore) { Op<0>() = val; Op<1>() = addr; + setStoreInstNumOperands(2); setVolatile(isVolatile); setAlignment(MaybeAlign(Align)); setAtomic(Order, SSID); AssertOK(); } -StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, - unsigned Align, AtomicOrdering Order, - SyncScope::ID SSID, +StoreInst::StoreInst(Value *val, Value *addr, bool isVolatile, unsigned Align, + AtomicOrdering Order, SyncScope::ID SSID, BasicBlock *InsertAtEnd) - : Instruction(Type::getVoidTy(val->getContext()), Store, - OperandTraits::op_begin(this), - OperandTraits::operands(this), - InsertAtEnd) { + : Instruction(Type::getVoidTy(val->getContext()), Store, + OperandTraits::op_begin(this), 3, InsertAtEnd) { Op<0>() = val; Op<1>() = addr; + setStoreInstNumOperands(2); setVolatile(isVolatile); setAlignment(MaybeAlign(Align)); setAtomic(Order, SSID); @@ -4138,14 +4140,32 @@ } LoadInst *LoadInst::cloneImpl() const { - return new LoadInst(getType(), getOperand(0), Twine(), isVolatile(), - getAlignment(), getOrdering(), getSyncScopeID()); + LoadInst *Result = + new LoadInst(getType(), getOperand(0), Twine(), isVolatile(), + getAlignment(), getOrdering(), getSyncScopeID()); + // - we must keep the same number of arguments (for vector optimizations) + // - if we duplicate the side channel, we can get into problems with passes + // that don't know how to handle it (Like MergeLoadStoreMotion shows) + // - safe alternative: keep the argument, but map it to undef. + if (hasNoaliasSideChannelOperand()) + Result->setNoaliasSideChannelOperand( + UndefValue::get(getNoaliasSideChannelOperand()->getType())); + return Result; } StoreInst *StoreInst::cloneImpl() const { - return new StoreInst(getOperand(0), getOperand(1), isVolatile(), - getAlignment(), getOrdering(), getSyncScopeID()); - + StoreInst *Result = + new StoreInst(getOperand(0), getOperand(1), isVolatile(), getAlignment(), + getOrdering(), getSyncScopeID()); + + // we must keep the same number of arguments (for vector optimizations) + // - if we duplicate the side channel, we can get into problems with passes + // that don't know how to handle it (Like MergeLoadStoreMotion shows) + // - safe alternative: keep the argument, but map it to undef. + if (hasNoaliasSideChannelOperand()) + Result->setNoaliasSideChannelOperand( + UndefValue::get(getNoaliasSideChannelOperand()->getType())); + return Result; } AtomicCmpXchgInst *AtomicCmpXchgInst::cloneImpl() const { Index: llvm/lib/IR/User.cpp =================================================================== --- llvm/lib/IR/User.cpp +++ llvm/lib/IR/User.cpp @@ -171,20 +171,19 @@ Use **HungOffOperandList = static_cast(Usr) - 1; // drop the hung off uses. - Use::zap(*HungOffOperandList, *HungOffOperandList + Obj->NumUserOperands, + Use::zap(*HungOffOperandList, *HungOffOperandList + Obj->getNumOperands(), /* Delete */ true); ::operator delete(HungOffOperandList); } else if (Obj->HasDescriptor) { Use *UseBegin = static_cast(Usr) - Obj->NumUserOperands; - Use::zap(UseBegin, UseBegin + Obj->NumUserOperands, /* Delete */ false); + Use::zap(UseBegin, UseBegin + Obj->getNumOperands(), /* Delete */ false); auto *DI = reinterpret_cast(UseBegin) - 1; uint8_t *Storage = reinterpret_cast(DI) - DI->SizeInBytes; ::operator delete(Storage); } else { Use *Storage = static_cast(Usr) - Obj->NumUserOperands; - Use::zap(Storage, Storage + Obj->NumUserOperands, - /* Delete */ false); + Use::zap(Storage, Storage + Obj->getNumOperands(), /* Delete */ false); ::operator delete(Storage); } } Index: llvm/lib/IR/Value.cpp =================================================================== --- llvm/lib/IR/Value.cpp +++ llvm/lib/IR/Value.cpp @@ -50,9 +50,9 @@ } Value::Value(Type *ty, unsigned scid) - : VTy(checkType(ty)), UseList(nullptr), SubclassID(scid), - HasValueHandle(0), SubclassOptionalData(0), SubclassData(0), - NumUserOperands(0), IsUsedByMD(false), HasName(false) { + : VTy(checkType(ty)), UseList(nullptr), SubclassID(scid), HasValueHandle(0), + SubclassOptionalData(0), SubclassData(0), NumUserOperands(0), + IsUsedByMD(false), HasName(false), NumUserOperandsDelta(0) { static_assert(ConstantFirstVal == 0, "!(SubclassID < ConstantFirstVal)"); // FIXME: Why isn't this in the subclass gunk?? // Note, we cannot call isa before the CallInst has been Index: llvm/unittests/IR/IRBuilderTest.cpp =================================================================== --- llvm/unittests/IR/IRBuilderTest.cpp +++ llvm/unittests/IR/IRBuilderTest.cpp @@ -284,6 +284,31 @@ EXPECT_FALSE(verifyModule(*M)); } +TEST_F(IRBuilderTest, SideChannel) { + IRBuilder<> Builder(BB); + + auto *L = Builder.CreateLoad(GV->getValueType(), GV); + EXPECT_TRUE(!L->hasNoaliasSideChannelOperand()); + + auto *S = Builder.CreateStore(L, GV); + EXPECT_TRUE(!S->hasNoaliasSideChannelOperand()); + + L->setNoaliasSideChannelOperand(GV); + EXPECT_TRUE(L->hasNoaliasSideChannelOperand()); + + S->setNoaliasSideChannelOperand(GV); + EXPECT_TRUE(S->hasNoaliasSideChannelOperand()); + + EXPECT_EQ(L->getNoaliasSideChannelOperand(), GV); + EXPECT_EQ(S->getNoaliasSideChannelOperand(), GV); + + L->removeNoaliasSideChannelOperand(); + EXPECT_TRUE(!L->hasNoaliasSideChannelOperand()); + + S->removeNoaliasSideChannelOperand(); + EXPECT_TRUE(!S->hasNoaliasSideChannelOperand()); +} + TEST_F(IRBuilderTest, Lifetime) { IRBuilder<> Builder(BB); AllocaInst *Var1 = Builder.CreateAlloca(Builder.getInt8Ty());