Index: llvm/include/llvm/IR/IRBuilder.h =================================================================== --- llvm/include/llvm/IR/IRBuilder.h +++ llvm/include/llvm/IR/IRBuilder.h @@ -641,9 +641,63 @@ /// assume that the provided condition will be true. CallInst *CreateAssumption(Value *Cond); - /// \brief Create an noalias intrinsic. - Instruction *CreateNoAliasPointer(Value *Ptr, MDNode *ScopeTag); + /// Create a llvm.noalias.decl intrinsic call. + Instruction *CreateNoAliasDeclaration(Value *AllocaPtr, Value *ObjId, + Value *Scope); + Instruction *CreateNoAliasDeclaration(Value *AllocaPtr, uint64_t ObjId, + Value *Scope) { + return CreateNoAliasDeclaration( + AllocaPtr, + ConstantInt::get(IntegerType::getInt64Ty(getContext()), ObjId), Scope); + } + Instruction *CreateNoAliasDeclaration(Value *AllocaPtr, MDNode *ScopeTag) { + uint64_t Zero = 0; + return CreateNoAliasDeclaration(AllocaPtr, Zero, + MetadataAsValue::get(Context, ScopeTag)); + } + + /// Create a llvm.noalias intrinsic call. + Instruction *CreateNoAliasPointer(Value *Ptr, Value *NoAliasDecl, + Value *AddrP, MDNode *ScopeTag, + const Twine &Name = "", + uint64_t ObjectId = 0) { + return CreateNoAliasPointer(Ptr, NoAliasDecl, AddrP, + MetadataAsValue::get(getContext(), ScopeTag), + Name, ObjectId); + } + Instruction *CreateNoAliasPointer(Value *Ptr, Value *NoAliasDecl, + Value *AddrP, Value *ScopeTag, + const Twine &Name = "", + uint64_t ObjectId = 0); + + /// Create a llvm.side.noalias intrinsic call. + Instruction *CreateSideNoAliasPlain(Value *Ptr, Value *NoAliasDecl, + Value *AddrP, Value *AddrP_Side, + Value *ObjId, MDNode *ScopeTag, + const Twine &Name = ""); + Instruction *CreateSideNoAliasPlain(Value *Ptr, Value *NoAliasDecl, + Value *AddrP, Value *AddrP_Side, + Value *ObjId, Value *ScopeValue, + const Twine &Name = ""); + + /// Create a llvm.noalias.arg.guard intrinsic call. + Instruction *CreateNoAliasArgGuard(Value *Ptr, Value *SideChannel, + const Twine &Name = ""); + + /// Create a llvm.noalias_copy_guard intrinsic call. + Instruction *CreateNoAliasCopyGuard(Value *BasePtr, Value *NoAliasDel, + ArrayRef EncodedIndices, + MDNode *ScopeTag, const Twine &Name = ""); +private: + /// Helper for creating noalias intrinsics + Instruction *CreateGenericNoAliasIntrinsic(Intrinsic::ID ID, Value *arg0, + ArrayRef args_opt, + ArrayRef MDNodes, + ArrayRef MDValues, + const Twine &Name = ""); + +public: /// Create a call to the experimental.gc.statepoint intrinsic to /// start a new statepoint sequence. CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes, Index: llvm/include/llvm/IR/IntrinsicInst.h =================================================================== --- llvm/include/llvm/IR/IntrinsicInst.h +++ llvm/include/llvm/IR/IntrinsicInst.h @@ -883,6 +883,21 @@ } }; + /// Returns true when two llvm.side.noalias represent the same noalias info. + inline bool areSideNoaliasCompatible(const IntrinsicInst *lhs, + const IntrinsicInst *rhs) { + assert(lhs->getIntrinsicID() == Intrinsic::side_noalias && + rhs->getIntrinsicID() == Intrinsic::side_noalias && + "Can only check noalias compatibility of side_noalias"); + return std::forward_as_tuple( + lhs->getOperand(Intrinsic::SideNoAliasScopeArg), + lhs->getOperand(Intrinsic::SideNoAliasIdentifyPObjIdArg), + lhs->getOperand(Intrinsic::SideNoAliasIdentifyPArg)) == + std::forward_as_tuple( + rhs->getOperand(Intrinsic::SideNoAliasScopeArg), + rhs->getOperand(Intrinsic::SideNoAliasIdentifyPObjIdArg), + rhs->getOperand(Intrinsic::SideNoAliasIdentifyPArg)); + } } // end namespace llvm #endif // LLVM_IR_INTRINSICINST_H Index: llvm/include/llvm/IR/Intrinsics.h =================================================================== --- llvm/include/llvm/IR/Intrinsics.h +++ llvm/include/llvm/IR/Intrinsics.h @@ -33,6 +33,27 @@ /// function known by LLVM. The enum values are returned by /// Function::getIntrinsicID(). namespace Intrinsic { + // Abstraction for the arguments of the noalias intrinsics + static const int SideNoAliasNoAliasDeclArg = 1; + static const int SideNoAliasIdentifyPArg = 2; + static const int SideNoAliasIdentifyPSideChannelArg = 3; + static const int SideNoAliasIdentifyPObjIdArg = 4; + static const int SideNoAliasScopeArg = 5; + + static const int NoAliasNoAliasDeclArg = 1; + static const int NoAliasIdentifyPArg = 2; + static const int NoAliasIdentifyPObjIdArg = 3; + static const int NoAliasScopeArg = 4; + + static const int NoAliasDeclAllocaArg = 0; + static const int NoAliasDeclObjIdArg = 1; + static const int NoAliasDeclScopeArg = 2; + + static const int NoAliasCopyGuardIdentifyPBaseObject = 0; + static const int NoAliasCopyGuardNoAliasDeclArg = 1; + static const int NoAliasCopyGuardIndicesArg = 2; + static const int NoAliasCopyGuardScopeArg = 3; + enum ID : unsigned { not_intrinsic = 0, // Must be zero Index: llvm/lib/IR/IRBuilder.cpp =================================================================== --- llvm/lib/IR/IRBuilder.cpp +++ llvm/lib/IR/IRBuilder.cpp @@ -23,7 +23,6 @@ #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/LLVMContext.h" -#include "llvm/IR/Metadata.h" #include "llvm/IR/Operator.h" #include "llvm/IR/Statepoint.h" #include "llvm/IR/Type.h" @@ -458,9 +457,97 @@ return createCallHelper(FnAssume, Ops, this); } -Instruction *IRBuilderBase::CreateNoAliasPointer(Value *Ptr, MDNode *ScopeTag) { +Instruction *IRBuilderBase::CreateNoAliasDeclaration(Value *AllocaPtr, + Value *ObjId, + Value *Scope) { + assert(AllocaPtr); + + SmallVector Types = {Type::getInt8PtrTy(getContext()), + AllocaPtr->getType(), ObjId->getType()}; + SmallVector Ops = {AllocaPtr, ObjId, Scope}; + + Module *M = BB->getModule(); + auto *FnIntrinsic = + Intrinsic::getDeclaration(M, Intrinsic::noalias_decl, Types); + return createCallHelper(FnIntrinsic, Ops, this); +} + +Instruction *IRBuilderBase::CreateNoAliasPointer(Value *Ptr, Value *NoAliasDecl, + Value *AddrP, Value *ScopeTag, + const Twine &Name, + uint64_t ObjectId) { + assert(Ptr && AddrP); + Value *I64_0 = + ConstantInt::get(IntegerType::getInt64Ty(getContext()), ObjectId); + return CreateGenericNoAliasIntrinsic(Intrinsic::noalias, Ptr, + {NoAliasDecl, AddrP, I64_0}, {}, + {ScopeTag}, Name); +} + +Instruction *IRBuilderBase::CreateSideNoAliasPlain( + Value *Ptr, Value *NoAliasDecl, Value *AddrP, Value *AddrP_Side, + Value *ObjId, MDNode *ScopeTag, const Twine &Name) { + assert(Ptr && AddrP && AddrP_Side); + return CreateGenericNoAliasIntrinsic(Intrinsic::side_noalias, Ptr, + {NoAliasDecl, AddrP, AddrP_Side, ObjId}, + {ScopeTag}, {}, Name); +} + +Instruction *IRBuilderBase::CreateSideNoAliasPlain( + Value *Ptr, Value *NoAliasDecl, Value *AddrP, Value *AddrP_Side, + Value *ObjId, Value *ScopeValue, const Twine &Name) { + assert(Ptr && AddrP && AddrP_Side); + return CreateGenericNoAliasIntrinsic(Intrinsic::side_noalias, Ptr, + {NoAliasDecl, AddrP, AddrP_Side, ObjId}, + {}, {ScopeValue}, Name); +} + +Instruction *IRBuilderBase::CreateNoAliasArgGuard(Value *Ptr, + Value *SideChannel, + const Twine &Name) { + return CreateGenericNoAliasIntrinsic(Intrinsic::noalias_arg_guard, Ptr, + {SideChannel}, {}, {}, Name); +} + +Instruction * +IRBuilderBase::CreateNoAliasCopyGuard(Value *BasePtr, Value *NoAliasDecl, + ArrayRef EncodedIndices, + MDNode *ScopeTag, const Twine &Name) { + SmallVector IndicesArray; + + assert(NoAliasDecl != nullptr); + assert(ScopeTag != nullptr); + assert(!EncodedIndices.empty() && + "NoAliasCopyGuard should have at least one set of indices"); + + for (unsigned i = 0, ci = EncodedIndices.size(); i < ci;) { + auto length = EncodedIndices[i++]; + assert(length > 0 && + "NoAliasCopyGuard: encoded indices problem: zero length seen"); + length += i; + assert(length <= ci && + "NoAliasCopyGuard: encoded indices problem: too few indices"); + + SmallVector Indices; + while (i < length) { + Indices.push_back( + llvm::ConstantAsMetadata::get(getInt64(EncodedIndices[i]))); + ++i; + } + IndicesArray.push_back(llvm::MDNode::get(Context, Indices)); + } + + return CreateGenericNoAliasIntrinsic( + Intrinsic::noalias_copy_guard, BasePtr, {NoAliasDecl}, + {llvm::MDNode::get(Context, IndicesArray), ScopeTag}, {}, Name); +} + +Instruction *IRBuilderBase::CreateGenericNoAliasIntrinsic( + Intrinsic::ID ID, Value *Ptr, ArrayRef args_opt, + ArrayRef MDNodes, ArrayRef MDValues, const Twine &Name) { // FIXME: We can currently mangle just about everything, but not literal // structs (for those, bitcast to i8*). + //@ JDO: FIXME: as far as I see in 'getMangledTypeStr', this should be ok now. Value *CPtr = Ptr; if (auto *STyp = dyn_cast(CPtr->getType()->getPointerElementType())) { @@ -468,14 +555,23 @@ CPtr = getCastedInt8PtrValue(CPtr); } - Type *Types[] = { CPtr->getType() }; - Value *Ops[] = { CPtr, MetadataAsValue::get(Context, ScopeTag) }; + SmallVector Types = {CPtr->getType()}; + SmallVector Ops = {CPtr}; + for (auto *A : args_opt) { + Types.push_back(A->getType()); + Ops.push_back(A); + } + // For the metadata info, types must not be added: + for (auto *MD : MDNodes) { + Ops.push_back(MetadataAsValue::get(Context, MD)); + } + Ops.insert(Ops.end(), MDValues.begin(), MDValues.end()); Module *M = BB->getModule(); - Function *FnNoAlias = Intrinsic::getDeclaration(M, Intrinsic::noalias, Types); - Instruction *Ret = createCallHelper(FnNoAlias, Ops, this); + auto *FnIntrinsic = Intrinsic::getDeclaration(M, ID, Types); + Instruction *Ret = createCallHelper(FnIntrinsic, Ops, this, Name); if (Ret->getType() != Ptr->getType()) { - BitCastInst *BCI = new BitCastInst(Ret, Ptr->getType(), ""); + BitCastInst *BCI = new BitCastInst(Ret, Ptr->getType(), Name + ".cast"); BB->getInstList().insert(InsertPt, BCI); SetInstDebugLocation(BCI); Ret = BCI; Index: llvm/lib/IR/Verifier.cpp =================================================================== --- llvm/lib/IR/Verifier.cpp +++ llvm/lib/IR/Verifier.cpp @@ -4719,6 +4719,81 @@ "Intrinsic does not support vectors", &Call); break; } + case Intrinsic::noalias_decl: { + Assert( + (isa( + Call.getArgOperand(Intrinsic::NoAliasDeclAllocaArg)) || + isa(Call.getArgOperand(Intrinsic::NoAliasDeclAllocaArg))), + "llvm.noalias.decl must refer to a null-ptr or an alloca instruction", + Call); + break; + } + case Intrinsic::noalias: { + auto *DA = Call.getArgOperand(Intrinsic::NoAliasNoAliasDeclArg); + if (!isa(DA) && !isa(DA)) { + auto *DeclSite = dyn_cast(DA); + Assert(DeclSite, "llvm.noalias arg1 must refer to a llvm.noalias.decl", + Call); + Assert((DeclSite->getArgOperand(Intrinsic::NoAliasDeclObjIdArg) == + Call.getArgOperand(Intrinsic::NoAliasIdentifyPObjIdArg)), + "llvm.noalias objId arg must match with llvm.noalias.decl", Call); + Assert((DeclSite->getArgOperand(Intrinsic::NoAliasDeclScopeArg) == + Call.getArgOperand(Intrinsic::NoAliasScopeArg)), + "llvm.noalias scope arg must match with llvm.noalias.decl", Call); + } + break; + } + case Intrinsic::side_noalias: { + auto *DA = Call.getArgOperand(Intrinsic::SideNoAliasNoAliasDeclArg); + if (!isa(DA) && !isa(DA)) { + // JumpThreading can duplicate llvm.noalias.decl. + // I do not think we should prohibit that. + SmallVector NoAliasDeclarations; + { + DenseSet Handled; + SmallVector Worklist = {DA}; + while (!Worklist.empty()) { + auto *V = Worklist.pop_back_val(); + if (Handled.insert(V).second) { + if (isa(V)) { + NoAliasDeclarations.push_back(cast(V)); + } else if (PHINode *PHI = cast(V)) { + auto OV = PHI->operand_values(); + Worklist.insert(Worklist.end(), OV.begin(), OV.end()); + } else if (isa(V)) { + // ignore undefs + } else if (isa(V)) { + // ignore nullptr + } else { + Assert(false, + "llvm.side.noalias depends on something that is not a " + "llvm.noalias.decl", + Call); + } + } + } + } + + Assert(!NoAliasDeclarations.empty(), + "llvm.side.noalias should depend on a llvm.noalias.decl", Call); + for (auto *CB : NoAliasDeclarations) { + Assert(CB, "llvm.side.noalias arg1 must refer to a llvm.noalias.decl", + Call); + Assert(CB->getIntrinsicID() == Intrinsic::noalias_decl, + "llvm.side.noalias arg1 must refer to a llvm.noalias.decl", + Call); + Assert((CB->getArgOperand(Intrinsic::NoAliasDeclObjIdArg) == + Call.getArgOperand(Intrinsic::SideNoAliasIdentifyPObjIdArg)), + "llvm.side.noalias objId arg must match with llvm.noalias.decl", + Call); + Assert((CB->getArgOperand(Intrinsic::NoAliasDeclScopeArg) == + Call.getArgOperand(Intrinsic::SideNoAliasScopeArg)), + "llvm.side.noalias scope arg must match with llvm.noalias.decl", + Call); + } + } + break; + } }; } Index: llvm/unittests/IR/IRBuilderTest.cpp =================================================================== --- llvm/unittests/IR/IRBuilderTest.cpp +++ llvm/unittests/IR/IRBuilderTest.cpp @@ -12,6 +12,7 @@ #include "llvm/IR/DataLayout.h" #include "llvm/IR/Function.h" #include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/MDBuilder.h" #include "llvm/IR/Module.h" @@ -288,17 +289,51 @@ IRBuilder<> Builder(BB); AllocaInst *Var1 = Builder.CreateAlloca(Builder.getInt8Ty()); + AllocaInst *VarP = Builder.CreateAlloca(Builder.getInt8PtrTy()); + Builder.CreateStore(Var1, VarP); + MDBuilder MDB(BB->getContext()); MDNode *NewDomain = MDB.createAnonymousAliasScopeDomain("Test Domain"); MDNode *AScope = MDB.createAnonymousAliasScope(NewDomain, "Test Scope"); - CallInst *NAP = cast(Builder.CreateNoAliasPointer(Var1, AScope)); - - EXPECT_EQ(NAP->getArgOperand(0), Var1); - EXPECT_EQ(cast(NAP->getArgOperand(1))->getMetadata(), - AScope); + CallInst *NAD = + cast(Builder.CreateNoAliasDeclaration(VarP, AScope)); + Value *Pointer = Builder.CreateLoad(VarP); + CallInst *NAP = + cast(Builder.CreateNoAliasPointer(Pointer, NAD, VarP, AScope)); + + // llvm.noalias.decl + EXPECT_EQ(NAD->getArgOperand(Intrinsic::NoAliasDeclAllocaArg), VarP); + EXPECT_EQ( + cast(NAD->getArgOperand(Intrinsic::NoAliasDeclObjIdArg)) + ->getZExtValue(), + 0ull); + EXPECT_EQ( + cast(NAD->getArgOperand(Intrinsic::NoAliasDeclScopeArg)) + ->getMetadata(), + AScope); + + IntrinsicInst *II_NAD = dyn_cast(NAD); + ASSERT_TRUE(II_NAD != nullptr); + EXPECT_EQ(II_NAD->getIntrinsicID(), Intrinsic::noalias_decl); + + EXPECT_TRUE(NAD->onlyAccessesArgMemory()); + EXPECT_TRUE(NAD->doesNotThrow()); + + // llvm.noalias + EXPECT_EQ(NAP->getArgOperand(0), Pointer); + EXPECT_EQ(NAP->getArgOperand(Intrinsic::NoAliasNoAliasDeclArg), NAD); + EXPECT_EQ(NAP->getArgOperand(Intrinsic::NoAliasIdentifyPArg), VarP); + EXPECT_EQ( + cast(NAP->getArgOperand(Intrinsic::NoAliasScopeArg)) + ->getMetadata(), + AScope); + EXPECT_EQ( + cast(NAP->getArgOperand(Intrinsic::NoAliasIdentifyPObjIdArg)) + ->getZExtValue(), + 0ull); // default object id is 0 IntrinsicInst *II_NAP = dyn_cast(NAP); ASSERT_TRUE(II_NAP != nullptr); @@ -308,6 +343,119 @@ EXPECT_TRUE(NAP->doesNotThrow()); } +TEST_F(IRBuilderTest, SideNoAliasAndArgGuard) { + IRBuilder<> Builder(BB); + AllocaInst *Var1 = Builder.CreateAlloca(Builder.getInt8Ty()); + + AllocaInst *VarP = Builder.CreateAlloca(Builder.getInt8PtrTy()); + Builder.CreateStore(Var1, VarP); + + MDBuilder MDB(BB->getContext()); + MDNode *NewDomain = MDB.createAnonymousAliasScopeDomain("Test Domain"); + MDNode *AScope = MDB.createAnonymousAliasScope(NewDomain, "Test Scope"); + + CallInst *NAD = + cast(Builder.CreateNoAliasDeclaration(VarP, AScope)); + Value *Pointer = Builder.CreateLoad(VarP); + CallInst *SNAP = cast(Builder.CreateSideNoAliasPlain( + Pointer, NAD, VarP, llvm::ConstantPointerNull::get(VarP->getType()), + NAD->getArgOperand(Intrinsic::NoAliasDeclObjIdArg), AScope)); + + CallInst *NAG = cast(Builder.CreateNoAliasArgGuard(Pointer, SNAP)); + + // create load on Pointer, with SNAP side channel + + // llvm.side.noalias + EXPECT_EQ(SNAP->getArgOperand(0), Pointer); + EXPECT_EQ(SNAP->getArgOperand(Intrinsic::SideNoAliasNoAliasDeclArg), NAD); + EXPECT_EQ(SNAP->getArgOperand(Intrinsic::SideNoAliasIdentifyPArg), VarP); + EXPECT_EQ( + cast(SNAP->getArgOperand(Intrinsic::SideNoAliasScopeArg)) + ->getMetadata(), + AScope); + ASSERT_TRUE(cast( + SNAP->getArgOperand(Intrinsic::SideNoAliasIdentifyPSideChannelArg))); + EXPECT_EQ(cast( + SNAP->getArgOperand(Intrinsic::SideNoAliasIdentifyPObjIdArg)) + ->getZExtValue(), + 0ull); // default object id is 0 + + IntrinsicInst *II_SNAP = dyn_cast(SNAP); + ASSERT_TRUE(II_SNAP != nullptr); + EXPECT_EQ(II_SNAP->getIntrinsicID(), Intrinsic::side_noalias); + + EXPECT_TRUE(SNAP->doesNotAccessMemory()); + EXPECT_TRUE(SNAP->doesNotThrow()); + + // llvm.noalias.arg.guard + EXPECT_EQ(NAG->getArgOperand(0), Pointer); + EXPECT_EQ(NAG->getArgOperand(1), SNAP); + + EXPECT_TRUE(NAG->doesNotAccessMemory()); + EXPECT_TRUE(NAG->doesNotThrow()); +} + +TEST_F(IRBuilderTest, NoAliasCopyGuard) { + // // roughly equivalent code for: + // struct FOO { char* __reststrict p; } + // char copy_restrict(struct FOO* p) { + // struct FOO local=*p; // Introduces llvm.noalias.copy.guard + // return *local.p; // NOTE: this part is omitted + // } + + IRBuilder<> Builder(BB); + StructType *ST = + StructType::create(BB->getContext(), {Builder.getInt8PtrTy()}); + + // pointer to incoming struct + AllocaInst *VarIn = Builder.CreateAlloca(ST->getPointerTo()); + + // local struct + AllocaInst *Var1 = Builder.CreateAlloca(ST); + // pointer to local struct + AllocaInst *VarP = Builder.CreateAlloca(ST->getPointerTo()); + Builder.CreateStore(Var1, VarP); + + MDBuilder MDB(BB->getContext()); + MDNode *NewDomain = MDB.createAnonymousAliasScopeDomain("Test Domain"); + MDNode *AScope = MDB.createAnonymousAliasScope(NewDomain, "Test Scope"); + + CallInst *NAD = + cast(Builder.CreateNoAliasDeclaration(VarP, AScope)); + (void)NAD; // not used as we don't do the 'return *local.p' here + + auto *SrcPointer = Builder.CreateLoad(VarIn); + // one entry of length 2, containing {-1, 0} : + const int64_t Indices[] = {2, -1, 0}; + auto Int8PtrNull = ConstantPointerNull::get(Builder.getInt8PtrTy()); + CallInst *GP = cast(Builder.CreateNoAliasCopyGuard( + SrcPointer, + Int8PtrNull, // out-of-function __restrict declaration + Indices, // For struct FOO* f, all f[X].p (X ~ -1, .p ~ 0) are restrict + AScope)); + auto *Copy = Builder.CreateMemCpy( + Var1, 4, GP, 4, + BB->getParent()->getParent()->getDataLayout().getTypeStoreSize(ST)); + (void)Copy; + + // llvm.side.noalias + EXPECT_EQ(GP->getArgOperand(Intrinsic::NoAliasCopyGuardIdentifyPBaseObject), + SrcPointer); + EXPECT_EQ(GP->getArgOperand(Intrinsic::NoAliasCopyGuardNoAliasDeclArg), + Int8PtrNull); + + // hmm. how to easily check that we are pointing to a { { -1, 0 } } array ? + EXPECT_TRUE(cast(GP->getArgOperand( + Intrinsic::NoAliasCopyGuardIndicesArg)) != nullptr); + EXPECT_EQ(cast( + GP->getArgOperand(Intrinsic::NoAliasCopyGuardScopeArg)) + ->getMetadata(), + AScope); + + EXPECT_TRUE(GP->doesNotAccessMemory()); + EXPECT_TRUE(GP->doesNotThrow()); +} + TEST_F(IRBuilderTest, SideChannel) { IRBuilder<> Builder(BB);