Index: llvm/include/llvm/IR/IRBuilder.h =================================================================== --- llvm/include/llvm/IR/IRBuilder.h +++ llvm/include/llvm/IR/IRBuilder.h @@ -780,6 +780,65 @@ MetadataAsValue::get(Context, ScopeTag)); } + /// Create a llvm.noalias.decl intrinsic call. + Instruction *CreateNoAliasDeclaration(Value *AllocaPtr, Value *ObjId, + Value *Scope); + Instruction *CreateNoAliasDeclaration(Value *AllocaPtr, uint64_t ObjId, + Value *Scope) { + return CreateNoAliasDeclaration( + AllocaPtr, + ConstantInt::get(IntegerType::getInt64Ty(getContext()), ObjId), Scope); + } + Instruction *CreateNoAliasDeclaration(Value *AllocaPtr, MDNode *ScopeTag) { + uint64_t Zero = 0; + return CreateNoAliasDeclaration(AllocaPtr, Zero, + MetadataAsValue::get(Context, ScopeTag)); + } + + /// Create a llvm.noalias intrinsic call. + Instruction *CreateNoAliasPointer(Value *Ptr, Value *NoAliasDecl, + Value *AddrP, MDNode *ScopeTag, + const Twine &Name = "", + uint64_t ObjectId = 0) { + return CreateNoAliasPointer(Ptr, NoAliasDecl, AddrP, + MetadataAsValue::get(getContext(), ScopeTag), + Name, ObjectId); + } + Instruction *CreateNoAliasPointer(Value *Ptr, Value *NoAliasDecl, + Value *AddrP, Value *ScopeTag, + const Twine &Name = "", + uint64_t ObjectId = 0); + + /// Create a llvm.provenance.noalias intrinsic call. + Instruction *CreateProvenanceNoAliasPlain(Value *Ptr, Value *NoAliasDecl, + Value *AddrP, + Value *AddrP_Provenance, + Value *ObjId, MDNode *ScopeTag, + const Twine &Name = ""); + Instruction *CreateProvenanceNoAliasPlain(Value *Ptr, Value *NoAliasDecl, + Value *AddrP, + Value *AddrP_Provenance, + Value *ObjId, Value *ScopeValue, + const Twine &Name = ""); + + /// Create a llvm.noalias.arg.guard intrinsic call. + Instruction *CreateNoAliasArgGuard(Value *Ptr, Value *Provenance, + const Twine &Name = ""); + + /// Create a llvm.noalias_copy_guard intrinsic call. + Instruction *CreateNoAliasCopyGuard(Value *BasePtr, Value *NoAliasDel, + ArrayRef EncodedIndices, + MDNode *ScopeTag, const Twine &Name = ""); + +private: + /// Helper for creating noalias intrinsics + Instruction *CreateGenericNoAliasIntrinsic(Intrinsic::ID ID, Value *arg0, + ArrayRef args_opt, + ArrayRef MDNodes, + ArrayRef MDValues, + const Twine &Name = ""); + +public: /// Create a call to the experimental.gc.statepoint intrinsic to /// start a new statepoint sequence. CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes, Index: llvm/include/llvm/IR/IntrinsicInst.h =================================================================== --- llvm/include/llvm/IR/IntrinsicInst.h +++ llvm/include/llvm/IR/IntrinsicInst.h @@ -1279,6 +1279,20 @@ } }; +/// Returns true when two llvm.provenance.noalias represent the same noalias +/// info. +inline bool areProvenanceNoAliasCompatible(const IntrinsicInst *lhs, + const IntrinsicInst *rhs) { + assert(lhs->getIntrinsicID() == Intrinsic::provenance_noalias && + rhs->getIntrinsicID() == Intrinsic::provenance_noalias && + "Can only check noalias compatibility of provenance.noalias"); + return (lhs->getOperand(Intrinsic::ProvenanceNoAliasScopeArg) == + rhs->getOperand(Intrinsic::ProvenanceNoAliasScopeArg)) && + (lhs->getOperand(Intrinsic::ProvenanceNoAliasIdentifyPObjIdArg) == + rhs->getOperand(Intrinsic::ProvenanceNoAliasIdentifyPObjIdArg)) && + (lhs->getOperand(Intrinsic::ProvenanceNoAliasIdentifyPArg) == + rhs->getOperand(Intrinsic::ProvenanceNoAliasIdentifyPArg)); +} } // end namespace llvm #endif // LLVM_IR_INTRINSICINST_H Index: llvm/include/llvm/IR/Intrinsics.h =================================================================== --- llvm/include/llvm/IR/Intrinsics.h +++ llvm/include/llvm/IR/Intrinsics.h @@ -37,6 +37,27 @@ // Abstraction for the arguments of the noalias intrinsics static const int NoAliasScopeDeclScopeArg = 0; + // Abstraction for the arguments of the noalias intrinsics + static const int ProvenanceNoAliasNoAliasDeclArg = 1; + static const int ProvenanceNoAliasIdentifyPArg = 2; + static const int ProvenanceNoAliasIdentifyPProvenanceArg = 3; + static const int ProvenanceNoAliasIdentifyPObjIdArg = 4; + static const int ProvenanceNoAliasScopeArg = 5; + + static const int NoAliasNoAliasDeclArg = 1; + static const int NoAliasIdentifyPArg = 2; + static const int NoAliasIdentifyPObjIdArg = 3; + static const int NoAliasScopeArg = 4; + + static const int NoAliasDeclAllocaArg = 0; + static const int NoAliasDeclObjIdArg = 1; + static const int NoAliasDeclScopeArg = 2; + + static const int NoAliasCopyGuardIdentifyPBaseObject = 0; + static const int NoAliasCopyGuardNoAliasDeclArg = 1; + static const int NoAliasCopyGuardIndicesArg = 2; + static const int NoAliasCopyGuardScopeArg = 3; + // Intrinsic ID type. This is an opaque typedef to facilitate splitting up // the enum into target-specific enums. typedef unsigned ID; Index: llvm/lib/IR/IRBuilder.cpp =================================================================== --- llvm/lib/IR/IRBuilder.cpp +++ llvm/lib/IR/IRBuilder.cpp @@ -478,6 +478,129 @@ return createCallHelper(FnIntrinsic, {Scope}, this); } +Instruction *IRBuilderBase::CreateNoAliasDeclaration(Value *AllocaPtr, + Value *ObjId, + Value *Scope) { + assert(AllocaPtr); + + SmallVector Types = {Type::getInt8PtrTy(getContext()), + AllocaPtr->getType(), ObjId->getType()}; + SmallVector Ops = {AllocaPtr, ObjId, Scope}; + + Module *M = BB->getModule(); + auto *FnIntrinsic = + Intrinsic::getDeclaration(M, Intrinsic::noalias_decl, Types); + return createCallHelper(FnIntrinsic, Ops, this); +} + +Instruction *IRBuilderBase::CreateNoAliasPointer(Value *Ptr, Value *NoAliasDecl, + Value *AddrP, Value *ScopeTag, + const Twine &Name, + uint64_t ObjectId) { + assert(Ptr && AddrP); + Value *I64_0 = + ConstantInt::get(IntegerType::getInt64Ty(getContext()), ObjectId); + return CreateGenericNoAliasIntrinsic(Intrinsic::noalias, Ptr, + {NoAliasDecl, AddrP, I64_0}, {}, + {ScopeTag}, Name); +} + +Instruction *IRBuilderBase::CreateProvenanceNoAliasPlain( + Value *Ptr, Value *NoAliasDecl, Value *AddrP, Value *AddrP_Provenance, + Value *ObjId, MDNode *ScopeTag, const Twine &Name) { + assert(Ptr && AddrP && AddrP_Provenance); + return CreateGenericNoAliasIntrinsic( + Intrinsic::provenance_noalias, Ptr, + {NoAliasDecl, AddrP, AddrP_Provenance, ObjId}, {ScopeTag}, {}, Name); +} + +Instruction *IRBuilderBase::CreateProvenanceNoAliasPlain( + Value *Ptr, Value *NoAliasDecl, Value *AddrP, Value *AddrP_Provenance, + Value *ObjId, Value *ScopeValue, const Twine &Name) { + assert(Ptr && AddrP && AddrP_Provenance); + return CreateGenericNoAliasIntrinsic( + Intrinsic::provenance_noalias, Ptr, + {NoAliasDecl, AddrP, AddrP_Provenance, ObjId}, {}, {ScopeValue}, Name); +} + +Instruction *IRBuilderBase::CreateNoAliasArgGuard(Value *Ptr, Value *Provenance, + const Twine &Name) { + return CreateGenericNoAliasIntrinsic(Intrinsic::noalias_arg_guard, Ptr, + {Provenance}, {}, {}, Name); +} + +Instruction * +IRBuilderBase::CreateNoAliasCopyGuard(Value *BasePtr, Value *NoAliasDecl, + ArrayRef EncodedIndices, + MDNode *ScopeTag, const Twine &Name) { + SmallVector IndicesArray; + + assert(NoAliasDecl != nullptr); + assert(ScopeTag != nullptr); + assert(!EncodedIndices.empty() && + "NoAliasCopyGuard should have at least one set of indices"); + + for (unsigned i = 0, ci = EncodedIndices.size(); i < ci;) { + auto length = EncodedIndices[i++]; + assert(length > 0 && + "NoAliasCopyGuard: encoded indices problem: zero length seen"); + length += i; + assert(length <= ci && + "NoAliasCopyGuard: encoded indices problem: too few indices"); + + SmallVector Indices; + while (i < length) { + // Note: getElementPtr requires i32 numbers + Indices.push_back( + llvm::ConstantAsMetadata::get(getInt32(EncodedIndices[i]))); + ++i; + } + IndicesArray.push_back(llvm::MDNode::get(Context, Indices)); + } + + return CreateGenericNoAliasIntrinsic( + Intrinsic::noalias_copy_guard, BasePtr, {NoAliasDecl}, + {llvm::MDNode::get(Context, IndicesArray), ScopeTag}, {}, Name); +} + +Instruction *IRBuilderBase::CreateGenericNoAliasIntrinsic( + Intrinsic::ID ID, Value *Ptr, ArrayRef args_opt, + ArrayRef MDNodes, ArrayRef MDValues, const Twine &Name) { + // FIXME: We can currently mangle just about everything, but not literal + // structs (for those, bitcast to i8*). + // FIXME: as far as I see in 'getMangledTypeStr', this should be ok now. + Value *CPtr = Ptr; + if (auto *STyp = + dyn_cast(CPtr->getType()->getPointerElementType())) { + if (STyp->isLiteral()) + CPtr = getCastedInt8PtrValue(CPtr); + } + + SmallVector Types = {CPtr->getType()}; + SmallVector Ops = {CPtr}; + for (auto *A : args_opt) { + Types.push_back(A->getType()); + Ops.push_back(A); + } + // For the metadata info, types must not be added: + for (auto *MD : MDNodes) { + Ops.push_back(MetadataAsValue::get(Context, MD)); + } + Ops.insert(Ops.end(), MDValues.begin(), MDValues.end()); + Module *M = BB->getModule(); + auto *FnIntrinsic = Intrinsic::getDeclaration(M, ID, Types); + Instruction *Ret = createCallHelper(FnIntrinsic, Ops, this, Name); + + if (Ret->getType() != Ptr->getType()) { + BitCastInst *BCI = new BitCastInst(Ret, Ptr->getType(), Name + ".cast"); + BB->getInstList().insert(InsertPt, BCI); + SetInstDebugLocation(BCI); + Ret = BCI; + } + + return Ret; +} + /// Create a call to a Masked Load intrinsic. /// \p Ptr - base pointer for the load /// \p Alignment - alignment of the source location Index: llvm/lib/IR/Verifier.cpp =================================================================== --- llvm/lib/IR/Verifier.cpp +++ llvm/lib/IR/Verifier.cpp @@ -5325,6 +5325,87 @@ NoAliasScopeDecls.push_back(cast(&Call)); break; } + case Intrinsic::noalias_decl: { + Assert( + (isa( + Call.getArgOperand(Intrinsic::NoAliasDeclAllocaArg)) || + isa(Call.getArgOperand(Intrinsic::NoAliasDeclAllocaArg))), + "llvm.noalias.decl must refer to a null-ptr or an alloca instruction", + Call); + break; + } + case Intrinsic::noalias: { + auto *DA = Call.getArgOperand(Intrinsic::NoAliasNoAliasDeclArg); + if (!isa(DA) && !isa(DA)) { + auto *DeclSite = dyn_cast(DA); + Assert(DeclSite, "llvm.noalias arg1 must refer to a llvm.noalias.decl", + Call); + Assert((DeclSite->getArgOperand(Intrinsic::NoAliasDeclObjIdArg) == + Call.getArgOperand(Intrinsic::NoAliasIdentifyPObjIdArg)), + "llvm.noalias objId arg must match with llvm.noalias.decl", Call); + Assert((DeclSite->getArgOperand(Intrinsic::NoAliasDeclScopeArg) == + Call.getArgOperand(Intrinsic::NoAliasScopeArg)), + "llvm.noalias scope arg must match with llvm.noalias.decl", Call); + } + break; + } + case Intrinsic::provenance_noalias: { + auto *DA = Call.getArgOperand(Intrinsic::ProvenanceNoAliasNoAliasDeclArg); + if (!isa(DA) && !isa(DA)) { + // JumpThreading can duplicate llvm.noalias.decl. + // I do not think we should prohibit that. + SmallVector NoAliasDeclarations; + { + DenseSet Handled; + SmallVector Worklist = {DA}; + while (!Worklist.empty()) { + auto *V = Worklist.pop_back_val(); + if (Handled.insert(V).second) { + if (isa(V)) { + NoAliasDeclarations.push_back(cast(V)); + } else if (PHINode *PHI = dyn_cast(V)) { + auto OV = PHI->operand_values(); + Worklist.insert(Worklist.end(), OV.begin(), OV.end()); + } else if (isa(V)) { + // ignore undefs + } else if (isa(V)) { + // ignore nullptr + } else { + Assert( + false, + "llvm.provenance.noalias depends on something that is not a " + "llvm.noalias.decl", + Call); + } + } + } + } + + Assert(!NoAliasDeclarations.empty(), + "llvm.provenance.noalias should depend on a llvm.noalias.decl", + Call); + for (auto *CB : NoAliasDeclarations) { + Assert(CB, + "llvm.provenance.noalias arg1 must refer to a llvm.noalias.decl", + Call); + Assert(CB->getIntrinsicID() == Intrinsic::noalias_decl, + "llvm.provenance.noalias arg1 must refer to a llvm.noalias.decl", + Call); + Assert( + (CB->getArgOperand(Intrinsic::NoAliasDeclObjIdArg) == + Call.getArgOperand(Intrinsic::ProvenanceNoAliasIdentifyPObjIdArg)), + "llvm.provenance.noalias objId arg must match with " + "llvm.noalias.decl", + Call); + Assert((CB->getArgOperand(Intrinsic::NoAliasDeclScopeArg) == + Call.getArgOperand(Intrinsic::ProvenanceNoAliasScopeArg)), + "llvm.provenance.noalias scope arg must match with " + "llvm.noalias.decl", + Call); + } + } + break; + } }; } Index: llvm/unittests/IR/IRBuilderTest.cpp =================================================================== --- llvm/unittests/IR/IRBuilderTest.cpp +++ llvm/unittests/IR/IRBuilderTest.cpp @@ -12,6 +12,7 @@ #include "llvm/IR/DataLayout.h" #include "llvm/IR/Function.h" #include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Intrinsics.h" #include "llvm/IR/IntrinsicsAArch64.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/MDBuilder.h" @@ -399,6 +400,178 @@ EXPECT_FALSE(verifyModule(*M)); } +TEST_F(IRBuilderTest, NoAlias) { + IRBuilder<> Builder(BB); + AllocaInst *Var1 = Builder.CreateAlloca(Builder.getInt8Ty()); + + AllocaInst *VarP = Builder.CreateAlloca(Builder.getInt8PtrTy()); + Builder.CreateStore(Var1, VarP); + + MDBuilder MDB(BB->getContext()); + MDNode *NewDomain = MDB.createAnonymousAliasScopeDomain("Test Domain"); + MDNode *AScope = MDB.createAnonymousAliasScope(NewDomain, "Test Scope"); + + CallInst *NAD = + cast(Builder.CreateNoAliasDeclaration(VarP, AScope)); + Value *Pointer = Builder.CreateLoad(Builder.getInt8PtrTy(), VarP); + CallInst *NAP = + cast(Builder.CreateNoAliasPointer(Pointer, NAD, VarP, AScope)); + + // llvm.noalias.decl + EXPECT_EQ(NAD->getArgOperand(Intrinsic::NoAliasDeclAllocaArg), VarP); + EXPECT_EQ( + cast(NAD->getArgOperand(Intrinsic::NoAliasDeclObjIdArg)) + ->getZExtValue(), + 0ull); + EXPECT_EQ( + cast(NAD->getArgOperand(Intrinsic::NoAliasDeclScopeArg)) + ->getMetadata(), + AScope); + + IntrinsicInst *II_NAD = dyn_cast(NAD); + ASSERT_TRUE(II_NAD != nullptr); + EXPECT_EQ(II_NAD->getIntrinsicID(), Intrinsic::noalias_decl); + + EXPECT_TRUE(NAD->onlyAccessesInaccessibleMemory()); + EXPECT_TRUE(NAD->doesNotThrow()); + + // llvm.noalias + EXPECT_EQ(NAP->getArgOperand(0), Pointer); + EXPECT_EQ(NAP->getArgOperand(Intrinsic::NoAliasNoAliasDeclArg), NAD); + EXPECT_EQ(NAP->getArgOperand(Intrinsic::NoAliasIdentifyPArg), VarP); + EXPECT_EQ( + cast(NAP->getArgOperand(Intrinsic::NoAliasScopeArg)) + ->getMetadata(), + AScope); + EXPECT_EQ( + cast(NAP->getArgOperand(Intrinsic::NoAliasIdentifyPObjIdArg)) + ->getZExtValue(), + 0ull); // default object id is 0 + + IntrinsicInst *II_NAP = dyn_cast(NAP); + ASSERT_TRUE(II_NAP != nullptr); + EXPECT_EQ(II_NAP->getIntrinsicID(), Intrinsic::noalias); + + EXPECT_TRUE(NAP->onlyAccessesArgMemory()); + EXPECT_TRUE(NAP->doesNotThrow()); +} + +TEST_F(IRBuilderTest, ProvenanceNoAliasAndArgGuard) { + IRBuilder<> Builder(BB); + AllocaInst *Var1 = Builder.CreateAlloca(Builder.getInt8Ty()); + + AllocaInst *VarP = Builder.CreateAlloca(Builder.getInt8PtrTy()); + Builder.CreateStore(Var1, VarP); + + MDBuilder MDB(BB->getContext()); + MDNode *NewDomain = MDB.createAnonymousAliasScopeDomain("Test Domain"); + MDNode *AScope = MDB.createAnonymousAliasScope(NewDomain, "Test Scope"); + + CallInst *NAD = + cast(Builder.CreateNoAliasDeclaration(VarP, AScope)); + Value *Pointer = Builder.CreateLoad(Builder.getInt8PtrTy(), VarP); + CallInst *SNAP = cast(Builder.CreateProvenanceNoAliasPlain( + Pointer, NAD, VarP, llvm::ConstantPointerNull::get(VarP->getType()), + NAD->getArgOperand(Intrinsic::NoAliasDeclObjIdArg), AScope)); + + CallInst *NAG = cast(Builder.CreateNoAliasArgGuard(Pointer, SNAP)); + + // create load on Pointer, with SNAP provenance + + // llvm.provenance.noalias + EXPECT_EQ(SNAP->getArgOperand(0), Pointer); + EXPECT_EQ(SNAP->getArgOperand(Intrinsic::ProvenanceNoAliasNoAliasDeclArg), + NAD); + EXPECT_EQ(SNAP->getArgOperand(Intrinsic::ProvenanceNoAliasIdentifyPArg), + VarP); + EXPECT_EQ(cast( + SNAP->getArgOperand(Intrinsic::ProvenanceNoAliasScopeArg)) + ->getMetadata(), + AScope); + ASSERT_TRUE(cast( + SNAP->getArgOperand(Intrinsic::ProvenanceNoAliasIdentifyPProvenanceArg))); + EXPECT_EQ( + cast( + SNAP->getArgOperand(Intrinsic::ProvenanceNoAliasIdentifyPObjIdArg)) + ->getZExtValue(), + 0ull); // default object id is 0 + + IntrinsicInst *II_SNAP = dyn_cast(SNAP); + ASSERT_TRUE(II_SNAP != nullptr); + EXPECT_EQ(II_SNAP->getIntrinsicID(), Intrinsic::provenance_noalias); + + EXPECT_TRUE(SNAP->doesNotAccessMemory()); + EXPECT_TRUE(SNAP->doesNotThrow()); + + // llvm.noalias.arg.guard + EXPECT_EQ(NAG->getArgOperand(0), Pointer); + EXPECT_EQ(NAG->getArgOperand(1), SNAP); + + EXPECT_TRUE(NAG->doesNotAccessMemory()); + EXPECT_TRUE(NAG->doesNotThrow()); +} + +TEST_F(IRBuilderTest, NoAliasCopyGuard) { + // // roughly equivalent code for: + // struct FOO { char* __reststrict p; } + // char copy_restrict(struct FOO* p) { + // struct FOO local=*p; // Introduces llvm.noalias.copy.guard + // return *local.p; // NOTE: this part is omitted + // } + + IRBuilder<> Builder(BB); + StructType *ST = + StructType::create(BB->getContext(), {Builder.getInt8PtrTy()}); + + // pointer to incoming struct + AllocaInst *VarIn = Builder.CreateAlloca(ST->getPointerTo()); + + // local struct + AllocaInst *Var1 = Builder.CreateAlloca(ST); + // pointer to local struct + AllocaInst *VarP = Builder.CreateAlloca(ST->getPointerTo()); + Builder.CreateStore(Var1, VarP); + + MDBuilder MDB(BB->getContext()); + MDNode *NewDomain = MDB.createAnonymousAliasScopeDomain("Test Domain"); + MDNode *AScope = MDB.createAnonymousAliasScope(NewDomain, "Test Scope"); + + CallInst *NAD = + cast(Builder.CreateNoAliasDeclaration(VarP, AScope)); + (void)NAD; // not used as we don't do the 'return *local.p' here + + auto *SrcPointer = Builder.CreateLoad(ST->getPointerTo(), VarIn); + // one entry of length 2, containing {-1, 0} : + const int64_t Indices[] = {2, -1, 0}; + auto Int8PtrNull = ConstantPointerNull::get(Builder.getInt8PtrTy()); + CallInst *GP = cast(Builder.CreateNoAliasCopyGuard( + SrcPointer, + Int8PtrNull, // out-of-function __restrict declaration + Indices, // For struct FOO* f, all f[X].p (X ~ -1, .p ~ 0) are restrict + AScope)); + auto *Copy = Builder.CreateMemCpy( + Var1, Align(4), GP, Align(4), + BB->getParent()->getParent()->getDataLayout().getTypeStoreSize(ST)); + (void)Copy; + + // llvm.provenance.noalias + EXPECT_EQ(GP->getArgOperand(Intrinsic::NoAliasCopyGuardIdentifyPBaseObject), + SrcPointer); + EXPECT_EQ(GP->getArgOperand(Intrinsic::NoAliasCopyGuardNoAliasDeclArg), + Int8PtrNull); + + // hmm. how to easily check that we are pointing to a { { -1, 0 } } array ? + EXPECT_TRUE(cast(GP->getArgOperand( + Intrinsic::NoAliasCopyGuardIndicesArg)) != nullptr); + EXPECT_EQ(cast( + GP->getArgOperand(Intrinsic::NoAliasCopyGuardScopeArg)) + ->getMetadata(), + AScope); + + EXPECT_TRUE(GP->doesNotAccessMemory()); + EXPECT_TRUE(GP->doesNotThrow()); +} + TEST_F(IRBuilderTest, Provenance) { IRBuilder<> Builder(BB);