diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp --- a/llvm/lib/Transforms/Utils/InlineFunction.cpp +++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp @@ -84,6 +84,10 @@ "update-return-attrs", cl::init(true), cl::Hidden, cl::desc("Update return attributes on calls within inlined body")); +static cl::opt UpdateLoadMetadataDuringInlining( + "update-load-metadata-during-inlining", cl::init(true), cl::Hidden, + cl::desc("Update metadata on loads within inlined body")); + static cl::opt PreserveAlignmentAssumptions("preserve-alignment-assumptions-during-inlining", cl::init(true), cl::Hidden, @@ -1146,8 +1150,9 @@ } } -static void AddReturnAttributes(CallSite CS, ValueToValueMapTy &VMap) { - if (!UpdateReturnAttributes) +static void AddReturnAttributesAndUpdateMetadata(CallSite CS, + ValueToValueMapTy &VMap) { + if (!UpdateReturnAttributes && !UpdateLoadMetadataDuringInlining) return; AttrBuilder AB(CS.getAttributes(), AttributeList::ReturnIndex); if (AB.empty()) @@ -1168,20 +1173,32 @@ return false; }; + auto getExpectedRV = [&](Value *V) -> Instruction * { + if (isa(V) && UpdateReturnAttributes) + return dyn_cast_or_null(VMap.lookup(V)); + if (isa(V) && UpdateLoadMetadataDuringInlining) + return dyn_cast_or_null(VMap.lookup(V)); + return nullptr; + }; + + MDBuilder MDB(Context); + auto CreateMDNode = [&](uint64_t Num) -> MDNode * { + auto *Int = ConstantInt::get(Type::getInt64Ty(Context), Num); + return MDNode::get(Context, MDB.createConstant(Int)); + }; for (auto &BB : *CalledFunction) { auto *RI = dyn_cast(BB.getTerminator()); - if (!RI || !isa(RI->getOperand(0))) + if (!RI) continue; // Sanity check that the cloned return instruction exists and is a return // instruction itself. auto *NewRI = dyn_cast_or_null(VMap.lookup(RI)); if (!NewRI) continue; - auto *RetVal = cast(RI->getOperand(0)); - // Sanity check that the cloned RetVal exists and is a call. + // Sanity check that the cloned RetVal exists and is a call or load. // Simplification during inlining could have transformed the cloned // instruction. - auto *NewRetVal = dyn_cast_or_null(VMap.lookup(RetVal)); + auto *NewRetVal = getExpectedRV(RI->getOperand(0)); if (!NewRetVal) continue; // Backward propagation of attributes to the returned value may be incorrect @@ -1206,11 +1223,26 @@ if (NewRI->getParent() != NewRetVal->getParent() || MayContainThrowingOrExitingCall(NewRetVal, NewRI)) continue; - // Add to the existing attributes. - AttributeList AL = NewRetVal->getAttributes(); - AttributeList NewAL = - AL.addAttributes(Context, AttributeList::ReturnIndex, AB); - NewRetVal->setAttributes(NewAL); + if (isa(NewRetVal)) { + // Add to the existing attributes. + AttributeList AL = cast(NewRetVal)->getAttributes(); + AttributeList NewAL = + AL.addAttributes(Context, AttributeList::ReturnIndex, AB); + cast(NewRetVal)->setAttributes(NewAL); + } else { + auto *NewLI = cast(NewRetVal); + // We do not need to check that NewLI is of PointerType, since the + // following return attributes on callsite only applies to pointer typed + // returns. + if (CS.isReturnNonNull()) + NewLI->setMetadata(LLVMContext::MD_nonnull, CreateMDNode(1)); + if (uint64_t DerefBytes = AB.getDereferenceableBytes()) + NewLI->setMetadata(LLVMContext::MD_dereferenceable, + CreateMDNode(DerefBytes)); + if (uint64_t DerefOrNullBytes = AB.getDereferenceableOrNullBytes()) + NewLI->setMetadata(LLVMContext::MD_dereferenceable_or_null, + CreateMDNode(DerefOrNullBytes)); + } } } @@ -1881,7 +1913,9 @@ // Clone return attributes on the callsite into the calls within the inlined // function which feed into its return value. - AddReturnAttributes(CS, VMap); + // Add metadata on loads that feed into the return value in the inlined + // body, based on return attributes on the callsite. + AddReturnAttributesAndUpdateMetadata(CS, VMap); // Propagate llvm.mem.parallel_loop_access if necessary. PropagateParallelLoopAccessMetadata(CS, VMap); diff --git a/llvm/test/Transforms/Inline/ret_load_metadata.ll b/llvm/test/Transforms/Inline/ret_load_metadata.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/Inline/ret_load_metadata.ll @@ -0,0 +1,90 @@ +; RUN: opt < %s -inline-threshold=0 -update-load-metadata-during-inlining=true -always-inline -S | FileCheck %s +; RUN: opt < %s -passes=always-inline -update-load-metadata-during-inlining=true -S | FileCheck %s + + +define i8* @callee(i8** %p) alwaysinline { +; CHECK: @callee( +; CHECK-NOT: nonnull + %r = load i8*, i8** %p, align 8 + ret i8* %r +} + +define i8* @test(i8** %ptr, i64 %x) { +; CHECK-LABEL: @test +; CHECK: load i8*, i8** %gep, align 8, !nonnull ![[NONNULL:[0-9]+]] + %gep = getelementptr inbounds i8*, i8** %ptr, i64 %x + %p = call nonnull i8* @callee(i8** %gep) + ret i8* %p +} + +declare void @does_not_return(i8*) nounwind +define internal i8* @callee_negative(i8** %p) alwaysinline { +; CHECK-NOT: @callee_negative( + %r = load i8*, i8** %p, align 8 + call void @does_not_return(i8* %r) + ret i8* %r +} + +define i8* @negative_test(i8** %ptr, i64 %x) { +; CHECK-LABEL: @negative_test +; CHECK: load i8*, i8** %gep, align 8 +; CHECK-NOT: nonnull + %gep = getelementptr inbounds i8*, i8** %ptr, i64 %x + %p = call nonnull i8* @callee_negative(i8** %gep) + ret i8* %p +} + + +define internal i8* @callee2(i8** %p) alwaysinline { +; CHECK-NOT: @callee2( + %r = load i8*, i8** %p, align 8 + ret i8* %r +} + +; dereferenceable attribute in default addrspace implies nonnull +define i8* @test2(i8** %ptr, i64 %x) { +; CHECK-LABEL: @test2 +; CHECK: load i8*, i8** %gep, align 8, !nonnull ![[NONNULL:[0-9]+]], !dereferenceable ![[DEREF:[0-9]+]] + %gep = getelementptr inbounds i8*, i8** %ptr, i64 %x + %p = call dereferenceable(12) i8* @callee(i8** %gep) + ret i8* %p +} + +declare void @bar(i8 addrspace(1)*) argmemonly nounwind + +define internal i8 addrspace(1)* @callee3(i8 addrspace(1)* addrspace(1)* %p) alwaysinline { +; CHECK-NOT: @callee3( + %r = load i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* %p, align 8 + call void @bar(i8 addrspace(1)* %r) + ret i8 addrspace(1)* %r +} + +define i8 addrspace(1)* @test3(i8 addrspace(1)* addrspace(1)* %ptr, i64 %x) { +; CHECK-LABEL: @test3 +; CHECK: load i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* %gep, align 8, !dereferenceable_or_null ![[DEREF_OR_NULL:[0-9]+]] + %gep = getelementptr inbounds i8 addrspace(1)*, i8 addrspace(1)* addrspace(1)* %ptr, i64 %x + %p = call dereferenceable_or_null(16) i8 addrspace(1)* @callee3(i8 addrspace(1)* addrspace(1)* %gep) + ret i8 addrspace(1)* %p +} + +; attribute is part of the callee itself +define nonnull i8* @callee4(i8** %p) alwaysinline { + %r = load i8*, i8** %p, align 8 + ret i8* %r +} + +; TODO : We should infer the attribute on the callee +; and add the nonnull on the load +define i8* @test4(i8** %ptr, i64 %x) { +; CHECK-LABEL: @test4 +; CHECK: load i8*, i8** %gep, align 8 +; CHECK-NOT: nonnull + %gep = getelementptr inbounds i8*, i8** %ptr, i64 %x + %p = call i8* @callee(i8** %gep) + ret i8* %p +} + + +; CHECK: ![[NONNULL]] = !{i64 1} +; CHECK: ![[DEREF]] = !{i64 12} +; CHECK: ![[DEREF_OR_NULL]] = !{i64 16}