diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -26162,6 +26162,28 @@ If the function's return value's second element is false, the value of the first element is undefined. +.. _type.checked.load.relative: + +'``llvm.type.checked.load.relative``' Intrinsic +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Syntax: +""""""" + +:: + + declare {ptr, i1} @llvm.type.checked.load.relative(ptr %ptr, i32 %offset, metadata %type) argmemonly nounwind readonly + +Overview: +""""""""" + +The ``llvm.type.checked.load.relative`` intrinsic loads a relative pointer to a +function from a virtual table pointer using metadata. Otherwise, its semantic is +identical to the ``llvm.type.checked.load`` intrinsic. + +A relative pointer is a pointer to an offset to the pointed to value. The +address of the underlying pointer of the relative pointer is obtained by adding +the offset to the address of the offset value. '``llvm.arithmetic.fence``' Intrinsic ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td --- a/llvm/include/llvm/IR/Intrinsics.td +++ b/llvm/include/llvm/IR/Intrinsics.td @@ -2234,6 +2234,11 @@ [llvm_ptr_ty, llvm_i32_ty, llvm_metadata_ty], [IntrNoMem, IntrWillReturn]>; +// Safely loads a relative function pointer from a virtual table pointer using type metadata. +def int_type_checked_load_relative : DefaultAttrsIntrinsic<[llvm_ptr_ty, llvm_i1_ty], + [llvm_ptr_ty, llvm_i32_ty, llvm_metadata_ty], + [IntrNoMem, IntrWillReturn]>; + // Test whether a pointer is associated with a type metadata identifier. Used // for public visibility classes that may later be refined to private // visibility. diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp --- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp +++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp @@ -198,6 +198,7 @@ break; } + case Intrinsic::type_checked_load_relative: case Intrinsic::type_checked_load: { auto *TypeMDVal = cast(CI->getArgOperand(2)); auto *TypeId = dyn_cast(TypeMDVal->getMetadata()); diff --git a/llvm/lib/Analysis/TypeMetadataUtils.cpp b/llvm/lib/Analysis/TypeMetadataUtils.cpp --- a/llvm/lib/Analysis/TypeMetadataUtils.cpp +++ b/llvm/lib/Analysis/TypeMetadataUtils.cpp @@ -99,7 +99,9 @@ SmallVectorImpl &Preds, bool &HasNonCallUses, const CallInst *CI, DominatorTree &DT) { assert(CI->getCalledFunction()->getIntrinsicID() == - Intrinsic::type_checked_load); + Intrinsic::type_checked_load || + CI->getCalledFunction()->getIntrinsicID() == + Intrinsic::type_checked_load_relative); auto *Offset = dyn_cast(CI->getArgOperand(1)); if (!Offset) { diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -1087,11 +1087,16 @@ Intrinsic::getName(Intrinsic::type_test)); Function *TypeCheckedLoadFunc = RegularLTO.CombinedModule->getFunction( Intrinsic::getName(Intrinsic::type_checked_load)); + Function *TypeCheckedLoadRelativeFunc = + RegularLTO.CombinedModule->getFunction( + Intrinsic::getName(Intrinsic::type_checked_load_relative)); // First check if there are type tests / type checked loads in the // merged regular LTO module IR. if ((TypeTestFunc && !TypeTestFunc->use_empty()) || - (TypeCheckedLoadFunc && !TypeCheckedLoadFunc->use_empty())) + (TypeCheckedLoadFunc && !TypeCheckedLoadFunc->use_empty()) || + (TypeCheckedLoadRelativeFunc && + !TypeCheckedLoadRelativeFunc->use_empty())) return make_error( "inconsistent LTO Unit splitting (recompile with -fsplit-lto-unit)", inconvertibleErrorCode()); diff --git a/llvm/lib/Transforms/IPO/GlobalDCE.cpp b/llvm/lib/Transforms/IPO/GlobalDCE.cpp --- a/llvm/lib/Transforms/IPO/GlobalDCE.cpp +++ b/llvm/lib/Transforms/IPO/GlobalDCE.cpp @@ -187,29 +187,36 @@ LLVM_DEBUG(dbgs() << "Scanning type.checked.load intrinsics\n"); Function *TypeCheckedLoadFunc = M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load)); - - if (!TypeCheckedLoadFunc) - return; - - for (auto *U : TypeCheckedLoadFunc->users()) { - auto CI = dyn_cast(U); - if (!CI) - continue; - - auto *Offset = dyn_cast(CI->getArgOperand(1)); - Value *TypeIdValue = CI->getArgOperand(2); - auto *TypeId = cast(TypeIdValue)->getMetadata(); - - if (Offset) { - ScanVTableLoad(CI->getFunction(), TypeId, Offset->getZExtValue()); - } else { - // type.checked.load with a non-constant offset, so assume every entry in - // every matching vtable is used. - for (const auto &VTableInfo : TypeIdMap[TypeId]) { - VFESafeVTables.erase(VTableInfo.first); + Function *TypeCheckedLoadRelativeFunc = + M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load_relative)); + + auto scan = [&](Function *CheckedLoadFunc) { + if (!CheckedLoadFunc) + return; + + for (auto *U : CheckedLoadFunc->users()) { + auto CI = dyn_cast(U); + if (!CI) + continue; + + auto *Offset = dyn_cast(CI->getArgOperand(1)); + Value *TypeIdValue = CI->getArgOperand(2); + auto *TypeId = cast(TypeIdValue)->getMetadata(); + + if (Offset) { + ScanVTableLoad(CI->getFunction(), TypeId, Offset->getZExtValue()); + } else { + // type.checked.load with a non-constant offset, so assume every entry + // in every matching vtable is used. + for (const auto &VTableInfo : TypeIdMap[TypeId]) { + VFESafeVTables.erase(VTableInfo.first); + } } } - } + }; + + scan(TypeCheckedLoadFunc); + scan(TypeCheckedLoadRelativeFunc); } void GlobalDCEPass::AddVirtualFunctionDependencies(Module &M) { diff --git a/llvm/lib/Transforms/IPO/GlobalSplit.cpp b/llvm/lib/Transforms/IPO/GlobalSplit.cpp --- a/llvm/lib/Transforms/IPO/GlobalSplit.cpp +++ b/llvm/lib/Transforms/IPO/GlobalSplit.cpp @@ -147,8 +147,12 @@ M.getFunction(Intrinsic::getName(Intrinsic::type_test)); Function *TypeCheckedLoadFunc = M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load)); + Function *TypeCheckedLoadRelativeFunc = + M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load_relative)); if ((!TypeTestFunc || TypeTestFunc->use_empty()) && - (!TypeCheckedLoadFunc || TypeCheckedLoadFunc->use_empty())) + (!TypeCheckedLoadFunc || TypeCheckedLoadFunc->use_empty()) && + (!TypeCheckedLoadRelativeFunc || + TypeCheckedLoadRelativeFunc->use_empty())) return false; bool Changed = false; diff --git a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp --- a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp +++ b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp @@ -146,6 +146,14 @@ } } + if (Function *TypeCheckedLoadRelativeFunc = M.getFunction( + Intrinsic::getName(Intrinsic::type_checked_load_relative))) { + for (const Use &U : TypeCheckedLoadRelativeFunc->uses()) { + auto CI = cast(U.getUser()); + ExternalizeTypeId(CI, 2); + } + } + for (GlobalObject &GO : M.global_objects()) { SmallVector MDs; GO.getMetadata(LLVMContext::MD_type, MDs); diff --git a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp --- a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp +++ b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp @@ -1006,7 +1006,7 @@ return false; Constant *Ptr = getPointerAtOffset(TM.Bits->GV->getInitializer(), - TM.Offset + ByteOffset, M); + TM.Offset + ByteOffset, M, TM.Bits->GV); if (!Ptr) return false; @@ -2002,9 +2002,23 @@ // This helps avoid unnecessary spills. IRBuilder<> LoadB( (LoadedPtrs.size() == 1 && !HasNonCallUses) ? LoadedPtrs[0] : CI); - Value *GEP = LoadB.CreateGEP(Int8Ty, Ptr, Offset); - Value *GEPPtr = LoadB.CreateBitCast(GEP, PointerType::getUnqual(Int8PtrTy)); - Value *LoadedValue = LoadB.CreateLoad(Int8PtrTy, GEPPtr); + + Value *LoadedValue = nullptr; + if (TypeCheckedLoadFunc->getIntrinsicID() == + Intrinsic::type_checked_load_relative) { + Value *GEP = LoadB.CreateGEP(Int8Ty, Ptr, Offset); + Value *GEPPtr = LoadB.CreateBitCast(GEP, PointerType::getUnqual(Int32Ty)); + LoadedValue = LoadB.CreateLoad(Int32Ty, GEPPtr); + LoadedValue = LoadB.CreateSExt(LoadedValue, IntPtrTy); + GEP = LoadB.CreatePtrToInt(GEP, IntPtrTy); + LoadedValue = LoadB.CreateAdd(GEP, LoadedValue); + LoadedValue = LoadB.CreateIntToPtr(LoadedValue, Int8PtrTy); + } else { + Value *GEP = LoadB.CreateGEP(Int8Ty, Ptr, Offset); + Value *GEPPtr = + LoadB.CreateBitCast(GEP, PointerType::getUnqual(Int8PtrTy)); + LoadedValue = LoadB.CreateLoad(Int8PtrTy, GEPPtr); + } for (Instruction *LoadedPtr : LoadedPtrs) { LoadedPtr->replaceAllUsesWith(LoadedValue); @@ -2185,6 +2199,8 @@ M.getFunction(Intrinsic::getName(Intrinsic::type_test)); Function *TypeCheckedLoadFunc = M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load)); + Function *TypeCheckedLoadRelativeFunc = + M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load_relative)); Function *AssumeFunc = M.getFunction(Intrinsic::getName(Intrinsic::assume)); // Normally if there are no users of the devirtualization intrinsics in the @@ -2193,7 +2209,9 @@ if (!ExportSummary && (!TypeTestFunc || TypeTestFunc->use_empty() || !AssumeFunc || AssumeFunc->use_empty()) && - (!TypeCheckedLoadFunc || TypeCheckedLoadFunc->use_empty())) + (!TypeCheckedLoadFunc || TypeCheckedLoadFunc->use_empty()) && + (!TypeCheckedLoadRelativeFunc || + TypeCheckedLoadRelativeFunc->use_empty())) return false; // Rebuild type metadata into a map for easy lookup. @@ -2207,6 +2225,9 @@ if (TypeCheckedLoadFunc) scanTypeCheckedLoadUsers(TypeCheckedLoadFunc); + if (TypeCheckedLoadRelativeFunc) + scanTypeCheckedLoadUsers(TypeCheckedLoadRelativeFunc); + if (ImportSummary) { for (auto &S : CallSlots) importResolution(S.first, S.second); diff --git a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check-relative.ll b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check-relative.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check-relative.ll @@ -0,0 +1,47 @@ +; RUN: opt -S -passes=wholeprogramdevirt -whole-program-visibility -pass-remarks=wholeprogramdevirt %s 2>&1 | FileCheck %s + +target datalayout = "e-p:64:64" +target triple = "x86_64-unknown-linux-gnu" + +; CHECK: remark: :0:0: single-impl: devirtualized a call to vf +; CHECK: remark: :0:0: devirtualized vf +; CHECK-NOT: devirtualized + +; A vtable with "relative pointers", slots don't contain pointers to implementations, but instead have an i32 offset from the vtable itself to the implementation. +@vtable = internal unnamed_addr constant { [2 x i32] } { [2 x i32] [ + i32 trunc (i64 sub (i64 ptrtoint (ptr @vfunc1_live to i64), i64 ptrtoint (ptr @vtable to i64)) to i32), + i32 trunc (i64 sub (i64 ptrtoint (ptr @vfunc2_dead to i64), i64 ptrtoint (ptr @vtable to i64)) to i32) +]}, align 8, !type !0, !type !1 +;, !vcall_visibility !{i64 2} +!0 = !{i64 0, !"vfunc1.type"} +!1 = !{i64 4, !"vfunc2.type"} + +define internal void @vfunc1_live() { + ret void +} + +define internal void @vfunc2_dead() { + ret void +} + +; CHECK: define void @call +define void @call(ptr %obj) { + %vtable = load ptr, ptr %obj + %pair = call {ptr, i1} @llvm.type.checked.load.relative(ptr %vtable, i32 0, metadata !"vfunc1.type") + %fptr = extractvalue {ptr, i1} %pair, 0 + %p = extractvalue {ptr, i1} %pair, 1 + ; CHECK: br i1 true, + br i1 %p, label %cont, label %trap + +cont: + ; CHECK: call void @vfunc1_live( + call void %fptr() + ret void + +trap: + call void @llvm.trap() + unreachable +} + +declare {ptr, i1} @llvm.type.checked.load.relative(ptr, i32, metadata) +declare void @llvm.trap() diff --git a/llvm/test/Transforms/WholeProgramDevirt/expand-check-relative.ll b/llvm/test/Transforms/WholeProgramDevirt/expand-check-relative.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/WholeProgramDevirt/expand-check-relative.ll @@ -0,0 +1,58 @@ +; RUN: opt -S -passes=wholeprogramdevirt %s | FileCheck %s + +; Test that we correctly expand the llvm.type.checked.load.relative intrinsic in +; cases where we cannot devirtualize. + +target datalayout = "e-p:64:64" +target triple = "x86_64-unknown-linux-gnu" + +@vt1 = constant { [2 x i32] } { [2 x i32] [ + i32 trunc (i64 sub (i64 ptrtoint (ptr @vf1 to i64), i64 ptrtoint (ptr @vt1 to i64)) to i32), + i32 trunc (i64 sub (i64 ptrtoint (ptr @vf2 to i64), i64 ptrtoint (ptr @vt1 to i64)) to i32) +]}, align 8, !type !0, !type !1 + +!0 = !{i64 0, !"vfunc1.type"} +!1 = !{i64 4, !"vfunc2.type"} + + +define void @vf1(ptr %this) { + ret void +} + +define void @vf2(ptr %this) { + ret void +} + +; CHECK: define void @call +; CHECK: [[TT:%.*]] = call i1 @llvm.type.test(ptr [[VT:%.*]], metadata !"vfunc1.type") +; CHECK: br i1 [[TT]] + +; Relative pointer computation at the address of the i32 value to the i32 value +; to get to the pointer value. + +; CHECK: [[T0:%.*]] = getelementptr i8, ptr [[VT]], i32 0 +; CHECK: [[T1:%.*]] = load i32, ptr [[T0]] +; CHECK: [[T2:%.*]] = sext i32 [[T1]] to i64 +; CHECK: [[T3:%.*]] = ptrtoint ptr [[T0]] to i64 +; CHECK: [[T4:%.*]] = add i64 [[T3]], [[T2]] +; CHECK: [[F:%.*]] = inttoptr i64 [[T4]] to ptr +; CHECK: call void [[F]](ptr + +define void @call(ptr %obj) { + %vtable = load ptr, ptr %obj + %pair = call {ptr, i1} @llvm.type.checked.load.relative(ptr %vtable, i32 0, metadata !"vfunc1.type") + %p = extractvalue {ptr, i1} %pair, 1 + br i1 %p, label %cont, label %trap + +cont: + %fptr = extractvalue {ptr, i1} %pair, 0 + call void %fptr(ptr %obj) + ret void + +trap: + call void @llvm.trap() + unreachable +} + +declare {ptr, i1} @llvm.type.checked.load.relative(ptr, i32, metadata) +declare void @llvm.trap()