diff --git a/llvm/lib/Analysis/TypeMetadataUtils.cpp b/llvm/lib/Analysis/TypeMetadataUtils.cpp --- a/llvm/lib/Analysis/TypeMetadataUtils.cpp +++ b/llvm/lib/Analysis/TypeMetadataUtils.cpp @@ -67,6 +67,14 @@ findLoadCallsAtConstantOffset(M, DevirtCalls, User, Offset + GEPOffset, CI, DT); } + } else if (auto *Call = dyn_cast(User)) { + if (Call->getIntrinsicID() == llvm::Intrinsic::load_relative) { + if (auto *LoadOffset = dyn_cast(Call->getOperand(1))) { + findCallsAtConstantOffset(DevirtCalls, nullptr, User, + Offset + LoadOffset->getSExtValue(), CI, + DT); + } + } } } } diff --git a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp --- a/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp +++ b/llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp @@ -1004,10 +1004,13 @@ return false; Constant *Ptr = getPointerAtOffset(TM.Bits->GV->getInitializer(), - TM.Offset + ByteOffset, M); + TM.Offset + ByteOffset, M, TM.Bits->GV); if (!Ptr) return false; + if (auto *Equiv = dyn_cast(Ptr)) + Ptr = Equiv->getGlobalValue(); + auto Fn = dyn_cast(Ptr->stripPointerCasts()); if (!Fn) return false; diff --git a/llvm/test/Transforms/WholeProgramDevirt/constant-arg.ll b/llvm/test/Transforms/WholeProgramDevirt/constant-arg.ll --- a/llvm/test/Transforms/WholeProgramDevirt/constant-arg.ll +++ b/llvm/test/Transforms/WholeProgramDevirt/constant-arg.ll @@ -7,6 +7,10 @@ ; CHECK: private constant { [8 x i8], [1 x i8*], [0 x i8] } { [8 x i8] c"\00\00\00\00\00\00\00\02", [1 x i8*] [i8* bitcast (i1 (i8*, i32)* @vf2 to i8*)], [0 x i8] zeroinitializer }, !type [[T8]] ; CHECK: private constant { [8 x i8], [1 x i8*], [0 x i8] } { [8 x i8] c"\00\00\00\00\00\00\00\01", [1 x i8*] [i8* bitcast (i1 (i8*, i32)* @vf4 to i8*)], [0 x i8] zeroinitializer }, !type [[T8]] ; CHECK: private constant { [8 x i8], [1 x i8*], [0 x i8] } { [8 x i8] c"\00\00\00\00\00\00\00\02", [1 x i8*] [i8* bitcast (i1 (i8*, i32)* @vf8 to i8*)], [0 x i8] zeroinitializer }, !type [[T8]] +; CHECK: private constant { [4 x i8], [1 x i32], [0 x i8] } { [4 x i8] c"\00\00\00\01", [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (i1 (i8*, i32)* dso_local_equivalent @vf1 to i64), i64 ptrtoint ([1 x i32]* @vt1_rv to i64)) to i32)], [0 x i8] zeroinitializer }, align 4, !type [[T4:![0-9]+]] +; CHECK: private constant { [4 x i8], [1 x i32], [0 x i8] } { [4 x i8] c"\00\00\00\02", [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (i1 (i8*, i32)* dso_local_equivalent @vf2 to i64), i64 ptrtoint ([1 x i32]* @vt2_rv to i64)) to i32)], [0 x i8] zeroinitializer }, align 4, !type [[T4]] +; CHECK: private constant { [4 x i8], [1 x i32], [0 x i8] } { [4 x i8] c"\00\00\00\01", [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (i1 (i8*, i32)* dso_local_equivalent @vf4 to i64), i64 ptrtoint ([1 x i32]* @vt4_rv to i64)) to i32)], [0 x i8] zeroinitializer }, align 4, !type [[T4]] +; CHECK: private constant { [4 x i8], [1 x i32], [0 x i8] } { [4 x i8] c"\00\00\00\02", [1 x i32] [i32 trunc (i64 sub (i64 ptrtoint (i1 (i8*, i32)* dso_local_equivalent @vf8 to i64), i64 ptrtoint ([1 x i32]* @vt8_rv to i64)) to i32)], [0 x i8] zeroinitializer }, align 4, !type [[T4]] @vt1 = constant [1 x i8*] [i8* bitcast (i1 (i8*, i32)* @vf1 to i8*)], !type !0 @vt2 = constant [1 x i8*] [i8* bitcast (i1 (i8*, i32)* @vf2 to i8*)], !type !0 @@ -69,8 +73,55 @@ ret i1 %result } +declare i8* @llvm.load.relative.i32(i8*, i32) + +@vt1_rv = private unnamed_addr constant [1 x i32] [ + i32 trunc (i64 sub (i64 ptrtoint (i1 (i8*, i32)* dso_local_equivalent @vf1 to i64), i64 ptrtoint (i32* getelementptr inbounds ([1 x i32], [1 x i32]* @vt1_rv, i32 0, i32 0) to i64)) to i32) +], align 4, !type !1 +@vt2_rv = private unnamed_addr constant [1 x i32] [ + i32 trunc (i64 sub (i64 ptrtoint (i1 (i8*, i32)* dso_local_equivalent @vf2 to i64), i64 ptrtoint (i32* getelementptr inbounds ([1 x i32], [1 x i32]* @vt2_rv, i32 0, i32 0) to i64)) to i32) +], align 4, !type !1 +@vt4_rv = private unnamed_addr constant [1 x i32] [ + i32 trunc (i64 sub (i64 ptrtoint (i1 (i8*, i32)* dso_local_equivalent @vf4 to i64), i64 ptrtoint (i32* getelementptr inbounds ([1 x i32], [1 x i32]* @vt4_rv, i32 0, i32 0) to i64)) to i32) +], align 4, !type !1 +@vt8_rv = private unnamed_addr constant [1 x i32] [ + i32 trunc (i64 sub (i64 ptrtoint (i1 (i8*, i32)* dso_local_equivalent @vf8 to i64), i64 ptrtoint (i32* getelementptr inbounds ([1 x i32], [1 x i32]* @vt8_rv, i32 0, i32 0) to i64)) to i32) +], align 4, !type !1 + +; CHECK: define i1 @call3 +define i1 @call3(i8* %obj) { + %vtableptr = bitcast i8* %obj to [1 x i32]** + %vtable = load [1 x i32]*, [1 x i32]** %vtableptr + %vtablei8 = bitcast [1 x i32]* %vtable to i8* + %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid2") + call void @llvm.assume(i1 %p) + %fptr = call i8* @llvm.load.relative.i32(i8* %vtablei8, i32 0) + %fptr_casted = bitcast i8* %fptr to i1 (i8*, i32)* + ; CHECK: getelementptr {{.*}} -1 + ; CHECK: and {{.*}}, 1 + %result = call i1 %fptr_casted(i8* %obj, i32 5) + ret i1 %result +} + +; CHECK: define i1 @call4 +define i1 @call4(i8* %obj) { + %vtableptr = bitcast i8* %obj to [1 x i32]** + %vtable = load [1 x i32]*, [1 x i32]** %vtableptr + %vtablei8 = bitcast [1 x i32]* %vtable to i8* + %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid2") + call void @llvm.assume(i1 %p) + %fptr = call i8* @llvm.load.relative.i32(i8* %vtablei8, i32 0) + %fptr_casted = bitcast i8* %fptr to i1 (i8*, i32)* + ; CHECK: getelementptr {{.*}} -1 + ; CHECK: and {{.*}}, 2 + %result = call i1 %fptr_casted(i8* %obj, i32 10) + ret i1 %result +} + declare i1 @llvm.type.test(i8*, metadata) declare void @llvm.assume(i1) ; CHECK: [[T8]] = !{i32 8, !"typeid"} +; CHECK: [[T4]] = !{i32 4, !"typeid2"} !0 = !{i32 0, !"typeid"} +!1 = !{i32 0, !"typeid2"} diff --git a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check.ll b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check.ll --- a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check.ll +++ b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-check.ll @@ -3,6 +3,7 @@ target datalayout = "e-p:64:64" target triple = "x86_64-unknown-linux-gnu" +; CHECK: remark: :0:0: single-impl: devirtualized a call to vf ; CHECK: remark: :0:0: single-impl: devirtualized a call to vf ; CHECK: remark: :0:0: devirtualized vf ; CHECK-NOT: devirtualized @@ -36,7 +37,34 @@ unreachable } +@vt3 = private unnamed_addr constant [1 x i32] [ + i32 trunc (i64 sub (i64 ptrtoint (void (i8*)* dso_local_equivalent @vf to i64), i64 ptrtoint (i32* getelementptr inbounds ([1 x i32], [1 x i32]* @vt3, i32 0, i32 0) to i64)) to i32) +], align 4, !type !1 + +; CHECK: define void @call2 +define void @call2(i8* %obj) { + %vtableptr = bitcast i8* %obj to [1 x i32]** + %vtable = load [1 x i32]*, [1 x i32]** %vtableptr + %vtablei8 = bitcast [1 x i32]* %vtable to i8* + %pair = call {i8*, i1} @llvm.type.checked.load(i8* %vtablei8, i32 0, metadata !"typeid2") + %fptr = extractvalue {i8*, i1} %pair, 0 + %p = extractvalue {i8*, i1} %pair, 1 + ; CHECK: br i1 true, + br i1 %p, label %cont, label %trap + +cont: + %fptr_casted = bitcast i8* %fptr to void (i8*)* + ; CHECK: call void @vf( + call void %fptr_casted(i8* %obj) + ret void + +trap: + call void @llvm.trap() + unreachable +} + declare {i8*, i1} @llvm.type.checked.load(i8*, i32, metadata) declare void @llvm.trap() !0 = !{i32 0, !"typeid"} +!1 = !{i32 0, !"typeid2"} diff --git a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-multiple-assumes.ll b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-multiple-assumes.ll --- a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-multiple-assumes.ll +++ b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl-multiple-assumes.ll @@ -27,7 +27,30 @@ ret void } +declare i8* @llvm.load.relative.i32(i8*, i32) + +@vt3 = private unnamed_addr constant [1 x i32] [ + i32 trunc (i64 sub (i64 ptrtoint (void (i8*)* dso_local_equivalent @vf to i64), i64 ptrtoint (i32* getelementptr inbounds ([1 x i32], [1 x i32]* @vt3, i32 0, i32 0) to i64)) to i32) +], align 4, !type !1 + +; CHECK: define void @call2 +define void @call2(i8* %obj) { + %vtableptr = bitcast i8* %obj to [1 x i32]** + %vtable = load [1 x i32]*, [1 x i32]** %vtableptr + %vtablei8 = bitcast [1 x i32]* %vtable to i8* + %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid2") + call void @llvm.assume(i1 %p) + %p2 = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid2") + call void @llvm.assume(i1 %p2) + %fptr = call i8* @llvm.load.relative.i32(i8* %vtablei8, i32 0) + %fptr_casted = bitcast i8* %fptr to void (i8*)* + ; CHECK: call void @vf( + call void %fptr_casted(i8* %obj) + ret void +} + declare i1 @llvm.type.test(i8*, metadata) declare void @llvm.assume(i1) !0 = !{i32 0, !"typeid"} +!1 = !{i32 0, !"typeid2"} diff --git a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl.ll b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl.ll --- a/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl.ll +++ b/llvm/test/Transforms/WholeProgramDevirt/devirt-single-impl.ll @@ -7,6 +7,8 @@ target triple = "x86_64-unknown-linux-gnu" ; CHECK: remark: devirt-single.cc:30:32: single-impl: devirtualized a call to vf +; CHECK: remark: devirt-single.cc:41:32: single-impl: devirtualized a call to vf +; CHECK: remark: devirt-single.cc:51:32: single-impl: devirtualized a call to vf ; CHECK: remark: devirt-single.cc:13:0: devirtualized vf ; CHECK-NOT: devirtualized @@ -32,6 +34,47 @@ ret void } +declare i8* @llvm.load.relative.i32(i8*, i32) + +@vt3 = private unnamed_addr constant [1 x i32] [ + i32 trunc (i64 sub (i64 ptrtoint (void (i8*)* dso_local_equivalent @vf to i64), i64 ptrtoint (i32* getelementptr inbounds ([1 x i32], [1 x i32]* @vt3, i32 0, i32 0) to i64)) to i32) +], align 4, !type !11 + +; CHECK: define void @call2 +define void @call2(i8* %obj) #1 !dbg !9 { + %vtableptr = bitcast i8* %obj to [1 x i32]** + %vtable = load [1 x i32]*, [1 x i32]** %vtableptr + %vtablei8 = bitcast [1 x i32]* %vtable to i8* + %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid2") + call void @llvm.assume(i1 %p) + %fptr = call i8* @llvm.load.relative.i32(i8* %vtablei8, i32 0) + %fptr_casted = bitcast i8* %fptr to void (i8*)* + ; CHECK: call void @vf( + call void %fptr_casted(i8* %obj), !dbg !10 + ret void +} + +@_ZTV1A.local = private unnamed_addr constant { [3 x i32] } { [3 x i32] [ + i32 0, ; offset to top + i32 0, ; rtti + i32 trunc (i64 sub (i64 ptrtoint (void (i8*)* dso_local_equivalent @vf to i64), i64 ptrtoint (i32* getelementptr inbounds ({ [3 x i32] }, { [3 x i32] }* @_ZTV1A.local, i32 0, i32 0, i32 2) to i64)) to i32) ; vfunc offset +] }, align 4, !type !14 + +; CHECK: define void @call3 +define void @call3(i8* %obj) #1 !dbg !12 { + %vtableptr = bitcast i8* %obj to [1 x i32]** + %vtable = load [1 x i32]*, [1 x i32]** %vtableptr + %vtablei8 = bitcast [1 x i32]* %vtable to i8* + %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid3") + call void @llvm.assume(i1 %p) + %fptr = call i8* @llvm.load.relative.i32(i8* %vtablei8, i32 8) + %fptr_casted = bitcast i8* %fptr to void (i8*)* + ; CHECK: call void @vf( + call void %fptr_casted(i8* %obj), !dbg !13 + ret void +} + + declare i1 @llvm.type.test(i8*, metadata) declare void @llvm.assume(i1) @@ -49,5 +92,13 @@ !7 = distinct !DISubprogram(name: "vf", linkageName: "_ZN3vt12vfEv", scope: !1, file: !1, line: 13, isLocal: false, isDefinition: true, scopeLine: 13, flags: DIFlagPrototyped, isOptimized: false, unit: !0) !8 = !{i32 0, !"typeid"} +!9 = distinct !DISubprogram(name: "call2", linkageName: "_Z5call2Pv", scope: !1, file: !1, line: 40, isLocal: false, isDefinition: true, scopeLine: 9, flags: DIFlagPrototyped, isOptimized: false, unit: !0) +!10 = !DILocation(line: 41, column: 32, scope: !9) +!11 = !{i32 0, !"typeid2"} + +!12 = distinct !DISubprogram(name: "call3", linkageName: "_Z5call3Pv", scope: !1, file: !1, line: 50, isLocal: false, isDefinition: true, scopeLine: 9, flags: DIFlagPrototyped, isOptimized: false, unit: !0) +!13 = !DILocation(line: 51, column: 32, scope: !12) +!14 = !{i32 0, !"typeid3"} + ; CHECK: 1 wholeprogramdevirt - Number of whole program devirtualization targets -; CHECK: 1 wholeprogramdevirt - Number of single implementation devirtualizations +; CHECK: 3 wholeprogramdevirt - Number of single implementation devirtualizations diff --git a/llvm/test/Transforms/WholeProgramDevirt/pointer-vtable.ll b/llvm/test/Transforms/WholeProgramDevirt/pointer-vtable.ll --- a/llvm/test/Transforms/WholeProgramDevirt/pointer-vtable.ll +++ b/llvm/test/Transforms/WholeProgramDevirt/pointer-vtable.ll @@ -24,7 +24,26 @@ ret void } +@vt2 = constant i32 trunc (i64 sub (i64 ptrtoint (void (i8*)* dso_local_equivalent @vf to i64), i64 ptrtoint (i32* getelementptr inbounds (i32, i32* @vt2, i32 0) to i64)) to i32), !type !1 + +declare i8* @llvm.load.relative.i32(i8*, i32) + +; CHECK: define void @call2 +define void @call2(i8* %obj) { + %vtableptr = bitcast i8* %obj to [1 x i8*]** + %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr + %vtablei8 = bitcast [1 x i8*]* %vtable to i8* + %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid2") + call void @llvm.assume(i1 %p) + %fptr = call i8* @llvm.load.relative.i32(i8* %vtablei8, i32 0) + %fptr_casted = bitcast i8* %fptr to void (i8*)* + ; CHECK: call void @vf( + call void %fptr_casted(i8* %obj) + ret void +} + declare i1 @llvm.type.test(i8*, metadata) declare void @llvm.assume(i1) !0 = !{i32 0, !"typeid"} +!1 = !{i32 0, !"typeid2"}