Index: include/llvm/IR/ModuleSummaryIndexYAML.h =================================================================== --- include/llvm/IR/ModuleSummaryIndexYAML.h +++ include/llvm/IR/ModuleSummaryIndexYAML.h @@ -138,6 +138,7 @@ struct FunctionSummaryYaml { unsigned Linkage; bool NotEligibleToImport, Live, IsLocal; + std::vector Refs; std::vector TypeTests; std::vector TypeTestAssumeVCalls, TypeCheckedLoadVCalls; @@ -180,6 +181,7 @@ io.mapOptional("NotEligibleToImport", summary.NotEligibleToImport); io.mapOptional("Live", summary.Live); io.mapOptional("Local", summary.IsLocal); + io.mapOptional("Refs", summary.Refs); io.mapOptional("TypeTests", summary.TypeTests); io.mapOptional("TypeTestAssumeVCalls", summary.TypeTestAssumeVCalls); io.mapOptional("TypeCheckedLoadVCalls", summary.TypeCheckedLoadVCalls); @@ -209,14 +211,21 @@ io.setError("key not an integer"); return; } - auto P = V.emplace(KeyInt, /*IsAnalysis=*/false); - auto &Elem = (*P.first).second; + if (!V.count(KeyInt)) + V.emplace(KeyInt, /*IsAnalysis=*/false); + auto &Elem = V.find(KeyInt)->second; for (auto &FSum : FSums) { + std::vector Refs; + for (auto &RefGUID : FSum.Refs) { + if (!V.count(RefGUID)) + V.emplace(RefGUID, /*IsAnalysis=*/false); + Refs.push_back(ValueInfo(/*IsAnalysis=*/false, &*V.find(RefGUID))); + } Elem.SummaryList.push_back(llvm::make_unique( GlobalValueSummary::GVFlags( static_cast(FSum.Linkage), FSum.NotEligibleToImport, FSum.Live, FSum.IsLocal), - 0, FunctionSummary::FFlags{}, ArrayRef{}, + 0, FunctionSummary::FFlags{}, Refs, ArrayRef{}, std::move(FSum.TypeTests), std::move(FSum.TypeTestAssumeVCalls), std::move(FSum.TypeCheckedLoadVCalls), @@ -228,15 +237,20 @@ for (auto &P : V) { std::vector FSums; for (auto &Sum : P.second.SummaryList) { - if (auto *FSum = dyn_cast(Sum.get())) + if (auto *FSum = dyn_cast(Sum.get())) { + std::vector Refs; + for (auto &VI : FSum->refs()) + Refs.push_back(VI.getGUID()); FSums.push_back(FunctionSummaryYaml{ FSum->flags().Linkage, static_cast(FSum->flags().NotEligibleToImport), static_cast(FSum->flags().Live), - static_cast(FSum->flags().DSOLocal), FSum->type_tests(), - FSum->type_test_assume_vcalls(), FSum->type_checked_load_vcalls(), + static_cast(FSum->flags().DSOLocal), Refs, + FSum->type_tests(), FSum->type_test_assume_vcalls(), + FSum->type_checked_load_vcalls(), FSum->type_test_assume_const_vcalls(), FSum->type_checked_load_const_vcalls()}); + } } if (!FSums.empty()) io.mapRequired(llvm::utostr(P.first).c_str(), FSums); Index: lib/Transforms/IPO/LowerTypeTests.cpp =================================================================== --- lib/Transforms/IPO/LowerTypeTests.cpp +++ lib/Transforms/IPO/LowerTypeTests.cpp @@ -1669,26 +1669,54 @@ unsigned CurUniqueId = 0; SmallVector Types; + // Cross-DSO CFI emits jumptable entries for exported functions as well as + // address taken functions in case they are address taken in other modules. + const bool CrossDsoCfi = M.getModuleFlag("Cross-DSO CFI") != nullptr; + struct ExportedFunctionInfo { CfiFunctionLinkage Linkage; MDNode *FuncMD; // {name, linkage, type[, type...]} }; DenseMap ExportedFunctions; if (ExportSummary) { + // A set of all functions that are address taken by a live global object. + DenseSet AddressTaken; + for (auto &I : *ExportSummary) + for (auto &GVS : I.second.SummaryList) + if (GVS->isLive()) + for (auto &Ref : GVS->refs()) + AddressTaken.insert(Ref.getGUID()); + NamedMDNode *CfiFunctionsMD = M.getNamedMetadata("cfi.functions"); if (CfiFunctionsMD) { for (auto FuncMD : CfiFunctionsMD->operands()) { assert(FuncMD->getNumOperands() >= 2); StringRef FunctionName = cast(FuncMD->getOperand(0))->getString(); - if (!ExportSummary->isGUIDLive(GlobalValue::getGUID( - GlobalValue::dropLLVMManglingEscape(FunctionName)))) - continue; CfiFunctionLinkage Linkage = static_cast( cast(FuncMD->getOperand(1)) ->getValue() ->getUniqueInteger() .getZExtValue()); + const GlobalValue::GUID GUID = GlobalValue::getGUID( + GlobalValue::dropLLVMManglingEscape(FunctionName)); + // Do not emit jumptable entries for functions that are not-live and + // have no live references (and are not external with cross-DSO CFI.) + if (!ExportSummary->isGUIDLive(GUID)) + continue; + if (!AddressTaken.count(GUID)) { + if (!CrossDsoCfi || Linkage != CFL_Definition) + continue; + + bool External = false; + if (auto VI = ExportSummary->getValueInfo(GUID)) + for (auto &GVS : VI.getSummaryList()) + if (GVS->isLive()) + External |= GVS->linkage() == GlobalValue::ExternalLinkage; + + if (!External) + continue; + } auto P = ExportedFunctions.insert({FunctionName, {Linkage, FuncMD}}); if (!P.second && P.first->second.Linkage != CFL_Definition) P.first->second = {Linkage, FuncMD}; @@ -1748,9 +1776,18 @@ bool IsDefinition = !GO.isDeclarationForLinker(); bool IsExported = false; - if (isa(GO) && ExportedFunctions.count(GO.getName())) { - IsDefinition |= ExportedFunctions[GO.getName()].Linkage == CFL_Definition; - IsExported = true; + if (isa(GO)) { + if (ExportedFunctions.count(GO.getName())) { + IsDefinition |= ExportedFunctions[GO.getName()].Linkage == CFL_Definition; + IsExported = true; + // TODO: The logic here checks only that the function is address taken, + // not that the address takers are live. This can be updated to check + // their liveness and emit fewer jumptable entries once monolithic LTO + // builds also emit summaries. + } else if (!cast(GO).hasAddressTaken()) { + if (!CrossDsoCfi || !IsDefinition || !GO.hasExternalLinkage()) + continue; + } } auto *GTM = Index: test/LTO/Resolution/X86/export-jumptable.ll =================================================================== --- test/LTO/Resolution/X86/export-jumptable.ll +++ test/LTO/Resolution/X86/export-jumptable.ll @@ -19,11 +19,11 @@ ret void } -define i1 @_start(i8* %p) { - %1 = call i1 @llvm.type.test(i8* %p, metadata !"typeid1") - call void @f1() - call void @f2() - ret i1 %1 +define i1 @_start(i1 %i) { + %1 = select i1 %i, void ()* @f1, void ()* @f2 + %2 = bitcast void ()* %1 to i8* + %3 = call i1 @llvm.type.test(i8* %2, metadata !"typeid1") + ret i1 %3 } declare i1 @llvm.type.test(i8*, metadata) Index: test/ThinLTO/X86/cfi-icall.ll =================================================================== --- test/ThinLTO/X86/cfi-icall.ll +++ test/ThinLTO/X86/cfi-icall.ll @@ -1,7 +1,8 @@ ; RUN: opt -thinlto-bc %s -o %t1.bc ; RUN: llvm-lto2 run -thinlto-distributed-indexes %t1.bc -o %t.out -save-temps \ ; RUN: -r %t1.bc,foo,plx \ -; RUN: -r %t1.bc,bar,x +; RUN: -r %t1.bc,bar,x \ +; RUN: -r %t1.bc,addrtaken,px ; RUN: llvm-bcanalyzer -dump %t.out.index.bc | FileCheck %s --check-prefix=COMBINED target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" @@ -13,7 +14,13 @@ ret i1 %x } -declare !type !0 void @bar() +declare !type !0 i1 @bar(i8*) + +; Functions must be address taken to have jump table entries emitted +define void @addrtaken(i1 %i) { + %1 = select i1 %i, i1(i8*)* @foo, i1(i8*)* @bar + ret void +} declare i1 @llvm.type.test(i8* %ptr, metadata %type) nounwind readnone Index: test/Transforms/LowerTypeTests/Inputs/export-icall.yaml =================================================================== --- /dev/null +++ test/Transforms/LowerTypeTests/Inputs/export-icall.yaml @@ -0,0 +1,29 @@ +--- +GlobalValueMap: + 42: + - Live: true + # guid("f"), guid("f2"), guid("f3"), guid("g"), guid("h"), guid("external"), guid("external_weak") + Refs: [14740650423002898831, 8471399308421654326, 4197650231481825559, 13146401226427987378, 8124147457056772133, 5224464028922159466, 5227079976482001346] + TypeTests: [14276520915468743435, 15427464259790519041] # guid("typeid1"), guid("typeid2") + 14740650423002898831: # guid("f") + - Linkage: 0 # external + Live: true + 8471399308421654326: # guid("f2") + - Linkage: 0 # external + Live: true + 4197650231481825559: # guid("f3") + - Linkage: 0 # external + Live: true + 13146401226427987378: # guid("g") + - Linkage: 0 # external + Live: true + 8124147457056772133: # guid("h") + - Linkage: 0 # external + Live: true + 5224464028922159466: # guid("external") + - Linkage: 0 # external + Live: true + 5227079976482001346: # guid("external_weak") + - Linkage: 9 # extern_weak + Live: true +... Index: test/Transforms/LowerTypeTests/Inputs/exported-funcs.yaml =================================================================== --- /dev/null +++ test/Transforms/LowerTypeTests/Inputs/exported-funcs.yaml @@ -0,0 +1,22 @@ +--- +GlobalValueMap: + 42: + - Live: true + Refs: [16594175687743574550, 2415377257478301385] # guid("external_addrtaken"), guid("external_addrtaken2") + TypeTests: [14276520915468743435, 15427464259790519041] # guid("typeid1"), guid("typeid2") + 5224464028922159466: # guid("external") + - Linkage: 0 # external + Live: true + 16430208882958242304: # guid("external2") + - Linkage: 0 # external + Live: true + 16594175687743574550: # guid("external_addrtaken") + - Linkage: 0 # external + Live: true + 2415377257478301385: # guid("external_addrtaken2") + - Linkage: 0 # external + Live: true + 15859245615183425489: # guid("internal") + - Linkage: 7 # internal + Live: true +... Index: test/Transforms/LowerTypeTests/export-alias.ll =================================================================== --- test/Transforms/LowerTypeTests/export-alias.ll +++ test/Transforms/LowerTypeTests/export-alias.ll @@ -1,21 +1,21 @@ -; RUN: opt -S %s -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/use-typeid1-typeid2.yaml | FileCheck %s +; RUN: opt -S %s -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/exported-funcs.yaml | FileCheck %s ; -; CHECK: @alias1 = weak alias void (), void ()* @f -; CHECK: @alias2 = hidden alias void (), void ()* @f -; CHECK: declare !type !1 void @alias3() +; CHECK: @alias1 = weak alias void (), void ()* @external_addrtaken +; CHECK: @alias2 = hidden alias void (), void ()* @external_addrtaken ; CHECK-NOT: @alias3 = alias +; CHECK-NOT: @not_present target triple = "x86_64-unknown-linux" !cfi.functions = !{!0, !2, !3} !aliases = !{!4, !5, !6} -!0 = !{!"f", i8 0, !1} +!0 = !{!"external_addrtaken", i8 0, !1} !1 = !{i64 0, !"typeid1"} !2 = !{!"alias1", i8 1, !1} ; alias2 not included here, this could happen if the only reference to alias2 ; is in a module compiled without cfi-icall !3 = !{!"alias3", i8 1, !1} -!4 = !{!"alias1", !"f", i8 0, i8 1} -!5 = !{!"alias2", !"f", i8 1, i8 0} +!4 = !{!"alias1", !"external_addrtaken", i8 0, i8 1} +!5 = !{!"alias2", !"external_addrtaken", i8 1, i8 0} !6 = !{!"alias3", !"not_present", i8 0, i8 0} Index: test/Transforms/LowerTypeTests/export-cross-dso-cfi.ll =================================================================== --- /dev/null +++ test/Transforms/LowerTypeTests/export-cross-dso-cfi.ll @@ -0,0 +1,39 @@ +; Test that external functions have jumptable entries emitted even if they are +; not address-taken when Cross-DSO CFI is used, but not otherwise. + +; RUN: opt -S -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/exported-funcs.yaml < %s | FileCheck --check-prefixes=CHECK,CROSSDSO %s +; RUN: cat %s | grep -v "llvm.module.flags" | opt -S -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/exported-funcs.yaml | FileCheck --check-prefixes=CHECK,NORMAL %s + +target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +;;; Defined in the ThinLTO portion of the build (e.g. the summary) +; CROSSDSO: declare !type !1 !type !2 void @external.cfi() +; NORMAL: declare !type !1 !type !2 void @external() +declare !type !1 !type !2 void @external() + +; Don't emit jumptable entries for external declarations/non-external definitions +; CHECK-NOT: @external2 +; CHECK-NOT: @internal + +;;; Defined in the regular LTO portion of the build +; CROSSDSO: define internal void @regularlto_external.cfi() +; NORMAL: define void @regularlto_external() +define void @regularlto_external() !type !1 !type !2 { + ret void +} + +; CHECK: define internal void @regularlto_internal() +define internal void @regularlto_internal() !type !1 !type !2 { + ret void +} + +!cfi.functions = !{!0, !3, !4} +!llvm.module.flags = !{!5} + +!0 = !{!"external", i8 0, !1, !2} +!1 = !{i64 0, !"typeid1"} +!2 = !{i64 0, i64 1234} +!3 = !{!"external2", i8 1, !1, !2} +!4 = !{!"internal", i8 0, !1, !2} +!5 = !{i32 4, !"Cross-DSO CFI", i32 1} Index: test/Transforms/LowerTypeTests/export-icall.ll =================================================================== --- test/Transforms/LowerTypeTests/export-icall.ll +++ test/Transforms/LowerTypeTests/export-icall.ll @@ -1,4 +1,4 @@ -; RUN: opt -S -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/use-typeid1-typeid2.yaml -lowertypetests-write-summary=%t < %s | FileCheck %s +; RUN: opt -S -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/export-icall.yaml -lowertypetests-write-summary=%t < %s | FileCheck %s ; RUN: FileCheck --check-prefix=SUMMARY %s < %t target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" Index: test/Transforms/LowerTypeTests/export-symver.ll =================================================================== --- test/Transforms/LowerTypeTests/export-symver.ll +++ test/Transforms/LowerTypeTests/export-symver.ll @@ -1,16 +1,16 @@ -; RUN: opt -S %s -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/use-typeid1-typeid2.yaml | FileCheck %s +; RUN: opt -S %s -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/exported-funcs.yaml | FileCheck %s ; -; CHECK: module asm ".symver exported_and_symver, alias1" -; CHECK-NOT: .symver exported -; CHECK-NOT: .symver symver +; CHECK: module asm ".symver external_addrtaken, alias1" +; CHECK-NOT: .symver external_addrtaken2 +; CHECK-NOT: .symver not_exported target triple = "x86_64-unknown-linux" !cfi.functions = !{!0, !1} !symvers = !{!3, !4} -!0 = !{!"exported_and_symver", i8 2, !2} -!1 = !{!"exported", i8 2, !2} +!0 = !{!"external_addrtaken", i8 0, !2} +!1 = !{!"external_addrtaken2", i8 0, !2} !2 = !{i64 0, !"typeid1"} -!3 = !{!"exported_and_symver", !"alias1"} -!4 = !{!"symver", !"alias2"} +!3 = !{!"external_addrtaken", !"alias1"} +!4 = !{!"not_exported", !"alias2"} Index: test/Transforms/LowerTypeTests/function-arm-thumb.ll =================================================================== --- test/Transforms/LowerTypeTests/function-arm-thumb.ll +++ test/Transforms/LowerTypeTests/function-arm-thumb.ll @@ -22,6 +22,12 @@ ret void } +declare void @takeaddr(void()*, void()*, void()*, void()*, void()*) +define void @addrtaken() { + call void @takeaddr(void()* @f1, void()* @g1, void()* @f2, void()* @g2, void()* @h2) + ret void +} + !0 = !{i32 0, !"typeid1"} !1 = !{i32 0, !"typeid2"} Index: test/Transforms/LowerTypeTests/function-ext.ll =================================================================== --- test/Transforms/LowerTypeTests/function-ext.ll +++ test/Transforms/LowerTypeTests/function-ext.ll @@ -10,7 +10,7 @@ declare !type !0 void @foo() define i1 @bar(i8* %ptr) { - ; X64: icmp eq i64 {{.*}}, ptrtoint (void ()* @[[JT:.*]] to i64) + ; X64: ret i1 false ; WASM32: ret i1 false %p = call i1 @llvm.type.test(i8* %ptr, metadata !"void") ret i1 %p @@ -21,6 +21,3 @@ !0 = !{i64 0, !"void"} ; WASM-NOT: !{i64 0} ; WASM-NOT: !{i64 1} - -; X64: define private void @[[JT]]() #{{.*}} align {{.*}} { -; X64: call void asm sideeffect "jmp ${0:c}@plt\0Aint3\0Aint3\0Aint3\0A", "s"(void ()* @foo) Index: test/Transforms/LowerTypeTests/pr37625.ll =================================================================== --- test/Transforms/LowerTypeTests/pr37625.ll +++ test/Transforms/LowerTypeTests/pr37625.ll @@ -1,14 +1,14 @@ -; RUN: opt -S -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/use-typeid1-typeid2.yaml -lowertypetests-write-summary=%t < %s | FileCheck %s +; RUN: opt -S -lowertypetests -lowertypetests-summary-action=export -lowertypetests-read-summary=%S/Inputs/exported-funcs.yaml -lowertypetests-write-summary=%t < %s | FileCheck %s target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" -declare !type !2 extern_weak void @h(i8) +declare !type !2 extern_weak void @external_addrtaken(i8) !cfi.functions = !{!0, !1} -!0 = !{!"h", i8 2, !2} -!1 = !{!"h", i8 0, !2} +!0 = !{!"external_addrtaken", i8 2, !2} +!1 = !{!"external_addrtaken", i8 0, !2} !2 = !{i64 0, !"typeid1"} -; CHECK-DAG: @h = alias void (i8), bitcast +; CHECK-DAG: @external_addrtaken = alias void (i8), bitcast