Index: llvm/include/llvm/IR/ModuleSummaryIndex.h =================================================================== --- llvm/include/llvm/IR/ModuleSummaryIndex.h +++ llvm/include/llvm/IR/ModuleSummaryIndex.h @@ -370,6 +370,14 @@ return TIdInfo->TypeCheckedLoadConstVCalls; return {}; } + + /// Add a type test to the summary. This is used by WholeProgramDevirt if we + /// were unable to devirtualize a checked call. + void addTypeTest(GlobalValue::GUID Guid) { + if (!TIdInfo) + TIdInfo = llvm::make_unique(); + TIdInfo->TypeTests.push_back(Guid); + } }; template <> struct DenseMapInfo { Index: llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp =================================================================== --- llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp +++ llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp @@ -284,6 +284,7 @@ struct CallSiteInfo { std::vector CallSites; + std::vector TypeCheckedLoadUsers; }; struct VTableSlotInfo { @@ -956,7 +957,8 @@ M.getFunction(Intrinsic::getName(Intrinsic::type_checked_load)); Function *AssumeFunc = M.getFunction(Intrinsic::getName(Intrinsic::assume)); - if ((!TypeTestFunc || TypeTestFunc->use_empty() || !AssumeFunc || + if (Action != PassSummaryAction::Export && + (!TypeTestFunc || TypeTestFunc->use_empty() || !AssumeFunc || AssumeFunc->use_empty()) && (!TypeCheckedLoadFunc || TypeCheckedLoadFunc->use_empty())) return false; @@ -974,6 +976,34 @@ if (TypeIdMap.empty()) return true; + // Collect information from summary about which calls to try to devirtualize. + if (Action == PassSummaryAction::Export) { + DenseMap> MetadataByGUID; + for (auto &P : TypeIdMap) { + if (auto *TypeId = dyn_cast(P.first)) + MetadataByGUID[GlobalValue::getGUID(TypeId->getString())].push_back( + TypeId); + } + + for (auto &P : *Summary) { + for (auto &S : P.second) { + auto *FS = dyn_cast(S.get()); + if (!FS) + continue; + // FIXME: Only add live functions. + for (FunctionSummary::VFuncId VF : FS->type_checked_load_vcalls()) + for (Metadata *MD : MetadataByGUID[VF.GUID]) + CallSlots[{MD, VF.Offset}].CSInfo.TypeCheckedLoadUsers.push_back( + FS); + for (const FunctionSummary::ConstVCall &VC : + FS->type_checked_load_const_vcalls()) + for (Metadata *MD : MetadataByGUID[VC.VFunc.GUID]) + CallSlots[{MD, VC.VFunc.Offset}] + .ConstCSInfo[VC.Args].TypeCheckedLoadUsers.push_back(FS); + } + } + } + // For each (type, offset) pair: bool DidVirtualConstProp = false; std::map DevirtTargets; @@ -982,19 +1012,32 @@ // function implementation at offset S.first.ByteOffset, and add to // TargetsForSlot. std::vector TargetsForSlot; - if (!tryFindVirtualCallTargets(TargetsForSlot, TypeIdMap[S.first.TypeID], - S.first.ByteOffset)) - continue; - - if (!trySingleImplDevirt(TargetsForSlot, S.second) && - tryVirtualConstProp(TargetsForSlot, S.second)) + if (tryFindVirtualCallTargets(TargetsForSlot, TypeIdMap[S.first.TypeID], + S.first.ByteOffset)) { + if (!trySingleImplDevirt(TargetsForSlot, S.second) && + tryVirtualConstProp(TargetsForSlot, S.second)) DidVirtualConstProp = true; - // Collect functions devirtualized at least for one call site for stats. - if (RemarksEnabled) - for (const auto &T : TargetsForSlot) - if (T.WasDevirt) - DevirtTargets[T.Fn->getName()] = T.Fn; + // Collect functions devirtualized at least for one call site for stats. + if (RemarksEnabled) + for (const auto &T : TargetsForSlot) + if (T.WasDevirt) + DevirtTargets[T.Fn->getName()] = T.Fn; + } + + // CFI-specific: if we are exporting and any llvm.type.checked.load + // intrinsics were *not* devirtualized, we need to add the resulting + // llvm.type.test intrinsics to the function summaries so that the + // LowerTypeTests pass will export them. + if (Action == PassSummaryAction::Export && isa(S.first.TypeID)) { + auto GUID = + GlobalValue::getGUID(cast(S.first.TypeID)->getString()); + for (auto FS : S.second.CSInfo.TypeCheckedLoadUsers) + FS->addTypeTest(GUID); + for (auto &CCS : S.second.ConstCSInfo) + for (auto FS : CCS.second.TypeCheckedLoadUsers) + FS->addTypeTest(GUID); + } } if (RemarksEnabled) { Index: llvm/test/Transforms/WholeProgramDevirt/Inputs/export.yaml =================================================================== --- /dev/null +++ llvm/test/Transforms/WholeProgramDevirt/Inputs/export.yaml @@ -0,0 +1,20 @@ +--- +GlobalValueMap: + 42: + - TypeTestAssumeVCalls: + - GUID: 14276520915468743435 # typeid1 + Offset: 0 + TypeCheckedLoadVCalls: + - GUID: 15427464259790519041 # typeid2 + Offset: 0 + TypeTestAssumeConstVCalls: + - VFunc: + GUID: 3515965990081467659 # typeid3 + Offset: 0 + Args: [12, 24] + TypeCheckedLoadConstVCalls: + - VFunc: + GUID: 17525413373118030901 # typeid4 + Offset: 0 + Args: [24, 12] +... Index: llvm/test/Transforms/WholeProgramDevirt/export-unsuccessful-checked.ll =================================================================== --- /dev/null +++ llvm/test/Transforms/WholeProgramDevirt/export-unsuccessful-checked.ll @@ -0,0 +1,30 @@ +; RUN: opt -wholeprogramdevirt -wholeprogramdevirt-summary-action=export -wholeprogramdevirt-read-summary=%S/Inputs/export.yaml -wholeprogramdevirt-write-summary=%t -o /dev/null %s +; RUN: FileCheck %s < %t + +; CHECK: - TypeTests: +; CHECK-NEXT: - 15427464259790519041 +; CHECK-NEXT: - 17525413373118030901 +; CHECK-NEXT: TypeTestAssumeVCalls: + +@vt1a = constant void (i8*)* @vf1a, !type !0 +@vt1b = constant void (i8*)* @vf1b, !type !0 +@vt2a = constant void (i8*)* @vf2a, !type !1 +@vt2b = constant void (i8*)* @vf2b, !type !1 +@vt3a = constant void (i8*)* @vf3a, !type !2 +@vt3b = constant void (i8*)* @vf3b, !type !2 +@vt4a = constant void (i8*)* @vf4a, !type !3 +@vt4b = constant void (i8*)* @vf4b, !type !3 + +declare void @vf1a(i8*) +declare void @vf1b(i8*) +declare void @vf2a(i8*) +declare void @vf2b(i8*) +declare void @vf3a(i8*) +declare void @vf3b(i8*) +declare void @vf4a(i8*) +declare void @vf4b(i8*) + +!0 = !{i32 0, !"typeid1"} +!1 = !{i32 0, !"typeid2"} +!2 = !{i32 0, !"typeid3"} +!3 = !{i32 0, !"typeid4"}