Index: llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp =================================================================== --- llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp +++ llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp @@ -320,15 +320,18 @@ /// the llvm.type.checked.load intrinsic and therefore will require /// resolutions for llvm.type.test in order to implement CFI checks if /// devirtualization was unsuccessful. If devirtualization was successful, the - /// pass will clear this vector. If at the end of the pass the vector is - /// non-empty, we will need to add a use of llvm.type.test to each of the - /// function summaries in the vector. + /// pass will clear this vector by calling markDevirt(). If at the end of the + /// pass the vector is non-empty, we will need to add a use of llvm.type.test + /// to each of the function summaries in the vector. std::vector SummaryTypeCheckedLoadUsers; bool isExported() const { return SummaryHasTypeTestAssumeUsers || !SummaryTypeCheckedLoadUsers.empty(); } + + /// As explained in the comment for SummaryTypeCheckedLoadUsers. + void markDevirt() { SummaryTypeCheckedLoadUsers.clear(); } }; // Call site information collected for a specific VTableSlot. @@ -431,17 +434,35 @@ CallSiteInfo &CSInfo, WholeProgramDevirtResolution::ByArg *Res); + // Returns the global symbol name that is used to export information about the + // given vtable slot and list of arguments. + std::string getGlobalName(VTableSlot Slot, ArrayRef Args, + StringRef Name); + + // This function is called during the export phase to create a symbol + // definition containing information about the given vtable slot and list of + // arguments. + void exportGlobal(VTableSlot Slot, ArrayRef Args, StringRef Name, + Constant *C); + + // This function is called during the import phase to create a reference to + // the symbol definition created during the export phase. + Constant *importGlobal(VTableSlot Slot, ArrayRef Args, + StringRef Name); + void applyUniqueRetValOpt(CallSiteInfo &CSInfo, StringRef FnName, bool IsOne, Constant *UniqueMemberAddr); bool tryUniqueRetValOpt(unsigned BitWidth, MutableArrayRef TargetsForSlot, - CallSiteInfo &CSInfo); + CallSiteInfo &CSInfo, + WholeProgramDevirtResolution::ByArg *Res, + VTableSlot Slot, ArrayRef Args); void applyVirtualConstProp(CallSiteInfo &CSInfo, StringRef FnName, Constant *Byte, Constant *Bit); bool tryVirtualConstProp(MutableArrayRef TargetsForSlot, VTableSlotInfo &SlotInfo, - WholeProgramDevirtResolution *Res); + WholeProgramDevirtResolution *Res, VTableSlot Slot); void rebuildGlobal(VTableBits &B); @@ -658,7 +679,7 @@ } if (CSInfo.isExported()) { IsExported = true; - CSInfo.SummaryTypeCheckedLoadUsers.clear(); + CSInfo.markDevirt(); } }; Apply(SlotInfo.CSInfo); @@ -736,7 +757,7 @@ Call.replaceAndErase( "uniform-ret-val", FnName, RemarksEnabled, ConstantInt::get(cast(Call.CS.getType()), TheRetVal)); - CSInfo.SummaryTypeCheckedLoadUsers.clear(); + CSInfo.markDevirt(); } bool DevirtModule::tryUniformRetValOpt( @@ -761,6 +782,35 @@ return true; } +std::string DevirtModule::getGlobalName(VTableSlot Slot, + ArrayRef Args, + StringRef Name) { + std::string FullName = "__typeid_"; + raw_string_ostream OS(FullName); + OS << cast(Slot.TypeID)->getString() << '_' << Slot.ByteOffset; + for (uint64_t Arg : Args) + OS << '_' << Arg; + OS << '_' << Name; + return OS.str(); +} + +void DevirtModule::exportGlobal(VTableSlot Slot, ArrayRef Args, + StringRef Name, Constant *C) { + GlobalAlias *GA = GlobalAlias::create(Int8Ty, 0, GlobalValue::ExternalLinkage, + getGlobalName(Slot, Args, Name), C, &M); + GA->setVisibility(GlobalValue::HiddenVisibility); +} + +Constant *DevirtModule::importGlobal(VTableSlot Slot, ArrayRef Args, + StringRef Name) { + Constant *C = M.getOrInsertGlobal(getGlobalName(Slot, Args, Name), Int8Ty); + auto *GV = dyn_cast(C); + if (!GV) + return C; + GV->setVisibility(GlobalValue::HiddenVisibility); + return GV; +} + void DevirtModule::applyUniqueRetValOpt(CallSiteInfo &CSInfo, StringRef FnName, bool IsOne, Constant *UniqueMemberAddr) { @@ -771,11 +821,13 @@ Cmp = B.CreateZExt(Cmp, Call.CS->getType()); Call.replaceAndErase("unique-ret-val", FnName, RemarksEnabled, Cmp); } + CSInfo.markDevirt(); } bool DevirtModule::tryUniqueRetValOpt( unsigned BitWidth, MutableArrayRef TargetsForSlot, - CallSiteInfo &CSInfo) { + CallSiteInfo &CSInfo, WholeProgramDevirtResolution::ByArg *Res, + VTableSlot Slot, ArrayRef Args) { // IsOne controls whether we look for a 0 or a 1. auto tryUniqueRetValOptFor = [&](bool IsOne) { const TypeMemberInfo *UniqueMember = nullptr; @@ -791,13 +843,20 @@ // checked for a uniform return value in tryUniformRetValOpt. assert(UniqueMember); - // Replace each call with the comparison. Constant *UniqueMemberAddr = ConstantExpr::getBitCast(UniqueMember->Bits->GV, Int8PtrTy); UniqueMemberAddr = ConstantExpr::getGetElementPtr( Int8Ty, UniqueMemberAddr, ConstantInt::get(Int64Ty, UniqueMember->Offset)); + if (CSInfo.isExported()) { + Res->TheKind = WholeProgramDevirtResolution::ByArg::UniqueRetVal; + Res->Info = IsOne; + + exportGlobal(Slot, Args, "unique_member", UniqueMemberAddr); + } + + // Replace each call with the comparison. applyUniqueRetValOpt(CSInfo, TargetsForSlot[0].Fn->getName(), IsOne, UniqueMemberAddr); @@ -839,8 +898,8 @@ } bool DevirtModule::tryVirtualConstProp( - MutableArrayRef TargetsForSlot, - VTableSlotInfo &SlotInfo, WholeProgramDevirtResolution *Res) { + MutableArrayRef TargetsForSlot, VTableSlotInfo &SlotInfo, + WholeProgramDevirtResolution *Res, VTableSlot Slot) { // This only works if the function returns an integer. auto RetType = dyn_cast(TargetsForSlot[0].Fn->getReturnType()); if (!RetType) @@ -879,7 +938,8 @@ if (tryUniformRetValOpt(TargetsForSlot, CSByConstantArg.second, ResByArg)) continue; - if (tryUniqueRetValOpt(BitWidth, TargetsForSlot, CSByConstantArg.second)) + if (tryUniqueRetValOpt(BitWidth, TargetsForSlot, CSByConstantArg.second, + ResByArg, Slot, CSByConstantArg.first)) continue; // Find an allocation offset in bits in all vtables associated with the @@ -1140,6 +1200,13 @@ case WholeProgramDevirtResolution::ByArg::UniformRetVal: applyUniformRetValOpt(CSByConstantArg.second, "", ResByArg.Info); break; + case WholeProgramDevirtResolution::ByArg::UniqueRetVal: { + Constant *UniqueMemberAddr = + importGlobal(Slot, CSByConstantArg.first, "unique_member"); + applyUniqueRetValOpt(CSByConstantArg.second, "", ResByArg.Info, + UniqueMemberAddr); + break; + } default: break; } @@ -1261,7 +1328,7 @@ .WPDRes[S.first.ByteOffset]; if (!trySingleImplDevirt(TargetsForSlot, S.second, Res) && - tryVirtualConstProp(TargetsForSlot, S.second, Res)) + tryVirtualConstProp(TargetsForSlot, S.second, Res, S.first)) DidVirtualConstProp = true; // Collect functions devirtualized at least for one call site for stats. Index: llvm/test/Transforms/WholeProgramDevirt/Inputs/import-unique-ret-val0.yaml =================================================================== --- /dev/null +++ llvm/test/Transforms/WholeProgramDevirt/Inputs/import-unique-ret-val0.yaml @@ -0,0 +1,11 @@ +--- +TypeIdMap: + typeid2: + WPDRes: + 8: + Kind: Indir + ResByArg: + 3: + Kind: UniqueRetVal + Info: 0 +... Index: llvm/test/Transforms/WholeProgramDevirt/Inputs/import-unique-ret-val1.yaml =================================================================== --- /dev/null +++ llvm/test/Transforms/WholeProgramDevirt/Inputs/import-unique-ret-val1.yaml @@ -0,0 +1,11 @@ +--- +TypeIdMap: + typeid2: + WPDRes: + 8: + Kind: Indir + ResByArg: + 3: + Kind: UniqueRetVal + Info: 1 +... Index: llvm/test/Transforms/WholeProgramDevirt/export-unique-ret-val.ll =================================================================== --- /dev/null +++ llvm/test/Transforms/WholeProgramDevirt/export-unique-ret-val.ll @@ -0,0 +1,79 @@ +; RUN: opt -wholeprogramdevirt -wholeprogramdevirt-summary-action=export -wholeprogramdevirt-read-summary=%S/Inputs/export.yaml -wholeprogramdevirt-write-summary=%t -S -o - %s | FileCheck %s +; RUN: FileCheck --check-prefix=SUMMARY %s < %t + +; SUMMARY: - TypeTests: +; SUMMARY-NEXT: TypeTestAssumeVCalls: + +; SUMMARY: TypeIdMap: +; SUMMARY-NEXT: typeid3: +; SUMMARY-NEXT: TTRes: +; SUMMARY-NEXT: Kind: Unsat +; SUMMARY-NEXT: SizeM1BitWidth: 0 +; SUMMARY-NEXT: WPDRes: +; SUMMARY-NEXT: 0: +; SUMMARY-NEXT: Kind: Indir +; SUMMARY-NEXT: SingleImplName: '' +; SUMMARY-NEXT: ResByArg: +; SUMMARY-NEXT: 12,24: +; SUMMARY-NEXT: Kind: UniqueRetVal +; SUMMARY-NEXT: Info: 0 +; SUMMARY-NEXT: typeid4: +; SUMMARY-NEXT: TTRes: +; SUMMARY-NEXT: Kind: Unsat +; SUMMARY-NEXT: SizeM1BitWidth: 0 +; SUMMARY-NEXT: WPDRes: +; SUMMARY-NEXT: 0: +; SUMMARY-NEXT: Kind: Indir +; SUMMARY-NEXT: SingleImplName: '' +; SUMMARY-NEXT: ResByArg: +; SUMMARY-NEXT: 24,12: +; SUMMARY-NEXT: Kind: UniqueRetVal +; SUMMARY-NEXT: Info: 1 + +; CHECK: @vt3a = constant i1 (i8*, i32, i32)* @vf3a +@vt3a = constant i1 (i8*, i32, i32)* @vf3a, !type !0 + +; CHECK: @vt3b = constant i1 (i8*, i32, i32)* @vf3b +@vt3b = constant i1 (i8*, i32, i32)* @vf3b, !type !0 + +; CHECK: @vt3c = constant i1 (i8*, i32, i32)* @vf3c +@vt3c = constant i1 (i8*, i32, i32)* @vf3c, !type !0 + +; CHECK: @vt4a = constant i1 (i8*, i32, i32)* @vf4a +@vt4a = constant i1 (i8*, i32, i32)* @vf4a, !type !1 + +; CHECK: @vt4b = constant i1 (i8*, i32, i32)* @vf4b +@vt4b = constant i1 (i8*, i32, i32)* @vf4b, !type !1 + +; CHECK: @vt4c = constant i1 (i8*, i32, i32)* @vf4c +@vt4c = constant i1 (i8*, i32, i32)* @vf4c, !type !1 + +; CHECK: @__typeid_typeid3_0_12_24_unique_member = hidden alias i8, bitcast (i1 (i8*, i32, i32)** @vt3b to i8*) +; CHECK: @__typeid_typeid4_0_24_12_unique_member = hidden alias i8, bitcast (i1 (i8*, i32, i32)** @vt4b to i8*) + +define i1 @vf3a(i8*, i32, i32) { + ret i1 true +} + +define i1 @vf3b(i8*, i32, i32) { + ret i1 false +} + +define i1 @vf3c(i8*, i32, i32) { + ret i1 true +} + +define i1 @vf4a(i8*, i32, i32) { + ret i1 false +} + +define i1 @vf4b(i8*, i32, i32) { + ret i1 true +} + +define i1 @vf4c(i8*, i32, i32) { + ret i1 false +} + +!0 = !{i32 0, !"typeid3"} +!1 = !{i32 0, !"typeid4"} Index: llvm/test/Transforms/WholeProgramDevirt/import.ll =================================================================== --- llvm/test/Transforms/WholeProgramDevirt/import.ll +++ llvm/test/Transforms/WholeProgramDevirt/import.ll @@ -1,5 +1,7 @@ ; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-single-impl.yaml < %s | FileCheck --check-prefixes=CHECK,SINGLE-IMPL %s ; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-uniform-ret-val.yaml < %s | FileCheck --check-prefixes=CHECK,UNIFORM-RET-VAL %s +; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-unique-ret-val0.yaml < %s | FileCheck --check-prefixes=CHECK,UNIQUE-RET-VAL0 %s +; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-unique-ret-val1.yaml < %s | FileCheck --check-prefixes=CHECK,UNIQUE-RET-VAL1 %s target datalayout = "e-p:64:64" target triple = "x86_64-unknown-linux-gnu" @@ -41,6 +43,8 @@ %fptr_casted = bitcast i8* %fptr to i1 (i8*, i32)* ; SINGLE-IMPL: call i1 bitcast (void ()* @singleimpl2 to i1 (i8*, i32)*) ; UNIFORM-RET-VAL: call i1 % + ; UNIQUE-RET-VAL0: call i1 % + ; UNIQUE-RET-VAL1: call i1 % %result = call i1 %fptr_casted(i8* %obj, i32 undef) ret i1 %result @@ -49,6 +53,28 @@ unreachable } +; CHECK: define i1 @call3 +define i1 @call3(i8* %obj) { + %vtableptr = bitcast i8* %obj to [1 x i8*]** + %vtable = load [1 x i8*]*, [1 x i8*]** %vtableptr + %vtablei8 = bitcast [1 x i8*]* %vtable to i8* + %pair = call {i8*, i1} @llvm.type.checked.load(i8* %vtablei8, i32 8, metadata !"typeid2") + %fptr = extractvalue {i8*, i1} %pair, 0 + %p = extractvalue {i8*, i1} %pair, 1 + br i1 %p, label %cont, label %trap + +cont: + %fptr_casted = bitcast i8* %fptr to i1 (i8*, i32)* + %result = call i1 %fptr_casted(i8* %obj, i32 3) + ; UNIQUE-RET-VAL0: icmp ne i8* %vtablei8, @__typeid_typeid2_8_3_unique_member + ; UNIQUE-RET-VAL1: icmp eq i8* %vtablei8, @__typeid_typeid2_8_3_unique_member + ret i1 %result + +trap: + call void @llvm.trap() + unreachable +} + ; SINGLE-IMPL-DAG: declare void @singleimpl1() ; SINGLE-IMPL-DAG: declare void @singleimpl2()