WholeProgramDevirt: Implement export/import support for VCP.

[pcc]

Summary

Depends on D29917

Depends on D30014

Depends on D30016

[tejohnson]

llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp

763 ↗

(On Diff #88635)

I noticed that this function largely duplicates handling in LowerTypeTests.cpp (ditto for exportGlobal). Would it make sense to refactor that into a general helper that can be invoked by both?

[pcc]

llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp

763 ↗

(On Diff #88635)

Maybe. This is a "rule of three" situation, I think -- if we add another pass that needs this, we can factor it out.

[tejohnson]

LGTM

llvm/lib/Transforms/IPO/WholeProgramDevirt.cpp

763 ↗

(On Diff #88635)

Ok, that's fine.

864 ↗

(On Diff #88635)

Make sure to update this to instead use the new markDevirt() before committing.

llvm/trunk/lib/Transforms/IPO/WholeProgramDevirt.cpp

struct DevirtModule {
 
   PassSummaryAction Action;
   ModuleSummaryIndex *Summary;
 
   IntegerType *Int8Ty;
   PointerType *Int8PtrTy;
   IntegerType *Int32Ty;
   IntegerType *Int64Ty;
+  IntegerType *IntPtrTy;
 
   bool RemarksEnabled;
 
   MapVector<VTableSlot, VTableSlotInfo> CallSlots;
 
   // This map keeps track of the number of "unsafe" uses of a loaded function
   // pointer. The key is the associated llvm.type.test intrinsic call generated
   // by this pass. An unsafe use is one that calls the loaded function pointer
   // directly. Every time we eliminate an unsafe use (for example, by
   // devirtualizing it or by applying virtual constant propagation), we
   // decrement the value stored in this map. If a value reaches zero, we can
   // eliminate the type check by RAUWing the associated llvm.type.test call with
   // true.
   std::map<CallInst *, unsigned> NumUnsafeUsesForTypeTest;
 
   DevirtModule(Module &M, function_ref<AAResults &(Function &)> AARGetter,
                PassSummaryAction Action, ModuleSummaryIndex *Summary)
       : M(M), AARGetter(AARGetter), Action(Action), Summary(Summary),
         Int8Ty(Type::getInt8Ty(M.getContext())),
         Int8PtrTy(Type::getInt8PtrTy(M.getContext())),
         Int32Ty(Type::getInt32Ty(M.getContext())),
         Int64Ty(Type::getInt64Ty(M.getContext())),
+        IntPtrTy(M.getDataLayout().getIntPtrType(M.getContext(), 0)),
         RemarksEnabled(areRemarksEnabled()) {}
 
   bool areRemarksEnabled();
 
   void scanTypeTestUsers(Function *TypeTestFunc, Function *AssumeFunc);
   void scanTypeCheckedLoadUsers(Function *TypeCheckedLoadFunc);
 
   void buildTypeIdentifierMap(
struct DevirtModule {
   // definition containing information about the given vtable slot and list of
   // arguments.
   void exportGlobal(VTableSlot Slot, ArrayRef<uint64_t> Args, StringRef Name,
                     Constant *C);
 
   // This function is called during the import phase to create a reference to
   // the symbol definition created during the export phase.
   Constant *importGlobal(VTableSlot Slot, ArrayRef<uint64_t> Args,
-                         StringRef Name);
+                         StringRef Name, unsigned AbsWidth = 0);
 
   void applyUniqueRetValOpt(CallSiteInfo &CSInfo, StringRef FnName, bool IsOne,
                             Constant *UniqueMemberAddr);
   bool tryUniqueRetValOpt(unsigned BitWidth,
                           MutableArrayRef<VirtualCallTarget> TargetsForSlot,
                           CallSiteInfo &CSInfo,
                           WholeProgramDevirtResolution::ByArg *Res,
                           VTableSlot Slot, ArrayRef<uint64_t> Args);

 void DevirtModule::exportGlobal(VTableSlot Slot, ArrayRef<uint64_t> Args,
                                 StringRef Name, Constant *C) {
   GlobalAlias *GA = GlobalAlias::create(Int8Ty, 0, GlobalValue::ExternalLinkage,
                                         getGlobalName(Slot, Args, Name), C, &M);
   GA->setVisibility(GlobalValue::HiddenVisibility);
 }
 
 Constant *DevirtModule::importGlobal(VTableSlot Slot, ArrayRef<uint64_t> Args,
-                                     StringRef Name) {
+                                     StringRef Name, unsigned AbsWidth) {
   Constant *C = M.getOrInsertGlobal(getGlobalName(Slot, Args, Name), Int8Ty);
   auto *GV = dyn_cast<GlobalVariable>(C);
-  if (!GV)
+  // We only need to set metadata if the global is newly created, in which
+  // case it would not have hidden visibility.
+  if (!GV || GV->getVisibility() == GlobalValue::HiddenVisibility)
     return C;
+
   GV->setVisibility(GlobalValue::HiddenVisibility);
+  auto SetAbsRange = [&](uint64_t Min, uint64_t Max) {
+    auto *MinC = ConstantAsMetadata::get(ConstantInt::get(IntPtrTy, Min));
+    auto *MaxC = ConstantAsMetadata::get(ConstantInt::get(IntPtrTy, Max));
+    GV->setMetadata(LLVMContext::MD_absolute_symbol,
+                    MDNode::get(M.getContext(), {MinC, MaxC}));
+  };
+  if (AbsWidth == IntPtrTy->getBitWidth())
+    SetAbsRange(~0ull, ~0ull); // Full set.
+  else if (AbsWidth)
+    SetAbsRange(0, 1ull << AbsWidth);
   return GV;
 }
 
 void DevirtModule::applyUniqueRetValOpt(CallSiteInfo &CSInfo, StringRef FnName,
                                         bool IsOne,
                                         Constant *UniqueMemberAddr) {
   for (auto &&Call : CSInfo.CallSites) {
     IRBuilder<> B(Call.CS.getInstruction());
if (RetType->getBitWidth() == 1) {
       Call.replaceAndErase("virtual-const-prop-1-bit", FnName, RemarksEnabled,
                            IsBitSet);
     } else {
       Value *ValAddr = B.CreateBitCast(Addr, RetType->getPointerTo());
       Value *Val = B.CreateLoad(RetType, ValAddr);
       Call.replaceAndErase("virtual-const-prop", FnName, RemarksEnabled, Val);
     }
   }
+  CSInfo.markDevirt();
 }
 
 bool DevirtModule::tryVirtualConstProp(
     MutableArrayRef<VirtualCallTarget> TargetsForSlot, VTableSlotInfo &SlotInfo,
     WholeProgramDevirtResolution *Res, VTableSlot Slot) {
   // This only works if the function returns an integer.
   auto RetType = dyn_cast<IntegerType>(TargetsForSlot[0].Fn->getReturnType());
   if (!RetType)
for (auto &&CSByConstantArg : SlotInfo.ConstCSInfo) {
     else
       setAfterReturnValues(TargetsForSlot, AllocAfter, BitWidth, OffsetByte,
                            OffsetBit);
 
     if (RemarksEnabled)
       for (auto &&Target : TargetsForSlot)
         Target.WasDevirt = true;
 
-    // Rewrite each call to a load from OffsetByte/OffsetBit.
     Constant *ByteConst = ConstantInt::get(Int32Ty, OffsetByte);
     Constant *BitConst = ConstantInt::get(Int8Ty, 1ULL << OffsetBit);
+
+    if (CSByConstantArg.second.isExported()) {
+      ResByArg->TheKind = WholeProgramDevirtResolution::ByArg::VirtualConstProp;
+      exportGlobal(Slot, CSByConstantArg.first, "byte",
+                   ConstantExpr::getIntToPtr(ByteConst, Int8PtrTy));
+      exportGlobal(Slot, CSByConstantArg.first, "bit",
+                   ConstantExpr::getIntToPtr(BitConst, Int8PtrTy));
+    }
+
+    // Rewrite each call to a load from OffsetByte/OffsetBit.
     applyVirtualConstProp(CSByConstantArg.second,
                           TargetsForSlot[0].Fn->getName(), ByteConst, BitConst);
   }
   return true;
 }
 
 void DevirtModule::rebuildGlobal(VTableBits &B) {
   if (B.Before.Bytes.empty() && B.After.Bytes.empty())
case WholeProgramDevirtResolution::ByArg::UniformRetVal:
       break;
     case WholeProgramDevirtResolution::ByArg::UniqueRetVal: {
       Constant *UniqueMemberAddr =
           importGlobal(Slot, CSByConstantArg.first, "unique_member");
       applyUniqueRetValOpt(CSByConstantArg.second, "", ResByArg.Info,
                            UniqueMemberAddr);
       break;
     }
+    case WholeProgramDevirtResolution::ByArg::VirtualConstProp: {
+      Constant *Byte = importGlobal(Slot, CSByConstantArg.first, "byte", 32);
+      Byte = ConstantExpr::getPtrToInt(Byte, Int32Ty);
+      Constant *Bit = importGlobal(Slot, CSByConstantArg.first, "bit", 8);
+      Bit = ConstantExpr::getPtrToInt(Bit, Int8Ty);
+      applyVirtualConstProp(CSByConstantArg.second, "", Byte, Bit);
+    }
     default:
       break;
     }
   }
 }
 
 void DevirtModule::removeRedundantTypeTests() {
   auto True = ConstantInt::getTrue(M.getContext());

llvm/trunk/test/Transforms/WholeProgramDevirt/Inputs/import-vcp.yaml

+---
+TypeIdMap:
+  typeid1:
+    WPDRes:
+      0:
+        Kind: Indir
+        ResByArg:
+          1:
+            Kind: VirtualConstProp
+            Info: 0
+  typeid2:
+    WPDRes:
+      8:
+        Kind: Indir
+        ResByArg:
+          3:
+            Kind: VirtualConstProp
+            Info: 0
+...

llvm/trunk/test/Transforms/WholeProgramDevirt/export-vcp.ll

+; RUN: opt -wholeprogramdevirt -wholeprogramdevirt-summary-action=export -wholeprogramdevirt-read-summary=%S/Inputs/export.yaml -wholeprogramdevirt-write-summary=%t -S -o - %s | FileCheck %s
+; RUN: FileCheck --check-prefix=SUMMARY %s < %t
+
+target datalayout = "e-p:64:64"
+target triple = "x86_64-unknown-linux-gnu"
+
+; SUMMARY:      TypeIdMap:
+; SUMMARY-NEXT:   typeid3:
+; SUMMARY-NEXT:     TTRes:
+; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       SizeM1BitWidth:  0
+; SUMMARY-NEXT:     WPDRes:
+; SUMMARY-NEXT:       0:
+; SUMMARY-NEXT:         Kind:            Indir
+; SUMMARY-NEXT:         SingleImplName:  ''
+; SUMMARY-NEXT:         ResByArg:
+; SUMMARY-NEXT:           12,24:
+; SUMMARY-NEXT:             Kind:            VirtualConstProp
+; SUMMARY-NEXT:             Info:            0
+; SUMMARY-NEXT:   typeid4:
+; SUMMARY-NEXT:     TTRes:
+; SUMMARY-NEXT:       Kind:            Unsat
+; SUMMARY-NEXT:       SizeM1BitWidth:  0
+; SUMMARY-NEXT:     WPDRes:
+; SUMMARY-NEXT:       0:
+; SUMMARY-NEXT:         Kind:            Indir
+; SUMMARY-NEXT:         SingleImplName:  ''
+; SUMMARY-NEXT:         ResByArg:
+; SUMMARY-NEXT:           24,12:
+; SUMMARY-NEXT:             Kind:            VirtualConstProp
+; SUMMARY-NEXT:             Info:            0
+
+; CHECK: [[CVT3A:.*]] = private constant { [8 x i8], i1 (i8*, i32, i32)*, [0 x i8] } { [8 x i8] zeroinitializer, i1 (i8*, i32, i32)* @vf0i1, [0 x i8] zeroinitializer }, !type !0
+@vt3a = constant i1 (i8*, i32, i32)* @vf0i1, !type !0
+
+; CHECK: [[CVT3B:.*]] = private constant { [8 x i8], i1 (i8*, i32, i32)*, [0 x i8] } { [8 x i8] c"\00\00\00\00\00\00\00\01", i1 (i8*, i32, i32)* @vf1i1, [0 x i8] zeroinitializer }, !type !0
+@vt3b = constant i1 (i8*, i32, i32)* @vf1i1, !type !0
+
+; CHECK: [[CVT3C:.*]] = private constant { [8 x i8], i1 (i8*, i32, i32)*, [0 x i8] } { [8 x i8] zeroinitializer, i1 (i8*, i32, i32)* @vf0i1, [0 x i8] zeroinitializer }, !type !0
+@vt3c = constant i1 (i8*, i32, i32)* @vf0i1, !type !0
+
+; CHECK: [[CVT3D:.*]] = private constant { [8 x i8], i1 (i8*, i32, i32)*, [0 x i8] } { [8 x i8] c"\00\00\00\00\00\00\00\01", i1 (i8*, i32, i32)* @vf1i1, [0 x i8] zeroinitializer }, !type !0
+@vt3d = constant i1 (i8*, i32, i32)* @vf1i1, !type !0
+
+; CHECK: [[CVT4A:.*]] = private constant { [8 x i8], i32 (i8*, i32, i32)*, [0 x i8] } { [8 x i8] c"\00\00\00\00\01\00\00\00", i32 (i8*, i32, i32)* @vf1i32, [0 x i8] zeroinitializer }, !type !1
+@vt4a = constant i32 (i8*, i32, i32)* @vf1i32, !type !1
+
+; CHECK: [[CVT4B:.*]] = private constant { [8 x i8], i32 (i8*, i32, i32)*, [0 x i8] } { [8 x i8] c"\00\00\00\00\02\00\00\00", i32 (i8*, i32, i32)* @vf2i32, [0 x i8] zeroinitializer }, !type !1
+@vt4b = constant i32 (i8*, i32, i32)* @vf2i32, !type !1
+
+; CHECK: @__typeid_typeid3_0_12_24_byte = hidden alias i8, inttoptr (i32 -1 to i8*)
+; CHECK: @__typeid_typeid3_0_12_24_bit = hidden alias i8, inttoptr (i8 1 to i8*)
+; CHECK: @__typeid_typeid4_0_24_12_byte = hidden alias i8, inttoptr (i32 -4 to i8*)
+; CHECK: @__typeid_typeid4_0_24_12_bit = hidden alias i8, inttoptr (i8 1 to i8*)
+
+; CHECK: @vt3a = alias i1 (i8*, i32, i32)*, getelementptr inbounds ({ [8 x i8], i1 (i8*, i32, i32)*, [0 x i8] }, { [8 x i8], i1 (i8*, i32, i32)*, [0 x i8] }* [[CVT3A]], i32 0, i32 1)
+; CHECK: @vt3b = alias i1 (i8*, i32, i32)*, getelementptr inbounds ({ [8 x i8], i1 (i8*, i32, i32)*, [0 x i8] }, { [8 x i8], i1 (i8*, i32, i32)*, [0 x i8] }* [[CVT3B]], i32 0, i32 1)
+; CHECK: @vt3c = alias i1 (i8*, i32, i32)*, getelementptr inbounds ({ [8 x i8], i1 (i8*, i32, i32)*, [0 x i8] }, { [8 x i8], i1 (i8*, i32, i32)*, [0 x i8] }* [[CVT3C]], i32 0, i32 1)
+; CHECK: @vt3d = alias i1 (i8*, i32, i32)*, getelementptr inbounds ({ [8 x i8], i1 (i8*, i32, i32)*, [0 x i8] }, { [8 x i8], i1 (i8*, i32, i32)*, [0 x i8] }* [[CVT3D]], i32 0, i32 1)
+; CHECK: @vt4a = alias i32 (i8*, i32, i32)*, getelementptr inbounds ({ [8 x i8], i32 (i8*, i32, i32)*, [0 x i8] }, { [8 x i8], i32 (i8*, i32, i32)*, [0 x i8] }* [[CVT4A]], i32 0, i32 1)
+; CHECK: @vt4b = alias i32 (i8*, i32, i32)*, getelementptr inbounds ({ [8 x i8], i32 (i8*, i32, i32)*, [0 x i8] }, { [8 x i8], i32 (i8*, i32, i32)*, [0 x i8] }* [[CVT4B]], i32 0, i32 1)
+
+define i1 @vf0i1(i8* %this, i32, i32) readnone {
+  ret i1 0
+}
+
+define i1 @vf1i1(i8* %this, i32, i32) readnone {
+  ret i1 1
+}
+
+define i32 @vf1i32(i8* %this, i32, i32) readnone {
+  ret i32 1
+}
+
+define i32 @vf2i32(i8* %this, i32, i32) readnone {
+  ret i32 2
+}
+
+; CHECK: !0 = !{i32 8, !"typeid3"}
+; CHECK: !1 = !{i32 8, !"typeid4"}
+
+!0 = !{i32 0, !"typeid3"}
+!1 = !{i32 0, !"typeid4"}

llvm/trunk/test/Transforms/WholeProgramDevirt/import.ll

 ; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-single-impl.yaml < %s | FileCheck --check-prefixes=CHECK,SINGLE-IMPL %s
 ; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-uniform-ret-val.yaml < %s | FileCheck --check-prefixes=CHECK,UNIFORM-RET-VAL %s
 ; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-unique-ret-val0.yaml < %s | FileCheck --check-prefixes=CHECK,UNIQUE-RET-VAL0 %s
 ; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-unique-ret-val1.yaml < %s | FileCheck --check-prefixes=CHECK,UNIQUE-RET-VAL1 %s
+; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-vcp.yaml < %s | FileCheck --check-prefixes=CHECK,VCP,VCP64 %s
+; RUN: opt -S -wholeprogramdevirt -wholeprogramdevirt-summary-action=import -wholeprogramdevirt-read-summary=%S/Inputs/import-vcp.yaml -mtriple=i686-unknown-linux -data-layout=e-p:32:32 < %s | FileCheck --check-prefixes=CHECK,VCP,VCP32 %s
 
 target datalayout = "e-p:64:64"
 target triple = "x86_64-unknown-linux-gnu"
 
+; VCP: @__typeid_typeid1_0_1_byte = external hidden global i8, !absolute_symbol !0
+; VCP: @__typeid_typeid1_0_1_bit = external hidden global i8, !absolute_symbol !1
+; VCP: @__typeid_typeid2_8_3_byte = external hidden global i8, !absolute_symbol !0
+; VCP: @__typeid_typeid2_8_3_bit = external hidden global i8, !absolute_symbol !1
+
 ; Test cases where the argument values are known and we can apply virtual
 ; constant propagation.
 
 ; CHECK: define i32 @call1
 define i32 @call1(i8* %obj) {
   %vtableptr = bitcast i8* %obj to [3 x i8*]**
   %vtable = load [3 x i8*]*, [3 x i8*]** %vtableptr
   %vtablei8 = bitcast [3 x i8*]* %vtable to i8*
   %p = call i1 @llvm.type.test(i8* %vtablei8, metadata !"typeid1")
   call void @llvm.assume(i1 %p)
   %fptrptr = getelementptr [3 x i8*], [3 x i8*]* %vtable, i32 0, i32 0
   %fptr = load i8*, i8** %fptrptr
   %fptr_casted = bitcast i8* %fptr to i32 (i8*, i32)*
   ; SINGLE-IMPL: call i32 bitcast (void ()* @singleimpl1 to i32 (i8*, i32)*)
   %result = call i32 %fptr_casted(i8* %obj, i32 1)
   ; UNIFORM-RET-VAL: ret i32 42
+  ; VCP: [[VT1:%.*]] = bitcast {{.*}} to i8*
+  ; VCP: [[GEP1:%.*]] = getelementptr i8, i8* [[VT1]], i32 ptrtoint (i8* @__typeid_typeid1_0_1_byte to i32)
+  ; VCP: [[BC1:%.*]] = bitcast i8* [[GEP1]] to i32*
+  ; VCP: [[LOAD1:%.*]] = load i32, i32* [[BC1]]
+  ; VCP: ret i32 [[LOAD1]]
   ret i32 %result
 }
 
 ; Test cases where the argument values are unknown, so we cannot apply virtual
 ; constant propagation.
 
 ; CHECK: define i1 @call2
 define i1 @call2(i8* %obj) {
define i1 @call3(i8* %obj) {
   %p = extractvalue {i8*, i1} %pair, 1
   br i1 %p, label %cont, label %trap
 
 cont:
   %fptr_casted = bitcast i8* %fptr to i1 (i8*, i32)*
   %result = call i1 %fptr_casted(i8* %obj, i32 3)
   ; UNIQUE-RET-VAL0: icmp ne i8* %vtablei8, @__typeid_typeid2_8_3_unique_member
   ; UNIQUE-RET-VAL1: icmp eq i8* %vtablei8, @__typeid_typeid2_8_3_unique_member
+  ; VCP: [[VT2:%.*]] = bitcast {{.*}} to i8*
+  ; VCP: [[GEP2:%.*]] = getelementptr i8, i8* [[VT2]], i32 ptrtoint (i8* @__typeid_typeid2_8_3_byte to i32)
+  ; VCP: [[LOAD2:%.*]] = load i8, i8* [[GEP2]]
+  ; VCP: [[AND2:%.*]] = and i8 [[LOAD2]], ptrtoint (i8* @__typeid_typeid2_8_3_bit to i8)
+  ; VCP: [[ICMP2:%.*]] = icmp ne i8 [[AND2]], 0
+  ; VCP: ret i1 [[ICMP2]]
   ret i1 %result
 
 trap:
   call void @llvm.trap()
   unreachable
 }
 
 ; SINGLE-IMPL-DAG: declare void @singleimpl1()
 ; SINGLE-IMPL-DAG: declare void @singleimpl2()
 
+; VCP32: !0 = !{i32 -1, i32 -1}
+; VCP64: !0 = !{i64 0, i64 4294967296}
+
+; VCP32: !1 = !{i32 0, i32 256}
+; VCP64: !1 = !{i64 0, i64 256}
+
 declare void @llvm.assume(i1)
 declare void @llvm.trap()
 declare {i8*, i1} @llvm.type.checked.load(i8*, i32, metadata)
 declare i1 @llvm.type.test(i8*, metadata)