[CSSPGO] Unblock optimizations with pseudo probe instrumentation part 2.

[hoy]

Summary

As a follow-up to D95982, this patch continues unblocking optimizations that are blocked by pseudu probe instrumention.

The optimizations unblocked are:

• In-block load propagation.
• In-block dead store elimination
• Memory copy optimization that turns stores to consecutive memories into a memset.

These optimizations are local to a block, so they shouldn't affect the profile quality.

[hoy]

Updating D100075: [CSSPGO] Unblock optimizations with pseudo probe instrumentation part 2.

[hoy]

Rebasing.

[hoy]

Rebasing.

[wmi]

LGTM.

llvm/lib/Analysis/Loads.cpp

Value *llvm::findAvailablePtrLoadStore(
 
   const DataLayout &DL = ScanBB->getModule()->getDataLayout();
   const Value *StrippedPtr = Loc.Ptr->stripPointerCasts();
 
   while (ScanFrom != ScanBB->begin()) {
     // We must ignore debug info directives when counting (otherwise they
     // would affect codegen).
     Instruction *Inst = &*--ScanFrom;
-    if (isa<DbgInfoIntrinsic>(Inst))
+    if (Inst->isDebugOrPseudoInst())
       continue;
 
     // Restore ScanFrom to expected value in case next test succeeds
     ScanFrom++;
 
     if (NumScanedInst)
       ++(*NumScanedInst);
 
if (!Load->isUnordered())
     return nullptr;
 
   // Try to find an available value first, and delay expensive alias analysis
   // queries until later.
   Value *Available = nullptr;;
   SmallVector<Instruction *> MustNotAliasInsts;
   for (Instruction &Inst : make_range(++Load->getReverseIterator(),
                                       ScanBB->rend())) {
-    if (isa<DbgInfoIntrinsic>(&Inst))
+    if (Inst.isDebugOrPseudoInst())
       continue;
 
     if (MaxInstsToScan-- == 0)
       return nullptr;
 
     Available = getAvailableLoadStore(&Inst, StrippedPtr, AccessTy,
                                       AtLeastAtomic, DL, IsLoadCSE);
     if (Available)

llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp

Instruction *InstCombinerImpl::visitStoreInst(StoreInst &SI) {
   // stores to the same location, separated by a few arithmetic operations. This
   // situation often occurs with bitfield accesses.
   BasicBlock::iterator BBI(SI);
   for (unsigned ScanInsts = 6; BBI != SI.getParent()->begin() && ScanInsts;
        --ScanInsts) {
     --BBI;
     // Don't count debug info directives, lest they affect codegen,
     // and we skip pointer-to-pointer bitcasts, which are NOPs.
-    if (isa<DbgInfoIntrinsic>(BBI) ||
+    if (BBI->isDebugOrPseudoInst() ||
         (isa<BitCastInst>(BBI) && BBI->getType()->isPointerTy())) {
       ScanInsts++;
       continue;
     }
 
     if (StoreInst *PrevSI = dyn_cast<StoreInst>(BBI)) {
       // Prev store isn't volatile, and stores to the same location?
       if (PrevSI->isUnordered() && equivalentAddressValues(PrevSI->getOperand(1),

llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp

if (MSSAU) {
           MSSAU->getMemorySSA()->getMemoryAccess(&*BI));
       if (CurrentAcc) {
         MemInsertPoint = CurrentAcc;
         if (auto *CurrentDef = dyn_cast<MemoryDef>(CurrentAcc))
           LastMemDef = CurrentDef;
       }
     }
 
+    // Calls that only access inaccessible memory do not block merging
+    // accessible stores.
+    if (auto *CB = dyn_cast<CallBase>(BI)) {
+      if (CB->onlyAccessesInaccessibleMemory())
+        continue;
+    }
+
     if (!isa<StoreInst>(BI) && !isa<MemSetInst>(BI)) {
       // If the instruction is readnone, ignore it, otherwise bail out.  We
       // don't even allow readonly here because we don't want something like:
       // A[1] = 2; strlen(A); A[2] = 2; -> memcpy(A, ...); strlen(A).
       if (BI->mayWriteToMemory() || BI->mayReadFromMemory())
         break;
       continue;
     }

llvm/lib/Transforms/Scalar/Sink.cpp

do {
     Instruction *Inst = &*I; // The instruction to sink.
 
     // Predecrement I (if it's not begin) so that it isn't invalidated by
     // sinking.
     ProcessedBegin = I == BB.begin();
     if (!ProcessedBegin)
       --I;
 
-    if (isa<DbgInfoIntrinsic>(Inst))
+    if (Inst->isDebugOrPseudoInst())
       continue;
 
     if (SinkInstruction(Inst, Stores, DT, LI, AA)) {
       ++NumSunk;
       MadeChange = true;
     }
 
     // If we just processed the first instruction in the block, we're done.

llvm/test/Transforms/SampleProfile/pseudo-probe-instcombine.ll

-; RUN: opt -passes=instcombine -S < %s | FileCheck %s
+; RUN: opt -passes=instcombine -available-load-scan-limit=2 -S < %s | FileCheck %s
 
 %struct.nonbonded = type { [2 x %struct.CompAtom*], [2 x %struct.CompAtomExt*], [2 x %struct.CompAtom*], [2 x %class.Vector*], [2 x %class.Vector*], [2 x i32], %class.Vector, double*, double*, %class.ComputeNonbondedWorkArrays*, %class.Pairlists*, i32, i32, double, double, i32, i32, i32, i32 }
 %struct.CompAtomExt = type { i32 }
 %struct.CompAtom = type { %class.Vector, float, i16, i8, i8 }
 %class.Vector = type { double, double, double }
 %class.ComputeNonbondedWorkArrays = type { %class.ResizeArray, %class.ResizeArray.0, %class.ResizeArray, %class.ResizeArray, %class.ResizeArray, %class.ResizeArray, %class.ResizeArray, %class.ResizeArray, %class.ResizeArray, %class.ResizeArray, %class.ResizeArray, %class.ResizeArray, %class.ResizeArray, %class.ResizeArray, %class.ResizeArray, %class.ResizeArray, %class.ResizeArray, %class.ResizeArray.2, %class.ResizeArray.2 }
 %class.ResizeArray.0 = type { i32 (...)**, %class.ResizeArrayRaw.1* }
 %class.ResizeArrayRaw.1 = type <{ double*, i8*, i32, i32, i32, float, i32, [4 x i8] }>
 %class.ResizeArray = type { i32 (...)**, %class.ResizeArrayRaw* }
 %class.ResizeArrayRaw = type <{ i16*, i8*, i32, i32, i32, float, i32, [4 x i8] }>
 %class.ResizeArray.2 = type { i32 (...)**, %class.ResizeArrayRaw.3* }
 %class.ResizeArrayRaw.3 = type <{ %class.Vector*, i8*, i32, i32, i32, float, i32, [4 x i8] }>
 %class.Pairlists = type { i16*, i32, i32 }
 
+define dso_local void @merge(%struct.nonbonded* nocapture readonly %params) local_unnamed_addr align 2 {
 ;; Check the minPart4 and minPart assignments are merged.
+; CHECK-LABEL: @merge(
 ; CHECK-COUNT-1: getelementptr inbounds %struct.nonbonded, %struct.nonbonded* %params, i64 0, i32 16
 ; CHECK-NOT: getelementptr inbounds %struct.nonbonded, %struct.nonbonded* %params, i64 0, i32 16
-
-define dso_local void @_ZN20ComputeNonbondedUtil9calc_pairEP9nonbonded(%struct.nonbonded* nocapture readonly %params) local_unnamed_addr align 2 {
 entry:
   %savePairlists3 = getelementptr inbounds %struct.nonbonded, %struct.nonbonded* %params, i64 0, i32 11
   %0 = load i32, i32* %savePairlists3, align 8
   %usePairlists4 = getelementptr inbounds %struct.nonbonded, %struct.nonbonded* %params, i64 0, i32 12
   %1 = load i32, i32* %usePairlists4, align 4
   %tobool54.not = icmp eq i32 %0, 0
   br i1 %tobool54.not, label %lor.lhs.false55, label %if.end109
 

 
 if.then140:                                       ; preds = %if.then138
   ret void
 
 if.else147:                                       ; preds = %if.then138
   ret void
 }
 
-declare dso_local void @_ZN9Pairlists8addIndexEv() align 2
+define i32 @load(i32* nocapture %a, i32* nocapture %b) {
+;; Check the last store is deleted.
+; CHECK-LABEL: @load(
+; CHECK-NEXT:  %1 = getelementptr inbounds i32, i32* %a, i64 1
+; CHECK-NEXT:  %2 = load i32, i32* %1, align 8
+; CHECK-NEXT:  %3 = getelementptr inbounds i32, i32* %b, i64 1
+; CHECK-NEXT:       store i32 %2, i32* %3, align 8
+; CHECK-NEXT:    call void @llvm.pseudoprobe(i64 5116412291814990879, i64 1, i32 0, i64 -1)
+; CHECK-NEXT:    ret i32 %[[#]]
+  %1 = getelementptr inbounds i32, i32* %a, i32 1
+  %2 = load i32, i32* %1, align 8
+  %3 = getelementptr inbounds i32, i32* %b, i32 1
+       store i32 %2, i32* %3, align 8
+  %4 = getelementptr inbounds i32, i32* %b, i32 1
+  call void @llvm.pseudoprobe(i64 5116412291814990879, i64 1, i32 0, i64 -1)
+  %5 = load i32, i32* %4, align 8
+  ret i32 %5
+}
+
+define void @dse(i32* %p) {
+;; Check the first store is deleted.
+; CHECK-LABEL: @dse(
+; CHECK-NEXT:    call void @llvm.pseudoprobe(i64 5116412291814990879, i64 1, i32 0, i64 -1)
+; CHECK-NEXT:    store i32 0, i32* [[P:%.*]], align 4
+; CHECK-NEXT:    ret void
+  store i32 0, i32* %p
+  call void @llvm.pseudoprobe(i64 5116412291814990879, i64 1, i32 0, i64 -1)
+  store i32 0, i32* %p
+  ret void
+}
 
 ; Function Attrs: inaccessiblememonly nounwind willreturn
 declare void @llvm.pseudoprobe(i64, i64, i32, i64) #0
 
 attributes #0 = { inaccessiblememonly nounwind willreturn }

llvm/test/Transforms/SampleProfile/pseudo-probe-memset.ll

+; RUN: opt < %s -memcpyopt -S | FileCheck %s
+
+%struct.MV = type { i16, i16 }
+
+define void @test(i32* nocapture %c) nounwind optsize {
+; All the stores in this example should be merged into a single memset.
+; CHECK-NOT:  store i32 -1
+; CHECK: call void @llvm.memset.p0i8.i64
+  store i32 -1, i32* %c, align 4
+  %1 = getelementptr inbounds i32, i32* %c, i32 1
+  store i32 -1, i32* %1, align 4
+  %2 = getelementptr inbounds i32, i32* %c, i32 2
+  store i32 -1, i32* %2, align 4
+  call void @llvm.pseudoprobe(i64 5116412291814990879, i64 1, i32 0, i64 -1)
+  %3 = getelementptr inbounds i32, i32* %c, i32 3
+  store i32 -1, i32* %3, align 4
+  %4 = getelementptr inbounds i32, i32* %c, i32 4
+  store i32 -1, i32* %4, align 4
+  ret void
+}
+
+; Function Attrs: inaccessiblememonly nounwind willreturn
+declare void @llvm.pseudoprobe(i64, i64, i32, i64) #0
+
+attributes #0 = { inaccessiblememonly nounwind willreturn }