[SystemZ] Experimental: Improve getMemoryOpCost() to find foldable loads that are converted.

[jonpa]

Summary

The SystemZ backend can do arithmetic of memory by loading and then extending one of the operands. Similarly, a load + truncate can be folded into an operand.

This was not recognized by the SystemZ TTI cost functions. I found a loop where this was obvious and decided to do a more full-fledged version of the check for folding of memory operands.

As it turned out, this seems to be the only case where this triggered, so it does not seem to be of much use to apply this patch at the moment - not even on z14 which has the 16->64 bit memory operands also.

I suppose then we don't use this at least for the moment, but save it here for the record.

[uweigand]

This does look like a nice refactoring, so it might be worthwhile getting it in anyway, even if there's not much change in codegen ...

[jonpa]

Updated patch to also handle compare with pointer correctly by using the new static getScalarSizeInBits(Ty*) . Added CHECKs for the new cases in test file.

On z13, the number of loads getting 0 cost in cost evaluation of VF=1 goes up from 737 to 778. No change in output.

On z14, the number of loads getting 0 cost in cost evaluation of VF=1 goes up from 737 to 779. One loop (and file) changed in perlbench (pp_pack / S_unpack_rec / while.body720)

Loop before vectorize pass:

while.body720:                                    ; preds = %while.body720.preheader, %while.body720
%dec7171149 = phi i32 [ %dec717, %while.body720 ], [ %dec7171143, %while.body720.preheader ]
%s.addr.211148 = phi i8* [ %add.ptr721, %while.body720 ], [ %s.addr.1, %while.body720.preheader ]
%cuv.131146 = phi i64 [ %add736, %while.body720 ], [ %cuv.1, %while.body720.preheader ]
%ai16.0..sroa_cast248 = bitcast i8* %s.addr.211148 to i16*
%ai16.0.copyload = load i16, i16* %ai16.0..sroa_cast248, align 1
%add.ptr721 = getelementptr inbounds i8, i8* %s.addr.211148, i64 2
%conv735 = sext i16 %ai16.0.copyload to i64
%add736 = add i64 %cuv.131146, %conv735
%dec717 = add nsw i32 %dec7171149, -1
%cmp718 = icmp sgt i32 %dec7171149, 0
br i1 %cmp718, label %while.body720, label %sw.epilog1397.loopexit1323

Trunk selects VF=2:

renamable $v1 = VLREPF renamable $r5d, 0, $noreg :: (load 4 from %ir.lsr.iv24332435, align 1)
renamable $v1 = VUPHH killed renamable $v1
renamable $v1 = VUPHF killed renamable $v1
renamable $v0 = VAG killed renamable $v0, killed renamable $v1

while with this patch VF=1 is instead selected, since it can do just one AGH per iteration. This seems to make sense, and this loop is not apparently in any hot path, so I would assume we don't need to check benchmarks.

[uweigand]

LGTM, thanks!

[jonpa]

Thanks for review.

r345327.

lib/Target/SystemZ/SystemZTargetTransformInfo.h

public:
   int getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index, Type *SubTp);
   unsigned getVectorTruncCost(Type *SrcTy, Type *DstTy);
   unsigned getVectorBitmaskConversionCost(Type *SrcTy, Type *DstTy);
   int getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                        const Instruction *I = nullptr);
   int getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
                          const Instruction *I = nullptr);
   int getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index);
+  bool isFoldableLoad(const LoadInst *Ld, const Instruction *&FoldedValue);
   int getMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment,
                       unsigned AddressSpace, const Instruction *I = nullptr);
 
   int getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy,
                                  unsigned Factor,
                                  ArrayRef<unsigned> Indices,
                                  unsigned Alignment,
                                  unsigned AddressSpace);
   /// @}
 };
 
 } // end namespace llvm
 
 #endif

lib/Target/SystemZ/SystemZTargetTransformInfo.cpp

if (Index == 0 && Val->isIntOrIntVectorTy())
       Cost += 1;
 
     return Cost;
   }
 
   return BaseT::getVectorInstrCost(Opcode, Val, Index);
 }
 
-int SystemZTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
-                                    unsigned Alignment, unsigned AddressSpace,
-                                    const Instruction *I) {
-  assert(!Src->isVoidTy() && "Invalid type");
-
-  if (!Src->isVectorTy() && Opcode == Instruction::Load &&
-      I != nullptr && I->hasOneUse()) {
-      const Instruction *UserI = cast<Instruction>(*I->user_begin());
-      unsigned Bits = Src->getScalarSizeInBits();
-      bool FoldsLoad = false;
-      switch (UserI->getOpcode()) {
-      case Instruction::ICmp:
-      case Instruction::Add:
-      case Instruction::Sub:
-      case Instruction::Mul:
-      case Instruction::SDiv:
-      case Instruction::UDiv:
-      case Instruction::And:
-      case Instruction::Or:
-      case Instruction::Xor:
-      // This also makes sense for float operations, but disabled for now due
-      // to regressions.
-      // case Instruction::FCmp:
-      // case Instruction::FAdd:
-      // case Instruction::FSub:
-      // case Instruction::FMul:
-      // case Instruction::FDiv:
-        FoldsLoad = (Bits == 32 || Bits == 64);
-        break;
-      }
+// Check if a load may be folded as a memory operand in its user.
+bool SystemZTTIImpl::
+isFoldableLoad(const LoadInst *Ld, const Instruction *&FoldedValue) {
+  if (!Ld->hasOneUse())
+    return false;
+  FoldedValue = Ld;
+  const Instruction *UserI = cast<Instruction>(*Ld->user_begin());
+  unsigned LoadedBits = Ld->getType()->getScalarSizeInBits();
+  unsigned TruncBits = 0;
+  unsigned SExtBits = 0;
+  unsigned ZExtBits = 0;
+  if (UserI->hasOneUse()) {
+    unsigned UserBits = UserI->getType()->getScalarSizeInBits();
+    if (isa<TruncInst>(UserI))
+      TruncBits = UserBits;
+    else if (isa<SExtInst>(UserI))
+      SExtBits = UserBits;
+    else if (isa<ZExtInst>(UserI))
+      ZExtBits = UserBits;
+  }
+  if (TruncBits || SExtBits || ZExtBits) {
+    FoldedValue = UserI;
+    UserI = cast<Instruction>(*UserI->user_begin());
+    // Load (single use) -> trunc/extend (single use) -> UserI
+  }
+  case Instruction::Add: // SE: 16->32, 16/32->64. ZE: 32->64
+  case Instruction::Sub: // SE: 16->32, 16/32->64. ZE: 32->64
+    if (LoadedBits == 32 && ZExtBits == 64)
+      return true;
+    LLVM_FALLTHROUGH;
+  case Instruction::Mul: // SE: 16->32, 16/32->64.
+    if (LoadedBits == 16 &&
+        (SExtBits == 32 ||
+         (SExtBits == 64  && ST->hasMiscellaneousExtensions2())))
+        return true;
+    LLVM_FALLTHROUGH;
+  case Instruction::SDiv:// SE: 32->64
+    if (LoadedBits == 32 && SExtBits == 64)
+      return true;
+    LLVM_FALLTHROUGH;
+  case Instruction::ICmp:
+
+    // All possible extensions of memory checked above.
+    if (SExtBits || ZExtBits)
+      return false;
+
+    unsigned LoadOrTruncBits = (TruncBits ? TruncBits : LoadedBits);
+    return (LoadOrTruncBits == 32 || LoadOrTruncBits == 64);
+  return false;
+}
 
-      if (FoldsLoad) {
-        assert (UserI->getNumOperands() == 2 &&
-                "Expected to only handle binops.");
+int SystemZTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
+                                    unsigned Alignment, unsigned AddressSpace,
+                                    const Instruction *I) {
+  assert(!Src->isVoidTy() && "Invalid type");
+
+  if (!Src->isVectorTy() && Opcode == Instruction::Load && I != nullptr) {
+    // Store the load or its truncated or extended value in FoldedValue.
+    const Instruction *FoldedValue = nullptr;
+    if (isFoldableLoad(cast<LoadInst>(I), FoldedValue)) {
+      const Instruction *UserI = cast<Instruction>(*FoldedValue->user_begin());
+      assert (UserI->getNumOperands() == 2 && "Expected a binop.");
 
-        // UserI can't fold two loads, so in that case return 0 cost only
-        // half of the time.
-        for (unsigned i = 0; i < 2; ++i) {
-          if (UserI->getOperand(i) == I)
-            continue;
-          if (LoadInst *LI = dyn_cast<LoadInst>(UserI->getOperand(i))) {
-            if (LI->hasOneUse())
-              return i == 0;
-          }
-        }
+        if (UserI->getOperand(i) == FoldedValue)
+
+        if (Instruction *OtherOp = dyn_cast<Instruction>(UserI->getOperand(i))){
+          LoadInst *OtherLoad = dyn_cast<LoadInst>(OtherOp);
+          if (!OtherLoad &&
+              (isa<TruncInst>(OtherOp) || isa<SExtInst>(OtherOp) ||
+               isa<ZExtInst>(OtherOp)))
+            OtherLoad = dyn_cast<LoadInst>(OtherOp->getOperand(0));
+          if (OtherLoad && isFoldableLoad(OtherLoad, FoldedValue/*dummy*/))
+            return i == 0; // Both operands foldable.
 
-        return 0;
-      }
+      return 0; // Only I is foldable in user.
   }
 
   unsigned NumOps = getNumberOfParts(Src);
 
   if (Src->getScalarSizeInBits() == 128)
     // 128 bit scalars are held in a pair of two 64 bit registers.
     NumOps *= 2;