[SystemZ] Experimental: Improve getMemoryOpCost() to find foldable loads that are converted.

[jonpa]

Summary

The SystemZ backend can do arithmetic of memory by loading and then extending one of the operands. Similarly, a load + truncate can be folded into an operand.

This was not recognized by the SystemZ TTI cost functions. I found a loop where this was obvious and decided to do a more full-fledged version of the check for folding of memory operands.

As it turned out, this seems to be the only case where this triggered, so it does not seem to be of much use to apply this patch at the moment - not even on z14 which has the 16->64 bit memory operands also.

I suppose then we don't use this at least for the moment, but save it here for the record.

[uweigand]

This does look like a nice refactoring, so it might be worthwhile getting it in anyway, even if there's not much change in codegen ...

[jonpa]

Updated patch to also handle compare with pointer correctly by using the new static getScalarSizeInBits(Ty*) . Added CHECKs for the new cases in test file.

On z13, the number of loads getting 0 cost in cost evaluation of VF=1 goes up from 737 to 778. No change in output.

On z14, the number of loads getting 0 cost in cost evaluation of VF=1 goes up from 737 to 779. One loop (and file) changed in perlbench (pp_pack / S_unpack_rec / while.body720)

Loop before vectorize pass:

while.body720:                                    ; preds = %while.body720.preheader, %while.body720
%dec7171149 = phi i32 [ %dec717, %while.body720 ], [ %dec7171143, %while.body720.preheader ]
%s.addr.211148 = phi i8* [ %add.ptr721, %while.body720 ], [ %s.addr.1, %while.body720.preheader ]
%cuv.131146 = phi i64 [ %add736, %while.body720 ], [ %cuv.1, %while.body720.preheader ]
%ai16.0..sroa_cast248 = bitcast i8* %s.addr.211148 to i16*
%ai16.0.copyload = load i16, i16* %ai16.0..sroa_cast248, align 1
%add.ptr721 = getelementptr inbounds i8, i8* %s.addr.211148, i64 2
%conv735 = sext i16 %ai16.0.copyload to i64
%add736 = add i64 %cuv.131146, %conv735
%dec717 = add nsw i32 %dec7171149, -1
%cmp718 = icmp sgt i32 %dec7171149, 0
br i1 %cmp718, label %while.body720, label %sw.epilog1397.loopexit1323

Trunk selects VF=2:

renamable $v1 = VLREPF renamable $r5d, 0, $noreg :: (load 4 from %ir.lsr.iv24332435, align 1)
renamable $v1 = VUPHH killed renamable $v1
renamable $v1 = VUPHF killed renamable $v1
renamable $v0 = VAG killed renamable $v0, killed renamable $v1

while with this patch VF=1 is instead selected, since it can do just one AGH per iteration. This seems to make sense, and this loop is not apparently in any hot path, so I would assume we don't need to check benchmarks.

[uweigand]

LGTM, thanks!

[jonpa]

Thanks for review.

r345327.

lib/Target/SystemZ/SystemZTargetTransformInfo.h

public:
   int getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index, Type *SubTp);
   unsigned getVectorTruncCost(Type *SrcTy, Type *DstTy);
   unsigned getVectorBitmaskConversionCost(Type *SrcTy, Type *DstTy);
   int getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
                        const Instruction *I = nullptr);
   int getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy,
                          const Instruction *I = nullptr);
   int getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index);
+  bool isFoldableLoad(const LoadInst *Ld, const Instruction *&FoldedValue);
   int getMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment,
                       unsigned AddressSpace, const Instruction *I = nullptr);
 
   int getInterleavedMemoryOpCost(unsigned Opcode, Type *VecTy,
                                  unsigned Factor,
                                  ArrayRef<unsigned> Indices,
                                  unsigned Alignment,
                                  unsigned AddressSpace);
   /// @}
 };
 
 } // end namespace llvm
 
 #endif

lib/Target/SystemZ/SystemZTargetTransformInfo.cpp

if (Index == 0 && Val->isIntOrIntVectorTy())
       Cost += 1;
 
     return Cost;
   }
 
   return BaseT::getVectorInstrCost(Opcode, Val, Index);
 }
 
-int SystemZTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
-                                    unsigned Alignment, unsigned AddressSpace,
-                                    const Instruction *I) {
-  assert(!Src->isVoidTy() && "Invalid type");
-
-  if (!Src->isVectorTy() && Opcode == Instruction::Load &&
-      I != nullptr && I->hasOneUse()) {
-      const Instruction *UserI = cast<Instruction>(*I->user_begin());
-      unsigned Bits = getScalarSizeInBits(Src);
-      bool FoldsLoad = false;
-      switch (UserI->getOpcode()) {
-      case Instruction::ICmp:
-      case Instruction::Add:
-      case Instruction::Sub:
-      case Instruction::Mul:
-      case Instruction::SDiv:
-      case Instruction::UDiv:
-      case Instruction::And:
-      case Instruction::Or:
-      case Instruction::Xor:
-      // This also makes sense for float operations, but disabled for now due
-      // to regressions.
-      // case Instruction::FCmp:
-      // case Instruction::FAdd:
-      // case Instruction::FSub:
-      // case Instruction::FMul:
-      // case Instruction::FDiv:
-        FoldsLoad = (Bits == 32 || Bits == 64);
-        break;
-      }
+// Check if a load may be folded as a memory operand in its user.
+bool SystemZTTIImpl::
+isFoldableLoad(const LoadInst *Ld, const Instruction *&FoldedValue) {
+  if (!Ld->hasOneUse())
+    return false;
+  FoldedValue = Ld;
+  const Instruction *UserI = cast<Instruction>(*Ld->user_begin());
+  unsigned LoadedBits = getScalarSizeInBits(Ld->getType());
+  unsigned TruncBits = 0;
+  unsigned SExtBits = 0;
+  unsigned ZExtBits = 0;
+  if (UserI->hasOneUse()) {
+    unsigned UserBits = UserI->getType()->getScalarSizeInBits();
+    if (isa<TruncInst>(UserI))
+      TruncBits = UserBits;
+    else if (isa<SExtInst>(UserI))
+      SExtBits = UserBits;
+    else if (isa<ZExtInst>(UserI))
+      ZExtBits = UserBits;
+  }
+  if (TruncBits || SExtBits || ZExtBits) {
+    FoldedValue = UserI;
+    UserI = cast<Instruction>(*UserI->user_begin());
+    // Load (single use) -> trunc/extend (single use) -> UserI
+  }
+  case Instruction::Add: // SE: 16->32, 16/32->64, z14:16->64. ZE: 32->64
+    if (LoadedBits == 32 && ZExtBits == 64)
+      return true;
+    LLVM_FALLTHROUGH;
+  case Instruction::Mul: // SE: 16->32, 32->64, z14:16->64
+    if (LoadedBits == 16 &&
+        (SExtBits == 32 ||
+         (SExtBits == 64 && ST->hasMiscellaneousExtensions2())))
+      return true;
+    LLVM_FALLTHROUGH;
+  case Instruction::SDiv:// SE: 32->64
+    if (LoadedBits == 32 && SExtBits == 64)
+      return true;
+    LLVM_FALLTHROUGH;
+  case Instruction::ICmp:
+
+    // All possible extensions of memory checked above.
+    if (SExtBits || ZExtBits)
+      return false;
+
+    unsigned LoadOrTruncBits = (TruncBits ? TruncBits : LoadedBits);
+    return (LoadOrTruncBits == 32 || LoadOrTruncBits == 64);
+  return false;
+}
+
+int SystemZTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
+                                    unsigned Alignment, unsigned AddressSpace,
+                                    const Instruction *I) {
+  assert(!Src->isVoidTy() && "Invalid type");
 
-      if (FoldsLoad) {
-        assert (UserI->getNumOperands() == 2 &&
-                "Expected to only handle binops.");
+  if (!Src->isVectorTy() && Opcode == Instruction::Load && I != nullptr) {
+    // Store the load or its truncated or extended value in FoldedValue.
+    const Instruction *FoldedValue = nullptr;
+    if (isFoldableLoad(cast<LoadInst>(I), FoldedValue)) {
+      const Instruction *UserI = cast<Instruction>(*FoldedValue->user_begin());
+      assert (UserI->getNumOperands() == 2 && "Expected a binop.");
 
-        // UserI can't fold two loads, so in that case return 0 cost only
-        // half of the time.
-        for (unsigned i = 0; i < 2; ++i) {
-          if (UserI->getOperand(i) == I)
-            continue;
-          if (LoadInst *LI = dyn_cast<LoadInst>(UserI->getOperand(i))) {
-            if (LI->hasOneUse())
-              return i == 0;
-          }
-        }
+        if (UserI->getOperand(i) == FoldedValue)
+
+        if (Instruction *OtherOp = dyn_cast<Instruction>(UserI->getOperand(i))){
+          LoadInst *OtherLoad = dyn_cast<LoadInst>(OtherOp);
+          if (!OtherLoad &&
+              (isa<TruncInst>(OtherOp) || isa<SExtInst>(OtherOp) ||
+               isa<ZExtInst>(OtherOp)))
+            OtherLoad = dyn_cast<LoadInst>(OtherOp->getOperand(0));
+          if (OtherLoad && isFoldableLoad(OtherLoad, FoldedValue/*dummy*/))
+            return i == 0; // Both operands foldable.
 
-        return 0;
-      }
+      return 0; // Only I is foldable in user.
   }
 
   unsigned NumOps =
     (Src->isVectorTy() ? getNumVectorRegs(Src) : getNumberOfParts(Src));
 
   if (Src->getScalarSizeInBits() == 128)
     // 128 bit scalars are held in a pair of two 64 bit registers.
     NumOps *= 2;

test/Analysis/CostModel/SystemZ/memop-folding-int-arith.ll

-; RUN: opt < %s -cost-model -analyze -mtriple=systemz-unknown -mcpu=z13 | FileCheck %s
+; RUN: opt < %s -cost-model -analyze -mtriple=systemz-unknown -mcpu=z13 \
+; RUN:  | FileCheck %s -check-prefixes=CHECK,Z13
+; RUN: opt < %s -cost-model -analyze -mtriple=systemz-unknown -mcpu=z14 \
+; RUN:  | FileCheck %s -check-prefixes=CHECK,Z14
 ;
 ; Test that loads into operations that can fold one memory operand get zero
 ; cost. In the case that both operands are loaded, one load should get a cost
 ; value.
 
 define void @add() {
   %li32 = load i32, i32* undef
   add i32 %li32, undef
 
   %li32_0 = load i32, i32* undef
   %li32_1 = load i32, i32* undef
   add i32 %li32_0, %li32_1
 
   %li64 = load i64, i64* undef
   add i64 %li64, undef
 
   %li64_0 = load i64, i64* undef
   %li64_1 = load i64, i64* undef
   add i64 %li64_0, %li64_1
 
+  ; Truncated load
+  %li64_2 = load i64, i64* undef
+  %tr = trunc i64 %li64_2 to i32
+  add i32 %tr, undef
+
+  ; Sign-extended loads
+  %li16_0 = load i16, i16* undef
+  %sext_0 = sext i16 %li16_0 to i32
+  add i32 %sext_0, undef
+
+  %li16_1 = load i16, i16* undef
+  %sext_1 = sext i16 %li16_1 to i64
+  add i64 %sext_1, undef
+
+  %li32_2 = load i32, i32* undef
+  %sext_2 = sext i32 %li32_2 to i64
+  add i64 %sext_2, undef
+
+  ; Zero-extended loads
+  %li32_3 = load i32, i32* undef
+  %zext_0 = zext i32 %li32_3 to i64
+  add i64 %zext_0, undef
+
+  ; Loads with multiple uses are *not* folded
+  %li16_3 = load i16, i16* undef
+  %sext_3 = sext i16 %li16_3 to i32
+  %sext_4 = sext i16 %li16_3 to i32
+  add i32 %sext_3, undef
+
   ret void;
 
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li32 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %1 = add i32 %li32, undef
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li32_0 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li32_1 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %2 = add i32 %li32_0, %li32_1
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %3 = add i64 %li64, undef
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64_0 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li64_1 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %4 = add i64 %li64_0, %li64_1
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64_2 = load i64, i64* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %tr = trunc i64 %li64_2 to i32
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %5 = add i32 %tr, undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li16_0 = load i16, i16* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %sext_0 = sext i16 %li16_0 to i32
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %6 = add i32 %sext_0, undef
+; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %li16_1 = load i16, i16* undef
+; Z14:   Cost Model: Found an estimated cost of 0 for instruction:   %li16_1 = load i16, i16* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %sext_1 = sext i16 %li16_1 to i64
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %7 = add i64 %sext_1, undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li32_2 = load i32, i32* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %sext_2 = sext i32 %li32_2 to i64
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %8 = add i64 %sext_2, undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li32_3 = load i32, i32* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %zext_0 = zext i32 %li32_3 to i64
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %9 = add i64 %zext_0, undef
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li16_3 = load i16, i16* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %sext_3 = sext i16 %li16_3 to i32
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %sext_4 = sext i16 %li16_3 to i32
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %10 = add i32 %sext_3, undef
 }
 
 define void @sub() {
   %li32 = load i32, i32* undef
   sub i32 %li32, undef
 
   %li32_0 = load i32, i32* undef
   %li32_1 = load i32, i32* undef
   sub i32 %li32_0, %li32_1
 
   %li64 = load i64, i64* undef
   sub i64 %li64, undef
 
   %li64_0 = load i64, i64* undef
   %li64_1 = load i64, i64* undef
   sub i64 %li64_0, %li64_1
 
+  ; Truncated load
+  %li64_2 = load i64, i64* undef
+  %tr = trunc i64 %li64_2 to i32
+  sub i32 %tr, undef
+
+  ; Sign-extended loads
+  %li16_0 = load i16, i16* undef
+  %sext_0 = sext i16 %li16_0 to i32
+  sub i32 %sext_0, undef
+
+  %li16_1 = load i16, i16* undef
+  %sext_1 = sext i16 %li16_1 to i64
+  sub i64 %sext_1, undef
+
+  %li32_2 = load i32, i32* undef
+  %sext_2 = sext i32 %li32_2 to i64
+  sub i64 %sext_2, undef
+
+  ; Zero-extended loads
+  %li32_3 = load i32, i32* undef
+  %zext_0 = zext i32 %li32_3 to i64
+  sub i64 %zext_0, undef
+
+  ; Loads with multiple uses are *not* folded
+  %li16_3 = load i16, i16* undef
+  %sext_3 = sext i16 %li16_3 to i32
+  %sext_4 = sext i16 %li16_3 to i32
+  sub i32 %sext_3, undef
+
   ret void;
 
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li32 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %1 = sub i32 %li32, undef
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li32_0 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li32_1 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %2 = sub i32 %li32_0, %li32_1
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %3 = sub i64 %li64, undef
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64_0 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li64_1 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %4 = sub i64 %li64_0, %li64_1
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64_2 = load i64, i64* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %tr = trunc i64 %li64_2 to i32
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %5 = sub i32 %tr, undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li16_0 = load i16, i16* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %sext_0 = sext i16 %li16_0 to i32
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %6 = sub i32 %sext_0, undef
+; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %li16_1 = load i16, i16* undef
+; Z14:   Cost Model: Found an estimated cost of 0 for instruction:   %li16_1 = load i16, i16* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %sext_1 = sext i16 %li16_1 to i64
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %7 = sub i64 %sext_1, undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li32_2 = load i32, i32* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %sext_2 = sext i32 %li32_2 to i64
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %8 = sub i64 %sext_2, undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li32_3 = load i32, i32* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %zext_0 = zext i32 %li32_3 to i64
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %9 = sub i64 %zext_0, undef
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li16_3 = load i16, i16* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %sext_3 = sext i16 %li16_3 to i32
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %sext_4 = sext i16 %li16_3 to i32
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %10 = sub i32 %sext_3, undef
 }
 
 define void @mul() {
   %li32 = load i32, i32* undef
   mul i32 %li32, undef
 
   %li32_0 = load i32, i32* undef
   %li32_1 = load i32, i32* undef
   mul i32 %li32_0, %li32_1
 
   %li64 = load i64, i64* undef
   mul i64 %li64, undef
 
   %li64_0 = load i64, i64* undef
   %li64_1 = load i64, i64* undef
   mul i64 %li64_0, %li64_1
 
+  ; Truncated load
+  %li64_2 = load i64, i64* undef
+  %tr = trunc i64 %li64_2 to i32
+  mul i32 %tr, undef
+
+  ; Sign-extended loads
+  %li16_0 = load i16, i16* undef
+  %sext_0 = sext i16 %li16_0 to i32
+  mul i32 %sext_0, undef
+
+  %li16_1 = load i16, i16* undef
+  %sext_1 = sext i16 %li16_1 to i64
+  mul i64 %sext_1, undef
+
+  %li32_2 = load i32, i32* undef
+  %sext_2 = sext i32 %li32_2 to i64
+  mul i64 %sext_2, undef
+
+  ; Zero-extended loads are *not* folded
+  %li16_2 = load i16, i16* undef
+  %zext_0 = zext i16 %li16_2 to i32
+  mul i32 %zext_0, undef
+
+  ; Loads with multiple uses are *not* folded
+  %li16_3 = load i16, i16* undef
+  %sext_3 = sext i16 %li16_3 to i32
+  %sext_4 = sext i16 %li16_3 to i32
+  mul i32 %sext_3, undef
+
   ret void;
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li32 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %1 = mul i32 %li32, undef
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li32_0 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li32_1 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %2 = mul i32 %li32_0, %li32_1
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %3 = mul i64 %li64, undef
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64_0 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li64_1 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %4 = mul i64 %li64_0, %li64_1
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64_2 = load i64, i64* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %tr = trunc i64 %li64_2 to i32
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %5 = mul i32 %tr, undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li16_0 = load i16, i16* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %sext_0 = sext i16 %li16_0 to i32
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %6 = mul i32 %sext_0, undef
+; Z13:   Cost Model: Found an estimated cost of 1 for instruction:   %li16_1 = load i16, i16* undef
+; Z14:   Cost Model: Found an estimated cost of 0 for instruction:   %li16_1 = load i16, i16* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %sext_1 = sext i16 %li16_1 to i64
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %7 = mul i64 %sext_1, undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li32_2 = load i32, i32* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %sext_2 = sext i32 %li32_2 to i64
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %8 = mul i64 %sext_2, undef
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li16_2 = load i16, i16* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %zext_0 = zext i16 %li16_2 to i32
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %9 = mul i32 %zext_0, undef
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li16_3 = load i16, i16* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %sext_3 = sext i16 %li16_3 to i32
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %sext_4 = sext i16 %li16_3 to i32
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %10 = mul i32 %sext_3, undef
 }
 
 define void @sdiv() {
   %li32 = load i32, i32* undef
   sdiv i32 %li32, undef
 
   %li32_0 = load i32, i32* undef
   %li32_1 = load i32, i32* undef
   sdiv i32 %li32_0, %li32_1
 
   %li64 = load i64, i64* undef
   sdiv i64 %li64, undef
 
   %li64_0 = load i64, i64* undef
   %li64_1 = load i64, i64* undef
   sdiv i64 %li64_0, %li64_1
 
+  ; Truncated load
+  %li64_2 = load i64, i64* undef
+  %tr = trunc i64 %li64_2 to i32
+  sdiv i32 %tr, undef
+
+  ; Sign-extended loads
+  %li32_2 = load i32, i32* undef
+  %sext_0 = sext i32 %li32_2 to i64
+  sdiv i64 %sext_0, undef
+
+  ; Loads with multiple uses are *not* folded
+  %li32_3 = load i32, i32* undef
+  %sext_1 = sext i32 %li32_3 to i64
+  %sext_2 = sext i32 %li32_3 to i64
+  sdiv i64 %sext_1, undef
+
   ret void;
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li32 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 2 for instruction:   %1 = sdiv i32 %li32, undef
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li32_0 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li32_1 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 2 for instruction:   %2 = sdiv i32 %li32_0, %li32_1
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %3 = sdiv i64 %li64, undef
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64_0 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li64_1 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %4 = sdiv i64 %li64_0, %li64_1
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64_2 = load i64, i64* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %tr = trunc i64 %li64_2 to i32
+; CHECK: Cost Model: Found an estimated cost of 2 for instruction:   %5 = sdiv i32 %tr, undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li32_2 = load i32, i32* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %sext_0 = sext i32 %li32_2 to i64
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %6 = sdiv i64 %sext_0, undef
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li32_3 = load i32, i32* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %sext_1 = sext i32 %li32_3 to i64
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %sext_2 = sext i32 %li32_3 to i64
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %7 = sdiv i64 %sext_1, undef
 }
 
 define void @udiv() {
   %li32 = load i32, i32* undef
   udiv i32 %li32, undef
 
   %li32_0 = load i32, i32* undef
   %li32_1 = load i32, i32* undef
   udiv i32 %li32_0, %li32_1
 
   %li64 = load i64, i64* undef
   udiv i64 %li64, undef
 
   %li64_0 = load i64, i64* undef
   %li64_1 = load i64, i64* undef
   udiv i64 %li64_0, %li64_1
 
+  ; Truncated load
+  %li64_2 = load i64, i64* undef
+  %tr_0 = trunc i64 %li64_2 to i32
+  udiv i32 %tr_0, undef
+
+  ; Loads with multiple uses are *not* folded
+  %li64_3 = load i64, i64* undef
+  %tr_1 = trunc i64 %li64_3 to i32
+  udiv i64 %li64_3, undef
+
   ret void;
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li32 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 2 for instruction:   %1 = udiv i32 %li32, undef
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li32_0 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li32_1 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 2 for instruction:   %2 = udiv i32 %li32_0, %li32_1
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 2 for instruction:   %3 = udiv i64 %li64, undef
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64_0 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li64_1 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 2 for instruction:   %4 = udiv i64 %li64_0, %li64_1
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64_2 = load i64, i64* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %tr_0 = trunc i64 %li64_2 to i32
+; CHECK: Cost Model: Found an estimated cost of 2 for instruction:   %5 = udiv i32 %tr_0, undef
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li64_3 = load i64, i64* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %tr_1 = trunc i64 %li64_3 to i32
+; CHECK: Cost Model: Found an estimated cost of 2 for instruction:   %6 = udiv i64 %li64_3, undef
 }
 
 define void @and() {
   %li32 = load i32, i32* undef
   and i32 %li32, undef
 
   %li32_0 = load i32, i32* undef
   %li32_1 = load i32, i32* undef
   and i32 %li32_0, %li32_1
 
   %li64 = load i64, i64* undef
   and i64 %li64, undef
 
   %li64_0 = load i64, i64* undef
   %li64_1 = load i64, i64* undef
   and i64 %li64_0, %li64_1
 
+  ; Truncated load
+  %li64_2 = load i64, i64* undef
+  %tr_0 = trunc i64 %li64_2 to i32
+  and i32 %tr_0, undef
+
+  ; Loads with multiple uses are *not* folded
+  %li64_3 = load i64, i64* undef
+  %tr_1 = trunc i64 %li64_3 to i32
+  and i64 %li64_3, undef
+
   ret void;
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li32 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %1 = and i32 %li32, undef
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li32_0 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li32_1 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %2 = and i32 %li32_0, %li32_1
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %3 = and i64 %li64, undef
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64_0 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li64_1 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %4 = and i64 %li64_0, %li64_1
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64_2 = load i64, i64* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %tr_0 = trunc i64 %li64_2 to i32
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %5 = and i32 %tr_0, undef
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li64_3 = load i64, i64* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %tr_1 = trunc i64 %li64_3 to i32
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %6 = and i64 %li64_3, undef
 }
 
 define void @or() {
   %li32 = load i32, i32* undef
   or i32 %li32, undef
 
   %li32_0 = load i32, i32* undef
   %li32_1 = load i32, i32* undef
   or i32 %li32_0, %li32_1
 
   %li64 = load i64, i64* undef
   or i64 %li64, undef
 
   %li64_0 = load i64, i64* undef
   %li64_1 = load i64, i64* undef
   or i64 %li64_0, %li64_1
 
+  ; Truncated load
+  %li64_2 = load i64, i64* undef
+  %tr_0 = trunc i64 %li64_2 to i32
+  or i32 %tr_0, undef
+
+  ; Loads with multiple uses are *not* folded
+  %li64_3 = load i64, i64* undef
+  %tr_1 = trunc i64 %li64_3 to i32
+  or i64 %li64_3, undef
+
   ret void;
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li32 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %1 = or i32 %li32, undef
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li32_0 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li32_1 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %2 = or i32 %li32_0, %li32_1
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %3 = or i64 %li64, undef
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64_0 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li64_1 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %4 = or i64 %li64_0, %li64_1
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64_2 = load i64, i64* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %tr_0 = trunc i64 %li64_2 to i32
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %5 = or i32 %tr_0, undef
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li64_3 = load i64, i64* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %tr_1 = trunc i64 %li64_3 to i32
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %6 = or i64 %li64_3, undef
 }
 
 define void @xor() {
   %li32 = load i32, i32* undef
   xor i32 %li32, undef
 
   %li32_0 = load i32, i32* undef
   %li32_1 = load i32, i32* undef
   xor i32 %li32_0, %li32_1
 
   %li64 = load i64, i64* undef
   xor i64 %li64, undef
 
   %li64_0 = load i64, i64* undef
   %li64_1 = load i64, i64* undef
   xor i64 %li64_0, %li64_1
 
+  ; Truncated load
+  %li64_2 = load i64, i64* undef
+  %tr_0 = trunc i64 %li64_2 to i32
+  xor i32 %tr_0, undef
+
+  ; Loads with multiple uses are *not* folded
+  %li64_3 = load i64, i64* undef
+  %tr_1 = trunc i64 %li64_3 to i32
+  xor i64 %li64_3, undef
+
   ret void;
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li32 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %1 = xor i32 %li32, undef
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li32_0 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li32_1 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %2 = xor i32 %li32_0, %li32_1
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %3 = xor i64 %li64, undef
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64_0 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li64_1 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %4 = xor i64 %li64_0, %li64_1
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64_2 = load i64, i64* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %tr_0 = trunc i64 %li64_2 to i32
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %5 = xor i32 %tr_0, undef
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li64_3 = load i64, i64* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %tr_1 = trunc i64 %li64_3 to i32
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %6 = xor i64 %li64_3, undef
 }
 
 define void @icmp() {
   %li32 = load i32, i32* undef
   icmp eq i32 %li32, undef
 
   %li32_0 = load i32, i32* undef
   %li32_1 = load i32, i32* undef
   icmp eq i32 %li32_0, %li32_1
 
   %li64 = load i64, i64* undef
   icmp eq i64 %li64, undef
 
   %li64_0 = load i64, i64* undef
   %li64_1 = load i64, i64* undef
   icmp eq i64 %li64_0, %li64_1
 
+  ; Truncated load
+  %li64_2 = load i64, i64* undef
+  %tr_0 = trunc i64 %li64_2 to i32
+  icmp eq i32 %tr_0, undef
+
+  ; Loads with multiple uses are *not* folded
+  %li64_3 = load i64, i64* undef
+  %tr_1 = trunc i64 %li64_3 to i32
+  icmp eq i64 %li64_3, undef
+
   ret void;
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li32 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %1 = icmp eq i32 %li32, undef
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li32_0 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li32_1 = load i32, i32* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %2 = icmp eq i32 %li32_0, %li32_1
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %3 = icmp eq i64 %li64, undef
 ; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64_0 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li64_1 = load i64, i64* undef
 ; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %4 = icmp eq i64 %li64_0, %li64_1
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %li64_2 = load i64, i64* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %tr_0 = trunc i64 %li64_2 to i32
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %5 = icmp eq i32 %tr_0, undef
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %li64_3 = load i64, i64* undef
+; CHECK: Cost Model: Found an estimated cost of 0 for instruction:   %tr_1 = trunc i64 %li64_3 to i32
+; CHECK: Cost Model: Found an estimated cost of 1 for instruction:   %6 = icmp eq i64 %li64_3, undef
 }