[InstCombine] don't widen an arbitrary sequence of vector ops (PR40032)

[spatel]

Summary

The problem is shown specifically for a case with vector multiply here:
https://bugs.llvm.org/show_bug.cgi?id=40032
...and this might mask the original backend bug for ARM shown in:
https://bugs.llvm.org/show_bug.cgi?id=39967

As the test diffs here show, we were (and probably still aren't) doing these kinds of transforms in a principled way. We are producing more or equal wide instructions than we started with in some cases, so we still need to restrict/correct other transforms from overstepping.

If there are perf regressions from this change, we can either carve out exceptions to the general IR rules, or improve the backend to do these transforms when we know the transform is profitable. That's probably similar to a change I just made in D55448.

[RKSimon]

lib/Transforms/InstCombine/InstCombineCasts.cpp

1112

What about putting !DestTy->isVectorTy() inside InstCombiner::shouldChangeType ?

[spatel]

lib/Transforms/InstCombine/InstCombineCasts.cpp

1112

Yes, that's cleaner - currently we just assert that we have scalar inside there.

[spatel]

Patch updated:
Move vector type check into shouldChangeType() rather than asserting we have scalars.

[RKSimon]

No more comments from me

[efriedma]

LGTM

lib/Transforms/InstCombine/InstCombineCasts.cpp

Instruction *InstCombiner::visitZExt(ZExtInst &CI) {
 
   // If one of the common conversion will work, do it.
   if (Instruction *Result = commonCastTransforms(CI))
     return Result;
 
   Value *Src = CI.getOperand(0);
   Type *SrcTy = Src->getType(), *DestTy = CI.getType();
 
-  // Attempt to extend the entire input expression tree to the destination
-  // type.   Only do this if the dest type is a simple type, don't convert the
-  // expression tree to something weird like i93 unless the source is also
-  // strange.
+  // Try to extend the entire expression tree to the wide destination type.
   unsigned BitsToClear;
-  if ((DestTy->isVectorTy() || shouldChangeType(SrcTy, DestTy)) &&
+  if (shouldChangeType(SrcTy, DestTy) &&
       canEvaluateZExtd(Src, DestTy, BitsToClear, *this, &CI)) {
     assert(BitsToClear <= SrcTy->getScalarSizeInBits() &&
            "Can't clear more bits than in SrcTy");
 
     // Okay, we can transform this!  Insert the new expression now.
     LLVM_DEBUG(
         dbgs() << "ICE: EvaluateInDifferentType converting expression type"
                   " to avoid zero extend: "
Instruction *InstCombiner::visitSExt(SExtInst &CI) {
   // If we know that the value being extended is positive, we can use a zext
   // instead.
   KnownBits Known = computeKnownBits(Src, 0, &CI);
   if (Known.isNonNegative()) {
     Value *ZExt = Builder.CreateZExt(Src, DestTy);
     return replaceInstUsesWith(CI, ZExt);
   }
 
-  // Attempt to extend the entire input expression tree to the destination
-  // type.   Only do this if the dest type is a simple type, don't convert the
-  // expression tree to something weird like i93 unless the source is also
-  // strange.
-  if ((DestTy->isVectorTy() || shouldChangeType(SrcTy, DestTy)) &&
-      canEvaluateSExtd(Src, DestTy)) {
+  // Try to extend the entire expression tree to the wide destination type.
+  if (shouldChangeType(SrcTy, DestTy) && canEvaluateSExtd(Src, DestTy)) {
     // Okay, we can transform this!  Insert the new expression now.
     LLVM_DEBUG(
         dbgs() << "ICE: EvaluateInDifferentType converting expression type"
                   " to avoid sign extend: "
                << CI << '\n');
     Value *Res = EvaluateInDifferentType(Src, DestTy, true);
     assert(Res->getType() == DestTy);
 

lib/Transforms/InstCombine/InstructionCombining.cpp

 }
 
 /// Return true if it is desirable to convert a computation from 'From' to 'To'.
 /// We don't want to convert from a legal to an illegal type or from a smaller
 /// to a larger illegal type. i1 is always treated as a legal type because it is
 /// a fundamental type in IR, and there are many specialized optimizations for
 /// i1 types.
 bool InstCombiner::shouldChangeType(Type *From, Type *To) const {
-  assert(From->isIntegerTy() && To->isIntegerTy());
+  // TODO: This could be extended to allow vectors. Datalayout changes might be
+  // needed to properly support that.
+  if (!From->isIntegerTy() || !To->isIntegerTy())
+    return false;
 
   unsigned FromWidth = From->getPrimitiveSizeInBits();
   unsigned ToWidth = To->getPrimitiveSizeInBits();
   return shouldChangeType(FromWidth, ToWidth);
 }
 
 // Return true, if No Signed Wrap should be maintained for I.
 // The No Signed Wrap flag can be kept if the operation "B (I.getOpcode) C",

test/Transforms/InstCombine/cast.ll

;
   %C = and i32 %B, 42
   %D = shl i32 %C, 8
   %E = zext i32 %D to i64
   ret i64 %E
 }
 
 define <2 x i64> @test46vec(<2 x i64> %A) {
 ; CHECK-LABEL: @test46vec(
-; CHECK-NEXT:    [[C:%.*]] = shl <2 x i64> [[A:%.*]], <i64 8, i64 8>
-; CHECK-NEXT:    [[D:%.*]] = and <2 x i64> [[C]], <i64 10752, i64 10752>
-; CHECK-NEXT:    ret <2 x i64> [[D]]
+; CHECK-NEXT:    [[B:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[C:%.*]] = shl <2 x i32> [[B]], <i32 8, i32 8>
+; CHECK-NEXT:    [[D:%.*]] = and <2 x i32> [[C]], <i32 10752, i32 10752>
+; CHECK-NEXT:    [[E:%.*]] = zext <2 x i32> [[D]] to <2 x i64>
+; CHECK-NEXT:    ret <2 x i64> [[E]]
 ;
   %B = trunc <2 x i64> %A to <2 x i32>
   %C = and <2 x i32> %B, <i32 42, i32 42>
   %D = shl <2 x i32> %C, <i32 8, i32 8>
   %E = zext <2 x i32> %D to <2 x i64>
   ret <2 x i64> %E
 }
 
;
   %p353 = sext i16 %A to i32
   %p354 = lshr i32 %p353, 5
   %p355 = zext i32 %p354 to i64
   ret i64 %p355
 }
 
 define <2 x i64> @test56vec(<2 x i16> %A) {
 ; CHECK-LABEL: @test56vec(
-; CHECK-NEXT:    [[P353:%.*]] = sext <2 x i16> [[A:%.*]] to <2 x i64>
-; CHECK-NEXT:    [[P354:%.*]] = lshr <2 x i64> [[P353]], <i64 5, i64 5>
-; CHECK-NEXT:    [[P355:%.*]] = and <2 x i64> [[P354]], <i64 134217727, i64 134217727>
+; CHECK-NEXT:    [[P353:%.*]] = sext <2 x i16> [[A:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[P354:%.*]] = lshr <2 x i32> [[P353]], <i32 5, i32 5>
+; CHECK-NEXT:    [[P355:%.*]] = zext <2 x i32> [[P354]] to <2 x i64>
 ; CHECK-NEXT:    ret <2 x i64> [[P355]]
 ;
   %p353 = sext <2 x i16> %A to <2 x i32>
   %p354 = lshr <2 x i32> %p353, <i32 5, i32 5>
   %p355 = zext <2 x i32> %p354 to <2 x i64>
   ret <2 x i64> %p355
 }
 
 define i64 @test57(i64 %A) {
 ; CHECK-LABEL: @test57(
 ; CHECK-NEXT:    [[C:%.*]] = lshr i64 [[A:%.*]], 8
 ; CHECK-NEXT:    [[E:%.*]] = and i64 [[C]], 16777215
 ; CHECK-NEXT:    ret i64 [[E]]
 ;
   %B = trunc i64 %A to i32
   %C = lshr i32 %B, 8
   %E = zext i32 %C to i64
   ret i64 %E
 }
 
 define <2 x i64> @test57vec(<2 x i64> %A) {
 ; CHECK-LABEL: @test57vec(
-; CHECK-NEXT:    [[C:%.*]] = lshr <2 x i64> [[A:%.*]], <i64 8, i64 8>
-; CHECK-NEXT:    [[E:%.*]] = and <2 x i64> [[C]], <i64 16777215, i64 16777215>
+; CHECK-NEXT:    [[B:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[C:%.*]] = lshr <2 x i32> [[B]], <i32 8, i32 8>
+; CHECK-NEXT:    [[E:%.*]] = zext <2 x i32> [[C]] to <2 x i64>
 ; CHECK-NEXT:    ret <2 x i64> [[E]]
 ;
   %B = trunc <2 x i64> %A to <2 x i32>
   %C = lshr <2 x i32> %B, <i32 8, i32 8>
   %E = zext <2 x i32> %C to <2 x i64>
   ret <2 x i64> %E
 }
 

test/Transforms/InstCombine/select-bitext.ll

;
   %trunc = trunc i32 %a to i16
   %sel = select i1 %cmp, i16 %trunc, i16 42
   %ext = sext i16 %sel to i64
   ret i64 %ext
 }
 
 define <2 x i64> @trunc_sel_larger_sext_vec(<2 x i32> %a, <2 x i1> %cmp) {
 ; CHECK-LABEL: @trunc_sel_larger_sext_vec(
-; CHECK-NEXT:    [[TRUNC:%.*]] = zext <2 x i32> [[A:%.*]] to <2 x i64>
-; CHECK-NEXT:    [[SEXT:%.*]] = shl <2 x i64> [[TRUNC]], <i64 48, i64 48>
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr exact <2 x i64> [[SEXT]], <i64 48, i64 48>
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <2 x i32> [[A:%.*]] to <2 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = sext <2 x i16> [[TRUNC]] to <2 x i64>
 ; CHECK-NEXT:    [[EXT:%.*]] = select <2 x i1> [[CMP:%.*]], <2 x i64> [[TMP1]], <2 x i64> <i64 42, i64 43>
 ; CHECK-NEXT:    ret <2 x i64> [[EXT]]
 ;
   %trunc = trunc <2 x i32> %a to <2 x i16>
   %sel = select <2 x i1> %cmp, <2 x i16> %trunc, <2 x i16> <i16 42, i16 43>
   %ext = sext <2 x i16> %sel to <2 x i64>
   ret <2 x i64> %ext
 }
 
 define i32 @trunc_sel_smaller_sext(i64 %a, i1 %cmp) {
 ; CHECK-LABEL: @trunc_sel_smaller_sext(
 ; CHECK-NEXT:    [[TRUNC:%.*]] = trunc i64 [[A:%.*]] to i16
 ; CHECK-NEXT:    [[TMP1:%.*]] = sext i16 [[TRUNC]] to i32
 ; CHECK-NEXT:    [[EXT:%.*]] = select i1 [[CMP:%.*]], i32 [[TMP1]], i32 42
 ; CHECK-NEXT:    ret i32 [[EXT]]
 ;
   %trunc = trunc i64 %a to i16
   %sel = select i1 %cmp, i16 %trunc, i16 42
   %ext = sext i16 %sel to i32
   ret i32 %ext
 }
 
 define <2 x i32> @trunc_sel_smaller_sext_vec(<2 x i64> %a, <2 x i1> %cmp) {
 ; CHECK-LABEL: @trunc_sel_smaller_sext_vec(
-; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32>
-; CHECK-NEXT:    [[SEXT:%.*]] = shl <2 x i32> [[TRUNC]], <i32 16, i32 16>
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr exact <2 x i32> [[SEXT]], <i32 16, i32 16>
+; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i16>
+; CHECK-NEXT:    [[TMP1:%.*]] = sext <2 x i16> [[TRUNC]] to <2 x i32>
 ; CHECK-NEXT:    [[EXT:%.*]] = select <2 x i1> [[CMP:%.*]], <2 x i32> [[TMP1]], <2 x i32> <i32 42, i32 43>
 ; CHECK-NEXT:    ret <2 x i32> [[EXT]]
 ;
   %trunc = trunc <2 x i64> %a to <2 x i16>
   %sel = select <2 x i1> %cmp, <2 x i16> %trunc, <2 x i16> <i16 42, i16 43>
   %ext = sext <2 x i16> %sel to <2 x i32>
   ret <2 x i32> %ext
 }
 
 define i32 @trunc_sel_equal_sext(i32 %a, i1 %cmp) {
 ; CHECK-LABEL: @trunc_sel_equal_sext(
 ; CHECK-NEXT:    [[TMP1:%.*]] = shl i32 [[A:%.*]], 16
 ; CHECK-NEXT:    [[TMP2:%.*]] = ashr exact i32 [[TMP1]], 16
 ; CHECK-NEXT:    [[EXT:%.*]] = select i1 [[CMP:%.*]], i32 [[TMP2]], i32 42
 ; CHECK-NEXT:    ret i32 [[EXT]]
 ;
   %trunc = trunc i32 %a to i16
   %sel = select i1 %cmp, i16 %trunc, i16 42
   %ext = sext i16 %sel to i32
   ret i32 %ext
 }
 
 define <2 x i32> @trunc_sel_equal_sext_vec(<2 x i32> %a, <2 x i1> %cmp) {
 ; CHECK-LABEL: @trunc_sel_equal_sext_vec(
-; CHECK-NEXT:    [[SEXT:%.*]] = shl <2 x i32> [[A:%.*]], <i32 16, i32 16>
-; CHECK-NEXT:    [[TMP1:%.*]] = ashr exact <2 x i32> [[SEXT]], <i32 16, i32 16>
-; CHECK-NEXT:    [[EXT:%.*]] = select <2 x i1> [[CMP:%.*]], <2 x i32> [[TMP1]], <2 x i32> <i32 42, i32 43>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i32> [[A:%.*]], <i32 16, i32 16>
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr exact <2 x i32> [[TMP1]], <i32 16, i32 16>
+; CHECK-NEXT:    [[EXT:%.*]] = select <2 x i1> [[CMP:%.*]], <2 x i32> [[TMP2]], <2 x i32> <i32 42, i32 43>
 ; CHECK-NEXT:    ret <2 x i32> [[EXT]]
 ;
   %trunc = trunc <2 x i32> %a to <2 x i16>
   %sel = select <2 x i1> %cmp, <2 x i16> %trunc, <2 x i16> <i16 42, i16 43>
   %ext = sext <2 x i16> %sel to <2 x i32>
   ret <2 x i32> %ext
 }
 
 define i64 @trunc_sel_larger_zext(i32 %a, i1 %cmp) {
 ; CHECK-LABEL: @trunc_sel_larger_zext(
 ; CHECK-NEXT:    [[TRUNC_MASK:%.*]] = and i32 [[A:%.*]], 65535
 ; CHECK-NEXT:    [[TMP1:%.*]] = zext i32 [[TRUNC_MASK]] to i64
 ; CHECK-NEXT:    [[EXT:%.*]] = select i1 [[CMP:%.*]], i64 [[TMP1]], i64 42
 ; CHECK-NEXT:    ret i64 [[EXT]]
 ;
   %trunc = trunc i32 %a to i16
   %sel = select i1 %cmp, i16 %trunc, i16 42
   %ext = zext i16 %sel to i64
   ret i64 %ext
 }
 
 define <2 x i64> @trunc_sel_larger_zext_vec(<2 x i32> %a, <2 x i1> %cmp) {
 ; CHECK-LABEL: @trunc_sel_larger_zext_vec(
-; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[A:%.*]], <i32 65535, i32 65535>
-; CHECK-NEXT:    [[TMP2:%.*]] = zext <2 x i32> [[TMP1]] to <2 x i64>
-; CHECK-NEXT:    [[EXT:%.*]] = select <2 x i1> [[CMP:%.*]], <2 x i64> [[TMP2]], <2 x i64> <i64 42, i64 43>
+; CHECK-NEXT:    [[TRUNC_MASK:%.*]] = and <2 x i32> [[A:%.*]], <i32 65535, i32 65535>
+; CHECK-NEXT:    [[TMP1:%.*]] = zext <2 x i32> [[TRUNC_MASK]] to <2 x i64>
+; CHECK-NEXT:    [[EXT:%.*]] = select <2 x i1> [[CMP:%.*]], <2 x i64> [[TMP1]], <2 x i64> <i64 42, i64 43>
 ; CHECK-NEXT:    ret <2 x i64> [[EXT]]
 ;
   %trunc = trunc <2 x i32> %a to <2 x i16>
   %sel = select <2 x i1> %cmp, <2 x i16> %trunc, <2 x i16> <i16 42, i16 43>
   %ext = zext <2 x i16> %sel to <2 x i64>
   ret <2 x i64> %ext
 }
 
 define i32 @trunc_sel_smaller_zext(i64 %a, i1 %cmp) {
 ; CHECK-LABEL: @trunc_sel_smaller_zext(
 ; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[A:%.*]] to i32
 ; CHECK-NEXT:    [[TMP2:%.*]] = and i32 [[TMP1]], 65535
 ; CHECK-NEXT:    [[EXT:%.*]] = select i1 [[CMP:%.*]], i32 [[TMP2]], i32 42
 ; CHECK-NEXT:    ret i32 [[EXT]]
 ;
   %trunc = trunc i64 %a to i16
   %sel = select i1 %cmp, i16 %trunc, i16 42
   %ext = zext i16 %sel to i32
   ret i32 %ext
 }
 
 define <2 x i32> @trunc_sel_smaller_zext_vec(<2 x i64> %a, <2 x i1> %cmp) {
 ; CHECK-LABEL: @trunc_sel_smaller_zext_vec(
-; CHECK-NEXT:    [[TRUNC:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32>
-; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i32> [[TRUNC]], <i32 65535, i32 65535>
-; CHECK-NEXT:    [[EXT:%.*]] = select <2 x i1> [[CMP:%.*]], <2 x i32> [[TMP1]], <2 x i32> <i32 42, i32 43>
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i64> [[A:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <2 x i32> [[TMP1]], <i32 65535, i32 65535>
+; CHECK-NEXT:    [[EXT:%.*]] = select <2 x i1> [[CMP:%.*]], <2 x i32> [[TMP2]], <2 x i32> <i32 42, i32 43>
 ; CHECK-NEXT:    ret <2 x i32> [[EXT]]
 ;
   %trunc = trunc <2 x i64> %a to <2 x i16>
   %sel = select <2 x i1> %cmp, <2 x i16> %trunc, <2 x i16> <i16 42, i16 43>
   %ext = zext <2 x i16> %sel to <2 x i32>
   ret <2 x i32> %ext
 }
 

test/Transforms/InstCombine/vector-casts.ll

;
   %val = trunc <2 x i64> %src to <2 x i32>
   %add = add <2 x i32> %val, <i32 1, i32 1>
   store <2 x i32> %add, <2 x i32>* %dst.addr
   ret void
 }
 
 define <2 x i65> @foo(<2 x i64> %t) {
 ; CHECK-LABEL: @foo(
-; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i64> [[T:%.*]], <i64 4294967295, i64 4294967295>
-; CHECK-NEXT:    [[B:%.*]] = zext <2 x i64> [[TMP1]] to <2 x i65>
+; CHECK-NEXT:    [[A_MASK:%.*]] = and <2 x i64> [[T:%.*]], <i64 4294967295, i64 4294967295>
+; CHECK-NEXT:    [[B:%.*]] = zext <2 x i64> [[A_MASK]] to <2 x i65>
 ; CHECK-NEXT:    ret <2 x i65> [[B]]
 ;
   %a = trunc <2 x i64> %t to <2 x i32>
   %b = zext <2 x i32> %a to <2 x i65>
   ret <2 x i65> %b
 }
 
 define <2 x i64> @bar(<2 x i65> %t) {
 ; CHECK-LABEL: @bar(
-; CHECK-NEXT:    [[A:%.*]] = trunc <2 x i65> [[T:%.*]] to <2 x i64>
-; CHECK-NEXT:    [[B:%.*]] = and <2 x i64> [[A]], <i64 4294967295, i64 4294967295>
+; CHECK-NEXT:    [[TMP1:%.*]] = trunc <2 x i65> [[T:%.*]] to <2 x i64>
+; CHECK-NEXT:    [[B:%.*]] = and <2 x i64> [[TMP1]], <i64 4294967295, i64 4294967295>
 ; CHECK-NEXT:    ret <2 x i64> [[B]]
 ;
   %a = trunc <2 x i65> %t to <2 x i32>
   %b = zext <2 x i32> %a to <2 x i64>
   ret <2 x i64> %b
 }
 
 define <2 x i64> @bars(<2 x i65> %t) {
 ; CHECK-LABEL: @bars(
-; CHECK-NEXT:    [[A:%.*]] = trunc <2 x i65> [[T:%.*]] to <2 x i64>
-; CHECK-NEXT:    [[SEXT:%.*]] = shl <2 x i64> [[A]], <i64 32, i64 32>
-; CHECK-NEXT:    [[B:%.*]] = ashr exact <2 x i64> [[SEXT]], <i64 32, i64 32>
+; CHECK-NEXT:    [[A:%.*]] = trunc <2 x i65> [[T:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[B:%.*]] = sext <2 x i32> [[A]] to <2 x i64>
 ; CHECK-NEXT:    ret <2 x i64> [[B]]
 ;
   %a = trunc <2 x i65> %t to <2 x i32>
   %b = sext <2 x i32> %a to <2 x i64>
   ret <2 x i64> %b
 }
 
 define <2 x i64> @quxs(<2 x i64> %t) {
 ; CHECK-LABEL: @quxs(
-; CHECK-NEXT:    [[SEXT:%.*]] = shl <2 x i64> [[T:%.*]], <i64 32, i64 32>
-; CHECK-NEXT:    [[B:%.*]] = ashr exact <2 x i64> [[SEXT]], <i64 32, i64 32>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i64> [[T:%.*]], <i64 32, i64 32>
+; CHECK-NEXT:    [[B:%.*]] = ashr exact <2 x i64> [[TMP1]], <i64 32, i64 32>
 ; CHECK-NEXT:    ret <2 x i64> [[B]]
 ;
   %a = trunc <2 x i64> %t to <2 x i32>
   %b = sext <2 x i32> %a to <2 x i64>
   ret <2 x i64> %b
 }
 
 define <2 x i64> @quxt(<2 x i64> %t) {

 
 ; Converting to a wide type might reduce instruction count,
 ; but we can not do that unless the backend can recover from
 ; the creation of a potentially illegal op (like a 64-bit vmul).
 ; PR40032 - https://bugs.llvm.org/show_bug.cgi?id=40032
 
 define <2 x i64> @sext_less_casting_with_wideop(<2 x i64> %x, <2 x i64> %y) {
 ; CHECK-LABEL: @sext_less_casting_with_wideop(
-; CHECK-NEXT:    [[MUL:%.*]] = mul <2 x i64> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[SEXT:%.*]] = shl <2 x i64> [[MUL]], <i64 32, i64 32>
-; CHECK-NEXT:    [[R:%.*]] = ashr exact <2 x i64> [[SEXT]], <i64 32, i64 32>
+; CHECK-NEXT:    [[XNARROW:%.*]] = trunc <2 x i64> [[X:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[YNARROW:%.*]] = trunc <2 x i64> [[Y:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[MUL:%.*]] = mul <2 x i32> [[XNARROW]], [[YNARROW]]
+; CHECK-NEXT:    [[R:%.*]] = sext <2 x i32> [[MUL]] to <2 x i64>
 ; CHECK-NEXT:    ret <2 x i64> [[R]]
 ;
   %xnarrow = trunc <2 x i64> %x to <2 x i32>
   %ynarrow = trunc <2 x i64> %y to <2 x i32>
   %mul = mul <2 x i32> %xnarrow, %ynarrow
   %r = sext <2 x i32> %mul to <2 x i64>
   ret <2 x i64> %r
 }
 
 define <2 x i64> @zext_less_casting_with_wideop(<2 x i64> %x, <2 x i64> %y) {
 ; CHECK-LABEL: @zext_less_casting_with_wideop(
-; CHECK-NEXT:    [[MUL:%.*]] = mul <2 x i64> [[X:%.*]], [[Y:%.*]]
-; CHECK-NEXT:    [[R:%.*]] = and <2 x i64> [[MUL]], <i64 4294967295, i64 4294967295>
+; CHECK-NEXT:    [[XNARROW:%.*]] = trunc <2 x i64> [[X:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[YNARROW:%.*]] = trunc <2 x i64> [[Y:%.*]] to <2 x i32>
+; CHECK-NEXT:    [[MUL:%.*]] = mul <2 x i32> [[XNARROW]], [[YNARROW]]
+; CHECK-NEXT:    [[R:%.*]] = zext <2 x i32> [[MUL]] to <2 x i64>
 ; CHECK-NEXT:    ret <2 x i64> [[R]]
 ;
   %xnarrow = trunc <2 x i64> %x to <2 x i32>
   %ynarrow = trunc <2 x i64> %y to <2 x i32>
   %mul = mul <2 x i32> %xnarrow, %ynarrow
   %r = zext <2 x i32> %mul to <2 x i64>
   ret <2 x i64> %r
 }