[TTI] BasicTTI::getInterleavedMemoryOpCost(): use getScalarizationOverhead()

[RKSimon]

Summary

getScalarizationOverhead() results in a somewhat better cost estimation
than counting the insertion/extraction costs directly.
Notably, this is still overestimating the costs.

[lebedev.ri]

(note that i haven't update all the affected tests, only the checkline blocks where there is at least one change have been updated)

[RKSimon]

I've done my own version and I have all the tests updated - would you mind if I commandeered this with my version?

[lebedev.ri]

In D110713#3030304, @RKSimon wrote:

I've done my own version and I have all the tests updated - would you mind if I commandeered this with my version?

Sure, go ahead!

[RKSimon]

Update all tests and just getScalarizationOverhead wrapper when all elements are demanded

[lebedev.ri]

LG, it's a bit of an improvement in modelling precision, but not much.
Thank you!

llvm/include/llvm/CodeGen/BasicTTIImpl.h

1257–1258

Please can you name the bool params?

[RKSimon]

cheers - it seems to be enough to cause an actual test change so is at least going in the right direction. A better step would be to get this converted to shuffle costs, but we have really poor handling of length changing shuffles

llvm/include/llvm/CodeGen/BasicTTIImpl.h

1257–1258

yes I'll add them

llvm/include/llvm/CodeGen/BasicTTIImpl.h

if (VecTySize > VecTyLTSize) {
       // Scale the cost of the load by the fraction of legal instructions that
       // will be used.
       Cost *= UsedInsts.count() / NumLegalInsts;
     }
 
     // Then plus the cost of interleave operation.
     assert(Indices.size() <= Factor &&
            "Interleaved memory op has too many members");
+
+    APInt DemandedLoadStoreElts = APInt::getNullValue(NumElts);
+    for (unsigned Index : Indices) {
+      assert(Index < Factor && "Invalid index for interleaved memory op");
+      for (unsigned Elm = 0; Elm < NumSubElts; Elm++)
+        DemandedLoadStoreElts.setBit(Index + Elm * Factor);
+    }
+
     if (Opcode == Instruction::Load) {
       // The interleave cost is similar to extract sub vectors' elements
       // from the wide vector, and insert them into sub vectors.
       //
       // E.g. An interleaved load of factor 2 (with one member of index 0):
       //      %vec = load <8 x i32>, <8 x i32>* %ptr
       //      %v0 = shuffle %vec, undef, <0, 2, 4, 6>         ; Index 0
       // The cost is estimated as extract elements at 0, 2, 4, 6 from the
       // <8 x i32> vector and insert them into a <4 x i32> vector.
-      for (unsigned Index : Indices) {
-        assert(Index < Factor && "Invalid index for interleaved memory op");
-
-        // Extract elements from loaded vector for each sub vector.
-        for (unsigned Elm = 0; Elm < NumSubElts; Elm++)
-          Cost += thisT()->getVectorInstrCost(Instruction::ExtractElement, VT,
-                                              Index + Elm * Factor);
-      }
-
-      InstructionCost InsSubCost = 0;
-      for (unsigned Elm = 0; Elm < NumSubElts; Elm++)
-        InsSubCost +=
-            thisT()->getVectorInstrCost(Instruction::InsertElement, SubVT, Elm);
-
+      InstructionCost InsSubCost =
+          getScalarizationOverhead(SubVT, /*Insert*/ true, /*Extract*/ false);
       Cost += Indices.size() * InsSubCost;
+      Cost +=
+          thisT()->getScalarizationOverhead(VT, DemandedLoadStoreElts,
+                                            /*Insert*/ false, /*Extract*/ true);
     } else {
       // The interleave cost is extract elements from sub vectors, and
       // insert them into the wide vector.
       //
       // E.g. An interleaved store of factor 3 with 2 members at indices 0,1:
       // (using VF=4):
       //    %v0_v1 = shuffle %v0, %v1, <0,4,undef,1,5,undef,2,6,undef,3,7,undef>
       //    %gaps.mask = <true, true, false, true, true, false,
       //                  true, true, false, true, true, false>
       //    call llvm.masked.store <12 x i32> %v0_v1, <12 x i32>* %ptr,
       //                           i32 Align, <12 x i1> %gaps.mask
       // The cost is estimated as extract all elements (of actual members,
       // excluding gaps) from both <4 x i32> vectors and insert into the <12 x
       // i32> vector.
-      InstructionCost ExtSubCost = 0;
-      for (unsigned Elm = 0; Elm < NumSubElts; Elm++)
-        ExtSubCost += thisT()->getVectorInstrCost(Instruction::ExtractElement,
-                                                  SubVT, Elm);
+      InstructionCost ExtSubCost =
+          getScalarizationOverhead(SubVT, /*Insert*/ false, /*Extract*/ true);
       Cost += ExtSubCost * Indices.size();
-
-      for (unsigned Index : Indices) {
-        assert(Index < Factor && "Invalid index for interleaved memory op");
-
-        // Insert elements from loaded vector for each sub vector.
-        for (unsigned Elm = 0; Elm < NumSubElts; Elm++)
-          Cost += thisT()->getVectorInstrCost(Instruction::InsertElement, VT,
-                                              Index + Elm * Factor);
-      }
+      Cost += thisT()->getScalarizationOverhead(VT, DemandedLoadStoreElts,
+                                                /*Insert*/ true,
+                                                /*Extract*/ false);
     }
 
     if (!UseMaskForCond)
       return Cost;
 
     Type *I8Type = Type::getInt8Ty(VT->getContext());
     auto *MaskVT = FixedVectorType::get(I8Type, NumElts);
     SubVT = FixedVectorType::get(I8Type, NumSubElts);
 
     // The Mask shuffling cost is extract all the elements of the Mask
     // and insert each of them Factor times into the wide vector:
     //
     // E.g. an interleaved group with factor 3:
     //    %mask = icmp ult <8 x i32> %vec1, %vec2
     //    %interleaved.mask = shufflevector <8 x i1> %mask, <8 x i1> undef,
     //        <24 x i32> <0,0,0,1,1,1,2,2,2,3,3,3,4,4,4,5,5,5,6,6,6,7,7,7>
     // The cost is estimated as extract all mask elements from the <8xi1> mask
     // vector and insert them factor times into the <24xi1> shuffled mask
     // vector.
-    for (unsigned i = 0; i < NumSubElts; i++)
-      Cost +=
-          thisT()->getVectorInstrCost(Instruction::ExtractElement, SubVT, i);
-
-    for (unsigned i = 0; i < NumElts; i++)
-      Cost +=
-          thisT()->getVectorInstrCost(Instruction::InsertElement, MaskVT, i);
+    Cost += getScalarizationOverhead(SubVT, /*Insert*/ false, /*Extract*/ true);
+        getScalarizationOverhead(MaskVT, /*Insert*/ true, /*Extract*/ false);
 
     // The Gaps mask is invariant and created outside the loop, therefore the
     // cost of creating it is not accounted for here. However if we have both
     // a MaskForGaps and some other mask that guards the execution of the
     // memory access, we need to account for the cost of And-ing the two masks
     // inside the loop.
     if (UseMaskForGaps)
       Cost += thisT()->getArithmeticInstrCost(BinaryOperator::And, MaskVT,

llvm/test/Analysis/CostModel/X86/interleaved-load-i16-stride-2.ll

 ; SSE2: LV: Found an estimated cost of 17 for VF 4 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; SSE2: LV: Found an estimated cost of 34 for VF 8 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; SSE2: LV: Found an estimated cost of 68 for VF 16 For instruction:   %v0 = load i16, i16* %in0, align 2
 ;
 ; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX1: LV: Found an estimated cost of 9 for VF 2 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX1: LV: Found an estimated cost of 17 for VF 4 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX1: LV: Found an estimated cost of 41 for VF 8 For instruction:   %v0 = load i16, i16* %in0, align 2
-; AVX1: LV: Found an estimated cost of 114 for VF 16 For instruction:   %v0 = load i16, i16* %in0, align 2
-; AVX1: LV: Found an estimated cost of 228 for VF 32 For instruction:   %v0 = load i16, i16* %in0, align 2
+; AVX1: LV: Found an estimated cost of 86 for VF 16 For instruction:   %v0 = load i16, i16* %in0, align 2
+; AVX1: LV: Found an estimated cost of 172 for VF 32 For instruction:   %v0 = load i16, i16* %in0, align 2
 ;
 ; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX2: LV: Found an estimated cost of 3 for VF 2 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX2: LV: Found an estimated cost of 3 for VF 4 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX2: LV: Found an estimated cost of 7 for VF 8 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX2: LV: Found an estimated cost of 11 for VF 16 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX2: LV: Found an estimated cost of 22 for VF 32 For instruction:   %v0 = load i16, i16* %in0, align 2
 ;

llvm/test/Analysis/CostModel/X86/interleaved-load-i16-stride-3.ll

 ; SSE2: LV: Found an estimated cost of 26 for VF 4 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; SSE2: LV: Found an estimated cost of 51 for VF 8 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; SSE2: LV: Found an estimated cost of 102 for VF 16 For instruction:   %v0 = load i16, i16* %in0, align 2
 ;
 ; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX1: LV: Found an estimated cost of 15 for VF 2 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX1: LV: Found an estimated cost of 31 for VF 4 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX1: LV: Found an estimated cost of 58 for VF 8 For instruction:   %v0 = load i16, i16* %in0, align 2
-; AVX1: LV: Found an estimated cost of 171 for VF 16 For instruction:   %v0 = load i16, i16* %in0, align 2
-; AVX1: LV: Found an estimated cost of 342 for VF 32 For instruction:   %v0 = load i16, i16* %in0, align 2
+; AVX1: LV: Found an estimated cost of 129 for VF 16 For instruction:   %v0 = load i16, i16* %in0, align 2
+; AVX1: LV: Found an estimated cost of 258 for VF 32 For instruction:   %v0 = load i16, i16* %in0, align 2
 ;
 ; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX2: LV: Found an estimated cost of 15 for VF 2 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX2: LV: Found an estimated cost of 31 for VF 4 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX2: LV: Found an estimated cost of 58 for VF 8 For instruction:   %v0 = load i16, i16* %in0, align 2
-; AVX2: LV: Found an estimated cost of 171 for VF 16 For instruction:   %v0 = load i16, i16* %in0, align 2
-; AVX2: LV: Found an estimated cost of 342 for VF 32 For instruction:   %v0 = load i16, i16* %in0, align 2
+; AVX2: LV: Found an estimated cost of 129 for VF 16 For instruction:   %v0 = load i16, i16* %in0, align 2
+; AVX2: LV: Found an estimated cost of 258 for VF 32 For instruction:   %v0 = load i16, i16* %in0, align 2
 ;
 ; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX512: LV: Found an estimated cost of 4 for VF 2 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX512: LV: Found an estimated cost of 7 for VF 4 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX512: LV: Found an estimated cost of 7 for VF 8 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX512: LV: Found an estimated cost of 9 for VF 16 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX512: LV: Found an estimated cost of 18 for VF 32 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX512: LV: Found an estimated cost of 81 for VF 64 For instruction:   %v0 = load i16, i16* %in0, align 2

llvm/test/Analysis/CostModel/X86/interleaved-load-i16-stride-4.ll

 ; SSE2: LV: Found an estimated cost of 34 for VF 4 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; SSE2: LV: Found an estimated cost of 68 for VF 8 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; SSE2: LV: Found an estimated cost of 136 for VF 16 For instruction:   %v0 = load i16, i16* %in0, align 2
 ;
 ; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX1: LV: Found an estimated cost of 17 for VF 2 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX1: LV: Found an estimated cost of 41 for VF 4 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX1: LV: Found an estimated cost of 82 for VF 8 For instruction:   %v0 = load i16, i16* %in0, align 2
-; AVX1: LV: Found an estimated cost of 228 for VF 16 For instruction:   %v0 = load i16, i16* %in0, align 2
-; AVX1: LV: Found an estimated cost of 456 for VF 32 For instruction:   %v0 = load i16, i16* %in0, align 2
+; AVX1: LV: Found an estimated cost of 172 for VF 16 For instruction:   %v0 = load i16, i16* %in0, align 2
+; AVX1: LV: Found an estimated cost of 344 for VF 32 For instruction:   %v0 = load i16, i16* %in0, align 2
 ;
 ; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX2: LV: Found an estimated cost of 7 for VF 2 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX2: LV: Found an estimated cost of 18 for VF 4 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX2: LV: Found an estimated cost of 35 for VF 8 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX2: LV: Found an estimated cost of 79 for VF 16 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX2: LV: Found an estimated cost of 158 for VF 32 For instruction:   %v0 = load i16, i16* %in0, align 2
 ;

llvm/test/Analysis/CostModel/X86/interleaved-load-i16-stride-5.ll

 ; SSE2: LV: Found an estimated cost of 22 for VF 2 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; SSE2: LV: Found an estimated cost of 43 for VF 4 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; SSE2: LV: Found an estimated cost of 85 for VF 8 For instruction:   %v0 = load i16, i16* %in0, align 2
 ;
 ; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX1: LV: Found an estimated cost of 26 for VF 2 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX1: LV: Found an estimated cost of 50 for VF 4 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX1: LV: Found an estimated cost of 99 for VF 8 For instruction:   %v0 = load i16, i16* %in0, align 2
-; AVX1: LV: Found an estimated cost of 285 for VF 16 For instruction:   %v0 = load i16, i16* %in0, align 2
+; AVX1: LV: Found an estimated cost of 215 for VF 16 For instruction:   %v0 = load i16, i16* %in0, align 2
 ;
 ; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX2: LV: Found an estimated cost of 26 for VF 2 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX2: LV: Found an estimated cost of 50 for VF 4 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX2: LV: Found an estimated cost of 99 for VF 8 For instruction:   %v0 = load i16, i16* %in0, align 2
-; AVX2: LV: Found an estimated cost of 285 for VF 16 For instruction:   %v0 = load i16, i16* %in0, align 2
+; AVX2: LV: Found an estimated cost of 215 for VF 16 For instruction:   %v0 = load i16, i16* %in0, align 2
 ;
 ; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX512: LV: Found an estimated cost of 11 for VF 2 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX512: LV: Found an estimated cost of 11 for VF 4 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX512: LV: Found an estimated cost of 14 for VF 8 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX512: LV: Found an estimated cost of 28 for VF 16 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX512: LV: Found an estimated cost of 55 for VF 32 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX512: LV: Found an estimated cost of 235 for VF 64 For instruction:   %v0 = load i16, i16* %in0, align 2

llvm/test/Analysis/CostModel/X86/interleaved-load-i16-stride-6.ll

 ; SSE2: LV: Found an estimated cost of 26 for VF 2 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; SSE2: LV: Found an estimated cost of 51 for VF 4 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; SSE2: LV: Found an estimated cost of 102 for VF 8 For instruction:   %v0 = load i16, i16* %in0, align 2
 ;
 ; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX1: LV: Found an estimated cost of 31 for VF 2 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX1: LV: Found an estimated cost of 58 for VF 4 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX1: LV: Found an estimated cost of 123 for VF 8 For instruction:   %v0 = load i16, i16* %in0, align 2
-; AVX1: LV: Found an estimated cost of 342 for VF 16 For instruction:   %v0 = load i16, i16* %in0, align 2
+; AVX1: LV: Found an estimated cost of 258 for VF 16 For instruction:   %v0 = load i16, i16* %in0, align 2
 ;
 ; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX2: LV: Found an estimated cost of 16 for VF 2 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX2: LV: Found an estimated cost of 11 for VF 4 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX2: LV: Found an estimated cost of 42 for VF 8 For instruction:   %v0 = load i16, i16* %in0, align 2
 ; AVX2: LV: Found an estimated cost of 112 for VF 16 For instruction:   %v0 = load i16, i16* %in0, align 2
 ;
 ; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction:   %v0 = load i16, i16* %in0, align 2

llvm/test/Analysis/CostModel/X86/interleaved-load-i8-stride-2.ll

 ; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,SSE2
 ; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX1
 ; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX2
 ; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw,+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @A = global [1024 x i8] zeroinitializer, align 128
 @B = global [1024 x i8] zeroinitializer, align 128
 
 ; CHECK: LV: Checking a loop in "test"
 ;
 ; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction:   %v0 = load i8, i8* %in0, align 1
-; SSE2: LV: Found an estimated cost of 20 for VF 2 For instruction:   %v0 = load i8, i8* %in0, align 1
-; SSE2: LV: Found an estimated cost of 56 for VF 4 For instruction:   %v0 = load i8, i8* %in0, align 1
-; SSE2: LV: Found an estimated cost of 160 for VF 8 For instruction:   %v0 = load i8, i8* %in0, align 1
-; SSE2: LV: Found an estimated cost of 478 for VF 16 For instruction:   %v0 = load i8, i8* %in0, align 1
+; SSE2: LV: Found an estimated cost of 14 for VF 2 For instruction:   %v0 = load i8, i8* %in0, align 1
+; SSE2: LV: Found an estimated cost of 30 for VF 4 For instruction:   %v0 = load i8, i8* %in0, align 1
+; SSE2: LV: Found an estimated cost of 62 for VF 8 For instruction:   %v0 = load i8, i8* %in0, align 1
+; SSE2: LV: Found an estimated cost of 126 for VF 16 For instruction:   %v0 = load i8, i8* %in0, align 1
 ;
 ; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction:   %v0 = load i8, i8* %in0, align 1
 ; AVX1: LV: Found an estimated cost of 9 for VF 2 For instruction:   %v0 = load i8, i8* %in0, align 1
 ; AVX1: LV: Found an estimated cost of 17 for VF 4 For instruction:   %v0 = load i8, i8* %in0, align 1
 ; AVX1: LV: Found an estimated cost of 33 for VF 8 For instruction:   %v0 = load i8, i8* %in0, align 1
 ; AVX1: LV: Found an estimated cost of 81 for VF 16 For instruction:   %v0 = load i8, i8* %in0, align 1
-; AVX1: LV: Found an estimated cost of 226 for VF 32 For instruction:   %v0 = load i8, i8* %in0, align 1
+; AVX1: LV: Found an estimated cost of 166 for VF 32 For instruction:   %v0 = load i8, i8* %in0, align 1
 ;
 ; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction:   %v0 = load i8, i8* %in0, align 1
 ; AVX2: LV: Found an estimated cost of 9 for VF 2 For instruction:   %v0 = load i8, i8* %in0, align 1
 ; AVX2: LV: Found an estimated cost of 17 for VF 4 For instruction:   %v0 = load i8, i8* %in0, align 1
 ; AVX2: LV: Found an estimated cost of 33 for VF 8 For instruction:   %v0 = load i8, i8* %in0, align 1
 ; AVX2: LV: Found an estimated cost of 81 for VF 16 For instruction:   %v0 = load i8, i8* %in0, align 1
-; AVX2: LV: Found an estimated cost of 226 for VF 32 For instruction:   %v0 = load i8, i8* %in0, align 1
+; AVX2: LV: Found an estimated cost of 166 for VF 32 For instruction:   %v0 = load i8, i8* %in0, align 1
 ;
 ; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction:   %v0 = load i8, i8* %in0, align 1
 ; AVX512: LV: Found an estimated cost of 3 for VF 2 For instruction:   %v0 = load i8, i8* %in0, align 1
 ; AVX512: LV: Found an estimated cost of 3 for VF 4 For instruction:   %v0 = load i8, i8* %in0, align 1
 ; AVX512: LV: Found an estimated cost of 3 for VF 8 For instruction:   %v0 = load i8, i8* %in0, align 1
 ; AVX512: LV: Found an estimated cost of 9 for VF 16 For instruction:   %v0 = load i8, i8* %in0, align 1
 ; AVX512: LV: Found an estimated cost of 17 for VF 32 For instruction:   %v0 = load i8, i8* %in0, align 1
 ; AVX512: LV: Found an estimated cost of 41 for VF 64 For instruction:   %v0 = load i8, i8* %in0, align 1

llvm/test/Analysis/CostModel/X86/interleaved-store-i16-stride-2.ll

 ; SSE2: LV: Found an estimated cost of 9 for VF 2 For instruction:   store i16 %v1, i16* %out1, align 2
 ; SSE2: LV: Found an estimated cost of 17 for VF 4 For instruction:   store i16 %v1, i16* %out1, align 2
 ; SSE2: LV: Found an estimated cost of 34 for VF 8 For instruction:   store i16 %v1, i16* %out1, align 2
 ; SSE2: LV: Found an estimated cost of 68 for VF 16 For instruction:   store i16 %v1, i16* %out1, align 2
 ;
 ; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction:   store i16 %v1, i16* %out1, align 2
 ; AVX1: LV: Found an estimated cost of 9 for VF 2 For instruction:   store i16 %v1, i16* %out1, align 2
 ; AVX1: LV: Found an estimated cost of 17 for VF 4 For instruction:   store i16 %v1, i16* %out1, align 2
-; AVX1: LV: Found an estimated cost of 49 for VF 8 For instruction:   store i16 %v1, i16* %out1, align 2
-; AVX1: LV: Found an estimated cost of 114 for VF 16 For instruction:   store i16 %v1, i16* %out1, align 2
-; AVX1: LV: Found an estimated cost of 228 for VF 32 For instruction:   store i16 %v1, i16* %out1, align 2
+; AVX1: LV: Found an estimated cost of 35 for VF 8 For instruction:   store i16 %v1, i16* %out1, align 2
+; AVX1: LV: Found an estimated cost of 86 for VF 16 For instruction:   store i16 %v1, i16* %out1, align 2
+; AVX1: LV: Found an estimated cost of 172 for VF 32 For instruction:   store i16 %v1, i16* %out1, align 2
 ;
 ; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction:   store i16 %v1, i16* %out1, align 2
 ; AVX2: LV: Found an estimated cost of 2 for VF 2 For instruction:   store i16 %v1, i16* %out1, align 2
 ; AVX2: LV: Found an estimated cost of 2 for VF 4 For instruction:   store i16 %v1, i16* %out1, align 2
 ; AVX2: LV: Found an estimated cost of 4 for VF 8 For instruction:   store i16 %v1, i16* %out1, align 2
 ; AVX2: LV: Found an estimated cost of 6 for VF 16 For instruction:   store i16 %v1, i16* %out1, align 2
 ; AVX2: LV: Found an estimated cost of 12 for VF 32 For instruction:   store i16 %v1, i16* %out1, align 2
 ;

llvm/test/Analysis/CostModel/X86/interleaved-store-i16-stride-3.ll

 ; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction:   store i16 %v2, i16* %out2, align 2
 ; SSE2: LV: Found an estimated cost of 16 for VF 2 For instruction:   store i16 %v2, i16* %out2, align 2
 ; SSE2: LV: Found an estimated cost of 26 for VF 4 For instruction:   store i16 %v2, i16* %out2, align 2
 ; SSE2: LV: Found an estimated cost of 51 for VF 8 For instruction:   store i16 %v2, i16* %out2, align 2
 ; SSE2: LV: Found an estimated cost of 102 for VF 16 For instruction:   store i16 %v2, i16* %out2, align 2
 ;
 ; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction:   store i16 %v2, i16* %out2, align 2
 ; AVX1: LV: Found an estimated cost of 15 for VF 2 For instruction:   store i16 %v2, i16* %out2, align 2
-; AVX1: LV: Found an estimated cost of 35 for VF 4 For instruction:   store i16 %v2, i16* %out2, align 2
-; AVX1: LV: Found an estimated cost of 66 for VF 8 For instruction:   store i16 %v2, i16* %out2, align 2
-; AVX1: LV: Found an estimated cost of 171 for VF 16 For instruction:   store i16 %v2, i16* %out2, align 2
-; AVX1: LV: Found an estimated cost of 342 for VF 32 For instruction:   store i16 %v2, i16* %out2, align 2
+; AVX1: LV: Found an estimated cost of 30 for VF 4 For instruction:   store i16 %v2, i16* %out2, align 2
+; AVX1: LV: Found an estimated cost of 53 for VF 8 For instruction:   store i16 %v2, i16* %out2, align 2
+; AVX1: LV: Found an estimated cost of 129 for VF 16 For instruction:   store i16 %v2, i16* %out2, align 2
+; AVX1: LV: Found an estimated cost of 258 for VF 32 For instruction:   store i16 %v2, i16* %out2, align 2
 ;
 ; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction:   store i16 %v2, i16* %out2, align 2
 ; AVX2: LV: Found an estimated cost of 15 for VF 2 For instruction:   store i16 %v2, i16* %out2, align 2
-; AVX2: LV: Found an estimated cost of 35 for VF 4 For instruction:   store i16 %v2, i16* %out2, align 2
-; AVX2: LV: Found an estimated cost of 66 for VF 8 For instruction:   store i16 %v2, i16* %out2, align 2
-; AVX2: LV: Found an estimated cost of 171 for VF 16 For instruction:   store i16 %v2, i16* %out2, align 2
-; AVX2: LV: Found an estimated cost of 342 for VF 32 For instruction:   store i16 %v2, i16* %out2, align 2
+; AVX2: LV: Found an estimated cost of 30 for VF 4 For instruction:   store i16 %v2, i16* %out2, align 2
+; AVX2: LV: Found an estimated cost of 53 for VF 8 For instruction:   store i16 %v2, i16* %out2, align 2
+; AVX2: LV: Found an estimated cost of 129 for VF 16 For instruction:   store i16 %v2, i16* %out2, align 2
+; AVX2: LV: Found an estimated cost of 258 for VF 32 For instruction:   store i16 %v2, i16* %out2, align 2
 ;
 ; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction:   store i16 %v2, i16* %out2, align 2
 ; AVX512: LV: Found an estimated cost of 6 for VF 2 For instruction:   store i16 %v2, i16* %out2, align 2
 ; AVX512: LV: Found an estimated cost of 6 for VF 4 For instruction:   store i16 %v2, i16* %out2, align 2
 ; AVX512: LV: Found an estimated cost of 6 for VF 8 For instruction:   store i16 %v2, i16* %out2, align 2
 ; AVX512: LV: Found an estimated cost of 12 for VF 16 For instruction:   store i16 %v2, i16* %out2, align 2
 ; AVX512: LV: Found an estimated cost of 18 for VF 32 For instruction:   store i16 %v2, i16* %out2, align 2
 ; AVX512: LV: Found an estimated cost of 36 for VF 64 For instruction:   store i16 %v2, i16* %out2, align 2

llvm/test/Analysis/CostModel/X86/interleaved-store-i16-stride-4.ll

 ; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction:   store i16 %v3, i16* %out3, align 2
 ; SSE2: LV: Found an estimated cost of 17 for VF 2 For instruction:   store i16 %v3, i16* %out3, align 2
 ; SSE2: LV: Found an estimated cost of 34 for VF 4 For instruction:   store i16 %v3, i16* %out3, align 2
 ; SSE2: LV: Found an estimated cost of 68 for VF 8 For instruction:   store i16 %v3, i16* %out3, align 2
 ; SSE2: LV: Found an estimated cost of 136 for VF 16 For instruction:   store i16 %v3, i16* %out3, align 2
 ;
 ; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction:   store i16 %v3, i16* %out3, align 2
 ; AVX1: LV: Found an estimated cost of 17 for VF 2 For instruction:   store i16 %v3, i16* %out3, align 2
-; AVX1: LV: Found an estimated cost of 49 for VF 4 For instruction:   store i16 %v3, i16* %out3, align 2
-; AVX1: LV: Found an estimated cost of 98 for VF 8 For instruction:   store i16 %v3, i16* %out3, align 2
-; AVX1: LV: Found an estimated cost of 228 for VF 16 For instruction:   store i16 %v3, i16* %out3, align 2
-; AVX1: LV: Found an estimated cost of 456 for VF 32 For instruction:   store i16 %v3, i16* %out3, align 2
+; AVX1: LV: Found an estimated cost of 35 for VF 4 For instruction:   store i16 %v3, i16* %out3, align 2
+; AVX1: LV: Found an estimated cost of 70 for VF 8 For instruction:   store i16 %v3, i16* %out3, align 2
+; AVX1: LV: Found an estimated cost of 172 for VF 16 For instruction:   store i16 %v3, i16* %out3, align 2
+; AVX1: LV: Found an estimated cost of 344 for VF 32 For instruction:   store i16 %v3, i16* %out3, align 2
 ;
 ; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction:   store i16 %v3, i16* %out3, align 2
 ; AVX2: LV: Found an estimated cost of 3 for VF 2 For instruction:   store i16 %v3, i16* %out3, align 2
 ; AVX2: LV: Found an estimated cost of 7 for VF 4 For instruction:   store i16 %v3, i16* %out3, align 2
 ; AVX2: LV: Found an estimated cost of 12 for VF 8 For instruction:   store i16 %v3, i16* %out3, align 2
 ; AVX2: LV: Found an estimated cost of 36 for VF 16 For instruction:   store i16 %v3, i16* %out3, align 2
 ; AVX2: LV: Found an estimated cost of 72 for VF 32 For instruction:   store i16 %v3, i16* %out3, align 2
 ;

llvm/test/Analysis/CostModel/X86/interleaved-store-i16-stride-5.ll

 ; CHECK: LV: Checking a loop in "test"
 ;
 ; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction:   store i16 %v4, i16* %out4, align 2
 ; SSE2: LV: Found an estimated cost of 22 for VF 2 For instruction:   store i16 %v4, i16* %out4, align 2
 ; SSE2: LV: Found an estimated cost of 43 for VF 4 For instruction:   store i16 %v4, i16* %out4, align 2
 ; SSE2: LV: Found an estimated cost of 85 for VF 8 For instruction:   store i16 %v4, i16* %out4, align 2
 ;
 ; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction:   store i16 %v4, i16* %out4, align 2
-; AVX1: LV: Found an estimated cost of 28 for VF 2 For instruction:   store i16 %v4, i16* %out4, align 2
-; AVX1: LV: Found an estimated cost of 58 for VF 4 For instruction:   store i16 %v4, i16* %out4, align 2
-; AVX1: LV: Found an estimated cost of 115 for VF 8 For instruction:   store i16 %v4, i16* %out4, align 2
-; AVX1: LV: Found an estimated cost of 285 for VF 16 For instruction:   store i16 %v4, i16* %out4, align 2
+; AVX1: LV: Found an estimated cost of 27 for VF 2 For instruction:   store i16 %v4, i16* %out4, align 2
+; AVX1: LV: Found an estimated cost of 45 for VF 4 For instruction:   store i16 %v4, i16* %out4, align 2
+; AVX1: LV: Found an estimated cost of 88 for VF 8 For instruction:   store i16 %v4, i16* %out4, align 2
+; AVX1: LV: Found an estimated cost of 215 for VF 16 For instruction:   store i16 %v4, i16* %out4, align 2
 ;
 ; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction:   store i16 %v4, i16* %out4, align 2
-; AVX2: LV: Found an estimated cost of 28 for VF 2 For instruction:   store i16 %v4, i16* %out4, align 2
-; AVX2: LV: Found an estimated cost of 58 for VF 4 For instruction:   store i16 %v4, i16* %out4, align 2
-; AVX2: LV: Found an estimated cost of 115 for VF 8 For instruction:   store i16 %v4, i16* %out4, align 2
-; AVX2: LV: Found an estimated cost of 285 for VF 16 For instruction:   store i16 %v4, i16* %out4, align 2
+; AVX2: LV: Found an estimated cost of 27 for VF 2 For instruction:   store i16 %v4, i16* %out4, align 2
+; AVX2: LV: Found an estimated cost of 45 for VF 4 For instruction:   store i16 %v4, i16* %out4, align 2
+; AVX2: LV: Found an estimated cost of 88 for VF 8 For instruction:   store i16 %v4, i16* %out4, align 2
+; AVX2: LV: Found an estimated cost of 215 for VF 16 For instruction:   store i16 %v4, i16* %out4, align 2
 ;
 ; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction:   store i16 %v4, i16* %out4, align 2
 ; AVX512: LV: Found an estimated cost of 11 for VF 2 For instruction:   store i16 %v4, i16* %out4, align 2
 ; AVX512: LV: Found an estimated cost of 11 for VF 4 For instruction:   store i16 %v4, i16* %out4, align 2
 ; AVX512: LV: Found an estimated cost of 22 for VF 8 For instruction:   store i16 %v4, i16* %out4, align 2
 ; AVX512: LV: Found an estimated cost of 33 for VF 16 For instruction:   store i16 %v4, i16* %out4, align 2
 ; AVX512: LV: Found an estimated cost of 55 for VF 32 For instruction:   store i16 %v4, i16* %out4, align 2
 ; AVX512: LV: Found an estimated cost of 110 for VF 64 For instruction:   store i16 %v4, i16* %out4, align 2

llvm/test/Analysis/CostModel/X86/interleaved-store-i16-stride-6.ll

 ; CHECK: LV: Checking a loop in "test"
 ;
 ; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction:   store i16 %v5, i16* %out5, align 2
 ; SSE2: LV: Found an estimated cost of 26 for VF 2 For instruction:   store i16 %v5, i16* %out5, align 2
 ; SSE2: LV: Found an estimated cost of 51 for VF 4 For instruction:   store i16 %v5, i16* %out5, align 2
 ; SSE2: LV: Found an estimated cost of 102 for VF 8 For instruction:   store i16 %v5, i16* %out5, align 2
 ;
 ; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction:   store i16 %v5, i16* %out5, align 2
-; AVX1: LV: Found an estimated cost of 35 for VF 2 For instruction:   store i16 %v5, i16* %out5, align 2
-; AVX1: LV: Found an estimated cost of 66 for VF 4 For instruction:   store i16 %v5, i16* %out5, align 2
-; AVX1: LV: Found an estimated cost of 147 for VF 8 For instruction:   store i16 %v5, i16* %out5, align 2
-; AVX1: LV: Found an estimated cost of 342 for VF 16 For instruction:   store i16 %v5, i16* %out5, align 2
+; AVX1: LV: Found an estimated cost of 30 for VF 2 For instruction:   store i16 %v5, i16* %out5, align 2
+; AVX1: LV: Found an estimated cost of 53 for VF 4 For instruction:   store i16 %v5, i16* %out5, align 2
+; AVX1: LV: Found an estimated cost of 105 for VF 8 For instruction:   store i16 %v5, i16* %out5, align 2
+; AVX1: LV: Found an estimated cost of 258 for VF 16 For instruction:   store i16 %v5, i16* %out5, align 2
 ;
 ; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction:   store i16 %v5, i16* %out5, align 2
 ; AVX2: LV: Found an estimated cost of 13 for VF 2 For instruction:   store i16 %v5, i16* %out5, align 2
 ; AVX2: LV: Found an estimated cost of 17 for VF 4 For instruction:   store i16 %v5, i16* %out5, align 2
 ; AVX2: LV: Found an estimated cost of 24 for VF 8 For instruction:   store i16 %v5, i16* %out5, align 2
 ; AVX2: LV: Found an estimated cost of 64 for VF 16 For instruction:   store i16 %v5, i16* %out5, align 2
 ;
 ; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction:   store i16 %v5, i16* %out5, align 2

llvm/test/Analysis/CostModel/X86/interleaved-store-i8-stride-2.ll

 ; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+sse2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,SSE2
 ; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx  --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX1
 ; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx2 --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX2
 ; RUN: opt -loop-vectorize -vectorizer-maximize-bandwidth -S -mattr=+avx512bw,+avx512vl --debug-only=loop-vectorize < %s 2>&1 | FileCheck %s --check-prefixes=CHECK,AVX512
 ; REQUIRES: asserts
 
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 @A = global [1024 x i8] zeroinitializer, align 128
 @B = global [1024 x i8] zeroinitializer, align 128
 
 ; CHECK: LV: Checking a loop in "test"
 ;
 ; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction:   store i8 %v1, i8* %out1, align 1
-; SSE2: LV: Found an estimated cost of 27 for VF 2 For instruction:   store i8 %v1, i8* %out1, align 1
-; SSE2: LV: Found an estimated cost of 79 for VF 4 For instruction:   store i8 %v1, i8* %out1, align 1
-; SSE2: LV: Found an estimated cost of 238 for VF 8 For instruction:   store i8 %v1, i8* %out1, align 1
-; SSE2: LV: Found an estimated cost of 478 for VF 16 For instruction:   store i8 %v1, i8* %out1, align 1
+; SSE2: LV: Found an estimated cost of 14 for VF 2 For instruction:   store i8 %v1, i8* %out1, align 1
+; SSE2: LV: Found an estimated cost of 30 for VF 4 For instruction:   store i8 %v1, i8* %out1, align 1
+; SSE2: LV: Found an estimated cost of 62 for VF 8 For instruction:   store i8 %v1, i8* %out1, align 1
+; SSE2: LV: Found an estimated cost of 126 for VF 16 For instruction:   store i8 %v1, i8* %out1, align 1
 ;
 ; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction:   store i8 %v1, i8* %out1, align 1
 ; AVX1: LV: Found an estimated cost of 9 for VF 2 For instruction:   store i8 %v1, i8* %out1, align 1
 ; AVX1: LV: Found an estimated cost of 17 for VF 4 For instruction:   store i8 %v1, i8* %out1, align 1
 ; AVX1: LV: Found an estimated cost of 33 for VF 8 For instruction:   store i8 %v1, i8* %out1, align 1
-; AVX1: LV: Found an estimated cost of 97 for VF 16 For instruction:   store i8 %v1, i8* %out1, align 1
-; AVX1: LV: Found an estimated cost of 226 for VF 32 For instruction:   store i8 %v1, i8* %out1, align 1
+; AVX1: LV: Found an estimated cost of 67 for VF 16 For instruction:   store i8 %v1, i8* %out1, align 1
+; AVX1: LV: Found an estimated cost of 166 for VF 32 For instruction:   store i8 %v1, i8* %out1, align 1
 
 ; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction:   store i8 %v1, i8* %out1, align 1
 ; AVX2: LV: Found an estimated cost of 9 for VF 2 For instruction:   store i8 %v1, i8* %out1, align 1
 ; AVX2: LV: Found an estimated cost of 17 for VF 4 For instruction:   store i8 %v1, i8* %out1, align 1
 ; AVX2: LV: Found an estimated cost of 33 for VF 8 For instruction:   store i8 %v1, i8* %out1, align 1
-; AVX2: LV: Found an estimated cost of 97 for VF 16 For instruction:   store i8 %v1, i8* %out1, align 1
-; AVX2: LV: Found an estimated cost of 226 for VF 32 For instruction:   store i8 %v1, i8* %out1, align 1
+; AVX2: LV: Found an estimated cost of 67 for VF 16 For instruction:   store i8 %v1, i8* %out1, align 1
+; AVX2: LV: Found an estimated cost of 166 for VF 32 For instruction:   store i8 %v1, i8* %out1, align 1
 ;
 ; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction:   store i8 %v1, i8* %out1, align 1
 ; AVX512: LV: Found an estimated cost of 4 for VF 2 For instruction:   store i8 %v1, i8* %out1, align 1
 ; AVX512: LV: Found an estimated cost of 4 for VF 4 For instruction:   store i8 %v1, i8* %out1, align 1
 ; AVX512: LV: Found an estimated cost of 4 for VF 8 For instruction:   store i8 %v1, i8* %out1, align 1
 ; AVX512: LV: Found an estimated cost of 8 for VF 16 For instruction:   store i8 %v1, i8* %out1, align 1
 ; AVX512: LV: Found an estimated cost of 20 for VF 32 For instruction:   store i8 %v1, i8* %out1, align 1
 ; AVX512: LV: Found an estimated cost of 41 for VF 64 For instruction:   store i8 %v1, i8* %out1, align 1

llvm/test/Transforms/LoopVectorize/X86/interleaving.ll

 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -S -mtriple=x86_64-pc_linux -loop-vectorize -instcombine < %s | FileCheck %s --check-prefix=SSE
-; RUN: opt -S -mtriple=x86_64-pc_linux -loop-vectorize -instcombine -mcpu=sandybridge < %s | FileCheck %s --check-prefix=AVX
-; RUN: opt -S -mtriple=x86_64-pc_linux -loop-vectorize -instcombine -mcpu=haswell < %s | FileCheck %s --check-prefix=AVX
+; RUN: opt -S -mtriple=x86_64-pc_linux -loop-vectorize -instcombine -mcpu=sandybridge < %s | FileCheck %s --check-prefix=AVX1
+; RUN: opt -S -mtriple=x86_64-pc_linux -loop-vectorize -instcombine -mcpu=haswell < %s | FileCheck %s --check-prefix=AVX2
 ; RUN: opt -S -mtriple=x86_64-pc_linux -loop-vectorize -instcombine -mcpu=slm < %s | FileCheck %s --check-prefix=SSE
 ; RUN: opt -S -mtriple=x86_64-pc_linux -loop-vectorize -instcombine -mcpu=atom < %s | FileCheck %s --check-prefix=SSE
 
 define void @foo(i32* noalias nocapture %a, i32* noalias nocapture readonly %b) {
 ; SSE-LABEL: @foo(
 ; SSE-NEXT:  entry:
 ; SSE-NEXT:    br label [[FOR_BODY:%.*]]
 ; SSE:       for.cond.cleanup:
 ; SSE-NEXT:    ret void
 ; SSE:       for.body:
 ; SSE-NEXT:    [[INDVARS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
 ; SSE-NEXT:    [[TMP0:%.*]] = shl nuw nsw i64 [[INDVARS_IV]], 1
 ; SSE-NEXT:    [[ARRAYIDX:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[TMP0]]
 ; SSE-NEXT:    [[TMP1:%.*]] = load i32, i32* [[ARRAYIDX]], align 4
 ; SSE-NEXT:    [[TMP2:%.*]] = or i64 [[TMP0]], 1
 ; SSE-NEXT:    [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, i32* [[B]], i64 [[TMP2]]
 ; SSE-NEXT:    [[TMP3:%.*]] = load i32, i32* [[ARRAYIDX3]], align 4
 ; SSE-NEXT:    [[ADD4:%.*]] = add nsw i32 [[TMP3]], [[TMP1]]
 ; SSE-NEXT:    [[ARRAYIDX6:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDVARS_IV]]
 ; SSE-NEXT:    store i32 [[ADD4]], i32* [[ARRAYIDX6]], align 4
 ; SSE-NEXT:    [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1
 ; SSE-NEXT:    [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
 ; SSE-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
 ;
-; AVX-LABEL: @foo(
-; AVX-NEXT:  entry:
-; AVX-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
-; AVX:       vector.ph:
-; AVX-NEXT:    br label [[VECTOR_BODY:%.*]]
-; AVX:       vector.body:
-; AVX-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; AVX-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
-; AVX-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[TMP0]]
-; AVX-NEXT:    [[TMP2:%.*]] = bitcast i32* [[TMP1]] to <8 x i32>*
-; AVX-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i32>, <8 x i32>* [[TMP2]], align 4
-; AVX-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; AVX-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
-; AVX-NEXT:    [[TMP3:%.*]] = add nsw <4 x i32> [[STRIDED_VEC1]], [[STRIDED_VEC]]
-; AVX-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDEX]]
-; AVX-NEXT:    [[TMP5:%.*]] = bitcast i32* [[TMP4]] to <4 x i32>*
-; AVX-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* [[TMP5]], align 4
-; AVX-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; AVX-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
-; AVX-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]]
-; AVX:       middle.block:
-; AVX-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
-; AVX:       scalar.ph:
-; AVX-NEXT:    br label [[FOR_BODY:%.*]]
-; AVX:       for.cond.cleanup:
-; AVX-NEXT:    ret void
-; AVX:       for.body:
-; AVX-NEXT:    br i1 undef, label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], [[LOOP2:!llvm.loop !.*]]
+; AVX1-LABEL: @foo(
+; AVX1-NEXT:  entry:
+; AVX1-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; AVX1:       vector.ph:
+; AVX1-NEXT:    br label [[VECTOR_BODY:%.*]]
+; AVX1:       vector.body:
+; AVX1-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; AVX1-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; AVX1-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[TMP0]]
+; AVX1-NEXT:    [[TMP2:%.*]] = bitcast i32* [[TMP1]] to <8 x i32>*
+; AVX1-NEXT:    [[WIDE_VEC:%.*]] = load <8 x i32>, <8 x i32>* [[TMP2]], align 4
+; AVX1-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
+; AVX1-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; AVX1-NEXT:    [[TMP3:%.*]] = add nsw <4 x i32> [[STRIDED_VEC1]], [[STRIDED_VEC]]
+; AVX1-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDEX]]
+; AVX1-NEXT:    [[TMP5:%.*]] = bitcast i32* [[TMP4]] to <4 x i32>*
+; AVX1-NEXT:    store <4 x i32> [[TMP3]], <4 x i32>* [[TMP5]], align 4
+; AVX1-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
+; AVX1-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; AVX1-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; AVX1:       middle.block:
+; AVX1-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; AVX1:       scalar.ph:
+; AVX1-NEXT:    br label [[FOR_BODY:%.*]]
+; AVX1:       for.cond.cleanup:
+; AVX1-NEXT:    ret void
+; AVX1:       for.body:
+; AVX1-NEXT:    br i1 undef, label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
+;
+; AVX2-LABEL: @foo(
+; AVX2-NEXT:  entry:
+; AVX2-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; AVX2:       vector.ph:
+; AVX2-NEXT:    br label [[VECTOR_BODY:%.*]]
+; AVX2:       vector.body:
+; AVX2-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; AVX2-NEXT:    [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
+; AVX2-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[TMP0]]
+; AVX2-NEXT:    [[TMP2:%.*]] = bitcast i32* [[TMP1]] to <16 x i32>*
+; AVX2-NEXT:    [[WIDE_VEC:%.*]] = load <16 x i32>, <16 x i32>* [[TMP2]], align 4
+; AVX2-NEXT:    [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
+; AVX2-NEXT:    [[STRIDED_VEC1:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
+; AVX2-NEXT:    [[TMP3:%.*]] = add nsw <8 x i32> [[STRIDED_VEC1]], [[STRIDED_VEC]]
+; AVX2-NEXT:    [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDEX]]
+; AVX2-NEXT:    [[TMP5:%.*]] = bitcast i32* [[TMP4]] to <8 x i32>*
+; AVX2-NEXT:    store <8 x i32> [[TMP3]], <8 x i32>* [[TMP5]], align 4
+; AVX2-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; AVX2-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; AVX2-NEXT:    br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; AVX2:       middle.block:
+; AVX2-NEXT:    br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
+; AVX2:       scalar.ph:
+; AVX2-NEXT:    br label [[FOR_BODY:%.*]]
+; AVX2:       for.cond.cleanup:
+; AVX2-NEXT:    ret void
+; AVX2:       for.body:
+; AVX2-NEXT:    br i1 undef, label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
 ;
 entry:
   br label %for.body
 
 for.cond.cleanup:                                 ; preds = %for.body
   ret void
 
 for.body:                                         ; preds = %for.body, %entry