This is an archive of the discontinued LLVM Phabricator instance.

Differential D131965

[SLP]Delay vectorization of postponable values for instructions with no users.
ClosedPublic

Authored by ABataev on Aug 16 2022, 6:46 AM.

Details

Reviewers
RKSimon
vdmitrie
Commits
rGc16702868485: [SLP]Delay vectorization of postponable values for instructions with no users.
Summary

SLP vectorizer tries to find the reductions starting the operands of the
instructions with no-users/void returns/etc. But such operands can be
postponable instructions, like Cmp, InsertElement or InsertValue. Such
operands still must be postponed, vectorizer should not try to vectorize
them immediately.

Diff Detail

Revision Contents

PathSize
llvm/
include/
llvm/
Transforms/
Vectorize/
6 lines
lib/
Transforms/
Vectorize/
29 lines
test/
Transforms/
SLPVectorizer/
X86/
54 lines
54 lines
30 lines
72 lines

Diff 454022

llvm/include/llvm/Transforms/Vectorize/SLPVectorizer.h

Show All 14 Lines
// //
//===----------------------------------------------------------------------===// //===----------------------------------------------------------------------===//
#ifndef LLVM_TRANSFORMS_VECTORIZE_SLPVECTORIZER_H #ifndef LLVM_TRANSFORMS_VECTORIZE_SLPVECTORIZER_H
#define LLVM_TRANSFORMS_VECTORIZE_SLPVECTORIZER_H #define LLVM_TRANSFORMS_VECTORIZE_SLPVECTORIZER_H
#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/MapVector.h" #include "llvm/ADT/MapVector.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/SmallVector.h"
#include "llvm/IR/PassManager.h" #include "llvm/IR/PassManager.h"
namespace llvm { namespace llvm {
class AAResults; class AAResults;
class AssumptionCache; class AssumptionCache;
class BasicBlock; class BasicBlock;
Show All 22 Lines
} // end namespace slpvectorizer } // end namespace slpvectorizer
struct SLPVectorizerPass : public PassInfoMixin<SLPVectorizerPass> { struct SLPVectorizerPass : public PassInfoMixin<SLPVectorizerPass> {
using StoreList = SmallVector<StoreInst *, 8>; using StoreList = SmallVector<StoreInst *, 8>;
using StoreListMap = MapVector<Value *, StoreList>; using StoreListMap = MapVector<Value *, StoreList>;
using GEPList = SmallVector<GetElementPtrInst *, 8>; using GEPList = SmallVector<GetElementPtrInst *, 8>;
using GEPListMap = MapVector<Value *, GEPList>; using GEPListMap = MapVector<Value *, GEPList>;
using InstSetVector = SmallSetVector<Instruction *, 8>;
ScalarEvolution *SE = nullptr; ScalarEvolution *SE = nullptr;
TargetTransformInfo *TTI = nullptr; TargetTransformInfo *TTI = nullptr;
TargetLibraryInfo *TLI = nullptr; TargetLibraryInfo *TLI = nullptr;
AAResults *AA = nullptr; AAResults *AA = nullptr;
LoopInfo *LI = nullptr; LoopInfo *LI = nullptr;
DominatorTree *DT = nullptr; DominatorTree *DT = nullptr;
AssumptionCache *AC = nullptr; AssumptionCache *AC = nullptr;
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines bool vectorizeInsertValueInst(InsertValueInst *IVI, BasicBlock *BB,
slpvectorizer::BoUpSLP &R); slpvectorizer::BoUpSLP &R);
/// Try to vectorize trees that start at insertelement instructions. /// Try to vectorize trees that start at insertelement instructions.
bool vectorizeInsertElementInst(InsertElementInst *IEI, BasicBlock *BB, bool vectorizeInsertElementInst(InsertElementInst *IEI, BasicBlock *BB,
slpvectorizer::BoUpSLP &R); slpvectorizer::BoUpSLP &R);
/// Tries to vectorize constructs started from CmpInst, InsertValueInst or /// Tries to vectorize constructs started from CmpInst, InsertValueInst or
/// InsertElementInst instructions. /// InsertElementInst instructions.
bool vectorizeSimpleInstructions(SmallVectorImpl<Instruction *> &Instructions, bool vectorizeSimpleInstructions(InstSetVector &Instructions, BasicBlock *BB,
BasicBlock *BB, slpvectorizer::BoUpSLP &R, slpvectorizer::BoUpSLP &R,
bool AtTerminator); bool AtTerminator);
/// Scan the basic block and look for patterns that are likely to start /// Scan the basic block and look for patterns that are likely to start
/// a vectorization chain. /// a vectorization chain.
bool vectorizeChainsInBlock(BasicBlock *BB, slpvectorizer::BoUpSLP &R); bool vectorizeChainsInBlock(BasicBlock *BB, slpvectorizer::BoUpSLP &R);
bool vectorizeStoreChain(ArrayRef<Value *> Chain, slpvectorizer::BoUpSLP &R, bool vectorizeStoreChain(ArrayRef<Value *> Chain, slpvectorizer::BoUpSLP &R,
unsigned Idx, unsigned MinVF); unsigned Idx, unsigned MinVF);
Show All 13 Lines

llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 11,960 Lines • ▼ Show 20 Lines if (auto *I1 = dyn_cast<Instruction>(Op1))
if (S.getOpcode()) if (S.getOpcode())
continue; continue;
return false; return false;
} }
} }
return IsCompatibility; return IsCompatibility;
} }
bool SLPVectorizerPass::vectorizeSimpleInstructions( bool SLPVectorizerPass::vectorizeSimpleInstructions(InstSetVector &Instructions,
SmallVectorImpl<Instruction *> &Instructions, BasicBlock *BB, BoUpSLP &R, BasicBlock *BB, BoUpSLP &R,
bool AtTerminator) { bool AtTerminator) {
bool OpsChanged = false; bool OpsChanged = false;
SmallVector<Instruction *, 4> PostponedCmps; SmallVector<Instruction *, 4> PostponedCmps;
for (auto *I : reverse(Instructions)) { for (auto *I : reverse(Instructions)) {
if (R.isDeleted(I)) if (R.isDeleted(I))
continue; continue;
if (auto *LastInsertValue = dyn_cast<InsertValueInst>(I)) { if (auto *LastInsertValue = dyn_cast<InsertValueInst>(I)) {
OpsChanged |= vectorizeInsertValueInst(LastInsertValue, BB, R); OpsChanged |= vectorizeInsertValueInst(LastInsertValue, BB, R);
} else if (auto *LastInsertElem = dyn_cast<InsertElementInst>(I)) { } else if (auto *LastInsertElem = dyn_cast<InsertElementInst>(I)) {
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines OpsChanged |= tryToVectorizeSequence<Value>(
}); });
if (ArePossiblyReducedInOtherBlock) if (ArePossiblyReducedInOtherBlock)
return false; return false;
return tryToVectorizeList(Candidates, R, LimitForRegisterSize); return tryToVectorizeList(Candidates, R, LimitForRegisterSize);
}, },
/*LimitForRegisterSize=*/true); /*LimitForRegisterSize=*/true);
Instructions.clear(); Instructions.clear();
} else { } else {
Instructions.clear();
// Insert in reverse order since the PostponedCmps vector was filled in // Insert in reverse order since the PostponedCmps vector was filled in
// reverse order. // reverse order.
Instructions.assign(PostponedCmps.rbegin(), PostponedCmps.rend()); Instructions.insert(PostponedCmps.rbegin(), PostponedCmps.rend());
} }
return OpsChanged; return OpsChanged;
} }
bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) { bool SLPVectorizerPass::vectorizeChainsInBlock(BasicBlock *BB, BoUpSLP &R) {
bool Changed = false; bool Changed = false;
SmallVector<Value *, 4> Incoming; SmallVector<Value *, 4> Incoming;
SmallPtrSet<Value *, 16> VisitedInstrs; SmallPtrSet<Value *, 16> VisitedInstrs;
▲ Show 20 Lines • Show All 136 Lines • ▼ Show 20 Lines HaveVectorizedPhiNodes = tryToVectorizeSequence<Value>(
}, },
/*LimitForRegisterSize=*/true); /*LimitForRegisterSize=*/true);
Changed |= HaveVectorizedPhiNodes; Changed |= HaveVectorizedPhiNodes;
VisitedInstrs.insert(Incoming.begin(), Incoming.end()); VisitedInstrs.insert(Incoming.begin(), Incoming.end());
} while (HaveVectorizedPhiNodes); } while (HaveVectorizedPhiNodes);
VisitedInstrs.clear(); VisitedInstrs.clear();
SmallVector<Instruction *, 8> PostProcessInstructions; InstSetVector PostProcessInstructions;
SmallDenseSet<Instruction *, 4> KeyNodes; SmallDenseSet<Instruction *, 4> KeyNodes;
for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; ++it) { for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; ++it) {
// Skip instructions with scalable type. The num of elements is unknown at // Skip instructions with scalable type. The num of elements is unknown at
// compile-time for scalable type. // compile-time for scalable type.
if (isa<ScalableVectorType>(it->getType())) if (isa<ScalableVectorType>(it->getType()))
continue; continue;
// Skip instructions marked for the deletion. // Skip instructions marked for the deletion.
Show All 35 Lines if (PHINode *P = dyn_cast<PHINode>(it)) {
// Skip if the incoming block is the current BB for now. Also, bypass // Skip if the incoming block is the current BB for now. Also, bypass
// unreachable IR for efficiency and to avoid crashing. // unreachable IR for efficiency and to avoid crashing.
// TODO: Collect the skipped incoming values and try to vectorize them // TODO: Collect the skipped incoming values and try to vectorize them
// after processing BB. // after processing BB.
if (BB == P->getIncomingBlock(I) || if (BB == P->getIncomingBlock(I) ||
!DT->isReachableFromEntry(P->getIncomingBlock(I))) !DT->isReachableFromEntry(P->getIncomingBlock(I)))
continue; continue;
// Postponed instructions should not be vectorized here, delay their
// vectorization.
if (auto *PI = dyn_cast<Instruction>(P->getIncomingValue(I));
PI && !PostProcessInstructions.contains(PI))
Changed |= vectorizeRootInstruction(nullptr, P->getIncomingValue(I), Changed |= vectorizeRootInstruction(nullptr, P->getIncomingValue(I),
P->getIncomingBlock(I), R, TTI); P->getIncomingBlock(I), R, TTI);
} }
continue; continue;
} }
// Ran into an instruction without users, like terminator, or function call // Ran into an instruction without users, like terminator, or function call
// with ignored return value, store. Ignore unused instructions (basing on // with ignored return value, store. Ignore unused instructions (basing on
// instruction type, except for CallInst and InvokeInst). // instruction type, except for CallInst and InvokeInst).
if (it->use_empty() && if (it->use_empty() &&
Show All 10 Lines if (it->use_empty() &&
// to investigate if we can safely turn on slp-vectorize-hor-store // to investigate if we can safely turn on slp-vectorize-hor-store
// instead to allow lookup for reduction chains in all non-vectorized // instead to allow lookup for reduction chains in all non-vectorized
// stores (need to check side effects and compile time). // stores (need to check side effects and compile time).
TryToVectorizeRoot = (I == Stores.end() || I->second.size() == 1) && TryToVectorizeRoot = (I == Stores.end() || I->second.size() == 1) &&
SI->getValueOperand()->hasOneUse(); SI->getValueOperand()->hasOneUse();
} }
if (TryToVectorizeRoot) { if (TryToVectorizeRoot) {
for (auto *V : it->operand_values()) { for (auto *V : it->operand_values()) {
// Postponed instructions should not be vectorized here, delay their
// vectorization.
if (auto *VI = dyn_cast<Instruction>(V);
VI && !PostProcessInstructions.contains(VI))
// Try to match and vectorize a horizontal reduction. // Try to match and vectorize a horizontal reduction.
OpsChanged |= vectorizeRootInstruction(nullptr, V, BB, R, TTI); OpsChanged |= vectorizeRootInstruction(nullptr, V, BB, R, TTI);
} }
} }
// Start vectorization of post-process list of instructions from the // Start vectorization of post-process list of instructions from the
// top-tree instructions to try to vectorize as many instructions as // top-tree instructions to try to vectorize as many instructions as
// possible. // possible.
OpsChanged |= vectorizeSimpleInstructions(PostProcessInstructions, BB, R, OpsChanged |= vectorizeSimpleInstructions(PostProcessInstructions, BB, R,
it->isTerminator()); it->isTerminator());
if (OpsChanged) { if (OpsChanged) {
// We would like to start over since some instructions are deleted // We would like to start over since some instructions are deleted
// and the iterator may become invalid value. // and the iterator may become invalid value.
Changed = true; Changed = true;
it = BB->begin(); it = BB->begin();
e = BB->end(); e = BB->end();
continue; continue;
} }
} }
if (isa<CmpInst, InsertElementInst, InsertValueInst>(it)) if (isa<CmpInst, InsertElementInst, InsertValueInst>(it))
PostProcessInstructions.push_back(&*it); PostProcessInstructions.insert(&*it);
} }
return Changed; return Changed;
} }
bool SLPVectorizerPass::vectorizeGEPIndices(BasicBlock *BB, BoUpSLP &R) { bool SLPVectorizerPass::vectorizeGEPIndices(BasicBlock *BB, BoUpSLP &R) {
auto Changed = false; auto Changed = false;
for (auto &Entry : GEPs) { for (auto &Entry : GEPs) {
▲ Show 20 Lines • Show All 194 Lines • Show Last 20 Lines

llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector-inseltpoison.ll

Show First 20 Lines • Show All 424 Lines • ▼ Show 20 Lines;
%10 = extractelement <2 x float> %8, i32 1 %10 = extractelement <2 x float> %8, i32 1
%rb = insertelement <4 x float> %ra, float %10, i32 1 %rb = insertelement <4 x float> %ra, float %10, i32 1
ret <4 x float> %rb ret <4 x float> %rb
} }
; Make sure that vectorization happens even if insertelements operations ; Make sure that vectorization happens even if insertelements operations
; must be rescheduled. The case here is from compiling Julia. ; must be rescheduled. The case here is from compiling Julia.
define <4 x float> @reschedule_extract(<4 x float> %a, <4 x float> %b) { define <4 x float> @reschedule_extract(<4 x float> %a, <4 x float> %b) {
; THRESHOLD-LABEL: @reschedule_extract( ; CHECK-LABEL: @reschedule_extract(
; THRESHOLD-NEXT: [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]] ; CHECK-NEXT: [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]]
; THRESHOLD-NEXT: ret <4 x float> [[TMP1]] ; CHECK-NEXT: ret <4 x float> [[TMP1]]
;
; NOTHRESHOLD-LABEL: @reschedule_extract(
; NOTHRESHOLD-NEXT: [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]]
; NOTHRESHOLD-NEXT: ret <4 x float> [[TMP1]]
;
; MINTREESIZE-LABEL: @reschedule_extract(
; MINTREESIZE-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[B:%.*]], i32 3
; MINTREESIZE-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[A:%.*]], i32 3
; MINTREESIZE-NEXT: [[TMP3:%.*]] = insertelement <2 x float> poison, float [[TMP2]], i32 0
; MINTREESIZE-NEXT: [[TMP4:%.*]] = insertelement <2 x float> [[TMP3]], float [[TMP1]], i32 1
; MINTREESIZE-NEXT: [[TMP5:%.*]] = fadd <4 x float> [[A]], [[B]]
; MINTREESIZE-NEXT: ret <4 x float> [[TMP5]]
; ;
%a0 = extractelement <4 x float> %a, i32 0 %a0 = extractelement <4 x float> %a, i32 0
%b0 = extractelement <4 x float> %b, i32 0 %b0 = extractelement <4 x float> %b, i32 0
%c0 = fadd float %a0, %b0 %c0 = fadd float %a0, %b0
%v0 = insertelement <4 x float> poison, float %c0, i32 0 %v0 = insertelement <4 x float> poison, float %c0, i32 0
%a1 = extractelement <4 x float> %a, i32 1 %a1 = extractelement <4 x float> %a, i32 1
%b1 = extractelement <4 x float> %b, i32 1 %b1 = extractelement <4 x float> %b, i32 1
%c1 = fadd float %a1, %b1 %c1 = fadd float %a1, %b1
%v1 = insertelement <4 x float> %v0, float %c1, i32 1 %v1 = insertelement <4 x float> %v0, float %c1, i32 1
%a2 = extractelement <4 x float> %a, i32 2 %a2 = extractelement <4 x float> %a, i32 2
%b2 = extractelement <4 x float> %b, i32 2 %b2 = extractelement <4 x float> %b, i32 2
%c2 = fadd float %a2, %b2 %c2 = fadd float %a2, %b2
%v2 = insertelement <4 x float> %v1, float %c2, i32 2 %v2 = insertelement <4 x float> %v1, float %c2, i32 2
%a3 = extractelement <4 x float> %a, i32 3 %a3 = extractelement <4 x float> %a, i32 3
%b3 = extractelement <4 x float> %b, i32 3 %b3 = extractelement <4 x float> %b, i32 3
%c3 = fadd float %a3, %b3 %c3 = fadd float %a3, %b3
%v3 = insertelement <4 x float> %v2, float %c3, i32 3 %v3 = insertelement <4 x float> %v2, float %c3, i32 3
ret <4 x float> %v3 ret <4 x float> %v3
} }
; Check that cost model for vectorization takes credit for ; Check that cost model for vectorization takes credit for
; instructions that are erased. ; instructions that are erased.
define <4 x float> @take_credit(<4 x float> %a, <4 x float> %b) { define <4 x float> @take_credit(<4 x float> %a, <4 x float> %b) {
; THRESHOLD-LABEL: @take_credit( ; CHECK-LABEL: @take_credit(
; THRESHOLD-NEXT: [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]] ; CHECK-NEXT: [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]]
; THRESHOLD-NEXT: ret <4 x float> [[TMP1]] ; CHECK-NEXT: ret <4 x float> [[TMP1]]
;
; NOTHRESHOLD-LABEL: @take_credit(
; NOTHRESHOLD-NEXT: [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]]
; NOTHRESHOLD-NEXT: ret <4 x float> [[TMP1]]
;
; MINTREESIZE-LABEL: @take_credit(
; MINTREESIZE-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[B:%.*]], i32 3
; MINTREESIZE-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[A:%.*]], i32 3
; MINTREESIZE-NEXT: [[TMP3:%.*]] = insertelement <2 x float> poison, float [[TMP2]], i32 0
; MINTREESIZE-NEXT: [[TMP4:%.*]] = insertelement <2 x float> [[TMP3]], float [[TMP1]], i32 1
; MINTREESIZE-NEXT: [[TMP5:%.*]] = fadd <4 x float> [[A]], [[B]]
; MINTREESIZE-NEXT: ret <4 x float> [[TMP5]]
; ;
%a0 = extractelement <4 x float> %a, i32 0 %a0 = extractelement <4 x float> %a, i32 0
%b0 = extractelement <4 x float> %b, i32 0 %b0 = extractelement <4 x float> %b, i32 0
%c0 = fadd float %a0, %b0 %c0 = fadd float %a0, %b0
%a1 = extractelement <4 x float> %a, i32 1 %a1 = extractelement <4 x float> %a, i32 1
%b1 = extractelement <4 x float> %b, i32 1 %b1 = extractelement <4 x float> %b, i32 1
%c1 = fadd float %a1, %b1 %c1 = fadd float %a1, %b1
%a2 = extractelement <4 x float> %a, i32 2 %a2 = extractelement <4 x float> %a, i32 2
Show All 31 Lines;
%t6 = fmul double %t2, 1.000000e+00 %t6 = fmul double %t2, 1.000000e+00
%i3 = insertelement <4 x double> %i2, double %t6, i32 1 %i3 = insertelement <4 x double> %i2, double %t6, i32 1
%t7 = fmul double %t3, 1.000000e+00 %t7 = fmul double %t3, 1.000000e+00
%i4 = insertelement <4 x double> %i3, double %t7, i32 0 %i4 = insertelement <4 x double> %i3, double %t7, i32 0
ret <4 x double> %i4 ret <4 x double> %i4
} }
define <8 x float> @_vadd256(<8 x float> %a, <8 x float> %b) local_unnamed_addr #0 { define <8 x float> @_vadd256(<8 x float> %a, <8 x float> %b) local_unnamed_addr #0 {
; THRESHOLD-LABEL: @_vadd256( ; CHECK-LABEL: @_vadd256(
; THRESHOLD-NEXT: [[TMP1:%.*]] = fadd <8 x float> [[A:%.*]], [[B:%.*]] ; CHECK-NEXT: [[TMP1:%.*]] = fadd <8 x float> [[A:%.*]], [[B:%.*]]
; THRESHOLD-NEXT: ret <8 x float> [[TMP1]] ; CHECK-NEXT: ret <8 x float> [[TMP1]]
;
; NOTHRESHOLD-LABEL: @_vadd256(
; NOTHRESHOLD-NEXT: [[TMP1:%.*]] = fadd <8 x float> [[A:%.*]], [[B:%.*]]
; NOTHRESHOLD-NEXT: ret <8 x float> [[TMP1]]
;
; MINTREESIZE-LABEL: @_vadd256(
; MINTREESIZE-NEXT: [[TMP1:%.*]] = extractelement <8 x float> [[B:%.*]], i32 7
; MINTREESIZE-NEXT: [[TMP2:%.*]] = extractelement <8 x float> [[A:%.*]], i32 7
; MINTREESIZE-NEXT: [[TMP3:%.*]] = insertelement <2 x float> poison, float [[TMP2]], i32 0
; MINTREESIZE-NEXT: [[TMP4:%.*]] = insertelement <2 x float> [[TMP3]], float [[TMP1]], i32 1
; MINTREESIZE-NEXT: [[TMP5:%.*]] = fadd <8 x float> [[A]], [[B]]
; MINTREESIZE-NEXT: ret <8 x float> [[TMP5]]
; ;
%vecext = extractelement <8 x float> %a, i32 0 %vecext = extractelement <8 x float> %a, i32 0
%vecext1 = extractelement <8 x float> %b, i32 0 %vecext1 = extractelement <8 x float> %b, i32 0
%add = fadd float %vecext, %vecext1 %add = fadd float %vecext, %vecext1
%vecext2 = extractelement <8 x float> %a, i32 1 %vecext2 = extractelement <8 x float> %a, i32 1
%vecext3 = extractelement <8 x float> %b, i32 1 %vecext3 = extractelement <8 x float> %b, i32 1
%add4 = fadd float %vecext2, %vecext3 %add4 = fadd float %vecext2, %vecext3
%vecext5 = extractelement <8 x float> %a, i32 2 %vecext5 = extractelement <8 x float> %a, i32 2
Show All 29 Lines

llvm/test/Transforms/SLPVectorizer/X86/insert-element-build-vector.ll

Show First 20 Lines • Show All 459 Lines • ▼ Show 20 Lines;
%10 = extractelement <2 x float> %8, i32 1 %10 = extractelement <2 x float> %8, i32 1
%rb = insertelement <4 x float> %ra, float %10, i32 1 %rb = insertelement <4 x float> %ra, float %10, i32 1
ret <4 x float> %rb ret <4 x float> %rb
} }
; Make sure that vectorization happens even if insertelements operations ; Make sure that vectorization happens even if insertelements operations
; must be rescheduled. The case here is from compiling Julia. ; must be rescheduled. The case here is from compiling Julia.
define <4 x float> @reschedule_extract(<4 x float> %a, <4 x float> %b) { define <4 x float> @reschedule_extract(<4 x float> %a, <4 x float> %b) {
; THRESHOLD-LABEL: @reschedule_extract( ; CHECK-LABEL: @reschedule_extract(
; THRESHOLD-NEXT: [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]] ; CHECK-NEXT: [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]]
; THRESHOLD-NEXT: ret <4 x float> [[TMP1]] ; CHECK-NEXT: ret <4 x float> [[TMP1]]
;
; NOTHRESHOLD-LABEL: @reschedule_extract(
; NOTHRESHOLD-NEXT: [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]]
; NOTHRESHOLD-NEXT: ret <4 x float> [[TMP1]]
;
; MINTREESIZE-LABEL: @reschedule_extract(
; MINTREESIZE-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[B:%.*]], i32 3
; MINTREESIZE-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[A:%.*]], i32 3
; MINTREESIZE-NEXT: [[TMP3:%.*]] = insertelement <2 x float> poison, float [[TMP2]], i32 0
; MINTREESIZE-NEXT: [[TMP4:%.*]] = insertelement <2 x float> [[TMP3]], float [[TMP1]], i32 1
; MINTREESIZE-NEXT: [[TMP5:%.*]] = fadd <4 x float> [[A]], [[B]]
; MINTREESIZE-NEXT: ret <4 x float> [[TMP5]]
; ;
%a0 = extractelement <4 x float> %a, i32 0 %a0 = extractelement <4 x float> %a, i32 0
%b0 = extractelement <4 x float> %b, i32 0 %b0 = extractelement <4 x float> %b, i32 0
%c0 = fadd float %a0, %b0 %c0 = fadd float %a0, %b0
%v0 = insertelement <4 x float> undef, float %c0, i32 0 %v0 = insertelement <4 x float> undef, float %c0, i32 0
%a1 = extractelement <4 x float> %a, i32 1 %a1 = extractelement <4 x float> %a, i32 1
%b1 = extractelement <4 x float> %b, i32 1 %b1 = extractelement <4 x float> %b, i32 1
%c1 = fadd float %a1, %b1 %c1 = fadd float %a1, %b1
%v1 = insertelement <4 x float> %v0, float %c1, i32 1 %v1 = insertelement <4 x float> %v0, float %c1, i32 1
%a2 = extractelement <4 x float> %a, i32 2 %a2 = extractelement <4 x float> %a, i32 2
%b2 = extractelement <4 x float> %b, i32 2 %b2 = extractelement <4 x float> %b, i32 2
%c2 = fadd float %a2, %b2 %c2 = fadd float %a2, %b2
%v2 = insertelement <4 x float> %v1, float %c2, i32 2 %v2 = insertelement <4 x float> %v1, float %c2, i32 2
%a3 = extractelement <4 x float> %a, i32 3 %a3 = extractelement <4 x float> %a, i32 3
%b3 = extractelement <4 x float> %b, i32 3 %b3 = extractelement <4 x float> %b, i32 3
%c3 = fadd float %a3, %b3 %c3 = fadd float %a3, %b3
%v3 = insertelement <4 x float> %v2, float %c3, i32 3 %v3 = insertelement <4 x float> %v2, float %c3, i32 3
ret <4 x float> %v3 ret <4 x float> %v3
} }
; Check that cost model for vectorization takes credit for ; Check that cost model for vectorization takes credit for
; instructions that are erased. ; instructions that are erased.
define <4 x float> @take_credit(<4 x float> %a, <4 x float> %b) { define <4 x float> @take_credit(<4 x float> %a, <4 x float> %b) {
; THRESHOLD-LABEL: @take_credit( ; CHECK-LABEL: @take_credit(
; THRESHOLD-NEXT: [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]] ; CHECK-NEXT: [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]]
; THRESHOLD-NEXT: ret <4 x float> [[TMP1]] ; CHECK-NEXT: ret <4 x float> [[TMP1]]
;
; NOTHRESHOLD-LABEL: @take_credit(
; NOTHRESHOLD-NEXT: [[TMP1:%.*]] = fadd <4 x float> [[A:%.*]], [[B:%.*]]
; NOTHRESHOLD-NEXT: ret <4 x float> [[TMP1]]
;
; MINTREESIZE-LABEL: @take_credit(
; MINTREESIZE-NEXT: [[TMP1:%.*]] = extractelement <4 x float> [[B:%.*]], i32 3
; MINTREESIZE-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[A:%.*]], i32 3
; MINTREESIZE-NEXT: [[TMP3:%.*]] = insertelement <2 x float> poison, float [[TMP2]], i32 0
; MINTREESIZE-NEXT: [[TMP4:%.*]] = insertelement <2 x float> [[TMP3]], float [[TMP1]], i32 1
; MINTREESIZE-NEXT: [[TMP5:%.*]] = fadd <4 x float> [[A]], [[B]]
; MINTREESIZE-NEXT: ret <4 x float> [[TMP5]]
; ;
%a0 = extractelement <4 x float> %a, i32 0 %a0 = extractelement <4 x float> %a, i32 0
%b0 = extractelement <4 x float> %b, i32 0 %b0 = extractelement <4 x float> %b, i32 0
%c0 = fadd float %a0, %b0 %c0 = fadd float %a0, %b0
%a1 = extractelement <4 x float> %a, i32 1 %a1 = extractelement <4 x float> %a, i32 1
%b1 = extractelement <4 x float> %b, i32 1 %b1 = extractelement <4 x float> %b, i32 1
%c1 = fadd float %a1, %b1 %c1 = fadd float %a1, %b1
%a2 = extractelement <4 x float> %a, i32 2 %a2 = extractelement <4 x float> %a, i32 2
Show All 31 Lines;
%t6 = fmul double %t2, 1.000000e+00 %t6 = fmul double %t2, 1.000000e+00
%i3 = insertelement <4 x double> %i2, double %t6, i32 1 %i3 = insertelement <4 x double> %i2, double %t6, i32 1
%t7 = fmul double %t3, 1.000000e+00 %t7 = fmul double %t3, 1.000000e+00
%i4 = insertelement <4 x double> %i3, double %t7, i32 0 %i4 = insertelement <4 x double> %i3, double %t7, i32 0
ret <4 x double> %i4 ret <4 x double> %i4
} }
define <8 x float> @_vadd256(<8 x float> %a, <8 x float> %b) local_unnamed_addr #0 { define <8 x float> @_vadd256(<8 x float> %a, <8 x float> %b) local_unnamed_addr #0 {
; THRESHOLD-LABEL: @_vadd256( ; CHECK-LABEL: @_vadd256(
; THRESHOLD-NEXT: [[TMP1:%.*]] = fadd <8 x float> [[A:%.*]], [[B:%.*]] ; CHECK-NEXT: [[TMP1:%.*]] = fadd <8 x float> [[A:%.*]], [[B:%.*]]
; THRESHOLD-NEXT: ret <8 x float> [[TMP1]] ; CHECK-NEXT: ret <8 x float> [[TMP1]]
;
; NOTHRESHOLD-LABEL: @_vadd256(
; NOTHRESHOLD-NEXT: [[TMP1:%.*]] = fadd <8 x float> [[A:%.*]], [[B:%.*]]
; NOTHRESHOLD-NEXT: ret <8 x float> [[TMP1]]
;
; MINTREESIZE-LABEL: @_vadd256(
; MINTREESIZE-NEXT: [[TMP1:%.*]] = extractelement <8 x float> [[B:%.*]], i32 7
; MINTREESIZE-NEXT: [[TMP2:%.*]] = extractelement <8 x float> [[A:%.*]], i32 7
; MINTREESIZE-NEXT: [[TMP3:%.*]] = insertelement <2 x float> poison, float [[TMP2]], i32 0
; MINTREESIZE-NEXT: [[TMP4:%.*]] = insertelement <2 x float> [[TMP3]], float [[TMP1]], i32 1
; MINTREESIZE-NEXT: [[TMP5:%.*]] = fadd <8 x float> [[A]], [[B]]
; MINTREESIZE-NEXT: ret <8 x float> [[TMP5]]
; ;
%vecext = extractelement <8 x float> %a, i32 0 %vecext = extractelement <8 x float> %a, i32 0
%vecext1 = extractelement <8 x float> %b, i32 0 %vecext1 = extractelement <8 x float> %b, i32 0
%add = fadd float %vecext, %vecext1 %add = fadd float %vecext, %vecext1
%vecext2 = extractelement <8 x float> %a, i32 1 %vecext2 = extractelement <8 x float> %a, i32 1
%vecext3 = extractelement <8 x float> %b, i32 1 %vecext3 = extractelement <8 x float> %b, i32 1
%add4 = fadd float %vecext2, %vecext3 %add4 = fadd float %vecext2, %vecext3
%vecext5 = extractelement <8 x float> %a, i32 2 %vecext5 = extractelement <8 x float> %a, i32 2
Show All 29 Lines

llvm/test/Transforms/SLPVectorizer/X86/insertelement-postpone.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -S -slp-vectorizer -mtriple x86_64-unknown-linux-gnu < %s | FileCheck %s ; RUN: opt -S -slp-vectorizer -mtriple x86_64-unknown-linux-gnu < %s | FileCheck %s
define <4 x double> @test(double* %p2, double %i1754, double %i1781, double %i1778) { define <4 x double> @test(double* %p2, double %i1754, double %i1781, double %i1778) {
; CHECK-LABEL: @test( ; CHECK-LABEL: @test(
; CHECK-NEXT: entry: ; CHECK-NEXT: entry:
; CHECK-NEXT: [[I1771:%.*]] = getelementptr inbounds double, double* [[P2:%.*]], i64 54 ; CHECK-NEXT: [[I1771:%.*]] = getelementptr inbounds double, double* [[P2:%.*]], i64 54
; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x double> poison, double [[I1754:%.*]], i32 0 ; CHECK-NEXT: [[I1772:%.*]] = load double, double* [[I1771]], align 8
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[I1778:%.*]], i32 1 ; CHECK-NEXT: [[I1795:%.*]] = getelementptr inbounds double, double* [[P2]], i64 55
; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> [[TMP0]], double [[I1754]], i32 1 ; CHECK-NEXT: [[I1796:%.*]] = load double, double* [[I1795]], align 8
; CHECK-NEXT: [[TMP3:%.*]] = fmul fast <2 x double> [[TMP1]], [[TMP2]] ; CHECK-NEXT: [[I1797:%.*]] = fmul fast double [[I1796]], [[I1781:%.*]]
; CHECK-NEXT: [[I1792:%.*]] = fmul fast double [[I1754]], [[I1781:%.*]] ; CHECK-NEXT: [[TMP0:%.*]] = insertelement <4 x double> poison, double [[I1754:%.*]], i32 0
; CHECK-NEXT: [[TMP4:%.*]] = bitcast double* [[I1771]] to <2 x double>* ; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x double> [[TMP0]], double [[I1778:%.*]], i32 1
; CHECK-NEXT: [[TMP5:%.*]] = load <2 x double>, <2 x double>* [[TMP4]], align 8 ; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x double> [[TMP1]], double [[I1781]], i32 2
; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x double> [[TMP0]], double [[I1781]], i32 1 ; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[I1772]], i32 3
; CHECK-NEXT: [[TMP7:%.*]] = fmul fast <2 x double> [[TMP5]], [[TMP6]] ; CHECK-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x double> [[TMP0]], <4 x double> poison, <4 x i32> zeroinitializer
; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x double> [[TMP7]], i32 0 ; CHECK-NEXT: [[TMP4:%.*]] = fmul fast <4 x double> [[TMP3]], [[SHUFFLE]]
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x double> [[TMP7]], i32 1 ; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x double> <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double poison>, double [[I1797]], i32 3
; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP3]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef> ; CHECK-NEXT: [[TMP6:%.*]] = fadd fast <4 x double> [[TMP4]], [[TMP5]]
; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x double> [[TMP10]], double [[I1792]], i32 2 ; CHECK-NEXT: ret <4 x double> [[TMP6]]
; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x double> [[TMP11]], double [[TMP8]], i32 3
; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x double> <double 1.000000e+00, double 1.000000e+00, double 1.000000e+00, double poison>, double [[TMP9]], i32 3
; CHECK-NEXT: [[TMP14:%.*]] = fadd fast <4 x double> [[TMP12]], [[TMP13]]
; CHECK-NEXT: ret <4 x double> [[TMP14]]
; ;
entry: entry:
%i1771 = getelementptr inbounds double, double* %p2, i64 54 %i1771 = getelementptr inbounds double, double* %p2, i64 54
%i1772 = load double, double* %i1771, align 8 %i1772 = load double, double* %i1771, align 8
%i1773 = fmul fast double %i1772, %i1754 %i1773 = fmul fast double %i1772, %i1754
%i1782 = fmul fast double %i1754, %i1754 %i1782 = fmul fast double %i1754, %i1754
%i1783 = fadd fast double %i1782, 1.000000e+00 %i1783 = fadd fast double %i1782, 1.000000e+00
%i1787 = fmul fast double %i1778, %i1754 %i1787 = fmul fast double %i1778, %i1754
Show All 13 Lines

llvm/test/Transforms/SLPVectorizer/X86/pr48879-sroa.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -mtriple=x86_64-unknown -slp-vectorizer -opaque-pointers -mcpu=x86-64 -S | FileCheck %s --check-prefixes=SSE ; RUN: opt < %s -mtriple=x86_64-unknown -slp-vectorizer -opaque-pointers -mcpu=x86-64 -S | FileCheck %s --check-prefixes=SSE
; RUN: opt < %s -mtriple=x86_64-unknown -slp-vectorizer -opaque-pointers -mcpu=x86-64-v2 -S | FileCheck %s --check-prefixes=AVX ; RUN: opt < %s -mtriple=x86_64-unknown -slp-vectorizer -opaque-pointers -mcpu=x86-64-v2 -S | FileCheck %s --check-prefixes=AVX
; RUN: opt < %s -mtriple=x86_64-unknown -slp-vectorizer -opaque-pointers -mcpu=x86-64-v3 -S | FileCheck %s --check-prefixes=AVX ; RUN: opt < %s -mtriple=x86_64-unknown -slp-vectorizer -opaque-pointers -mcpu=x86-64-v3 -S | FileCheck %s --check-prefixes=AVX2
; RUN: opt < %s -mtriple=x86_64-unknown -slp-vectorizer -opaque-pointers -mcpu=x86-64-v4 -S | FileCheck %s --check-prefixes=AVX ; RUN: opt < %s -mtriple=x86_64-unknown -slp-vectorizer -opaque-pointers -mcpu=x86-64-v4 -S | FileCheck %s --check-prefixes=AVX2
define { i64, i64 } @compute_min(ptr nocapture noundef nonnull readonly align 2 dereferenceable(16) %x, ptr nocapture noundef nonnull readonly align 2 dereferenceable(16) %y) { define { i64, i64 } @compute_min(ptr nocapture noundef nonnull readonly align 2 dereferenceable(16) %x, ptr nocapture noundef nonnull readonly align 2 dereferenceable(16) %y) {
; SSE-LABEL: @compute_min( ; SSE-LABEL: @compute_min(
; SSE-NEXT: entry: ; SSE-NEXT: entry:
; SSE-NEXT: [[TMP0:%.*]] = load i16, ptr [[Y:%.*]], align 2 ; SSE-NEXT: [[TMP0:%.*]] = load i16, ptr [[Y:%.*]], align 2
; SSE-NEXT: [[TMP1:%.*]] = load i16, ptr [[X:%.*]], align 2 ; SSE-NEXT: [[TMP1:%.*]] = load i16, ptr [[X:%.*]], align 2
; SSE-NEXT: [[TMP2:%.*]] = tail call i16 @llvm.smin.i16(i16 [[TMP0]], i16 [[TMP1]]) ; SSE-NEXT: [[TMP2:%.*]] = tail call i16 @llvm.smin.i16(i16 [[TMP0]], i16 [[TMP1]])
; SSE-NEXT: [[ARRAYIDX_I_I_1:%.*]] = getelementptr inbounds [8 x i16], ptr [[X]], i64 0, i64 1 ; SSE-NEXT: [[ARRAYIDX_I_I_1:%.*]] = getelementptr inbounds [8 x i16], ptr [[X]], i64 0, i64 1
▲ Show 20 Lines • Show All 104 Lines • ▼ Show 20 Lines
; AVX-NEXT: [[RETVAL_SROA_7_8_INSERT_EXT:%.*]] = zext i16 [[TMP11]] to i64 ; AVX-NEXT: [[RETVAL_SROA_7_8_INSERT_EXT:%.*]] = zext i16 [[TMP11]] to i64
; AVX-NEXT: [[RETVAL_SROA_7_8_INSERT_SHIFT:%.*]] = shl nuw nsw i64 [[RETVAL_SROA_7_8_INSERT_EXT]], 16 ; AVX-NEXT: [[RETVAL_SROA_7_8_INSERT_SHIFT:%.*]] = shl nuw nsw i64 [[RETVAL_SROA_7_8_INSERT_EXT]], 16
; AVX-NEXT: [[RETVAL_SROA_7_8_INSERT_INSERT:%.*]] = or i64 [[RETVAL_SROA_8_8_INSERT_INSERT]], [[RETVAL_SROA_7_8_INSERT_SHIFT]] ; AVX-NEXT: [[RETVAL_SROA_7_8_INSERT_INSERT:%.*]] = or i64 [[RETVAL_SROA_8_8_INSERT_INSERT]], [[RETVAL_SROA_7_8_INSERT_SHIFT]]
; AVX-NEXT: [[RETVAL_SROA_5_8_INSERT_EXT:%.*]] = zext i16 [[TMP8]] to i64 ; AVX-NEXT: [[RETVAL_SROA_5_8_INSERT_EXT:%.*]] = zext i16 [[TMP8]] to i64
; AVX-NEXT: [[RETVAL_SROA_5_8_INSERT_INSERT:%.*]] = or i64 [[RETVAL_SROA_7_8_INSERT_INSERT]], [[RETVAL_SROA_5_8_INSERT_EXT]] ; AVX-NEXT: [[RETVAL_SROA_5_8_INSERT_INSERT:%.*]] = or i64 [[RETVAL_SROA_7_8_INSERT_INSERT]], [[RETVAL_SROA_5_8_INSERT_EXT]]
; AVX-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue { i64, i64 } [[DOTFCA_0_INSERT]], i64 [[RETVAL_SROA_5_8_INSERT_INSERT]], 1 ; AVX-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue { i64, i64 } [[DOTFCA_0_INSERT]], i64 [[RETVAL_SROA_5_8_INSERT_INSERT]], 1
; AVX-NEXT: ret { i64, i64 } [[DOTFCA_1_INSERT]] ; AVX-NEXT: ret { i64, i64 } [[DOTFCA_1_INSERT]]
; ;
; AVX2-LABEL: @compute_min(
; AVX2-NEXT: entry:
; AVX2-NEXT: [[ARRAYIDX_I_I_1:%.*]] = getelementptr inbounds [8 x i16], ptr [[X:%.*]], i64 0, i64 1
; AVX2-NEXT: [[ARRAYIDX_I_I10_1:%.*]] = getelementptr inbounds [8 x i16], ptr [[Y:%.*]], i64 0, i64 1
; AVX2-NEXT: [[ARRAYIDX_I_I_2:%.*]] = getelementptr inbounds [8 x i16], ptr [[X]], i64 0, i64 2
; AVX2-NEXT: [[ARRAYIDX_I_I10_2:%.*]] = getelementptr inbounds [8 x i16], ptr [[Y]], i64 0, i64 2
; AVX2-NEXT: [[ARRAYIDX_I_I_3:%.*]] = getelementptr inbounds [8 x i16], ptr [[X]], i64 0, i64 3
; AVX2-NEXT: [[ARRAYIDX_I_I10_3:%.*]] = getelementptr inbounds [8 x i16], ptr [[Y]], i64 0, i64 3
; AVX2-NEXT: [[ARRAYIDX_I_I_4:%.*]] = getelementptr inbounds [8 x i16], ptr [[X]], i64 0, i64 4
; AVX2-NEXT: [[ARRAYIDX_I_I10_4:%.*]] = getelementptr inbounds [8 x i16], ptr [[Y]], i64 0, i64 4
; AVX2-NEXT: [[ARRAYIDX_I_I_5:%.*]] = getelementptr inbounds [8 x i16], ptr [[X]], i64 0, i64 5
; AVX2-NEXT: [[ARRAYIDX_I_I10_5:%.*]] = getelementptr inbounds [8 x i16], ptr [[Y]], i64 0, i64 5
; AVX2-NEXT: [[ARRAYIDX_I_I_6:%.*]] = getelementptr inbounds [8 x i16], ptr [[X]], i64 0, i64 6
; AVX2-NEXT: [[ARRAYIDX_I_I10_6:%.*]] = getelementptr inbounds [8 x i16], ptr [[Y]], i64 0, i64 6
; AVX2-NEXT: [[ARRAYIDX_I_I_7:%.*]] = getelementptr inbounds [8 x i16], ptr [[X]], i64 0, i64 7
; AVX2-NEXT: [[ARRAYIDX_I_I10_7:%.*]] = getelementptr inbounds [8 x i16], ptr [[Y]], i64 0, i64 7
; AVX2-NEXT: [[TMP0:%.*]] = load i16, ptr [[Y]], align 2
; AVX2-NEXT: [[TMP1:%.*]] = load i16, ptr [[X]], align 2
; AVX2-NEXT: [[TMP2:%.*]] = load i16, ptr [[ARRAYIDX_I_I10_1]], align 2
; AVX2-NEXT: [[TMP3:%.*]] = load i16, ptr [[ARRAYIDX_I_I_1]], align 2
; AVX2-NEXT: [[TMP4:%.*]] = load i16, ptr [[ARRAYIDX_I_I10_2]], align 2
; AVX2-NEXT: [[TMP5:%.*]] = load i16, ptr [[ARRAYIDX_I_I_2]], align 2
; AVX2-NEXT: [[TMP6:%.*]] = load i16, ptr [[ARRAYIDX_I_I10_3]], align 2
; AVX2-NEXT: [[TMP7:%.*]] = load i16, ptr [[ARRAYIDX_I_I_3]], align 2
; AVX2-NEXT: [[TMP8:%.*]] = load i16, ptr [[ARRAYIDX_I_I10_4]], align 2
; AVX2-NEXT: [[TMP9:%.*]] = load i16, ptr [[ARRAYIDX_I_I_4]], align 2
; AVX2-NEXT: [[TMP10:%.*]] = insertelement <2 x i16> poison, i16 [[TMP0]], i32 0
; AVX2-NEXT: [[TMP11:%.*]] = insertelement <2 x i16> [[TMP10]], i16 [[TMP8]], i32 1
; AVX2-NEXT: [[TMP12:%.*]] = insertelement <2 x i16> poison, i16 [[TMP1]], i32 0
; AVX2-NEXT: [[TMP13:%.*]] = insertelement <2 x i16> [[TMP12]], i16 [[TMP9]], i32 1
; AVX2-NEXT: [[TMP14:%.*]] = call <2 x i16> @llvm.smin.v2i16(<2 x i16> [[TMP11]], <2 x i16> [[TMP13]])
; AVX2-NEXT: [[TMP15:%.*]] = load i16, ptr [[ARRAYIDX_I_I10_5]], align 2
; AVX2-NEXT: [[TMP16:%.*]] = load i16, ptr [[ARRAYIDX_I_I_5]], align 2
; AVX2-NEXT: [[TMP17:%.*]] = insertelement <2 x i16> poison, i16 [[TMP2]], i32 0
; AVX2-NEXT: [[TMP18:%.*]] = insertelement <2 x i16> [[TMP17]], i16 [[TMP15]], i32 1
; AVX2-NEXT: [[TMP19:%.*]] = insertelement <2 x i16> poison, i16 [[TMP3]], i32 0
; AVX2-NEXT: [[TMP20:%.*]] = insertelement <2 x i16> [[TMP19]], i16 [[TMP16]], i32 1
; AVX2-NEXT: [[TMP21:%.*]] = call <2 x i16> @llvm.smin.v2i16(<2 x i16> [[TMP18]], <2 x i16> [[TMP20]])
; AVX2-NEXT: [[TMP22:%.*]] = load i16, ptr [[ARRAYIDX_I_I10_6]], align 2
; AVX2-NEXT: [[TMP23:%.*]] = load i16, ptr [[ARRAYIDX_I_I_6]], align 2
; AVX2-NEXT: [[TMP24:%.*]] = insertelement <2 x i16> poison, i16 [[TMP6]], i32 0
; AVX2-NEXT: [[TMP25:%.*]] = insertelement <2 x i16> [[TMP24]], i16 [[TMP22]], i32 1
; AVX2-NEXT: [[TMP26:%.*]] = insertelement <2 x i16> poison, i16 [[TMP7]], i32 0
; AVX2-NEXT: [[TMP27:%.*]] = insertelement <2 x i16> [[TMP26]], i16 [[TMP23]], i32 1
; AVX2-NEXT: [[TMP28:%.*]] = call <2 x i16> @llvm.smin.v2i16(<2 x i16> [[TMP25]], <2 x i16> [[TMP27]])
; AVX2-NEXT: [[TMP29:%.*]] = load i16, ptr [[ARRAYIDX_I_I10_7]], align 2
; AVX2-NEXT: [[TMP30:%.*]] = load i16, ptr [[ARRAYIDX_I_I_7]], align 2
; AVX2-NEXT: [[TMP31:%.*]] = insertelement <2 x i16> poison, i16 [[TMP4]], i32 0
; AVX2-NEXT: [[TMP32:%.*]] = insertelement <2 x i16> [[TMP31]], i16 [[TMP29]], i32 1
; AVX2-NEXT: [[TMP33:%.*]] = insertelement <2 x i16> poison, i16 [[TMP5]], i32 0
; AVX2-NEXT: [[TMP34:%.*]] = insertelement <2 x i16> [[TMP33]], i16 [[TMP30]], i32 1
; AVX2-NEXT: [[TMP35:%.*]] = call <2 x i16> @llvm.smin.v2i16(<2 x i16> [[TMP32]], <2 x i16> [[TMP34]])
; AVX2-NEXT: [[TMP36:%.*]] = zext <2 x i16> [[TMP35]] to <2 x i64>
; AVX2-NEXT: [[TMP37:%.*]] = shl nuw <2 x i64> [[TMP36]], <i64 32, i64 48>
; AVX2-NEXT: [[TMP38:%.*]] = zext <2 x i16> [[TMP28]] to <2 x i64>
; AVX2-NEXT: [[TMP39:%.*]] = shl nuw <2 x i64> [[TMP38]], <i64 48, i64 32>
; AVX2-NEXT: [[TMP40:%.*]] = or <2 x i64> [[TMP37]], [[TMP39]]
; AVX2-NEXT: [[TMP41:%.*]] = zext <2 x i16> [[TMP21]] to <2 x i64>
; AVX2-NEXT: [[TMP42:%.*]] = shl nuw nsw <2 x i64> [[TMP41]], <i64 16, i64 16>
; AVX2-NEXT: [[TMP43:%.*]] = or <2 x i64> [[TMP40]], [[TMP42]]
; AVX2-NEXT: [[TMP44:%.*]] = zext <2 x i16> [[TMP14]] to <2 x i64>
; AVX2-NEXT: [[TMP45:%.*]] = or <2 x i64> [[TMP43]], [[TMP44]]
; AVX2-NEXT: [[TMP46:%.*]] = extractelement <2 x i64> [[TMP45]], i32 0
; AVX2-NEXT: [[DOTFCA_0_INSERT:%.*]] = insertvalue { i64, i64 } poison, i64 [[TMP46]], 0
; AVX2-NEXT: [[TMP47:%.*]] = extractelement <2 x i64> [[TMP45]], i32 1
; AVX2-NEXT: [[DOTFCA_1_INSERT:%.*]] = insertvalue { i64, i64 } [[DOTFCA_0_INSERT]], i64 [[TMP47]], 1
; AVX2-NEXT: ret { i64, i64 } [[DOTFCA_1_INSERT]]
;
entry: entry:
%0 = load i16, ptr %y, align 2 %0 = load i16, ptr %y, align 2
%1 = load i16, ptr %x, align 2 %1 = load i16, ptr %x, align 2
%2 = tail call i16 @llvm.smin.i16(i16 %0, i16 %1) %2 = tail call i16 @llvm.smin.i16(i16 %0, i16 %1)
%arrayidx.i.i.1 = getelementptr inbounds [8 x i16], ptr %x, i64 0, i64 1 %arrayidx.i.i.1 = getelementptr inbounds [8 x i16], ptr %x, i64 0, i64 1
%arrayidx.i.i10.1 = getelementptr inbounds [8 x i16], ptr %y, i64 0, i64 1 %arrayidx.i.i10.1 = getelementptr inbounds [8 x i16], ptr %y, i64 0, i64 1
%3 = load i16, ptr %arrayidx.i.i10.1, align 2 %3 = load i16, ptr %arrayidx.i.i10.1, align 2
%4 = load i16, ptr %arrayidx.i.i.1, align 2 %4 = load i16, ptr %arrayidx.i.i.1, align 2
▲ Show 20 Lines • Show All 56 Lines • Show Last 20 Lines