This is an archive of the discontinued LLVM Phabricator instance.

Differential D115712

[LV] Make sure prefer-predicate-over-epilogue works for short TC loops
Needs ReviewPublic

Authored by ebrevnov on Dec 14 2021, 1:37 AM.

Details

Reviewers
fhahn
anna
nikic
lebedev.ri
Summary

Currently "short trip count" heuristic overrides user settings of PreferPredicateOverEpilogue and llvm.loop.vectorize.predicate.enable. That doesn't seem right for several reasons. Typically user settings are expected to override compiler chosen heuristics. Another reason is surprising difference in behavior for "known" short trip count loops and all other cases.

PS: This change is exploited in https://reviews.llvm.org/D115713 to be able to keep testing coverage.

Diff Detail

Event Timeline

ebrevnov created this revision.Dec 14 2021, 1:37 AM
Herald added a subscriber: hiraditya. · View Herald TranscriptDec 14 2021, 1:37 AM
ebrevnov requested review of this revision.Dec 14 2021, 1:37 AM
Herald added a project: Restricted Project. · View Herald TranscriptDec 14 2021, 1:37 AM
Herald added a subscriber: llvm-commits. · View Herald Transcript
Harbormaster completed remote builds in B139184: Diff 394186.Dec 14 2021, 1:38 AM
ebrevnov mentioned this in D115711: [LV][NFC] New test case to check prefer-predicate-over-epilogue for short TC loops.Dec 14 2021, 1:41 AM
ebrevnov added a parent revision: D115711: [LV][NFC] New test case to check prefer-predicate-over-epilogue for short TC loops.
ebrevnov edited the summary of this revision. (Show Details)Dec 14 2021, 1:53 AM
ebrevnov added a reviewer: fhahn.
ebrevnov added a child revision: D115713: [LV] Don't apply "TinyTripCountVectorThreshold" for loops with compile time known TC..
xbolva00 added a subscriber: xbolva00.Jan 12 2022, 1:48 AM
ebrevnov added a comment.Jan 24 2022, 4:59 AM

Ping

ebrevnov added reviewers: anna, nikic, lebedev.ri.Oct 9 2022, 7:24 AM
Herald added a project: Restricted Project. · View Herald TranscriptOct 9 2022, 7:24 AM
Herald added a subscriber: • pcwang-thead. · View Herald Transcript
ebrevnov added a comment.Oct 9 2022, 7:24 AM

ping

ebrevnov updated this revision to Diff 467816.Oct 14 2022, 10:09 AM

rebase

ebrevnov removed a child revision: D115713: [LV] Don't apply "TinyTripCountVectorThreshold" for loops with compile time known TC..Oct 14 2022, 10:13 AM
ebrevnov added a child revision: D135971: [LV][NFC][WIP] Restructure getScalarEpilogueLowering.
Harbormaster completed remote builds in B192209: Diff 467816.Oct 14 2022, 11:06 AM
lebedev.ri resigned from this revision.Jan 12 2023, 5:31 PM

This review may be stuck/dead, consider abandoning if no longer relevant.
Removing myself as reviewer in attempt to clean dashboard.

Herald added a subscriber: StephenFan. · View Herald TranscriptJan 12 2023, 5:31 PM
nikic resigned from this revision.Jan 20 2023, 12:33 PM

(not familiar with LV)

Revision Contents

PathSize
llvm/
lib/
Transforms/
Vectorize/
40 lines
test/
Transforms/
LoopVectorize/
641 lines

Diff 394186

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 10,037 Lines • ▼ Show 20 Linesstatic ScalarEpilogueLowering getScalarEpilogueLowering(
// 3) If set, obey the hints // 3) If set, obey the hints
switch (Hints.getPredicate()) { switch (Hints.getPredicate()) {
case LoopVectorizeHints::FK_Enabled: case LoopVectorizeHints::FK_Enabled:
return CM_ScalarEpilogueNotNeededUsePredicate; return CM_ScalarEpilogueNotNeededUsePredicate;
case LoopVectorizeHints::FK_Disabled: case LoopVectorizeHints::FK_Disabled:
return CM_ScalarEpilogueAllowed; return CM_ScalarEpilogueAllowed;
}; };
// 4) if the TTI hook indicates this is profitable, request predication. // 4) Check the loop for a trip count threshold. Vectorize loops with a tiny
// trip count by optimizing for size, to minimize overheads.
auto ExpectedTC = getSmallBestKnownTC(*SE, L);
if (ExpectedTC && *ExpectedTC < TinyTripCountVectorThreshold) {
LLVM_DEBUG(dbgs() << "LV: Found a loop with a very small trip count. "
<< "This loop is worth vectorizing only if no scalar "
<< "iteration overheads are incurred.");
if (Hints.getForce() == LoopVectorizeHints::FK_Enabled)
LLVM_DEBUG(dbgs() << " But vectorizing was explicitly forced.\n");
else {
LLVM_DEBUG(dbgs() << "\n");
return CM_ScalarEpilogueNotAllowedLowTripLoop;
}
}
// 5) if the TTI hook indicates this is profitable, request predication.
if (TTI->preferPredicateOverEpilogue(L, LI, *SE, *AC, TLI, DT, if (TTI->preferPredicateOverEpilogue(L, LI, *SE, *AC, TLI, DT,
LVL.getLAI())) LVL.getLAI()))
return CM_ScalarEpilogueNotNeededUsePredicate; return CM_ScalarEpilogueNotNeededUsePredicate;
return CM_ScalarEpilogueAllowed; return CM_ScalarEpilogueAllowed;
} }
Value *VPTransformState::get(VPValue *Def, unsigned Part) { Value *VPTransformState::get(VPValue *Def, unsigned Part) {
▲ Show 20 Lines • Show All 226 Lines • ▼ Show 20 Lines#endif /* NDEBUG */
LoopVectorizationLegality LVL(L, PSE, DT, TTI, TLI, AA, F, GetLAA, LI, ORE, LoopVectorizationLegality LVL(L, PSE, DT, TTI, TLI, AA, F, GetLAA, LI, ORE,
&Requirements, &Hints, DB, AC, BFI, PSI); &Requirements, &Hints, DB, AC, BFI, PSI);
if (!LVL.canVectorize(EnableVPlanNativePath)) { if (!LVL.canVectorize(EnableVPlanNativePath)) {
LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n"); LLVM_DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n");
Hints.emitRemarkWithHints(); Hints.emitRemarkWithHints();
return false; return false;
} }
// Check the function attributes and profiles to find out if this function
// should be optimized for size.
ScalarEpilogueLowering SEL = getScalarEpilogueLowering(
F, L, Hints, PSI, BFI, TTI, TLI, AC, LI, PSE.getSE(), DT, LVL);
// Entrance to the VPlan-native vectorization path. Outer loops are processed // Entrance to the VPlan-native vectorization path. Outer loops are processed
// here. They may require CFG and instruction level transformations before // here. They may require CFG and instruction level transformations before
// even evaluating whether vectorization is profitable. Since we cannot modify // even evaluating whether vectorization is profitable. Since we cannot modify
// the incoming IR, we need to build VPlan upfront in the vectorization // the incoming IR, we need to build VPlan upfront in the vectorization
// pipeline. // pipeline.
if (!L->isInnermost()) if (!L->isInnermost())
return processLoopInVPlanNativePath(L, PSE, LI, DT, &LVL, TTI, TLI, DB, AC, return processLoopInVPlanNativePath(L, PSE, LI, DT, &LVL, TTI, TLI, DB, AC,
ORE, BFI, PSI, Hints, Requirements); ORE, BFI, PSI, Hints, Requirements);
assert(L->isInnermost() && "Inner loop expected."); assert(L->isInnermost() && "Inner loop expected.");
// Check the loop for a trip count threshold: vectorize loops with a tiny trip // Check the function attributes and profiles to find out if this function
// count by optimizing for size, to minimize overheads. // should be optimized for size.
auto ExpectedTC = getSmallBestKnownTC(*SE, L); ScalarEpilogueLowering SEL = getScalarEpilogueLowering(
if (ExpectedTC && *ExpectedTC < TinyTripCountVectorThreshold) { F, L, Hints, PSI, BFI, TTI, TLI, AC, LI, PSE.getSE(), DT, LVL);
LLVM_DEBUG(dbgs() << "LV: Found a loop with a very small trip count. "
<< "This loop is worth vectorizing only if no scalar "
<< "iteration overheads are incurred.");
if (Hints.getForce() == LoopVectorizeHints::FK_Enabled)
LLVM_DEBUG(dbgs() << " But vectorizing was explicitly forced.\n");
else {
LLVM_DEBUG(dbgs() << "\n");
SEL = CM_ScalarEpilogueNotAllowedLowTripLoop;
}
}
// Check the function attributes to see if implicit floats are allowed. // Check the function attributes to see if implicit floats are allowed.
// FIXME: This check doesn't seem possibly correct -- what if the loop is // FIXME: This check doesn't seem possibly correct -- what if the loop is
// an integer loop and the vector instructions selected are purely integer // an integer loop and the vector instructions selected are purely integer
// vector instructions? // vector instructions?
if (F->hasFnAttribute(Attribute::NoImplicitFloat)) { if (F->hasFnAttribute(Attribute::NoImplicitFloat)) {
reportVectorizationFailure( reportVectorizationFailure(
"Can't vectorize when the NoImplicitFloat attribute is used", "Can't vectorize when the NoImplicitFloat attribute is used",
▲ Show 20 Lines • Show All 399 Lines • Show Last 20 Lines

llvm/test/Transforms/LoopVectorize/override-short-tc-heuristic.ll

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt < %s -passes=loop-vectorize -S | FileCheck %s ; RUN: opt < %s -passes=loop-vectorize -S | FileCheck %s
; RUN: opt < %s -passes=loop-vectorize -prefer-predicate-over-epilogue=scalar-epilogue -S | FileCheck -check-prefix CHECK-SCALAR %s ; RUN: opt < %s -passes=loop-vectorize -prefer-predicate-over-epilogue=scalar-epilogue -S | FileCheck -check-prefix CHECK-SCALAR %s
; RUN: opt < %s -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -S | FileCheck -check-prefix CHECK-PRED-OR-SCALAR %s ; RUN: opt < %s -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-else-scalar-epilogue -S | FileCheck -check-prefix CHECK-PRED-OR-SCALAR %s
; RUN: opt < %s -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-dont-vectorize -S | FileCheck -check-prefix PRED-OR-DONTVEC %s ; RUN: opt < %s -passes=loop-vectorize -prefer-predicate-over-epilogue=predicate-dont-vectorize -S | FileCheck -check-prefix PRED-OR-DONTVEC %s
;@src = common global [32 x i8] zeroinitializer, align 1 ;@src = common global [32 x i8] zeroinitializer, align 1
;@dst = common global [32 x i8] zeroinitializer, align 1 ;@dst = common global [32 x i8] zeroinitializer, align 1
; Simple loop with small constant trip count. ; Simple loop with small constant trip count.
; TODO: Check that -prefer-predicate-over-epilogue and "llvm.loop.vectorize.predicate.enable=true" ; Check that -prefer-predicate-over-epilogue and "llvm.loop.vectorize.predicate.enable=true"
; hint override "small trip count" heuristic. ; hint override "small trip count" heuristic.
define i32 @const_low_trip_count_hint_pred(i8 *%dst, i8 *%src) { define i32 @const_low_trip_count_hint_pred(i8 *%dst, i8 *%src) {
; CHECK-LABEL: @const_low_trip_count_hint_pred( ; CHECK-LABEL: @const_low_trip_count_hint_pred(
; CHECK-NEXT: entry: ; CHECK-NEXT: entry:
; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; CHECK: vector.memcheck:
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, i8* [[DST:%.*]], i64 9
; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, i8* [[SRC:%.*]], i64 9
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[DST]], [[SCEVGEP1]]
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SRC]], [[SCEVGEP]]
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE13:%.*]] ]
; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE13]] ]
; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0
; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1
; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 2
; CHECK-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 3
; CHECK-NEXT: [[TMP4:%.*]] = icmp ule <4 x i32> [[VEC_IND]], <i32 8, i32 8, i32 8, i32 8>
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
; CHECK-NEXT: br i1 [[TMP5]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
; CHECK: pred.load.if:
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[TMP0]]
; CHECK-NEXT: [[TMP7:%.*]] = load i8, i8* [[TMP6]], align 1, !alias.scope !0
; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x i8> poison, i8 [[TMP7]], i32 0
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]]
; CHECK: pred.load.continue:
; CHECK-NEXT: [[TMP9:%.*]] = phi <4 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_LOAD_IF]] ]
; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
; CHECK-NEXT: br i1 [[TMP10]], label [[PRED_LOAD_IF2:%.*]], label [[PRED_LOAD_CONTINUE3:%.*]]
; CHECK: pred.load.if2:
; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[TMP1]]
; CHECK-NEXT: [[TMP12:%.*]] = load i8, i8* [[TMP11]], align 1, !alias.scope !0
; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i8> [[TMP9]], i8 [[TMP12]], i32 1
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE3]]
; CHECK: pred.load.continue3:
; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i8> [ [[TMP9]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], [[PRED_LOAD_IF2]] ]
; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
; CHECK-NEXT: br i1 [[TMP15]], label [[PRED_LOAD_IF4:%.*]], label [[PRED_LOAD_CONTINUE5:%.*]]
; CHECK: pred.load.if4:
; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[TMP2]]
; CHECK-NEXT: [[TMP17:%.*]] = load i8, i8* [[TMP16]], align 1, !alias.scope !0
; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i8> [[TMP14]], i8 [[TMP17]], i32 2
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE5]]
; CHECK: pred.load.continue5:
; CHECK-NEXT: [[TMP19:%.*]] = phi <4 x i8> [ [[TMP14]], [[PRED_LOAD_CONTINUE3]] ], [ [[TMP18]], [[PRED_LOAD_IF4]] ]
; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF6:%.*]], label [[PRED_LOAD_CONTINUE7:%.*]]
; CHECK: pred.load.if6:
; CHECK-NEXT: [[TMP21:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[TMP3]]
; CHECK-NEXT: [[TMP22:%.*]] = load i8, i8* [[TMP21]], align 1, !alias.scope !0
; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP22]], i32 3
; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE7]]
; CHECK: pred.load.continue7:
; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x i8> [ [[TMP19]], [[PRED_LOAD_CONTINUE5]] ], [ [[TMP23]], [[PRED_LOAD_IF6]] ]
; CHECK-NEXT: [[TMP25:%.*]] = icmp eq <4 x i8> [[TMP24]], <i8 5, i8 5, i8 5, i8 5>
; CHECK-NEXT: [[TMP26:%.*]] = select <4 x i1> [[TMP25]], <4 x i8> <i8 1, i8 1, i8 1, i8 1>, <4 x i8> <i8 2, i8 2, i8 2, i8 2>
; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
; CHECK-NEXT: br i1 [[TMP27]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; CHECK: pred.store.if:
; CHECK-NEXT: [[TMP28:%.*]] = getelementptr i8, i8* [[DST]], i32 [[TMP0]]
; CHECK-NEXT: [[TMP29:%.*]] = extractelement <4 x i8> [[TMP26]], i32 0
; CHECK-NEXT: store i8 [[TMP29]], i8* [[TMP28]], align 1, !alias.scope !3, !noalias !0
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE]]
; CHECK: pred.store.continue:
; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
; CHECK-NEXT: br i1 [[TMP30]], label [[PRED_STORE_IF8:%.*]], label [[PRED_STORE_CONTINUE9:%.*]]
; CHECK: pred.store.if8:
; CHECK-NEXT: [[TMP31:%.*]] = getelementptr i8, i8* [[DST]], i32 [[TMP1]]
; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i8> [[TMP26]], i32 1
; CHECK-NEXT: store i8 [[TMP32]], i8* [[TMP31]], align 1, !alias.scope !3, !noalias !0
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE9]]
; CHECK: pred.store.continue9:
; CHECK-NEXT: [[TMP33:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
; CHECK-NEXT: br i1 [[TMP33]], label [[PRED_STORE_IF10:%.*]], label [[PRED_STORE_CONTINUE11:%.*]]
; CHECK: pred.store.if10:
; CHECK-NEXT: [[TMP34:%.*]] = getelementptr i8, i8* [[DST]], i32 [[TMP2]]
; CHECK-NEXT: [[TMP35:%.*]] = extractelement <4 x i8> [[TMP26]], i32 2
; CHECK-NEXT: store i8 [[TMP35]], i8* [[TMP34]], align 1, !alias.scope !3, !noalias !0
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE11]]
; CHECK: pred.store.continue11:
; CHECK-NEXT: [[TMP36:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
; CHECK-NEXT: br i1 [[TMP36]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13]]
; CHECK: pred.store.if12:
; CHECK-NEXT: [[TMP37:%.*]] = getelementptr i8, i8* [[DST]], i32 [[TMP3]]
; CHECK-NEXT: [[TMP38:%.*]] = extractelement <4 x i8> [[TMP26]], i32 3
; CHECK-NEXT: store i8 [[TMP38]], i8* [[TMP37]], align 1, !alias.scope !3, !noalias !0
; CHECK-NEXT: br label [[PRED_STORE_CONTINUE13]]
; CHECK: pred.store.continue13:
; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
; CHECK-NEXT: [[TMP39:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
; CHECK-NEXT: br i1 [[TMP39]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 12, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: br label [[FOR_BODY:%.*]] ; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.body: ; CHECK: for.body:
; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ] ; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
; CHECK-NEXT: [[LDIDX:%.*]] = getelementptr i8, i8* [[SRC:%.*]], i32 [[I]] ; CHECK-NEXT: [[LDIDX:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[I]]
; CHECK-NEXT: [[STIDX:%.*]] = getelementptr i8, i8* [[DST:%.*]], i32 [[I]] ; CHECK-NEXT: [[STIDX:%.*]] = getelementptr i8, i8* [[DST]], i32 [[I]]
; CHECK-NEXT: [[LDVAL:%.*]] = load i8, i8* [[LDIDX]], align 1 ; CHECK-NEXT: [[LDVAL:%.*]] = load i8, i8* [[LDIDX]], align 1
; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LDVAL]], 5 ; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LDVAL]], 5
; CHECK-NEXT: [[VAL:%.*]] = select i1 [[CMP1]], i8 1, i8 2 ; CHECK-NEXT: [[VAL:%.*]] = select i1 [[CMP1]], i8 1, i8 2
; CHECK-NEXT: store i8 [[VAL]], i8* [[STIDX]], align 1 ; CHECK-NEXT: store i8 [[VAL]], i8* [[STIDX]], align 1
; CHECK-NEXT: [[INC]] = add nsw i32 [[I]], 1 ; CHECK-NEXT: [[INC]] = add nsw i32 [[I]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[I]], 8 ; CHECK-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[I]], 8
; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END:%.*]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK: for.end: ; CHECK: for.end:
; CHECK-NEXT: ret i32 0 ; CHECK-NEXT: ret i32 0
; ;
; CHECK-SCALAR-LABEL: @const_low_trip_count_hint_pred( ; CHECK-SCALAR-LABEL: @const_low_trip_count_hint_pred(
; CHECK-SCALAR-NEXT: entry: ; CHECK-SCALAR-NEXT: entry:
; CHECK-SCALAR-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; CHECK-SCALAR: vector.memcheck:
; CHECK-SCALAR-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, i8* [[DST:%.*]], i64 9
; CHECK-SCALAR-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, i8* [[SRC:%.*]], i64 9
; CHECK-SCALAR-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[DST]], [[SCEVGEP1]]
; CHECK-SCALAR-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SRC]], [[SCEVGEP]]
; CHECK-SCALAR-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-SCALAR-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; CHECK-SCALAR: vector.ph:
; CHECK-SCALAR-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-SCALAR: vector.body:
; CHECK-SCALAR-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-SCALAR-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0
; CHECK-SCALAR-NEXT: [[TMP1:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[TMP0]]
; CHECK-SCALAR-NEXT: [[TMP2:%.*]] = getelementptr i8, i8* [[DST]], i32 [[TMP0]]
; CHECK-SCALAR-NEXT: [[TMP3:%.*]] = getelementptr i8, i8* [[TMP1]], i32 0
; CHECK-SCALAR-NEXT: [[TMP4:%.*]] = bitcast i8* [[TMP3]] to <4 x i8>*
; CHECK-SCALAR-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, <4 x i8>* [[TMP4]], align 1, !alias.scope !0
; CHECK-SCALAR-NEXT: [[TMP5:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], <i8 5, i8 5, i8 5, i8 5>
; CHECK-SCALAR-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP5]], <4 x i8> <i8 1, i8 1, i8 1, i8 1>, <4 x i8> <i8 2, i8 2, i8 2, i8 2>
; CHECK-SCALAR-NEXT: [[TMP7:%.*]] = getelementptr i8, i8* [[TMP2]], i32 0
; CHECK-SCALAR-NEXT: [[TMP8:%.*]] = bitcast i8* [[TMP7]] to <4 x i8>*
; CHECK-SCALAR-NEXT: store <4 x i8> [[TMP6]], <4 x i8>* [[TMP8]], align 1, !alias.scope !3, !noalias !0
; CHECK-SCALAR-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-SCALAR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], 8
; CHECK-SCALAR-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK-SCALAR: middle.block:
; CHECK-SCALAR-NEXT: [[CMP_N:%.*]] = icmp eq i32 9, 8
; CHECK-SCALAR-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
; CHECK-SCALAR: scalar.ph:
; CHECK-SCALAR-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 8, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
; CHECK-SCALAR-NEXT: br label [[FOR_BODY:%.*]] ; CHECK-SCALAR-NEXT: br label [[FOR_BODY:%.*]]
; CHECK-SCALAR: for.body: ; CHECK-SCALAR: for.body:
; CHECK-SCALAR-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ] ; CHECK-SCALAR-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
; CHECK-SCALAR-NEXT: [[LDIDX:%.*]] = getelementptr i8, i8* [[SRC:%.*]], i32 [[I]] ; CHECK-SCALAR-NEXT: [[LDIDX:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[I]]
; CHECK-SCALAR-NEXT: [[STIDX:%.*]] = getelementptr i8, i8* [[DST:%.*]], i32 [[I]] ; CHECK-SCALAR-NEXT: [[STIDX:%.*]] = getelementptr i8, i8* [[DST]], i32 [[I]]
; CHECK-SCALAR-NEXT: [[LDVAL:%.*]] = load i8, i8* [[LDIDX]], align 1 ; CHECK-SCALAR-NEXT: [[LDVAL:%.*]] = load i8, i8* [[LDIDX]], align 1
; CHECK-SCALAR-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LDVAL]], 5 ; CHECK-SCALAR-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LDVAL]], 5
; CHECK-SCALAR-NEXT: [[VAL:%.*]] = select i1 [[CMP1]], i8 1, i8 2 ; CHECK-SCALAR-NEXT: [[VAL:%.*]] = select i1 [[CMP1]], i8 1, i8 2
; CHECK-SCALAR-NEXT: store i8 [[VAL]], i8* [[STIDX]], align 1 ; CHECK-SCALAR-NEXT: store i8 [[VAL]], i8* [[STIDX]], align 1
; CHECK-SCALAR-NEXT: [[INC]] = add nsw i32 [[I]], 1 ; CHECK-SCALAR-NEXT: [[INC]] = add nsw i32 [[I]], 1
; CHECK-SCALAR-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[I]], 8 ; CHECK-SCALAR-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[I]], 8
; CHECK-SCALAR-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END:%.*]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK-SCALAR-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK-SCALAR: for.end: ; CHECK-SCALAR: for.end:
; CHECK-SCALAR-NEXT: ret i32 0 ; CHECK-SCALAR-NEXT: ret i32 0
; ;
; CHECK-PRED-OR-SCALAR-LABEL: @const_low_trip_count_hint_pred( ; CHECK-PRED-OR-SCALAR-LABEL: @const_low_trip_count_hint_pred(
; CHECK-PRED-OR-SCALAR-NEXT: entry: ; CHECK-PRED-OR-SCALAR-NEXT: entry:
; CHECK-PRED-OR-SCALAR-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; CHECK-PRED-OR-SCALAR: vector.memcheck:
; CHECK-PRED-OR-SCALAR-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, i8* [[DST:%.*]], i64 9
; CHECK-PRED-OR-SCALAR-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, i8* [[SRC:%.*]], i64 9
; CHECK-PRED-OR-SCALAR-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[DST]], [[SCEVGEP1]]
; CHECK-PRED-OR-SCALAR-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SRC]], [[SCEVGEP]]
; CHECK-PRED-OR-SCALAR-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-PRED-OR-SCALAR-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; CHECK-PRED-OR-SCALAR: vector.ph:
; CHECK-PRED-OR-SCALAR-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-PRED-OR-SCALAR: vector.body:
; CHECK-PRED-OR-SCALAR-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE13:%.*]] ]
; CHECK-PRED-OR-SCALAR-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE13]] ]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 2
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 3
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP4:%.*]] = icmp ule <4 x i32> [[VEC_IND]], <i32 8, i32 8, i32 8, i32 8>
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
; CHECK-PRED-OR-SCALAR-NEXT: br i1 [[TMP5]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
; CHECK-PRED-OR-SCALAR: pred.load.if:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP6:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[TMP0]]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP7:%.*]] = load i8, i8* [[TMP6]], align 1, !alias.scope !0
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP8:%.*]] = insertelement <4 x i8> poison, i8 [[TMP7]], i32 0
; CHECK-PRED-OR-SCALAR-NEXT: br label [[PRED_LOAD_CONTINUE]]
; CHECK-PRED-OR-SCALAR: pred.load.continue:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP9:%.*]] = phi <4 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_LOAD_IF]] ]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
; CHECK-PRED-OR-SCALAR-NEXT: br i1 [[TMP10]], label [[PRED_LOAD_IF2:%.*]], label [[PRED_LOAD_CONTINUE3:%.*]]
; CHECK-PRED-OR-SCALAR: pred.load.if2:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP11:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[TMP1]]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP12:%.*]] = load i8, i8* [[TMP11]], align 1, !alias.scope !0
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP13:%.*]] = insertelement <4 x i8> [[TMP9]], i8 [[TMP12]], i32 1
; CHECK-PRED-OR-SCALAR-NEXT: br label [[PRED_LOAD_CONTINUE3]]
; CHECK-PRED-OR-SCALAR: pred.load.continue3:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP14:%.*]] = phi <4 x i8> [ [[TMP9]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], [[PRED_LOAD_IF2]] ]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
; CHECK-PRED-OR-SCALAR-NEXT: br i1 [[TMP15]], label [[PRED_LOAD_IF4:%.*]], label [[PRED_LOAD_CONTINUE5:%.*]]
; CHECK-PRED-OR-SCALAR: pred.load.if4:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP16:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[TMP2]]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP17:%.*]] = load i8, i8* [[TMP16]], align 1, !alias.scope !0
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP18:%.*]] = insertelement <4 x i8> [[TMP14]], i8 [[TMP17]], i32 2
; CHECK-PRED-OR-SCALAR-NEXT: br label [[PRED_LOAD_CONTINUE5]]
; CHECK-PRED-OR-SCALAR: pred.load.continue5:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP19:%.*]] = phi <4 x i8> [ [[TMP14]], [[PRED_LOAD_CONTINUE3]] ], [ [[TMP18]], [[PRED_LOAD_IF4]] ]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
; CHECK-PRED-OR-SCALAR-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF6:%.*]], label [[PRED_LOAD_CONTINUE7:%.*]]
; CHECK-PRED-OR-SCALAR: pred.load.if6:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP21:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[TMP3]]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP22:%.*]] = load i8, i8* [[TMP21]], align 1, !alias.scope !0
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP23:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP22]], i32 3
; CHECK-PRED-OR-SCALAR-NEXT: br label [[PRED_LOAD_CONTINUE7]]
; CHECK-PRED-OR-SCALAR: pred.load.continue7:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP24:%.*]] = phi <4 x i8> [ [[TMP19]], [[PRED_LOAD_CONTINUE5]] ], [ [[TMP23]], [[PRED_LOAD_IF6]] ]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP25:%.*]] = icmp eq <4 x i8> [[TMP24]], <i8 5, i8 5, i8 5, i8 5>
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP26:%.*]] = select <4 x i1> [[TMP25]], <4 x i8> <i8 1, i8 1, i8 1, i8 1>, <4 x i8> <i8 2, i8 2, i8 2, i8 2>
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
; CHECK-PRED-OR-SCALAR-NEXT: br i1 [[TMP27]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; CHECK-PRED-OR-SCALAR: pred.store.if:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP28:%.*]] = getelementptr i8, i8* [[DST]], i32 [[TMP0]]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP29:%.*]] = extractelement <4 x i8> [[TMP26]], i32 0
; CHECK-PRED-OR-SCALAR-NEXT: store i8 [[TMP29]], i8* [[TMP28]], align 1, !alias.scope !3, !noalias !0
; CHECK-PRED-OR-SCALAR-NEXT: br label [[PRED_STORE_CONTINUE]]
; CHECK-PRED-OR-SCALAR: pred.store.continue:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP30:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
; CHECK-PRED-OR-SCALAR-NEXT: br i1 [[TMP30]], label [[PRED_STORE_IF8:%.*]], label [[PRED_STORE_CONTINUE9:%.*]]
; CHECK-PRED-OR-SCALAR: pred.store.if8:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP31:%.*]] = getelementptr i8, i8* [[DST]], i32 [[TMP1]]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP32:%.*]] = extractelement <4 x i8> [[TMP26]], i32 1
; CHECK-PRED-OR-SCALAR-NEXT: store i8 [[TMP32]], i8* [[TMP31]], align 1, !alias.scope !3, !noalias !0
; CHECK-PRED-OR-SCALAR-NEXT: br label [[PRED_STORE_CONTINUE9]]
; CHECK-PRED-OR-SCALAR: pred.store.continue9:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP33:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
; CHECK-PRED-OR-SCALAR-NEXT: br i1 [[TMP33]], label [[PRED_STORE_IF10:%.*]], label [[PRED_STORE_CONTINUE11:%.*]]
; CHECK-PRED-OR-SCALAR: pred.store.if10:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP34:%.*]] = getelementptr i8, i8* [[DST]], i32 [[TMP2]]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP35:%.*]] = extractelement <4 x i8> [[TMP26]], i32 2
; CHECK-PRED-OR-SCALAR-NEXT: store i8 [[TMP35]], i8* [[TMP34]], align 1, !alias.scope !3, !noalias !0
; CHECK-PRED-OR-SCALAR-NEXT: br label [[PRED_STORE_CONTINUE11]]
; CHECK-PRED-OR-SCALAR: pred.store.continue11:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP36:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
; CHECK-PRED-OR-SCALAR-NEXT: br i1 [[TMP36]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13]]
; CHECK-PRED-OR-SCALAR: pred.store.if12:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP37:%.*]] = getelementptr i8, i8* [[DST]], i32 [[TMP3]]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP38:%.*]] = extractelement <4 x i8> [[TMP26]], i32 3
; CHECK-PRED-OR-SCALAR-NEXT: store i8 [[TMP38]], i8* [[TMP37]], align 1, !alias.scope !3, !noalias !0
; CHECK-PRED-OR-SCALAR-NEXT: br label [[PRED_STORE_CONTINUE13]]
; CHECK-PRED-OR-SCALAR: pred.store.continue13:
; CHECK-PRED-OR-SCALAR-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
; CHECK-PRED-OR-SCALAR-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP39:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
; CHECK-PRED-OR-SCALAR-NEXT: br i1 [[TMP39]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; CHECK-PRED-OR-SCALAR: middle.block:
; CHECK-PRED-OR-SCALAR-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
; CHECK-PRED-OR-SCALAR: scalar.ph:
; CHECK-PRED-OR-SCALAR-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 12, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
; CHECK-PRED-OR-SCALAR-NEXT: br label [[FOR_BODY:%.*]] ; CHECK-PRED-OR-SCALAR-NEXT: br label [[FOR_BODY:%.*]]
; CHECK-PRED-OR-SCALAR: for.body: ; CHECK-PRED-OR-SCALAR: for.body:
; CHECK-PRED-OR-SCALAR-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ] ; CHECK-PRED-OR-SCALAR-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
; CHECK-PRED-OR-SCALAR-NEXT: [[LDIDX:%.*]] = getelementptr i8, i8* [[SRC:%.*]], i32 [[I]] ; CHECK-PRED-OR-SCALAR-NEXT: [[LDIDX:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[I]]
; CHECK-PRED-OR-SCALAR-NEXT: [[STIDX:%.*]] = getelementptr i8, i8* [[DST:%.*]], i32 [[I]] ; CHECK-PRED-OR-SCALAR-NEXT: [[STIDX:%.*]] = getelementptr i8, i8* [[DST]], i32 [[I]]
; CHECK-PRED-OR-SCALAR-NEXT: [[LDVAL:%.*]] = load i8, i8* [[LDIDX]], align 1 ; CHECK-PRED-OR-SCALAR-NEXT: [[LDVAL:%.*]] = load i8, i8* [[LDIDX]], align 1
; CHECK-PRED-OR-SCALAR-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LDVAL]], 5 ; CHECK-PRED-OR-SCALAR-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LDVAL]], 5
; CHECK-PRED-OR-SCALAR-NEXT: [[VAL:%.*]] = select i1 [[CMP1]], i8 1, i8 2 ; CHECK-PRED-OR-SCALAR-NEXT: [[VAL:%.*]] = select i1 [[CMP1]], i8 1, i8 2
; CHECK-PRED-OR-SCALAR-NEXT: store i8 [[VAL]], i8* [[STIDX]], align 1 ; CHECK-PRED-OR-SCALAR-NEXT: store i8 [[VAL]], i8* [[STIDX]], align 1
; CHECK-PRED-OR-SCALAR-NEXT: [[INC]] = add nsw i32 [[I]], 1 ; CHECK-PRED-OR-SCALAR-NEXT: [[INC]] = add nsw i32 [[I]], 1
; CHECK-PRED-OR-SCALAR-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[I]], 8 ; CHECK-PRED-OR-SCALAR-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[I]], 8
; CHECK-PRED-OR-SCALAR-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END:%.*]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK-PRED-OR-SCALAR-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP7:![0-9]+]]
; CHECK-PRED-OR-SCALAR: for.end: ; CHECK-PRED-OR-SCALAR: for.end:
; CHECK-PRED-OR-SCALAR-NEXT: ret i32 0 ; CHECK-PRED-OR-SCALAR-NEXT: ret i32 0
; ;
; PRED-OR-DONTVEC-LABEL: @const_low_trip_count_hint_pred( ; PRED-OR-DONTVEC-LABEL: @const_low_trip_count_hint_pred(
; PRED-OR-DONTVEC-NEXT: entry: ; PRED-OR-DONTVEC-NEXT: entry:
; PRED-OR-DONTVEC-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; PRED-OR-DONTVEC: vector.memcheck:
; PRED-OR-DONTVEC-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, i8* [[DST:%.*]], i64 9
; PRED-OR-DONTVEC-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, i8* [[SRC:%.*]], i64 9
; PRED-OR-DONTVEC-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[DST]], [[SCEVGEP1]]
; PRED-OR-DONTVEC-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SRC]], [[SCEVGEP]]
; PRED-OR-DONTVEC-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; PRED-OR-DONTVEC-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; PRED-OR-DONTVEC: vector.ph:
; PRED-OR-DONTVEC-NEXT: br label [[VECTOR_BODY:%.*]]
; PRED-OR-DONTVEC: vector.body:
; PRED-OR-DONTVEC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE13:%.*]] ]
; PRED-OR-DONTVEC-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE13]] ]
; PRED-OR-DONTVEC-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0
; PRED-OR-DONTVEC-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1
; PRED-OR-DONTVEC-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 2
; PRED-OR-DONTVEC-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 3
; PRED-OR-DONTVEC-NEXT: [[TMP4:%.*]] = icmp ule <4 x i32> [[VEC_IND]], <i32 8, i32 8, i32 8, i32 8>
; PRED-OR-DONTVEC-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
; PRED-OR-DONTVEC-NEXT: br i1 [[TMP5]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
; PRED-OR-DONTVEC: pred.load.if:
; PRED-OR-DONTVEC-NEXT: [[TMP6:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[TMP0]]
; PRED-OR-DONTVEC-NEXT: [[TMP7:%.*]] = load i8, i8* [[TMP6]], align 1, !alias.scope !0
; PRED-OR-DONTVEC-NEXT: [[TMP8:%.*]] = insertelement <4 x i8> poison, i8 [[TMP7]], i32 0
; PRED-OR-DONTVEC-NEXT: br label [[PRED_LOAD_CONTINUE]]
; PRED-OR-DONTVEC: pred.load.continue:
; PRED-OR-DONTVEC-NEXT: [[TMP9:%.*]] = phi <4 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_LOAD_IF]] ]
; PRED-OR-DONTVEC-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
; PRED-OR-DONTVEC-NEXT: br i1 [[TMP10]], label [[PRED_LOAD_IF2:%.*]], label [[PRED_LOAD_CONTINUE3:%.*]]
; PRED-OR-DONTVEC: pred.load.if2:
; PRED-OR-DONTVEC-NEXT: [[TMP11:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[TMP1]]
; PRED-OR-DONTVEC-NEXT: [[TMP12:%.*]] = load i8, i8* [[TMP11]], align 1, !alias.scope !0
; PRED-OR-DONTVEC-NEXT: [[TMP13:%.*]] = insertelement <4 x i8> [[TMP9]], i8 [[TMP12]], i32 1
; PRED-OR-DONTVEC-NEXT: br label [[PRED_LOAD_CONTINUE3]]
; PRED-OR-DONTVEC: pred.load.continue3:
; PRED-OR-DONTVEC-NEXT: [[TMP14:%.*]] = phi <4 x i8> [ [[TMP9]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], [[PRED_LOAD_IF2]] ]
; PRED-OR-DONTVEC-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
; PRED-OR-DONTVEC-NEXT: br i1 [[TMP15]], label [[PRED_LOAD_IF4:%.*]], label [[PRED_LOAD_CONTINUE5:%.*]]
; PRED-OR-DONTVEC: pred.load.if4:
; PRED-OR-DONTVEC-NEXT: [[TMP16:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[TMP2]]
; PRED-OR-DONTVEC-NEXT: [[TMP17:%.*]] = load i8, i8* [[TMP16]], align 1, !alias.scope !0
; PRED-OR-DONTVEC-NEXT: [[TMP18:%.*]] = insertelement <4 x i8> [[TMP14]], i8 [[TMP17]], i32 2
; PRED-OR-DONTVEC-NEXT: br label [[PRED_LOAD_CONTINUE5]]
; PRED-OR-DONTVEC: pred.load.continue5:
; PRED-OR-DONTVEC-NEXT: [[TMP19:%.*]] = phi <4 x i8> [ [[TMP14]], [[PRED_LOAD_CONTINUE3]] ], [ [[TMP18]], [[PRED_LOAD_IF4]] ]
; PRED-OR-DONTVEC-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
; PRED-OR-DONTVEC-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF6:%.*]], label [[PRED_LOAD_CONTINUE7:%.*]]
; PRED-OR-DONTVEC: pred.load.if6:
; PRED-OR-DONTVEC-NEXT: [[TMP21:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[TMP3]]
; PRED-OR-DONTVEC-NEXT: [[TMP22:%.*]] = load i8, i8* [[TMP21]], align 1, !alias.scope !0
; PRED-OR-DONTVEC-NEXT: [[TMP23:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP22]], i32 3
; PRED-OR-DONTVEC-NEXT: br label [[PRED_LOAD_CONTINUE7]]
; PRED-OR-DONTVEC: pred.load.continue7:
; PRED-OR-DONTVEC-NEXT: [[TMP24:%.*]] = phi <4 x i8> [ [[TMP19]], [[PRED_LOAD_CONTINUE5]] ], [ [[TMP23]], [[PRED_LOAD_IF6]] ]
; PRED-OR-DONTVEC-NEXT: [[TMP25:%.*]] = icmp eq <4 x i8> [[TMP24]], <i8 5, i8 5, i8 5, i8 5>
; PRED-OR-DONTVEC-NEXT: [[TMP26:%.*]] = select <4 x i1> [[TMP25]], <4 x i8> <i8 1, i8 1, i8 1, i8 1>, <4 x i8> <i8 2, i8 2, i8 2, i8 2>
; PRED-OR-DONTVEC-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
; PRED-OR-DONTVEC-NEXT: br i1 [[TMP27]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; PRED-OR-DONTVEC: pred.store.if:
; PRED-OR-DONTVEC-NEXT: [[TMP28:%.*]] = getelementptr i8, i8* [[DST]], i32 [[TMP0]]
; PRED-OR-DONTVEC-NEXT: [[TMP29:%.*]] = extractelement <4 x i8> [[TMP26]], i32 0
; PRED-OR-DONTVEC-NEXT: store i8 [[TMP29]], i8* [[TMP28]], align 1, !alias.scope !3, !noalias !0
; PRED-OR-DONTVEC-NEXT: br label [[PRED_STORE_CONTINUE]]
; PRED-OR-DONTVEC: pred.store.continue:
; PRED-OR-DONTVEC-NEXT: [[TMP30:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
; PRED-OR-DONTVEC-NEXT: br i1 [[TMP30]], label [[PRED_STORE_IF8:%.*]], label [[PRED_STORE_CONTINUE9:%.*]]
; PRED-OR-DONTVEC: pred.store.if8:
; PRED-OR-DONTVEC-NEXT: [[TMP31:%.*]] = getelementptr i8, i8* [[DST]], i32 [[TMP1]]
; PRED-OR-DONTVEC-NEXT: [[TMP32:%.*]] = extractelement <4 x i8> [[TMP26]], i32 1
; PRED-OR-DONTVEC-NEXT: store i8 [[TMP32]], i8* [[TMP31]], align 1, !alias.scope !3, !noalias !0
; PRED-OR-DONTVEC-NEXT: br label [[PRED_STORE_CONTINUE9]]
; PRED-OR-DONTVEC: pred.store.continue9:
; PRED-OR-DONTVEC-NEXT: [[TMP33:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
; PRED-OR-DONTVEC-NEXT: br i1 [[TMP33]], label [[PRED_STORE_IF10:%.*]], label [[PRED_STORE_CONTINUE11:%.*]]
; PRED-OR-DONTVEC: pred.store.if10:
; PRED-OR-DONTVEC-NEXT: [[TMP34:%.*]] = getelementptr i8, i8* [[DST]], i32 [[TMP2]]
; PRED-OR-DONTVEC-NEXT: [[TMP35:%.*]] = extractelement <4 x i8> [[TMP26]], i32 2
; PRED-OR-DONTVEC-NEXT: store i8 [[TMP35]], i8* [[TMP34]], align 1, !alias.scope !3, !noalias !0
; PRED-OR-DONTVEC-NEXT: br label [[PRED_STORE_CONTINUE11]]
; PRED-OR-DONTVEC: pred.store.continue11:
; PRED-OR-DONTVEC-NEXT: [[TMP36:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
; PRED-OR-DONTVEC-NEXT: br i1 [[TMP36]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13]]
; PRED-OR-DONTVEC: pred.store.if12:
; PRED-OR-DONTVEC-NEXT: [[TMP37:%.*]] = getelementptr i8, i8* [[DST]], i32 [[TMP3]]
; PRED-OR-DONTVEC-NEXT: [[TMP38:%.*]] = extractelement <4 x i8> [[TMP26]], i32 3
; PRED-OR-DONTVEC-NEXT: store i8 [[TMP38]], i8* [[TMP37]], align 1, !alias.scope !3, !noalias !0
; PRED-OR-DONTVEC-NEXT: br label [[PRED_STORE_CONTINUE13]]
; PRED-OR-DONTVEC: pred.store.continue13:
; PRED-OR-DONTVEC-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
; PRED-OR-DONTVEC-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
; PRED-OR-DONTVEC-NEXT: [[TMP39:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
; PRED-OR-DONTVEC-NEXT: br i1 [[TMP39]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]]
; PRED-OR-DONTVEC: middle.block:
; PRED-OR-DONTVEC-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
; PRED-OR-DONTVEC: scalar.ph:
; PRED-OR-DONTVEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 12, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
; PRED-OR-DONTVEC-NEXT: br label [[FOR_BODY:%.*]] ; PRED-OR-DONTVEC-NEXT: br label [[FOR_BODY:%.*]]
; PRED-OR-DONTVEC: for.body: ; PRED-OR-DONTVEC: for.body:
; PRED-OR-DONTVEC-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ] ; PRED-OR-DONTVEC-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
; PRED-OR-DONTVEC-NEXT: [[LDIDX:%.*]] = getelementptr i8, i8* [[SRC:%.*]], i32 [[I]] ; PRED-OR-DONTVEC-NEXT: [[LDIDX:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[I]]
; PRED-OR-DONTVEC-NEXT: [[STIDX:%.*]] = getelementptr i8, i8* [[DST:%.*]], i32 [[I]] ; PRED-OR-DONTVEC-NEXT: [[STIDX:%.*]] = getelementptr i8, i8* [[DST]], i32 [[I]]
; PRED-OR-DONTVEC-NEXT: [[LDVAL:%.*]] = load i8, i8* [[LDIDX]], align 1 ; PRED-OR-DONTVEC-NEXT: [[LDVAL:%.*]] = load i8, i8* [[LDIDX]], align 1
; PRED-OR-DONTVEC-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LDVAL]], 5 ; PRED-OR-DONTVEC-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LDVAL]], 5
; PRED-OR-DONTVEC-NEXT: [[VAL:%.*]] = select i1 [[CMP1]], i8 1, i8 2 ; PRED-OR-DONTVEC-NEXT: [[VAL:%.*]] = select i1 [[CMP1]], i8 1, i8 2
; PRED-OR-DONTVEC-NEXT: store i8 [[VAL]], i8* [[STIDX]], align 1 ; PRED-OR-DONTVEC-NEXT: store i8 [[VAL]], i8* [[STIDX]], align 1
; PRED-OR-DONTVEC-NEXT: [[INC]] = add nsw i32 [[I]], 1 ; PRED-OR-DONTVEC-NEXT: [[INC]] = add nsw i32 [[I]], 1
; PRED-OR-DONTVEC-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[I]], 8 ; PRED-OR-DONTVEC-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[I]], 8
; PRED-OR-DONTVEC-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END:%.*]], !llvm.loop [[LOOP0:![0-9]+]] ; PRED-OR-DONTVEC-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP7:![0-9]+]]
; PRED-OR-DONTVEC: for.end: ; PRED-OR-DONTVEC: for.end:
; PRED-OR-DONTVEC-NEXT: ret i32 0 ; PRED-OR-DONTVEC-NEXT: ret i32 0
; ;
entry: entry:
br label %for.body br label %for.body
for.body: ; preds = %for.body, %entry for.body: ; preds = %for.body, %entry
%i = phi i32 [ 0, %entry ], [ %inc, %for.body ] %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]
%ldidx = getelementptr i8, i8* %src, i32 %i %ldidx = getelementptr i8, i8* %src, i32 %i
%stidx = getelementptr i8, i8* %dst, i32 %i %stidx = getelementptr i8, i8* %dst, i32 %i
%ldval = load i8, i8* %ldidx, align 1 %ldval = load i8, i8* %ldidx, align 1
%cmp1 = icmp eq i8 %ldval, 5 %cmp1 = icmp eq i8 %ldval, 5
%val = select i1 %cmp1, i8 1, i8 2 %val = select i1 %cmp1, i8 1, i8 2
store i8 %val, i8* %stidx, align 1 store i8 %val, i8* %stidx, align 1
%inc = add nsw i32 %i, 1 %inc = add nsw i32 %i, 1
%exitcond = icmp slt i32 %i, 8 %exitcond = icmp slt i32 %i, 8
br i1 %exitcond, label %for.body, label %for.end, !llvm.loop !1 br i1 %exitcond, label %for.body, label %for.end, !llvm.loop !1
for.end: ; preds = %for.body for.end: ; preds = %for.body
ret i32 0 ret i32 0
} }
; Simple loop with small constant trip count. ; Simple loop with small constant trip count.
; TODO: Check that -prefer-predicate-over-epilogue and "llvm.loop.vectorize.predicate.enable=false" ; Check that -prefer-predicate-over-epilogue and "llvm.loop.vectorize.predicate.enable=false"
; hint override "small trip count" heuristic. ; hint override "small trip count" heuristic.
define i32 @const_low_trip_count_hint_no_pred(i8 *%dst, i8 *%src) { define i32 @const_low_trip_count_hint_no_pred(i8 *%dst, i8 *%src) {
; CHECK-LABEL: @const_low_trip_count_hint_no_pred( ; CHECK-LABEL: @const_low_trip_count_hint_no_pred(
; CHECK-NEXT: entry: ; CHECK-NEXT: entry:
; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; CHECK: vector.memcheck:
; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, i8* [[DST:%.*]], i64 9
; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, i8* [[SRC:%.*]], i64 9
; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[DST]], [[SCEVGEP1]]
; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SRC]], [[SCEVGEP]]
; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; CHECK: vector.ph:
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0
; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[TMP0]]
; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i8, i8* [[DST]], i32 [[TMP0]]
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, i8* [[TMP1]], i32 0
; CHECK-NEXT: [[TMP4:%.*]] = bitcast i8* [[TMP3]] to <4 x i8>*
; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, <4 x i8>* [[TMP4]], align 1, !alias.scope !8
; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], <i8 5, i8 5, i8 5, i8 5>
; CHECK-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP5]], <4 x i8> <i8 1, i8 1, i8 1, i8 1>, <4 x i8> <i8 2, i8 2, i8 2, i8 2>
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i8, i8* [[TMP2]], i32 0
; CHECK-NEXT: [[TMP8:%.*]] = bitcast i8* [[TMP7]] to <4 x i8>*
; CHECK-NEXT: store <4 x i8> [[TMP6]], <4 x i8>* [[TMP8]], align 1, !alias.scope !11, !noalias !8
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], 8
; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK: middle.block:
; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 9, 8
; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
; CHECK: scalar.ph:
; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 8, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
; CHECK-NEXT: br label [[FOR_BODY:%.*]] ; CHECK-NEXT: br label [[FOR_BODY:%.*]]
; CHECK: for.body: ; CHECK: for.body:
; CHECK-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ] ; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
; CHECK-NEXT: [[LDIDX:%.*]] = getelementptr i8, i8* [[SRC:%.*]], i32 [[I]] ; CHECK-NEXT: [[LDIDX:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[I]]
; CHECK-NEXT: [[STIDX:%.*]] = getelementptr i8, i8* [[DST:%.*]], i32 [[I]] ; CHECK-NEXT: [[STIDX:%.*]] = getelementptr i8, i8* [[DST]], i32 [[I]]
; CHECK-NEXT: [[LDVAL:%.*]] = load i8, i8* [[LDIDX]], align 1 ; CHECK-NEXT: [[LDVAL:%.*]] = load i8, i8* [[LDIDX]], align 1
; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LDVAL]], 5 ; CHECK-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LDVAL]], 5
; CHECK-NEXT: [[VAL:%.*]] = select i1 [[CMP1]], i8 1, i8 2 ; CHECK-NEXT: [[VAL:%.*]] = select i1 [[CMP1]], i8 1, i8 2
; CHECK-NEXT: store i8 [[VAL]], i8* [[STIDX]], align 1 ; CHECK-NEXT: store i8 [[VAL]], i8* [[STIDX]], align 1
; CHECK-NEXT: [[INC]] = add nsw i32 [[I]], 1 ; CHECK-NEXT: [[INC]] = add nsw i32 [[I]], 1
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[I]], 8 ; CHECK-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[I]], 8
; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END:%.*]], !llvm.loop [[LOOP3:![0-9]+]] ; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK: for.end: ; CHECK: for.end:
; CHECK-NEXT: ret i32 0 ; CHECK-NEXT: ret i32 0
; ;
; CHECK-SCALAR-LABEL: @const_low_trip_count_hint_no_pred( ; CHECK-SCALAR-LABEL: @const_low_trip_count_hint_no_pred(
; CHECK-SCALAR-NEXT: entry: ; CHECK-SCALAR-NEXT: entry:
; CHECK-SCALAR-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; CHECK-SCALAR: vector.memcheck:
; CHECK-SCALAR-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, i8* [[DST:%.*]], i64 9
; CHECK-SCALAR-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, i8* [[SRC:%.*]], i64 9
; CHECK-SCALAR-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[DST]], [[SCEVGEP1]]
; CHECK-SCALAR-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SRC]], [[SCEVGEP]]
; CHECK-SCALAR-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-SCALAR-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; CHECK-SCALAR: vector.ph:
; CHECK-SCALAR-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-SCALAR: vector.body:
; CHECK-SCALAR-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; CHECK-SCALAR-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0
; CHECK-SCALAR-NEXT: [[TMP1:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[TMP0]]
; CHECK-SCALAR-NEXT: [[TMP2:%.*]] = getelementptr i8, i8* [[DST]], i32 [[TMP0]]
; CHECK-SCALAR-NEXT: [[TMP3:%.*]] = getelementptr i8, i8* [[TMP1]], i32 0
; CHECK-SCALAR-NEXT: [[TMP4:%.*]] = bitcast i8* [[TMP3]] to <4 x i8>*
; CHECK-SCALAR-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, <4 x i8>* [[TMP4]], align 1, !alias.scope !8
; CHECK-SCALAR-NEXT: [[TMP5:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], <i8 5, i8 5, i8 5, i8 5>
; CHECK-SCALAR-NEXT: [[TMP6:%.*]] = select <4 x i1> [[TMP5]], <4 x i8> <i8 1, i8 1, i8 1, i8 1>, <4 x i8> <i8 2, i8 2, i8 2, i8 2>
; CHECK-SCALAR-NEXT: [[TMP7:%.*]] = getelementptr i8, i8* [[TMP2]], i32 0
; CHECK-SCALAR-NEXT: [[TMP8:%.*]] = bitcast i8* [[TMP7]] to <4 x i8>*
; CHECK-SCALAR-NEXT: store <4 x i8> [[TMP6]], <4 x i8>* [[TMP8]], align 1, !alias.scope !11, !noalias !8
; CHECK-SCALAR-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4
; CHECK-SCALAR-NEXT: [[TMP9:%.*]] = icmp eq i32 [[INDEX_NEXT]], 8
; CHECK-SCALAR-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK-SCALAR: middle.block:
; CHECK-SCALAR-NEXT: [[CMP_N:%.*]] = icmp eq i32 9, 8
; CHECK-SCALAR-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]]
; CHECK-SCALAR: scalar.ph:
; CHECK-SCALAR-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 8, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
; CHECK-SCALAR-NEXT: br label [[FOR_BODY:%.*]] ; CHECK-SCALAR-NEXT: br label [[FOR_BODY:%.*]]
; CHECK-SCALAR: for.body: ; CHECK-SCALAR: for.body:
; CHECK-SCALAR-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ] ; CHECK-SCALAR-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
; CHECK-SCALAR-NEXT: [[LDIDX:%.*]] = getelementptr i8, i8* [[SRC:%.*]], i32 [[I]] ; CHECK-SCALAR-NEXT: [[LDIDX:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[I]]
; CHECK-SCALAR-NEXT: [[STIDX:%.*]] = getelementptr i8, i8* [[DST:%.*]], i32 [[I]] ; CHECK-SCALAR-NEXT: [[STIDX:%.*]] = getelementptr i8, i8* [[DST]], i32 [[I]]
; CHECK-SCALAR-NEXT: [[LDVAL:%.*]] = load i8, i8* [[LDIDX]], align 1 ; CHECK-SCALAR-NEXT: [[LDVAL:%.*]] = load i8, i8* [[LDIDX]], align 1
; CHECK-SCALAR-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LDVAL]], 5 ; CHECK-SCALAR-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LDVAL]], 5
; CHECK-SCALAR-NEXT: [[VAL:%.*]] = select i1 [[CMP1]], i8 1, i8 2 ; CHECK-SCALAR-NEXT: [[VAL:%.*]] = select i1 [[CMP1]], i8 1, i8 2
; CHECK-SCALAR-NEXT: store i8 [[VAL]], i8* [[STIDX]], align 1 ; CHECK-SCALAR-NEXT: store i8 [[VAL]], i8* [[STIDX]], align 1
; CHECK-SCALAR-NEXT: [[INC]] = add nsw i32 [[I]], 1 ; CHECK-SCALAR-NEXT: [[INC]] = add nsw i32 [[I]], 1
; CHECK-SCALAR-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[I]], 8 ; CHECK-SCALAR-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[I]], 8
; CHECK-SCALAR-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END:%.*]], !llvm.loop [[LOOP3:![0-9]+]] ; CHECK-SCALAR-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK-SCALAR: for.end: ; CHECK-SCALAR: for.end:
; CHECK-SCALAR-NEXT: ret i32 0 ; CHECK-SCALAR-NEXT: ret i32 0
; ;
; CHECK-PRED-OR-SCALAR-LABEL: @const_low_trip_count_hint_no_pred( ; CHECK-PRED-OR-SCALAR-LABEL: @const_low_trip_count_hint_no_pred(
; CHECK-PRED-OR-SCALAR-NEXT: entry: ; CHECK-PRED-OR-SCALAR-NEXT: entry:
; CHECK-PRED-OR-SCALAR-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; CHECK-PRED-OR-SCALAR: vector.memcheck:
; CHECK-PRED-OR-SCALAR-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, i8* [[DST:%.*]], i64 9
; CHECK-PRED-OR-SCALAR-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, i8* [[SRC:%.*]], i64 9
; CHECK-PRED-OR-SCALAR-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[DST]], [[SCEVGEP1]]
; CHECK-PRED-OR-SCALAR-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SRC]], [[SCEVGEP]]
; CHECK-PRED-OR-SCALAR-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; CHECK-PRED-OR-SCALAR-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; CHECK-PRED-OR-SCALAR: vector.ph:
; CHECK-PRED-OR-SCALAR-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK-PRED-OR-SCALAR: vector.body:
; CHECK-PRED-OR-SCALAR-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE13:%.*]] ]
; CHECK-PRED-OR-SCALAR-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE13]] ]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 2
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 3
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP4:%.*]] = icmp ule <4 x i32> [[VEC_IND]], <i32 8, i32 8, i32 8, i32 8>
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
; CHECK-PRED-OR-SCALAR-NEXT: br i1 [[TMP5]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
; CHECK-PRED-OR-SCALAR: pred.load.if:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP6:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[TMP0]]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP7:%.*]] = load i8, i8* [[TMP6]], align 1, !alias.scope !8
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP8:%.*]] = insertelement <4 x i8> poison, i8 [[TMP7]], i32 0
; CHECK-PRED-OR-SCALAR-NEXT: br label [[PRED_LOAD_CONTINUE]]
; CHECK-PRED-OR-SCALAR: pred.load.continue:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP9:%.*]] = phi <4 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_LOAD_IF]] ]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
; CHECK-PRED-OR-SCALAR-NEXT: br i1 [[TMP10]], label [[PRED_LOAD_IF2:%.*]], label [[PRED_LOAD_CONTINUE3:%.*]]
; CHECK-PRED-OR-SCALAR: pred.load.if2:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP11:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[TMP1]]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP12:%.*]] = load i8, i8* [[TMP11]], align 1, !alias.scope !8
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP13:%.*]] = insertelement <4 x i8> [[TMP9]], i8 [[TMP12]], i32 1
; CHECK-PRED-OR-SCALAR-NEXT: br label [[PRED_LOAD_CONTINUE3]]
; CHECK-PRED-OR-SCALAR: pred.load.continue3:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP14:%.*]] = phi <4 x i8> [ [[TMP9]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], [[PRED_LOAD_IF2]] ]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
; CHECK-PRED-OR-SCALAR-NEXT: br i1 [[TMP15]], label [[PRED_LOAD_IF4:%.*]], label [[PRED_LOAD_CONTINUE5:%.*]]
; CHECK-PRED-OR-SCALAR: pred.load.if4:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP16:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[TMP2]]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP17:%.*]] = load i8, i8* [[TMP16]], align 1, !alias.scope !8
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP18:%.*]] = insertelement <4 x i8> [[TMP14]], i8 [[TMP17]], i32 2
; CHECK-PRED-OR-SCALAR-NEXT: br label [[PRED_LOAD_CONTINUE5]]
; CHECK-PRED-OR-SCALAR: pred.load.continue5:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP19:%.*]] = phi <4 x i8> [ [[TMP14]], [[PRED_LOAD_CONTINUE3]] ], [ [[TMP18]], [[PRED_LOAD_IF4]] ]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
; CHECK-PRED-OR-SCALAR-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF6:%.*]], label [[PRED_LOAD_CONTINUE7:%.*]]
; CHECK-PRED-OR-SCALAR: pred.load.if6:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP21:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[TMP3]]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP22:%.*]] = load i8, i8* [[TMP21]], align 1, !alias.scope !8
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP23:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP22]], i32 3
; CHECK-PRED-OR-SCALAR-NEXT: br label [[PRED_LOAD_CONTINUE7]]
; CHECK-PRED-OR-SCALAR: pred.load.continue7:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP24:%.*]] = phi <4 x i8> [ [[TMP19]], [[PRED_LOAD_CONTINUE5]] ], [ [[TMP23]], [[PRED_LOAD_IF6]] ]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP25:%.*]] = icmp eq <4 x i8> [[TMP24]], <i8 5, i8 5, i8 5, i8 5>
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP26:%.*]] = select <4 x i1> [[TMP25]], <4 x i8> <i8 1, i8 1, i8 1, i8 1>, <4 x i8> <i8 2, i8 2, i8 2, i8 2>
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
; CHECK-PRED-OR-SCALAR-NEXT: br i1 [[TMP27]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; CHECK-PRED-OR-SCALAR: pred.store.if:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP28:%.*]] = getelementptr i8, i8* [[DST]], i32 [[TMP0]]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP29:%.*]] = extractelement <4 x i8> [[TMP26]], i32 0
; CHECK-PRED-OR-SCALAR-NEXT: store i8 [[TMP29]], i8* [[TMP28]], align 1, !alias.scope !11, !noalias !8
; CHECK-PRED-OR-SCALAR-NEXT: br label [[PRED_STORE_CONTINUE]]
; CHECK-PRED-OR-SCALAR: pred.store.continue:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP30:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
; CHECK-PRED-OR-SCALAR-NEXT: br i1 [[TMP30]], label [[PRED_STORE_IF8:%.*]], label [[PRED_STORE_CONTINUE9:%.*]]
; CHECK-PRED-OR-SCALAR: pred.store.if8:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP31:%.*]] = getelementptr i8, i8* [[DST]], i32 [[TMP1]]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP32:%.*]] = extractelement <4 x i8> [[TMP26]], i32 1
; CHECK-PRED-OR-SCALAR-NEXT: store i8 [[TMP32]], i8* [[TMP31]], align 1, !alias.scope !11, !noalias !8
; CHECK-PRED-OR-SCALAR-NEXT: br label [[PRED_STORE_CONTINUE9]]
; CHECK-PRED-OR-SCALAR: pred.store.continue9:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP33:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
; CHECK-PRED-OR-SCALAR-NEXT: br i1 [[TMP33]], label [[PRED_STORE_IF10:%.*]], label [[PRED_STORE_CONTINUE11:%.*]]
; CHECK-PRED-OR-SCALAR: pred.store.if10:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP34:%.*]] = getelementptr i8, i8* [[DST]], i32 [[TMP2]]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP35:%.*]] = extractelement <4 x i8> [[TMP26]], i32 2
; CHECK-PRED-OR-SCALAR-NEXT: store i8 [[TMP35]], i8* [[TMP34]], align 1, !alias.scope !11, !noalias !8
; CHECK-PRED-OR-SCALAR-NEXT: br label [[PRED_STORE_CONTINUE11]]
; CHECK-PRED-OR-SCALAR: pred.store.continue11:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP36:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
; CHECK-PRED-OR-SCALAR-NEXT: br i1 [[TMP36]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13]]
; CHECK-PRED-OR-SCALAR: pred.store.if12:
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP37:%.*]] = getelementptr i8, i8* [[DST]], i32 [[TMP3]]
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP38:%.*]] = extractelement <4 x i8> [[TMP26]], i32 3
; CHECK-PRED-OR-SCALAR-NEXT: store i8 [[TMP38]], i8* [[TMP37]], align 1, !alias.scope !11, !noalias !8
; CHECK-PRED-OR-SCALAR-NEXT: br label [[PRED_STORE_CONTINUE13]]
; CHECK-PRED-OR-SCALAR: pred.store.continue13:
; CHECK-PRED-OR-SCALAR-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
; CHECK-PRED-OR-SCALAR-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
; CHECK-PRED-OR-SCALAR-NEXT: [[TMP39:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
; CHECK-PRED-OR-SCALAR-NEXT: br i1 [[TMP39]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; CHECK-PRED-OR-SCALAR: middle.block:
; CHECK-PRED-OR-SCALAR-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
; CHECK-PRED-OR-SCALAR: scalar.ph:
; CHECK-PRED-OR-SCALAR-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 12, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
; CHECK-PRED-OR-SCALAR-NEXT: br label [[FOR_BODY:%.*]] ; CHECK-PRED-OR-SCALAR-NEXT: br label [[FOR_BODY:%.*]]
; CHECK-PRED-OR-SCALAR: for.body: ; CHECK-PRED-OR-SCALAR: for.body:
; CHECK-PRED-OR-SCALAR-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ] ; CHECK-PRED-OR-SCALAR-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
; CHECK-PRED-OR-SCALAR-NEXT: [[LDIDX:%.*]] = getelementptr i8, i8* [[SRC:%.*]], i32 [[I]] ; CHECK-PRED-OR-SCALAR-NEXT: [[LDIDX:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[I]]
; CHECK-PRED-OR-SCALAR-NEXT: [[STIDX:%.*]] = getelementptr i8, i8* [[DST:%.*]], i32 [[I]] ; CHECK-PRED-OR-SCALAR-NEXT: [[STIDX:%.*]] = getelementptr i8, i8* [[DST]], i32 [[I]]
; CHECK-PRED-OR-SCALAR-NEXT: [[LDVAL:%.*]] = load i8, i8* [[LDIDX]], align 1 ; CHECK-PRED-OR-SCALAR-NEXT: [[LDVAL:%.*]] = load i8, i8* [[LDIDX]], align 1
; CHECK-PRED-OR-SCALAR-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LDVAL]], 5 ; CHECK-PRED-OR-SCALAR-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LDVAL]], 5
; CHECK-PRED-OR-SCALAR-NEXT: [[VAL:%.*]] = select i1 [[CMP1]], i8 1, i8 2 ; CHECK-PRED-OR-SCALAR-NEXT: [[VAL:%.*]] = select i1 [[CMP1]], i8 1, i8 2
; CHECK-PRED-OR-SCALAR-NEXT: store i8 [[VAL]], i8* [[STIDX]], align 1 ; CHECK-PRED-OR-SCALAR-NEXT: store i8 [[VAL]], i8* [[STIDX]], align 1
; CHECK-PRED-OR-SCALAR-NEXT: [[INC]] = add nsw i32 [[I]], 1 ; CHECK-PRED-OR-SCALAR-NEXT: [[INC]] = add nsw i32 [[I]], 1
; CHECK-PRED-OR-SCALAR-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[I]], 8 ; CHECK-PRED-OR-SCALAR-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[I]], 8
; CHECK-PRED-OR-SCALAR-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END:%.*]], !llvm.loop [[LOOP3:![0-9]+]] ; CHECK-PRED-OR-SCALAR-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP14:![0-9]+]]
; CHECK-PRED-OR-SCALAR: for.end: ; CHECK-PRED-OR-SCALAR: for.end:
; CHECK-PRED-OR-SCALAR-NEXT: ret i32 0 ; CHECK-PRED-OR-SCALAR-NEXT: ret i32 0
; ;
; PRED-OR-DONTVEC-LABEL: @const_low_trip_count_hint_no_pred( ; PRED-OR-DONTVEC-LABEL: @const_low_trip_count_hint_no_pred(
; PRED-OR-DONTVEC-NEXT: entry: ; PRED-OR-DONTVEC-NEXT: entry:
; PRED-OR-DONTVEC-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
; PRED-OR-DONTVEC: vector.memcheck:
; PRED-OR-DONTVEC-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, i8* [[DST:%.*]], i64 9
; PRED-OR-DONTVEC-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, i8* [[SRC:%.*]], i64 9
; PRED-OR-DONTVEC-NEXT: [[BOUND0:%.*]] = icmp ult i8* [[DST]], [[SCEVGEP1]]
; PRED-OR-DONTVEC-NEXT: [[BOUND1:%.*]] = icmp ult i8* [[SRC]], [[SCEVGEP]]
; PRED-OR-DONTVEC-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
; PRED-OR-DONTVEC-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]]
; PRED-OR-DONTVEC: vector.ph:
; PRED-OR-DONTVEC-NEXT: br label [[VECTOR_BODY:%.*]]
; PRED-OR-DONTVEC: vector.body:
; PRED-OR-DONTVEC-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_STORE_CONTINUE13:%.*]] ]
; PRED-OR-DONTVEC-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ <i32 0, i32 1, i32 2, i32 3>, [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[PRED_STORE_CONTINUE13]] ]
; PRED-OR-DONTVEC-NEXT: [[TMP0:%.*]] = add i32 [[INDEX]], 0
; PRED-OR-DONTVEC-NEXT: [[TMP1:%.*]] = add i32 [[INDEX]], 1
; PRED-OR-DONTVEC-NEXT: [[TMP2:%.*]] = add i32 [[INDEX]], 2
; PRED-OR-DONTVEC-NEXT: [[TMP3:%.*]] = add i32 [[INDEX]], 3
; PRED-OR-DONTVEC-NEXT: [[TMP4:%.*]] = icmp ule <4 x i32> [[VEC_IND]], <i32 8, i32 8, i32 8, i32 8>
; PRED-OR-DONTVEC-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
; PRED-OR-DONTVEC-NEXT: br i1 [[TMP5]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]]
; PRED-OR-DONTVEC: pred.load.if:
; PRED-OR-DONTVEC-NEXT: [[TMP6:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[TMP0]]
; PRED-OR-DONTVEC-NEXT: [[TMP7:%.*]] = load i8, i8* [[TMP6]], align 1, !alias.scope !8
; PRED-OR-DONTVEC-NEXT: [[TMP8:%.*]] = insertelement <4 x i8> poison, i8 [[TMP7]], i32 0
; PRED-OR-DONTVEC-NEXT: br label [[PRED_LOAD_CONTINUE]]
; PRED-OR-DONTVEC: pred.load.continue:
; PRED-OR-DONTVEC-NEXT: [[TMP9:%.*]] = phi <4 x i8> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_LOAD_IF]] ]
; PRED-OR-DONTVEC-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
; PRED-OR-DONTVEC-NEXT: br i1 [[TMP10]], label [[PRED_LOAD_IF2:%.*]], label [[PRED_LOAD_CONTINUE3:%.*]]
; PRED-OR-DONTVEC: pred.load.if2:
; PRED-OR-DONTVEC-NEXT: [[TMP11:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[TMP1]]
; PRED-OR-DONTVEC-NEXT: [[TMP12:%.*]] = load i8, i8* [[TMP11]], align 1, !alias.scope !8
; PRED-OR-DONTVEC-NEXT: [[TMP13:%.*]] = insertelement <4 x i8> [[TMP9]], i8 [[TMP12]], i32 1
; PRED-OR-DONTVEC-NEXT: br label [[PRED_LOAD_CONTINUE3]]
; PRED-OR-DONTVEC: pred.load.continue3:
; PRED-OR-DONTVEC-NEXT: [[TMP14:%.*]] = phi <4 x i8> [ [[TMP9]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], [[PRED_LOAD_IF2]] ]
; PRED-OR-DONTVEC-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
; PRED-OR-DONTVEC-NEXT: br i1 [[TMP15]], label [[PRED_LOAD_IF4:%.*]], label [[PRED_LOAD_CONTINUE5:%.*]]
; PRED-OR-DONTVEC: pred.load.if4:
; PRED-OR-DONTVEC-NEXT: [[TMP16:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[TMP2]]
; PRED-OR-DONTVEC-NEXT: [[TMP17:%.*]] = load i8, i8* [[TMP16]], align 1, !alias.scope !8
; PRED-OR-DONTVEC-NEXT: [[TMP18:%.*]] = insertelement <4 x i8> [[TMP14]], i8 [[TMP17]], i32 2
; PRED-OR-DONTVEC-NEXT: br label [[PRED_LOAD_CONTINUE5]]
; PRED-OR-DONTVEC: pred.load.continue5:
; PRED-OR-DONTVEC-NEXT: [[TMP19:%.*]] = phi <4 x i8> [ [[TMP14]], [[PRED_LOAD_CONTINUE3]] ], [ [[TMP18]], [[PRED_LOAD_IF4]] ]
; PRED-OR-DONTVEC-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
; PRED-OR-DONTVEC-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF6:%.*]], label [[PRED_LOAD_CONTINUE7:%.*]]
; PRED-OR-DONTVEC: pred.load.if6:
; PRED-OR-DONTVEC-NEXT: [[TMP21:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[TMP3]]
; PRED-OR-DONTVEC-NEXT: [[TMP22:%.*]] = load i8, i8* [[TMP21]], align 1, !alias.scope !8
; PRED-OR-DONTVEC-NEXT: [[TMP23:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP22]], i32 3
; PRED-OR-DONTVEC-NEXT: br label [[PRED_LOAD_CONTINUE7]]
; PRED-OR-DONTVEC: pred.load.continue7:
; PRED-OR-DONTVEC-NEXT: [[TMP24:%.*]] = phi <4 x i8> [ [[TMP19]], [[PRED_LOAD_CONTINUE5]] ], [ [[TMP23]], [[PRED_LOAD_IF6]] ]
; PRED-OR-DONTVEC-NEXT: [[TMP25:%.*]] = icmp eq <4 x i8> [[TMP24]], <i8 5, i8 5, i8 5, i8 5>
; PRED-OR-DONTVEC-NEXT: [[TMP26:%.*]] = select <4 x i1> [[TMP25]], <4 x i8> <i8 1, i8 1, i8 1, i8 1>, <4 x i8> <i8 2, i8 2, i8 2, i8 2>
; PRED-OR-DONTVEC-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[TMP4]], i32 0
; PRED-OR-DONTVEC-NEXT: br i1 [[TMP27]], label [[PRED_STORE_IF:%.*]], label [[PRED_STORE_CONTINUE:%.*]]
; PRED-OR-DONTVEC: pred.store.if:
; PRED-OR-DONTVEC-NEXT: [[TMP28:%.*]] = getelementptr i8, i8* [[DST]], i32 [[TMP0]]
; PRED-OR-DONTVEC-NEXT: [[TMP29:%.*]] = extractelement <4 x i8> [[TMP26]], i32 0
; PRED-OR-DONTVEC-NEXT: store i8 [[TMP29]], i8* [[TMP28]], align 1, !alias.scope !11, !noalias !8
; PRED-OR-DONTVEC-NEXT: br label [[PRED_STORE_CONTINUE]]
; PRED-OR-DONTVEC: pred.store.continue:
; PRED-OR-DONTVEC-NEXT: [[TMP30:%.*]] = extractelement <4 x i1> [[TMP4]], i32 1
; PRED-OR-DONTVEC-NEXT: br i1 [[TMP30]], label [[PRED_STORE_IF8:%.*]], label [[PRED_STORE_CONTINUE9:%.*]]
; PRED-OR-DONTVEC: pred.store.if8:
; PRED-OR-DONTVEC-NEXT: [[TMP31:%.*]] = getelementptr i8, i8* [[DST]], i32 [[TMP1]]
; PRED-OR-DONTVEC-NEXT: [[TMP32:%.*]] = extractelement <4 x i8> [[TMP26]], i32 1
; PRED-OR-DONTVEC-NEXT: store i8 [[TMP32]], i8* [[TMP31]], align 1, !alias.scope !11, !noalias !8
; PRED-OR-DONTVEC-NEXT: br label [[PRED_STORE_CONTINUE9]]
; PRED-OR-DONTVEC: pred.store.continue9:
; PRED-OR-DONTVEC-NEXT: [[TMP33:%.*]] = extractelement <4 x i1> [[TMP4]], i32 2
; PRED-OR-DONTVEC-NEXT: br i1 [[TMP33]], label [[PRED_STORE_IF10:%.*]], label [[PRED_STORE_CONTINUE11:%.*]]
; PRED-OR-DONTVEC: pred.store.if10:
; PRED-OR-DONTVEC-NEXT: [[TMP34:%.*]] = getelementptr i8, i8* [[DST]], i32 [[TMP2]]
; PRED-OR-DONTVEC-NEXT: [[TMP35:%.*]] = extractelement <4 x i8> [[TMP26]], i32 2
; PRED-OR-DONTVEC-NEXT: store i8 [[TMP35]], i8* [[TMP34]], align 1, !alias.scope !11, !noalias !8
; PRED-OR-DONTVEC-NEXT: br label [[PRED_STORE_CONTINUE11]]
; PRED-OR-DONTVEC: pred.store.continue11:
; PRED-OR-DONTVEC-NEXT: [[TMP36:%.*]] = extractelement <4 x i1> [[TMP4]], i32 3
; PRED-OR-DONTVEC-NEXT: br i1 [[TMP36]], label [[PRED_STORE_IF12:%.*]], label [[PRED_STORE_CONTINUE13]]
; PRED-OR-DONTVEC: pred.store.if12:
; PRED-OR-DONTVEC-NEXT: [[TMP37:%.*]] = getelementptr i8, i8* [[DST]], i32 [[TMP3]]
; PRED-OR-DONTVEC-NEXT: [[TMP38:%.*]] = extractelement <4 x i8> [[TMP26]], i32 3
; PRED-OR-DONTVEC-NEXT: store i8 [[TMP38]], i8* [[TMP37]], align 1, !alias.scope !11, !noalias !8
; PRED-OR-DONTVEC-NEXT: br label [[PRED_STORE_CONTINUE13]]
; PRED-OR-DONTVEC: pred.store.continue13:
; PRED-OR-DONTVEC-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], 4
; PRED-OR-DONTVEC-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], <i32 4, i32 4, i32 4, i32 4>
; PRED-OR-DONTVEC-NEXT: [[TMP39:%.*]] = icmp eq i32 [[INDEX_NEXT]], 12
; PRED-OR-DONTVEC-NEXT: br i1 [[TMP39]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]]
; PRED-OR-DONTVEC: middle.block:
; PRED-OR-DONTVEC-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]]
; PRED-OR-DONTVEC: scalar.ph:
; PRED-OR-DONTVEC-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 12, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ], [ 0, [[VECTOR_MEMCHECK]] ]
; PRED-OR-DONTVEC-NEXT: br label [[FOR_BODY:%.*]] ; PRED-OR-DONTVEC-NEXT: br label [[FOR_BODY:%.*]]
; PRED-OR-DONTVEC: for.body: ; PRED-OR-DONTVEC: for.body:
; PRED-OR-DONTVEC-NEXT: [[I:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ] ; PRED-OR-DONTVEC-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
; PRED-OR-DONTVEC-NEXT: [[LDIDX:%.*]] = getelementptr i8, i8* [[SRC:%.*]], i32 [[I]] ; PRED-OR-DONTVEC-NEXT: [[LDIDX:%.*]] = getelementptr i8, i8* [[SRC]], i32 [[I]]
; PRED-OR-DONTVEC-NEXT: [[STIDX:%.*]] = getelementptr i8, i8* [[DST:%.*]], i32 [[I]] ; PRED-OR-DONTVEC-NEXT: [[STIDX:%.*]] = getelementptr i8, i8* [[DST]], i32 [[I]]
; PRED-OR-DONTVEC-NEXT: [[LDVAL:%.*]] = load i8, i8* [[LDIDX]], align 1 ; PRED-OR-DONTVEC-NEXT: [[LDVAL:%.*]] = load i8, i8* [[LDIDX]], align 1
; PRED-OR-DONTVEC-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LDVAL]], 5 ; PRED-OR-DONTVEC-NEXT: [[CMP1:%.*]] = icmp eq i8 [[LDVAL]], 5
; PRED-OR-DONTVEC-NEXT: [[VAL:%.*]] = select i1 [[CMP1]], i8 1, i8 2 ; PRED-OR-DONTVEC-NEXT: [[VAL:%.*]] = select i1 [[CMP1]], i8 1, i8 2
; PRED-OR-DONTVEC-NEXT: store i8 [[VAL]], i8* [[STIDX]], align 1 ; PRED-OR-DONTVEC-NEXT: store i8 [[VAL]], i8* [[STIDX]], align 1
; PRED-OR-DONTVEC-NEXT: [[INC]] = add nsw i32 [[I]], 1 ; PRED-OR-DONTVEC-NEXT: [[INC]] = add nsw i32 [[I]], 1
; PRED-OR-DONTVEC-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[I]], 8 ; PRED-OR-DONTVEC-NEXT: [[EXITCOND:%.*]] = icmp slt i32 [[I]], 8
; PRED-OR-DONTVEC-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END:%.*]], !llvm.loop [[LOOP3:![0-9]+]] ; PRED-OR-DONTVEC-NEXT: br i1 [[EXITCOND]], label [[FOR_BODY]], label [[FOR_END]], !llvm.loop [[LOOP14:![0-9]+]]
; PRED-OR-DONTVEC: for.end: ; PRED-OR-DONTVEC: for.end:
; PRED-OR-DONTVEC-NEXT: ret i32 0 ; PRED-OR-DONTVEC-NEXT: ret i32 0
; ;
entry: entry:
br label %for.body br label %for.body
for.body: ; preds = %for.body, %entry for.body: ; preds = %for.body, %entry
%i = phi i32 [ 0, %entry ], [ %inc, %for.body ] %i = phi i32 [ 0, %entry ], [ %inc, %for.body ]
Show All 19 Lines