This is an archive of the discontinued LLVM Phabricator instance.

Differential D83133

[ARM][MVE] Refactor option -disable-mve-tail-predication
ClosedPublic

Authored by SjoerdMeijer on Jul 3 2020, 7:45 AM.

Details

Reviewers
samparker
dmgreen
Commits
rG595270ae3967: [ARM][MVE] Refactor option -disable-mve-tail-predication
Summary

This refactors option -disable-mve-tail-predication to take different arguments so that we have 1 option to control tail-predication rather than several different ones.




This is also a prep step for D82953, in which we want to reject reductions unless that is requested with this option.
Diff Detail
Event Timeline
SjoerdMeijer created this revision.Jul 3 2020, 7:45 AM
Herald added a project: Restricted Project.  ·  View Herald TranscriptJul 3 2020, 7:45 AM
Herald added subscribers: danielkiss, zzheng, hiraditya, kristof.beyls.  ·  View Herald Transcript
dmgreen accepted this revision.Jul 13 2020, 2:49 AM
Sounds good to me. It may get a bit unwieldy to have one option with many preferences, but looks OK with these ones.
This revision is now accepted and ready to land.Jul 13 2020, 2:49 AM
SjoerdMeijer added a comment.Jul 13 2020, 3:01 AM
Thanks, and yes, we can/should revise this later. For now this should be an improvement, because I needed 3 different options for tail-predication, which was a becoming unwieldy :-) It should only become simpler from now on as reductions are a special case at the moment, but that should disappear at some point.
Closed by commit rG595270ae3967: [ARM][MVE] Refactor option -disable-mve-tail-predication (authored by SjoerdMeijer).  ·  Explain WhyJul 13 2020, 5:41 AM
This revision was automatically updated to reflect the committed changes.
SjoerdMeijer mentioned this in D82953: [ARM][MVE] Only tail-fold integer add reductions.Jul 13 2020, 9:25 AM
Revision Contents
Diff 277396
llvm/lib/Target/ARM/ARMTargetTransformInfo.h
Show All 32 Lines
class ARMTargetLowering;
class ARMTargetLowering;

class Instruction;
class Instruction;

class Loop;
class Loop;

class SCEV;
class SCEV;

class ScalarEvolution;
class ScalarEvolution;

class Type;
class Type;

class Value;
class Value;




namespace TailPredication {

  enum Mode {

    Disabled = 0,

    EnabledNoReductions,

    Enabled,

    ForceEnabledNoReductions,

    ForceEnabled

  };

}



class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {
class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {

  using BaseT = BasicTTIImplBase<ARMTTIImpl>;
  using BaseT = BasicTTIImplBase<ARMTTIImpl>;

  using TTI = TargetTransformInfo;
  using TTI = TargetTransformInfo;




  friend BaseT;
  friend BaseT;




  const ARMSubtarget *ST;
  const ARMSubtarget *ST;

  const ARMTargetLowering *TLI;
  const ARMTargetLowering *TLI;

▲ Show 20 Lines • Show All 222 Lines • Show Last 20 Lines
llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
Show All 39 Lines
static cl::opt<bool> EnableMaskedLoadStores(
static cl::opt<bool> EnableMaskedLoadStores(

  "enable-arm-maskedldst", cl::Hidden, cl::init(true),
  "enable-arm-maskedldst", cl::Hidden, cl::init(true),

  cl::desc("Enable the generation of masked loads and stores"));
  cl::desc("Enable the generation of masked loads and stores"));




static cl::opt<bool> DisableLowOverheadLoops(
static cl::opt<bool> DisableLowOverheadLoops(

  "disable-arm-loloops", cl::Hidden, cl::init(false),
  "disable-arm-loloops", cl::Hidden, cl::init(false),

  cl::desc("Disable the generation of low-overhead loops"));
  cl::desc("Disable the generation of low-overhead loops"));




extern cl::opt<bool> DisableTailPredication;
extern cl::opt<TailPredication::Mode> EnableTailPredication;




extern cl::opt<bool> EnableMaskedGatherScatters;
extern cl::opt<bool> EnableMaskedGatherScatters;




bool ARMTTIImpl::areInlineCompatible(const Function *Caller,
bool ARMTTIImpl::areInlineCompatible(const Function *Caller,

                                     const Function *Callee) const {
                                     const Function *Callee) const {

  const TargetMachine &TM = getTLI()->getTargetMachine();
  const TargetMachine &TM = getTLI()->getTargetMachine();

  const FeatureBitset &CallerBits =
  const FeatureBitset &CallerBits =

      TM.getSubtargetImpl(*Caller)->getFeatureBits();
      TM.getSubtargetImpl(*Caller)->getFeatureBits();

▲ Show 20 Lines • Show All 1,396 Lines • ▼ Show 20 Lines
}
}




bool ARMTTIImpl::preferPredicateOverEpilogue(Loop *L, LoopInfo *LI,
bool ARMTTIImpl::preferPredicateOverEpilogue(Loop *L, LoopInfo *LI,

                                             ScalarEvolution &SE,
                                             ScalarEvolution &SE,

                                             AssumptionCache &AC,
                                             AssumptionCache &AC,

                                             TargetLibraryInfo *TLI,
                                             TargetLibraryInfo *TLI,

                                             DominatorTree *DT,
                                             DominatorTree *DT,

                                             const LoopAccessInfo *LAI) {
                                             const LoopAccessInfo *LAI) {

  if (DisableTailPredication)
  if (!EnableTailPredication)

    return false;
    return false;




  // Creating a predicated vector loop is the first step for generating a
  // Creating a predicated vector loop is the first step for generating a

  // tail-predicated hardware loop, for which we need the MVE masked
  // tail-predicated hardware loop, for which we need the MVE masked

  // load/stores instructions:
  // load/stores instructions:

  if (!ST->hasMVEIntegerOps())
  if (!ST->hasMVEIntegerOps())

    return false;
    return false;




Show All 26 Lines    LLVM_DEBUG(dbgs() << "preferPredicateOverEpilogue: hardware-loop is not "

                         "a candidate.\n");
                         "a candidate.\n");

    return false;
    return false;

  }
  }




  return canTailPredicateLoop(L, LI, SE, DL, LAI);
  return canTailPredicateLoop(L, LI, SE, DL, LAI);

}
}




bool ARMTTIImpl::emitGetActiveLaneMask() const {
bool ARMTTIImpl::emitGetActiveLaneMask() const {

  if (!ST->hasMVEIntegerOps() || DisableTailPredication)
  if (!ST->hasMVEIntegerOps() || !EnableTailPredication)

    return false;
    return false;




  // Intrinsic @llvm.get.active.lane.mask is supported.
  // Intrinsic @llvm.get.active.lane.mask is supported.

  // It is used in the MVETailPredication pass, which requires the number of
  // It is used in the MVETailPredication pass, which requires the number of

  // elements processed by this vector loop to setup the tail-predicated
  // elements processed by this vector loop to setup the tail-predicated

  // loop.
  // loop.

  return true;
  return true;

}
}

▲ Show 20 Lines • Show All 81 Lines • Show Last 20 Lines
llvm/lib/Target/ARM/MVETailPredication.cpp
Show All 36 Lines
///    which we extract to set up the number of elements processed by the loop.
///    which we extract to set up the number of elements processed by the loop.

/// 2) Intrinsic @llvm.get.active.lanes() is then replaced by the MVE target
/// 2) Intrinsic @llvm.get.active.lanes() is then replaced by the MVE target

///    specific VCTP intrinsic to represent the effect of tail predication.
///    specific VCTP intrinsic to represent the effect of tail predication.

///    This will be picked up by the ARM Low-overhead loop pass, which performs
///    This will be picked up by the ARM Low-overhead loop pass, which performs

///    the final transformation to a DLSTP or WLSTP tail-predicated loop.
///    the final transformation to a DLSTP or WLSTP tail-predicated loop.




#include "ARM.h"
#include "ARM.h"

#include "ARMSubtarget.h"
#include "ARMSubtarget.h"

#include "ARMTargetTransformInfo.h"

#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopInfo.h"

#include "llvm/Analysis/LoopPass.h"
#include "llvm/Analysis/LoopPass.h"

#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolution.h"

#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"

#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetLibraryInfo.h"

#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"

#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/CodeGen/TargetPassConfig.h"

#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/IRBuilder.h"

#include "llvm/IR/Instructions.h"
#include "llvm/IR/Instructions.h"

#include "llvm/IR/IntrinsicsARM.h"
#include "llvm/IR/IntrinsicsARM.h"

#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/PatternMatch.h"

#include "llvm/InitializePasses.h"
#include "llvm/InitializePasses.h"

#include "llvm/Support/Debug.h"
#include "llvm/Support/Debug.h"

#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"

#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/LoopUtils.h"

#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"




using namespace llvm;
using namespace llvm;




#define DEBUG_TYPE "mve-tail-predication"
#define DEBUG_TYPE "mve-tail-predication"

#define DESC "Transform predicated vector loops to use MVE tail predication"
#define DESC "Transform predicated vector loops to use MVE tail predication"




static cl::opt<bool>
cl::opt<TailPredication::Mode> EnableTailPredication(

ForceTailPredication("force-mve-tail-predication", cl::Hidden, cl::init(false),
   "tail-predication", cl::desc("MVE tail-predication options"),

                     cl::desc("Force MVE tail-predication even if it might be "
   cl::init(TailPredication::Disabled),

                              "unsafe (e.g. possible overflow in loop "
   cl::values(clEnumValN(TailPredication::Disabled, "disabled",

                              "counters)"));
                         "Don't tail-predicate loops"),


              clEnumValN(TailPredication::EnabledNoReductions,

cl::opt<bool>
                         "enabled-no-reductions",

DisableTailPredication("disable-mve-tail-predication", cl::Hidden,
                         "Enable tail-predication, but not for reduction loops"),

                       cl::init(true),
              clEnumValN(TailPredication::Enabled,

                       cl::desc("Disable MVE Tail Predication"));
                         "enabled",

                         "Enable tail-predication, including reduction loops"),

              clEnumValN(TailPredication::ForceEnabledNoReductions,

                         "force-enabled-no-reductions",

                         "Enable tail-predication, but not for reduction loops, "

                         "and force this which might be unsafe"),

              clEnumValN(TailPredication::ForceEnabled,

                         "force-enabled",

                         "Enable tail-predication, including reduction loops, "

                         "and force this which might be unsafe")));





namespace {
namespace {




class MVETailPredication : public LoopPass {
class MVETailPredication : public LoopPass {

  SmallVector<IntrinsicInst*, 4> MaskedInsts;
  SmallVector<IntrinsicInst*, 4> MaskedInsts;

  Loop *L = nullptr;
  Loop *L = nullptr;

  ScalarEvolution *SE = nullptr;
  ScalarEvolution *SE = nullptr;

  TargetTransformInfo *TTI = nullptr;
  TargetTransformInfo *TTI = nullptr;

  const ARMSubtarget *ST = nullptr;
  const ARMSubtarget *ST = nullptr;

▲ Show 20 Lines • Show All 56 Lines • ▼ Show 20 Lines  if (!Call)

    return false;
    return false;




  Intrinsic::ID ID = Call->getIntrinsicID();
  Intrinsic::ID ID = Call->getIntrinsicID();

  // TODO: Support gather/scatter expand/compress operations.
  // TODO: Support gather/scatter expand/compress operations.

  return ID == Intrinsic::masked_store || ID == Intrinsic::masked_load;
  return ID == Intrinsic::masked_store || ID == Intrinsic::masked_load;

}
}




bool MVETailPredication::runOnLoop(Loop *L, LPPassManager&) {
bool MVETailPredication::runOnLoop(Loop *L, LPPassManager&) {

  if (skipLoop(L) || DisableTailPredication)
  if (skipLoop(L) || !EnableTailPredication)

    return false;
    return false;




  MaskedInsts.clear();
  MaskedInsts.clear();

  Function &F = *L->getHeader()->getParent();
  Function &F = *L->getHeader()->getParent();

  auto &TPC = getAnalysis<TargetPassConfig>();
  auto &TPC = getAnalysis<TargetPassConfig>();

  auto &TM = TPC.getTM<TargetMachine>();
  auto &TM = TPC.getTM<TargetMachine>();

  ST = &TM.getSubtarget<ARMSubtarget>(F);
  ST = &TM.getSubtarget<ARMSubtarget>(F);

  TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
  TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);

▲ Show 20 Lines • Show All 183 Lines • ▼ Show 20 Lines
//    by rounding up ElementCount up:
//    by rounding up ElementCount up:

//        ((ElementCount + (VectorWidth - 1)) / VectorWidth
//        ((ElementCount + (VectorWidth - 1)) / VectorWidth

//    and evaluate if expression isKnownNonNegative:
//    and evaluate if expression isKnownNonNegative:

//        (((ElementCount + (VectorWidth - 1)) / VectorWidth) - TripCount
//        (((ElementCount + (VectorWidth - 1)) / VectorWidth) - TripCount

// 3) The IV must be an induction phi with an increment equal to the
// 3) The IV must be an induction phi with an increment equal to the

//    vector width.
//    vector width.

bool MVETailPredication::IsSafeActiveMask(IntrinsicInst *ActiveLaneMask,
bool MVETailPredication::IsSafeActiveMask(IntrinsicInst *ActiveLaneMask,

    Value *TripCount, FixedVectorType *VecTy) {
    Value *TripCount, FixedVectorType *VecTy) {

  bool ForceTailPredication =

    EnableTailPredication == TailPredication::ForceEnabledNoReductions ||

    EnableTailPredication == TailPredication::ForceEnabled;

  // 1) Test whether entry to the loop is protected by a conditional
  // 1) Test whether entry to the loop is protected by a conditional

  // BTC + 1 < 0. In other words, if the scalar trip count overflows,
  // BTC + 1 < 0. In other words, if the scalar trip count overflows,

  // becomes negative, we shouldn't enter the loop and creating
  // becomes negative, we shouldn't enter the loop and creating

  // tripcount expression BTC + 1 is not safe. So, check that BTC
  // tripcount expression BTC + 1 is not safe. So, check that BTC

  // isn't max. This is evaluated in unsigned, because the semantics
  // isn't max. This is evaluated in unsigned, because the semantics

  // of @get.active.lane.mask is a ULE comparison.
  // of @get.active.lane.mask is a ULE comparison.




  int VectorWidth = VecTy->getNumElements();
  int VectorWidth = VecTy->getNumElements();

▲ Show 20 Lines • Show All 247 Lines • Show Last 20 Lines
llvm/test/CodeGen/Thumb2/LowOverheadLoops/basic-tail-pred.ll
; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -disable-mve-tail-predication=false -mattr=+mve,+lob %s -S -o - | FileCheck %s
; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -tail-predication=enabled -mattr=+mve,+lob %s -S -o - | FileCheck %s




; CHECK-LABEL: mul_v16i8
; CHECK-LABEL: mul_v16i8

; CHECK-NOT: %num.elements = add i32 %trip.count.minus.1, 1
; CHECK-NOT: %num.elements = add i32 %trip.count.minus.1, 1

; CHECK: vector.body:
; CHECK: vector.body:

; CHECK: %index = phi i32
; CHECK: %index = phi i32

; CHECK: [[ELEMS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[REMAINING:%[^ ]+]], %vector.body ]
; CHECK: [[ELEMS:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[REMAINING:%[^ ]+]], %vector.body ]

; CHECK: [[VCTP:%[^ ]+]] = call <16 x i1> @llvm.arm.mve.vctp8(i32 [[ELEMS]])
; CHECK: [[VCTP:%[^ ]+]] = call <16 x i1> @llvm.arm.mve.vctp8(i32 [[ELEMS]])

; CHECK: [[REMAINING]] = sub i32 [[ELEMS]], 16
; CHECK: [[REMAINING]] = sub i32 [[ELEMS]], 16

▲ Show 20 Lines • Show All 436 Lines • Show Last 20 Lines
llvm/test/CodeGen/Thumb2/LowOverheadLoops/clear-maskedinsts.ll
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py

; RUN: opt -mtriple=thumbv8.1m.main -mattr=+mve.fp -mve-tail-predication -disable-mve-tail-predication=false %s -S -o - | FileCheck %s
; RUN: opt -mtriple=thumbv8.1m.main -mattr=+mve.fp -mve-tail-predication -tail-predication=enabled %s -S -o - | FileCheck %s




define hidden i32 @_Z4loopPiPjiS0_i(i32* noalias nocapture readonly %s1, i32* noalias nocapture readonly %s2, i32 %x, i32* noalias nocapture %d, i32 %n) {
define hidden i32 @_Z4loopPiPjiS0_i(i32* noalias nocapture readonly %s1, i32* noalias nocapture readonly %s2, i32 %x, i32* noalias nocapture %d, i32 %n) {

; CHECK-LABEL: @_Z4loopPiPjiS0_i(
; CHECK-LABEL: @_Z4loopPiPjiS0_i(

; CHECK-NEXT:  entry:
; CHECK-NEXT:  entry:

; CHECK-NEXT:    [[CMP63:%.*]] = icmp sgt i32 [[N:%.*]], 0
; CHECK-NEXT:    [[CMP63:%.*]] = icmp sgt i32 [[N:%.*]], 0

; CHECK-NEXT:    br i1 [[CMP63]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]
; CHECK-NEXT:    br i1 [[CMP63]], label [[FOR_BODY_LR_PH:%.*]], label [[FOR_COND_CLEANUP:%.*]]

; CHECK:       for.body.lr.ph:
; CHECK:       for.body.lr.ph:

; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0
; CHECK-NEXT:    [[TOBOOL:%.*]] = icmp eq i32 [[X:%.*]], 0

▲ Show 20 Lines • Show All 147 Lines • Show Last 20 Lines
llvm/test/CodeGen/Thumb2/LowOverheadLoops/cond-vector-reduce-mve-codegen.ll
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py

; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve -disable-mve-tail-predication=false --verify-machineinstrs %s -o - | FileCheck %s
; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve -tail-predication=enabled --verify-machineinstrs %s -o - | FileCheck %s




define dso_local i32 @vpsel_mul_reduce_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture readonly %c, i32 %N) {
define dso_local i32 @vpsel_mul_reduce_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture readonly %c, i32 %N) {

; CHECK-LABEL: vpsel_mul_reduce_add:
; CHECK-LABEL: vpsel_mul_reduce_add:

; CHECK:       @ %bb.0: @ %entry
; CHECK:       @ %bb.0: @ %entry

; CHECK-NEXT:    cmp r3, #0
; CHECK-NEXT:    cmp r3, #0

; CHECK-NEXT:    itt eq
; CHECK-NEXT:    itt eq

; CHECK-NEXT:    moveq r0, #0
; CHECK-NEXT:    moveq r0, #0

; CHECK-NEXT:    bxeq lr
; CHECK-NEXT:    bxeq lr

▲ Show 20 Lines • Show All 527 Lines • Show Last 20 Lines
llvm/test/CodeGen/Thumb2/LowOverheadLoops/extending-loads.ll
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py

; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve -disable-mve-tail-predication=false %s -o - | FileCheck %s
; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve -tail-predication=enabled %s -o - | FileCheck %s




define dso_local arm_aapcs_vfpcc void @sext_i8(i16* noalias nocapture %a, i8* nocapture readonly %b, i32 %N) {
define dso_local arm_aapcs_vfpcc void @sext_i8(i16* noalias nocapture %a, i8* nocapture readonly %b, i32 %N) {

; CHECK-LABEL: sext_i8:
; CHECK-LABEL: sext_i8:

; CHECK:       @ %bb.0: @ %entry
; CHECK:       @ %bb.0: @ %entry

; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    push {r7, lr}

; CHECK-NEXT:    cmp r2, #0
; CHECK-NEXT:    cmp r2, #0

; CHECK-NEXT:    it eq
; CHECK-NEXT:    it eq

; CHECK-NEXT:    popeq {r7, pc}
; CHECK-NEXT:    popeq {r7, pc}

▲ Show 20 Lines • Show All 237 Lines • Show Last 20 Lines
llvm/test/CodeGen/Thumb2/LowOverheadLoops/fast-fp-loops.ll
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py

; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp,+fp-armv8d16sp,+fp16,+fullfp16 -disable-mve-tail-predication=false %s -o - | FileCheck %s
; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve.fp,+fp-armv8d16sp,+fp16,+fullfp16 -tail-predication=enabled %s -o - | FileCheck %s




define arm_aapcs_vfpcc void @fast_float_mul(float* nocapture %a, float* nocapture readonly %b, float* nocapture readonly %c, i32 %N) {
define arm_aapcs_vfpcc void @fast_float_mul(float* nocapture %a, float* nocapture readonly %b, float* nocapture readonly %c, i32 %N) {

; CHECK-LABEL: fast_float_mul:
; CHECK-LABEL: fast_float_mul:

; CHECK:       @ %bb.0: @ %entry
; CHECK:       @ %bb.0: @ %entry

; CHECK-NEXT:    push {r4, r5, r6, r7, lr}
; CHECK-NEXT:    push {r4, r5, r6, r7, lr}

; CHECK-NEXT:    cmp r3, #0
; CHECK-NEXT:    cmp r3, #0

; CHECK-NEXT:    beq.w .LBB0_11
; CHECK-NEXT:    beq.w .LBB0_11

; CHECK-NEXT:  @ %bb.1: @ %vector.memcheck
; CHECK-NEXT:  @ %bb.1: @ %vector.memcheck

▲ Show 20 Lines • Show All 584 Lines • Show Last 20 Lines
llvm/test/CodeGen/Thumb2/LowOverheadLoops/lsr-profitable-chain.ll
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py

; RUN: llc -O3 -disable-mve-tail-predication=false -mtriple=thumbv8.1m.main -mattr=+mve,+mve.fp %s -o - | FileCheck %s
; RUN: llc -O3 -tail-predication=enabled -mtriple=thumbv8.1m.main -mattr=+mve,+mve.fp %s -o - | FileCheck %s




target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"
target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64"

target triple = "thumbv8.1m-arm-none-eabi"
target triple = "thumbv8.1m-arm-none-eabi"




; Tests that LSR will not interfere with the VCTP intrinsic,
; Tests that LSR will not interfere with the VCTP intrinsic,

; and that this loop will correctly become tail-predicated.
; and that this loop will correctly become tail-predicated.




define arm_aapcs_vfpcc float @vctpi32(float* %0, i32 %1) {
define arm_aapcs_vfpcc float @vctpi32(float* %0, i32 %1) {

▲ Show 20 Lines • Show All 59 Lines • Show Last 20 Lines
llvm/test/CodeGen/Thumb2/LowOverheadLoops/mve-tail-data-types.ll
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py

; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve -disable-mve-tail-predication=false %s -o - | FileCheck %s
; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve -tail-predication=enabled %s -o - | FileCheck %s




define arm_aapcs_vfpcc i32 @test_acc_scalar_char(i8 zeroext %a, i8* nocapture readonly %b, i32 %N) {
define arm_aapcs_vfpcc i32 @test_acc_scalar_char(i8 zeroext %a, i8* nocapture readonly %b, i32 %N) {

; CHECK-LABEL: test_acc_scalar_char:
; CHECK-LABEL: test_acc_scalar_char:

; CHECK:       @ %bb.0: @ %entry
; CHECK:       @ %bb.0: @ %entry

; CHECK-NEXT:    cmp r2, #0
; CHECK-NEXT:    cmp r2, #0

; CHECK-NEXT:    itt eq
; CHECK-NEXT:    itt eq

; CHECK-NEXT:    moveq r0, #0
; CHECK-NEXT:    moveq r0, #0

; CHECK-NEXT:    bxeq lr
; CHECK-NEXT:    bxeq lr

▲ Show 20 Lines • Show All 1,264 Lines • Show Last 20 Lines
llvm/test/CodeGen/Thumb2/LowOverheadLoops/nested.ll
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py

; RUN: opt -mtriple=armv8.1m.main -mattr=+mve -S -mve-tail-predication -disable-mve-tail-predication=false %s -o - | FileCheck %s
; RUN: opt -mtriple=armv8.1m.main -mattr=+mve -S -mve-tail-predication -tail-predication=enabled %s -o - | FileCheck %s




define void @mat_vec_sext_i16(i16** nocapture readonly %A, i16* nocapture readonly %B, i32* noalias nocapture %C, i32 %N) {
define void @mat_vec_sext_i16(i16** nocapture readonly %A, i16* nocapture readonly %B, i32* noalias nocapture %C, i32 %N) {

; CHECK-LABEL: @mat_vec_sext_i16(
; CHECK-LABEL: @mat_vec_sext_i16(

; CHECK-NEXT:  entry:
; CHECK-NEXT:  entry:

; CHECK-NEXT:    [[CMP24:%.*]] = icmp eq i32 [[N:%.*]], 0
; CHECK-NEXT:    [[CMP24:%.*]] = icmp eq i32 [[N:%.*]], 0

; CHECK-NEXT:    br i1 [[CMP24]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER_US_PREHEADER:%.*]]
; CHECK-NEXT:    br i1 [[CMP24]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_COND1_PREHEADER_US_PREHEADER:%.*]]

; CHECK:       for.cond1.preheader.us.preheader:
; CHECK:       for.cond1.preheader.us.preheader:

; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 3
; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i32 [[N]], 3

▲ Show 20 Lines • Show All 254 Lines • Show Last 20 Lines
llvm/test/CodeGen/Thumb2/LowOverheadLoops/reductions.ll
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py

; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve -disable-mve-tail-predication=false %s -o - | FileCheck %s
; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve -tail-predication=enabled %s -o - | FileCheck %s




define dso_local arm_aapcs_vfpcc zeroext i8 @one_loop_add_add_v16i8(i8* nocapture readonly %a, i8* nocapture readonly %b, i32 %N) local_unnamed_addr {
define dso_local arm_aapcs_vfpcc zeroext i8 @one_loop_add_add_v16i8(i8* nocapture readonly %a, i8* nocapture readonly %b, i32 %N) local_unnamed_addr {

; CHECK-LABEL: one_loop_add_add_v16i8:
; CHECK-LABEL: one_loop_add_add_v16i8:

; CHECK:       @ %bb.0: @ %entry
; CHECK:       @ %bb.0: @ %entry

; CHECK-NEXT:    cmp r2, #0
; CHECK-NEXT:    cmp r2, #0

; CHECK-NEXT:    ittt eq
; CHECK-NEXT:    ittt eq

; CHECK-NEXT:    moveq r0, #0
; CHECK-NEXT:    moveq r0, #0

; CHECK-NEXT:    uxtbeq r0, r0
; CHECK-NEXT:    uxtbeq r0, r0

▲ Show 20 Lines • Show All 598 Lines • Show Last 20 Lines
llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-const.ll
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py

; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -disable-mve-tail-predication=false -mattr=+mve %s -S -o - | FileCheck %s
; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -tail-predication=enabled -mattr=+mve %s -S -o - | FileCheck %s




define dso_local void @foo(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C, i32* noalias nocapture readnone %D, i32 %N) local_unnamed_addr #0 {
define dso_local void @foo(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C, i32* noalias nocapture readnone %D, i32 %N) local_unnamed_addr #0 {

; CHECK-LABEL: @foo(
; CHECK-LABEL: @foo(

; CHECK-NEXT:  entry:
; CHECK-NEXT:  entry:

; CHECK-NEXT:    call void @llvm.set.loop.iterations.i32(i32 8001)
; CHECK-NEXT:    call void @llvm.set.loop.iterations.i32(i32 8001)

; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]

; CHECK:       vector.body:
; CHECK:       vector.body:

; CHECK-NEXT:    [[LSR_IV14:%.*]] = phi i32* [ [[SCEVGEP15:%.*]], [[VECTOR_BODY]] ], [ [[A:%.*]], [[ENTRY:%.*]] ]
; CHECK-NEXT:    [[LSR_IV14:%.*]] = phi i32* [ [[SCEVGEP15:%.*]], [[VECTOR_BODY]] ], [ [[A:%.*]], [[ENTRY:%.*]] ]

▲ Show 20 Lines • Show All 596 Lines • Show Last 20 Lines
llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-intrinsic-add-sat.ll
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py

; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs -disable-mve-tail-predication=false -o - %s | FileCheck %s
; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs -tail-predication=enabled -o - %s | FileCheck %s



define arm_aapcs_vfpcc void @uadd_sat(i16* noalias nocapture readonly %pSrcA, i16* noalias nocapture readonly %pSrcB, i16* noalias nocapture %pDst, i32 %blockSize) {
define arm_aapcs_vfpcc void @uadd_sat(i16* noalias nocapture readonly %pSrcA, i16* noalias nocapture readonly %pSrcB, i16* noalias nocapture %pDst, i32 %blockSize) {

; CHECK-LABEL: uadd_sat:
; CHECK-LABEL: uadd_sat:

; CHECK:       @ %bb.0: @ %entry
; CHECK:       @ %bb.0: @ %entry

; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    .save {r7, lr}

; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    push {r7, lr}

; CHECK-NEXT:    cmp r3, #0
; CHECK-NEXT:    cmp r3, #0

; CHECK-NEXT:    it eq
; CHECK-NEXT:    it eq

; CHECK-NEXT:    popeq {r7, pc}
; CHECK-NEXT:    popeq {r7, pc}

▲ Show 20 Lines • Show All 99 Lines • Show Last 20 Lines
llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-intrinsic-fabs.ll
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py

; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -verify-machineinstrs -disable-mve-tail-predication=false -o - %s | FileCheck %s
; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -verify-machineinstrs -tail-predication=enabled -o - %s | FileCheck %s



define arm_aapcs_vfpcc void @fabs(float* noalias nocapture readonly %pSrcA, float* noalias nocapture %pDst, i32 %blockSize) {
define arm_aapcs_vfpcc void @fabs(float* noalias nocapture readonly %pSrcA, float* noalias nocapture %pDst, i32 %blockSize) {

; CHECK-LABEL: fabs:
; CHECK-LABEL: fabs:

; CHECK:       @ %bb.0: @ %entry
; CHECK:       @ %bb.0: @ %entry

; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    .save {r7, lr}

; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    push {r7, lr}

; CHECK-NEXT:    cmp r2, #0
; CHECK-NEXT:    cmp r2, #0

; CHECK-NEXT:    it eq
; CHECK-NEXT:    it eq

; CHECK-NEXT:    popeq {r7, pc}
; CHECK-NEXT:    popeq {r7, pc}

▲ Show 20 Lines • Show All 44 Lines • Show Last 20 Lines
llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-intrinsic-round.ll
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py

; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -verify-machineinstrs -disable-mve-tail-predication=false -o - %s | FileCheck %s
; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -verify-machineinstrs -tail-predication=enabled -o - %s | FileCheck %s



define arm_aapcs_vfpcc void @round(float* noalias nocapture readonly %pSrcA, float* noalias nocapture %pDst, i32 %n) #0 {
define arm_aapcs_vfpcc void @round(float* noalias nocapture readonly %pSrcA, float* noalias nocapture %pDst, i32 %n) #0 {

; CHECK-LABEL: round:
; CHECK-LABEL: round:

; CHECK:       @ %bb.0: @ %entry
; CHECK:       @ %bb.0: @ %entry

; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    .save {r7, lr}

; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    push {r7, lr}

; CHECK-NEXT:    cmp r2, #0
; CHECK-NEXT:    cmp r2, #0

; CHECK-NEXT:    it eq
; CHECK-NEXT:    it eq

; CHECK-NEXT:    popeq {r7, pc}
; CHECK-NEXT:    popeq {r7, pc}

▲ Show 20 Lines • Show All 308 Lines • Show Last 20 Lines
llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-intrinsic-sub-sat.ll
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py

; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs -disable-mve-tail-predication=false -o - %s | FileCheck %s
; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve -verify-machineinstrs -tail-predication=enabled -o - %s | FileCheck %s



define arm_aapcs_vfpcc void @usub_sat(i16* noalias nocapture readonly %pSrcA, i16* noalias nocapture readonly %pSrcB, i16* noalias nocapture %pDst, i32 %blockSize) {
define arm_aapcs_vfpcc void @usub_sat(i16* noalias nocapture readonly %pSrcA, i16* noalias nocapture readonly %pSrcB, i16* noalias nocapture %pDst, i32 %blockSize) {

; CHECK-LABEL: usub_sat:
; CHECK-LABEL: usub_sat:

; CHECK:       @ %bb.0: @ %entry
; CHECK:       @ %bb.0: @ %entry

; CHECK-NEXT:    .save {r7, lr}
; CHECK-NEXT:    .save {r7, lr}

; CHECK-NEXT:    push {r7, lr}
; CHECK-NEXT:    push {r7, lr}

; CHECK-NEXT:    cmp r3, #0
; CHECK-NEXT:    cmp r3, #0

; CHECK-NEXT:    it eq
; CHECK-NEXT:    it eq

; CHECK-NEXT:    popeq {r7, pc}
; CHECK-NEXT:    popeq {r7, pc}

▲ Show 20 Lines • Show All 99 Lines • Show Last 20 Lines
llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-narrow.ll
; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -disable-mve-tail-predication=false -mattr=+mve,+lob %s -S -o - | FileCheck %s
; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -tail-predication=enabled -mattr=+mve,+lob %s -S -o - | FileCheck %s




; TODO: We should be able to generate a vctp for the loads.
; TODO: We should be able to generate a vctp for the loads.

; CHECK-LABEL: trunc_v4i32_v4i16
; CHECK-LABEL: trunc_v4i32_v4i16

; CHECK-NOT: vcpt
; CHECK-NOT: vcpt

define void @trunc_v4i32_v4i16(i32* readonly %a, i32* readonly %b, i16* %c, i32 %N) {
define void @trunc_v4i32_v4i16(i32* readonly %a, i32* readonly %b, i16* %c, i32 %N) {

entry:
entry:

  %cmp8 = icmp eq i32 %N, 0
  %cmp8 = icmp eq i32 %N, 0

  %tmp8 = add i32 %N, 3
  %tmp8 = add i32 %N, 3

▲ Show 20 Lines • Show All 45 Lines • Show Last 20 Lines
llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-pattern-fail.ll
; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -disable-mve-tail-predication=false -mattr=+mve,+lob %s -S -o - | FileCheck %s
; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -tail-predication=enabled -mattr=+mve,+lob %s -S -o - | FileCheck %s




; The following functions should all fail to become tail-predicated.
; The following functions should all fail to become tail-predicated.

; CHECK-NOT: call i32 @llvm.arm.vctp
; CHECK-NOT: call i32 @llvm.arm.vctp




; trip.count.minus.1 has been inserted into element 1, not 0.
; trip.count.minus.1 has been inserted into element 1, not 0.

define dso_local arm_aapcs_vfpcc void @wrong_ph_insert_0(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) {
define dso_local arm_aapcs_vfpcc void @wrong_ph_insert_0(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) {

entry:
entry:

  %cmp8 = icmp eq i32 %N, 0
  %cmp8 = icmp eq i32 %N, 0

▲ Show 20 Lines • Show All 496 Lines • Show Last 20 Lines
llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-pred-widen.ll
; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -disable-mve-tail-predication=false -mattr=+mve,+lob %s -S -o - | FileCheck %s
; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -tail-predication=enabled -mattr=+mve,+lob %s -S -o - | FileCheck %s




; CHECK-LABEL: expand_v8i16_v8i32
; CHECK-LABEL: expand_v8i16_v8i32

; CHECK-NOT: call i32 @llvm.arm.mve.vctp
; CHECK-NOT: call i32 @llvm.arm.mve.vctp

define void @expand_v8i16_v8i32(i16* noalias nocapture readonly %a, i16* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) {
define void @expand_v8i16_v8i32(i16* noalias nocapture readonly %a, i16* noalias nocapture readonly %b, i32* noalias nocapture %c, i32 %N) {

entry:
entry:

  %cmp8 = icmp eq i32 %N, 0
  %cmp8 = icmp eq i32 %N, 0

  %tmp8 = add i32 %N, 7
  %tmp8 = add i32 %N, 7

  %tmp9 = lshr i32 %tmp8, 3
  %tmp9 = lshr i32 %tmp8, 3

▲ Show 20 Lines • Show All 175 Lines • Show Last 20 Lines
llvm/test/CodeGen/Thumb2/LowOverheadLoops/tail-reduce.ll
; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -disable-mve-tail-predication=false -mattr=+mve %s -S -o - | FileCheck %s
; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -tail-predication=enabled -mattr=+mve %s -S -o - | FileCheck %s

; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -disable-mve-tail-predication=false \
; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -tail-predication=force-enabled \

; RUN:    -force-mve-tail-predication -mattr=+mve %s -S -o - | FileCheck %s --check-prefix=FORCE
; RUN:    -mattr=+mve %s -S -o - | FileCheck %s --check-prefix=FORCE




; CHECK-LABEL: reduction_i32
; CHECK-LABEL: reduction_i32

; CHECK: phi i32 [ 0, %vector.ph ]
; CHECK: phi i32 [ 0, %vector.ph ]

; CHECK: phi <8 x i16> [ zeroinitializer, %vector.ph ]
; CHECK: phi <8 x i16> [ zeroinitializer, %vector.ph ]

; CHECK: phi i32
; CHECK: phi i32

; CHECK: [[PHI:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[ELEMS:%[^ ]+]], %vector.body ]
; CHECK: [[PHI:%[^ ]+]] = phi i32 [ %N, %vector.ph ], [ [[ELEMS:%[^ ]+]], %vector.body ]

; CHECK: [[VCTP:%[^ ]+]] = call <8 x i1> @llvm.arm.mve.vctp16(i32 [[PHI]])
; CHECK: [[VCTP:%[^ ]+]] = call <8 x i1> @llvm.arm.mve.vctp16(i32 [[PHI]])

; CHECK: [[ELEMS]] = sub i32 [[PHI]], 8
; CHECK: [[ELEMS]] = sub i32 [[PHI]], 8

▲ Show 20 Lines • Show All 292 Lines • Show Last 20 Lines
llvm/test/CodeGen/Thumb2/LowOverheadLoops/varying-outer-2d-reduction.ll
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py

; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve -disable-mve-tail-predication=false %s -o - | FileCheck %s
; RUN: llc -mtriple=thumbv8.1m.main -mattr=+mve -tail-predication=enabled %s -o - | FileCheck %s




define dso_local void @varying_outer_2d_reduction(i16* nocapture readonly %Input, i16* nocapture %Output, i16 signext %Size, i16 signext %N, i16 signext %Scale) local_unnamed_addr {
define dso_local void @varying_outer_2d_reduction(i16* nocapture readonly %Input, i16* nocapture %Output, i16 signext %Size, i16 signext %N, i16 signext %Scale) local_unnamed_addr {

; CHECK-LABEL: varying_outer_2d_reduction:
; CHECK-LABEL: varying_outer_2d_reduction:

; CHECK:       @ %bb.0: @ %entry
; CHECK:       @ %bb.0: @ %entry

; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}
; CHECK-NEXT:    push.w {r4, r5, r6, r7, r8, r9, r10, r11, lr}

; CHECK-NEXT:    sub sp, #8
; CHECK-NEXT:    sub sp, #8

; CHECK-NEXT:    cmp r3, #1
; CHECK-NEXT:    cmp r3, #1

; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill
; CHECK-NEXT:    str r0, [sp, #4] @ 4-byte Spill

▲ Show 20 Lines • Show All 163 Lines • Show Last 20 Lines
llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-arith-codegen.ll
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py

; RUN: llc -mtriple=armv8.1m.main -mattr=+mve -disable-mve-tail-predication=false --verify-machineinstrs %s -o - | FileCheck %s
; RUN: llc -mtriple=armv8.1m.main -mattr=+mve -tail-predication=enabled --verify-machineinstrs %s -o - | FileCheck %s




define dso_local i32 @mul_reduce_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32 %N) {
define dso_local i32 @mul_reduce_add(i32* noalias nocapture readonly %a, i32* noalias nocapture readonly %b, i32 %N) {

; CHECK-LABEL: mul_reduce_add:
; CHECK-LABEL: mul_reduce_add:

; CHECK:       @ %bb.0: @ %entry
; CHECK:       @ %bb.0: @ %entry

; CHECK-NEXT:    cmp r2, #0
; CHECK-NEXT:    cmp r2, #0

; CHECK-NEXT:    itt eq
; CHECK-NEXT:    itt eq

; CHECK-NEXT:    moveq r0, #0
; CHECK-NEXT:    moveq r0, #0

; CHECK-NEXT:    bxeq lr
; CHECK-NEXT:    bxeq lr

▲ Show 20 Lines • Show All 421 Lines • Show Last 20 Lines
llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-reduce-mve-tail.ll



; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -disable-mve-tail-predication=false -mattr=+mve %s -S -o - | FileCheck %s
; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -tail-predication=enabled -mattr=+mve %s -S -o - | FileCheck %s




; CHECK-LABEL: vec_mul_reduce_add
; CHECK-LABEL: vec_mul_reduce_add




; CHECK: vector.ph:
; CHECK: vector.ph:

; CHECK:  call void @llvm.set.loop.iterations.i32
; CHECK:  call void @llvm.set.loop.iterations.i32

; CHECK:  br label %vector.body
; CHECK:  br label %vector.body




; CHECK: vector.body:
; CHECK: vector.body:

▲ Show 20 Lines • Show All 69 Lines • Show Last 20 Lines
llvm/test/CodeGen/Thumb2/LowOverheadLoops/vector-unroll.ll
; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -disable-mve-tail-predication=false -mattr=+mve,+lob %s -S -o - | FileCheck %s
; RUN: opt -mtriple=thumbv8.1m.main -mve-tail-predication -tail-predication=enabled -mattr=+mve,+lob %s -S -o - | FileCheck %s




; TODO: The unrolled pattern is preventing the transform
; TODO: The unrolled pattern is preventing the transform

; CHECK-LABEL: mul_v16i8_unroll
; CHECK-LABEL: mul_v16i8_unroll

; CHECK-NOT: call i32 @llvm.arm.vcpt
; CHECK-NOT: call i32 @llvm.arm.vcpt

define void @mul_v16i8_unroll(i8* noalias nocapture readonly %a, i8* noalias nocapture readonly %b, i8* noalias nocapture %c, i32 %N) {
define void @mul_v16i8_unroll(i8* noalias nocapture readonly %a, i8* noalias nocapture readonly %b, i8* noalias nocapture %c, i32 %N) {

entry:
entry:

  %cmp8 = icmp eq i32 %N, 0
  %cmp8 = icmp eq i32 %N, 0

  %tmp8 = add i32 %N, 15
  %tmp8 = add i32 %N, 15

▲ Show 20 Lines • Show All 109 Lines • Show Last 20 Lines
llvm/test/CodeGen/Thumb2/mve-fma-loops.ll
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py

; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -verify-machineinstrs -disable-mve-tail-predication=false %s -o - | FileCheck %s
; RUN: llc -mtriple=thumbv8.1m.main-none-none-eabi -mattr=+mve.fp -verify-machineinstrs -tail-predication=enabled %s -o - | FileCheck %s




define arm_aapcs_vfpcc void @fmas1(float* nocapture readonly %x, float* nocapture readonly %y, float* noalias nocapture %z, float %a, i32 %n) {
define arm_aapcs_vfpcc void @fmas1(float* nocapture readonly %x, float* nocapture readonly %y, float* noalias nocapture %z, float %a, i32 %n) {

; CHECK-LABEL: fmas1:
; CHECK-LABEL: fmas1:

; CHECK:       @ %bb.0: @ %entry
; CHECK:       @ %bb.0: @ %entry

; CHECK-NEXT:    .save {r4, lr}
; CHECK-NEXT:    .save {r4, lr}

; CHECK-NEXT:    push {r4, lr}
; CHECK-NEXT:    push {r4, lr}

; CHECK-NEXT:    cmp r3, #1
; CHECK-NEXT:    cmp r3, #1

; CHECK-NEXT:    it lt
; CHECK-NEXT:    it lt

▲ Show 20 Lines • Show All 763 Lines • Show Last 20 Lines
llvm/test/Transforms/LoopVectorize/ARM/prefer-tail-loop-folding.ll
; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf \
; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf \

; RUN:   -disable-mve-tail-predication=false -loop-vectorize -S < %s | \
; RUN:   -tail-predication=enabled -loop-vectorize -S < %s | \

; RUN:   FileCheck %s -check-prefixes=CHECK,PREFER-FOLDING
; RUN:   FileCheck %s -check-prefixes=CHECK,PREFER-FOLDING




; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=-mve \
; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=-mve \

; RUN:   -disable-mve-tail-predication=false -loop-vectorize \
; RUN:   -tail-predication=enabled -loop-vectorize \

; RUN:   -enable-arm-maskedldst=true -S < %s | \
; RUN:   -enable-arm-maskedldst=true -S < %s | \

; RUN:   FileCheck %s -check-prefixes=CHECK,NO-FOLDING
; RUN:   FileCheck %s -check-prefixes=CHECK,NO-FOLDING




; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=+mve \
; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=+mve \

; RUN:   -disable-mve-tail-predication=false -loop-vectorize \
; RUN:   -tail-predication=enabled -loop-vectorize \

; RUN:   -enable-arm-maskedldst=false -S < %s | \
; RUN:   -enable-arm-maskedldst=false -S < %s | \

; RUN:   FileCheck %s -check-prefixes=CHECK,NO-FOLDING
; RUN:   FileCheck %s -check-prefixes=CHECK,NO-FOLDING




; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=+mve \
; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=+mve \

; RUN:   -disable-mve-tail-predication=true -loop-vectorize \
; RUN:   -tail-predication=disabled -loop-vectorize \

; RUN:   -enable-arm-maskedldst=true -S < %s | \
; RUN:   -enable-arm-maskedldst=true -S < %s | \

; RUN:   FileCheck %s -check-prefixes=CHECK,NO-FOLDING
; RUN:   FileCheck %s -check-prefixes=CHECK,NO-FOLDING




; Disabling the low-overhead branch extension will make
; Disabling the low-overhead branch extension will make

; 'isHardwareLoopProfitable' return false, so that we test avoiding folding for
; 'isHardwareLoopProfitable' return false, so that we test avoiding folding for

; these cases.
; these cases.

; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=+mve,-lob \
; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=+mve,-lob \

; RUN:   -disable-mve-tail-predication=false -loop-vectorize \
; RUN:   -tail-predication=enabled -loop-vectorize \

; RUN:   -enable-arm-maskedldst=true -S < %s | \
; RUN:   -enable-arm-maskedldst=true -S < %s | \

; RUN:   FileCheck %s -check-prefixes=CHECK,NO-FOLDING
; RUN:   FileCheck %s -check-prefixes=CHECK,NO-FOLDING




; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=+mve.fp \
; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=+mve.fp \

; RUN:   -disable-mve-tail-predication=false -loop-vectorize \
; RUN:   -tail-predication=enabled -loop-vectorize \

; RUN:   -enable-arm-maskedldst=true -S < %s | \
; RUN:   -enable-arm-maskedldst=true -S < %s | \

; RUN:   FileCheck %s -check-prefixes=CHECK,PREFER-FOLDING
; RUN:   FileCheck %s -check-prefixes=CHECK,PREFER-FOLDING




; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=+mve.fp \
; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=+mve.fp \

; RUN:   -prefer-predicate-over-epilog=false \
; RUN:   -prefer-predicate-over-epilog=false \

; RUN:   -disable-mve-tail-predication=false -loop-vectorize \
; RUN:   -tail-predication=enabled -loop-vectorize \

; RUN:   -enable-arm-maskedldst=true -S < %s | \
; RUN:   -enable-arm-maskedldst=true -S < %s | \

; RUN:   FileCheck %s -check-prefixes=CHECK,NO-FOLDING
; RUN:   FileCheck %s -check-prefixes=CHECK,NO-FOLDING




; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=+mve.fp \
; RUN: opt -mtriple=thumbv8.1m.main-arm-eabihf -mattr=+mve.fp \

; RUN:   -prefer-predicate-over-epilog=true \
; RUN:   -prefer-predicate-over-epilog=true \

; RUN:   -disable-mve-tail-predication=false -loop-vectorize \
; RUN:   -tail-predication=enabled -loop-vectorize \

; RUN:   -enable-arm-maskedldst=true -S < %s | \
; RUN:   -enable-arm-maskedldst=true -S < %s | \

; RUN:   FileCheck %s -check-prefixes=CHECK,FOLDING-OPT
; RUN:   FileCheck %s -check-prefixes=CHECK,FOLDING-OPT




define void @prefer_folding(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) #0 {
define void @prefer_folding(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) #0 {

; CHECK-LABEL:    prefer_folding(
; CHECK-LABEL:    prefer_folding(

; PREFER-FOLDING: vector.body:
; PREFER-FOLDING: vector.body:

; PREFER-FOLDING: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]
; PREFER-FOLDING: %index = phi i32 [ 0, %vector.ph ], [ %index.next, %vector.body ]

; PREFER-FOLDING: %[[VIVELEM0:.*]] = add i32 %index, 0
; PREFER-FOLDING: %[[VIVELEM0:.*]] = add i32 %index, 0

▲ Show 20 Lines • Show All 609 Lines • Show Last 20 Lines
llvm/test/Transforms/LoopVectorize/ARM/tail-folding-counting-down.ll
; RUN: opt < %s -loop-vectorize -S | FileCheck %s --check-prefixes=COMMON,DEFAULT
; RUN: opt < %s -loop-vectorize -S | FileCheck %s --check-prefixes=COMMON,DEFAULT

; RUN: opt < %s -loop-vectorize -disable-mve-tail-predication=false  -prefer-predicate-over-epilog -S | FileCheck %s --check-prefixes=COMMON,CHECK-TF,CHECK-PREFER
; RUN: opt < %s -loop-vectorize -tail-predication=enabled -prefer-predicate-over-epilog -S | FileCheck %s --check-prefixes=COMMON,CHECK-TF,CHECK-PREFER

; RUN: opt < %s -loop-vectorize -disable-mve-tail-predication=false -S | FileCheck %s --check-prefixes=COMMON,CHECK-TF,CHECK-ENABLE-TP
; RUN: opt < %s -loop-vectorize -tail-predication=enabled -S | FileCheck %s --check-prefixes=COMMON,CHECK-TF,CHECK-ENABLE-TP




target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"

target triple = "thumbv8.1m.main-arm-unknown-eabihf"
target triple = "thumbv8.1m.main-arm-unknown-eabihf"




; This IR corresponds to this type of C-code:
; This IR corresponds to this type of C-code:

;
;

;  void f(char *a, char *b, char *c, int N) {
;  void f(char *a, char *b, char *c, int N) {

;    while (N-- > 0)
;    while (N-- > 0)

▲ Show 20 Lines • Show All 427 Lines • Show Last 20 Lines
llvm/test/Transforms/LoopVectorize/ARM/tail-loop-folding.ll
; RUN: opt < %s -loop-vectorize -disable-mve-tail-predication=false -S | \
; RUN: opt < %s -loop-vectorize -tail-predication=enabled -S | \

; RUN:  FileCheck %s -check-prefixes=COMMON,CHECK
; RUN:  FileCheck %s -check-prefixes=COMMON,CHECK




; RUN: opt < %s -loop-vectorize -disable-mve-tail-predication=false -prefer-predicate-over-epilog -S | \
; RUN: opt < %s -loop-vectorize -tail-predication=enabled -prefer-predicate-over-epilog -S | \

; RUN:   FileCheck -check-prefixes=COMMON,PREDFLAG %s
; RUN:   FileCheck -check-prefixes=COMMON,PREDFLAG %s




target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"

target triple = "thumbv8.1m.main-arm-unknown-eabihf"
target triple = "thumbv8.1m.main-arm-unknown-eabihf"




define dso_local void @tail_folding(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) #0 {
define dso_local void @tail_folding(i32* noalias nocapture %A, i32* noalias nocapture readonly %B, i32* noalias nocapture readonly %C) #0 {

; CHECK-LABEL: tail_folding(
; CHECK-LABEL: tail_folding(

; CHECK: vector.body:
; CHECK: vector.body:

▲ Show 20 Lines • Show All 172 Lines • Show Last 20 Lines