Index: llvm/include/llvm/Analysis/TargetTransformInfo.h =================================================================== --- llvm/include/llvm/Analysis/TargetTransformInfo.h +++ llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -519,6 +519,9 @@ bool isLSRCostLess(TargetTransformInfo::LSRCost &C1, TargetTransformInfo::LSRCost &C2) const; + /// \returns true if LSR should not optimize a chain that includes \p I. + bool isProfitableLSRChainElement(Instruction *I) const; + /// Return true if the target can fuse a compare and branch. /// Loop-strength-reduction (LSR) uses that knowledge to adjust its cost /// calculation for the instructions in a loop. @@ -1233,6 +1236,7 @@ Instruction *I) = 0; virtual bool isLSRCostLess(TargetTransformInfo::LSRCost &C1, TargetTransformInfo::LSRCost &C2) = 0; + virtual bool isProfitableLSRChainElement(Instruction *I) = 0; virtual bool canMacroFuseCmp() = 0; virtual bool canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE, LoopInfo *LI, DominatorTree *DT, AssumptionCache *AC, @@ -1542,6 +1546,9 @@ TargetTransformInfo::LSRCost &C2) override { return Impl.isLSRCostLess(C1, C2); } + bool isProfitableLSRChainElement(Instruction *I) override { + return Impl.isProfitableLSRChainElement(I); + } bool canMacroFuseCmp() override { return Impl.canMacroFuseCmp(); } bool canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE, LoopInfo *LI, DominatorTree *DT, AssumptionCache *AC, Index: llvm/include/llvm/Analysis/TargetTransformInfoImpl.h =================================================================== --- llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -166,6 +166,8 @@ C2.ScaleCost, C2.ImmCost, C2.SetupCost); } + bool isProfitableLSRChainElement(Instruction *I) { return false; } + bool canMacroFuseCmp() { return false; } bool canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE, LoopInfo *LI, Index: llvm/include/llvm/CodeGen/BasicTTIImpl.h =================================================================== --- llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -262,6 +262,10 @@ return TargetTransformInfoImplBase::isLSRCostLess(C1, C2); } + bool isProfitableLSRChainElement(Instruction *I) { + return TargetTransformInfoImplBase::isProfitableLSRChainElement(I); + } + int getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, bool HasBaseReg, int64_t Scale, unsigned AddrSpace) { TargetLoweringBase::AddrMode AM; Index: llvm/lib/Analysis/TargetTransformInfo.cpp =================================================================== --- llvm/lib/Analysis/TargetTransformInfo.cpp +++ llvm/lib/Analysis/TargetTransformInfo.cpp @@ -261,6 +261,10 @@ return TTIImpl->isLSRCostLess(C1, C2); } +bool TargetTransformInfo::isProfitableLSRChainElement(Instruction *I) const { + return TTIImpl->isProfitableLSRChainElement(I); +} + bool TargetTransformInfo::canMacroFuseCmp() const { return TTIImpl->canMacroFuseCmp(); } Index: llvm/lib/Target/ARM/ARMTargetTransformInfo.h =================================================================== --- llvm/lib/Target/ARM/ARMTargetTransformInfo.h +++ llvm/lib/Target/ARM/ARMTargetTransformInfo.h @@ -151,6 +151,8 @@ return ST->getMaxInterleaveFactor(); } + bool isProfitableLSRChainElement(Instruction *I); + bool isLegalMaskedLoad(Type *DataTy, MaybeAlign Alignment); bool isLegalMaskedStore(Type *DataTy, MaybeAlign Alignment) { Index: llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp =================================================================== --- llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp +++ llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp @@ -21,6 +21,7 @@ #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/IntrinsicsARM.h" #include "llvm/IR/PatternMatch.h" #include "llvm/IR/Type.h" #include "llvm/MC/SubtargetFeature.h" @@ -515,6 +516,23 @@ return BaseT::getAddressComputationCost(Ty, SE, Ptr); } +bool ARMTTIImpl::isProfitableLSRChainElement(Instruction *I) { + if (IntrinsicInst *II = dyn_cast(I)) { + // If a VCTP is part of a chain, it's already profitable and shouldn't be + // optimized, else LSR may block tail-predication. + switch (II->getIntrinsicID()) { + case Intrinsic::arm_mve_vctp8: + case Intrinsic::arm_mve_vctp16: + case Intrinsic::arm_mve_vctp32: + case Intrinsic::arm_mve_vctp64: + return true; + default: + break; + } + } + return false; +} + bool ARMTTIImpl::isLegalMaskedLoad(Type *DataTy, MaybeAlign Alignment) { if (!EnableMaskedLoadStores || !ST->hasMVEIntegerOps()) return false; Index: llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp =================================================================== --- llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp +++ llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp @@ -2820,9 +2820,10 @@ /// increments can be computed in fewer registers when chained. /// /// TODO: Consider IVInc free if it's already used in another chains. -static bool -isProfitableChain(IVChain &Chain, SmallPtrSetImpl &Users, - ScalarEvolution &SE) { +static bool isProfitableChain(IVChain &Chain, + SmallPtrSetImpl &Users, + ScalarEvolution &SE, + const TargetTransformInfo &TTI) { if (StressIVChain) return true; @@ -2851,7 +2852,14 @@ unsigned NumConstIncrements = 0; unsigned NumVarIncrements = 0; unsigned NumReusedIncrements = 0; + + if (TTI.isProfitableLSRChainElement(Chain.Incs[0].UserInst)) + return true; + for (const IVInc &Inc : Chain) { + if (TTI.isProfitableLSRChainElement(Inc.UserInst)) + return true; + if (Inc.IncExpr->isZero()) continue; @@ -3082,7 +3090,7 @@ for (unsigned UsersIdx = 0, NChains = IVChainVec.size(); UsersIdx < NChains; ++UsersIdx) { if (!isProfitableChain(IVChainVec[UsersIdx], - ChainUsersVec[UsersIdx].FarUsers, SE)) + ChainUsersVec[UsersIdx].FarUsers, SE, TTI)) continue; // Preserve the chain at UsesIdx. if (ChainIdx != UsersIdx) Index: llvm/test/Transforms/LoopStrengthReduce/ARM/vctp-chains.ll =================================================================== --- /dev/null +++ llvm/test/Transforms/LoopStrengthReduce/ARM/vctp-chains.ll @@ -0,0 +1,262 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -mtriple=thumbv8.1m.main -mattr=+mve %s -S -loop-reduce -o - | FileCheck %s +target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-a:0:32-n32-S64" +target triple = "thumbv8.1m-arm-none-eabi" + +define dso_local arm_aapcs_vfpcc float @vctp8(float* %0, i32 %1) local_unnamed_addr #0 { +; CHECK-LABEL: @vctp8( +; CHECK-NEXT: [[TMP3:%.*]] = tail call { <4 x i32>, i32 } @llvm.arm.mve.vidup.v4i32(i32 0, i32 8) +; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP3]], 0 +; CHECK-NEXT: [[TMP5:%.*]] = add nsw i32 [[TMP1:%.*]], -1 +; CHECK-NEXT: [[TMP6:%.*]] = ptrtoint float* [[TMP0:%.*]] to i32 +; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> undef, i32 [[TMP6]], i32 0 +; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i32> [[TMP7]], +; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> undef, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP10:%.*]] = add <4 x i32> [[TMP4]], [[TMP9]] +; CHECK-NEXT: br label [[TMP11:%.*]] +; CHECK: 11: +; CHECK-NEXT: [[TMP12:%.*]] = phi i32 [ [[TMP5]], [[TMP2:%.*]] ], [ [[TMP21:%.*]], [[TMP11]] ] +; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x float> [ zeroinitializer, [[TMP2]] ], [ [[TMP19:%.*]], [[TMP11]] ] +; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP10]], [[TMP2]] ], [ [[TMP17:%.*]], [[TMP11]] ] +; CHECK-NEXT: [[TMP15:%.*]] = tail call <16 x i1> @llvm.arm.mve.vctp8(i32 [[TMP12]]) +; CHECK-NEXT: [[MASK:%.*]] = tail call <4 x i1> @v16i1_to_v4i1(<16 x i1> [[TMP15]]) +; CHECK-NEXT: [[TMP16:%.*]] = tail call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4f32.v4i32.v4i1(<4 x i32> [[TMP14]], i32 32, <4 x i1> [[MASK]]) +; CHECK-NEXT: [[TMP17]] = extractvalue { <4 x float>, <4 x i32> } [[TMP16]], 1 +; CHECK-NEXT: [[TMP18:%.*]] = extractvalue { <4 x float>, <4 x i32> } [[TMP16]], 0 +; CHECK-NEXT: [[TMP19]] = tail call <4 x float> @llvm.arm.mve.add.predicated.v4f32.v4i1(<4 x float> [[TMP13]], <4 x float> [[TMP18]], <4 x i1> [[MASK]], <4 x float> [[TMP13]]) +; CHECK-NEXT: [[TMP20:%.*]] = icmp sgt i32 [[TMP12]], 4 +; CHECK-NEXT: [[TMP21]] = add i32 [[TMP12]], -4 +; CHECK-NEXT: br i1 [[TMP20]], label [[TMP11]], label [[TMP22:%.*]] +; CHECK: 22: +; CHECK-NEXT: [[TMP23:%.*]] = tail call arm_aapcs_vfpcc i32 bitcast (i32 (...)* @vecAddAcrossF32Mve to i32 (<4 x float>)*)(<4 x float> [[TMP19]]) #4 +; CHECK-NEXT: [[TMP24:%.*]] = sitofp i32 [[TMP23]] to float +; CHECK-NEXT: [[TMP25:%.*]] = tail call float @llvm.fabs.f32(float [[TMP24]]) +; CHECK-NEXT: ret float [[TMP25]] +; + %3 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vidup.v4i32(i32 0, i32 8) + %4 = extractvalue { <4 x i32>, i32 } %3, 0 + %5 = add nsw i32 %1, -1 + %6 = ptrtoint float* %0 to i32 + %7 = insertelement <4 x i32> undef, i32 %6, i32 0 + %8 = add <4 x i32> %7, + %9 = shufflevector <4 x i32> %8, <4 x i32> undef, <4 x i32> zeroinitializer + %10 = add <4 x i32> %4, %9 + br label %11 + +11: ; preds = %11, %2 + %12 = phi i32 [ %5, %2 ], [ %20, %11 ] + %13 = phi <4 x float> [ zeroinitializer, %2 ], [ %19, %11 ] + %14 = phi <4 x i32> [ %10, %2 ], [ %17, %11 ] + %15 = tail call <16 x i1> @llvm.arm.mve.vctp8(i32 %12) + %mask = tail call <4 x i1> @v16i1_to_v4i1(<16 x i1> %15) + %16 = tail call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4f32.v4i32.v4i1(<4 x i32> %14, i32 32, <4 x i1> %mask) + %17 = extractvalue { <4 x float>, <4 x i32> } %16, 1 + %18 = extractvalue { <4 x float>, <4 x i32> } %16, 0 + %19 = tail call <4 x float> @llvm.arm.mve.add.predicated.v4f32.v4i1(<4 x float> %13, <4 x float> %18, <4 x i1> %mask, <4 x float> %13) + %20 = add nsw i32 %12, -4 + %21 = icmp sgt i32 %12, 4 + br i1 %21, label %11, label %22 + +22: ; preds = %11 + %23 = tail call arm_aapcs_vfpcc i32 bitcast (i32 (...)* @vecAddAcrossF32Mve to i32 (<4 x float>)*)(<4 x float> %19) #0 + %24 = sitofp i32 %23 to float + %25 = tail call float @llvm.fabs.f32(float %24) + ret float %25 +} + +define dso_local arm_aapcs_vfpcc float @vctp16(float* %0, i32 %1) local_unnamed_addr #0 { +; CHECK-LABEL: @vctp16( +; CHECK-NEXT: [[TMP3:%.*]] = tail call { <4 x i32>, i32 } @llvm.arm.mve.vidup.v4i32(i32 0, i32 8) +; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP3]], 0 +; CHECK-NEXT: [[TMP5:%.*]] = add nsw i32 [[TMP1:%.*]], -1 +; CHECK-NEXT: [[TMP6:%.*]] = ptrtoint float* [[TMP0:%.*]] to i32 +; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> undef, i32 [[TMP6]], i32 0 +; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i32> [[TMP7]], +; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> undef, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP10:%.*]] = add <4 x i32> [[TMP4]], [[TMP9]] +; CHECK-NEXT: br label [[TMP11:%.*]] +; CHECK: 11: +; CHECK-NEXT: [[TMP12:%.*]] = phi i32 [ [[TMP5]], [[TMP2:%.*]] ], [ [[TMP21:%.*]], [[TMP11]] ] +; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x float> [ zeroinitializer, [[TMP2]] ], [ [[TMP19:%.*]], [[TMP11]] ] +; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP10]], [[TMP2]] ], [ [[TMP17:%.*]], [[TMP11]] ] +; CHECK-NEXT: [[TMP15:%.*]] = tail call <8 x i1> @llvm.arm.mve.vctp16(i32 [[TMP12]]) +; CHECK-NEXT: [[MASK:%.*]] = tail call <4 x i1> @v8i1_to_v4i1(<8 x i1> [[TMP15]]) +; CHECK-NEXT: [[TMP16:%.*]] = tail call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4f32.v4i32.v4i1(<4 x i32> [[TMP14]], i32 32, <4 x i1> [[MASK]]) +; CHECK-NEXT: [[TMP17]] = extractvalue { <4 x float>, <4 x i32> } [[TMP16]], 1 +; CHECK-NEXT: [[TMP18:%.*]] = extractvalue { <4 x float>, <4 x i32> } [[TMP16]], 0 +; CHECK-NEXT: [[TMP19]] = tail call <4 x float> @llvm.arm.mve.add.predicated.v4f32.v4i1(<4 x float> [[TMP13]], <4 x float> [[TMP18]], <4 x i1> [[MASK]], <4 x float> [[TMP13]]) +; CHECK-NEXT: [[TMP20:%.*]] = icmp sgt i32 [[TMP12]], 4 +; CHECK-NEXT: [[TMP21]] = add i32 [[TMP12]], -4 +; CHECK-NEXT: br i1 [[TMP20]], label [[TMP11]], label [[TMP22:%.*]] +; CHECK: 22: +; CHECK-NEXT: [[TMP23:%.*]] = tail call arm_aapcs_vfpcc i32 bitcast (i32 (...)* @vecAddAcrossF32Mve to i32 (<4 x float>)*)(<4 x float> [[TMP19]]) #4 +; CHECK-NEXT: [[TMP24:%.*]] = sitofp i32 [[TMP23]] to float +; CHECK-NEXT: [[TMP25:%.*]] = tail call float @llvm.fabs.f32(float [[TMP24]]) +; CHECK-NEXT: ret float [[TMP25]] +; + %3 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vidup.v4i32(i32 0, i32 8) + %4 = extractvalue { <4 x i32>, i32 } %3, 0 + %5 = add nsw i32 %1, -1 + %6 = ptrtoint float* %0 to i32 + %7 = insertelement <4 x i32> undef, i32 %6, i32 0 + %8 = add <4 x i32> %7, + %9 = shufflevector <4 x i32> %8, <4 x i32> undef, <4 x i32> zeroinitializer + %10 = add <4 x i32> %4, %9 + br label %11 + +11: ; preds = %11, %2 + %12 = phi i32 [ %5, %2 ], [ %20, %11 ] + %13 = phi <4 x float> [ zeroinitializer, %2 ], [ %19, %11 ] + %14 = phi <4 x i32> [ %10, %2 ], [ %17, %11 ] + %15 = tail call <8 x i1> @llvm.arm.mve.vctp16(i32 %12) + %mask = tail call <4 x i1> @v8i1_to_v4i1(<8 x i1> %15) + %16 = tail call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4f32.v4i32.v4i1(<4 x i32> %14, i32 32, <4 x i1> %mask) + %17 = extractvalue { <4 x float>, <4 x i32> } %16, 1 + %18 = extractvalue { <4 x float>, <4 x i32> } %16, 0 + %19 = tail call <4 x float> @llvm.arm.mve.add.predicated.v4f32.v4i1(<4 x float> %13, <4 x float> %18, <4 x i1> %mask, <4 x float> %13) + %20 = add nsw i32 %12, -4 + %21 = icmp sgt i32 %12, 4 + br i1 %21, label %11, label %22 + +22: ; preds = %11 + %23 = tail call arm_aapcs_vfpcc i32 bitcast (i32 (...)* @vecAddAcrossF32Mve to i32 (<4 x float>)*)(<4 x float> %19) #0 + %24 = sitofp i32 %23 to float + %25 = tail call float @llvm.fabs.f32(float %24) + ret float %25 +} + +define dso_local arm_aapcs_vfpcc float @vctpi32(float* %0, i32 %1) local_unnamed_addr #0 { +; CHECK-LABEL: @vctpi32( +; CHECK-NEXT: [[TMP3:%.*]] = tail call { <4 x i32>, i32 } @llvm.arm.mve.vidup.v4i32(i32 0, i32 8) +; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP3]], 0 +; CHECK-NEXT: [[TMP5:%.*]] = add nsw i32 [[TMP1:%.*]], -1 +; CHECK-NEXT: [[TMP6:%.*]] = ptrtoint float* [[TMP0:%.*]] to i32 +; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> undef, i32 [[TMP6]], i32 0 +; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i32> [[TMP7]], +; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> undef, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP10:%.*]] = add <4 x i32> [[TMP4]], [[TMP9]] +; CHECK-NEXT: br label [[TMP11:%.*]] +; CHECK: 11: +; CHECK-NEXT: [[TMP12:%.*]] = phi i32 [ [[TMP5]], [[TMP2:%.*]] ], [ [[TMP21:%.*]], [[TMP11]] ] +; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x float> [ zeroinitializer, [[TMP2]] ], [ [[TMP19:%.*]], [[TMP11]] ] +; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP10]], [[TMP2]] ], [ [[TMP17:%.*]], [[TMP11]] ] +; CHECK-NEXT: [[TMP15:%.*]] = tail call <4 x i1> @llvm.arm.mve.vctp32(i32 [[TMP12]]) +; CHECK-NEXT: [[TMP16:%.*]] = tail call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4f32.v4i32.v4i1(<4 x i32> [[TMP14]], i32 32, <4 x i1> [[TMP15]]) +; CHECK-NEXT: [[TMP17]] = extractvalue { <4 x float>, <4 x i32> } [[TMP16]], 1 +; CHECK-NEXT: [[TMP18:%.*]] = extractvalue { <4 x float>, <4 x i32> } [[TMP16]], 0 +; CHECK-NEXT: [[TMP19]] = tail call <4 x float> @llvm.arm.mve.add.predicated.v4f32.v4i1(<4 x float> [[TMP13]], <4 x float> [[TMP18]], <4 x i1> [[TMP15]], <4 x float> [[TMP13]]) +; CHECK-NEXT: [[TMP20:%.*]] = icmp sgt i32 [[TMP12]], 4 +; CHECK-NEXT: [[TMP21]] = add i32 [[TMP12]], -4 +; CHECK-NEXT: br i1 [[TMP20]], label [[TMP11]], label [[TMP22:%.*]] +; CHECK: 22: +; CHECK-NEXT: [[TMP23:%.*]] = tail call arm_aapcs_vfpcc i32 bitcast (i32 (...)* @vecAddAcrossF32Mve to i32 (<4 x float>)*)(<4 x float> [[TMP19]]) +; CHECK-NEXT: [[TMP24:%.*]] = sitofp i32 [[TMP23]] to float +; CHECK-NEXT: [[TMP25:%.*]] = tail call float @llvm.fabs.f32(float [[TMP24]]) +; CHECK-NEXT: ret float [[TMP25]] +; + %3 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vidup.v4i32(i32 0, i32 8) + %4 = extractvalue { <4 x i32>, i32 } %3, 0 + %5 = add nsw i32 %1, -1 + %6 = ptrtoint float* %0 to i32 + %7 = insertelement <4 x i32> undef, i32 %6, i32 0 + %8 = add <4 x i32> %7, + %9 = shufflevector <4 x i32> %8, <4 x i32> undef, <4 x i32> zeroinitializer + %10 = add <4 x i32> %4, %9 + br label %11 + +11: ; preds = %11, %2 + %12 = phi i32 [ %5, %2 ], [ %20, %11 ] + %13 = phi <4 x float> [ zeroinitializer, %2 ], [ %19, %11 ] + %14 = phi <4 x i32> [ %10, %2 ], [ %17, %11 ] + %15 = tail call <4 x i1> @llvm.arm.mve.vctp32(i32 %12) + %16 = tail call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4f32.v4i32.v4i1(<4 x i32> %14, i32 32, <4 x i1> %15) + %17 = extractvalue { <4 x float>, <4 x i32> } %16, 1 + %18 = extractvalue { <4 x float>, <4 x i32> } %16, 0 + %19 = tail call <4 x float> @llvm.arm.mve.add.predicated.v4f32.v4i1(<4 x float> %13, <4 x float> %18, <4 x i1> %15, <4 x float> %13) + %20 = add nsw i32 %12, -4 + %21 = icmp sgt i32 %12, 4 + br i1 %21, label %11, label %22 + +22: ; preds = %11 + %23 = tail call arm_aapcs_vfpcc i32 bitcast (i32 (...)* @vecAddAcrossF32Mve to i32 (<4 x float>)*)(<4 x float> %19) #5 + %24 = sitofp i32 %23 to float + %25 = tail call float @llvm.fabs.f32(float %24) + ret float %25 +} + + +define dso_local arm_aapcs_vfpcc float @vctpi64(float* %0, i32 %1) local_unnamed_addr #0 { +; CHECK-LABEL: @vctpi64( +; CHECK-NEXT: [[TMP3:%.*]] = tail call { <4 x i32>, i32 } @llvm.arm.mve.vidup.v4i32(i32 0, i32 8) +; CHECK-NEXT: [[TMP4:%.*]] = extractvalue { <4 x i32>, i32 } [[TMP3]], 0 +; CHECK-NEXT: [[TMP5:%.*]] = add nsw i32 [[TMP1:%.*]], -1 +; CHECK-NEXT: [[TMP6:%.*]] = ptrtoint float* [[TMP0:%.*]] to i32 +; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> undef, i32 [[TMP6]], i32 0 +; CHECK-NEXT: [[TMP8:%.*]] = add <4 x i32> [[TMP7]], +; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> undef, <4 x i32> zeroinitializer +; CHECK-NEXT: [[TMP10:%.*]] = add <4 x i32> [[TMP4]], [[TMP9]] +; CHECK-NEXT: br label [[TMP11:%.*]] +; CHECK: 11: +; CHECK-NEXT: [[TMP12:%.*]] = phi i32 [ [[TMP5]], [[TMP2:%.*]] ], [ [[TMP21:%.*]], [[TMP11]] ] +; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x float> [ zeroinitializer, [[TMP2]] ], [ [[TMP19:%.*]], [[TMP11]] ] +; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP10]], [[TMP2]] ], [ [[TMP17:%.*]], [[TMP11]] ] +; CHECK-NEXT: [[TMP15:%.*]] = tail call <4 x i1> @llvm.arm.mve.vctp64(i32 [[TMP12]]) +; CHECK-NEXT: [[TMP16:%.*]] = tail call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4f32.v4i32.v4i1(<4 x i32> [[TMP14]], i32 32, <4 x i1> [[TMP15]]) +; CHECK-NEXT: [[TMP17]] = extractvalue { <4 x float>, <4 x i32> } [[TMP16]], 1 +; CHECK-NEXT: [[TMP18:%.*]] = extractvalue { <4 x float>, <4 x i32> } [[TMP16]], 0 +; CHECK-NEXT: [[TMP19]] = tail call <4 x float> @llvm.arm.mve.add.predicated.v4f32.v4i1(<4 x float> [[TMP13]], <4 x float> [[TMP18]], <4 x i1> [[TMP15]], <4 x float> [[TMP13]]) +; CHECK-NEXT: [[TMP20:%.*]] = icmp sgt i32 [[TMP12]], 4 +; CHECK-NEXT: [[TMP21]] = add i32 [[TMP12]], -4 +; CHECK-NEXT: br i1 [[TMP20]], label [[TMP11]], label [[TMP22:%.*]] +; CHECK: 22: +; CHECK-NEXT: [[TMP23:%.*]] = tail call arm_aapcs_vfpcc i32 bitcast (i32 (...)* @vecAddAcrossF32Mve to i32 (<4 x float>)*)(<4 x float> [[TMP19]]) #4 +; CHECK-NEXT: [[TMP24:%.*]] = sitofp i32 [[TMP23]] to float +; CHECK-NEXT: [[TMP25:%.*]] = tail call float @llvm.fabs.f32(float [[TMP24]]) +; CHECK-NEXT: ret float [[TMP25]] +; + %3 = tail call { <4 x i32>, i32 } @llvm.arm.mve.vidup.v4i32(i32 0, i32 8) + %4 = extractvalue { <4 x i32>, i32 } %3, 0 + %5 = add nsw i32 %1, -1 + %6 = ptrtoint float* %0 to i32 + %7 = insertelement <4 x i32> undef, i32 %6, i32 0 + %8 = add <4 x i32> %7, + %9 = shufflevector <4 x i32> %8, <4 x i32> undef, <4 x i32> zeroinitializer + %10 = add <4 x i32> %4, %9 + br label %11 + +11: ; preds = %11, %2 + %12 = phi i32 [ %5, %2 ], [ %20, %11 ] + %13 = phi <4 x float> [ zeroinitializer, %2 ], [ %19, %11 ] + %14 = phi <4 x i32> [ %10, %2 ], [ %17, %11 ] + %15 = tail call <4 x i1> @llvm.arm.mve.vctp64(i32 %12) + %16 = tail call { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4f32.v4i32.v4i1(<4 x i32> %14, i32 32, <4 x i1> %15) + %17 = extractvalue { <4 x float>, <4 x i32> } %16, 1 + %18 = extractvalue { <4 x float>, <4 x i32> } %16, 0 + %19 = tail call <4 x float> @llvm.arm.mve.add.predicated.v4f32.v4i1(<4 x float> %13, <4 x float> %18, <4 x i1> %15, <4 x float> %13) + %20 = add nsw i32 %12, -4 + %21 = icmp sgt i32 %12, 4 + br i1 %21, label %11, label %22 + +22: ; preds = %11 + %23 = tail call arm_aapcs_vfpcc i32 bitcast (i32 (...)* @vecAddAcrossF32Mve to i32 (<4 x float>)*)(<4 x float> %19) #0 + %24 = sitofp i32 %23 to float + %25 = tail call float @llvm.fabs.f32(float %24) + ret float %25 +} + +declare { <4 x i32>, i32 } @llvm.arm.mve.vidup.v4i32(i32, i32) #1 +declare <16 x i1> @llvm.arm.mve.vctp8(i32) #1 +declare <8 x i1> @llvm.arm.mve.vctp16(i32) #1 +declare <4 x i1> @llvm.arm.mve.vctp32(i32) #1 +declare <4 x i1> @llvm.arm.mve.vctp64(i32) #1 +declare { <4 x float>, <4 x i32> } @llvm.arm.mve.vldr.gather.base.wb.predicated.v4f32.v4i32.v4i1(<4 x i32>, i32, <4 x i1>) #2 +declare <4 x float> @llvm.arm.mve.add.predicated.v4f32.v4i1(<4 x float>, <4 x float>, <4 x i1>, <4 x float>) #1 +declare dso_local arm_aapcs_vfpcc i32 @vecAddAcrossF32Mve(...) local_unnamed_addr #0 +declare dso_local arm_aapcs_vfpcc <4 x i1> @v8i1_to_v4i1(<8 x i1>) local_unnamed_addr #0 +declare dso_local arm_aapcs_vfpcc <4 x i1> @v16i1_to_v4i1(<16 x i1>) local_unnamed_addr #0 +declare float @llvm.fabs.f32(float) #4 + +attributes #0 = { nounwind } +attributes #1 = { nounwind readnone } +attributes #2 = { nounwind readonly } +attributes #4 = { nounwind readnone speculatable willreturn }