diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp @@ -35,37 +35,44 @@ namespace { static cl::opt WidenLoads( - "amdgpu-codegenprepare-widen-constant-loads", - cl::desc("Widen sub-dword constant address space loads in AMDGPUCodeGenPrepare"), - cl::ReallyHidden, - cl::init(false)); + "amdgpu-codegenprepare-widen-constant-loads", + cl::desc( + "Widen sub-dword constant address space loads in AMDGPUCodeGenPrepare"), + cl::ReallyHidden, cl::init(false)); static cl::opt Widen16BitOps( - "amdgpu-codegenprepare-widen-16-bit-ops", - cl::desc("Widen uniform 16-bit instructions to 32-bit in AMDGPUCodeGenPrepare"), - cl::ReallyHidden, - cl::init(true)); + "amdgpu-codegenprepare-widen-16-bit-ops", + cl::desc( + "Widen uniform 16-bit instructions to 32-bit in AMDGPUCodeGenPrepare"), + cl::ReallyHidden, cl::init(true)); + +static cl::opt + ScalarizeLargePHIs("amdgpu-codegenprepare-scalarize-large-phis", + cl::desc("Scalarize large PHI nodes for DAGISel"), + cl::ReallyHidden, cl::init(true)); + +static cl::opt ScalarizeLargePHIsThreshold( + "amdgpu-codegenprepare-scalarize-large-phis-threshold", + cl::desc("Minimum type size in bits for large PHI node scalarization."), + cl::ReallyHidden, cl::init(256)); static cl::opt UseMul24Intrin( - "amdgpu-codegenprepare-mul24", - cl::desc("Introduce mul24 intrinsics in AMDGPUCodeGenPrepare"), - cl::ReallyHidden, - cl::init(true)); + "amdgpu-codegenprepare-mul24", + cl::desc("Introduce mul24 intrinsics in AMDGPUCodeGenPrepare"), + cl::ReallyHidden, cl::init(true)); // Legalize 64-bit division by using the generic IR expansion. -static cl::opt ExpandDiv64InIR( - "amdgpu-codegenprepare-expand-div64", - cl::desc("Expand 64-bit division in AMDGPUCodeGenPrepare"), - cl::ReallyHidden, - cl::init(false)); +static cl::opt + ExpandDiv64InIR("amdgpu-codegenprepare-expand-div64", + cl::desc("Expand 64-bit division in AMDGPUCodeGenPrepare"), + cl::ReallyHidden, cl::init(false)); // Leave all division operations as they are. This supersedes ExpandDiv64InIR // and is used for testing the legalizer. static cl::opt DisableIDivExpand( - "amdgpu-codegenprepare-disable-idiv-expansion", - cl::desc("Prevent expanding integer division in AMDGPUCodeGenPrepare"), - cl::ReallyHidden, - cl::init(false)); + "amdgpu-codegenprepare-disable-idiv-expansion", + cl::desc("Prevent expanding integer division in AMDGPUCodeGenPrepare"), + cl::ReallyHidden, cl::init(false)); class AMDGPUCodeGenPrepare : public FunctionPass, public InstVisitor { @@ -212,6 +219,7 @@ bool visitLoadInst(LoadInst &I); bool visitICmpInst(ICmpInst &I); bool visitSelectInst(SelectInst &I); + bool visitPHINode(PHINode &I); bool visitIntrinsicInst(IntrinsicInst &I); bool visitBitreverseIntrinsicInst(IntrinsicInst &I); @@ -228,7 +236,7 @@ // FIXME: Division expansion needs to preserve the dominator tree. if (!ExpandDiv64InIR) AU.setPreservesAll(); - } + } }; } // end anonymous namespace @@ -687,15 +695,15 @@ return Builder.CreateCall(Decl, { Den }); } - // Same as for 1.0, but expand the sign out of the constant. + // Same as for 1.0, but expand the sign out of the constant. if (CLHS->isExactlyValue(-1.0)) { Function *Decl = Intrinsic::getDeclaration( Mod, Intrinsic::amdgcn_rcp, Ty); - // -1.0 / x -> rcp (fneg x) - Value *FNeg = Builder.CreateFNeg(Den); - return Builder.CreateCall(Decl, { FNeg }); - } + // -1.0 / x -> rcp (fneg x) + Value *FNeg = Builder.CreateFNeg(Den); + return Builder.CreateCall(Decl, {FNeg}); + } } } @@ -782,8 +790,9 @@ // rcp_f16 is accurate for !fpmath >= 1.0ulp. // rcp_f32 is accurate for !fpmath >= 1.0ulp and denormals are flushed. // rcp_f64 is never accurate. - const bool RcpIsAccurate = (Ty->isHalfTy() && ReqdAccuracy >= 1.0f) || - (Ty->isFloatTy() && !HasFP32Denormals && ReqdAccuracy >= 1.0f); + const bool RcpIsAccurate = + (Ty->isHalfTy() && ReqdAccuracy >= 1.0f) || + (Ty->isFloatTy() && !HasFP32Denormals && ReqdAccuracy >= 1.0f); IRBuilder<> Builder(FDiv.getParent(), std::next(FDiv.getIterator())); Builder.setFastMathFlags(FMF); @@ -1340,10 +1349,10 @@ WidenLoad->setMetadata(LLVMContext::MD_range, nullptr); } else { Metadata *LowAndHigh[] = { - ConstantAsMetadata::get(ConstantInt::get(I32Ty, Lower->getValue().zext(32))), - // Don't make assumptions about the high bits. - ConstantAsMetadata::get(ConstantInt::get(I32Ty, 0)) - }; + ConstantAsMetadata::get( + ConstantInt::get(I32Ty, Lower->getValue().zext(32))), + // Don't make assumptions about the high bits. + ConstantAsMetadata::get(ConstantInt::get(I32Ty, 0))}; WidenLoad->setMetadata(LLVMContext::MD_range, MDNode::get(Mod->getContext(), LowAndHigh)); @@ -1382,6 +1391,59 @@ return Changed; } +bool AMDGPUCodeGenPrepare::visitPHINode(PHINode &I) { + // Scalarize large, non power-of-two sized PHIs: Create one PHI node per + // vector element. + // + // This is only helpful for DAGISel because it doesn't handle large PHIs as + // well as GlobalISel. DAGISel lowers PHIs by using CopyToReg/CopyFromReg. + // With large, odd-sized PHIs we may end up needing many `build_vector` + // operations with most elements being "undef". This inhibits a lot of + // optimization opportunities and can result in unreasonably high register + // pressure and the inevitable stack spilling. + if (!ScalarizeLargePHIs || getCGPassBuilderOption().EnableGlobalISelOption) + return false; + + FixedVectorType *FVT = dyn_cast(I.getType()); + if (!FVT) + return false; + + const unsigned NumElts = FVT->getNumElements(); + Type *ScalarTy = FVT->getElementType(); + if (isPowerOf2_32(NumElts) || + DL->getTypeSizeInBits(FVT) < ScalarizeLargePHIsThreshold) + return false; + + // Scalarize this PHI by breaking it up using insert/extract vector elts. + SmallVector, 2> BrokenIncomingValues; + for (unsigned Idx = 0; Idx < I.getNumIncomingValues(); ++Idx) { + Value *Inc = I.getIncomingValue(Idx); + + auto &Results = BrokenIncomingValues.emplace_back(); + IRBuilder<> B(I.getIncomingBlock(Idx)->getTerminator()); + for (unsigned K = 0; K < NumElts; ++K) + Results.push_back(B.CreateExtractElement(Inc, K)); + } + + // Now create one PHI per vector element + IRBuilder<> B(I.getParent()->getFirstNonPHI()); + std::vector NewPHIs; + for (unsigned Elt = 0; Elt < NumElts; ++Elt) { + PHINode *NewPHI = B.CreatePHI(ScalarTy, I.getNumIncomingValues()); + for (auto &[Idx, BB] : enumerate(I.blocks())) + NewPHI->addIncoming(BrokenIncomingValues[Idx][Elt], BB); + NewPHIs.push_back(NewPHI); + } + + // And replace this PHI with a vector of all the previous PHI values. + Value *Vec = PoisonValue::get(FVT); + for (unsigned K = 0; K < NumElts; ++K) + Vec = B.CreateInsertElement(Vec, NewPHIs[K], K); + I.replaceAllUsesWith(Vec); + I.eraseFromParent(); + return true; +} + bool AMDGPUCodeGenPrepare::visitIntrinsicInst(IntrinsicInst &I) { switch (I.getIntrinsicID()) { case Intrinsic::bitreverse: diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-scalarize-large-phis.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-scalarize-large-phis.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-scalarize-large-phis.ll @@ -0,0 +1,612 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -S -mtriple=amdgcn-- -amdgpu-codegenprepare %s | FileCheck %s --check-prefixes=SCALARIZED +; RUN: opt -S -mtriple=amdgcn-- -amdgpu-codegenprepare -global-isel %s | FileCheck %s --check-prefixes=NOSCALARIZE +; RUN: opt -S -mtriple=amdgcn-- -amdgpu-codegenprepare -amdgpu-codegenprepare-scalarize-large-phis=0 %s | FileCheck %s --check-prefixes=NOSCALARIZE + +define amdgpu_kernel void @phi_v5f64(<5 x double> %in, ptr %out, i1 %cond) { +; SCALARIZED-LABEL: @phi_v5f64( +; SCALARIZED-NEXT: entry: +; SCALARIZED-NEXT: br i1 [[COND:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]] +; SCALARIZED: then: +; SCALARIZED-NEXT: [[X:%.*]] = insertelement <5 x double> [[IN:%.*]], double 3.140000e+00, i32 3 +; SCALARIZED-NEXT: [[TMP0:%.*]] = extractelement <5 x double> [[X]], i64 0 +; SCALARIZED-NEXT: [[TMP1:%.*]] = extractelement <5 x double> [[X]], i64 1 +; SCALARIZED-NEXT: [[TMP2:%.*]] = extractelement <5 x double> [[X]], i64 2 +; SCALARIZED-NEXT: [[TMP3:%.*]] = extractelement <5 x double> [[X]], i64 3 +; SCALARIZED-NEXT: [[TMP4:%.*]] = extractelement <5 x double> [[X]], i64 4 +; SCALARIZED-NEXT: br label [[FINALLY:%.*]] +; SCALARIZED: else: +; SCALARIZED-NEXT: [[Y:%.*]] = insertelement <5 x double> [[IN]], double 9.140000e+00, i32 2 +; SCALARIZED-NEXT: [[TMP5:%.*]] = extractelement <5 x double> [[Y]], i64 0 +; SCALARIZED-NEXT: [[TMP6:%.*]] = extractelement <5 x double> [[Y]], i64 1 +; SCALARIZED-NEXT: [[TMP7:%.*]] = extractelement <5 x double> [[Y]], i64 2 +; SCALARIZED-NEXT: [[TMP8:%.*]] = extractelement <5 x double> [[Y]], i64 3 +; SCALARIZED-NEXT: [[TMP9:%.*]] = extractelement <5 x double> [[Y]], i64 4 +; SCALARIZED-NEXT: br label [[FINALLY]] +; SCALARIZED: finally: +; SCALARIZED-NEXT: [[TMP10:%.*]] = phi double [ [[TMP0]], [[THEN]] ], [ [[TMP5]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP11:%.*]] = phi double [ [[TMP1]], [[THEN]] ], [ [[TMP6]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP12:%.*]] = phi double [ [[TMP2]], [[THEN]] ], [ [[TMP7]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP13:%.*]] = phi double [ [[TMP3]], [[THEN]] ], [ [[TMP8]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP14:%.*]] = phi double [ [[TMP4]], [[THEN]] ], [ [[TMP9]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP15:%.*]] = insertelement <5 x double> poison, double [[TMP10]], i64 0 +; SCALARIZED-NEXT: [[TMP16:%.*]] = insertelement <5 x double> [[TMP15]], double [[TMP11]], i64 1 +; SCALARIZED-NEXT: [[TMP17:%.*]] = insertelement <5 x double> [[TMP16]], double [[TMP12]], i64 2 +; SCALARIZED-NEXT: [[TMP18:%.*]] = insertelement <5 x double> [[TMP17]], double [[TMP13]], i64 3 +; SCALARIZED-NEXT: [[TMP19:%.*]] = insertelement <5 x double> [[TMP18]], double [[TMP14]], i64 4 +; SCALARIZED-NEXT: store <5 x double> [[TMP19]], ptr [[OUT:%.*]], align 1 +; SCALARIZED-NEXT: ret void +; +; NOSCALARIZE-LABEL: @phi_v5f64( +; NOSCALARIZE-NEXT: entry: +; NOSCALARIZE-NEXT: br i1 [[COND:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]] +; NOSCALARIZE: then: +; NOSCALARIZE-NEXT: [[X:%.*]] = insertelement <5 x double> [[IN:%.*]], double 3.140000e+00, i32 3 +; NOSCALARIZE-NEXT: br label [[FINALLY:%.*]] +; NOSCALARIZE: else: +; NOSCALARIZE-NEXT: [[Y:%.*]] = insertelement <5 x double> [[IN]], double 9.140000e+00, i32 2 +; NOSCALARIZE-NEXT: br label [[FINALLY]] +; NOSCALARIZE: finally: +; NOSCALARIZE-NEXT: [[VAL:%.*]] = phi <5 x double> [ [[X]], [[THEN]] ], [ [[Y]], [[ELSE]] ] +; NOSCALARIZE-NEXT: store <5 x double> [[VAL]], ptr [[OUT:%.*]], align 1 +; NOSCALARIZE-NEXT: ret void +; +entry: + br i1 %cond, label %then, label %else +then: + %x = insertelement <5 x double> %in, double 3.14, i32 3 + br label %finally +else: + %y = insertelement <5 x double> %in, double 9.14, i32 2 + br label %finally +finally: + %val = phi <5 x double> [%x, %then], [%y, %else] + store <5 x double> %val, ptr %out, align 1 + ret void +} + +define amdgpu_kernel void @phi_v7f64(<7 x double> %in, ptr %out, i1 %cond) { +; SCALARIZED-LABEL: @phi_v7f64( +; SCALARIZED-NEXT: entry: +; SCALARIZED-NEXT: br i1 [[COND:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]] +; SCALARIZED: then: +; SCALARIZED-NEXT: [[X:%.*]] = insertelement <7 x double> [[IN:%.*]], double 3.140000e+00, i32 3 +; SCALARIZED-NEXT: [[TMP0:%.*]] = extractelement <7 x double> [[X]], i64 0 +; SCALARIZED-NEXT: [[TMP1:%.*]] = extractelement <7 x double> [[X]], i64 1 +; SCALARIZED-NEXT: [[TMP2:%.*]] = extractelement <7 x double> [[X]], i64 2 +; SCALARIZED-NEXT: [[TMP3:%.*]] = extractelement <7 x double> [[X]], i64 3 +; SCALARIZED-NEXT: [[TMP4:%.*]] = extractelement <7 x double> [[X]], i64 4 +; SCALARIZED-NEXT: [[TMP5:%.*]] = extractelement <7 x double> [[X]], i64 5 +; SCALARIZED-NEXT: [[TMP6:%.*]] = extractelement <7 x double> [[X]], i64 6 +; SCALARIZED-NEXT: br label [[FINALLY:%.*]] +; SCALARIZED: else: +; SCALARIZED-NEXT: [[Y:%.*]] = insertelement <7 x double> [[IN]], double 9.140000e+00, i32 6 +; SCALARIZED-NEXT: [[TMP7:%.*]] = extractelement <7 x double> [[Y]], i64 0 +; SCALARIZED-NEXT: [[TMP8:%.*]] = extractelement <7 x double> [[Y]], i64 1 +; SCALARIZED-NEXT: [[TMP9:%.*]] = extractelement <7 x double> [[Y]], i64 2 +; SCALARIZED-NEXT: [[TMP10:%.*]] = extractelement <7 x double> [[Y]], i64 3 +; SCALARIZED-NEXT: [[TMP11:%.*]] = extractelement <7 x double> [[Y]], i64 4 +; SCALARIZED-NEXT: [[TMP12:%.*]] = extractelement <7 x double> [[Y]], i64 5 +; SCALARIZED-NEXT: [[TMP13:%.*]] = extractelement <7 x double> [[Y]], i64 6 +; SCALARIZED-NEXT: br label [[FINALLY]] +; SCALARIZED: finally: +; SCALARIZED-NEXT: [[TMP14:%.*]] = phi double [ [[TMP0]], [[THEN]] ], [ [[TMP7]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP15:%.*]] = phi double [ [[TMP1]], [[THEN]] ], [ [[TMP8]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP16:%.*]] = phi double [ [[TMP2]], [[THEN]] ], [ [[TMP9]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP17:%.*]] = phi double [ [[TMP3]], [[THEN]] ], [ [[TMP10]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP18:%.*]] = phi double [ [[TMP4]], [[THEN]] ], [ [[TMP11]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP19:%.*]] = phi double [ [[TMP5]], [[THEN]] ], [ [[TMP12]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP20:%.*]] = phi double [ [[TMP6]], [[THEN]] ], [ [[TMP13]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP21:%.*]] = insertelement <7 x double> poison, double [[TMP14]], i64 0 +; SCALARIZED-NEXT: [[TMP22:%.*]] = insertelement <7 x double> [[TMP21]], double [[TMP15]], i64 1 +; SCALARIZED-NEXT: [[TMP23:%.*]] = insertelement <7 x double> [[TMP22]], double [[TMP16]], i64 2 +; SCALARIZED-NEXT: [[TMP24:%.*]] = insertelement <7 x double> [[TMP23]], double [[TMP17]], i64 3 +; SCALARIZED-NEXT: [[TMP25:%.*]] = insertelement <7 x double> [[TMP24]], double [[TMP18]], i64 4 +; SCALARIZED-NEXT: [[TMP26:%.*]] = insertelement <7 x double> [[TMP25]], double [[TMP19]], i64 5 +; SCALARIZED-NEXT: [[TMP27:%.*]] = insertelement <7 x double> [[TMP26]], double [[TMP20]], i64 6 +; SCALARIZED-NEXT: store <7 x double> [[TMP27]], ptr [[OUT:%.*]], align 1 +; SCALARIZED-NEXT: ret void +; +; NOSCALARIZE-LABEL: @phi_v7f64( +; NOSCALARIZE-NEXT: entry: +; NOSCALARIZE-NEXT: br i1 [[COND:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]] +; NOSCALARIZE: then: +; NOSCALARIZE-NEXT: [[X:%.*]] = insertelement <7 x double> [[IN:%.*]], double 3.140000e+00, i32 3 +; NOSCALARIZE-NEXT: br label [[FINALLY:%.*]] +; NOSCALARIZE: else: +; NOSCALARIZE-NEXT: [[Y:%.*]] = insertelement <7 x double> [[IN]], double 9.140000e+00, i32 6 +; NOSCALARIZE-NEXT: br label [[FINALLY]] +; NOSCALARIZE: finally: +; NOSCALARIZE-NEXT: [[VAL:%.*]] = phi <7 x double> [ [[X]], [[THEN]] ], [ [[Y]], [[ELSE]] ] +; NOSCALARIZE-NEXT: store <7 x double> [[VAL]], ptr [[OUT:%.*]], align 1 +; NOSCALARIZE-NEXT: ret void +; +entry: + br i1 %cond, label %then, label %else +then: + %x = insertelement <7 x double> %in, double 3.14, i32 3 + br label %finally +else: + %y = insertelement <7 x double> %in, double 9.14, i32 6 + br label %finally +finally: + %val = phi <7 x double> [%x, %then], [%y, %else] + store <7 x double> %val, ptr %out, align 1 + ret void +} + +define amdgpu_kernel void @phi_v11f64(<11 x double> %in, ptr %out, i1 %cond) { +; SCALARIZED-LABEL: @phi_v11f64( +; SCALARIZED-NEXT: entry: +; SCALARIZED-NEXT: br i1 [[COND:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]] +; SCALARIZED: then: +; SCALARIZED-NEXT: [[X:%.*]] = insertelement <11 x double> [[IN:%.*]], double 3.140000e+00, i32 3 +; SCALARIZED-NEXT: [[TMP0:%.*]] = extractelement <11 x double> [[X]], i64 0 +; SCALARIZED-NEXT: [[TMP1:%.*]] = extractelement <11 x double> [[X]], i64 1 +; SCALARIZED-NEXT: [[TMP2:%.*]] = extractelement <11 x double> [[X]], i64 2 +; SCALARIZED-NEXT: [[TMP3:%.*]] = extractelement <11 x double> [[X]], i64 3 +; SCALARIZED-NEXT: [[TMP4:%.*]] = extractelement <11 x double> [[X]], i64 4 +; SCALARIZED-NEXT: [[TMP5:%.*]] = extractelement <11 x double> [[X]], i64 5 +; SCALARIZED-NEXT: [[TMP6:%.*]] = extractelement <11 x double> [[X]], i64 6 +; SCALARIZED-NEXT: [[TMP7:%.*]] = extractelement <11 x double> [[X]], i64 7 +; SCALARIZED-NEXT: [[TMP8:%.*]] = extractelement <11 x double> [[X]], i64 8 +; SCALARIZED-NEXT: [[TMP9:%.*]] = extractelement <11 x double> [[X]], i64 9 +; SCALARIZED-NEXT: [[TMP10:%.*]] = extractelement <11 x double> [[X]], i64 10 +; SCALARIZED-NEXT: br label [[FINALLY:%.*]] +; SCALARIZED: else: +; SCALARIZED-NEXT: [[Y:%.*]] = insertelement <11 x double> [[IN]], double 9.140000e+00, i32 6 +; SCALARIZED-NEXT: [[TMP11:%.*]] = extractelement <11 x double> [[Y]], i64 0 +; SCALARIZED-NEXT: [[TMP12:%.*]] = extractelement <11 x double> [[Y]], i64 1 +; SCALARIZED-NEXT: [[TMP13:%.*]] = extractelement <11 x double> [[Y]], i64 2 +; SCALARIZED-NEXT: [[TMP14:%.*]] = extractelement <11 x double> [[Y]], i64 3 +; SCALARIZED-NEXT: [[TMP15:%.*]] = extractelement <11 x double> [[Y]], i64 4 +; SCALARIZED-NEXT: [[TMP16:%.*]] = extractelement <11 x double> [[Y]], i64 5 +; SCALARIZED-NEXT: [[TMP17:%.*]] = extractelement <11 x double> [[Y]], i64 6 +; SCALARIZED-NEXT: [[TMP18:%.*]] = extractelement <11 x double> [[Y]], i64 7 +; SCALARIZED-NEXT: [[TMP19:%.*]] = extractelement <11 x double> [[Y]], i64 8 +; SCALARIZED-NEXT: [[TMP20:%.*]] = extractelement <11 x double> [[Y]], i64 9 +; SCALARIZED-NEXT: [[TMP21:%.*]] = extractelement <11 x double> [[Y]], i64 10 +; SCALARIZED-NEXT: br label [[FINALLY]] +; SCALARIZED: finally: +; SCALARIZED-NEXT: [[TMP22:%.*]] = phi double [ [[TMP0]], [[THEN]] ], [ [[TMP11]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP23:%.*]] = phi double [ [[TMP1]], [[THEN]] ], [ [[TMP12]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP24:%.*]] = phi double [ [[TMP2]], [[THEN]] ], [ [[TMP13]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP25:%.*]] = phi double [ [[TMP3]], [[THEN]] ], [ [[TMP14]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP26:%.*]] = phi double [ [[TMP4]], [[THEN]] ], [ [[TMP15]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP27:%.*]] = phi double [ [[TMP5]], [[THEN]] ], [ [[TMP16]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP28:%.*]] = phi double [ [[TMP6]], [[THEN]] ], [ [[TMP17]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP29:%.*]] = phi double [ [[TMP7]], [[THEN]] ], [ [[TMP18]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP30:%.*]] = phi double [ [[TMP8]], [[THEN]] ], [ [[TMP19]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP31:%.*]] = phi double [ [[TMP9]], [[THEN]] ], [ [[TMP20]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP32:%.*]] = phi double [ [[TMP10]], [[THEN]] ], [ [[TMP21]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP33:%.*]] = insertelement <11 x double> poison, double [[TMP22]], i64 0 +; SCALARIZED-NEXT: [[TMP34:%.*]] = insertelement <11 x double> [[TMP33]], double [[TMP23]], i64 1 +; SCALARIZED-NEXT: [[TMP35:%.*]] = insertelement <11 x double> [[TMP34]], double [[TMP24]], i64 2 +; SCALARIZED-NEXT: [[TMP36:%.*]] = insertelement <11 x double> [[TMP35]], double [[TMP25]], i64 3 +; SCALARIZED-NEXT: [[TMP37:%.*]] = insertelement <11 x double> [[TMP36]], double [[TMP26]], i64 4 +; SCALARIZED-NEXT: [[TMP38:%.*]] = insertelement <11 x double> [[TMP37]], double [[TMP27]], i64 5 +; SCALARIZED-NEXT: [[TMP39:%.*]] = insertelement <11 x double> [[TMP38]], double [[TMP28]], i64 6 +; SCALARIZED-NEXT: [[TMP40:%.*]] = insertelement <11 x double> [[TMP39]], double [[TMP29]], i64 7 +; SCALARIZED-NEXT: [[TMP41:%.*]] = insertelement <11 x double> [[TMP40]], double [[TMP30]], i64 8 +; SCALARIZED-NEXT: [[TMP42:%.*]] = insertelement <11 x double> [[TMP41]], double [[TMP31]], i64 9 +; SCALARIZED-NEXT: [[TMP43:%.*]] = insertelement <11 x double> [[TMP42]], double [[TMP32]], i64 10 +; SCALARIZED-NEXT: store <11 x double> [[TMP43]], ptr [[OUT:%.*]], align 1 +; SCALARIZED-NEXT: ret void +; +; NOSCALARIZE-LABEL: @phi_v11f64( +; NOSCALARIZE-NEXT: entry: +; NOSCALARIZE-NEXT: br i1 [[COND:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]] +; NOSCALARIZE: then: +; NOSCALARIZE-NEXT: [[X:%.*]] = insertelement <11 x double> [[IN:%.*]], double 3.140000e+00, i32 3 +; NOSCALARIZE-NEXT: br label [[FINALLY:%.*]] +; NOSCALARIZE: else: +; NOSCALARIZE-NEXT: [[Y:%.*]] = insertelement <11 x double> [[IN]], double 9.140000e+00, i32 6 +; NOSCALARIZE-NEXT: br label [[FINALLY]] +; NOSCALARIZE: finally: +; NOSCALARIZE-NEXT: [[VAL:%.*]] = phi <11 x double> [ [[X]], [[THEN]] ], [ [[Y]], [[ELSE]] ] +; NOSCALARIZE-NEXT: store <11 x double> [[VAL]], ptr [[OUT:%.*]], align 1 +; NOSCALARIZE-NEXT: ret void +; +entry: + br i1 %cond, label %then, label %else +then: + %x = insertelement <11 x double> %in, double 3.14, i32 3 + br label %finally +else: + %y = insertelement <11 x double> %in, double 9.14, i32 6 + br label %finally +finally: + %val = phi <11 x double> [%x, %then], [%y, %else] + store <11 x double> %val, ptr %out, align 1 + ret void +} + +define amdgpu_kernel void @phi_v11f64_cst(<11 x double> %in, ptr %out, i1 %cond) { +; SCALARIZED-LABEL: @phi_v11f64_cst( +; SCALARIZED-NEXT: entry: +; SCALARIZED-NEXT: br i1 [[COND:%.*]], label [[THEN:%.*]], label [[FINALLY:%.*]] +; SCALARIZED: then: +; SCALARIZED-NEXT: [[X:%.*]] = insertelement <11 x double> [[IN:%.*]], double 3.140000e+00, i32 3 +; SCALARIZED-NEXT: [[TMP0:%.*]] = extractelement <11 x double> [[X]], i64 0 +; SCALARIZED-NEXT: [[TMP1:%.*]] = extractelement <11 x double> [[X]], i64 1 +; SCALARIZED-NEXT: [[TMP2:%.*]] = extractelement <11 x double> [[X]], i64 2 +; SCALARIZED-NEXT: [[TMP3:%.*]] = extractelement <11 x double> [[X]], i64 3 +; SCALARIZED-NEXT: [[TMP4:%.*]] = extractelement <11 x double> [[X]], i64 4 +; SCALARIZED-NEXT: [[TMP5:%.*]] = extractelement <11 x double> [[X]], i64 5 +; SCALARIZED-NEXT: [[TMP6:%.*]] = extractelement <11 x double> [[X]], i64 6 +; SCALARIZED-NEXT: [[TMP7:%.*]] = extractelement <11 x double> [[X]], i64 7 +; SCALARIZED-NEXT: [[TMP8:%.*]] = extractelement <11 x double> [[X]], i64 8 +; SCALARIZED-NEXT: [[TMP9:%.*]] = extractelement <11 x double> [[X]], i64 9 +; SCALARIZED-NEXT: [[TMP10:%.*]] = extractelement <11 x double> [[X]], i64 10 +; SCALARIZED-NEXT: br label [[FINALLY]] +; SCALARIZED: finally: +; SCALARIZED-NEXT: [[TMP11:%.*]] = phi double [ [[TMP0]], [[THEN]] ], [ 0.000000e+00, [[ENTRY:%.*]] ] +; SCALARIZED-NEXT: [[TMP12:%.*]] = phi double [ [[TMP1]], [[THEN]] ], [ 0.000000e+00, [[ENTRY]] ] +; SCALARIZED-NEXT: [[TMP13:%.*]] = phi double [ [[TMP2]], [[THEN]] ], [ 0.000000e+00, [[ENTRY]] ] +; SCALARIZED-NEXT: [[TMP14:%.*]] = phi double [ [[TMP3]], [[THEN]] ], [ 0.000000e+00, [[ENTRY]] ] +; SCALARIZED-NEXT: [[TMP15:%.*]] = phi double [ [[TMP4]], [[THEN]] ], [ 0.000000e+00, [[ENTRY]] ] +; SCALARIZED-NEXT: [[TMP16:%.*]] = phi double [ [[TMP5]], [[THEN]] ], [ 0.000000e+00, [[ENTRY]] ] +; SCALARIZED-NEXT: [[TMP17:%.*]] = phi double [ [[TMP6]], [[THEN]] ], [ 0.000000e+00, [[ENTRY]] ] +; SCALARIZED-NEXT: [[TMP18:%.*]] = phi double [ [[TMP7]], [[THEN]] ], [ 0.000000e+00, [[ENTRY]] ] +; SCALARIZED-NEXT: [[TMP19:%.*]] = phi double [ [[TMP8]], [[THEN]] ], [ 0.000000e+00, [[ENTRY]] ] +; SCALARIZED-NEXT: [[TMP20:%.*]] = phi double [ [[TMP9]], [[THEN]] ], [ 0.000000e+00, [[ENTRY]] ] +; SCALARIZED-NEXT: [[TMP21:%.*]] = phi double [ [[TMP10]], [[THEN]] ], [ 0.000000e+00, [[ENTRY]] ] +; SCALARIZED-NEXT: [[TMP22:%.*]] = insertelement <11 x double> poison, double [[TMP11]], i64 0 +; SCALARIZED-NEXT: [[TMP23:%.*]] = insertelement <11 x double> [[TMP22]], double [[TMP12]], i64 1 +; SCALARIZED-NEXT: [[TMP24:%.*]] = insertelement <11 x double> [[TMP23]], double [[TMP13]], i64 2 +; SCALARIZED-NEXT: [[TMP25:%.*]] = insertelement <11 x double> [[TMP24]], double [[TMP14]], i64 3 +; SCALARIZED-NEXT: [[TMP26:%.*]] = insertelement <11 x double> [[TMP25]], double [[TMP15]], i64 4 +; SCALARIZED-NEXT: [[TMP27:%.*]] = insertelement <11 x double> [[TMP26]], double [[TMP16]], i64 5 +; SCALARIZED-NEXT: [[TMP28:%.*]] = insertelement <11 x double> [[TMP27]], double [[TMP17]], i64 6 +; SCALARIZED-NEXT: [[TMP29:%.*]] = insertelement <11 x double> [[TMP28]], double [[TMP18]], i64 7 +; SCALARIZED-NEXT: [[TMP30:%.*]] = insertelement <11 x double> [[TMP29]], double [[TMP19]], i64 8 +; SCALARIZED-NEXT: [[TMP31:%.*]] = insertelement <11 x double> [[TMP30]], double [[TMP20]], i64 9 +; SCALARIZED-NEXT: [[TMP32:%.*]] = insertelement <11 x double> [[TMP31]], double [[TMP21]], i64 10 +; SCALARIZED-NEXT: store <11 x double> [[TMP32]], ptr [[OUT:%.*]], align 1 +; SCALARIZED-NEXT: ret void +; +; NOSCALARIZE-LABEL: @phi_v11f64_cst( +; NOSCALARIZE-NEXT: entry: +; NOSCALARIZE-NEXT: br i1 [[COND:%.*]], label [[THEN:%.*]], label [[FINALLY:%.*]] +; NOSCALARIZE: then: +; NOSCALARIZE-NEXT: [[X:%.*]] = insertelement <11 x double> [[IN:%.*]], double 3.140000e+00, i32 3 +; NOSCALARIZE-NEXT: br label [[FINALLY]] +; NOSCALARIZE: finally: +; NOSCALARIZE-NEXT: [[VAL:%.*]] = phi <11 x double> [ [[X]], [[THEN]] ], [ zeroinitializer, [[ENTRY:%.*]] ] +; NOSCALARIZE-NEXT: store <11 x double> [[VAL]], ptr [[OUT:%.*]], align 1 +; NOSCALARIZE-NEXT: ret void +; +entry: + br i1 %cond, label %then, label %finally +then: + %x = insertelement <11 x double> %in, double 3.14, i32 3 + br label %finally +finally: + %val = phi <11 x double> [%x, %then], [zeroinitializer, %entry] + store <11 x double> %val, ptr %out, align 1 + ret void +} + +define amdgpu_kernel void @phi_v15i64(<15 x i64> %in, ptr %out, i1 %cond) { +; SCALARIZED-LABEL: @phi_v15i64( +; SCALARIZED-NEXT: entry: +; SCALARIZED-NEXT: br i1 [[COND:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]] +; SCALARIZED: then: +; SCALARIZED-NEXT: [[X:%.*]] = insertelement <15 x i64> [[IN:%.*]], i64 42, i32 3 +; SCALARIZED-NEXT: [[TMP0:%.*]] = extractelement <15 x i64> [[X]], i64 0 +; SCALARIZED-NEXT: [[TMP1:%.*]] = extractelement <15 x i64> [[X]], i64 1 +; SCALARIZED-NEXT: [[TMP2:%.*]] = extractelement <15 x i64> [[X]], i64 2 +; SCALARIZED-NEXT: [[TMP3:%.*]] = extractelement <15 x i64> [[X]], i64 3 +; SCALARIZED-NEXT: [[TMP4:%.*]] = extractelement <15 x i64> [[X]], i64 4 +; SCALARIZED-NEXT: [[TMP5:%.*]] = extractelement <15 x i64> [[X]], i64 5 +; SCALARIZED-NEXT: [[TMP6:%.*]] = extractelement <15 x i64> [[X]], i64 6 +; SCALARIZED-NEXT: [[TMP7:%.*]] = extractelement <15 x i64> [[X]], i64 7 +; SCALARIZED-NEXT: [[TMP8:%.*]] = extractelement <15 x i64> [[X]], i64 8 +; SCALARIZED-NEXT: [[TMP9:%.*]] = extractelement <15 x i64> [[X]], i64 9 +; SCALARIZED-NEXT: [[TMP10:%.*]] = extractelement <15 x i64> [[X]], i64 10 +; SCALARIZED-NEXT: [[TMP11:%.*]] = extractelement <15 x i64> [[X]], i64 11 +; SCALARIZED-NEXT: [[TMP12:%.*]] = extractelement <15 x i64> [[X]], i64 12 +; SCALARIZED-NEXT: [[TMP13:%.*]] = extractelement <15 x i64> [[X]], i64 13 +; SCALARIZED-NEXT: [[TMP14:%.*]] = extractelement <15 x i64> [[X]], i64 14 +; SCALARIZED-NEXT: br label [[FINALLY:%.*]] +; SCALARIZED: else: +; SCALARIZED-NEXT: [[Y:%.*]] = insertelement <15 x i64> [[IN]], i64 64, i32 6 +; SCALARIZED-NEXT: [[TMP15:%.*]] = extractelement <15 x i64> [[Y]], i64 0 +; SCALARIZED-NEXT: [[TMP16:%.*]] = extractelement <15 x i64> [[Y]], i64 1 +; SCALARIZED-NEXT: [[TMP17:%.*]] = extractelement <15 x i64> [[Y]], i64 2 +; SCALARIZED-NEXT: [[TMP18:%.*]] = extractelement <15 x i64> [[Y]], i64 3 +; SCALARIZED-NEXT: [[TMP19:%.*]] = extractelement <15 x i64> [[Y]], i64 4 +; SCALARIZED-NEXT: [[TMP20:%.*]] = extractelement <15 x i64> [[Y]], i64 5 +; SCALARIZED-NEXT: [[TMP21:%.*]] = extractelement <15 x i64> [[Y]], i64 6 +; SCALARIZED-NEXT: [[TMP22:%.*]] = extractelement <15 x i64> [[Y]], i64 7 +; SCALARIZED-NEXT: [[TMP23:%.*]] = extractelement <15 x i64> [[Y]], i64 8 +; SCALARIZED-NEXT: [[TMP24:%.*]] = extractelement <15 x i64> [[Y]], i64 9 +; SCALARIZED-NEXT: [[TMP25:%.*]] = extractelement <15 x i64> [[Y]], i64 10 +; SCALARIZED-NEXT: [[TMP26:%.*]] = extractelement <15 x i64> [[Y]], i64 11 +; SCALARIZED-NEXT: [[TMP27:%.*]] = extractelement <15 x i64> [[Y]], i64 12 +; SCALARIZED-NEXT: [[TMP28:%.*]] = extractelement <15 x i64> [[Y]], i64 13 +; SCALARIZED-NEXT: [[TMP29:%.*]] = extractelement <15 x i64> [[Y]], i64 14 +; SCALARIZED-NEXT: br label [[FINALLY]] +; SCALARIZED: finally: +; SCALARIZED-NEXT: [[TMP30:%.*]] = phi i64 [ [[TMP0]], [[THEN]] ], [ [[TMP15]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP31:%.*]] = phi i64 [ [[TMP1]], [[THEN]] ], [ [[TMP16]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP32:%.*]] = phi i64 [ [[TMP2]], [[THEN]] ], [ [[TMP17]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP33:%.*]] = phi i64 [ [[TMP3]], [[THEN]] ], [ [[TMP18]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP34:%.*]] = phi i64 [ [[TMP4]], [[THEN]] ], [ [[TMP19]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP35:%.*]] = phi i64 [ [[TMP5]], [[THEN]] ], [ [[TMP20]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP36:%.*]] = phi i64 [ [[TMP6]], [[THEN]] ], [ [[TMP21]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP37:%.*]] = phi i64 [ [[TMP7]], [[THEN]] ], [ [[TMP22]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP38:%.*]] = phi i64 [ [[TMP8]], [[THEN]] ], [ [[TMP23]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP39:%.*]] = phi i64 [ [[TMP9]], [[THEN]] ], [ [[TMP24]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP40:%.*]] = phi i64 [ [[TMP10]], [[THEN]] ], [ [[TMP25]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP41:%.*]] = phi i64 [ [[TMP11]], [[THEN]] ], [ [[TMP26]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP42:%.*]] = phi i64 [ [[TMP12]], [[THEN]] ], [ [[TMP27]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP43:%.*]] = phi i64 [ [[TMP13]], [[THEN]] ], [ [[TMP28]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP44:%.*]] = phi i64 [ [[TMP14]], [[THEN]] ], [ [[TMP29]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP45:%.*]] = insertelement <15 x i64> poison, i64 [[TMP30]], i64 0 +; SCALARIZED-NEXT: [[TMP46:%.*]] = insertelement <15 x i64> [[TMP45]], i64 [[TMP31]], i64 1 +; SCALARIZED-NEXT: [[TMP47:%.*]] = insertelement <15 x i64> [[TMP46]], i64 [[TMP32]], i64 2 +; SCALARIZED-NEXT: [[TMP48:%.*]] = insertelement <15 x i64> [[TMP47]], i64 [[TMP33]], i64 3 +; SCALARIZED-NEXT: [[TMP49:%.*]] = insertelement <15 x i64> [[TMP48]], i64 [[TMP34]], i64 4 +; SCALARIZED-NEXT: [[TMP50:%.*]] = insertelement <15 x i64> [[TMP49]], i64 [[TMP35]], i64 5 +; SCALARIZED-NEXT: [[TMP51:%.*]] = insertelement <15 x i64> [[TMP50]], i64 [[TMP36]], i64 6 +; SCALARIZED-NEXT: [[TMP52:%.*]] = insertelement <15 x i64> [[TMP51]], i64 [[TMP37]], i64 7 +; SCALARIZED-NEXT: [[TMP53:%.*]] = insertelement <15 x i64> [[TMP52]], i64 [[TMP38]], i64 8 +; SCALARIZED-NEXT: [[TMP54:%.*]] = insertelement <15 x i64> [[TMP53]], i64 [[TMP39]], i64 9 +; SCALARIZED-NEXT: [[TMP55:%.*]] = insertelement <15 x i64> [[TMP54]], i64 [[TMP40]], i64 10 +; SCALARIZED-NEXT: [[TMP56:%.*]] = insertelement <15 x i64> [[TMP55]], i64 [[TMP41]], i64 11 +; SCALARIZED-NEXT: [[TMP57:%.*]] = insertelement <15 x i64> [[TMP56]], i64 [[TMP42]], i64 12 +; SCALARIZED-NEXT: [[TMP58:%.*]] = insertelement <15 x i64> [[TMP57]], i64 [[TMP43]], i64 13 +; SCALARIZED-NEXT: [[TMP59:%.*]] = insertelement <15 x i64> [[TMP58]], i64 [[TMP44]], i64 14 +; SCALARIZED-NEXT: store <15 x i64> [[TMP59]], ptr [[OUT:%.*]], align 1 +; SCALARIZED-NEXT: ret void +; +; NOSCALARIZE-LABEL: @phi_v15i64( +; NOSCALARIZE-NEXT: entry: +; NOSCALARIZE-NEXT: br i1 [[COND:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]] +; NOSCALARIZE: then: +; NOSCALARIZE-NEXT: [[X:%.*]] = insertelement <15 x i64> [[IN:%.*]], i64 42, i32 3 +; NOSCALARIZE-NEXT: br label [[FINALLY:%.*]] +; NOSCALARIZE: else: +; NOSCALARIZE-NEXT: [[Y:%.*]] = insertelement <15 x i64> [[IN]], i64 64, i32 6 +; NOSCALARIZE-NEXT: br label [[FINALLY]] +; NOSCALARIZE: finally: +; NOSCALARIZE-NEXT: [[VAL:%.*]] = phi <15 x i64> [ [[X]], [[THEN]] ], [ [[Y]], [[ELSE]] ] +; NOSCALARIZE-NEXT: store <15 x i64> [[VAL]], ptr [[OUT:%.*]], align 1 +; NOSCALARIZE-NEXT: ret void +; +entry: + br i1 %cond, label %then, label %else +then: + %x = insertelement <15 x i64> %in, i64 42, i32 3 + br label %finally +else: + %y = insertelement <15 x i64> %in, i64 64, i32 6 + br label %finally +finally: + %val = phi <15 x i64> [%x, %then], [%y, %else] + store <15 x i64> %val, ptr %out, align 1 + ret void +} + +define amdgpu_kernel void @phi_v23i32(<23 x i32> %in, ptr %out, i1 %cond) { +; SCALARIZED-LABEL: @phi_v23i32( +; SCALARIZED-NEXT: entry: +; SCALARIZED-NEXT: br i1 [[COND:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]] +; SCALARIZED: then: +; SCALARIZED-NEXT: [[X:%.*]] = insertelement <23 x i32> [[IN:%.*]], i32 42, i32 3 +; SCALARIZED-NEXT: [[TMP0:%.*]] = extractelement <23 x i32> [[X]], i64 0 +; SCALARIZED-NEXT: [[TMP1:%.*]] = extractelement <23 x i32> [[X]], i64 1 +; SCALARIZED-NEXT: [[TMP2:%.*]] = extractelement <23 x i32> [[X]], i64 2 +; SCALARIZED-NEXT: [[TMP3:%.*]] = extractelement <23 x i32> [[X]], i64 3 +; SCALARIZED-NEXT: [[TMP4:%.*]] = extractelement <23 x i32> [[X]], i64 4 +; SCALARIZED-NEXT: [[TMP5:%.*]] = extractelement <23 x i32> [[X]], i64 5 +; SCALARIZED-NEXT: [[TMP6:%.*]] = extractelement <23 x i32> [[X]], i64 6 +; SCALARIZED-NEXT: [[TMP7:%.*]] = extractelement <23 x i32> [[X]], i64 7 +; SCALARIZED-NEXT: [[TMP8:%.*]] = extractelement <23 x i32> [[X]], i64 8 +; SCALARIZED-NEXT: [[TMP9:%.*]] = extractelement <23 x i32> [[X]], i64 9 +; SCALARIZED-NEXT: [[TMP10:%.*]] = extractelement <23 x i32> [[X]], i64 10 +; SCALARIZED-NEXT: [[TMP11:%.*]] = extractelement <23 x i32> [[X]], i64 11 +; SCALARIZED-NEXT: [[TMP12:%.*]] = extractelement <23 x i32> [[X]], i64 12 +; SCALARIZED-NEXT: [[TMP13:%.*]] = extractelement <23 x i32> [[X]], i64 13 +; SCALARIZED-NEXT: [[TMP14:%.*]] = extractelement <23 x i32> [[X]], i64 14 +; SCALARIZED-NEXT: [[TMP15:%.*]] = extractelement <23 x i32> [[X]], i64 15 +; SCALARIZED-NEXT: [[TMP16:%.*]] = extractelement <23 x i32> [[X]], i64 16 +; SCALARIZED-NEXT: [[TMP17:%.*]] = extractelement <23 x i32> [[X]], i64 17 +; SCALARIZED-NEXT: [[TMP18:%.*]] = extractelement <23 x i32> [[X]], i64 18 +; SCALARIZED-NEXT: [[TMP19:%.*]] = extractelement <23 x i32> [[X]], i64 19 +; SCALARIZED-NEXT: [[TMP20:%.*]] = extractelement <23 x i32> [[X]], i64 20 +; SCALARIZED-NEXT: [[TMP21:%.*]] = extractelement <23 x i32> [[X]], i64 21 +; SCALARIZED-NEXT: [[TMP22:%.*]] = extractelement <23 x i32> [[X]], i64 22 +; SCALARIZED-NEXT: br label [[FINALLY:%.*]] +; SCALARIZED: else: +; SCALARIZED-NEXT: [[Y:%.*]] = insertelement <23 x i32> [[IN]], i32 64, i32 6 +; SCALARIZED-NEXT: [[TMP23:%.*]] = extractelement <23 x i32> [[Y]], i64 0 +; SCALARIZED-NEXT: [[TMP24:%.*]] = extractelement <23 x i32> [[Y]], i64 1 +; SCALARIZED-NEXT: [[TMP25:%.*]] = extractelement <23 x i32> [[Y]], i64 2 +; SCALARIZED-NEXT: [[TMP26:%.*]] = extractelement <23 x i32> [[Y]], i64 3 +; SCALARIZED-NEXT: [[TMP27:%.*]] = extractelement <23 x i32> [[Y]], i64 4 +; SCALARIZED-NEXT: [[TMP28:%.*]] = extractelement <23 x i32> [[Y]], i64 5 +; SCALARIZED-NEXT: [[TMP29:%.*]] = extractelement <23 x i32> [[Y]], i64 6 +; SCALARIZED-NEXT: [[TMP30:%.*]] = extractelement <23 x i32> [[Y]], i64 7 +; SCALARIZED-NEXT: [[TMP31:%.*]] = extractelement <23 x i32> [[Y]], i64 8 +; SCALARIZED-NEXT: [[TMP32:%.*]] = extractelement <23 x i32> [[Y]], i64 9 +; SCALARIZED-NEXT: [[TMP33:%.*]] = extractelement <23 x i32> [[Y]], i64 10 +; SCALARIZED-NEXT: [[TMP34:%.*]] = extractelement <23 x i32> [[Y]], i64 11 +; SCALARIZED-NEXT: [[TMP35:%.*]] = extractelement <23 x i32> [[Y]], i64 12 +; SCALARIZED-NEXT: [[TMP36:%.*]] = extractelement <23 x i32> [[Y]], i64 13 +; SCALARIZED-NEXT: [[TMP37:%.*]] = extractelement <23 x i32> [[Y]], i64 14 +; SCALARIZED-NEXT: [[TMP38:%.*]] = extractelement <23 x i32> [[Y]], i64 15 +; SCALARIZED-NEXT: [[TMP39:%.*]] = extractelement <23 x i32> [[Y]], i64 16 +; SCALARIZED-NEXT: [[TMP40:%.*]] = extractelement <23 x i32> [[Y]], i64 17 +; SCALARIZED-NEXT: [[TMP41:%.*]] = extractelement <23 x i32> [[Y]], i64 18 +; SCALARIZED-NEXT: [[TMP42:%.*]] = extractelement <23 x i32> [[Y]], i64 19 +; SCALARIZED-NEXT: [[TMP43:%.*]] = extractelement <23 x i32> [[Y]], i64 20 +; SCALARIZED-NEXT: [[TMP44:%.*]] = extractelement <23 x i32> [[Y]], i64 21 +; SCALARIZED-NEXT: [[TMP45:%.*]] = extractelement <23 x i32> [[Y]], i64 22 +; SCALARIZED-NEXT: br label [[FINALLY]] +; SCALARIZED: finally: +; SCALARIZED-NEXT: [[TMP46:%.*]] = phi i32 [ [[TMP0]], [[THEN]] ], [ [[TMP23]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP47:%.*]] = phi i32 [ [[TMP1]], [[THEN]] ], [ [[TMP24]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP48:%.*]] = phi i32 [ [[TMP2]], [[THEN]] ], [ [[TMP25]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP49:%.*]] = phi i32 [ [[TMP3]], [[THEN]] ], [ [[TMP26]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP50:%.*]] = phi i32 [ [[TMP4]], [[THEN]] ], [ [[TMP27]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP51:%.*]] = phi i32 [ [[TMP5]], [[THEN]] ], [ [[TMP28]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP52:%.*]] = phi i32 [ [[TMP6]], [[THEN]] ], [ [[TMP29]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP53:%.*]] = phi i32 [ [[TMP7]], [[THEN]] ], [ [[TMP30]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP54:%.*]] = phi i32 [ [[TMP8]], [[THEN]] ], [ [[TMP31]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP55:%.*]] = phi i32 [ [[TMP9]], [[THEN]] ], [ [[TMP32]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP56:%.*]] = phi i32 [ [[TMP10]], [[THEN]] ], [ [[TMP33]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP57:%.*]] = phi i32 [ [[TMP11]], [[THEN]] ], [ [[TMP34]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP58:%.*]] = phi i32 [ [[TMP12]], [[THEN]] ], [ [[TMP35]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP59:%.*]] = phi i32 [ [[TMP13]], [[THEN]] ], [ [[TMP36]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP60:%.*]] = phi i32 [ [[TMP14]], [[THEN]] ], [ [[TMP37]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP61:%.*]] = phi i32 [ [[TMP15]], [[THEN]] ], [ [[TMP38]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP62:%.*]] = phi i32 [ [[TMP16]], [[THEN]] ], [ [[TMP39]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP63:%.*]] = phi i32 [ [[TMP17]], [[THEN]] ], [ [[TMP40]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP64:%.*]] = phi i32 [ [[TMP18]], [[THEN]] ], [ [[TMP41]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP65:%.*]] = phi i32 [ [[TMP19]], [[THEN]] ], [ [[TMP42]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP66:%.*]] = phi i32 [ [[TMP20]], [[THEN]] ], [ [[TMP43]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP67:%.*]] = phi i32 [ [[TMP21]], [[THEN]] ], [ [[TMP44]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP68:%.*]] = phi i32 [ [[TMP22]], [[THEN]] ], [ [[TMP45]], [[ELSE]] ] +; SCALARIZED-NEXT: [[TMP69:%.*]] = insertelement <23 x i32> poison, i32 [[TMP46]], i64 0 +; SCALARIZED-NEXT: [[TMP70:%.*]] = insertelement <23 x i32> [[TMP69]], i32 [[TMP47]], i64 1 +; SCALARIZED-NEXT: [[TMP71:%.*]] = insertelement <23 x i32> [[TMP70]], i32 [[TMP48]], i64 2 +; SCALARIZED-NEXT: [[TMP72:%.*]] = insertelement <23 x i32> [[TMP71]], i32 [[TMP49]], i64 3 +; SCALARIZED-NEXT: [[TMP73:%.*]] = insertelement <23 x i32> [[TMP72]], i32 [[TMP50]], i64 4 +; SCALARIZED-NEXT: [[TMP74:%.*]] = insertelement <23 x i32> [[TMP73]], i32 [[TMP51]], i64 5 +; SCALARIZED-NEXT: [[TMP75:%.*]] = insertelement <23 x i32> [[TMP74]], i32 [[TMP52]], i64 6 +; SCALARIZED-NEXT: [[TMP76:%.*]] = insertelement <23 x i32> [[TMP75]], i32 [[TMP53]], i64 7 +; SCALARIZED-NEXT: [[TMP77:%.*]] = insertelement <23 x i32> [[TMP76]], i32 [[TMP54]], i64 8 +; SCALARIZED-NEXT: [[TMP78:%.*]] = insertelement <23 x i32> [[TMP77]], i32 [[TMP55]], i64 9 +; SCALARIZED-NEXT: [[TMP79:%.*]] = insertelement <23 x i32> [[TMP78]], i32 [[TMP56]], i64 10 +; SCALARIZED-NEXT: [[TMP80:%.*]] = insertelement <23 x i32> [[TMP79]], i32 [[TMP57]], i64 11 +; SCALARIZED-NEXT: [[TMP81:%.*]] = insertelement <23 x i32> [[TMP80]], i32 [[TMP58]], i64 12 +; SCALARIZED-NEXT: [[TMP82:%.*]] = insertelement <23 x i32> [[TMP81]], i32 [[TMP59]], i64 13 +; SCALARIZED-NEXT: [[TMP83:%.*]] = insertelement <23 x i32> [[TMP82]], i32 [[TMP60]], i64 14 +; SCALARIZED-NEXT: [[TMP84:%.*]] = insertelement <23 x i32> [[TMP83]], i32 [[TMP61]], i64 15 +; SCALARIZED-NEXT: [[TMP85:%.*]] = insertelement <23 x i32> [[TMP84]], i32 [[TMP62]], i64 16 +; SCALARIZED-NEXT: [[TMP86:%.*]] = insertelement <23 x i32> [[TMP85]], i32 [[TMP63]], i64 17 +; SCALARIZED-NEXT: [[TMP87:%.*]] = insertelement <23 x i32> [[TMP86]], i32 [[TMP64]], i64 18 +; SCALARIZED-NEXT: [[TMP88:%.*]] = insertelement <23 x i32> [[TMP87]], i32 [[TMP65]], i64 19 +; SCALARIZED-NEXT: [[TMP89:%.*]] = insertelement <23 x i32> [[TMP88]], i32 [[TMP66]], i64 20 +; SCALARIZED-NEXT: [[TMP90:%.*]] = insertelement <23 x i32> [[TMP89]], i32 [[TMP67]], i64 21 +; SCALARIZED-NEXT: [[TMP91:%.*]] = insertelement <23 x i32> [[TMP90]], i32 [[TMP68]], i64 22 +; SCALARIZED-NEXT: store <23 x i32> [[TMP91]], ptr [[OUT:%.*]], align 1 +; SCALARIZED-NEXT: ret void +; +; NOSCALARIZE-LABEL: @phi_v23i32( +; NOSCALARIZE-NEXT: entry: +; NOSCALARIZE-NEXT: br i1 [[COND:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]] +; NOSCALARIZE: then: +; NOSCALARIZE-NEXT: [[X:%.*]] = insertelement <23 x i32> [[IN:%.*]], i32 42, i32 3 +; NOSCALARIZE-NEXT: br label [[FINALLY:%.*]] +; NOSCALARIZE: else: +; NOSCALARIZE-NEXT: [[Y:%.*]] = insertelement <23 x i32> [[IN]], i32 64, i32 6 +; NOSCALARIZE-NEXT: br label [[FINALLY]] +; NOSCALARIZE: finally: +; NOSCALARIZE-NEXT: [[VAL:%.*]] = phi <23 x i32> [ [[X]], [[THEN]] ], [ [[Y]], [[ELSE]] ] +; NOSCALARIZE-NEXT: store <23 x i32> [[VAL]], ptr [[OUT:%.*]], align 1 +; NOSCALARIZE-NEXT: ret void +; +entry: + br i1 %cond, label %then, label %else +then: + %x = insertelement <23 x i32> %in, i32 42, i32 3 + br label %finally +else: + %y = insertelement <23 x i32> %in, i32 64, i32 6 + br label %finally +finally: + %val = phi <23 x i32> [%x, %then], [%y, %else] + store <23 x i32> %val, ptr %out, align 1 + ret void +} + +; Power-of-two-sized PHIs are not scalarized. +define amdgpu_kernel void @phi_v16i64(<16 x i64> %in, ptr %out, i1 %cond) { +; SCALARIZED-LABEL: @phi_v16i64( +; SCALARIZED-NEXT: entry: +; SCALARIZED-NEXT: br i1 [[COND:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]] +; SCALARIZED: then: +; SCALARIZED-NEXT: [[X:%.*]] = insertelement <16 x i64> [[IN:%.*]], i64 42, i32 3 +; SCALARIZED-NEXT: br label [[FINALLY:%.*]] +; SCALARIZED: else: +; SCALARIZED-NEXT: [[Y:%.*]] = insertelement <16 x i64> [[IN]], i64 64, i32 6 +; SCALARIZED-NEXT: br label [[FINALLY]] +; SCALARIZED: finally: +; SCALARIZED-NEXT: [[VAL:%.*]] = phi <16 x i64> [ [[X]], [[THEN]] ], [ [[Y]], [[ELSE]] ] +; SCALARIZED-NEXT: store <16 x i64> [[VAL]], ptr [[OUT:%.*]], align 1 +; SCALARIZED-NEXT: ret void +; +; NOSCALARIZE-LABEL: @phi_v16i64( +; NOSCALARIZE-NEXT: entry: +; NOSCALARIZE-NEXT: br i1 [[COND:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]] +; NOSCALARIZE: then: +; NOSCALARIZE-NEXT: [[X:%.*]] = insertelement <16 x i64> [[IN:%.*]], i64 42, i32 3 +; NOSCALARIZE-NEXT: br label [[FINALLY:%.*]] +; NOSCALARIZE: else: +; NOSCALARIZE-NEXT: [[Y:%.*]] = insertelement <16 x i64> [[IN]], i64 64, i32 6 +; NOSCALARIZE-NEXT: br label [[FINALLY]] +; NOSCALARIZE: finally: +; NOSCALARIZE-NEXT: [[VAL:%.*]] = phi <16 x i64> [ [[X]], [[THEN]] ], [ [[Y]], [[ELSE]] ] +; NOSCALARIZE-NEXT: store <16 x i64> [[VAL]], ptr [[OUT:%.*]], align 1 +; NOSCALARIZE-NEXT: ret void +; +entry: + br i1 %cond, label %then, label %else +then: + %x = insertelement <16 x i64> %in, i64 42, i32 3 + br label %finally +else: + %y = insertelement <16 x i64> %in, i64 64, i32 6 + br label %finally +finally: + %val = phi <16 x i64> [%x, %then], [%y, %else] + store <16 x i64> %val, ptr %out, align 1 + ret void +} + +; <256 bit vector +define amdgpu_kernel void @phi_v7i16(<7 x i16> %in, ptr %out, i1 %cond) { +; SCALARIZED-LABEL: @phi_v7i16( +; SCALARIZED-NEXT: entry: +; SCALARIZED-NEXT: br i1 [[COND:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]] +; SCALARIZED: then: +; SCALARIZED-NEXT: [[X:%.*]] = insertelement <7 x i16> [[IN:%.*]], i16 3, i32 3 +; SCALARIZED-NEXT: br label [[FINALLY:%.*]] +; SCALARIZED: else: +; SCALARIZED-NEXT: [[Y:%.*]] = insertelement <7 x i16> [[IN]], i16 9, i32 6 +; SCALARIZED-NEXT: br label [[FINALLY]] +; SCALARIZED: finally: +; SCALARIZED-NEXT: [[VAL:%.*]] = phi <7 x i16> [ [[X]], [[THEN]] ], [ [[Y]], [[ELSE]] ] +; SCALARIZED-NEXT: store <7 x i16> [[VAL]], ptr [[OUT:%.*]], align 1 +; SCALARIZED-NEXT: ret void +; +; NOSCALARIZE-LABEL: @phi_v7i16( +; NOSCALARIZE-NEXT: entry: +; NOSCALARIZE-NEXT: br i1 [[COND:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]] +; NOSCALARIZE: then: +; NOSCALARIZE-NEXT: [[X:%.*]] = insertelement <7 x i16> [[IN:%.*]], i16 3, i32 3 +; NOSCALARIZE-NEXT: br label [[FINALLY:%.*]] +; NOSCALARIZE: else: +; NOSCALARIZE-NEXT: [[Y:%.*]] = insertelement <7 x i16> [[IN]], i16 9, i32 6 +; NOSCALARIZE-NEXT: br label [[FINALLY]] +; NOSCALARIZE: finally: +; NOSCALARIZE-NEXT: [[VAL:%.*]] = phi <7 x i16> [ [[X]], [[THEN]] ], [ [[Y]], [[ELSE]] ] +; NOSCALARIZE-NEXT: store <7 x i16> [[VAL]], ptr [[OUT:%.*]], align 1 +; NOSCALARIZE-NEXT: ret void +; +entry: + br i1 %cond, label %then, label %else +then: + %x = insertelement <7 x i16> %in, i16 3, i32 3 + br label %finally +else: + %y = insertelement <7 x i16> %in, i16 9, i32 6 + br label %finally +finally: + %val = phi <7 x i16> [%x, %then], [%y, %else] + store <7 x i16> %val, ptr %out, align 1 + ret void +} diff --git a/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll b/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll --- a/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll +++ b/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll @@ -32,67 +32,76 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i32 %tmp5.i.i, i32 %tmp427.i, i1 %tmp438.i, double %tmp27.i, i1 %tmp48.i) { ; GLOBALNESS1-LABEL: kernel: ; GLOBALNESS1: ; %bb.0: ; %bb -; GLOBALNESS1-NEXT: s_mov_b64 s[54:55], s[6:7] -; GLOBALNESS1-NEXT: s_load_dwordx4 s[36:39], s[8:9], 0x0 +; GLOBALNESS1-NEXT: s_mov_b64 s[36:37], s[6:7] +; GLOBALNESS1-NEXT: s_load_dwordx4 s[76:79], s[8:9], 0x0 ; GLOBALNESS1-NEXT: s_load_dword s6, s[8:9], 0x14 -; GLOBALNESS1-NEXT: v_mov_b32_e32 v43, v0 +; GLOBALNESS1-NEXT: v_mov_b32_e32 v42, v0 ; GLOBALNESS1-NEXT: v_mov_b32_e32 v40, 0 ; GLOBALNESS1-NEXT: v_pk_mov_b32 v[0:1], 0, 0 ; GLOBALNESS1-NEXT: global_store_dword v[0:1], v40, off ; GLOBALNESS1-NEXT: s_waitcnt lgkmcnt(0) -; GLOBALNESS1-NEXT: global_load_dword v0, v40, s[36:37] -; GLOBALNESS1-NEXT: s_add_u32 flat_scratch_lo, s12, s17 -; GLOBALNESS1-NEXT: s_mov_b64 s[64:65], s[4:5] +; GLOBALNESS1-NEXT: global_load_dword v0, v40, s[76:77] +; GLOBALNESS1-NEXT: s_mov_b64 s[40:41], s[4:5] ; GLOBALNESS1-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x18 ; GLOBALNESS1-NEXT: s_load_dword s7, s[8:9], 0x20 +; GLOBALNESS1-NEXT: s_add_u32 flat_scratch_lo, s12, s17 ; GLOBALNESS1-NEXT: s_addc_u32 flat_scratch_hi, s13, 0 ; GLOBALNESS1-NEXT: s_add_u32 s0, s0, s17 ; GLOBALNESS1-NEXT: s_addc_u32 s1, s1, 0 ; GLOBALNESS1-NEXT: v_mov_b32_e32 v41, 0x40994400 -; GLOBALNESS1-NEXT: s_bitcmp1_b32 s38, 0 +; GLOBALNESS1-NEXT: s_bitcmp1_b32 s78, 0 ; GLOBALNESS1-NEXT: s_waitcnt lgkmcnt(0) -; GLOBALNESS1-NEXT: v_cmp_ngt_f64_e64 s[36:37], s[4:5], v[40:41] -; GLOBALNESS1-NEXT: v_cmp_ngt_f64_e64 s[40:41], s[4:5], 0 +; GLOBALNESS1-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[40:41] +; GLOBALNESS1-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[4:5], 0 +; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5] ; GLOBALNESS1-NEXT: s_cselect_b64 s[4:5], -1, 0 -; GLOBALNESS1-NEXT: s_xor_b64 s[94:95], s[4:5], -1 +; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v3, 0, 1, s[4:5] +; GLOBALNESS1-NEXT: s_xor_b64 s[4:5], s[4:5], -1 +; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc ; GLOBALNESS1-NEXT: s_bitcmp1_b32 s6, 0 +; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[42:43], 1, v1 ; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5] ; GLOBALNESS1-NEXT: s_cselect_b64 s[4:5], -1, 0 -; GLOBALNESS1-NEXT: s_xor_b64 s[88:89], s[4:5], -1 +; GLOBALNESS1-NEXT: s_xor_b64 s[4:5], s[4:5], -1 ; GLOBALNESS1-NEXT: s_bitcmp1_b32 s7, 0 +; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[48:49], 1, v1 +; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5] ; GLOBALNESS1-NEXT: s_cselect_b64 s[4:5], -1, 0 ; GLOBALNESS1-NEXT: s_getpc_b64 s[6:7] ; GLOBALNESS1-NEXT: s_add_u32 s6, s6, wobble@gotpcrel32@lo+4 ; GLOBALNESS1-NEXT: s_addc_u32 s7, s7, wobble@gotpcrel32@hi+12 -; GLOBALNESS1-NEXT: s_xor_b64 s[86:87], s[4:5], -1 -; GLOBALNESS1-NEXT: s_load_dwordx2 s[66:67], s[6:7], 0x0 -; GLOBALNESS1-NEXT: s_mov_b32 s98, s16 -; GLOBALNESS1-NEXT: s_mov_b64 s[62:63], s[8:9] -; GLOBALNESS1-NEXT: s_mov_b32 s99, s15 -; GLOBALNESS1-NEXT: s_mov_b32 s100, s14 +; GLOBALNESS1-NEXT: s_xor_b64 s[4:5], s[4:5], -1 +; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[50:51], 1, v1 +; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5] +; GLOBALNESS1-NEXT: s_load_dwordx2 s[76:77], s[6:7], 0x0 +; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[52:53], 1, v1 +; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[44:45], 1, v2 +; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[46:47], 1, v3 +; GLOBALNESS1-NEXT: s_mov_b32 s70, s16 +; GLOBALNESS1-NEXT: s_mov_b64 s[38:39], s[8:9] +; GLOBALNESS1-NEXT: s_mov_b32 s71, s15 +; GLOBALNESS1-NEXT: s_mov_b32 s72, s14 ; GLOBALNESS1-NEXT: s_mov_b64 s[34:35], s[10:11] -; GLOBALNESS1-NEXT: s_mov_b64 s[92:93], 0x80 -; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[42:43], 1, v1 -; GLOBALNESS1-NEXT: s_mov_b32 s69, 0x3ff00000 +; GLOBALNESS1-NEXT: s_mov_b64 s[74:75], 0x80 ; GLOBALNESS1-NEXT: s_mov_b32 s32, 0 -; GLOBALNESS1-NEXT: ; implicit-def: $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39_agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47_agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55_agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63 +; GLOBALNESS1-NEXT: ; implicit-def: $vgpr44_vgpr45 ; GLOBALNESS1-NEXT: s_waitcnt vmcnt(0) -; GLOBALNESS1-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v0 -; GLOBALNESS1-NEXT: v_writelane_b32 v42, s4, 0 -; GLOBALNESS1-NEXT: v_writelane_b32 v42, s5, 1 -; GLOBALNESS1-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v0 -; GLOBALNESS1-NEXT: v_writelane_b32 v42, s4, 2 -; GLOBALNESS1-NEXT: v_writelane_b32 v42, s5, 3 -; GLOBALNESS1-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v0 -; GLOBALNESS1-NEXT: v_writelane_b32 v42, s4, 4 -; GLOBALNESS1-NEXT: v_cmp_gt_i32_e64 s[90:91], 1, v0 -; GLOBALNESS1-NEXT: v_writelane_b32 v42, s5, 5 +; GLOBALNESS1-NEXT: v_cmp_gt_i32_e32 vcc, 0, v0 +; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc +; GLOBALNESS1-NEXT: v_cmp_gt_i32_e32 vcc, 1, v0 +; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc +; GLOBALNESS1-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc +; GLOBALNESS1-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc +; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[54:55], 1, v1 +; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[56:57], 1, v2 +; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[58:59], 1, v3 +; GLOBALNESS1-NEXT: v_cmp_ne_u32_e64 s[60:61], 1, v0 ; GLOBALNESS1-NEXT: s_branch .LBB1_4 ; GLOBALNESS1-NEXT: .LBB1_1: ; %bb70.i ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS1-NEXT: v_readlane_b32 s6, v42, 4 -; GLOBALNESS1-NEXT: v_readlane_b32 s7, v42, 5 -; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[6:7] +; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[60:61] ; GLOBALNESS1-NEXT: s_cbranch_vccz .LBB1_29 ; GLOBALNESS1-NEXT: .LBB1_2: ; %Flow6 ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1 @@ -101,69 +110,38 @@ ; GLOBALNESS1-NEXT: ; implicit-def: $sgpr4_sgpr5 ; GLOBALNESS1-NEXT: .LBB1_3: ; %Flow19 ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a63, v31 ; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[6:7] -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a62, v30 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a61, v29 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a60, v28 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a59, v27 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a58, v26 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a57, v25 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a56, v24 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a55, v23 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a54, v22 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a53, v21 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a52, v20 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a51, v19 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a50, v18 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a49, v17 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a48, v16 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a47, v15 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a46, v14 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a45, v13 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a44, v12 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a43, v11 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a42, v10 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a41, v9 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a40, v8 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a39, v7 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a38, v6 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a37, v5 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a36, v4 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a35, v3 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a34, v2 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a33, v1 -; GLOBALNESS1-NEXT: v_accvgpr_write_b32 a32, v0 +; GLOBALNESS1-NEXT: v_pk_mov_b32 v[44:45], v[0:1], v[0:1] op_sel:[0,1] ; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_30 ; GLOBALNESS1-NEXT: .LBB1_4: ; %bb5 ; GLOBALNESS1-NEXT: ; =>This Loop Header: Depth=1 ; GLOBALNESS1-NEXT: ; Child Loop BB1_15 Depth 2 -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[0:1], s[92:93], s[92:93] op_sel:[0,1] -; GLOBALNESS1-NEXT: flat_load_dword v44, v[0:1] -; GLOBALNESS1-NEXT: s_add_u32 s8, s62, 40 +; GLOBALNESS1-NEXT: v_pk_mov_b32 v[0:1], s[74:75], s[74:75] op_sel:[0,1] +; GLOBALNESS1-NEXT: flat_load_dword v43, v[0:1] +; GLOBALNESS1-NEXT: s_add_u32 s8, s38, 40 ; GLOBALNESS1-NEXT: buffer_store_dword v40, off, s[0:3], 0 -; GLOBALNESS1-NEXT: flat_load_dword v45, v[0:1] -; GLOBALNESS1-NEXT: s_addc_u32 s9, s63, 0 -; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], s[64:65] -; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], s[54:55] +; GLOBALNESS1-NEXT: flat_load_dword v46, v[0:1] +; GLOBALNESS1-NEXT: s_addc_u32 s9, s39, 0 +; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], s[40:41] +; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], s[36:37] ; GLOBALNESS1-NEXT: s_mov_b64 s[10:11], s[34:35] -; GLOBALNESS1-NEXT: s_mov_b32 s12, s100 -; GLOBALNESS1-NEXT: s_mov_b32 s13, s99 -; GLOBALNESS1-NEXT: s_mov_b32 s14, s98 -; GLOBALNESS1-NEXT: v_mov_b32_e32 v31, v43 +; GLOBALNESS1-NEXT: s_mov_b32 s12, s72 +; GLOBALNESS1-NEXT: s_mov_b32 s13, s71 +; GLOBALNESS1-NEXT: s_mov_b32 s14, s70 +; GLOBALNESS1-NEXT: v_mov_b32_e32 v31, v42 ; GLOBALNESS1-NEXT: s_waitcnt lgkmcnt(0) -; GLOBALNESS1-NEXT: s_swappc_b64 s[30:31], s[66:67] -; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[42:43] +; GLOBALNESS1-NEXT: s_swappc_b64 s[30:31], s[76:77] +; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[46:47] ; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], -1 ; GLOBALNESS1-NEXT: ; implicit-def: $sgpr4_sgpr5 ; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_8 ; GLOBALNESS1-NEXT: ; %bb.5: ; %NodeBlock ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS1-NEXT: s_cmp_lt_i32 s39, 1 +; GLOBALNESS1-NEXT: s_cmp_lt_i32 s79, 1 ; GLOBALNESS1-NEXT: s_cbranch_scc1 .LBB1_7 ; GLOBALNESS1-NEXT: ; %bb.6: ; %LeafBlock3 ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS1-NEXT: s_cmp_lg_u32 s39, 1 +; GLOBALNESS1-NEXT: s_cmp_lg_u32 s79, 1 ; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], -1 ; GLOBALNESS1-NEXT: s_cselect_b64 s[6:7], -1, 0 ; GLOBALNESS1-NEXT: s_cbranch_execnz .LBB1_8 @@ -178,51 +156,18 @@ ; GLOBALNESS1-NEXT: s_cbranch_vccz .LBB1_24 ; GLOBALNESS1-NEXT: .LBB1_9: ; %baz.exit.i ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[32:33], 0, 0 -; GLOBALNESS1-NEXT: flat_load_dword v0, v[32:33] -; GLOBALNESS1-NEXT: s_mov_b32 s68, s93 -; GLOBALNESS1-NEXT: s_mov_b32 s70, s93 -; GLOBALNESS1-NEXT: s_mov_b32 s71, s69 -; GLOBALNESS1-NEXT: s_mov_b32 s72, s93 -; GLOBALNESS1-NEXT: s_mov_b32 s73, s69 -; GLOBALNESS1-NEXT: s_mov_b32 s74, s93 -; GLOBALNESS1-NEXT: s_mov_b32 s75, s69 -; GLOBALNESS1-NEXT: s_mov_b32 s76, s93 -; GLOBALNESS1-NEXT: s_mov_b32 s77, s69 -; GLOBALNESS1-NEXT: s_mov_b32 s78, s93 -; GLOBALNESS1-NEXT: s_mov_b32 s79, s69 -; GLOBALNESS1-NEXT: s_mov_b32 s80, s93 -; GLOBALNESS1-NEXT: s_mov_b32 s81, s69 -; GLOBALNESS1-NEXT: s_mov_b32 s82, s93 -; GLOBALNESS1-NEXT: s_mov_b32 s83, s69 -; GLOBALNESS1-NEXT: s_mov_b32 s84, s93 -; GLOBALNESS1-NEXT: s_mov_b32 s85, s69 +; GLOBALNESS1-NEXT: v_pk_mov_b32 v[2:3], 0, 0 +; GLOBALNESS1-NEXT: flat_load_dword v0, v[2:3] ; GLOBALNESS1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GLOBALNESS1-NEXT: v_cmp_gt_i32_e64 s[96:97], 0, v0 -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[0:1], s[68:69], s[68:69] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[2:3], s[70:71], s[70:71] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[4:5], s[72:73], s[72:73] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[6:7], s[74:75], s[74:75] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[8:9], s[76:77], s[76:77] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[10:11], s[78:79], s[78:79] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[12:13], s[80:81], s[80:81] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[14:15], s[82:83], s[82:83] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[16:17], s[84:85], s[84:85] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[18:19], s[86:87], s[86:87] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[20:21], s[88:89], s[88:89] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[22:23], s[90:91], s[90:91] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[24:25], s[92:93], s[92:93] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[26:27], s[94:95], s[94:95] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[28:29], s[96:97], s[96:97] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[30:31], s[98:99], s[98:99] op_sel:[0,1] -; GLOBALNESS1-NEXT: s_and_saveexec_b64 s[70:71], s[96:97] +; GLOBALNESS1-NEXT: v_cmp_gt_i32_e64 s[62:63], 0, v0 +; GLOBALNESS1-NEXT: v_mov_b32_e32 v0, 0 +; GLOBALNESS1-NEXT: v_mov_b32_e32 v1, 0x3ff00000 +; GLOBALNESS1-NEXT: s_and_saveexec_b64 s[80:81], s[62:63] ; GLOBALNESS1-NEXT: s_cbranch_execz .LBB1_26 ; GLOBALNESS1-NEXT: ; %bb.10: ; %bb33.i ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS1-NEXT: global_load_dwordx2 v[0:1], v[32:33], off -; GLOBALNESS1-NEXT: v_readlane_b32 s4, v42, 0 -; GLOBALNESS1-NEXT: v_readlane_b32 s5, v42, 1 -; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[4:5] +; GLOBALNESS1-NEXT: global_load_dwordx2 v[0:1], v[2:3], off +; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[54:55] ; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_12 ; GLOBALNESS1-NEXT: ; %bb.11: ; %bb39.i ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1 @@ -231,166 +176,123 @@ ; GLOBALNESS1-NEXT: global_store_dwordx2 v[2:3], v[40:41], off ; GLOBALNESS1-NEXT: .LBB1_12: ; %bb44.lr.ph.i ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS1-NEXT: v_cmp_ne_u32_e32 vcc, 0, v45 -; GLOBALNESS1-NEXT: v_cndmask_b32_e32 v2, 0, v44, vcc -; GLOBALNESS1-NEXT: s_mov_b64 s[72:73], s[42:43] -; GLOBALNESS1-NEXT: s_mov_b32 s75, s39 +; GLOBALNESS1-NEXT: v_cmp_ne_u32_e32 vcc, 0, v46 +; GLOBALNESS1-NEXT: v_cndmask_b32_e32 v2, 0, v43, vcc ; GLOBALNESS1-NEXT: s_waitcnt vmcnt(0) -; GLOBALNESS1-NEXT: v_cmp_nlt_f64_e64 s[56:57], 0, v[0:1] -; GLOBALNESS1-NEXT: v_cmp_eq_u32_e64 s[58:59], 0, v2 +; GLOBALNESS1-NEXT: v_cmp_nlt_f64_e64 s[64:65], 0, v[0:1] +; GLOBALNESS1-NEXT: v_cmp_eq_u32_e64 s[66:67], 0, v2 ; GLOBALNESS1-NEXT: s_branch .LBB1_15 ; GLOBALNESS1-NEXT: .LBB1_13: ; %Flow7 ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_15 Depth=2 ; GLOBALNESS1-NEXT: s_or_b64 exec, exec, s[4:5] ; GLOBALNESS1-NEXT: .LBB1_14: ; %bb63.i ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_15 Depth=2 -; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[86:87] +; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[52:53] ; GLOBALNESS1-NEXT: s_cbranch_vccz .LBB1_25 ; GLOBALNESS1-NEXT: .LBB1_15: ; %bb44.i ; GLOBALNESS1-NEXT: ; Parent Loop BB1_4 Depth=1 ; GLOBALNESS1-NEXT: ; => This Inner Loop Header: Depth=2 -; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[94:95] +; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[48:49] ; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_14 ; GLOBALNESS1-NEXT: ; %bb.16: ; %bb46.i ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_15 Depth=2 -; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[88:89] +; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[50:51] ; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_14 ; GLOBALNESS1-NEXT: ; %bb.17: ; %bb50.i ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_15 Depth=2 -; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[36:37] +; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[42:43] ; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_20 ; GLOBALNESS1-NEXT: ; %bb.18: ; %bb3.i.i ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_15 Depth=2 -; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[40:41] +; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[44:45] ; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_20 ; GLOBALNESS1-NEXT: ; %bb.19: ; %bb6.i.i ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_15 Depth=2 -; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[56:57] +; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[64:65] ; GLOBALNESS1-NEXT: .LBB1_20: ; %spam.exit.i ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_15 Depth=2 -; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[90:91] +; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[56:57] ; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_14 ; GLOBALNESS1-NEXT: ; %bb.21: ; %bb55.i ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_15 Depth=2 -; GLOBALNESS1-NEXT: s_add_u32 s60, s62, 40 -; GLOBALNESS1-NEXT: s_addc_u32 s61, s63, 0 -; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], s[64:65] -; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], s[54:55] -; GLOBALNESS1-NEXT: s_mov_b64 s[8:9], s[60:61] +; GLOBALNESS1-NEXT: s_add_u32 s68, s38, 40 +; GLOBALNESS1-NEXT: s_addc_u32 s69, s39, 0 +; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], s[40:41] +; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], s[36:37] +; GLOBALNESS1-NEXT: s_mov_b64 s[8:9], s[68:69] ; GLOBALNESS1-NEXT: s_mov_b64 s[10:11], s[34:35] -; GLOBALNESS1-NEXT: s_mov_b32 s12, s100 -; GLOBALNESS1-NEXT: s_mov_b32 s13, s99 -; GLOBALNESS1-NEXT: s_mov_b32 s14, s98 -; GLOBALNESS1-NEXT: v_mov_b32_e32 v31, v43 -; GLOBALNESS1-NEXT: s_swappc_b64 s[30:31], s[66:67] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[44:45], 0, 0 -; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], s[64:65] -; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], s[54:55] -; GLOBALNESS1-NEXT: s_mov_b64 s[8:9], s[60:61] +; GLOBALNESS1-NEXT: s_mov_b32 s12, s72 +; GLOBALNESS1-NEXT: s_mov_b32 s13, s71 +; GLOBALNESS1-NEXT: s_mov_b32 s14, s70 +; GLOBALNESS1-NEXT: v_mov_b32_e32 v31, v42 +; GLOBALNESS1-NEXT: s_swappc_b64 s[30:31], s[76:77] +; GLOBALNESS1-NEXT: v_pk_mov_b32 v[46:47], 0, 0 +; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], s[40:41] +; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], s[36:37] +; GLOBALNESS1-NEXT: s_mov_b64 s[8:9], s[68:69] ; GLOBALNESS1-NEXT: s_mov_b64 s[10:11], s[34:35] -; GLOBALNESS1-NEXT: s_mov_b32 s12, s100 -; GLOBALNESS1-NEXT: s_mov_b32 s13, s99 -; GLOBALNESS1-NEXT: s_mov_b32 s14, s98 -; GLOBALNESS1-NEXT: v_mov_b32_e32 v31, v43 -; GLOBALNESS1-NEXT: global_store_dwordx2 v[44:45], a[32:33], off -; GLOBALNESS1-NEXT: s_swappc_b64 s[30:31], s[66:67] -; GLOBALNESS1-NEXT: s_and_saveexec_b64 s[4:5], s[58:59] +; GLOBALNESS1-NEXT: s_mov_b32 s12, s72 +; GLOBALNESS1-NEXT: s_mov_b32 s13, s71 +; GLOBALNESS1-NEXT: s_mov_b32 s14, s70 +; GLOBALNESS1-NEXT: v_mov_b32_e32 v31, v42 +; GLOBALNESS1-NEXT: global_store_dwordx2 v[46:47], v[44:45], off +; GLOBALNESS1-NEXT: s_swappc_b64 s[30:31], s[76:77] +; GLOBALNESS1-NEXT: s_and_saveexec_b64 s[4:5], s[66:67] ; GLOBALNESS1-NEXT: s_cbranch_execz .LBB1_13 ; GLOBALNESS1-NEXT: ; %bb.22: ; %bb62.i ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_15 Depth=2 ; GLOBALNESS1-NEXT: v_mov_b32_e32 v41, v40 -; GLOBALNESS1-NEXT: global_store_dwordx2 v[44:45], v[40:41], off +; GLOBALNESS1-NEXT: global_store_dwordx2 v[46:47], v[40:41], off ; GLOBALNESS1-NEXT: s_branch .LBB1_13 ; GLOBALNESS1-NEXT: .LBB1_23: ; %LeafBlock ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS1-NEXT: s_cmp_lg_u32 s39, 0 +; GLOBALNESS1-NEXT: s_cmp_lg_u32 s79, 0 ; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], 0 ; GLOBALNESS1-NEXT: s_cselect_b64 s[6:7], -1, 0 ; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[6:7] ; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_9 ; GLOBALNESS1-NEXT: .LBB1_24: ; in Loop: Header=BB1_4 Depth=1 ; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], -1 -; GLOBALNESS1-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 +; GLOBALNESS1-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GLOBALNESS1-NEXT: s_branch .LBB1_3 ; GLOBALNESS1-NEXT: .LBB1_25: ; %Flow14 ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], s[36:37] -; GLOBALNESS1-NEXT: s_mov_b32 s36, s93 -; GLOBALNESS1-NEXT: s_mov_b32 s37, s93 -; GLOBALNESS1-NEXT: s_mov_b32 s38, s93 -; GLOBALNESS1-NEXT: s_mov_b32 s39, s93 -; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], s[40:41] -; GLOBALNESS1-NEXT: s_mov_b32 s40, s93 -; GLOBALNESS1-NEXT: s_mov_b32 s41, s93 -; GLOBALNESS1-NEXT: s_mov_b32 s42, s93 -; GLOBALNESS1-NEXT: s_mov_b32 s43, s93 -; GLOBALNESS1-NEXT: s_mov_b32 s44, s93 -; GLOBALNESS1-NEXT: s_mov_b32 s45, s93 -; GLOBALNESS1-NEXT: s_mov_b32 s46, s93 -; GLOBALNESS1-NEXT: s_mov_b32 s47, s93 -; GLOBALNESS1-NEXT: s_mov_b32 s48, s93 -; GLOBALNESS1-NEXT: s_mov_b32 s49, s93 -; GLOBALNESS1-NEXT: s_mov_b32 s50, s93 -; GLOBALNESS1-NEXT: s_mov_b32 s51, s93 -; GLOBALNESS1-NEXT: s_mov_b32 s52, s93 -; GLOBALNESS1-NEXT: s_mov_b32 s53, s93 -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[0:1], s[36:37], s[36:37] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[2:3], s[38:39], s[38:39] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[4:5], s[40:41], s[40:41] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[6:7], s[42:43], s[42:43] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[8:9], s[44:45], s[44:45] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[10:11], s[46:47], s[46:47] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[12:13], s[48:49], s[48:49] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[14:15], s[50:51], s[50:51] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[16:17], s[52:53], s[52:53] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[18:19], s[54:55], s[54:55] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[20:21], s[56:57], s[56:57] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[22:23], s[58:59], s[58:59] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[24:25], s[60:61], s[60:61] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[26:27], s[62:63], s[62:63] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[28:29], s[64:65], s[64:65] op_sel:[0,1] -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[30:31], s[66:67], s[66:67] op_sel:[0,1] -; GLOBALNESS1-NEXT: s_mov_b64 s[40:41], s[6:7] -; GLOBALNESS1-NEXT: s_mov_b64 s[36:37], s[4:5] -; GLOBALNESS1-NEXT: s_mov_b32 s39, s75 -; GLOBALNESS1-NEXT: s_mov_b64 s[42:43], s[72:73] +; GLOBALNESS1-NEXT: v_pk_mov_b32 v[0:1], 0, 0 ; GLOBALNESS1-NEXT: .LBB1_26: ; %Flow15 ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS1-NEXT: s_or_b64 exec, exec, s[70:71] -; GLOBALNESS1-NEXT: s_and_saveexec_b64 s[4:5], s[96:97] +; GLOBALNESS1-NEXT: s_or_b64 exec, exec, s[80:81] +; GLOBALNESS1-NEXT: s_and_saveexec_b64 s[4:5], s[62:63] ; GLOBALNESS1-NEXT: s_cbranch_execz .LBB1_2 ; GLOBALNESS1-NEXT: ; %bb.27: ; %bb67.i ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS1-NEXT: v_readlane_b32 s6, v42, 2 -; GLOBALNESS1-NEXT: v_readlane_b32 s7, v42, 3 -; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[6:7] +; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[58:59] ; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_1 ; GLOBALNESS1-NEXT: ; %bb.28: ; %bb69.i ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1 ; GLOBALNESS1-NEXT: v_mov_b32_e32 v41, v40 -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[32:33], 0, 0 -; GLOBALNESS1-NEXT: global_store_dwordx2 v[32:33], v[40:41], off +; GLOBALNESS1-NEXT: v_pk_mov_b32 v[2:3], 0, 0 +; GLOBALNESS1-NEXT: global_store_dwordx2 v[2:3], v[40:41], off ; GLOBALNESS1-NEXT: s_branch .LBB1_1 ; GLOBALNESS1-NEXT: .LBB1_29: ; %bb73.i ; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1 ; GLOBALNESS1-NEXT: v_mov_b32_e32 v41, v40 -; GLOBALNESS1-NEXT: v_pk_mov_b32 v[32:33], 0, 0 -; GLOBALNESS1-NEXT: global_store_dwordx2 v[32:33], v[40:41], off +; GLOBALNESS1-NEXT: v_pk_mov_b32 v[2:3], 0, 0 +; GLOBALNESS1-NEXT: global_store_dwordx2 v[2:3], v[40:41], off ; GLOBALNESS1-NEXT: s_branch .LBB1_2 ; GLOBALNESS1-NEXT: .LBB1_30: ; %loop.exit.guard ; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[4:5] ; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], -1 ; GLOBALNESS1-NEXT: s_cbranch_vccz .LBB1_32 ; GLOBALNESS1-NEXT: ; %bb.31: ; %bb7.i.i -; GLOBALNESS1-NEXT: s_add_u32 s8, s62, 40 -; GLOBALNESS1-NEXT: s_addc_u32 s9, s63, 0 -; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], s[64:65] -; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], s[54:55] +; GLOBALNESS1-NEXT: s_add_u32 s8, s38, 40 +; GLOBALNESS1-NEXT: s_addc_u32 s9, s39, 0 +; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], s[40:41] +; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], s[36:37] ; GLOBALNESS1-NEXT: s_mov_b64 s[10:11], s[34:35] -; GLOBALNESS1-NEXT: s_mov_b32 s12, s100 -; GLOBALNESS1-NEXT: s_mov_b32 s13, s99 -; GLOBALNESS1-NEXT: s_mov_b32 s14, s98 -; GLOBALNESS1-NEXT: v_mov_b32_e32 v31, v43 +; GLOBALNESS1-NEXT: s_mov_b32 s12, s72 +; GLOBALNESS1-NEXT: s_mov_b32 s13, s71 +; GLOBALNESS1-NEXT: s_mov_b32 s14, s70 +; GLOBALNESS1-NEXT: v_mov_b32_e32 v31, v42 ; GLOBALNESS1-NEXT: s_getpc_b64 s[16:17] ; GLOBALNESS1-NEXT: s_add_u32 s16, s16, widget@rel32@lo+4 ; GLOBALNESS1-NEXT: s_addc_u32 s17, s17, widget@rel32@hi+12 @@ -400,15 +302,15 @@ ; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[4:5] ; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_34 ; GLOBALNESS1-NEXT: ; %bb.33: ; %bb11.i.i -; GLOBALNESS1-NEXT: s_add_u32 s8, s62, 40 -; GLOBALNESS1-NEXT: s_addc_u32 s9, s63, 0 -; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], s[64:65] -; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], s[54:55] +; GLOBALNESS1-NEXT: s_add_u32 s8, s38, 40 +; GLOBALNESS1-NEXT: s_addc_u32 s9, s39, 0 +; GLOBALNESS1-NEXT: s_mov_b64 s[4:5], s[40:41] +; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], s[36:37] ; GLOBALNESS1-NEXT: s_mov_b64 s[10:11], s[34:35] -; GLOBALNESS1-NEXT: s_mov_b32 s12, s100 -; GLOBALNESS1-NEXT: s_mov_b32 s13, s99 -; GLOBALNESS1-NEXT: s_mov_b32 s14, s98 -; GLOBALNESS1-NEXT: v_mov_b32_e32 v31, v43 +; GLOBALNESS1-NEXT: s_mov_b32 s12, s72 +; GLOBALNESS1-NEXT: s_mov_b32 s13, s71 +; GLOBALNESS1-NEXT: s_mov_b32 s14, s70 +; GLOBALNESS1-NEXT: v_mov_b32_e32 v31, v42 ; GLOBALNESS1-NEXT: s_getpc_b64 s[16:17] ; GLOBALNESS1-NEXT: s_add_u32 s16, s16, widget@rel32@lo+4 ; GLOBALNESS1-NEXT: s_addc_u32 s17, s17, widget@rel32@hi+12 @@ -417,67 +319,76 @@ ; ; GLOBALNESS0-LABEL: kernel: ; GLOBALNESS0: ; %bb.0: ; %bb -; GLOBALNESS0-NEXT: s_mov_b64 s[54:55], s[6:7] -; GLOBALNESS0-NEXT: s_load_dwordx4 s[36:39], s[8:9], 0x0 +; GLOBALNESS0-NEXT: s_mov_b64 s[36:37], s[6:7] +; GLOBALNESS0-NEXT: s_load_dwordx4 s[72:75], s[8:9], 0x0 ; GLOBALNESS0-NEXT: s_load_dword s6, s[8:9], 0x14 -; GLOBALNESS0-NEXT: v_mov_b32_e32 v43, v0 +; GLOBALNESS0-NEXT: v_mov_b32_e32 v42, v0 ; GLOBALNESS0-NEXT: v_mov_b32_e32 v40, 0 ; GLOBALNESS0-NEXT: v_pk_mov_b32 v[0:1], 0, 0 ; GLOBALNESS0-NEXT: global_store_dword v[0:1], v40, off ; GLOBALNESS0-NEXT: s_waitcnt lgkmcnt(0) -; GLOBALNESS0-NEXT: global_load_dword v0, v40, s[36:37] -; GLOBALNESS0-NEXT: s_add_u32 flat_scratch_lo, s12, s17 -; GLOBALNESS0-NEXT: s_mov_b64 s[62:63], s[4:5] +; GLOBALNESS0-NEXT: global_load_dword v0, v40, s[72:73] +; GLOBALNESS0-NEXT: s_mov_b64 s[40:41], s[4:5] ; GLOBALNESS0-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x18 ; GLOBALNESS0-NEXT: s_load_dword s7, s[8:9], 0x20 +; GLOBALNESS0-NEXT: s_add_u32 flat_scratch_lo, s12, s17 ; GLOBALNESS0-NEXT: s_addc_u32 flat_scratch_hi, s13, 0 ; GLOBALNESS0-NEXT: s_add_u32 s0, s0, s17 ; GLOBALNESS0-NEXT: s_addc_u32 s1, s1, 0 ; GLOBALNESS0-NEXT: v_mov_b32_e32 v41, 0x40994400 -; GLOBALNESS0-NEXT: s_bitcmp1_b32 s38, 0 +; GLOBALNESS0-NEXT: s_bitcmp1_b32 s74, 0 ; GLOBALNESS0-NEXT: s_waitcnt lgkmcnt(0) -; GLOBALNESS0-NEXT: v_cmp_ngt_f64_e64 s[36:37], s[4:5], v[40:41] -; GLOBALNESS0-NEXT: v_cmp_ngt_f64_e64 s[40:41], s[4:5], 0 +; GLOBALNESS0-NEXT: v_cmp_ngt_f64_e32 vcc, s[4:5], v[40:41] +; GLOBALNESS0-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[4:5], 0 +; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[4:5] ; GLOBALNESS0-NEXT: s_cselect_b64 s[4:5], -1, 0 -; GLOBALNESS0-NEXT: s_xor_b64 s[94:95], s[4:5], -1 +; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v3, 0, 1, s[4:5] +; GLOBALNESS0-NEXT: s_xor_b64 s[4:5], s[4:5], -1 +; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc ; GLOBALNESS0-NEXT: s_bitcmp1_b32 s6, 0 +; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[42:43], 1, v1 ; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5] ; GLOBALNESS0-NEXT: s_cselect_b64 s[4:5], -1, 0 -; GLOBALNESS0-NEXT: s_xor_b64 s[88:89], s[4:5], -1 +; GLOBALNESS0-NEXT: s_xor_b64 s[4:5], s[4:5], -1 ; GLOBALNESS0-NEXT: s_bitcmp1_b32 s7, 0 +; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[48:49], 1, v1 +; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5] ; GLOBALNESS0-NEXT: s_cselect_b64 s[4:5], -1, 0 ; GLOBALNESS0-NEXT: s_getpc_b64 s[6:7] ; GLOBALNESS0-NEXT: s_add_u32 s6, s6, wobble@gotpcrel32@lo+4 ; GLOBALNESS0-NEXT: s_addc_u32 s7, s7, wobble@gotpcrel32@hi+12 -; GLOBALNESS0-NEXT: s_xor_b64 s[86:87], s[4:5], -1 -; GLOBALNESS0-NEXT: s_load_dwordx2 s[66:67], s[6:7], 0x0 -; GLOBALNESS0-NEXT: s_mov_b32 s98, s16 -; GLOBALNESS0-NEXT: s_mov_b64 s[60:61], s[8:9] -; GLOBALNESS0-NEXT: s_mov_b32 s99, s15 -; GLOBALNESS0-NEXT: s_mov_b32 s100, s14 +; GLOBALNESS0-NEXT: s_xor_b64 s[4:5], s[4:5], -1 +; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[50:51], 1, v1 +; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5] +; GLOBALNESS0-NEXT: s_load_dwordx2 s[78:79], s[6:7], 0x0 +; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[52:53], 1, v1 +; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[44:45], 1, v2 +; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[46:47], 1, v3 +; GLOBALNESS0-NEXT: s_mov_b32 s68, s16 +; GLOBALNESS0-NEXT: s_mov_b64 s[38:39], s[8:9] +; GLOBALNESS0-NEXT: s_mov_b32 s69, s15 +; GLOBALNESS0-NEXT: s_mov_b32 s70, s14 ; GLOBALNESS0-NEXT: s_mov_b64 s[34:35], s[10:11] -; GLOBALNESS0-NEXT: s_mov_b64 s[92:93], 0x80 -; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[42:43], 1, v1 -; GLOBALNESS0-NEXT: s_mov_b32 s69, 0x3ff00000 +; GLOBALNESS0-NEXT: s_mov_b64 s[76:77], 0x80 ; GLOBALNESS0-NEXT: s_mov_b32 s32, 0 -; GLOBALNESS0-NEXT: ; implicit-def: $agpr32_agpr33_agpr34_agpr35_agpr36_agpr37_agpr38_agpr39_agpr40_agpr41_agpr42_agpr43_agpr44_agpr45_agpr46_agpr47_agpr48_agpr49_agpr50_agpr51_agpr52_agpr53_agpr54_agpr55_agpr56_agpr57_agpr58_agpr59_agpr60_agpr61_agpr62_agpr63 +; GLOBALNESS0-NEXT: ; implicit-def: $vgpr44_vgpr45 ; GLOBALNESS0-NEXT: s_waitcnt vmcnt(0) -; GLOBALNESS0-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v0 -; GLOBALNESS0-NEXT: v_writelane_b32 v42, s4, 0 -; GLOBALNESS0-NEXT: v_writelane_b32 v42, s5, 1 -; GLOBALNESS0-NEXT: v_cmp_eq_u32_e64 s[4:5], 1, v0 -; GLOBALNESS0-NEXT: v_writelane_b32 v42, s4, 2 -; GLOBALNESS0-NEXT: v_writelane_b32 v42, s5, 3 -; GLOBALNESS0-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v0 -; GLOBALNESS0-NEXT: v_writelane_b32 v42, s4, 4 -; GLOBALNESS0-NEXT: v_cmp_gt_i32_e64 s[90:91], 1, v0 -; GLOBALNESS0-NEXT: v_writelane_b32 v42, s5, 5 +; GLOBALNESS0-NEXT: v_cmp_gt_i32_e32 vcc, 0, v0 +; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v1, 0, 1, vcc +; GLOBALNESS0-NEXT: v_cmp_gt_i32_e32 vcc, 1, v0 +; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc +; GLOBALNESS0-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0 +; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc +; GLOBALNESS0-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 +; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc +; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[54:55], 1, v1 +; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[56:57], 1, v2 +; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[58:59], 1, v3 +; GLOBALNESS0-NEXT: v_cmp_ne_u32_e64 s[60:61], 1, v0 ; GLOBALNESS0-NEXT: s_branch .LBB1_4 ; GLOBALNESS0-NEXT: .LBB1_1: ; %bb70.i ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS0-NEXT: v_readlane_b32 s6, v42, 4 -; GLOBALNESS0-NEXT: v_readlane_b32 s7, v42, 5 -; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[6:7] +; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[60:61] ; GLOBALNESS0-NEXT: s_cbranch_vccz .LBB1_29 ; GLOBALNESS0-NEXT: .LBB1_2: ; %Flow6 ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1 @@ -486,69 +397,38 @@ ; GLOBALNESS0-NEXT: ; implicit-def: $sgpr4_sgpr5 ; GLOBALNESS0-NEXT: .LBB1_3: ; %Flow19 ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a63, v31 ; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[6:7] -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a62, v30 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a61, v29 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a60, v28 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a59, v27 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a58, v26 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a57, v25 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a56, v24 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a55, v23 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a54, v22 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a53, v21 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a52, v20 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a51, v19 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a50, v18 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a49, v17 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a48, v16 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a47, v15 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a46, v14 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a45, v13 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a44, v12 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a43, v11 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a42, v10 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a41, v9 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a40, v8 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a39, v7 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a38, v6 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a37, v5 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a36, v4 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a35, v3 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a34, v2 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a33, v1 -; GLOBALNESS0-NEXT: v_accvgpr_write_b32 a32, v0 +; GLOBALNESS0-NEXT: v_pk_mov_b32 v[44:45], v[0:1], v[0:1] op_sel:[0,1] ; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_30 ; GLOBALNESS0-NEXT: .LBB1_4: ; %bb5 ; GLOBALNESS0-NEXT: ; =>This Loop Header: Depth=1 ; GLOBALNESS0-NEXT: ; Child Loop BB1_15 Depth 2 -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[0:1], s[92:93], s[92:93] op_sel:[0,1] -; GLOBALNESS0-NEXT: flat_load_dword v44, v[0:1] -; GLOBALNESS0-NEXT: s_add_u32 s8, s60, 40 +; GLOBALNESS0-NEXT: v_pk_mov_b32 v[0:1], s[76:77], s[76:77] op_sel:[0,1] +; GLOBALNESS0-NEXT: flat_load_dword v43, v[0:1] +; GLOBALNESS0-NEXT: s_add_u32 s8, s38, 40 ; GLOBALNESS0-NEXT: buffer_store_dword v40, off, s[0:3], 0 -; GLOBALNESS0-NEXT: flat_load_dword v45, v[0:1] -; GLOBALNESS0-NEXT: s_addc_u32 s9, s61, 0 -; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], s[62:63] -; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], s[54:55] +; GLOBALNESS0-NEXT: flat_load_dword v46, v[0:1] +; GLOBALNESS0-NEXT: s_addc_u32 s9, s39, 0 +; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], s[40:41] +; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], s[36:37] ; GLOBALNESS0-NEXT: s_mov_b64 s[10:11], s[34:35] -; GLOBALNESS0-NEXT: s_mov_b32 s12, s100 -; GLOBALNESS0-NEXT: s_mov_b32 s13, s99 -; GLOBALNESS0-NEXT: s_mov_b32 s14, s98 -; GLOBALNESS0-NEXT: v_mov_b32_e32 v31, v43 +; GLOBALNESS0-NEXT: s_mov_b32 s12, s70 +; GLOBALNESS0-NEXT: s_mov_b32 s13, s69 +; GLOBALNESS0-NEXT: s_mov_b32 s14, s68 +; GLOBALNESS0-NEXT: v_mov_b32_e32 v31, v42 ; GLOBALNESS0-NEXT: s_waitcnt lgkmcnt(0) -; GLOBALNESS0-NEXT: s_swappc_b64 s[30:31], s[66:67] -; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[42:43] +; GLOBALNESS0-NEXT: s_swappc_b64 s[30:31], s[78:79] +; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[46:47] ; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], -1 ; GLOBALNESS0-NEXT: ; implicit-def: $sgpr4_sgpr5 ; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_8 ; GLOBALNESS0-NEXT: ; %bb.5: ; %NodeBlock ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS0-NEXT: s_cmp_lt_i32 s39, 1 +; GLOBALNESS0-NEXT: s_cmp_lt_i32 s75, 1 ; GLOBALNESS0-NEXT: s_cbranch_scc1 .LBB1_7 ; GLOBALNESS0-NEXT: ; %bb.6: ; %LeafBlock3 ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS0-NEXT: s_cmp_lg_u32 s39, 1 +; GLOBALNESS0-NEXT: s_cmp_lg_u32 s75, 1 ; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], -1 ; GLOBALNESS0-NEXT: s_cselect_b64 s[6:7], -1, 0 ; GLOBALNESS0-NEXT: s_cbranch_execnz .LBB1_8 @@ -563,51 +443,18 @@ ; GLOBALNESS0-NEXT: s_cbranch_vccz .LBB1_24 ; GLOBALNESS0-NEXT: .LBB1_9: ; %baz.exit.i ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[32:33], 0, 0 -; GLOBALNESS0-NEXT: flat_load_dword v0, v[32:33] -; GLOBALNESS0-NEXT: s_mov_b32 s68, s93 -; GLOBALNESS0-NEXT: s_mov_b32 s70, s93 -; GLOBALNESS0-NEXT: s_mov_b32 s71, s69 -; GLOBALNESS0-NEXT: s_mov_b32 s72, s93 -; GLOBALNESS0-NEXT: s_mov_b32 s73, s69 -; GLOBALNESS0-NEXT: s_mov_b32 s74, s93 -; GLOBALNESS0-NEXT: s_mov_b32 s75, s69 -; GLOBALNESS0-NEXT: s_mov_b32 s76, s93 -; GLOBALNESS0-NEXT: s_mov_b32 s77, s69 -; GLOBALNESS0-NEXT: s_mov_b32 s78, s93 -; GLOBALNESS0-NEXT: s_mov_b32 s79, s69 -; GLOBALNESS0-NEXT: s_mov_b32 s80, s93 -; GLOBALNESS0-NEXT: s_mov_b32 s81, s69 -; GLOBALNESS0-NEXT: s_mov_b32 s82, s93 -; GLOBALNESS0-NEXT: s_mov_b32 s83, s69 -; GLOBALNESS0-NEXT: s_mov_b32 s84, s93 -; GLOBALNESS0-NEXT: s_mov_b32 s85, s69 +; GLOBALNESS0-NEXT: v_pk_mov_b32 v[2:3], 0, 0 +; GLOBALNESS0-NEXT: flat_load_dword v0, v[2:3] ; GLOBALNESS0-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) -; GLOBALNESS0-NEXT: v_cmp_gt_i32_e64 s[96:97], 0, v0 -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[0:1], s[68:69], s[68:69] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[2:3], s[70:71], s[70:71] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[4:5], s[72:73], s[72:73] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[6:7], s[74:75], s[74:75] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[8:9], s[76:77], s[76:77] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[10:11], s[78:79], s[78:79] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[12:13], s[80:81], s[80:81] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[14:15], s[82:83], s[82:83] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[16:17], s[84:85], s[84:85] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[18:19], s[86:87], s[86:87] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[20:21], s[88:89], s[88:89] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[22:23], s[90:91], s[90:91] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[24:25], s[92:93], s[92:93] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[26:27], s[94:95], s[94:95] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[28:29], s[96:97], s[96:97] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[30:31], s[98:99], s[98:99] op_sel:[0,1] -; GLOBALNESS0-NEXT: s_and_saveexec_b64 s[70:71], s[96:97] +; GLOBALNESS0-NEXT: v_cmp_gt_i32_e64 s[62:63], 0, v0 +; GLOBALNESS0-NEXT: v_mov_b32_e32 v0, 0 +; GLOBALNESS0-NEXT: v_mov_b32_e32 v1, 0x3ff00000 +; GLOBALNESS0-NEXT: s_and_saveexec_b64 s[80:81], s[62:63] ; GLOBALNESS0-NEXT: s_cbranch_execz .LBB1_26 ; GLOBALNESS0-NEXT: ; %bb.10: ; %bb33.i ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS0-NEXT: global_load_dwordx2 v[0:1], v[32:33], off -; GLOBALNESS0-NEXT: v_readlane_b32 s4, v42, 0 -; GLOBALNESS0-NEXT: v_readlane_b32 s5, v42, 1 -; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[4:5] +; GLOBALNESS0-NEXT: global_load_dwordx2 v[0:1], v[2:3], off +; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[54:55] ; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_12 ; GLOBALNESS0-NEXT: ; %bb.11: ; %bb39.i ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1 @@ -616,166 +463,123 @@ ; GLOBALNESS0-NEXT: global_store_dwordx2 v[2:3], v[40:41], off ; GLOBALNESS0-NEXT: .LBB1_12: ; %bb44.lr.ph.i ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS0-NEXT: v_cmp_ne_u32_e32 vcc, 0, v45 -; GLOBALNESS0-NEXT: v_cndmask_b32_e32 v2, 0, v44, vcc -; GLOBALNESS0-NEXT: s_mov_b64 s[72:73], s[42:43] -; GLOBALNESS0-NEXT: s_mov_b32 s75, s39 +; GLOBALNESS0-NEXT: v_cmp_ne_u32_e32 vcc, 0, v46 +; GLOBALNESS0-NEXT: v_cndmask_b32_e32 v2, 0, v43, vcc ; GLOBALNESS0-NEXT: s_waitcnt vmcnt(0) -; GLOBALNESS0-NEXT: v_cmp_nlt_f64_e64 s[56:57], 0, v[0:1] -; GLOBALNESS0-NEXT: v_cmp_eq_u32_e64 s[58:59], 0, v2 +; GLOBALNESS0-NEXT: v_cmp_nlt_f64_e64 s[64:65], 0, v[0:1] +; GLOBALNESS0-NEXT: v_cmp_eq_u32_e64 s[66:67], 0, v2 ; GLOBALNESS0-NEXT: s_branch .LBB1_15 ; GLOBALNESS0-NEXT: .LBB1_13: ; %Flow7 ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_15 Depth=2 ; GLOBALNESS0-NEXT: s_or_b64 exec, exec, s[4:5] ; GLOBALNESS0-NEXT: .LBB1_14: ; %bb63.i ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_15 Depth=2 -; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[86:87] +; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[52:53] ; GLOBALNESS0-NEXT: s_cbranch_vccz .LBB1_25 ; GLOBALNESS0-NEXT: .LBB1_15: ; %bb44.i ; GLOBALNESS0-NEXT: ; Parent Loop BB1_4 Depth=1 ; GLOBALNESS0-NEXT: ; => This Inner Loop Header: Depth=2 -; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[94:95] +; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[48:49] ; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_14 ; GLOBALNESS0-NEXT: ; %bb.16: ; %bb46.i ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_15 Depth=2 -; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[88:89] +; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[50:51] ; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_14 ; GLOBALNESS0-NEXT: ; %bb.17: ; %bb50.i ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_15 Depth=2 -; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[36:37] +; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[42:43] ; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_20 ; GLOBALNESS0-NEXT: ; %bb.18: ; %bb3.i.i ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_15 Depth=2 -; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[40:41] +; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[44:45] ; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_20 ; GLOBALNESS0-NEXT: ; %bb.19: ; %bb6.i.i ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_15 Depth=2 -; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[56:57] +; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[64:65] ; GLOBALNESS0-NEXT: .LBB1_20: ; %spam.exit.i ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_15 Depth=2 -; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[90:91] +; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[56:57] ; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_14 ; GLOBALNESS0-NEXT: ; %bb.21: ; %bb55.i ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_15 Depth=2 -; GLOBALNESS0-NEXT: s_add_u32 s64, s60, 40 -; GLOBALNESS0-NEXT: s_addc_u32 s65, s61, 0 -; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], s[62:63] -; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], s[54:55] -; GLOBALNESS0-NEXT: s_mov_b64 s[8:9], s[64:65] +; GLOBALNESS0-NEXT: s_add_u32 s72, s38, 40 +; GLOBALNESS0-NEXT: s_addc_u32 s73, s39, 0 +; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], s[40:41] +; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], s[36:37] +; GLOBALNESS0-NEXT: s_mov_b64 s[8:9], s[72:73] ; GLOBALNESS0-NEXT: s_mov_b64 s[10:11], s[34:35] -; GLOBALNESS0-NEXT: s_mov_b32 s12, s100 -; GLOBALNESS0-NEXT: s_mov_b32 s13, s99 -; GLOBALNESS0-NEXT: s_mov_b32 s14, s98 -; GLOBALNESS0-NEXT: v_mov_b32_e32 v31, v43 -; GLOBALNESS0-NEXT: s_swappc_b64 s[30:31], s[66:67] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[44:45], 0, 0 -; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], s[62:63] -; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], s[54:55] -; GLOBALNESS0-NEXT: s_mov_b64 s[8:9], s[64:65] +; GLOBALNESS0-NEXT: s_mov_b32 s12, s70 +; GLOBALNESS0-NEXT: s_mov_b32 s13, s69 +; GLOBALNESS0-NEXT: s_mov_b32 s14, s68 +; GLOBALNESS0-NEXT: v_mov_b32_e32 v31, v42 +; GLOBALNESS0-NEXT: s_swappc_b64 s[30:31], s[78:79] +; GLOBALNESS0-NEXT: v_pk_mov_b32 v[46:47], 0, 0 +; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], s[40:41] +; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], s[36:37] +; GLOBALNESS0-NEXT: s_mov_b64 s[8:9], s[72:73] ; GLOBALNESS0-NEXT: s_mov_b64 s[10:11], s[34:35] -; GLOBALNESS0-NEXT: s_mov_b32 s12, s100 -; GLOBALNESS0-NEXT: s_mov_b32 s13, s99 -; GLOBALNESS0-NEXT: s_mov_b32 s14, s98 -; GLOBALNESS0-NEXT: v_mov_b32_e32 v31, v43 -; GLOBALNESS0-NEXT: global_store_dwordx2 v[44:45], a[32:33], off -; GLOBALNESS0-NEXT: s_swappc_b64 s[30:31], s[66:67] -; GLOBALNESS0-NEXT: s_and_saveexec_b64 s[4:5], s[58:59] +; GLOBALNESS0-NEXT: s_mov_b32 s12, s70 +; GLOBALNESS0-NEXT: s_mov_b32 s13, s69 +; GLOBALNESS0-NEXT: s_mov_b32 s14, s68 +; GLOBALNESS0-NEXT: v_mov_b32_e32 v31, v42 +; GLOBALNESS0-NEXT: global_store_dwordx2 v[46:47], v[44:45], off +; GLOBALNESS0-NEXT: s_swappc_b64 s[30:31], s[78:79] +; GLOBALNESS0-NEXT: s_and_saveexec_b64 s[4:5], s[66:67] ; GLOBALNESS0-NEXT: s_cbranch_execz .LBB1_13 ; GLOBALNESS0-NEXT: ; %bb.22: ; %bb62.i ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_15 Depth=2 ; GLOBALNESS0-NEXT: v_mov_b32_e32 v41, v40 -; GLOBALNESS0-NEXT: global_store_dwordx2 v[44:45], v[40:41], off +; GLOBALNESS0-NEXT: global_store_dwordx2 v[46:47], v[40:41], off ; GLOBALNESS0-NEXT: s_branch .LBB1_13 ; GLOBALNESS0-NEXT: .LBB1_23: ; %LeafBlock ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS0-NEXT: s_cmp_lg_u32 s39, 0 +; GLOBALNESS0-NEXT: s_cmp_lg_u32 s75, 0 ; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], 0 ; GLOBALNESS0-NEXT: s_cselect_b64 s[6:7], -1, 0 ; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[6:7] ; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_9 ; GLOBALNESS0-NEXT: .LBB1_24: ; in Loop: Header=BB1_4 Depth=1 ; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], -1 -; GLOBALNESS0-NEXT: ; implicit-def: $vgpr0_vgpr1_vgpr2_vgpr3_vgpr4_vgpr5_vgpr6_vgpr7_vgpr8_vgpr9_vgpr10_vgpr11_vgpr12_vgpr13_vgpr14_vgpr15_vgpr16_vgpr17_vgpr18_vgpr19_vgpr20_vgpr21_vgpr22_vgpr23_vgpr24_vgpr25_vgpr26_vgpr27_vgpr28_vgpr29_vgpr30_vgpr31 +; GLOBALNESS0-NEXT: ; implicit-def: $vgpr0_vgpr1 ; GLOBALNESS0-NEXT: s_branch .LBB1_3 ; GLOBALNESS0-NEXT: .LBB1_25: ; %Flow14 ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], s[36:37] -; GLOBALNESS0-NEXT: s_mov_b32 s36, s93 -; GLOBALNESS0-NEXT: s_mov_b32 s37, s93 -; GLOBALNESS0-NEXT: s_mov_b32 s38, s93 -; GLOBALNESS0-NEXT: s_mov_b32 s39, s93 -; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], s[40:41] -; GLOBALNESS0-NEXT: s_mov_b32 s40, s93 -; GLOBALNESS0-NEXT: s_mov_b32 s41, s93 -; GLOBALNESS0-NEXT: s_mov_b32 s42, s93 -; GLOBALNESS0-NEXT: s_mov_b32 s43, s93 -; GLOBALNESS0-NEXT: s_mov_b32 s44, s93 -; GLOBALNESS0-NEXT: s_mov_b32 s45, s93 -; GLOBALNESS0-NEXT: s_mov_b32 s46, s93 -; GLOBALNESS0-NEXT: s_mov_b32 s47, s93 -; GLOBALNESS0-NEXT: s_mov_b32 s48, s93 -; GLOBALNESS0-NEXT: s_mov_b32 s49, s93 -; GLOBALNESS0-NEXT: s_mov_b32 s50, s93 -; GLOBALNESS0-NEXT: s_mov_b32 s51, s93 -; GLOBALNESS0-NEXT: s_mov_b32 s52, s93 -; GLOBALNESS0-NEXT: s_mov_b32 s53, s93 -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[0:1], s[36:37], s[36:37] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[2:3], s[38:39], s[38:39] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[4:5], s[40:41], s[40:41] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[6:7], s[42:43], s[42:43] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[8:9], s[44:45], s[44:45] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[10:11], s[46:47], s[46:47] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[12:13], s[48:49], s[48:49] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[14:15], s[50:51], s[50:51] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[16:17], s[52:53], s[52:53] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[18:19], s[54:55], s[54:55] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[20:21], s[56:57], s[56:57] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[22:23], s[58:59], s[58:59] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[24:25], s[60:61], s[60:61] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[26:27], s[62:63], s[62:63] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[28:29], s[64:65], s[64:65] op_sel:[0,1] -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[30:31], s[66:67], s[66:67] op_sel:[0,1] -; GLOBALNESS0-NEXT: s_mov_b64 s[40:41], s[6:7] -; GLOBALNESS0-NEXT: s_mov_b64 s[36:37], s[4:5] -; GLOBALNESS0-NEXT: s_mov_b32 s39, s75 -; GLOBALNESS0-NEXT: s_mov_b64 s[42:43], s[72:73] +; GLOBALNESS0-NEXT: v_pk_mov_b32 v[0:1], 0, 0 ; GLOBALNESS0-NEXT: .LBB1_26: ; %Flow15 ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS0-NEXT: s_or_b64 exec, exec, s[70:71] -; GLOBALNESS0-NEXT: s_and_saveexec_b64 s[4:5], s[96:97] +; GLOBALNESS0-NEXT: s_or_b64 exec, exec, s[80:81] +; GLOBALNESS0-NEXT: s_and_saveexec_b64 s[4:5], s[62:63] ; GLOBALNESS0-NEXT: s_cbranch_execz .LBB1_2 ; GLOBALNESS0-NEXT: ; %bb.27: ; %bb67.i ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1 -; GLOBALNESS0-NEXT: v_readlane_b32 s6, v42, 2 -; GLOBALNESS0-NEXT: v_readlane_b32 s7, v42, 3 -; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[6:7] +; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[58:59] ; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_1 ; GLOBALNESS0-NEXT: ; %bb.28: ; %bb69.i ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1 ; GLOBALNESS0-NEXT: v_mov_b32_e32 v41, v40 -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[32:33], 0, 0 -; GLOBALNESS0-NEXT: global_store_dwordx2 v[32:33], v[40:41], off +; GLOBALNESS0-NEXT: v_pk_mov_b32 v[2:3], 0, 0 +; GLOBALNESS0-NEXT: global_store_dwordx2 v[2:3], v[40:41], off ; GLOBALNESS0-NEXT: s_branch .LBB1_1 ; GLOBALNESS0-NEXT: .LBB1_29: ; %bb73.i ; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1 ; GLOBALNESS0-NEXT: v_mov_b32_e32 v41, v40 -; GLOBALNESS0-NEXT: v_pk_mov_b32 v[32:33], 0, 0 -; GLOBALNESS0-NEXT: global_store_dwordx2 v[32:33], v[40:41], off +; GLOBALNESS0-NEXT: v_pk_mov_b32 v[2:3], 0, 0 +; GLOBALNESS0-NEXT: global_store_dwordx2 v[2:3], v[40:41], off ; GLOBALNESS0-NEXT: s_branch .LBB1_2 ; GLOBALNESS0-NEXT: .LBB1_30: ; %loop.exit.guard ; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[4:5] ; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], -1 ; GLOBALNESS0-NEXT: s_cbranch_vccz .LBB1_32 ; GLOBALNESS0-NEXT: ; %bb.31: ; %bb7.i.i -; GLOBALNESS0-NEXT: s_add_u32 s8, s60, 40 -; GLOBALNESS0-NEXT: s_addc_u32 s9, s61, 0 -; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], s[62:63] -; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], s[54:55] +; GLOBALNESS0-NEXT: s_add_u32 s8, s38, 40 +; GLOBALNESS0-NEXT: s_addc_u32 s9, s39, 0 +; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], s[40:41] +; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], s[36:37] ; GLOBALNESS0-NEXT: s_mov_b64 s[10:11], s[34:35] -; GLOBALNESS0-NEXT: s_mov_b32 s12, s100 -; GLOBALNESS0-NEXT: s_mov_b32 s13, s99 -; GLOBALNESS0-NEXT: s_mov_b32 s14, s98 -; GLOBALNESS0-NEXT: v_mov_b32_e32 v31, v43 +; GLOBALNESS0-NEXT: s_mov_b32 s12, s70 +; GLOBALNESS0-NEXT: s_mov_b32 s13, s69 +; GLOBALNESS0-NEXT: s_mov_b32 s14, s68 +; GLOBALNESS0-NEXT: v_mov_b32_e32 v31, v42 ; GLOBALNESS0-NEXT: s_getpc_b64 s[16:17] ; GLOBALNESS0-NEXT: s_add_u32 s16, s16, widget@rel32@lo+4 ; GLOBALNESS0-NEXT: s_addc_u32 s17, s17, widget@rel32@hi+12 @@ -785,15 +589,15 @@ ; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[4:5] ; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_34 ; GLOBALNESS0-NEXT: ; %bb.33: ; %bb11.i.i -; GLOBALNESS0-NEXT: s_add_u32 s8, s60, 40 -; GLOBALNESS0-NEXT: s_addc_u32 s9, s61, 0 -; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], s[62:63] -; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], s[54:55] +; GLOBALNESS0-NEXT: s_add_u32 s8, s38, 40 +; GLOBALNESS0-NEXT: s_addc_u32 s9, s39, 0 +; GLOBALNESS0-NEXT: s_mov_b64 s[4:5], s[40:41] +; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], s[36:37] ; GLOBALNESS0-NEXT: s_mov_b64 s[10:11], s[34:35] -; GLOBALNESS0-NEXT: s_mov_b32 s12, s100 -; GLOBALNESS0-NEXT: s_mov_b32 s13, s99 -; GLOBALNESS0-NEXT: s_mov_b32 s14, s98 -; GLOBALNESS0-NEXT: v_mov_b32_e32 v31, v43 +; GLOBALNESS0-NEXT: s_mov_b32 s12, s70 +; GLOBALNESS0-NEXT: s_mov_b32 s13, s69 +; GLOBALNESS0-NEXT: s_mov_b32 s14, s68 +; GLOBALNESS0-NEXT: v_mov_b32_e32 v31, v42 ; GLOBALNESS0-NEXT: s_getpc_b64 s[16:17] ; GLOBALNESS0-NEXT: s_add_u32 s16, s16, widget@rel32@lo+4 ; GLOBALNESS0-NEXT: s_addc_u32 s17, s17, widget@rel32@hi+12