diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp --- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp @@ -854,13 +854,6 @@ (VecTy && TTI.getLoadVectorFactor(VF, TySize, TySize / 8, VecTy) == 0)) continue; - // Make sure all the users of a vector are constant-index extracts. - if (isa(Ty) && !llvm::all_of(LI->users(), [](const User *U) { - const ExtractElementInst *EEI = dyn_cast(U); - return EEI && isa(EEI->getOperand(1)); - })) - continue; - // Save the load locations. const ChainID ID = getChainID(Ptr); LoadRefs[ID].push_back(LI); @@ -901,12 +894,6 @@ (VecTy && TTI.getStoreVectorFactor(VF, TySize, TySize / 8, VecTy) == 0)) continue; - if (isa(Ty) && !llvm::all_of(SI->users(), [](const User *U) { - const ExtractElementInst *EEI = dyn_cast(U); - return EEI && isa(EEI->getOperand(1)); - })) - continue; - // Save store location. const ChainID ID = getChainID(Ptr); StoreRefs[ID].push_back(SI); @@ -1290,52 +1277,32 @@ Builder.CreateAlignedLoad(VecTy, Bitcast, MaybeAlign(Alignment)); propagateMetadata(LI, Chain); - if (VecLoadTy) { - SmallVector InstrsToErase; - - unsigned VecWidth = VecLoadTy->getNumElements(); - for (unsigned I = 0, E = Chain.size(); I != E; ++I) { - for (auto Use : Chain[I]->users()) { - // All users of vector loads are ExtractElement instructions with - // constant indices, otherwise we would have bailed before now. - Instruction *UI = cast(Use); - unsigned Idx = cast(UI->getOperand(1))->getZExtValue(); - unsigned NewIdx = Idx + I * VecWidth; - Value *V = Builder.CreateExtractElement(LI, Builder.getInt32(NewIdx), - UI->getName()); - if (V->getType() != UI->getType()) - V = Builder.CreateBitCast(V, UI->getType()); - - // Replace the old instruction. - UI->replaceAllUsesWith(V); - InstrsToErase.push_back(UI); - } + for (unsigned I = 0, E = Chain.size(); I != E; ++I) { + Value *CV = Chain[I]; + Value *V; + if (VecLoadTy) { + // Extract a subvector using shufflevector. + unsigned VecWidth = VecLoadTy->getNumElements(); + auto Mask = + llvm::to_vector<8>(llvm::seq(I * VecWidth, (I + 1) * VecWidth)); + V = Builder.CreateShuffleVector(LI, Mask, CV->getName()); + } else { + V = Builder.CreateExtractElement(LI, Builder.getInt32(I), CV->getName()); } - // Bitcast might not be an Instruction, if the value being loaded is a - // constant. In that case, no need to reorder anything. - if (Instruction *BitcastInst = dyn_cast(Bitcast)) - reorder(BitcastInst); - - for (auto I : InstrsToErase) - I->eraseFromParent(); - } else { - for (unsigned I = 0, E = Chain.size(); I != E; ++I) { - Value *CV = Chain[I]; - Value *V = - Builder.CreateExtractElement(LI, Builder.getInt32(I), CV->getName()); - if (V->getType() != CV->getType()) { - V = Builder.CreateBitOrPointerCast(V, CV->getType()); - } - - // Replace the old instruction. - CV->replaceAllUsesWith(V); + if (V->getType() != CV->getType()) { + V = Builder.CreateBitOrPointerCast(V, CV->getType()); } - if (Instruction *BitcastInst = dyn_cast(Bitcast)) - reorder(BitcastInst); + // Replace the old instruction. + CV->replaceAllUsesWith(V); } + // Bitcast might not be an Instruction, if the value being loaded is a + // constant. In that case, no need to reorder anything. + if (Instruction *BitcastInst = dyn_cast(Bitcast)) + reorder(BitcastInst); + eraseInstructions(Chain); ++NumVectorInstructions; diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll @@ -667,69 +667,68 @@ define amdgpu_kernel void @sdivrem_v2i32(<2 x i32> addrspace(1)* %out0, <2 x i32> addrspace(1)* %out1, <2 x i32> %x, <2 x i32> %y) { ; GFX8-LABEL: sdivrem_v2i32: ; GFX8: ; %bb.0: -; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x18 -; GFX8-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x10 +; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x10 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: s_ashr_i32 s8, s0, 31 -; GFX8-NEXT: s_add_i32 s0, s0, s8 -; GFX8-NEXT: s_xor_b32 s9, s0, s8 -; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s9 -; GFX8-NEXT: s_ashr_i32 s11, s1, 31 -; GFX8-NEXT: s_add_i32 s0, s1, s11 -; GFX8-NEXT: s_sub_i32 s1, 0, s9 +; GFX8-NEXT: s_ashr_i32 s2, s10, 31 +; GFX8-NEXT: s_add_i32 s0, s10, s2 +; GFX8-NEXT: s_xor_b32 s3, s0, s2 +; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s3 +; GFX8-NEXT: s_sub_i32 s1, 0, s3 +; GFX8-NEXT: s_ashr_i32 s12, s11, 31 +; GFX8-NEXT: s_add_i32 s0, s11, s12 ; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX8-NEXT: s_xor_b32 s12, s0, s11 -; GFX8-NEXT: v_cvt_f32_u32_e32 v2, s12 -; GFX8-NEXT: s_ashr_i32 s10, s2, 31 +; GFX8-NEXT: s_xor_b32 s11, s0, s12 +; GFX8-NEXT: v_cvt_f32_u32_e32 v2, s11 +; GFX8-NEXT: s_ashr_i32 s10, s8, 31 ; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX8-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX8-NEXT: s_add_i32 s0, s2, s10 +; GFX8-NEXT: s_add_i32 s0, s8, s10 ; GFX8-NEXT: s_xor_b32 s0, s0, s10 ; GFX8-NEXT: v_rcp_iflag_f32_e32 v2, v2 ; GFX8-NEXT: v_mul_lo_u32 v1, s1, v0 -; GFX8-NEXT: s_ashr_i32 s2, s3, 31 +; GFX8-NEXT: s_sub_i32 s8, 0, s11 ; GFX8-NEXT: v_mul_hi_u32 v1, v0, v1 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v1 ; GFX8-NEXT: v_mul_hi_u32 v0, s0, v0 ; GFX8-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v2 ; GFX8-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX8-NEXT: v_mul_lo_u32 v2, v0, s9 +; GFX8-NEXT: v_mul_lo_u32 v2, v0, s3 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 1, v0 ; GFX8-NEXT: v_sub_u32_e32 v2, vcc, s0, v2 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s9, v2 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s3, v2 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX8-NEXT: v_subrev_u32_e64 v3, s[0:1], s9, v2 +; GFX8-NEXT: v_subrev_u32_e64 v3, s[0:1], s3, v2 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 1, v0 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s9, v2 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s3, v2 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX8-NEXT: v_subrev_u32_e64 v3, s[0:1], s9, v2 -; GFX8-NEXT: s_sub_i32 s0, 0, s12 +; GFX8-NEXT: v_subrev_u32_e64 v3, s[0:1], s3, v2 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc -; GFX8-NEXT: v_mul_lo_u32 v3, s0, v1 -; GFX8-NEXT: s_add_i32 s1, s3, s2 -; GFX8-NEXT: s_xor_b32 s1, s1, s2 -; GFX8-NEXT: s_xor_b32 s0, s10, s8 +; GFX8-NEXT: v_mul_lo_u32 v3, s8, v1 +; GFX8-NEXT: s_xor_b32 s0, s10, s2 +; GFX8-NEXT: s_ashr_i32 s2, s9, 31 +; GFX8-NEXT: s_add_i32 s1, s9, s2 ; GFX8-NEXT: v_mul_hi_u32 v3, v1, v3 +; GFX8-NEXT: s_xor_b32 s1, s1, s2 ; GFX8-NEXT: v_xor_b32_e32 v0, s0, v0 ; GFX8-NEXT: v_xor_b32_e32 v2, s10, v2 -; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, s0, v0 ; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v3 ; GFX8-NEXT: v_mul_hi_u32 v1, s1, v1 +; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, s0, v0 ; GFX8-NEXT: v_subrev_u32_e32 v2, vcc, s10, v2 -; GFX8-NEXT: v_mul_lo_u32 v3, v1, s12 +; GFX8-NEXT: v_mul_lo_u32 v3, v1, s11 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 1, v1 ; GFX8-NEXT: v_sub_u32_e32 v3, vcc, s1, v3 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s12, v3 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s11, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s12, v3 +; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s11, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 1, v1 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s12, v3 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s11, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s12, v3 -; GFX8-NEXT: s_xor_b32 s0, s2, s11 +; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s11, v3 +; GFX8-NEXT: s_xor_b32 s0, s2, s12 ; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc ; GFX8-NEXT: v_xor_b32_e32 v1, s0, v1 ; GFX8-NEXT: v_mov_b32_e32 v4, s4 @@ -745,74 +744,73 @@ ; ; GFX9-LABEL: sdivrem_v2i32: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x18 -; GFX9-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x10 +; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x10 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_ashr_i32 s10, s6, 31 -; GFX9-NEXT: s_add_i32 s0, s6, s10 -; GFX9-NEXT: s_xor_b32 s6, s0, s10 -; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s6 +; GFX9-NEXT: s_ashr_i32 s6, s10, 31 +; GFX9-NEXT: s_add_i32 s0, s10, s6 +; GFX9-NEXT: s_xor_b32 s7, s0, s6 +; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s7 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 -; GFX9-NEXT: s_ashr_i32 s5, s7, 31 -; GFX9-NEXT: s_add_i32 s7, s7, s5 +; GFX9-NEXT: s_ashr_i32 s4, s11, 31 +; GFX9-NEXT: s_add_i32 s5, s11, s4 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX9-NEXT: s_xor_b32 s7, s7, s5 -; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s7 -; GFX9-NEXT: s_sub_i32 s11, 0, s6 +; GFX9-NEXT: s_xor_b32 s5, s5, s4 +; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s5 +; GFX9-NEXT: s_sub_i32 s11, 0, s7 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v1, v1 -; GFX9-NEXT: s_ashr_i32 s4, s8, 31 -; GFX9-NEXT: s_add_i32 s8, s8, s4 +; GFX9-NEXT: s_ashr_i32 s10, s8, 31 +; GFX9-NEXT: s_add_i32 s8, s8, s10 ; GFX9-NEXT: v_mul_lo_u32 v2, s11, v0 ; GFX9-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX9-NEXT: s_xor_b32 s8, s8, s4 +; GFX9-NEXT: s_xor_b32 s8, s8, s10 ; GFX9-NEXT: v_mul_hi_u32 v2, v0, v2 -; GFX9-NEXT: s_sub_i32 s12, 0, s7 +; GFX9-NEXT: s_sub_i32 s11, 0, s5 +; GFX9-NEXT: v_mul_lo_u32 v3, s11, v1 ; GFX9-NEXT: s_ashr_i32 s11, s9, 31 -; GFX9-NEXT: s_add_i32 s9, s9, s11 ; GFX9-NEXT: v_add_u32_e32 v0, v0, v2 ; GFX9-NEXT: v_mul_hi_u32 v0, s8, v0 -; GFX9-NEXT: v_mul_lo_u32 v2, s12, v1 +; GFX9-NEXT: v_mul_hi_u32 v2, v1, v3 +; GFX9-NEXT: s_add_i32 s9, s9, s11 ; GFX9-NEXT: s_xor_b32 s9, s9, s11 -; GFX9-NEXT: v_mul_lo_u32 v3, v0, s6 -; GFX9-NEXT: v_mul_hi_u32 v2, v1, v2 -; GFX9-NEXT: v_add_u32_e32 v4, 1, v0 -; GFX9-NEXT: v_sub_u32_e32 v3, s8, v3 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s6, v3 +; GFX9-NEXT: v_mul_lo_u32 v3, v0, s7 ; GFX9-NEXT: v_add_u32_e32 v1, v1, v2 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX9-NEXT: v_subrev_u32_e32 v4, s6, v3 +; GFX9-NEXT: v_add_u32_e32 v2, 1, v0 ; GFX9-NEXT: v_mul_hi_u32 v1, s9, v1 -; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc -; GFX9-NEXT: v_add_u32_e32 v4, 1, v0 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s6, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX9-NEXT: v_subrev_u32_e32 v4, s6, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc -; GFX9-NEXT: v_mul_lo_u32 v3, v1, s7 +; GFX9-NEXT: v_sub_u32_e32 v3, s8, v3 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s7, v3 +; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc +; GFX9-NEXT: v_subrev_u32_e32 v2, s7, v3 +; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc +; GFX9-NEXT: v_add_u32_e32 v3, 1, v0 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s7, v2 +; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc +; GFX9-NEXT: v_subrev_u32_e32 v3, s7, v2 +; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc +; GFX9-NEXT: v_mul_lo_u32 v3, v1, s5 ; GFX9-NEXT: v_add_u32_e32 v4, 1, v1 -; GFX9-NEXT: v_xor_b32_e32 v2, s4, v2 -; GFX9-NEXT: s_xor_b32 s6, s4, s10 +; GFX9-NEXT: s_xor_b32 s6, s10, s6 +; GFX9-NEXT: s_xor_b32 s4, s11, s4 ; GFX9-NEXT: v_sub_u32_e32 v3, s9, v3 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s7, v3 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s5, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX9-NEXT: v_subrev_u32_e32 v4, s7, v3 +; GFX9-NEXT: v_subrev_u32_e32 v4, s5, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc ; GFX9-NEXT: v_add_u32_e32 v4, 1, v1 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s7, v3 -; GFX9-NEXT: v_subrev_u32_e32 v2, s4, v2 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s5, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX9-NEXT: v_subrev_u32_e32 v4, s7, v3 -; GFX9-NEXT: s_xor_b32 s4, s11, s5 +; GFX9-NEXT: v_subrev_u32_e32 v4, s5, v3 ; GFX9-NEXT: v_xor_b32_e32 v0, s6, v0 ; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc ; GFX9-NEXT: v_xor_b32_e32 v1, s4, v1 ; GFX9-NEXT: v_subrev_u32_e32 v0, s6, v0 +; GFX9-NEXT: v_xor_b32_e32 v2, s10, v2 ; GFX9-NEXT: v_subrev_u32_e32 v1, s4, v1 ; GFX9-NEXT: v_xor_b32_e32 v3, s11, v3 ; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: v_subrev_u32_e32 v2, s10, v2 ; GFX9-NEXT: v_subrev_u32_e32 v3, s11, v3 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1] @@ -821,34 +819,32 @@ ; ; GFX10-LABEL: sdivrem_v2i32: ; GFX10: ; %bb.0: -; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x18 +; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x10 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_ashr_i32 s2, s0, 31 -; GFX10-NEXT: s_ashr_i32 s3, s1, 31 -; GFX10-NEXT: s_add_i32 s0, s0, s2 -; GFX10-NEXT: s_add_i32 s1, s1, s3 -; GFX10-NEXT: s_xor_b32 s8, s0, s2 -; GFX10-NEXT: s_xor_b32 s9, s1, s3 -; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s8 -; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s9 -; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10 -; GFX10-NEXT: s_sub_i32 s6, 0, s8 -; GFX10-NEXT: s_sub_i32 s7, 0, s9 +; GFX10-NEXT: s_ashr_i32 s8, s2, 31 +; GFX10-NEXT: s_ashr_i32 s9, s3, 31 +; GFX10-NEXT: s_add_i32 s2, s2, s8 +; GFX10-NEXT: s_add_i32 s3, s3, s9 +; GFX10-NEXT: s_xor_b32 s2, s2, s8 +; GFX10-NEXT: s_xor_b32 s3, s3, s9 +; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s2 +; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s3 +; GFX10-NEXT: s_sub_i32 s6, 0, s2 +; GFX10-NEXT: s_sub_i32 s7, 0, s3 +; GFX10-NEXT: s_ashr_i32 s10, s0, 31 ; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX10-NEXT: v_rcp_iflag_f32_e32 v1, v1 +; GFX10-NEXT: s_ashr_i32 s11, s1, 31 +; GFX10-NEXT: s_add_i32 s0, s0, s10 +; GFX10-NEXT: s_add_i32 s1, s1, s11 +; GFX10-NEXT: s_xor_b32 s0, s0, s10 +; GFX10-NEXT: s_xor_b32 s1, s1, s11 ; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX10-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX10-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_ashr_i32 s10, s0, 31 -; GFX10-NEXT: s_ashr_i32 s11, s1, 31 -; GFX10-NEXT: s_add_i32 s0, s0, s10 ; GFX10-NEXT: v_mul_lo_u32 v2, s6, v0 ; GFX10-NEXT: v_mul_lo_u32 v3, s7, v1 -; GFX10-NEXT: s_add_i32 s1, s1, s11 -; GFX10-NEXT: s_xor_b32 s0, s0, s10 -; GFX10-NEXT: s_xor_b32 s1, s1, s11 ; GFX10-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0 ; GFX10-NEXT: v_mul_hi_u32 v2, v0, v2 ; GFX10-NEXT: v_mul_hi_u32 v3, v1, v3 @@ -856,32 +852,32 @@ ; GFX10-NEXT: v_add_nc_u32_e32 v1, v1, v3 ; GFX10-NEXT: v_mul_hi_u32 v0, s0, v0 ; GFX10-NEXT: v_mul_hi_u32 v1, s1, v1 -; GFX10-NEXT: v_mul_lo_u32 v2, v0, s8 -; GFX10-NEXT: v_mul_lo_u32 v3, v1, s9 +; GFX10-NEXT: v_mul_lo_u32 v2, v0, s2 +; GFX10-NEXT: v_mul_lo_u32 v3, v1, s3 ; GFX10-NEXT: v_add_nc_u32_e32 v4, 1, v0 ; GFX10-NEXT: v_add_nc_u32_e32 v5, 1, v1 ; GFX10-NEXT: v_sub_nc_u32_e32 v2, s0, v2 ; GFX10-NEXT: v_sub_nc_u32_e32 v3, s1, v3 -; GFX10-NEXT: s_xor_b32 s1, s10, s2 -; GFX10-NEXT: v_subrev_nc_u32_e32 v6, s8, v2 -; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s9, v3 -; GFX10-NEXT: v_cmp_le_u32_e64 s0, s8, v2 -; GFX10-NEXT: v_subrev_nc_u32_e32 v7, s9, v3 +; GFX10-NEXT: s_xor_b32 s1, s10, s8 +; GFX10-NEXT: v_subrev_nc_u32_e32 v6, s2, v2 +; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s3, v3 +; GFX10-NEXT: v_cmp_le_u32_e64 s0, s2, v2 +; GFX10-NEXT: v_subrev_nc_u32_e32 v7, s3, v3 ; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo ; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0 ; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v6, s0 ; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo ; GFX10-NEXT: v_add_nc_u32_e32 v5, 1, v1 ; GFX10-NEXT: v_add_nc_u32_e32 v4, 1, v0 -; GFX10-NEXT: v_cmp_le_u32_e64 s0, s8, v2 -; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s9, v3 -; GFX10-NEXT: v_subrev_nc_u32_e32 v6, s8, v2 -; GFX10-NEXT: v_subrev_nc_u32_e32 v7, s9, v3 +; GFX10-NEXT: v_cmp_le_u32_e64 s0, s2, v2 +; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s3, v3 +; GFX10-NEXT: v_subrev_nc_u32_e32 v6, s2, v2 +; GFX10-NEXT: v_subrev_nc_u32_e32 v7, s3, v3 ; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0 ; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo ; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v6, s0 ; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo -; GFX10-NEXT: s_xor_b32 s0, s11, s3 +; GFX10-NEXT: s_xor_b32 s0, s11, s9 ; GFX10-NEXT: v_xor_b32_e32 v0, s1, v0 ; GFX10-NEXT: v_xor_b32_e32 v1, s0, v1 ; GFX10-NEXT: v_xor_b32_e32 v2, s10, v2 @@ -905,142 +901,141 @@ define amdgpu_kernel void @sdivrem_v4i32(<4 x i32> addrspace(1)* %out0, <4 x i32> addrspace(1)* %out1, <4 x i32> %x, <4 x i32> %y) { ; GFX8-LABEL: sdivrem_v4i32: ; GFX8: ; %bb.0: -; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x20 -; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x10 -; GFX8-NEXT: v_mov_b32_e32 v3, 0x4f7ffffe +; GFX8-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x10 +; GFX8-NEXT: v_mov_b32_e32 v2, 0x4f7ffffe ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: s_ashr_i32 s12, s0, 31 -; GFX8-NEXT: s_add_i32 s0, s0, s12 -; GFX8-NEXT: s_xor_b32 s13, s0, s12 -; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s13 -; GFX8-NEXT: s_ashr_i32 s15, s1, 31 -; GFX8-NEXT: s_add_i32 s0, s1, s15 -; GFX8-NEXT: s_sub_i32 s1, 0, s13 +; GFX8-NEXT: s_ashr_i32 s2, s12, 31 +; GFX8-NEXT: s_add_i32 s0, s12, s2 +; GFX8-NEXT: s_xor_b32 s3, s0, s2 +; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s3 +; GFX8-NEXT: s_sub_i32 s1, 0, s3 +; GFX8-NEXT: s_ashr_i32 s16, s13, 31 +; GFX8-NEXT: s_add_i32 s0, s13, s16 ; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX8-NEXT: s_xor_b32 s16, s0, s15 -; GFX8-NEXT: v_cvt_f32_u32_e32 v2, s16 -; GFX8-NEXT: s_ashr_i32 s14, s8, 31 +; GFX8-NEXT: s_xor_b32 s13, s0, s16 +; GFX8-NEXT: v_cvt_f32_u32_e32 v3, s13 +; GFX8-NEXT: s_ashr_i32 s12, s8, 31 ; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX8-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX8-NEXT: s_add_i32 s0, s8, s14 -; GFX8-NEXT: s_xor_b32 s0, s0, s14 -; GFX8-NEXT: v_rcp_iflag_f32_e32 v2, v2 +; GFX8-NEXT: s_add_i32 s0, s8, s12 +; GFX8-NEXT: s_xor_b32 s0, s0, s12 +; GFX8-NEXT: v_rcp_iflag_f32_e32 v3, v3 ; GFX8-NEXT: v_mul_lo_u32 v1, s1, v0 -; GFX8-NEXT: s_ashr_i32 s8, s9, 31 ; GFX8-NEXT: v_mul_hi_u32 v1, v0, v1 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v1 ; GFX8-NEXT: v_mul_hi_u32 v0, s0, v0 -; GFX8-NEXT: v_mul_f32_e32 v1, v2, v3 +; GFX8-NEXT: v_mul_f32_e32 v1, v3, v2 ; GFX8-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX8-NEXT: v_mul_lo_u32 v2, v0, s13 +; GFX8-NEXT: v_mul_lo_u32 v3, v0, s3 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 1, v0 -; GFX8-NEXT: v_sub_u32_e32 v2, vcc, s0, v2 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s13, v2 +; GFX8-NEXT: v_sub_u32_e32 v3, vcc, s0, v3 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s3, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s13, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc +; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s3, v3 +; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 1, v0 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s13, v2 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s3, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s13, v2 -; GFX8-NEXT: s_sub_i32 s0, 0, s16 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc +; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s3, v3 +; GFX8-NEXT: s_sub_i32 s0, 0, s13 +; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc ; GFX8-NEXT: v_mul_lo_u32 v4, s0, v1 -; GFX8-NEXT: s_add_i32 s1, s9, s8 -; GFX8-NEXT: s_xor_b32 s1, s1, s8 -; GFX8-NEXT: s_xor_b32 s0, s14, s12 +; GFX8-NEXT: s_xor_b32 s0, s12, s2 +; GFX8-NEXT: s_ashr_i32 s2, s9, 31 +; GFX8-NEXT: s_add_i32 s1, s9, s2 ; GFX8-NEXT: v_mul_hi_u32 v4, v1, v4 +; GFX8-NEXT: s_xor_b32 s1, s1, s2 ; GFX8-NEXT: v_xor_b32_e32 v0, s0, v0 -; GFX8-NEXT: v_xor_b32_e32 v2, s14, v2 -; GFX8-NEXT: s_ashr_i32 s9, s2, 31 +; GFX8-NEXT: v_xor_b32_e32 v3, s12, v3 ; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v4 ; GFX8-NEXT: v_mul_hi_u32 v1, s1, v1 +; GFX8-NEXT: s_ashr_i32 s3, s14, 31 ; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, s0, v0 -; GFX8-NEXT: v_subrev_u32_e32 v4, vcc, s14, v2 -; GFX8-NEXT: v_mul_lo_u32 v5, v1, s16 -; GFX8-NEXT: s_add_i32 s0, s2, s9 -; GFX8-NEXT: s_xor_b32 s2, s0, s9 -; GFX8-NEXT: s_ashr_i32 s12, s10, 31 -; GFX8-NEXT: v_sub_u32_e32 v2, vcc, s1, v5 +; GFX8-NEXT: v_mul_lo_u32 v5, v1, s13 +; GFX8-NEXT: v_subrev_u32_e32 v4, vcc, s12, v3 +; GFX8-NEXT: s_add_i32 s0, s14, s3 +; GFX8-NEXT: v_sub_u32_e32 v3, vcc, s1, v5 ; GFX8-NEXT: v_add_u32_e32 v5, vcc, 1, v1 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s16, v2 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s13, v3 +; GFX8-NEXT: s_xor_b32 s8, s0, s3 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc -; GFX8-NEXT: v_cvt_f32_u32_e32 v5, s2 -; GFX8-NEXT: v_subrev_u32_e64 v6, s[0:1], s16, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc +; GFX8-NEXT: v_cvt_f32_u32_e32 v5, s8 +; GFX8-NEXT: v_subrev_u32_e64 v6, s[0:1], s13, v3 +; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc ; GFX8-NEXT: v_rcp_iflag_f32_e32 v5, v5 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, 1, v1 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s16, v2 -; GFX8-NEXT: v_mul_f32_e32 v5, v5, v3 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s13, v3 +; GFX8-NEXT: v_mul_f32_e32 v5, v5, v2 ; GFX8-NEXT: v_cvt_u32_f32_e32 v5, v5 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc -; GFX8-NEXT: v_subrev_u32_e64 v6, s[0:1], s16, v2 -; GFX8-NEXT: s_sub_i32 s0, 0, s2 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc +; GFX8-NEXT: v_subrev_u32_e64 v6, s[0:1], s13, v3 +; GFX8-NEXT: s_sub_i32 s0, 0, s8 +; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc ; GFX8-NEXT: v_mul_lo_u32 v6, s0, v5 -; GFX8-NEXT: s_add_i32 s1, s10, s12 -; GFX8-NEXT: s_xor_b32 s1, s1, s12 -; GFX8-NEXT: s_xor_b32 s0, s8, s15 +; GFX8-NEXT: s_ashr_i32 s9, s10, 31 +; GFX8-NEXT: s_add_i32 s1, s10, s9 +; GFX8-NEXT: s_xor_b32 s1, s1, s9 ; GFX8-NEXT: v_mul_hi_u32 v6, v5, v6 -; GFX8-NEXT: v_xor_b32_e32 v2, s8, v2 +; GFX8-NEXT: s_xor_b32 s0, s2, s16 +; GFX8-NEXT: v_xor_b32_e32 v3, s2, v3 ; GFX8-NEXT: v_xor_b32_e32 v1, s0, v1 -; GFX8-NEXT: v_subrev_u32_e32 v1, vcc, s0, v1 ; GFX8-NEXT: v_add_u32_e32 v5, vcc, v5, v6 ; GFX8-NEXT: v_mul_hi_u32 v6, s1, v5 -; GFX8-NEXT: v_subrev_u32_e32 v5, vcc, s8, v2 -; GFX8-NEXT: s_ashr_i32 s8, s3, 31 -; GFX8-NEXT: v_mul_lo_u32 v7, v6, s2 -; GFX8-NEXT: s_add_i32 s0, s3, s8 -; GFX8-NEXT: s_xor_b32 s3, s0, s8 -; GFX8-NEXT: v_sub_u32_e32 v2, vcc, s1, v7 +; GFX8-NEXT: v_subrev_u32_e32 v5, vcc, s2, v3 +; GFX8-NEXT: s_ashr_i32 s2, s15, 31 +; GFX8-NEXT: v_mul_lo_u32 v7, v6, s8 +; GFX8-NEXT: v_subrev_u32_e32 v1, vcc, s0, v1 +; GFX8-NEXT: s_add_i32 s0, s15, s2 +; GFX8-NEXT: v_sub_u32_e32 v3, vcc, s1, v7 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 1, v6 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s2, v2 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s8, v3 +; GFX8-NEXT: s_xor_b32 s10, s0, s2 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v7, vcc -; GFX8-NEXT: v_cvt_f32_u32_e32 v7, s3 -; GFX8-NEXT: v_subrev_u32_e64 v8, s[0:1], s2, v2 -; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc +; GFX8-NEXT: v_cvt_f32_u32_e32 v7, s10 +; GFX8-NEXT: v_subrev_u32_e64 v8, s[0:1], s8, v3 +; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v8, vcc ; GFX8-NEXT: v_rcp_iflag_f32_e32 v7, v7 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 1, v6 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s2, v2 -; GFX8-NEXT: v_mul_f32_e32 v3, v7, v3 -; GFX8-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GFX8-NEXT: v_subrev_u32_e64 v7, s[0:1], s2, v2 -; GFX8-NEXT: s_sub_i32 s0, 0, s3 -; GFX8-NEXT: v_cndmask_b32_e32 v7, v2, v7, vcc -; GFX8-NEXT: v_mul_lo_u32 v2, s0, v3 -; GFX8-NEXT: s_ashr_i32 s2, s11, 31 -; GFX8-NEXT: s_add_i32 s1, s11, s2 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s8, v3 +; GFX8-NEXT: v_mul_f32_e32 v2, v7, v2 +; GFX8-NEXT: v_cvt_u32_f32_e32 v2, v2 +; GFX8-NEXT: v_subrev_u32_e64 v7, s[0:1], s8, v3 +; GFX8-NEXT: s_sub_i32 s0, 0, s10 +; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc +; GFX8-NEXT: v_mul_lo_u32 v7, s0, v2 +; GFX8-NEXT: s_xor_b32 s0, s9, s3 +; GFX8-NEXT: s_ashr_i32 s3, s11, 31 +; GFX8-NEXT: s_add_i32 s1, s11, s3 +; GFX8-NEXT: v_mul_hi_u32 v7, v2, v7 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v6, v8, vcc -; GFX8-NEXT: v_mul_hi_u32 v2, v3, v2 -; GFX8-NEXT: s_xor_b32 s1, s1, s2 -; GFX8-NEXT: s_xor_b32 s0, s12, s9 +; GFX8-NEXT: s_xor_b32 s1, s1, s3 ; GFX8-NEXT: v_xor_b32_e32 v6, s0, v6 -; GFX8-NEXT: v_add_u32_e32 v2, vcc, v3, v2 -; GFX8-NEXT: v_mul_hi_u32 v3, s1, v2 +; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v7 +; GFX8-NEXT: v_mul_hi_u32 v7, s1, v2 +; GFX8-NEXT: v_xor_b32_e32 v3, s9, v3 ; GFX8-NEXT: v_subrev_u32_e32 v2, vcc, s0, v6 -; GFX8-NEXT: v_xor_b32_e32 v6, s12, v7 -; GFX8-NEXT: v_mul_lo_u32 v7, v3, s3 -; GFX8-NEXT: v_subrev_u32_e32 v6, vcc, s12, v6 -; GFX8-NEXT: v_add_u32_e32 v8, vcc, 1, v3 -; GFX8-NEXT: v_sub_u32_e32 v7, vcc, s1, v7 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s3, v7 -; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v8, vcc -; GFX8-NEXT: v_subrev_u32_e64 v8, s[0:1], s3, v7 +; GFX8-NEXT: v_mul_lo_u32 v8, v7, s10 +; GFX8-NEXT: v_subrev_u32_e32 v6, vcc, s9, v3 +; GFX8-NEXT: v_sub_u32_e32 v3, vcc, s1, v8 +; GFX8-NEXT: v_add_u32_e32 v8, vcc, 1, v7 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s10, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v8, vcc -; GFX8-NEXT: v_add_u32_e32 v8, vcc, 1, v3 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s3, v7 +; GFX8-NEXT: v_subrev_u32_e64 v8, s[0:1], s10, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v8, vcc -; GFX8-NEXT: v_subrev_u32_e64 v8, s[0:1], s3, v7 -; GFX8-NEXT: s_xor_b32 s0, s2, s8 +; GFX8-NEXT: v_add_u32_e32 v8, vcc, 1, v7 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s10, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v8, vcc -; GFX8-NEXT: v_xor_b32_e32 v3, s0, v3 +; GFX8-NEXT: v_subrev_u32_e64 v8, s[0:1], s10, v3 +; GFX8-NEXT: v_cndmask_b32_e32 v8, v3, v8, vcc +; GFX8-NEXT: s_xor_b32 s0, s3, s2 +; GFX8-NEXT: v_xor_b32_e32 v3, s0, v7 +; GFX8-NEXT: v_xor_b32_e32 v7, s3, v8 ; GFX8-NEXT: v_mov_b32_e32 v9, s5 ; GFX8-NEXT: v_subrev_u32_e32 v3, vcc, s0, v3 ; GFX8-NEXT: v_mov_b32_e32 v8, s4 -; GFX8-NEXT: v_xor_b32_e32 v7, s2, v7 ; GFX8-NEXT: flat_store_dwordx4 v[8:9], v[0:3] -; GFX8-NEXT: v_subrev_u32_e32 v7, vcc, s2, v7 +; GFX8-NEXT: v_subrev_u32_e32 v7, vcc, s3, v7 ; GFX8-NEXT: v_mov_b32_e32 v0, s6 ; GFX8-NEXT: v_mov_b32_e32 v1, s7 ; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[4:7] @@ -1048,7 +1043,7 @@ ; ; GFX9-LABEL: sdivrem_v4i32: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dwordx4 s[12:15], s[4:5], 0x20 +; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x10 ; GFX9-NEXT: v_mov_b32_e32 v2, 0x4f7ffffe ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_ashr_i32 s6, s12, 31 @@ -1056,116 +1051,114 @@ ; GFX9-NEXT: s_xor_b32 s7, s0, s6 ; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s7 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 -; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x10 -; GFX9-NEXT: s_ashr_i32 s4, s13, 31 -; GFX9-NEXT: s_add_i32 s5, s13, s4 +; GFX9-NEXT: s_ashr_i32 s5, s13, 31 +; GFX9-NEXT: s_add_i32 s12, s13, s5 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX9-NEXT: s_sub_i32 s12, 0, s7 -; GFX9-NEXT: s_xor_b32 s5, s5, s4 -; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s5 +; GFX9-NEXT: s_xor_b32 s12, s12, s5 +; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s12 +; GFX9-NEXT: s_sub_i32 s13, 0, s7 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX9-NEXT: s_sub_i32 s13, 0, s5 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v1, v1 -; GFX9-NEXT: v_mul_lo_u32 v3, s12, v0 -; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_ashr_i32 s12, s8, 31 -; GFX9-NEXT: s_add_i32 s8, s8, s12 -; GFX9-NEXT: s_xor_b32 s8, s8, s12 -; GFX9-NEXT: v_mul_hi_u32 v3, v0, v3 +; GFX9-NEXT: s_ashr_i32 s4, s8, 31 +; GFX9-NEXT: s_add_i32 s8, s8, s4 +; GFX9-NEXT: v_mul_lo_u32 v3, s13, v0 ; GFX9-NEXT: v_mul_f32_e32 v1, v1, v2 ; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX9-NEXT: s_xor_b32 s6, s12, s6 -; GFX9-NEXT: v_add_u32_e32 v0, v0, v3 -; GFX9-NEXT: v_mul_hi_u32 v0, s8, v0 -; GFX9-NEXT: v_mul_lo_u32 v3, s13, v1 +; GFX9-NEXT: s_xor_b32 s8, s8, s4 +; GFX9-NEXT: v_mul_hi_u32 v3, v0, v3 +; GFX9-NEXT: s_sub_i32 s16, 0, s12 ; GFX9-NEXT: s_ashr_i32 s13, s9, 31 ; GFX9-NEXT: s_add_i32 s9, s9, s13 +; GFX9-NEXT: v_add_u32_e32 v0, v0, v3 +; GFX9-NEXT: v_mul_hi_u32 v0, s8, v0 +; GFX9-NEXT: v_mul_lo_u32 v3, s16, v1 +; GFX9-NEXT: s_xor_b32 s9, s9, s13 +; GFX9-NEXT: s_xor_b32 s6, s4, s6 ; GFX9-NEXT: v_mul_lo_u32 v4, v0, s7 ; GFX9-NEXT: v_mul_hi_u32 v3, v1, v3 ; GFX9-NEXT: v_add_u32_e32 v5, 1, v0 -; GFX9-NEXT: s_xor_b32 s4, s13, s4 +; GFX9-NEXT: s_xor_b32 s5, s13, s5 ; GFX9-NEXT: v_sub_u32_e32 v4, s8, v4 ; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s7, v4 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc ; GFX9-NEXT: v_subrev_u32_e32 v5, s7, v4 +; GFX9-NEXT: v_add_u32_e32 v1, v1, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v5, vcc +; GFX9-NEXT: v_mul_hi_u32 v1, s9, v1 ; GFX9-NEXT: v_add_u32_e32 v5, 1, v0 ; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s7, v4 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc ; GFX9-NEXT: v_subrev_u32_e32 v5, s7, v4 -; GFX9-NEXT: s_xor_b32 s7, s9, s13 -; GFX9-NEXT: v_add_u32_e32 v1, v1, v3 -; GFX9-NEXT: v_mul_hi_u32 v1, s7, v1 ; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v5, vcc +; GFX9-NEXT: v_mul_lo_u32 v5, v1, s12 +; GFX9-NEXT: v_xor_b32_e32 v3, s4, v3 ; GFX9-NEXT: v_xor_b32_e32 v0, s6, v0 +; GFX9-NEXT: v_subrev_u32_e32 v4, s4, v3 +; GFX9-NEXT: s_ashr_i32 s4, s14, 31 ; GFX9-NEXT: v_subrev_u32_e32 v0, s6, v0 -; GFX9-NEXT: v_mul_lo_u32 v5, v1, s5 -; GFX9-NEXT: v_xor_b32_e32 v3, s12, v3 -; GFX9-NEXT: s_ashr_i32 s6, s14, 31 -; GFX9-NEXT: v_subrev_u32_e32 v4, s12, v3 -; GFX9-NEXT: v_sub_u32_e32 v3, s7, v5 -; GFX9-NEXT: s_add_i32 s7, s14, s6 -; GFX9-NEXT: s_xor_b32 s7, s7, s6 -; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s7 +; GFX9-NEXT: s_add_i32 s6, s14, s4 +; GFX9-NEXT: s_xor_b32 s6, s6, s4 +; GFX9-NEXT: v_sub_u32_e32 v3, s9, v5 +; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s6 ; GFX9-NEXT: v_add_u32_e32 v6, 1, v1 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s5, v3 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s12, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc ; GFX9-NEXT: v_rcp_iflag_f32_e32 v5, v5 -; GFX9-NEXT: v_subrev_u32_e32 v6, s5, v3 +; GFX9-NEXT: v_subrev_u32_e32 v6, s12, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc ; GFX9-NEXT: v_add_u32_e32 v6, 1, v1 ; GFX9-NEXT: v_mul_f32_e32 v5, v5, v2 ; GFX9-NEXT: v_cvt_u32_f32_e32 v5, v5 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s5, v3 -; GFX9-NEXT: s_sub_i32 s8, 0, s7 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s12, v3 +; GFX9-NEXT: s_sub_i32 s7, 0, s6 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc -; GFX9-NEXT: v_mul_lo_u32 v6, s8, v5 -; GFX9-NEXT: v_xor_b32_e32 v1, s4, v1 -; GFX9-NEXT: v_subrev_u32_e32 v1, s4, v1 -; GFX9-NEXT: s_ashr_i32 s4, s15, 31 -; GFX9-NEXT: s_add_i32 s9, s15, s4 +; GFX9-NEXT: v_mul_lo_u32 v6, s7, v5 +; GFX9-NEXT: v_xor_b32_e32 v1, s5, v1 +; GFX9-NEXT: v_subrev_u32_e32 v1, s5, v1 +; GFX9-NEXT: s_ashr_i32 s5, s15, 31 +; GFX9-NEXT: s_add_i32 s9, s15, s5 ; GFX9-NEXT: v_mul_hi_u32 v6, v5, v6 -; GFX9-NEXT: s_xor_b32 s9, s9, s4 +; GFX9-NEXT: s_xor_b32 s9, s9, s5 ; GFX9-NEXT: v_cvt_f32_u32_e32 v8, s9 -; GFX9-NEXT: v_subrev_u32_e32 v7, s5, v3 -; GFX9-NEXT: s_ashr_i32 s5, s10, 31 -; GFX9-NEXT: s_add_i32 s8, s10, s5 -; GFX9-NEXT: s_xor_b32 s8, s8, s5 +; GFX9-NEXT: s_ashr_i32 s7, s10, 31 +; GFX9-NEXT: s_add_i32 s8, s10, s7 +; GFX9-NEXT: s_xor_b32 s8, s8, s7 ; GFX9-NEXT: v_add_u32_e32 v5, v5, v6 ; GFX9-NEXT: v_mul_hi_u32 v6, s8, v5 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v8, v8 +; GFX9-NEXT: v_subrev_u32_e32 v7, s12, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc -; GFX9-NEXT: v_xor_b32_e32 v3, s13, v3 -; GFX9-NEXT: v_mul_lo_u32 v7, v6, s7 +; GFX9-NEXT: v_mul_lo_u32 v7, v6, s6 ; GFX9-NEXT: v_mul_f32_e32 v2, v8, v2 ; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2 +; GFX9-NEXT: v_xor_b32_e32 v3, s13, v3 ; GFX9-NEXT: v_subrev_u32_e32 v5, s13, v3 ; GFX9-NEXT: v_sub_u32_e32 v3, s8, v7 ; GFX9-NEXT: s_sub_i32 s8, 0, s9 ; GFX9-NEXT: v_mul_lo_u32 v8, s8, v2 ; GFX9-NEXT: v_add_u32_e32 v7, 1, v6 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s7, v3 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s6, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v7, vcc -; GFX9-NEXT: v_subrev_u32_e32 v7, s7, v3 +; GFX9-NEXT: v_subrev_u32_e32 v7, s6, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc ; GFX9-NEXT: v_mul_hi_u32 v8, v2, v8 ; GFX9-NEXT: v_add_u32_e32 v7, 1, v6 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s7, v3 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s6, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v7, vcc -; GFX9-NEXT: v_subrev_u32_e32 v7, s7, v3 -; GFX9-NEXT: s_ashr_i32 s7, s11, 31 -; GFX9-NEXT: s_add_i32 s8, s11, s7 -; GFX9-NEXT: s_xor_b32 s8, s8, s7 +; GFX9-NEXT: v_subrev_u32_e32 v7, s6, v3 +; GFX9-NEXT: s_ashr_i32 s6, s11, 31 +; GFX9-NEXT: s_add_i32 s8, s11, s6 +; GFX9-NEXT: s_xor_b32 s8, s8, s6 ; GFX9-NEXT: v_add_u32_e32 v2, v2, v8 ; GFX9-NEXT: v_mul_hi_u32 v8, s8, v2 ; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc -; GFX9-NEXT: s_xor_b32 s6, s5, s6 -; GFX9-NEXT: v_xor_b32_e32 v3, s5, v3 -; GFX9-NEXT: v_mul_lo_u32 v7, v8, s9 -; GFX9-NEXT: v_xor_b32_e32 v2, s6, v6 -; GFX9-NEXT: v_subrev_u32_e32 v6, s5, v3 ; GFX9-NEXT: s_xor_b32 s4, s7, s4 +; GFX9-NEXT: v_xor_b32_e32 v3, s7, v3 +; GFX9-NEXT: v_mul_lo_u32 v7, v8, s9 +; GFX9-NEXT: v_xor_b32_e32 v2, s4, v6 +; GFX9-NEXT: v_subrev_u32_e32 v6, s7, v3 +; GFX9-NEXT: v_subrev_u32_e32 v2, s4, v2 ; GFX9-NEXT: v_sub_u32_e32 v3, s8, v7 ; GFX9-NEXT: v_add_u32_e32 v7, 1, v8 ; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s9, v3 @@ -1176,148 +1169,147 @@ ; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s9, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v7, v7, v8, vcc ; GFX9-NEXT: v_subrev_u32_e32 v8, s9, v3 +; GFX9-NEXT: s_xor_b32 s4, s6, s5 ; GFX9-NEXT: v_cndmask_b32_e32 v8, v3, v8, vcc ; GFX9-NEXT: v_xor_b32_e32 v3, s4, v7 -; GFX9-NEXT: v_subrev_u32_e32 v2, s6, v2 ; GFX9-NEXT: v_subrev_u32_e32 v3, s4, v3 -; GFX9-NEXT: v_xor_b32_e32 v7, s7, v8 +; GFX9-NEXT: v_xor_b32_e32 v7, s6, v8 ; GFX9-NEXT: v_mov_b32_e32 v8, 0 -; GFX9-NEXT: v_subrev_u32_e32 v7, s7, v7 +; GFX9-NEXT: v_subrev_u32_e32 v7, s6, v7 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] ; GFX9-NEXT: global_store_dwordx4 v8, v[4:7], s[2:3] ; GFX9-NEXT: s_endpgm ; ; GFX10-LABEL: sdivrem_v4i32: ; GFX10: ; %bb.0: -; GFX10-NEXT: s_clause 0x1 -; GFX10-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x20 -; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x10 +; GFX10-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x10 ; GFX10-NEXT: v_mov_b32_e32 v4, 0x4f7ffffe ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_ashr_i32 s12, s8, 31 -; GFX10-NEXT: s_ashr_i32 s14, s10, 31 -; GFX10-NEXT: s_add_i32 s6, s8, s12 -; GFX10-NEXT: s_add_i32 s8, s10, s14 -; GFX10-NEXT: s_xor_b32 s10, s6, s12 -; GFX10-NEXT: s_ashr_i32 s13, s9, 31 -; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s10 -; GFX10-NEXT: s_ashr_i32 s15, s11, 31 -; GFX10-NEXT: s_add_i32 s7, s9, s13 -; GFX10-NEXT: s_add_i32 s9, s11, s15 -; GFX10-NEXT: s_xor_b32 s11, s7, s13 -; GFX10-NEXT: s_xor_b32 s8, s8, s14 +; GFX10-NEXT: s_ashr_i32 s0, s12, 31 +; GFX10-NEXT: s_ashr_i32 s2, s14, 31 +; GFX10-NEXT: s_add_i32 s6, s12, s0 +; GFX10-NEXT: s_add_i32 s12, s14, s2 +; GFX10-NEXT: s_xor_b32 s14, s6, s0 +; GFX10-NEXT: s_ashr_i32 s1, s13, 31 +; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s14 +; GFX10-NEXT: s_ashr_i32 s3, s15, 31 +; GFX10-NEXT: s_add_i32 s7, s13, s1 +; GFX10-NEXT: s_add_i32 s13, s15, s3 +; GFX10-NEXT: s_xor_b32 s15, s7, s1 +; GFX10-NEXT: s_xor_b32 s12, s12, s2 +; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s15 +; GFX10-NEXT: v_cvt_f32_u32_e32 v2, s12 ; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s11 -; GFX10-NEXT: v_cvt_f32_u32_e32 v2, s8 -; GFX10-NEXT: s_xor_b32 s9, s9, s15 -; GFX10-NEXT: s_sub_i32 s6, 0, s10 -; GFX10-NEXT: v_cvt_f32_u32_e32 v3, s9 +; GFX10-NEXT: s_xor_b32 s13, s13, s3 +; GFX10-NEXT: s_sub_i32 s6, 0, s14 +; GFX10-NEXT: v_cvt_f32_u32_e32 v3, s13 ; GFX10-NEXT: v_rcp_iflag_f32_e32 v1, v1 ; GFX10-NEXT: v_rcp_iflag_f32_e32 v2, v2 -; GFX10-NEXT: s_sub_i32 s7, 0, s11 -; GFX10-NEXT: s_sub_i32 s19, 0, s8 +; GFX10-NEXT: s_sub_i32 s7, 0, s15 +; GFX10-NEXT: s_sub_i32 s19, 0, s12 ; GFX10-NEXT: v_rcp_iflag_f32_e32 v3, v3 +; GFX10-NEXT: s_ashr_i32 s16, s8, 31 ; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 -; GFX10-NEXT: s_ashr_i32 s16, s0, 31 -; GFX10-NEXT: s_ashr_i32 s17, s1, 31 -; GFX10-NEXT: s_add_i32 s0, s0, s16 -; GFX10-NEXT: s_ashr_i32 s18, s2, 31 -; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GFX10-NEXT: s_ashr_i32 s18, s10, 31 +; GFX10-NEXT: s_ashr_i32 s17, s9, 31 +; GFX10-NEXT: s_xor_b32 s20, s16, s0 ; GFX10-NEXT: v_mul_f32_e32 v1, v1, v4 ; GFX10-NEXT: v_mul_f32_e32 v2, v2, v4 -; GFX10-NEXT: s_xor_b32 s0, s0, s16 +; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0 +; GFX10-NEXT: s_xor_b32 s21, s17, s1 ; GFX10-NEXT: v_mul_f32_e32 v3, v3, v4 -; GFX10-NEXT: v_mul_lo_u32 v4, s6, v0 ; GFX10-NEXT: v_cvt_u32_f32_e32 v1, v1 ; GFX10-NEXT: v_cvt_u32_f32_e32 v2, v2 -; GFX10-NEXT: s_sub_i32 s6, 0, s9 +; GFX10-NEXT: v_mul_lo_u32 v4, s6, v0 +; GFX10-NEXT: s_sub_i32 s6, 0, s13 ; GFX10-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GFX10-NEXT: s_add_i32 s1, s1, s17 ; GFX10-NEXT: v_mul_lo_u32 v5, s7, v1 ; GFX10-NEXT: v_mul_lo_u32 v6, s19, v2 -; GFX10-NEXT: v_mul_hi_u32 v4, v0, v4 +; GFX10-NEXT: s_ashr_i32 s19, s11, 31 +; GFX10-NEXT: s_add_i32 s7, s9, s17 ; GFX10-NEXT: v_mul_lo_u32 v7, s6, v3 -; GFX10-NEXT: s_add_i32 s2, s2, s18 -; GFX10-NEXT: s_ashr_i32 s19, s3, 31 -; GFX10-NEXT: s_xor_b32 s1, s1, s17 -; GFX10-NEXT: s_xor_b32 s2, s2, s18 +; GFX10-NEXT: v_mul_hi_u32 v4, v0, v4 +; GFX10-NEXT: s_add_i32 s6, s8, s16 +; GFX10-NEXT: s_add_i32 s8, s10, s18 ; GFX10-NEXT: v_mul_hi_u32 v5, v1, v5 ; GFX10-NEXT: v_mul_hi_u32 v6, v2, v6 -; GFX10-NEXT: v_add_nc_u32_e32 v0, v0, v4 +; GFX10-NEXT: s_xor_b32 s10, s6, s16 +; GFX10-NEXT: s_add_i32 s9, s11, s19 ; GFX10-NEXT: v_mul_hi_u32 v7, v3, v7 -; GFX10-NEXT: s_add_i32 s3, s3, s19 -; GFX10-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0 -; GFX10-NEXT: s_xor_b32 s3, s3, s19 -; GFX10-NEXT: v_mul_hi_u32 v0, s0, v0 +; GFX10-NEXT: v_add_nc_u32_e32 v0, v0, v4 +; GFX10-NEXT: s_xor_b32 s11, s7, s17 +; GFX10-NEXT: s_xor_b32 s8, s8, s18 ; GFX10-NEXT: v_add_nc_u32_e32 v1, v1, v5 ; GFX10-NEXT: v_add_nc_u32_e32 v2, v2, v6 -; GFX10-NEXT: s_xor_b32 s12, s16, s12 +; GFX10-NEXT: v_mul_hi_u32 v0, s10, v0 +; GFX10-NEXT: s_xor_b32 s9, s9, s19 ; GFX10-NEXT: v_add_nc_u32_e32 v3, v3, v7 -; GFX10-NEXT: s_xor_b32 s13, s17, s13 -; GFX10-NEXT: v_mul_hi_u32 v1, s1, v1 -; GFX10-NEXT: v_mul_hi_u32 v2, s2, v2 -; GFX10-NEXT: v_mul_lo_u32 v4, v0, s10 -; GFX10-NEXT: v_mul_hi_u32 v3, s3, v3 +; GFX10-NEXT: v_mul_hi_u32 v1, s11, v1 +; GFX10-NEXT: v_mul_hi_u32 v2, s8, v2 +; GFX10-NEXT: s_xor_b32 s22, s18, s2 +; GFX10-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0 +; GFX10-NEXT: v_mul_hi_u32 v3, s9, v3 +; GFX10-NEXT: v_mul_lo_u32 v4, v0, s14 ; GFX10-NEXT: v_add_nc_u32_e32 v8, 1, v0 -; GFX10-NEXT: s_xor_b32 s14, s18, s14 -; GFX10-NEXT: v_mul_lo_u32 v5, v1, s11 -; GFX10-NEXT: v_mul_lo_u32 v6, v2, s8 -; GFX10-NEXT: v_sub_nc_u32_e32 v4, s0, v4 -; GFX10-NEXT: v_mul_lo_u32 v7, v3, s9 +; GFX10-NEXT: v_mul_lo_u32 v5, v1, s15 +; GFX10-NEXT: v_mul_lo_u32 v6, v2, s12 ; GFX10-NEXT: v_add_nc_u32_e32 v9, 1, v1 ; GFX10-NEXT: v_add_nc_u32_e32 v10, 1, v2 +; GFX10-NEXT: v_mul_lo_u32 v7, v3, s13 +; GFX10-NEXT: v_sub_nc_u32_e32 v4, s10, v4 ; GFX10-NEXT: v_add_nc_u32_e32 v11, 1, v3 -; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s10, v4 -; GFX10-NEXT: v_sub_nc_u32_e32 v5, s1, v5 -; GFX10-NEXT: v_sub_nc_u32_e32 v6, s2, v6 -; GFX10-NEXT: v_sub_nc_u32_e32 v7, s3, v7 +; GFX10-NEXT: v_sub_nc_u32_e32 v5, s11, v5 +; GFX10-NEXT: v_sub_nc_u32_e32 v6, s8, v6 +; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s14, v4 +; GFX10-NEXT: v_sub_nc_u32_e32 v7, s9, v7 +; GFX10-NEXT: v_cmp_le_u32_e64 s0, s15, v5 +; GFX10-NEXT: v_cmp_le_u32_e64 s1, s12, v6 ; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc_lo -; GFX10-NEXT: v_subrev_nc_u32_e32 v8, s10, v4 -; GFX10-NEXT: v_cmp_le_u32_e64 s0, s11, v5 -; GFX10-NEXT: v_cmp_le_u32_e64 s1, s8, v6 -; GFX10-NEXT: v_cmp_le_u32_e64 s2, s9, v7 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo +; GFX10-NEXT: v_subrev_nc_u32_e32 v8, s14, v4 +; GFX10-NEXT: v_cmp_le_u32_e64 s2, s13, v7 ; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v9, s0 -; GFX10-NEXT: v_subrev_nc_u32_e32 v9, s11, v5 +; GFX10-NEXT: v_subrev_nc_u32_e32 v9, s15, v5 ; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v10, s1 -; GFX10-NEXT: v_subrev_nc_u32_e32 v10, s8, v6 +; GFX10-NEXT: v_subrev_nc_u32_e32 v10, s12, v6 +; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo ; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v11, s2 -; GFX10-NEXT: v_subrev_nc_u32_e32 v11, s9, v7 +; GFX10-NEXT: v_subrev_nc_u32_e32 v11, s13, v7 ; GFX10-NEXT: v_add_nc_u32_e32 v8, 1, v0 -; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s10, v4 ; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v9, s0 ; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v10, s1 +; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s14, v4 ; GFX10-NEXT: v_cndmask_b32_e64 v7, v7, v11, s2 ; GFX10-NEXT: v_add_nc_u32_e32 v9, 1, v1 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc_lo -; GFX10-NEXT: v_subrev_nc_u32_e32 v8, s10, v4 ; GFX10-NEXT: v_add_nc_u32_e32 v10, 1, v2 -; GFX10-NEXT: v_cmp_le_u32_e64 s0, s11, v5 -; GFX10-NEXT: v_cmp_le_u32_e64 s1, s8, v6 +; GFX10-NEXT: v_cmp_le_u32_e64 s0, s15, v5 +; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc_lo +; GFX10-NEXT: v_subrev_nc_u32_e32 v8, s14, v4 +; GFX10-NEXT: v_cmp_le_u32_e64 s1, s12, v6 ; GFX10-NEXT: v_add_nc_u32_e32 v11, 1, v3 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo -; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s9, v7 +; GFX10-NEXT: v_cmp_le_u32_e64 s2, s13, v7 ; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v9, s0 -; GFX10-NEXT: v_subrev_nc_u32_e32 v9, s11, v5 +; GFX10-NEXT: v_subrev_nc_u32_e32 v9, s15, v5 ; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v10, s1 -; GFX10-NEXT: v_subrev_nc_u32_e32 v10, s8, v6 -; GFX10-NEXT: v_subrev_nc_u32_e32 v12, s9, v7 -; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v11, vcc_lo +; GFX10-NEXT: v_subrev_nc_u32_e32 v10, s12, v6 +; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo +; GFX10-NEXT: v_subrev_nc_u32_e32 v8, s13, v7 +; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v11, s2 ; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v9, s0 -; GFX10-NEXT: s_xor_b32 s0, s19, s15 ; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v10, s1 -; GFX10-NEXT: v_cndmask_b32_e32 v7, v7, v12, vcc_lo -; GFX10-NEXT: v_xor_b32_e32 v0, s12, v0 -; GFX10-NEXT: v_xor_b32_e32 v1, s13, v1 -; GFX10-NEXT: v_xor_b32_e32 v2, s14, v2 +; GFX10-NEXT: s_xor_b32 s0, s19, s3 +; GFX10-NEXT: v_cndmask_b32_e64 v7, v7, v8, s2 +; GFX10-NEXT: v_xor_b32_e32 v0, s20, v0 +; GFX10-NEXT: v_xor_b32_e32 v1, s21, v1 +; GFX10-NEXT: v_xor_b32_e32 v2, s22, v2 ; GFX10-NEXT: v_xor_b32_e32 v3, s0, v3 ; GFX10-NEXT: v_xor_b32_e32 v4, s16, v4 ; GFX10-NEXT: v_xor_b32_e32 v5, s17, v5 ; GFX10-NEXT: v_xor_b32_e32 v6, s18, v6 ; GFX10-NEXT: v_xor_b32_e32 v7, s19, v7 -; GFX10-NEXT: v_subrev_nc_u32_e32 v0, s12, v0 -; GFX10-NEXT: v_subrev_nc_u32_e32 v1, s13, v1 -; GFX10-NEXT: v_subrev_nc_u32_e32 v2, s14, v2 +; GFX10-NEXT: v_subrev_nc_u32_e32 v0, s20, v0 +; GFX10-NEXT: v_subrev_nc_u32_e32 v1, s21, v1 +; GFX10-NEXT: v_subrev_nc_u32_e32 v2, s22, v2 ; GFX10-NEXT: v_subrev_nc_u32_e32 v3, s0, v3 ; GFX10-NEXT: v_mov_b32_e32 v8, 0 ; GFX10-NEXT: v_subrev_nc_u32_e32 v4, s16, v4 @@ -1338,27 +1330,26 @@ define amdgpu_kernel void @sdivrem_v2i64(<2 x i64> addrspace(1)* %out0, <2 x i64> addrspace(1)* %out1, <2 x i64> %x, <2 x i64> %y) { ; GFX8-LABEL: sdivrem_v2i64: ; GFX8: ; %bb.0: -; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x10 -; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x20 +; GFX8-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x10 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: s_ashr_i32 s6, s9, 31 -; GFX8-NEXT: s_ashr_i32 s12, s1, 31 -; GFX8-NEXT: s_add_u32 s14, s8, s6 -; GFX8-NEXT: s_cselect_b32 s7, 1, 0 -; GFX8-NEXT: s_and_b32 s7, s7, 1 -; GFX8-NEXT: s_cmp_lg_u32 s7, 0 -; GFX8-NEXT: s_addc_u32 s15, s9, s6 -; GFX8-NEXT: s_add_u32 s0, s0, s12 -; GFX8-NEXT: s_cselect_b32 s7, 1, 0 -; GFX8-NEXT: s_and_b32 s7, s7, 1 -; GFX8-NEXT: s_cmp_lg_u32 s7, 0 -; GFX8-NEXT: s_mov_b32 s13, s12 -; GFX8-NEXT: s_addc_u32 s1, s1, s12 -; GFX8-NEXT: s_xor_b64 s[8:9], s[0:1], s[12:13] +; GFX8-NEXT: s_ashr_i32 s2, s9, 31 +; GFX8-NEXT: s_ashr_i32 s6, s13, 31 +; GFX8-NEXT: s_add_u32 s0, s8, s2 +; GFX8-NEXT: s_cselect_b32 s1, 1, 0 +; GFX8-NEXT: s_and_b32 s1, s1, 1 +; GFX8-NEXT: s_cmp_lg_u32 s1, 0 +; GFX8-NEXT: s_addc_u32 s1, s9, s2 +; GFX8-NEXT: s_add_u32 s8, s12, s6 +; GFX8-NEXT: s_cselect_b32 s3, 1, 0 +; GFX8-NEXT: s_and_b32 s3, s3, 1 +; GFX8-NEXT: s_cmp_lg_u32 s3, 0 +; GFX8-NEXT: s_mov_b32 s7, s6 +; GFX8-NEXT: s_addc_u32 s9, s13, s6 +; GFX8-NEXT: s_xor_b64 s[8:9], s[8:9], s[6:7] ; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s9 ; GFX8-NEXT: v_cvt_f32_u32_e32 v1, s8 -; GFX8-NEXT: s_mov_b32 s7, s6 -; GFX8-NEXT: s_xor_b64 s[14:15], s[14:15], s[6:7] +; GFX8-NEXT: s_mov_b32 s3, s2 +; GFX8-NEXT: s_xor_b64 s[12:13], s[0:1], s[2:3] ; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f800000, v0 ; GFX8-NEXT: v_add_f32_e32 v0, v0, v1 ; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0 @@ -1433,24 +1424,24 @@ ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v4 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v3 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v2, vcc -; GFX8-NEXT: v_mul_lo_u32 v2, s15, v0 -; GFX8-NEXT: v_mul_lo_u32 v3, s14, v1 -; GFX8-NEXT: v_mul_hi_u32 v5, s14, v0 -; GFX8-NEXT: v_mul_hi_u32 v0, s15, v0 -; GFX8-NEXT: v_mov_b32_e32 v4, s15 +; GFX8-NEXT: v_mul_lo_u32 v2, s13, v0 +; GFX8-NEXT: v_mul_lo_u32 v3, s12, v1 +; GFX8-NEXT: v_mul_hi_u32 v5, s12, v0 +; GFX8-NEXT: v_mul_hi_u32 v0, s13, v0 +; GFX8-NEXT: v_mov_b32_e32 v4, s13 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v3 ; GFX8-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v5 ; GFX8-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc -; GFX8-NEXT: v_mul_lo_u32 v5, s15, v1 +; GFX8-NEXT: v_mul_lo_u32 v5, s13, v1 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v3, v2 -; GFX8-NEXT: v_mul_hi_u32 v3, s14, v1 +; GFX8-NEXT: v_mul_hi_u32 v3, s12, v1 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, v5, v0 ; GFX8-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc ; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v3 ; GFX8-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc ; GFX8-NEXT: v_add_u32_e32 v3, vcc, v5, v3 -; GFX8-NEXT: v_mul_hi_u32 v1, s15, v1 +; GFX8-NEXT: v_mul_hi_u32 v1, s13, v1 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2 ; GFX8-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v3, v2 @@ -1461,9 +1452,9 @@ ; GFX8-NEXT: v_mul_lo_u32 v5, s8, v0 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v3 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v7 -; GFX8-NEXT: v_sub_u32_e32 v3, vcc, s14, v5 +; GFX8-NEXT: v_sub_u32_e32 v3, vcc, s12, v5 ; GFX8-NEXT: v_subb_u32_e64 v4, s[0:1], v4, v2, vcc -; GFX8-NEXT: v_sub_u32_e64 v2, s[0:1], s15, v2 +; GFX8-NEXT: v_sub_u32_e64 v2, s[0:1], s13, v2 ; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s9, v4 ; GFX8-NEXT: v_cndmask_b32_e64 v5, 0, -1, s[0:1] ; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s8, v3 @@ -1494,37 +1485,37 @@ ; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc ; GFX8-NEXT: v_cndmask_b32_e64 v5, v7, v6, s[0:1] ; GFX8-NEXT: v_cndmask_b32_e64 v2, v8, v2, s[0:1] -; GFX8-NEXT: s_xor_b64 s[0:1], s[6:7], s[12:13] +; GFX8-NEXT: s_xor_b64 s[0:1], s[2:3], s[6:7] ; GFX8-NEXT: v_xor_b32_e32 v0, s0, v0 -; GFX8-NEXT: s_ashr_i32 s8, s11, 31 -; GFX8-NEXT: s_ashr_i32 s12, s3, 31 +; GFX8-NEXT: s_ashr_i32 s6, s11, 31 +; GFX8-NEXT: s_ashr_i32 s8, s15, 31 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc ; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc ; GFX8-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc ; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, s0, v0 -; GFX8-NEXT: s_add_u32 s0, s10, s8 +; GFX8-NEXT: s_add_u32 s0, s10, s6 ; GFX8-NEXT: v_xor_b32_e32 v1, s1, v1 ; GFX8-NEXT: v_mov_b32_e32 v4, s1 ; GFX8-NEXT: s_cselect_b32 s1, 1, 0 ; GFX8-NEXT: s_and_b32 s1, s1, 1 ; GFX8-NEXT: s_cmp_lg_u32 s1, 0 -; GFX8-NEXT: s_addc_u32 s1, s11, s8 -; GFX8-NEXT: s_add_u32 s2, s2, s12 -; GFX8-NEXT: s_cselect_b32 s7, 1, 0 -; GFX8-NEXT: s_and_b32 s7, s7, 1 -; GFX8-NEXT: s_cmp_lg_u32 s7, 0 -; GFX8-NEXT: s_mov_b32 s13, s12 -; GFX8-NEXT: s_addc_u32 s3, s3, s12 -; GFX8-NEXT: s_xor_b64 s[2:3], s[2:3], s[12:13] -; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v1, v4, vcc -; GFX8-NEXT: v_cvt_f32_u32_e32 v4, s3 -; GFX8-NEXT: v_cvt_f32_u32_e32 v5, s2 +; GFX8-NEXT: s_addc_u32 s1, s11, s6 +; GFX8-NEXT: s_add_u32 s10, s14, s8 +; GFX8-NEXT: s_cselect_b32 s3, 1, 0 +; GFX8-NEXT: s_and_b32 s3, s3, 1 +; GFX8-NEXT: s_cmp_lg_u32 s3, 0 ; GFX8-NEXT: s_mov_b32 s9, s8 -; GFX8-NEXT: s_xor_b64 s[10:11], s[0:1], s[8:9] +; GFX8-NEXT: s_addc_u32 s11, s15, s8 +; GFX8-NEXT: s_xor_b64 s[10:11], s[10:11], s[8:9] +; GFX8-NEXT: v_subb_u32_e32 v1, vcc, v1, v4, vcc +; GFX8-NEXT: v_cvt_f32_u32_e32 v4, s11 +; GFX8-NEXT: v_cvt_f32_u32_e32 v5, s10 +; GFX8-NEXT: s_mov_b32 s7, s6 +; GFX8-NEXT: s_xor_b64 s[12:13], s[0:1], s[6:7] ; GFX8-NEXT: v_mul_f32_e32 v4, 0x4f800000, v4 ; GFX8-NEXT: v_add_f32_e32 v4, v4, v5 ; GFX8-NEXT: v_rcp_iflag_f32_e32 v4, v4 -; GFX8-NEXT: s_sub_u32 s0, 0, s2 +; GFX8-NEXT: s_sub_u32 s0, 0, s10 ; GFX8-NEXT: s_cselect_b32 s1, 1, 0 ; GFX8-NEXT: s_and_b32 s1, s1, 1 ; GFX8-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4 @@ -1535,19 +1526,19 @@ ; GFX8-NEXT: v_cvt_u32_f32_e32 v7, v4 ; GFX8-NEXT: v_cvt_u32_f32_e32 v6, v6 ; GFX8-NEXT: s_cmp_lg_u32 s1, 0 -; GFX8-NEXT: s_subb_u32 s1, 0, s3 +; GFX8-NEXT: s_subb_u32 s1, 0, s11 ; GFX8-NEXT: v_mul_lo_u32 v4, s1, v7 ; GFX8-NEXT: v_mul_lo_u32 v8, s0, v6 ; GFX8-NEXT: v_mul_hi_u32 v10, s0, v7 ; GFX8-NEXT: v_mul_lo_u32 v9, s0, v7 -; GFX8-NEXT: v_xor_b32_e32 v3, s6, v3 +; GFX8-NEXT: v_xor_b32_e32 v3, s2, v3 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v8 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, v4, v10 -; GFX8-NEXT: v_xor_b32_e32 v2, s6, v2 -; GFX8-NEXT: v_mov_b32_e32 v5, s6 +; GFX8-NEXT: v_xor_b32_e32 v2, s2, v2 +; GFX8-NEXT: v_mov_b32_e32 v5, s2 ; GFX8-NEXT: v_mul_lo_u32 v10, v6, v9 ; GFX8-NEXT: v_mul_lo_u32 v11, v7, v8 -; GFX8-NEXT: v_subrev_u32_e32 v4, vcc, s6, v3 +; GFX8-NEXT: v_subrev_u32_e32 v4, vcc, s2, v3 ; GFX8-NEXT: v_subb_u32_e32 v5, vcc, v2, v5, vcc ; GFX8-NEXT: v_mul_hi_u32 v2, v7, v9 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, v10, v11 @@ -1574,14 +1565,13 @@ ; GFX8-NEXT: v_mul_lo_u32 v7, s0, v3 ; GFX8-NEXT: v_mul_hi_u32 v9, s0, v2 ; GFX8-NEXT: v_mul_lo_u32 v8, s0, v2 -; GFX8-NEXT: v_mov_b32_e32 v10, s3 +; GFX8-NEXT: v_mov_b32_e32 v10, s11 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v7 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v9 ; GFX8-NEXT: v_mul_lo_u32 v7, v3, v8 ; GFX8-NEXT: v_mul_lo_u32 v9, v2, v6 ; GFX8-NEXT: v_mul_hi_u32 v11, v2, v8 ; GFX8-NEXT: v_mul_hi_u32 v8, v3, v8 -; GFX8-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v9 ; GFX8-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v7, v11 @@ -1601,51 +1591,51 @@ ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v8 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v7 ; GFX8-NEXT: v_addc_u32_e32 v3, vcc, v3, v6, vcc -; GFX8-NEXT: v_mul_lo_u32 v6, s11, v2 -; GFX8-NEXT: v_mul_lo_u32 v7, s10, v3 -; GFX8-NEXT: v_mul_hi_u32 v9, s10, v2 -; GFX8-NEXT: v_mul_hi_u32 v2, s11, v2 -; GFX8-NEXT: v_mov_b32_e32 v8, s11 +; GFX8-NEXT: v_mul_lo_u32 v6, s13, v2 +; GFX8-NEXT: v_mul_lo_u32 v7, s12, v3 +; GFX8-NEXT: v_mul_hi_u32 v9, s12, v2 +; GFX8-NEXT: v_mul_hi_u32 v2, s13, v2 +; GFX8-NEXT: v_mov_b32_e32 v8, s13 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v7 ; GFX8-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v9 ; GFX8-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc -; GFX8-NEXT: v_mul_lo_u32 v9, s11, v3 +; GFX8-NEXT: v_mul_lo_u32 v9, s13, v3 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v7, v6 -; GFX8-NEXT: v_mul_hi_u32 v7, s10, v3 +; GFX8-NEXT: v_mul_hi_u32 v7, s12, v3 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v9, v2 ; GFX8-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v7 ; GFX8-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v9, v7 -; GFX8-NEXT: v_mul_hi_u32 v3, s11, v3 +; GFX8-NEXT: v_mul_hi_u32 v3, s13, v3 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v6 ; GFX8-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v7, v6 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v6 -; GFX8-NEXT: v_mul_lo_u32 v6, s3, v2 -; GFX8-NEXT: v_mul_lo_u32 v7, s2, v3 -; GFX8-NEXT: v_mul_hi_u32 v11, s2, v2 -; GFX8-NEXT: v_mul_lo_u32 v9, s2, v2 +; GFX8-NEXT: v_mul_lo_u32 v6, s11, v2 +; GFX8-NEXT: v_mul_lo_u32 v7, s10, v3 +; GFX8-NEXT: v_mul_hi_u32 v11, s10, v2 +; GFX8-NEXT: v_mul_lo_u32 v9, s10, v2 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v7 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v11 -; GFX8-NEXT: v_sub_u32_e32 v7, vcc, s10, v9 +; GFX8-NEXT: v_sub_u32_e32 v7, vcc, s12, v9 ; GFX8-NEXT: v_subb_u32_e64 v8, s[0:1], v8, v6, vcc -; GFX8-NEXT: v_sub_u32_e64 v6, s[0:1], s11, v6 -; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s3, v8 +; GFX8-NEXT: v_sub_u32_e64 v6, s[0:1], s13, v6 +; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s11, v8 ; GFX8-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[0:1] -; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s2, v7 +; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s10, v7 ; GFX8-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[0:1] -; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], s3, v8 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], s11, v8 ; GFX8-NEXT: v_subb_u32_e32 v6, vcc, v6, v10, vcc ; GFX8-NEXT: v_cndmask_b32_e64 v9, v9, v11, s[0:1] -; GFX8-NEXT: v_subrev_u32_e32 v11, vcc, s2, v7 +; GFX8-NEXT: v_subrev_u32_e32 v11, vcc, s10, v7 ; GFX8-NEXT: v_subbrev_u32_e64 v12, s[0:1], 0, v6, vcc -; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s3, v12 +; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s11, v12 ; GFX8-NEXT: v_cndmask_b32_e64 v13, 0, -1, s[0:1] -; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s2, v11 +; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s10, v11 ; GFX8-NEXT: v_cndmask_b32_e64 v14, 0, -1, s[0:1] -; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], s3, v12 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], s11, v12 ; GFX8-NEXT: v_cndmask_b32_e64 v13, v13, v14, s[0:1] ; GFX8-NEXT: v_add_u32_e64 v14, s[0:1], 1, v2 ; GFX8-NEXT: v_subb_u32_e32 v6, vcc, v6, v10, vcc @@ -1653,61 +1643,61 @@ ; GFX8-NEXT: v_add_u32_e32 v10, vcc, 1, v14 ; GFX8-NEXT: v_addc_u32_e32 v16, vcc, 0, v15, vcc ; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, 0, v13 -; GFX8-NEXT: v_subrev_u32_e64 v13, s[0:1], s2, v11 +; GFX8-NEXT: v_subrev_u32_e64 v13, s[0:1], s10, v11 ; GFX8-NEXT: v_subbrev_u32_e64 v6, s[0:1], 0, v6, s[0:1] ; GFX8-NEXT: v_cndmask_b32_e32 v10, v14, v10, vcc ; GFX8-NEXT: v_cndmask_b32_e32 v14, v15, v16, vcc ; GFX8-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v9 ; GFX8-NEXT: v_cndmask_b32_e32 v9, v11, v13, vcc ; GFX8-NEXT: v_cndmask_b32_e32 v6, v12, v6, vcc +; GFX8-NEXT: s_load_dwordx4 s[12:15], s[4:5], 0x0 ; GFX8-NEXT: v_cndmask_b32_e64 v2, v2, v10, s[0:1] ; GFX8-NEXT: v_cndmask_b32_e64 v3, v3, v14, s[0:1] ; GFX8-NEXT: v_cndmask_b32_e64 v7, v7, v9, s[0:1] ; GFX8-NEXT: v_cndmask_b32_e64 v6, v8, v6, s[0:1] -; GFX8-NEXT: s_xor_b64 s[0:1], s[8:9], s[12:13] +; GFX8-NEXT: s_xor_b64 s[0:1], s[6:7], s[8:9] ; GFX8-NEXT: v_xor_b32_e32 v2, s0, v2 ; GFX8-NEXT: v_xor_b32_e32 v3, s1, v3 ; GFX8-NEXT: v_mov_b32_e32 v8, s1 ; GFX8-NEXT: v_subrev_u32_e32 v2, vcc, s0, v2 ; GFX8-NEXT: v_subb_u32_e32 v3, vcc, v3, v8, vcc -; GFX8-NEXT: v_xor_b32_e32 v7, s8, v7 -; GFX8-NEXT: v_xor_b32_e32 v8, s8, v6 -; GFX8-NEXT: v_mov_b32_e32 v9, s8 -; GFX8-NEXT: v_subrev_u32_e32 v6, vcc, s8, v7 +; GFX8-NEXT: v_xor_b32_e32 v7, s6, v7 +; GFX8-NEXT: v_xor_b32_e32 v8, s6, v6 +; GFX8-NEXT: v_mov_b32_e32 v9, s6 +; GFX8-NEXT: v_subrev_u32_e32 v6, vcc, s6, v7 ; GFX8-NEXT: v_subb_u32_e32 v7, vcc, v8, v9, vcc ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: v_mov_b32_e32 v9, s5 -; GFX8-NEXT: v_mov_b32_e32 v8, s4 +; GFX8-NEXT: v_mov_b32_e32 v8, s12 +; GFX8-NEXT: v_mov_b32_e32 v9, s13 ; GFX8-NEXT: flat_store_dwordx4 v[8:9], v[0:3] ; GFX8-NEXT: s_nop 0 -; GFX8-NEXT: v_mov_b32_e32 v0, s6 -; GFX8-NEXT: v_mov_b32_e32 v1, s7 +; GFX8-NEXT: v_mov_b32_e32 v0, s14 +; GFX8-NEXT: v_mov_b32_e32 v1, s15 ; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[4:7] ; GFX8-NEXT: s_endpgm ; ; GFX9-LABEL: sdivrem_v2i64: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x10 -; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x20 +; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x10 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_ashr_i32 s6, s9, 31 -; GFX9-NEXT: s_ashr_i32 s12, s1, 31 -; GFX9-NEXT: s_add_u32 s14, s8, s6 -; GFX9-NEXT: s_cselect_b32 s7, 1, 0 -; GFX9-NEXT: s_and_b32 s7, s7, 1 -; GFX9-NEXT: s_cmp_lg_u32 s7, 0 -; GFX9-NEXT: s_addc_u32 s15, s9, s6 -; GFX9-NEXT: s_add_u32 s0, s0, s12 -; GFX9-NEXT: s_cselect_b32 s7, 1, 0 -; GFX9-NEXT: s_and_b32 s7, s7, 1 -; GFX9-NEXT: s_cmp_lg_u32 s7, 0 -; GFX9-NEXT: s_mov_b32 s13, s12 -; GFX9-NEXT: s_addc_u32 s1, s1, s12 -; GFX9-NEXT: s_xor_b64 s[8:9], s[0:1], s[12:13] +; GFX9-NEXT: s_ashr_i32 s2, s9, 31 +; GFX9-NEXT: s_ashr_i32 s6, s13, 31 +; GFX9-NEXT: s_add_u32 s0, s8, s2 +; GFX9-NEXT: s_cselect_b32 s1, 1, 0 +; GFX9-NEXT: s_and_b32 s1, s1, 1 +; GFX9-NEXT: s_cmp_lg_u32 s1, 0 +; GFX9-NEXT: s_addc_u32 s1, s9, s2 +; GFX9-NEXT: s_add_u32 s8, s12, s6 +; GFX9-NEXT: s_cselect_b32 s3, 1, 0 +; GFX9-NEXT: s_and_b32 s3, s3, 1 +; GFX9-NEXT: s_cmp_lg_u32 s3, 0 +; GFX9-NEXT: s_mov_b32 s7, s6 +; GFX9-NEXT: s_addc_u32 s9, s13, s6 +; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[6:7] ; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s9 ; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s8 -; GFX9-NEXT: s_mov_b32 s7, s6 -; GFX9-NEXT: s_xor_b64 s[14:15], s[14:15], s[6:7] +; GFX9-NEXT: s_mov_b32 s3, s2 +; GFX9-NEXT: s_xor_b64 s[12:13], s[0:1], s[2:3] ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f800000, v0 ; GFX9-NEXT: v_add_f32_e32 v0, v0, v1 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 @@ -1727,7 +1717,7 @@ ; GFX9-NEXT: v_mul_lo_u32 v3, s1, v0 ; GFX9-NEXT: v_mul_hi_u32 v4, s0, v0 ; GFX9-NEXT: v_mul_lo_u32 v5, s0, v0 -; GFX9-NEXT: v_mov_b32_e32 v7, s15 +; GFX9-NEXT: v_mov_b32_e32 v7, s13 ; GFX9-NEXT: v_add3_u32 v2, v3, v2, v4 ; GFX9-NEXT: v_mul_lo_u32 v3, v1, v5 ; GFX9-NEXT: v_mul_lo_u32 v4, v0, v2 @@ -1778,19 +1768,19 @@ ; GFX9-NEXT: v_add3_u32 v2, v5, v4, v2 ; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v3 ; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v2, vcc -; GFX9-NEXT: v_mul_lo_u32 v2, s15, v0 -; GFX9-NEXT: v_mul_lo_u32 v3, s14, v1 -; GFX9-NEXT: v_mul_hi_u32 v4, s14, v0 -; GFX9-NEXT: v_mul_hi_u32 v0, s15, v0 +; GFX9-NEXT: v_mul_lo_u32 v2, s13, v0 +; GFX9-NEXT: v_mul_lo_u32 v3, s12, v1 +; GFX9-NEXT: v_mul_hi_u32 v4, s12, v0 +; GFX9-NEXT: v_mul_hi_u32 v0, s13, v0 ; GFX9-NEXT: v_mov_b32_e32 v5, s9 ; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v3 ; GFX9-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc ; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 ; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc -; GFX9-NEXT: v_mul_lo_u32 v4, s15, v1 +; GFX9-NEXT: v_mul_lo_u32 v4, s13, v1 ; GFX9-NEXT: v_add_u32_e32 v2, v3, v2 -; GFX9-NEXT: v_mul_hi_u32 v3, s14, v1 -; GFX9-NEXT: v_mul_hi_u32 v1, s15, v1 +; GFX9-NEXT: v_mul_hi_u32 v3, s12, v1 +; GFX9-NEXT: v_mul_hi_u32 v1, s13, v1 ; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v4, v0 ; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc ; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v3 @@ -1804,10 +1794,10 @@ ; GFX9-NEXT: v_mul_hi_u32 v4, s8, v0 ; GFX9-NEXT: v_mul_lo_u32 v6, s8, v0 ; GFX9-NEXT: v_add3_u32 v2, v2, v3, v4 -; GFX9-NEXT: v_sub_co_u32_e32 v3, vcc, s14, v6 +; GFX9-NEXT: v_sub_co_u32_e32 v3, vcc, s12, v6 ; GFX9-NEXT: v_subb_co_u32_e64 v4, s[0:1], v7, v2, vcc ; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s9, v4 -; GFX9-NEXT: v_sub_u32_e32 v2, s15, v2 +; GFX9-NEXT: v_sub_u32_e32 v2, s13, v2 ; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[0:1] ; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s8, v3 ; GFX9-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[0:1] @@ -1832,35 +1822,35 @@ ; GFX9-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v11 ; GFX9-NEXT: v_cndmask_b32_e64 v5, v7, v5, s[0:1] ; GFX9-NEXT: v_cndmask_b32_e64 v2, v8, v2, s[0:1] -; GFX9-NEXT: s_xor_b64 s[0:1], s[6:7], s[12:13] -; GFX9-NEXT: s_ashr_i32 s8, s11, 31 -; GFX9-NEXT: s_ashr_i32 s12, s3, 31 -; GFX9-NEXT: s_add_u32 s10, s10, s8 -; GFX9-NEXT: s_cselect_b32 s7, 1, 0 -; GFX9-NEXT: s_and_b32 s7, s7, 1 -; GFX9-NEXT: s_cmp_lg_u32 s7, 0 -; GFX9-NEXT: s_addc_u32 s11, s11, s8 -; GFX9-NEXT: s_add_u32 s2, s2, s12 -; GFX9-NEXT: s_cselect_b32 s7, 1, 0 -; GFX9-NEXT: s_and_b32 s7, s7, 1 -; GFX9-NEXT: s_cmp_lg_u32 s7, 0 -; GFX9-NEXT: s_mov_b32 s13, s12 -; GFX9-NEXT: s_addc_u32 s3, s3, s12 +; GFX9-NEXT: s_xor_b64 s[0:1], s[2:3], s[6:7] +; GFX9-NEXT: s_ashr_i32 s6, s11, 31 +; GFX9-NEXT: s_ashr_i32 s8, s15, 31 +; GFX9-NEXT: s_add_u32 s12, s10, s6 +; GFX9-NEXT: s_cselect_b32 s3, 1, 0 +; GFX9-NEXT: s_and_b32 s3, s3, 1 +; GFX9-NEXT: s_cmp_lg_u32 s3, 0 +; GFX9-NEXT: s_addc_u32 s13, s11, s6 +; GFX9-NEXT: s_add_u32 s10, s14, s8 +; GFX9-NEXT: s_cselect_b32 s3, 1, 0 +; GFX9-NEXT: s_and_b32 s3, s3, 1 +; GFX9-NEXT: s_cmp_lg_u32 s3, 0 +; GFX9-NEXT: s_mov_b32 s9, s8 +; GFX9-NEXT: s_addc_u32 s11, s15, s8 ; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v11 -; GFX9-NEXT: s_xor_b64 s[2:3], s[2:3], s[12:13] +; GFX9-NEXT: s_xor_b64 s[10:11], s[10:11], s[8:9] ; GFX9-NEXT: v_cndmask_b32_e32 v9, v9, v12, vcc ; GFX9-NEXT: v_cndmask_b32_e32 v10, v10, v13, vcc ; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v6 -; GFX9-NEXT: v_cvt_f32_u32_e32 v6, s3 -; GFX9-NEXT: v_cvt_f32_u32_e32 v7, s2 +; GFX9-NEXT: v_cvt_f32_u32_e32 v6, s11 +; GFX9-NEXT: v_cvt_f32_u32_e32 v7, s10 ; GFX9-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc ; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc ; GFX9-NEXT: v_mul_f32_e32 v4, 0x4f800000, v6 ; GFX9-NEXT: v_add_f32_e32 v4, v4, v7 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v4, v4 -; GFX9-NEXT: s_mov_b32 s9, s8 -; GFX9-NEXT: s_xor_b64 s[10:11], s[10:11], s[8:9] -; GFX9-NEXT: s_sub_u32 s7, 0, s2 +; GFX9-NEXT: s_mov_b32 s7, s6 +; GFX9-NEXT: s_xor_b64 s[12:13], s[12:13], s[6:7] +; GFX9-NEXT: s_sub_u32 s3, 0, s10 ; GFX9-NEXT: v_mul_f32_e32 v4, 0x5f7ffffc, v4 ; GFX9-NEXT: v_mul_f32_e32 v5, 0x2f800000, v4 ; GFX9-NEXT: v_trunc_f32_e32 v5, v5 @@ -1871,12 +1861,12 @@ ; GFX9-NEXT: s_cselect_b32 s14, 1, 0 ; GFX9-NEXT: s_and_b32 s14, s14, 1 ; GFX9-NEXT: s_cmp_lg_u32 s14, 0 -; GFX9-NEXT: s_subb_u32 s14, 0, s3 +; GFX9-NEXT: s_subb_u32 s14, 0, s11 ; GFX9-NEXT: v_mul_lo_u32 v6, s14, v4 -; GFX9-NEXT: v_mul_lo_u32 v7, s7, v5 -; GFX9-NEXT: v_mul_hi_u32 v8, s7, v4 +; GFX9-NEXT: v_mul_lo_u32 v7, s3, v5 +; GFX9-NEXT: v_mul_hi_u32 v8, s3, v4 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc -; GFX9-NEXT: v_mul_lo_u32 v9, s7, v4 +; GFX9-NEXT: v_mul_lo_u32 v9, s3, v4 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc ; GFX9-NEXT: v_xor_b32_e32 v0, s0, v0 ; GFX9-NEXT: v_add3_u32 v6, v6, v7, v8 @@ -1907,17 +1897,17 @@ ; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v7 ; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v5, v6, vcc ; GFX9-NEXT: v_mul_lo_u32 v6, s14, v4 -; GFX9-NEXT: v_mul_lo_u32 v7, s7, v5 -; GFX9-NEXT: v_mul_hi_u32 v8, s7, v4 -; GFX9-NEXT: v_mul_lo_u32 v9, s7, v4 -; GFX9-NEXT: v_xor_b32_e32 v3, s6, v3 -; GFX9-NEXT: v_xor_b32_e32 v2, s6, v2 +; GFX9-NEXT: v_mul_lo_u32 v7, s3, v5 +; GFX9-NEXT: v_mul_hi_u32 v8, s3, v4 +; GFX9-NEXT: v_mul_lo_u32 v9, s3, v4 +; GFX9-NEXT: v_xor_b32_e32 v3, s2, v3 +; GFX9-NEXT: v_xor_b32_e32 v2, s2, v2 ; GFX9-NEXT: v_add3_u32 v6, v6, v7, v8 ; GFX9-NEXT: v_mul_lo_u32 v7, v5, v9 ; GFX9-NEXT: v_mul_lo_u32 v8, v4, v6 ; GFX9-NEXT: v_mul_hi_u32 v11, v4, v9 ; GFX9-NEXT: v_mul_hi_u32 v9, v5, v9 -; GFX9-NEXT: v_mov_b32_e32 v10, s6 +; GFX9-NEXT: v_mov_b32_e32 v10, s2 ; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, v7, v8 ; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc ; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, v7, v11 @@ -1936,20 +1926,20 @@ ; GFX9-NEXT: v_add3_u32 v6, v9, v8, v6 ; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, v4, v7 ; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, v5, v6, vcc -; GFX9-NEXT: v_mul_lo_u32 v8, s11, v7 -; GFX9-NEXT: v_mul_lo_u32 v9, s10, v6 -; GFX9-NEXT: v_subrev_co_u32_e32 v4, vcc, s6, v3 +; GFX9-NEXT: v_mul_lo_u32 v8, s13, v7 +; GFX9-NEXT: v_mul_lo_u32 v9, s12, v6 +; GFX9-NEXT: v_subrev_co_u32_e32 v4, vcc, s2, v3 ; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v2, v10, vcc -; GFX9-NEXT: v_mul_hi_u32 v2, s10, v7 +; GFX9-NEXT: v_mul_hi_u32 v2, s12, v7 ; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, v8, v9 ; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc ; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v3, v2 ; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc -; GFX9-NEXT: v_mul_lo_u32 v3, s11, v6 -; GFX9-NEXT: v_mul_hi_u32 v7, s11, v7 +; GFX9-NEXT: v_mul_lo_u32 v3, s13, v6 +; GFX9-NEXT: v_mul_hi_u32 v7, s13, v7 ; GFX9-NEXT: v_add_u32_e32 v2, v8, v2 -; GFX9-NEXT: v_mul_hi_u32 v8, s10, v6 -; GFX9-NEXT: v_mul_hi_u32 v6, s11, v6 +; GFX9-NEXT: v_mul_hi_u32 v8, s12, v6 +; GFX9-NEXT: v_mul_hi_u32 v6, s13, v6 ; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, v3, v7 ; GFX9-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc ; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, v3, v8 @@ -1958,30 +1948,30 @@ ; GFX9-NEXT: v_add_u32_e32 v7, v7, v8 ; GFX9-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc ; GFX9-NEXT: v_add3_u32 v3, v7, v3, v6 -; GFX9-NEXT: v_mul_lo_u32 v6, s3, v2 -; GFX9-NEXT: v_mul_lo_u32 v7, s2, v3 -; GFX9-NEXT: v_mul_hi_u32 v8, s2, v2 -; GFX9-NEXT: v_mul_lo_u32 v10, s2, v2 -; GFX9-NEXT: v_mov_b32_e32 v11, s11 -; GFX9-NEXT: v_mov_b32_e32 v9, s3 +; GFX9-NEXT: v_mul_lo_u32 v6, s11, v2 +; GFX9-NEXT: v_mul_lo_u32 v7, s10, v3 +; GFX9-NEXT: v_mul_hi_u32 v8, s10, v2 +; GFX9-NEXT: v_mul_lo_u32 v10, s10, v2 +; GFX9-NEXT: v_mov_b32_e32 v11, s13 +; GFX9-NEXT: v_mov_b32_e32 v9, s11 ; GFX9-NEXT: v_add3_u32 v6, v6, v7, v8 -; GFX9-NEXT: v_sub_co_u32_e32 v7, vcc, s10, v10 +; GFX9-NEXT: v_sub_co_u32_e32 v7, vcc, s12, v10 ; GFX9-NEXT: v_subb_co_u32_e64 v8, s[0:1], v11, v6, vcc -; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s3, v8 -; GFX9-NEXT: v_sub_u32_e32 v6, s11, v6 +; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s11, v8 +; GFX9-NEXT: v_sub_u32_e32 v6, s13, v6 ; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[0:1] -; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s2, v7 +; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s10, v7 ; GFX9-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[0:1] -; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s3, v8 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s11, v8 ; GFX9-NEXT: v_subb_co_u32_e32 v6, vcc, v6, v9, vcc ; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, v11, s[0:1] -; GFX9-NEXT: v_subrev_co_u32_e32 v11, vcc, s2, v7 +; GFX9-NEXT: v_subrev_co_u32_e32 v11, vcc, s10, v7 ; GFX9-NEXT: v_subbrev_co_u32_e64 v12, s[0:1], 0, v6, vcc -; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s3, v12 +; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s11, v12 ; GFX9-NEXT: v_cndmask_b32_e64 v13, 0, -1, s[0:1] -; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s2, v11 +; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s10, v11 ; GFX9-NEXT: v_cndmask_b32_e64 v14, 0, -1, s[0:1] -; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s3, v12 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s11, v12 ; GFX9-NEXT: v_cndmask_b32_e64 v13, v13, v14, s[0:1] ; GFX9-NEXT: v_add_co_u32_e64 v14, s[0:1], 1, v2 ; GFX9-NEXT: v_subb_co_u32_e32 v6, vcc, v6, v9, vcc @@ -1991,96 +1981,94 @@ ; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v13 ; GFX9-NEXT: v_cndmask_b32_e32 v9, v14, v9, vcc ; GFX9-NEXT: v_cndmask_b32_e32 v14, v15, v16, vcc -; GFX9-NEXT: v_subrev_co_u32_e64 v15, s[0:1], s2, v11 +; GFX9-NEXT: v_subrev_co_u32_e64 v15, s[0:1], s10, v11 ; GFX9-NEXT: v_subbrev_co_u32_e64 v6, s[0:1], 0, v6, s[0:1] ; GFX9-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v10 ; GFX9-NEXT: v_cndmask_b32_e64 v2, v2, v9, s[0:1] ; GFX9-NEXT: v_cndmask_b32_e32 v9, v11, v15, vcc ; GFX9-NEXT: v_cndmask_b32_e32 v6, v12, v6, vcc -; GFX9-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0 +; GFX9-NEXT: s_load_dwordx4 s[12:15], s[4:5], 0x0 ; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v14, s[0:1] ; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, v9, s[0:1] ; GFX9-NEXT: v_cndmask_b32_e64 v6, v8, v6, s[0:1] -; GFX9-NEXT: s_xor_b64 s[0:1], s[8:9], s[12:13] +; GFX9-NEXT: s_xor_b64 s[0:1], s[6:7], s[8:9] ; GFX9-NEXT: v_xor_b32_e32 v2, s0, v2 ; GFX9-NEXT: v_xor_b32_e32 v3, s1, v3 ; GFX9-NEXT: v_mov_b32_e32 v8, s1 ; GFX9-NEXT: v_subrev_co_u32_e32 v2, vcc, s0, v2 ; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v8, vcc -; GFX9-NEXT: v_xor_b32_e32 v7, s8, v7 +; GFX9-NEXT: v_xor_b32_e32 v7, s6, v7 ; GFX9-NEXT: v_mov_b32_e32 v13, 0 -; GFX9-NEXT: v_xor_b32_e32 v8, s8, v6 -; GFX9-NEXT: v_mov_b32_e32 v9, s8 -; GFX9-NEXT: v_subrev_co_u32_e32 v6, vcc, s8, v7 +; GFX9-NEXT: v_xor_b32_e32 v8, s6, v6 +; GFX9-NEXT: v_mov_b32_e32 v9, s6 +; GFX9-NEXT: v_subrev_co_u32_e32 v6, vcc, s6, v7 ; GFX9-NEXT: v_subb_co_u32_e32 v7, vcc, v8, v9, vcc ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: global_store_dwordx4 v13, v[0:3], s[4:5] -; GFX9-NEXT: global_store_dwordx4 v13, v[4:7], s[6:7] +; GFX9-NEXT: global_store_dwordx4 v13, v[0:3], s[12:13] +; GFX9-NEXT: global_store_dwordx4 v13, v[4:7], s[14:15] ; GFX9-NEXT: s_endpgm ; ; GFX10-LABEL: sdivrem_v2i64: ; GFX10: ; %bb.0: -; GFX10-NEXT: s_clause 0x1 -; GFX10-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x10 -; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x20 +; GFX10-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x10 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_ashr_i32 s12, s9, 31 -; GFX10-NEXT: s_ashr_i32 s6, s1, 31 -; GFX10-NEXT: s_add_u32 s14, s8, s12 -; GFX10-NEXT: s_cselect_b32 s7, 1, 0 -; GFX10-NEXT: s_mov_b32 s13, s12 -; GFX10-NEXT: s_and_b32 s7, s7, 1 -; GFX10-NEXT: s_cmp_lg_u32 s7, 0 -; GFX10-NEXT: s_addc_u32 s15, s9, s12 -; GFX10-NEXT: s_add_u32 s0, s0, s6 -; GFX10-NEXT: s_cselect_b32 s7, 1, 0 -; GFX10-NEXT: s_and_b32 s8, s7, 1 +; GFX10-NEXT: s_ashr_i32 s2, s9, 31 +; GFX10-NEXT: s_ashr_i32 s6, s13, 31 +; GFX10-NEXT: s_add_u32 s0, s8, s2 +; GFX10-NEXT: s_cselect_b32 s1, 1, 0 ; GFX10-NEXT: s_mov_b32 s7, s6 -; GFX10-NEXT: s_cmp_lg_u32 s8, 0 -; GFX10-NEXT: s_addc_u32 s1, s1, s6 -; GFX10-NEXT: s_xor_b64 s[8:9], s[0:1], s[6:7] -; GFX10-NEXT: s_xor_b64 s[0:1], s[14:15], s[12:13] +; GFX10-NEXT: s_and_b32 s1, s1, 1 +; GFX10-NEXT: s_cmp_lg_u32 s1, 0 +; GFX10-NEXT: s_addc_u32 s1, s9, s2 +; GFX10-NEXT: s_add_u32 s8, s12, s6 +; GFX10-NEXT: s_cselect_b32 s3, 1, 0 +; GFX10-NEXT: s_and_b32 s3, s3, 1 +; GFX10-NEXT: s_cmp_lg_u32 s3, 0 +; GFX10-NEXT: s_mov_b32 s3, s2 +; GFX10-NEXT: s_addc_u32 s9, s13, s6 +; GFX10-NEXT: s_xor_b64 s[0:1], s[0:1], s[2:3] +; GFX10-NEXT: s_xor_b64 s[8:9], s[8:9], s[6:7] ; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s9 ; GFX10-NEXT: s_sub_u32 s20, 0, s8 -; GFX10-NEXT: s_cselect_b32 s14, 1, 0 +; GFX10-NEXT: s_cselect_b32 s12, 1, 0 ; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s8 -; GFX10-NEXT: s_and_b32 s14, s14, 1 +; GFX10-NEXT: s_and_b32 s12, s12, 1 ; GFX10-NEXT: v_mul_f32_e32 v1, 0x4f800000, v1 -; GFX10-NEXT: s_cmp_lg_u32 s14, 0 +; GFX10-NEXT: s_cmp_lg_u32 s12, 0 ; GFX10-NEXT: s_subb_u32 s21, 0, s9 -; GFX10-NEXT: s_ashr_i32 s14, s11, 31 -; GFX10-NEXT: s_xor_b64 s[18:19], s[12:13], s[6:7] -; GFX10-NEXT: s_ashr_i32 s16, s3, 31 +; GFX10-NEXT: s_ashr_i32 s12, s11, 31 +; GFX10-NEXT: s_xor_b64 s[18:19], s[2:3], s[6:7] +; GFX10-NEXT: s_ashr_i32 s16, s15, 31 ; GFX10-NEXT: v_add_f32_e32 v0, v1, v0 -; GFX10-NEXT: s_add_u32 s6, s10, s14 -; GFX10-NEXT: s_cselect_b32 s7, 1, 0 +; GFX10-NEXT: s_add_u32 s6, s10, s12 +; GFX10-NEXT: s_cselect_b32 s3, 1, 0 ; GFX10-NEXT: s_mov_b32 s17, s16 -; GFX10-NEXT: s_and_b32 s7, s7, 1 +; GFX10-NEXT: s_and_b32 s3, s3, 1 ; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX10-NEXT: s_cmp_lg_u32 s7, 0 -; GFX10-NEXT: s_mov_b32 s15, s14 -; GFX10-NEXT: s_addc_u32 s7, s11, s14 -; GFX10-NEXT: s_add_u32 s2, s2, s16 -; GFX10-NEXT: s_cselect_b32 s10, 1, 0 -; GFX10-NEXT: s_and_b32 s10, s10, 1 -; GFX10-NEXT: s_cmp_lg_u32 s10, 0 +; GFX10-NEXT: s_cmp_lg_u32 s3, 0 +; GFX10-NEXT: s_mov_b32 s13, s12 +; GFX10-NEXT: s_addc_u32 s7, s11, s12 +; GFX10-NEXT: s_add_u32 s10, s14, s16 +; GFX10-NEXT: s_cselect_b32 s3, 1, 0 +; GFX10-NEXT: s_and_b32 s3, s3, 1 +; GFX10-NEXT: s_cmp_lg_u32 s3, 0 ; GFX10-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 -; GFX10-NEXT: s_addc_u32 s3, s3, s16 -; GFX10-NEXT: s_xor_b64 s[10:11], s[6:7], s[14:15] -; GFX10-NEXT: s_xor_b64 s[2:3], s[2:3], s[16:17] -; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s3 +; GFX10-NEXT: s_addc_u32 s11, s15, s16 +; GFX10-NEXT: s_xor_b64 s[14:15], s[6:7], s[12:13] +; GFX10-NEXT: s_xor_b64 s[10:11], s[10:11], s[16:17] +; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s11 ; GFX10-NEXT: v_mul_f32_e32 v2, 0x2f800000, v0 -; GFX10-NEXT: v_cvt_f32_u32_e32 v3, s2 -; GFX10-NEXT: s_sub_u32 s6, 0, s2 -; GFX10-NEXT: s_cselect_b32 s7, 1, 0 +; GFX10-NEXT: v_cvt_f32_u32_e32 v3, s10 +; GFX10-NEXT: s_sub_u32 s3, 0, s10 +; GFX10-NEXT: s_cselect_b32 s6, 1, 0 ; GFX10-NEXT: v_mul_f32_e32 v1, 0x4f800000, v1 ; GFX10-NEXT: v_trunc_f32_e32 v2, v2 -; GFX10-NEXT: s_and_b32 s7, s7, 1 -; GFX10-NEXT: s_cmp_lg_u32 s7, 0 +; GFX10-NEXT: s_and_b32 s6, s6, 1 +; GFX10-NEXT: s_cmp_lg_u32 s6, 0 ; GFX10-NEXT: v_add_f32_e32 v1, v1, v3 ; GFX10-NEXT: v_mul_f32_e32 v3, 0xcf800000, v2 ; GFX10-NEXT: v_cvt_u32_f32_e32 v2, v2 -; GFX10-NEXT: s_subb_u32 s7, 0, s3 +; GFX10-NEXT: s_subb_u32 s6, 0, s11 ; GFX10-NEXT: v_rcp_iflag_f32_e32 v1, v1 ; GFX10-NEXT: v_add_f32_e32 v0, v3, v0 ; GFX10-NEXT: v_mul_lo_u32 v3, s20, v2 @@ -2102,23 +2090,23 @@ ; GFX10-NEXT: v_cvt_u32_f32_e32 v4, v4 ; GFX10-NEXT: v_mul_hi_u32 v3, v2, v3 ; GFX10-NEXT: v_add_f32_e32 v1, v9, v1 -; GFX10-NEXT: v_add_co_u32 v5, s13, v5, v8 -; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, s13 -; GFX10-NEXT: v_add_co_u32 v6, s13, v10, v6 -; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, 1, s13 -; GFX10-NEXT: v_add_co_u32 v5, s13, v5, v7 +; GFX10-NEXT: v_add_co_u32 v5, s7, v5, v8 +; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, s7 +; GFX10-NEXT: v_add_co_u32 v6, s7, v10, v6 +; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, 1, s7 +; GFX10-NEXT: v_add_co_u32 v5, s7, v5, v7 ; GFX10-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s13 -; GFX10-NEXT: v_mul_lo_u32 v9, s6, v4 -; GFX10-NEXT: v_add_co_u32 v6, s13, v6, v11 -; GFX10-NEXT: v_mul_lo_u32 v12, s7, v1 -; GFX10-NEXT: v_mul_hi_u32 v13, s6, v1 +; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s7 +; GFX10-NEXT: v_mul_lo_u32 v9, s3, v4 +; GFX10-NEXT: v_add_co_u32 v6, s7, v6, v11 +; GFX10-NEXT: v_mul_lo_u32 v12, s6, v1 +; GFX10-NEXT: v_mul_hi_u32 v13, s3, v1 ; GFX10-NEXT: v_add_nc_u32_e32 v5, v8, v5 -; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, s13 -; GFX10-NEXT: v_mul_lo_u32 v11, s6, v1 -; GFX10-NEXT: v_add_co_u32 v5, s13, v6, v5 +; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, s7 +; GFX10-NEXT: v_mul_lo_u32 v11, s3, v1 +; GFX10-NEXT: v_add_co_u32 v5, s7, v6, v5 ; GFX10-NEXT: v_add_nc_u32_e32 v7, v10, v7 -; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s13 +; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s7 ; GFX10-NEXT: v_add3_u32 v8, v12, v9, v13 ; GFX10-NEXT: v_mul_lo_u32 v9, v4, v11 ; GFX10-NEXT: v_mul_hi_u32 v10, v1, v11 @@ -2130,218 +2118,218 @@ ; GFX10-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v3, vcc_lo ; GFX10-NEXT: v_mul_hi_u32 v5, v1, v8 ; GFX10-NEXT: v_mul_lo_u32 v12, s21, v0 -; GFX10-NEXT: v_add_co_u32 v6, s13, v9, v6 +; GFX10-NEXT: v_add_co_u32 v6, s7, v9, v6 ; GFX10-NEXT: v_mul_hi_u32 v13, s20, v0 ; GFX10-NEXT: v_mul_lo_u32 v14, s20, v2 -; GFX10-NEXT: v_cndmask_b32_e64 v9, 0, 1, s13 -; GFX10-NEXT: v_add_co_u32 v7, s13, v7, v11 -; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, 1, s13 -; GFX10-NEXT: v_add_co_u32 v6, s13, v6, v10 +; GFX10-NEXT: v_cndmask_b32_e64 v9, 0, 1, s7 +; GFX10-NEXT: v_add_co_u32 v7, s7, v7, v11 +; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, 1, s7 +; GFX10-NEXT: v_add_co_u32 v6, s7, v6, v10 ; GFX10-NEXT: v_mul_lo_u32 v3, s20, v0 -; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s13 -; GFX10-NEXT: v_add_co_u32 v5, s13, v7, v5 -; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, s13 +; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s7 +; GFX10-NEXT: v_add_co_u32 v5, s7, v7, v5 +; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, s7 ; GFX10-NEXT: v_add3_u32 v12, v12, v14, v13 ; GFX10-NEXT: v_add_nc_u32_e32 v6, v9, v6 ; GFX10-NEXT: v_mul_hi_u32 v8, v4, v8 ; GFX10-NEXT: v_mul_lo_u32 v10, v2, v3 ; GFX10-NEXT: v_add_nc_u32_e32 v7, v11, v7 ; GFX10-NEXT: v_mul_lo_u32 v11, v0, v12 -; GFX10-NEXT: v_add_co_u32 v5, s13, v5, v6 +; GFX10-NEXT: v_add_co_u32 v5, s7, v5, v6 ; GFX10-NEXT: v_mul_hi_u32 v9, v0, v3 -; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s13 +; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s7 ; GFX10-NEXT: v_mul_hi_u32 v3, v2, v3 ; GFX10-NEXT: v_mul_lo_u32 v13, v2, v12 ; GFX10-NEXT: v_add_co_u32 v1, vcc_lo, v1, v5 -; GFX10-NEXT: v_add_co_u32 v5, s13, v10, v11 -; GFX10-NEXT: v_mul_hi_u32 v14, v0, v12 +; GFX10-NEXT: v_add_co_u32 v5, s7, v10, v11 ; GFX10-NEXT: v_add3_u32 v6, v7, v6, v8 -; GFX10-NEXT: v_mul_lo_u32 v10, s7, v1 -; GFX10-NEXT: v_add_co_u32 v5, s7, v5, v9 -; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, s13 -; GFX10-NEXT: v_add_co_u32 v3, s13, v13, v3 -; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s7 +; GFX10-NEXT: v_mul_hi_u32 v14, v0, v12 +; GFX10-NEXT: v_mul_lo_u32 v10, s6, v1 +; GFX10-NEXT: v_add_co_u32 v5, s6, v5, v9 +; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, s7 +; GFX10-NEXT: v_add_co_u32 v3, s7, v13, v3 ; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v6, vcc_lo -; GFX10-NEXT: v_add_co_u32 v3, s7, v3, v14 +; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s6 +; GFX10-NEXT: v_mul_hi_u32 v11, s3, v1 +; GFX10-NEXT: v_add_co_u32 v3, s6, v3, v14 +; GFX10-NEXT: v_mul_lo_u32 v13, s3, v4 ; GFX10-NEXT: v_add_nc_u32_e32 v5, v7, v5 -; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, s13 -; GFX10-NEXT: v_mul_hi_u32 v11, s6, v1 -; GFX10-NEXT: v_cndmask_b32_e64 v9, 0, 1, s7 -; GFX10-NEXT: v_mul_lo_u32 v13, s6, v4 +; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, s7 +; GFX10-NEXT: v_cndmask_b32_e64 v9, 0, 1, s6 ; GFX10-NEXT: v_mul_hi_u32 v7, v2, v12 -; GFX10-NEXT: v_mul_lo_u32 v6, s6, v1 -; GFX10-NEXT: v_add_co_u32 v3, s6, v3, v5 +; GFX10-NEXT: v_mul_lo_u32 v6, s3, v1 +; GFX10-NEXT: v_add_co_u32 v3, s3, v3, v5 ; GFX10-NEXT: v_add_nc_u32_e32 v8, v8, v9 -; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s6 +; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s3 ; GFX10-NEXT: v_add3_u32 v9, v10, v13, v11 ; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v3 -; GFX10-NEXT: v_add3_u32 v5, v8, v5, v7 ; GFX10-NEXT: v_mul_lo_u32 v10, v4, v6 +; GFX10-NEXT: v_add3_u32 v5, v8, v5, v7 ; GFX10-NEXT: v_mul_lo_u32 v7, v1, v9 ; GFX10-NEXT: v_mul_hi_u32 v11, v1, v6 ; GFX10-NEXT: v_mul_hi_u32 v6, v4, v6 +; GFX10-NEXT: v_mul_lo_u32 v8, v4, v9 ; GFX10-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo -; GFX10-NEXT: v_mul_lo_u32 v3, v4, v9 +; GFX10-NEXT: v_mul_hi_u32 v3, v1, v9 ; GFX10-NEXT: v_mul_lo_u32 v5, s1, v0 -; GFX10-NEXT: v_mul_hi_u32 v12, s0, v0 +; GFX10-NEXT: v_add_co_u32 v7, s3, v10, v7 ; GFX10-NEXT: v_mul_lo_u32 v13, s0, v2 -; GFX10-NEXT: v_add_co_u32 v7, s6, v10, v7 -; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, 1, s6 +; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, 1, s3 +; GFX10-NEXT: v_add_co_u32 v6, s3, v8, v6 +; GFX10-NEXT: v_mul_hi_u32 v12, s0, v0 ; GFX10-NEXT: v_mul_hi_u32 v0, s1, v0 -; GFX10-NEXT: v_add_co_u32 v3, s6, v3, v6 -; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s6 -; GFX10-NEXT: v_add_co_u32 v7, s6, v7, v11 ; GFX10-NEXT: v_mul_lo_u32 v14, s1, v2 -; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, s6 -; GFX10-NEXT: v_add_co_u32 v5, s6, v5, v13 +; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, s3 +; GFX10-NEXT: v_add_co_u32 v7, s3, v7, v11 +; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, s3 +; GFX10-NEXT: v_add_co_u32 v3, s3, v6, v3 +; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s3 +; GFX10-NEXT: v_add_co_u32 v5, s3, v5, v13 +; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, 1, s3 +; GFX10-NEXT: v_add_co_u32 v0, s3, v14, v0 ; GFX10-NEXT: v_mul_hi_u32 v15, s0, v2 -; GFX10-NEXT: v_mul_hi_u32 v8, v1, v9 -; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, 1, s6 -; GFX10-NEXT: v_add_co_u32 v5, s7, v5, v12 -; GFX10-NEXT: v_add_co_u32 v0, s6, v14, v0 -; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s7 -; GFX10-NEXT: v_cndmask_b32_e64 v12, 0, 1, s6 +; GFX10-NEXT: v_cndmask_b32_e64 v13, 0, 1, s3 +; GFX10-NEXT: v_add_co_u32 v5, s3, v5, v12 +; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s3 ; GFX10-NEXT: v_mul_hi_u32 v2, s1, v2 -; GFX10-NEXT: v_add_co_u32 v0, s6, v0, v15 -; GFX10-NEXT: v_add_nc_u32_e32 v5, v11, v5 -; GFX10-NEXT: v_cndmask_b32_e64 v13, 0, 1, s6 -; GFX10-NEXT: v_add_co_u32 v3, s6, v3, v8 -; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, s6 -; GFX10-NEXT: v_add_co_u32 v0, s6, v0, v5 -; GFX10-NEXT: v_add_nc_u32_e32 v11, v12, v13 -; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s6 ; GFX10-NEXT: v_add_nc_u32_e32 v7, v10, v7 +; GFX10-NEXT: v_add_nc_u32_e32 v6, v8, v6 +; GFX10-NEXT: v_add_co_u32 v0, s3, v0, v15 +; GFX10-NEXT: v_add_nc_u32_e32 v5, v11, v5 +; GFX10-NEXT: v_cndmask_b32_e64 v12, 0, 1, s3 ; GFX10-NEXT: v_mul_hi_u32 v9, v4, v9 -; GFX10-NEXT: v_add_nc_u32_e32 v6, v6, v8 -; GFX10-NEXT: v_mul_hi_u32 v8, s8, v0 -; GFX10-NEXT: v_add3_u32 v2, v11, v5, v2 -; GFX10-NEXT: v_add_co_u32 v3, s6, v3, v7 -; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s6 -; GFX10-NEXT: v_mul_lo_u32 v7, s9, v0 -; GFX10-NEXT: v_mul_lo_u32 v10, s8, v2 -; GFX10-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3 -; GFX10-NEXT: v_add3_u32 v5, v6, v5, v9 -; GFX10-NEXT: v_mul_lo_u32 v6, s8, v0 ; GFX10-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0 -; GFX10-NEXT: v_mov_b32_e32 v9, 0 -; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v4, v5, vcc_lo -; GFX10-NEXT: v_add3_u32 v4, v7, v10, v8 -; GFX10-NEXT: v_mul_lo_u32 v5, s11, v1 -; GFX10-NEXT: v_sub_co_u32 v6, vcc_lo, s0, v6 -; GFX10-NEXT: v_mul_lo_u32 v14, s10, v3 -; GFX10-NEXT: v_sub_nc_u32_e32 v8, s1, v4 -; GFX10-NEXT: v_sub_co_ci_u32_e64 v4, s0, s1, v4, vcc_lo -; GFX10-NEXT: v_cmp_le_u32_e64 s0, s8, v6 -; GFX10-NEXT: v_mul_hi_u32 v7, s11, v1 -; GFX10-NEXT: v_subrev_co_ci_u32_e32 v8, vcc_lo, s9, v8, vcc_lo -; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s9, v4 -; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, -1, s0 -; GFX10-NEXT: v_mul_lo_u32 v15, s11, v3 -; GFX10-NEXT: v_mul_hi_u32 v1, s10, v1 -; GFX10-NEXT: v_mul_hi_u32 v17, s10, v3 -; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, -1, vcc_lo -; GFX10-NEXT: v_sub_co_u32 v12, vcc_lo, v6, s8 -; GFX10-NEXT: v_subrev_co_ci_u32_e64 v13, s0, 0, v8, vcc_lo -; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s9, v4 -; GFX10-NEXT: v_mul_hi_u32 v3, s11, v3 -; GFX10-NEXT: v_subrev_co_ci_u32_e32 v8, vcc_lo, s9, v8, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v10, v11, v10, s0 -; GFX10-NEXT: v_cmp_le_u32_e64 s0, s9, v13 -; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, -1, s0 -; GFX10-NEXT: v_cmp_le_u32_e64 s0, s8, v12 +; GFX10-NEXT: v_add_co_u32 v0, s3, v0, v5 +; GFX10-NEXT: v_add_nc_u32_e32 v8, v13, v12 +; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s3 +; GFX10-NEXT: v_add_co_u32 v3, s3, v3, v7 +; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, s3 +; GFX10-NEXT: v_add3_u32 v2, v8, v5, v2 +; GFX10-NEXT: v_add_co_u32 v1, vcc_lo, v1, v3 +; GFX10-NEXT: v_add3_u32 v5, v6, v7, v9 +; GFX10-NEXT: v_mul_lo_u32 v6, s9, v0 +; GFX10-NEXT: v_mul_hi_u32 v7, s8, v0 +; GFX10-NEXT: v_mul_lo_u32 v9, s8, v2 +; GFX10-NEXT: v_mul_lo_u32 v3, s8, v0 +; GFX10-NEXT: v_add_co_ci_u32_e32 v4, vcc_lo, v4, v5, vcc_lo +; GFX10-NEXT: v_mul_lo_u32 v5, s15, v1 +; GFX10-NEXT: v_mul_hi_u32 v10, s15, v1 +; GFX10-NEXT: v_mul_hi_u32 v1, s14, v1 +; GFX10-NEXT: v_mul_hi_u32 v17, s14, v4 +; GFX10-NEXT: v_add3_u32 v6, v6, v9, v7 +; GFX10-NEXT: v_sub_co_u32 v3, vcc_lo, s0, v3 +; GFX10-NEXT: v_mul_lo_u32 v7, s14, v4 +; GFX10-NEXT: v_mul_lo_u32 v9, s15, v4 +; GFX10-NEXT: v_sub_nc_u32_e32 v11, s1, v6 +; GFX10-NEXT: v_sub_co_ci_u32_e64 v6, s0, s1, v6, vcc_lo +; GFX10-NEXT: v_cmp_le_u32_e64 s0, s8, v3 +; GFX10-NEXT: v_mul_hi_u32 v4, s15, v4 +; GFX10-NEXT: v_subrev_co_ci_u32_e32 v11, vcc_lo, s9, v11, vcc_lo +; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s9, v6 +; GFX10-NEXT: v_cndmask_b32_e64 v12, 0, -1, s0 +; GFX10-NEXT: v_mov_b32_e32 v8, 0 +; GFX10-NEXT: v_cndmask_b32_e64 v13, 0, -1, vcc_lo +; GFX10-NEXT: v_sub_co_u32 v14, vcc_lo, v3, s8 +; GFX10-NEXT: v_subrev_co_ci_u32_e64 v15, s0, 0, v11, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s9, v6 +; GFX10-NEXT: v_subrev_co_ci_u32_e32 v11, vcc_lo, s9, v11, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v12, v13, v12, s0 +; GFX10-NEXT: v_cmp_le_u32_e64 s0, s8, v14 +; GFX10-NEXT: v_cndmask_b32_e64 v13, 0, -1, s0 +; GFX10-NEXT: v_cmp_le_u32_e64 s0, s9, v15 ; GFX10-NEXT: v_cndmask_b32_e64 v16, 0, -1, s0 -; GFX10-NEXT: v_add_co_u32 v5, s0, v5, v14 -; GFX10-NEXT: v_cndmask_b32_e64 v14, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v7, s0, v15, v7 +; GFX10-NEXT: v_add_co_u32 v5, s0, v5, v7 +; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0 +; GFX10-NEXT: v_add_co_u32 v9, s0, v9, v10 ; GFX10-NEXT: v_add_co_u32 v1, s1, v5, v1 ; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 1, s1 ; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v7, s0, v7, v17 -; GFX10-NEXT: v_cndmask_b32_e64 v15, 0, 1, s0 +; GFX10-NEXT: v_add_co_u32 v9, s0, v9, v17 +; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0 ; GFX10-NEXT: v_add_co_u32 v17, s0, v0, 1 ; GFX10-NEXT: v_add_co_ci_u32_e64 v18, s0, 0, v2, s0 -; GFX10-NEXT: v_add_nc_u32_e32 v1, v14, v1 -; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s9, v13 -; GFX10-NEXT: v_add_nc_u32_e32 v5, v5, v15 -; GFX10-NEXT: v_cndmask_b32_e64 v11, v11, v16, s0 -; GFX10-NEXT: v_add_co_u32 v7, s0, v7, v1 +; GFX10-NEXT: v_add_nc_u32_e32 v1, v7, v1 +; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s9, v15 +; GFX10-NEXT: v_add_nc_u32_e32 v5, v5, v10 +; GFX10-NEXT: v_cndmask_b32_e64 v7, v16, v13, s0 +; GFX10-NEXT: v_add_co_u32 v9, s0, v9, v1 ; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v14, s0, v17, 1 -; GFX10-NEXT: v_add_co_ci_u32_e64 v15, s0, 0, v18, s0 -; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v11 -; GFX10-NEXT: v_add3_u32 v3, v5, v1, v3 -; GFX10-NEXT: v_sub_co_u32 v1, s0, v12, s8 -; GFX10-NEXT: v_subrev_co_ci_u32_e64 v5, s0, 0, v8, s0 -; GFX10-NEXT: v_cndmask_b32_e32 v8, v17, v14, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v14, v18, v15, vcc_lo -; GFX10-NEXT: v_mul_lo_u32 v15, s3, v7 -; GFX10-NEXT: v_mul_lo_u32 v16, s2, v3 -; GFX10-NEXT: v_mul_hi_u32 v17, s2, v7 -; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v10 -; GFX10-NEXT: v_mul_lo_u32 v10, s2, v7 -; GFX10-NEXT: v_cmp_ne_u32_e64 s0, 0, v11 -; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v8, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v14, vcc_lo -; GFX10-NEXT: v_add3_u32 v8, v15, v16, v17 -; GFX10-NEXT: v_cndmask_b32_e64 v1, v12, v1, s0 -; GFX10-NEXT: v_cndmask_b32_e64 v5, v13, v5, s0 -; GFX10-NEXT: v_sub_co_u32 v10, s0, s10, v10 -; GFX10-NEXT: v_sub_co_ci_u32_e64 v11, s1, s11, v8, s0 -; GFX10-NEXT: v_cndmask_b32_e32 v6, v6, v1, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v5, vcc_lo -; GFX10-NEXT: v_sub_nc_u32_e32 v1, s11, v8 -; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s3, v11 +; GFX10-NEXT: v_add_co_u32 v10, s0, v17, 1 +; GFX10-NEXT: v_add_co_ci_u32_e64 v13, s0, 0, v18, s0 +; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v7 +; GFX10-NEXT: v_add3_u32 v4, v5, v1, v4 +; GFX10-NEXT: v_sub_co_u32 v1, s0, v14, s8 +; GFX10-NEXT: v_subrev_co_ci_u32_e64 v5, s0, 0, v11, s0 +; GFX10-NEXT: v_cndmask_b32_e32 v10, v17, v10, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v11, v18, v13, vcc_lo +; GFX10-NEXT: v_mul_lo_u32 v13, s11, v9 +; GFX10-NEXT: v_mul_lo_u32 v16, s10, v4 +; GFX10-NEXT: v_mul_hi_u32 v17, s10, v9 +; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v12 +; GFX10-NEXT: v_cmp_ne_u32_e64 s0, 0, v7 +; GFX10-NEXT: v_mul_lo_u32 v7, s10, v9 +; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v10, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v1, v14, v1, s0 +; GFX10-NEXT: v_add3_u32 v10, v13, v16, v17 +; GFX10-NEXT: v_cndmask_b32_e64 v5, v15, v5, s0 +; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v11, vcc_lo +; GFX10-NEXT: v_sub_co_u32 v7, s0, s14, v7 +; GFX10-NEXT: v_sub_co_ci_u32_e64 v11, s1, s15, v10, s0 +; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v1, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v5, v6, v5, vcc_lo +; GFX10-NEXT: v_sub_nc_u32_e32 v1, s15, v10 +; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s11, v11 ; GFX10-NEXT: v_xor_b32_e32 v0, s18, v0 ; GFX10-NEXT: v_xor_b32_e32 v2, s19, v2 -; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc_lo -; GFX10-NEXT: v_subrev_co_ci_u32_e64 v8, vcc_lo, s3, v1, s0 -; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s2, v10 +; GFX10-NEXT: v_xor_b32_e32 v5, s2, v5 +; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc_lo +; GFX10-NEXT: v_subrev_co_ci_u32_e64 v10, vcc_lo, s11, v1, s0 +; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s10, v7 ; GFX10-NEXT: v_cndmask_b32_e64 v12, 0, -1, vcc_lo -; GFX10-NEXT: v_sub_co_u32 v13, vcc_lo, v10, s2 -; GFX10-NEXT: v_subrev_co_ci_u32_e64 v14, s0, 0, v8, vcc_lo +; GFX10-NEXT: v_sub_co_u32 v13, vcc_lo, v7, s10 +; GFX10-NEXT: v_subrev_co_ci_u32_e64 v14, s0, 0, v10, vcc_lo ; GFX10-NEXT: v_sub_co_u32 v0, s0, v0, s18 ; GFX10-NEXT: v_subrev_co_ci_u32_e64 v1, s0, s19, v2, s0 -; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s3, v11 -; GFX10-NEXT: v_xor_b32_e32 v2, s12, v6 -; GFX10-NEXT: v_subrev_co_ci_u32_e32 v8, vcc_lo, s3, v8, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v12, s0 -; GFX10-NEXT: v_cmp_le_u32_e64 s0, s3, v14 +; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s11, v11 +; GFX10-NEXT: v_xor_b32_e32 v2, s2, v3 +; GFX10-NEXT: v_subrev_co_ci_u32_e32 v10, vcc_lo, s11, v10, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v3, v6, v12, s0 +; GFX10-NEXT: v_cmp_le_u32_e64 s0, s11, v14 ; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, -1, s0 -; GFX10-NEXT: v_cmp_le_u32_e64 s0, s2, v13 +; GFX10-NEXT: v_cmp_le_u32_e64 s0, s10, v13 ; GFX10-NEXT: v_cndmask_b32_e64 v12, 0, -1, s0 -; GFX10-NEXT: v_add_co_u32 v15, s0, v7, 1 -; GFX10-NEXT: v_add_co_ci_u32_e64 v16, s0, 0, v3, s0 -; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s3, v14 +; GFX10-NEXT: v_add_co_u32 v15, s0, v9, 1 +; GFX10-NEXT: v_add_co_ci_u32_e64 v16, s0, 0, v4, s0 +; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s11, v14 ; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v12, s0 ; GFX10-NEXT: v_add_co_u32 v12, s0, v15, 1 ; GFX10-NEXT: v_add_co_ci_u32_e64 v17, s0, 0, v16, s0 ; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v6 -; GFX10-NEXT: v_sub_co_u32 v6, s0, v13, s2 -; GFX10-NEXT: v_subrev_co_ci_u32_e64 v8, s0, 0, v8, s0 +; GFX10-NEXT: v_sub_co_u32 v6, s0, v13, s10 +; GFX10-NEXT: v_subrev_co_ci_u32_e64 v10, s0, 0, v10, s0 ; GFX10-NEXT: v_cndmask_b32_e32 v12, v15, v12, vcc_lo -; GFX10-NEXT: v_cmp_ne_u32_e64 s0, 0, v5 +; GFX10-NEXT: v_cmp_ne_u32_e64 s0, 0, v3 ; GFX10-NEXT: v_cndmask_b32_e32 v15, v16, v17, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v5, v13, v6, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e32 v6, v14, v8, vcc_lo -; GFX10-NEXT: v_xor_b32_e32 v8, s12, v4 -; GFX10-NEXT: v_cndmask_b32_e64 v7, v7, v12, s0 -; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v15, s0 -; GFX10-NEXT: v_cndmask_b32_e64 v10, v10, v5, s0 +; GFX10-NEXT: v_cndmask_b32_e32 v3, v13, v6, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e32 v6, v14, v10, vcc_lo +; GFX10-NEXT: v_cndmask_b32_e64 v9, v9, v12, s0 +; GFX10-NEXT: v_cndmask_b32_e64 v10, v4, v15, s0 +; GFX10-NEXT: v_cndmask_b32_e64 v3, v7, v3, s0 ; GFX10-NEXT: v_cndmask_b32_e64 v6, v11, v6, s0 -; GFX10-NEXT: s_xor_b64 s[0:1], s[14:15], s[16:17] -; GFX10-NEXT: v_sub_co_u32 v4, vcc_lo, v2, s12 -; GFX10-NEXT: v_xor_b32_e32 v2, s0, v7 -; GFX10-NEXT: v_xor_b32_e32 v3, s1, v3 -; GFX10-NEXT: v_xor_b32_e32 v7, s14, v10 -; GFX10-NEXT: v_subrev_co_ci_u32_e32 v5, vcc_lo, s12, v8, vcc_lo -; GFX10-NEXT: v_xor_b32_e32 v8, s14, v6 +; GFX10-NEXT: s_xor_b64 s[0:1], s[12:13], s[16:17] +; GFX10-NEXT: v_sub_co_u32 v4, vcc_lo, v2, s2 +; GFX10-NEXT: v_xor_b32_e32 v2, s0, v9 +; GFX10-NEXT: v_xor_b32_e32 v7, s1, v10 +; GFX10-NEXT: v_xor_b32_e32 v9, s12, v3 +; GFX10-NEXT: v_subrev_co_ci_u32_e32 v5, vcc_lo, s2, v5, vcc_lo +; GFX10-NEXT: v_xor_b32_e32 v10, s12, v6 ; GFX10-NEXT: v_sub_co_u32 v2, vcc_lo, v2, s0 -; GFX10-NEXT: v_subrev_co_ci_u32_e32 v3, vcc_lo, s1, v3, vcc_lo -; GFX10-NEXT: v_sub_co_u32 v6, vcc_lo, v7, s14 -; GFX10-NEXT: v_subrev_co_ci_u32_e32 v7, vcc_lo, s14, v8, vcc_lo +; GFX10-NEXT: v_subrev_co_ci_u32_e32 v3, vcc_lo, s1, v7, vcc_lo +; GFX10-NEXT: v_sub_co_u32 v6, vcc_lo, v9, s12 +; GFX10-NEXT: v_subrev_co_ci_u32_e32 v7, vcc_lo, s12, v10, vcc_lo ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: global_store_dwordx4 v9, v[0:3], s[4:5] -; GFX10-NEXT: global_store_dwordx4 v9, v[4:7], s[6:7] +; GFX10-NEXT: global_store_dwordx4 v8, v[0:3], s[4:5] +; GFX10-NEXT: global_store_dwordx4 v8, v[4:7], s[6:7] ; GFX10-NEXT: s_endpgm %div = sdiv <2 x i64> %x, %y store <2 x i64> %div, <2 x i64> addrspace(1)* %out0 @@ -2909,63 +2897,62 @@ define amdgpu_kernel void @sdivrem_v2i16(<2 x i16> addrspace(1)* %out0, <2 x i16> addrspace(1)* %out1, <2 x i16> %x, <2 x i16> %y) { ; GFX8-LABEL: sdivrem_v2i16: ; GFX8: ; %bb.0: -; GFX8-NEXT: s_load_dword s0, s[4:5], 0x14 -; GFX8-NEXT: s_load_dword s8, s[4:5], 0x10 -; GFX8-NEXT: s_mov_b32 s9, 0x100010 +; GFX8-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x10 +; GFX8-NEXT: s_mov_b32 s10, 0x100010 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: s_sext_i32_i16 s1, s0 -; GFX8-NEXT: s_ashr_i32 s2, s1, 31 -; GFX8-NEXT: s_add_i32 s1, s1, s2 -; GFX8-NEXT: s_xor_b32 s3, s1, s2 -; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s3 -; GFX8-NEXT: s_sub_i32 s6, 0, s3 -; GFX8-NEXT: s_sext_i32_i16 s1, s8 -; GFX8-NEXT: s_bfe_i32 s0, s0, s9 +; GFX8-NEXT: s_sext_i32_i16 s0, s3 +; GFX8-NEXT: s_ashr_i32 s8, s0, 31 +; GFX8-NEXT: s_add_i32 s0, s0, s8 +; GFX8-NEXT: s_xor_b32 s9, s0, s8 +; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s9 +; GFX8-NEXT: s_sub_i32 s6, 0, s9 +; GFX8-NEXT: s_sext_i32_i16 s0, s2 +; GFX8-NEXT: s_bfe_i32 s1, s3, s10 ; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX8-NEXT: s_ashr_i32 s10, s1, 31 -; GFX8-NEXT: s_ashr_i32 s11, s0, 31 -; GFX8-NEXT: s_add_i32 s1, s1, s10 +; GFX8-NEXT: s_ashr_i32 s3, s0, 31 +; GFX8-NEXT: s_ashr_i32 s11, s1, 31 +; GFX8-NEXT: s_add_i32 s0, s0, s3 ; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX8-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX8-NEXT: s_add_i32 s0, s0, s11 -; GFX8-NEXT: s_xor_b32 s1, s1, s10 -; GFX8-NEXT: s_xor_b32 s12, s0, s11 +; GFX8-NEXT: s_add_i32 s1, s1, s11 +; GFX8-NEXT: s_xor_b32 s0, s0, s3 +; GFX8-NEXT: s_xor_b32 s12, s1, s11 ; GFX8-NEXT: v_mul_lo_u32 v1, s6, v0 ; GFX8-NEXT: v_cvt_f32_u32_e32 v2, s12 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0 ; GFX8-NEXT: v_mul_hi_u32 v1, v0, v1 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v1 -; GFX8-NEXT: v_mul_hi_u32 v0, s1, v0 +; GFX8-NEXT: v_mul_hi_u32 v0, s0, v0 ; GFX8-NEXT: v_rcp_iflag_f32_e32 v1, v2 -; GFX8-NEXT: v_mul_lo_u32 v2, v0, s3 +; GFX8-NEXT: v_mul_lo_u32 v2, v0, s9 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 1, v0 ; GFX8-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 -; GFX8-NEXT: v_sub_u32_e32 v2, vcc, s1, v2 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s3, v2 +; GFX8-NEXT: v_sub_u32_e32 v2, vcc, s0, v2 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s9, v2 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX8-NEXT: v_subrev_u32_e64 v3, s[0:1], s3, v2 +; GFX8-NEXT: v_subrev_u32_e64 v3, s[0:1], s9, v2 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 1, v0 ; GFX8-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s3, v2 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s9, v2 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX8-NEXT: v_subrev_u32_e64 v3, s[0:1], s3, v2 +; GFX8-NEXT: v_subrev_u32_e64 v3, s[0:1], s9, v2 ; GFX8-NEXT: s_sub_i32 s1, 0, s12 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc ; GFX8-NEXT: v_mul_lo_u32 v3, s1, v1 -; GFX8-NEXT: s_bfe_i32 s1, s8, s9 -; GFX8-NEXT: s_xor_b32 s0, s10, s2 +; GFX8-NEXT: s_bfe_i32 s1, s2, s10 ; GFX8-NEXT: s_ashr_i32 s2, s1, 31 -; GFX8-NEXT: v_mul_hi_u32 v3, v1, v3 ; GFX8-NEXT: s_add_i32 s1, s1, s2 +; GFX8-NEXT: v_mul_hi_u32 v3, v1, v3 ; GFX8-NEXT: s_xor_b32 s1, s1, s2 +; GFX8-NEXT: s_xor_b32 s0, s3, s8 ; GFX8-NEXT: v_xor_b32_e32 v0, s0, v0 ; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v3 ; GFX8-NEXT: v_mul_hi_u32 v1, s1, v1 -; GFX8-NEXT: v_xor_b32_e32 v2, s10, v2 +; GFX8-NEXT: v_xor_b32_e32 v2, s3, v2 ; GFX8-NEXT: v_subrev_u32_e32 v0, vcc, s0, v0 ; GFX8-NEXT: v_mul_lo_u32 v3, v1, s12 -; GFX8-NEXT: v_subrev_u32_e32 v2, vcc, s10, v2 +; GFX8-NEXT: v_subrev_u32_e32 v2, vcc, s3, v2 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 1, v1 ; GFX8-NEXT: v_sub_u32_e32 v3, vcc, s1, v3 ; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s12, v3 @@ -3000,106 +2987,104 @@ ; ; GFX9-LABEL: sdivrem_v2i16: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dword s6, s[4:5], 0x14 +; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x10 +; GFX9-NEXT: s_mov_b32 s10, 0x100010 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_sext_i32_i16 s0, s6 -; GFX9-NEXT: s_ashr_i32 s7, s0, 31 -; GFX9-NEXT: s_add_i32 s0, s0, s7 -; GFX9-NEXT: s_xor_b32 s8, s0, s7 -; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s8 -; GFX9-NEXT: s_load_dword s9, s[4:5], 0x10 -; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 -; GFX9-NEXT: s_mov_b32 s4, 0x100010 -; GFX9-NEXT: s_bfe_i32 s6, s6, s4 +; GFX9-NEXT: s_sext_i32_i16 s0, s7 +; GFX9-NEXT: s_ashr_i32 s8, s0, 31 +; GFX9-NEXT: s_add_i32 s0, s0, s8 +; GFX9-NEXT: s_xor_b32 s9, s0, s8 +; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s9 +; GFX9-NEXT: s_bfe_i32 s1, s7, s10 +; GFX9-NEXT: s_ashr_i32 s7, s1, 31 +; GFX9-NEXT: s_add_i32 s1, s1, s7 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX9-NEXT: s_ashr_i32 s10, s6, 31 -; GFX9-NEXT: s_add_i32 s6, s6, s10 -; GFX9-NEXT: s_sub_i32 s11, 0, s8 +; GFX9-NEXT: s_xor_b32 s11, s1, s7 +; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s11 +; GFX9-NEXT: s_sub_i32 s1, 0, s9 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX9-NEXT: s_xor_b32 s6, s6, s10 -; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s6 -; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_sext_i32_i16 s5, s9 -; GFX9-NEXT: v_mul_lo_u32 v1, s11, v0 -; GFX9-NEXT: s_ashr_i32 s11, s5, 31 -; GFX9-NEXT: v_rcp_iflag_f32_e32 v2, v2 -; GFX9-NEXT: s_add_i32 s5, s5, s11 -; GFX9-NEXT: v_mul_hi_u32 v1, v0, v1 -; GFX9-NEXT: s_xor_b32 s5, s5, s11 -; GFX9-NEXT: s_bfe_i32 s4, s9, s4 -; GFX9-NEXT: s_sub_i32 s9, 0, s6 -; GFX9-NEXT: v_add_u32_e32 v0, v0, v1 -; GFX9-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v2 +; GFX9-NEXT: v_rcp_iflag_f32_e32 v1, v1 +; GFX9-NEXT: s_sext_i32_i16 s0, s6 +; GFX9-NEXT: s_ashr_i32 s12, s0, 31 +; GFX9-NEXT: v_mul_lo_u32 v2, s1, v0 +; GFX9-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX9-NEXT: v_mul_hi_u32 v0, s5, v0 -; GFX9-NEXT: s_xor_b32 s7, s11, s7 -; GFX9-NEXT: v_mul_lo_u32 v3, s9, v1 -; GFX9-NEXT: v_mul_lo_u32 v2, v0, s8 -; GFX9-NEXT: v_add_u32_e32 v4, 1, v0 -; GFX9-NEXT: v_mul_hi_u32 v3, v1, v3 -; GFX9-NEXT: v_sub_u32_e32 v2, s5, v2 +; GFX9-NEXT: s_add_i32 s0, s0, s12 +; GFX9-NEXT: v_mul_hi_u32 v2, v0, v2 +; GFX9-NEXT: s_xor_b32 s13, s0, s12 +; GFX9-NEXT: s_sub_i32 s0, 0, s11 +; GFX9-NEXT: v_add_u32_e32 v0, v0, v2 +; GFX9-NEXT: v_mul_hi_u32 v0, s13, v0 +; GFX9-NEXT: v_mul_lo_u32 v2, s0, v1 +; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 +; GFX9-NEXT: s_bfe_i32 s4, s6, s10 +; GFX9-NEXT: v_mul_lo_u32 v3, v0, s9 +; GFX9-NEXT: v_mul_hi_u32 v2, v1, v2 ; GFX9-NEXT: s_ashr_i32 s5, s4, 31 ; GFX9-NEXT: s_add_i32 s4, s4, s5 +; GFX9-NEXT: v_sub_u32_e32 v3, s13, v3 +; GFX9-NEXT: v_add_u32_e32 v4, 1, v0 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s9, v3 ; GFX9-NEXT: s_xor_b32 s4, s4, s5 -; GFX9-NEXT: v_add_u32_e32 v1, v1, v3 -; GFX9-NEXT: v_mul_hi_u32 v1, s4, v1 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s8, v2 +; GFX9-NEXT: v_add_u32_e32 v1, v1, v2 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX9-NEXT: v_subrev_u32_e32 v4, s8, v2 -; GFX9-NEXT: v_mul_lo_u32 v3, v1, s6 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc +; GFX9-NEXT: v_subrev_u32_e32 v4, s9, v3 +; GFX9-NEXT: v_mul_hi_u32 v1, s4, v1 +; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc ; GFX9-NEXT: v_add_u32_e32 v4, 1, v0 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s8, v2 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s9, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX9-NEXT: v_subrev_u32_e32 v4, s8, v2 -; GFX9-NEXT: v_sub_u32_e32 v3, s4, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc +; GFX9-NEXT: v_subrev_u32_e32 v4, s9, v3 +; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc +; GFX9-NEXT: v_mul_lo_u32 v3, v1, s11 ; GFX9-NEXT: v_add_u32_e32 v4, 1, v1 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s6, v3 +; GFX9-NEXT: s_xor_b32 s6, s12, s8 +; GFX9-NEXT: v_xor_b32_e32 v0, s6, v0 +; GFX9-NEXT: v_sub_u32_e32 v3, s4, v3 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s11, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX9-NEXT: v_subrev_u32_e32 v4, s6, v3 +; GFX9-NEXT: v_subrev_u32_e32 v4, s11, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc ; GFX9-NEXT: v_add_u32_e32 v4, 1, v1 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s6, v3 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s11, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX9-NEXT: v_subrev_u32_e32 v4, s6, v3 +; GFX9-NEXT: v_subrev_u32_e32 v4, s11, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc -; GFX9-NEXT: s_xor_b32 s4, s5, s10 -; GFX9-NEXT: v_xor_b32_e32 v0, s7, v0 -; GFX9-NEXT: v_xor_b32_e32 v2, s11, v2 +; GFX9-NEXT: s_xor_b32 s4, s5, s7 +; GFX9-NEXT: v_xor_b32_e32 v2, s12, v2 ; GFX9-NEXT: v_xor_b32_e32 v1, s4, v1 ; GFX9-NEXT: v_xor_b32_e32 v3, s5, v3 -; GFX9-NEXT: v_subrev_u32_e32 v0, s7, v0 -; GFX9-NEXT: v_subrev_u32_e32 v2, s11, v2 +; GFX9-NEXT: v_subrev_u32_e32 v0, s6, v0 +; GFX9-NEXT: v_subrev_u32_e32 v2, s12, v2 ; GFX9-NEXT: v_sub_u32_sdwa v1, v1, s4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX9-NEXT: v_sub_u32_sdwa v3, v3, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX9-NEXT: v_mov_b32_e32 v4, 0xffff ; GFX9-NEXT: v_and_or_b32 v0, v0, v4, v1 ; GFX9-NEXT: v_and_or_b32 v1, v2, v4, v3 ; GFX9-NEXT: v_mov_b32_e32 v2, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: global_store_dword v2, v0, s[0:1] ; GFX9-NEXT: global_store_dword v2, v1, s[2:3] ; GFX9-NEXT: s_endpgm ; ; GFX10-LABEL: sdivrem_v2i16: ; GFX10: ; %bb.0: -; GFX10-NEXT: s_load_dword s0, s[4:5], 0x14 -; GFX10-NEXT: s_mov_b32 s1, 0x100010 +; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10 +; GFX10-NEXT: s_mov_b32 s2, 0x100010 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_sext_i32_i16 s2, s0 -; GFX10-NEXT: s_bfe_i32 s0, s0, s1 -; GFX10-NEXT: s_ashr_i32 s3, s2, 31 -; GFX10-NEXT: s_ashr_i32 s8, s0, 31 -; GFX10-NEXT: s_add_i32 s2, s2, s3 -; GFX10-NEXT: s_add_i32 s0, s0, s8 -; GFX10-NEXT: s_xor_b32 s2, s2, s3 -; GFX10-NEXT: s_xor_b32 s9, s0, s8 -; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s2 -; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s9 -; GFX10-NEXT: s_load_dword s0, s[4:5], 0x10 -; GFX10-NEXT: s_sub_i32 s6, 0, s2 -; GFX10-NEXT: s_sub_i32 s7, 0, s9 +; GFX10-NEXT: s_sext_i32_i16 s3, s1 +; GFX10-NEXT: s_bfe_i32 s1, s1, s2 +; GFX10-NEXT: s_ashr_i32 s8, s3, 31 +; GFX10-NEXT: s_ashr_i32 s9, s1, 31 +; GFX10-NEXT: s_add_i32 s3, s3, s8 +; GFX10-NEXT: s_add_i32 s1, s1, s9 +; GFX10-NEXT: s_xor_b32 s3, s3, s8 +; GFX10-NEXT: s_xor_b32 s1, s1, s9 +; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s3 +; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s1 +; GFX10-NEXT: s_sub_i32 s6, 0, s3 +; GFX10-NEXT: s_sub_i32 s7, 0, s1 ; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX10-NEXT: v_rcp_iflag_f32_e32 v1, v1 ; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 @@ -3108,56 +3093,55 @@ ; GFX10-NEXT: v_cvt_u32_f32_e32 v1, v1 ; GFX10-NEXT: v_mul_lo_u32 v2, s6, v0 ; GFX10-NEXT: v_mul_lo_u32 v3, s7, v1 -; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_sext_i32_i16 s6, s0 -; GFX10-NEXT: s_bfe_i32 s0, s0, s1 -; GFX10-NEXT: s_ashr_i32 s1, s6, 31 +; GFX10-NEXT: s_bfe_i32 s0, s0, s2 +; GFX10-NEXT: s_ashr_i32 s2, s6, 31 ; GFX10-NEXT: s_ashr_i32 s10, s0, 31 -; GFX10-NEXT: s_add_i32 s6, s6, s1 +; GFX10-NEXT: s_add_i32 s6, s6, s2 ; GFX10-NEXT: s_add_i32 s0, s0, s10 ; GFX10-NEXT: v_mul_hi_u32 v2, v0, v2 ; GFX10-NEXT: v_mul_hi_u32 v3, v1, v3 -; GFX10-NEXT: s_xor_b32 s6, s6, s1 +; GFX10-NEXT: s_xor_b32 s6, s6, s2 ; GFX10-NEXT: s_xor_b32 s0, s0, s10 ; GFX10-NEXT: v_add_nc_u32_e32 v0, v0, v2 ; GFX10-NEXT: v_add_nc_u32_e32 v1, v1, v3 ; GFX10-NEXT: v_mul_hi_u32 v0, s6, v0 ; GFX10-NEXT: v_mul_hi_u32 v1, s0, v1 -; GFX10-NEXT: v_mul_lo_u32 v2, v0, s2 -; GFX10-NEXT: v_mul_lo_u32 v3, v1, s9 +; GFX10-NEXT: v_mul_lo_u32 v2, v0, s3 +; GFX10-NEXT: v_mul_lo_u32 v3, v1, s1 ; GFX10-NEXT: v_add_nc_u32_e32 v4, 1, v0 ; GFX10-NEXT: v_add_nc_u32_e32 v5, 1, v1 ; GFX10-NEXT: v_sub_nc_u32_e32 v2, s6, v2 ; GFX10-NEXT: v_sub_nc_u32_e32 v3, s0, v3 ; GFX10-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0 -; GFX10-NEXT: v_subrev_nc_u32_e32 v6, s2, v2 -; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s9, v3 -; GFX10-NEXT: v_cmp_le_u32_e64 s0, s2, v2 -; GFX10-NEXT: v_subrev_nc_u32_e32 v7, s9, v3 +; GFX10-NEXT: v_subrev_nc_u32_e32 v6, s3, v2 +; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s1, v3 +; GFX10-NEXT: v_cmp_le_u32_e64 s0, s3, v2 +; GFX10-NEXT: v_subrev_nc_u32_e32 v7, s1, v3 ; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo ; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0 ; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v6, s0 ; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo ; GFX10-NEXT: v_add_nc_u32_e32 v5, 1, v1 ; GFX10-NEXT: v_add_nc_u32_e32 v4, 1, v0 -; GFX10-NEXT: v_cmp_le_u32_e64 s0, s2, v2 -; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s9, v3 -; GFX10-NEXT: v_subrev_nc_u32_e32 v6, s2, v2 -; GFX10-NEXT: v_subrev_nc_u32_e32 v7, s9, v3 -; GFX10-NEXT: s_xor_b32 s2, s1, s3 +; GFX10-NEXT: v_cmp_le_u32_e64 s0, s3, v2 +; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s1, v3 +; GFX10-NEXT: v_subrev_nc_u32_e32 v6, s3, v2 +; GFX10-NEXT: v_subrev_nc_u32_e32 v7, s1, v3 +; GFX10-NEXT: s_xor_b32 s1, s2, s8 ; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, v4, s0 ; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo ; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v6, s0 ; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo -; GFX10-NEXT: s_xor_b32 s0, s10, s8 -; GFX10-NEXT: v_xor_b32_e32 v0, s2, v0 +; GFX10-NEXT: s_xor_b32 s0, s10, s9 +; GFX10-NEXT: v_xor_b32_e32 v0, s1, v0 ; GFX10-NEXT: v_xor_b32_e32 v1, s0, v1 -; GFX10-NEXT: v_xor_b32_e32 v2, s1, v2 +; GFX10-NEXT: v_xor_b32_e32 v2, s2, v2 ; GFX10-NEXT: v_xor_b32_e32 v3, s10, v3 ; GFX10-NEXT: v_mov_b32_e32 v4, 0xffff -; GFX10-NEXT: v_subrev_nc_u32_e32 v0, s2, v0 +; GFX10-NEXT: v_subrev_nc_u32_e32 v0, s1, v0 ; GFX10-NEXT: v_sub_nc_u32_sdwa v1, v1, s0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD -; GFX10-NEXT: v_subrev_nc_u32_e32 v2, s1, v2 +; GFX10-NEXT: v_subrev_nc_u32_e32 v2, s2, v2 ; GFX10-NEXT: v_sub_nc_u32_sdwa v3, v3, s10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX10-NEXT: v_and_or_b32 v0, v0, v4, v1 ; GFX10-NEXT: v_mov_b32_e32 v1, 0 diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll @@ -555,13 +555,12 @@ define amdgpu_kernel void @udivrem_v2i32(<2 x i32> addrspace(1)* %out0, <2 x i32> addrspace(1)* %out1, <2 x i32> %x, <2 x i32> %y) { ; GFX8-LABEL: udivrem_v2i32: ; GFX8: ; %bb.0: -; GFX8-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x18 -; GFX8-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x10 +; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x10 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s2 -; GFX8-NEXT: v_cvt_f32_u32_e32 v1, s3 -; GFX8-NEXT: s_sub_i32 s0, 0, s2 +; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s10 +; GFX8-NEXT: v_cvt_f32_u32_e32 v1, s11 +; GFX8-NEXT: s_sub_i32 s0, 0, s10 ; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX8-NEXT: v_rcp_iflag_f32_e32 v1, v1 ; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 @@ -569,7 +568,7 @@ ; GFX8-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ; GFX8-NEXT: v_cvt_u32_f32_e32 v1, v1 ; GFX8-NEXT: v_mul_lo_u32 v2, s0, v0 -; GFX8-NEXT: s_sub_i32 s0, 0, s3 +; GFX8-NEXT: s_sub_i32 s0, 0, s11 ; GFX8-NEXT: v_mul_lo_u32 v3, s0, v1 ; GFX8-NEXT: v_mul_hi_u32 v2, v0, v2 ; GFX8-NEXT: v_mul_hi_u32 v3, v1, v3 @@ -577,29 +576,29 @@ ; GFX8-NEXT: v_mul_hi_u32 v0, s8, v0 ; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v3 ; GFX8-NEXT: v_mul_hi_u32 v1, s9, v1 -; GFX8-NEXT: v_mul_lo_u32 v2, v0, s2 +; GFX8-NEXT: v_mul_lo_u32 v2, v0, s10 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 1, v0 -; GFX8-NEXT: v_mul_lo_u32 v4, v1, s3 +; GFX8-NEXT: v_mul_lo_u32 v4, v1, s11 ; GFX8-NEXT: v_sub_u32_e32 v2, vcc, s8, v2 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s2, v2 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s10, v2 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX8-NEXT: v_subrev_u32_e64 v3, s[0:1], s2, v2 +; GFX8-NEXT: v_subrev_u32_e64 v3, s[0:1], s10, v2 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 1, v0 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s2, v2 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s10, v2 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX8-NEXT: v_subrev_u32_e64 v3, s[0:1], s2, v2 +; GFX8-NEXT: v_subrev_u32_e64 v3, s[0:1], s10, v2 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc ; GFX8-NEXT: v_sub_u32_e32 v3, vcc, s9, v4 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 1, v1 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s3, v3 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s11, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s3, v3 +; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s11, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 1, v1 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s3, v3 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s11, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s3, v3 +; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s11, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc ; GFX8-NEXT: v_mov_b32_e32 v4, s4 ; GFX8-NEXT: v_mov_b32_e32 v5, s5 @@ -611,26 +610,24 @@ ; ; GFX9-LABEL: udivrem_v2i32: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x18 +; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x10 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s2 ; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s3 -; GFX9-NEXT: s_sub_i32 s0, 0, s2 -; GFX9-NEXT: s_sub_i32 s1, 0, s3 +; GFX9-NEXT: s_sub_i32 s6, 0, s2 +; GFX9-NEXT: s_sub_i32 s7, 0, s3 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v1, v1 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX9-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX9-NEXT: v_mul_lo_u32 v2, s0, v0 -; GFX9-NEXT: v_mul_lo_u32 v3, s1, v1 -; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10 -; GFX9-NEXT: v_mul_hi_u32 v2, v0, v2 +; GFX9-NEXT: v_mul_lo_u32 v2, s6, v0 +; GFX9-NEXT: v_mul_lo_u32 v3, s7, v1 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0 +; GFX9-NEXT: v_mul_hi_u32 v2, v0, v2 ; GFX9-NEXT: v_mul_hi_u32 v3, v1, v3 ; GFX9-NEXT: v_add_u32_e32 v0, v0, v2 -; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: v_mul_hi_u32 v0, s0, v0 ; GFX9-NEXT: v_add_u32_e32 v1, v1, v3 ; GFX9-NEXT: v_mul_hi_u32 v1, s1, v1 @@ -652,41 +649,40 @@ ; GFX9-NEXT: v_cndmask_b32_e64 v3, v3, v5, s[0:1] ; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc ; GFX9-NEXT: v_subrev_u32_e32 v4, s2, v2 +; GFX9-NEXT: v_add_u32_e32 v5, 1, v1 ; GFX9-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc -; GFX9-NEXT: v_add_u32_e32 v4, 1, v1 ; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s3, v3 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc ; GFX9-NEXT: v_subrev_u32_e32 v4, s3, v3 +; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc ; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc ; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[4:5] ; GFX9-NEXT: global_store_dwordx2 v4, v[2:3], s[6:7] ; GFX9-NEXT: s_endpgm ; ; GFX10-LABEL: udivrem_v2i32: ; GFX10: ; %bb.0: -; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x18 +; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x10 ; GFX10-NEXT: v_mov_b32_e32 v8, 0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s2 ; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s3 -; GFX10-NEXT: s_sub_i32 s0, 0, s2 -; GFX10-NEXT: s_sub_i32 s1, 0, s3 +; GFX10-NEXT: s_sub_i32 s6, 0, s2 +; GFX10-NEXT: s_sub_i32 s7, 0, s3 ; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX10-NEXT: v_rcp_iflag_f32_e32 v1, v1 ; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX10-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX10-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX10-NEXT: v_mul_lo_u32 v2, s0, v0 -; GFX10-NEXT: v_mul_lo_u32 v3, s1, v1 -; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10 +; GFX10-NEXT: v_mul_lo_u32 v2, s6, v0 +; GFX10-NEXT: v_mul_lo_u32 v3, s7, v1 +; GFX10-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0 ; GFX10-NEXT: v_mul_hi_u32 v2, v0, v2 ; GFX10-NEXT: v_mul_hi_u32 v3, v1, v3 -; GFX10-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0 ; GFX10-NEXT: v_add_nc_u32_e32 v0, v0, v2 ; GFX10-NEXT: v_add_nc_u32_e32 v1, v1, v3 -; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: v_mul_hi_u32 v0, s0, v0 ; GFX10-NEXT: v_mul_hi_u32 v1, s1, v1 ; GFX10-NEXT: v_mul_lo_u32 v2, v0, s2 @@ -713,6 +709,7 @@ ; GFX10-NEXT: v_cndmask_b32_e32 v1, v1, v5, vcc_lo ; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v6, s0 ; GFX10-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo +; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_store_dwordx2 v8, v[0:1], s[4:5] ; GFX10-NEXT: global_store_dwordx2 v8, v[2:3], s[6:7] ; GFX10-NEXT: s_endpgm @@ -726,15 +723,14 @@ define amdgpu_kernel void @udivrem_v4i32(<4 x i32> addrspace(1)* %out0, <4 x i32> addrspace(1)* %out1, <4 x i32> %x, <4 x i32> %y) { ; GFX8-LABEL: udivrem_v4i32: ; GFX8: ; %bb.0: -; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x20 -; GFX8-NEXT: s_load_dwordx4 s[12:15], s[4:5], 0x10 +; GFX8-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x10 ; GFX8-NEXT: v_mov_b32_e32 v2, 0x4f7ffffe ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s8 -; GFX8-NEXT: v_cvt_f32_u32_e32 v1, s9 -; GFX8-NEXT: s_sub_i32 s0, 0, s8 -; GFX8-NEXT: v_cvt_f32_u32_e32 v6, s10 +; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s12 +; GFX8-NEXT: v_cvt_f32_u32_e32 v1, s13 +; GFX8-NEXT: s_sub_i32 s0, 0, s12 +; GFX8-NEXT: v_cvt_f32_u32_e32 v6, s14 ; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX8-NEXT: v_rcp_iflag_f32_e32 v1, v1 ; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 @@ -742,78 +738,78 @@ ; GFX8-NEXT: v_mul_f32_e32 v1, v1, v2 ; GFX8-NEXT: v_cvt_u32_f32_e32 v1, v1 ; GFX8-NEXT: v_mul_lo_u32 v3, s0, v0 -; GFX8-NEXT: s_sub_i32 s0, 0, s9 +; GFX8-NEXT: s_sub_i32 s0, 0, s13 ; GFX8-NEXT: v_mul_lo_u32 v4, s0, v1 ; GFX8-NEXT: v_mul_hi_u32 v3, v0, v3 ; GFX8-NEXT: v_mul_hi_u32 v4, v1, v4 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v3 -; GFX8-NEXT: v_mul_hi_u32 v0, s12, v0 +; GFX8-NEXT: v_mul_hi_u32 v0, s8, v0 ; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v4 -; GFX8-NEXT: v_mul_hi_u32 v1, s13, v1 -; GFX8-NEXT: v_mul_lo_u32 v3, v0, s8 +; GFX8-NEXT: v_mul_hi_u32 v1, s9, v1 +; GFX8-NEXT: v_mul_lo_u32 v3, v0, s12 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 1, v0 -; GFX8-NEXT: v_mul_lo_u32 v5, v1, s9 -; GFX8-NEXT: v_sub_u32_e32 v3, vcc, s12, v3 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s8, v3 +; GFX8-NEXT: v_mul_lo_u32 v5, v1, s13 +; GFX8-NEXT: v_sub_u32_e32 v3, vcc, s8, v3 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s12, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s8, v3 +; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s12, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 1, v0 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s8, v3 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s12, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s8, v3 +; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s12, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v4, v3, v4, vcc -; GFX8-NEXT: v_sub_u32_e32 v3, vcc, s13, v5 +; GFX8-NEXT: v_sub_u32_e32 v3, vcc, s9, v5 ; GFX8-NEXT: v_rcp_iflag_f32_e32 v5, v6 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, 1, v1 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s9, v3 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s13, v3 ; GFX8-NEXT: v_mul_f32_e32 v5, v5, v2 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc ; GFX8-NEXT: v_cvt_u32_f32_e32 v5, v5 -; GFX8-NEXT: v_subrev_u32_e64 v6, s[0:1], s9, v3 +; GFX8-NEXT: v_subrev_u32_e64 v6, s[0:1], s13, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v6, vcc ; GFX8-NEXT: v_add_u32_e32 v6, vcc, 1, v1 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s9, v3 -; GFX8-NEXT: s_sub_i32 s0, 0, s10 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s13, v3 +; GFX8-NEXT: s_sub_i32 s0, 0, s14 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc -; GFX8-NEXT: v_cvt_f32_u32_e32 v6, s11 +; GFX8-NEXT: v_cvt_f32_u32_e32 v6, s15 ; GFX8-NEXT: v_mul_lo_u32 v7, s0, v5 ; GFX8-NEXT: v_rcp_iflag_f32_e32 v6, v6 ; GFX8-NEXT: v_mul_hi_u32 v7, v5, v7 ; GFX8-NEXT: v_mul_f32_e32 v2, v6, v2 ; GFX8-NEXT: v_add_u32_e64 v5, s[0:1], v5, v7 ; GFX8-NEXT: v_cvt_u32_f32_e32 v2, v2 -; GFX8-NEXT: v_mul_hi_u32 v7, s14, v5 -; GFX8-NEXT: v_subrev_u32_e64 v5, s[0:1], s9, v3 -; GFX8-NEXT: s_sub_i32 s0, 0, s11 +; GFX8-NEXT: v_mul_hi_u32 v7, s10, v5 +; GFX8-NEXT: v_subrev_u32_e64 v5, s[0:1], s13, v3 +; GFX8-NEXT: s_sub_i32 s0, 0, s15 ; GFX8-NEXT: v_mul_lo_u32 v6, s0, v2 ; GFX8-NEXT: v_cndmask_b32_e32 v5, v3, v5, vcc -; GFX8-NEXT: v_mul_lo_u32 v3, v7, s10 +; GFX8-NEXT: v_mul_lo_u32 v3, v7, s14 ; GFX8-NEXT: v_add_u32_e32 v8, vcc, 1, v7 ; GFX8-NEXT: v_mul_hi_u32 v6, v2, v6 -; GFX8-NEXT: v_sub_u32_e32 v3, vcc, s14, v3 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s10, v3 +; GFX8-NEXT: v_sub_u32_e32 v3, vcc, s10, v3 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s14, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v7, v8, vcc -; GFX8-NEXT: v_subrev_u32_e64 v8, s[0:1], s10, v3 +; GFX8-NEXT: v_subrev_u32_e64 v8, s[0:1], s14, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v8, vcc ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v6 -; GFX8-NEXT: v_mul_hi_u32 v8, s15, v2 +; GFX8-NEXT: v_mul_hi_u32 v8, s11, v2 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, 1, v7 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s10, v3 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s14, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v7, v2, vcc -; GFX8-NEXT: v_mul_lo_u32 v7, v8, s11 -; GFX8-NEXT: v_subrev_u32_e64 v6, s[0:1], s10, v3 +; GFX8-NEXT: v_mul_lo_u32 v7, v8, s15 +; GFX8-NEXT: v_subrev_u32_e64 v6, s[0:1], s14, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v6, v3, v6, vcc -; GFX8-NEXT: v_sub_u32_e32 v3, vcc, s15, v7 +; GFX8-NEXT: v_sub_u32_e32 v3, vcc, s11, v7 ; GFX8-NEXT: v_add_u32_e32 v7, vcc, 1, v8 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s11, v3 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s15, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc -; GFX8-NEXT: v_subrev_u32_e64 v8, s[0:1], s11, v3 +; GFX8-NEXT: v_subrev_u32_e64 v8, s[0:1], s15, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v8, v3, v8, vcc ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 1, v7 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s11, v8 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s15, v8 ; GFX8-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX8-NEXT: v_subrev_u32_e64 v7, s[0:1], s11, v8 +; GFX8-NEXT: v_subrev_u32_e64 v7, s[0:1], s15, v8 ; GFX8-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc ; GFX8-NEXT: v_mov_b32_e32 v9, s5 ; GFX8-NEXT: v_mov_b32_e32 v8, s4 @@ -826,120 +822,118 @@ ; ; GFX9-LABEL: udivrem_v4i32: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x20 -; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x10 +; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x10 ; GFX9-NEXT: v_mov_b32_e32 v2, 0x4f7ffffe ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s0 -; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s1 -; GFX9-NEXT: s_sub_i32 s6, 0, s0 -; GFX9-NEXT: s_sub_i32 s7, 0, s1 +; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s12 +; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s13 +; GFX9-NEXT: s_sub_i32 s0, 0, s12 +; GFX9-NEXT: s_sub_i32 s1, 0, s13 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v1, v1 -; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s2 +; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s14 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX9-NEXT: v_mul_f32_e32 v1, v1, v2 ; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v5, v5 -; GFX9-NEXT: v_mul_lo_u32 v3, s6, v0 -; GFX9-NEXT: v_mul_lo_u32 v4, s7, v1 -; GFX9-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0 +; GFX9-NEXT: v_mul_lo_u32 v3, s0, v0 +; GFX9-NEXT: v_mul_lo_u32 v4, s1, v1 +; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX9-NEXT: v_mul_hi_u32 v3, v0, v3 +; GFX9-NEXT: s_sub_i32 s4, 0, s14 ; GFX9-NEXT: v_mul_hi_u32 v4, v1, v4 ; GFX9-NEXT: v_add_u32_e32 v0, v0, v3 ; GFX9-NEXT: v_mul_hi_u32 v0, s8, v0 ; GFX9-NEXT: v_add_u32_e32 v1, v1, v4 ; GFX9-NEXT: v_mul_hi_u32 v1, s9, v1 ; GFX9-NEXT: v_mul_f32_e32 v3, v5, v2 -; GFX9-NEXT: v_mul_lo_u32 v4, v0, s0 +; GFX9-NEXT: v_mul_lo_u32 v4, v0, s12 ; GFX9-NEXT: v_add_u32_e32 v6, 1, v0 -; GFX9-NEXT: v_mul_lo_u32 v5, v1, s1 -; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3 +; GFX9-NEXT: v_mul_lo_u32 v5, v1, s13 +; GFX9-NEXT: v_add_u32_e32 v7, 1, v1 ; GFX9-NEXT: v_sub_u32_e32 v4, s8, v4 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s0, v4 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s12, v4 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc -; GFX9-NEXT: v_subrev_u32_e32 v6, s0, v4 +; GFX9-NEXT: v_subrev_u32_e32 v6, s12, v4 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc ; GFX9-NEXT: v_add_u32_e32 v6, 1, v0 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s0, v4 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s12, v4 ; GFX9-NEXT: v_sub_u32_e32 v5, s9, v5 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v6, vcc -; GFX9-NEXT: v_subrev_u32_e32 v6, s0, v4 +; GFX9-NEXT: v_subrev_u32_e32 v6, s12, v4 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc -; GFX9-NEXT: v_add_u32_e32 v6, 1, v1 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s1, v5 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc -; GFX9-NEXT: v_cvt_f32_u32_e32 v6, s3 -; GFX9-NEXT: s_sub_i32 s0, 0, s2 -; GFX9-NEXT: v_mul_lo_u32 v7, s0, v3 -; GFX9-NEXT: s_sub_i32 s0, 0, s3 -; GFX9-NEXT: v_rcp_iflag_f32_e32 v6, v6 -; GFX9-NEXT: v_subrev_u32_e32 v8, s1, v5 -; GFX9-NEXT: v_mul_hi_u32 v7, v3, v7 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s13, v5 +; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc +; GFX9-NEXT: v_cvt_f32_u32_e32 v7, s15 +; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3 +; GFX9-NEXT: v_subrev_u32_e32 v8, s13, v5 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v8, vcc -; GFX9-NEXT: v_mul_f32_e32 v2, v6, v2 -; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2 -; GFX9-NEXT: v_add_u32_e32 v3, v3, v7 -; GFX9-NEXT: v_mul_hi_u32 v3, s10, v3 +; GFX9-NEXT: v_rcp_iflag_f32_e32 v7, v7 +; GFX9-NEXT: v_mul_lo_u32 v6, s4, v3 +; GFX9-NEXT: s_sub_i32 s4, 0, s15 ; GFX9-NEXT: v_add_u32_e32 v8, 1, v1 -; GFX9-NEXT: v_mul_lo_u32 v7, s0, v2 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s1, v5 -; GFX9-NEXT: v_mul_lo_u32 v6, v3, s2 +; GFX9-NEXT: v_mul_f32_e32 v2, v7, v2 +; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2 +; GFX9-NEXT: v_mul_hi_u32 v6, v3, v6 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s13, v5 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v8, vcc +; GFX9-NEXT: v_mul_lo_u32 v7, s4, v2 +; GFX9-NEXT: v_add_u32_e32 v3, v3, v6 +; GFX9-NEXT: v_mul_hi_u32 v3, s10, v3 +; GFX9-NEXT: v_subrev_u32_e32 v8, s13, v5 ; GFX9-NEXT: v_mul_hi_u32 v7, v2, v7 -; GFX9-NEXT: v_subrev_u32_e32 v8, s1, v5 -; GFX9-NEXT: v_sub_u32_e32 v6, s10, v6 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v5, v8, vcc +; GFX9-NEXT: v_mul_lo_u32 v6, v3, s14 +; GFX9-NEXT: v_add_u32_e32 v8, 1, v3 ; GFX9-NEXT: v_add_u32_e32 v2, v2, v7 ; GFX9-NEXT: v_mul_hi_u32 v7, s11, v2 -; GFX9-NEXT: v_add_u32_e32 v8, 1, v3 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s2, v6 +; GFX9-NEXT: v_sub_u32_e32 v6, s10, v6 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s14, v6 ; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v8, vcc -; GFX9-NEXT: v_subrev_u32_e32 v2, s2, v6 -; GFX9-NEXT: v_mul_lo_u32 v8, v7, s3 +; GFX9-NEXT: v_subrev_u32_e32 v2, s14, v6 +; GFX9-NEXT: v_mul_lo_u32 v8, v7, s15 ; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v2, vcc ; GFX9-NEXT: v_add_u32_e32 v2, 1, v3 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s2, v6 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s14, v6 ; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc -; GFX9-NEXT: v_subrev_u32_e32 v3, s2, v6 +; GFX9-NEXT: v_subrev_u32_e32 v3, s14, v6 ; GFX9-NEXT: v_cndmask_b32_e32 v6, v6, v3, vcc ; GFX9-NEXT: v_sub_u32_e32 v3, s11, v8 ; GFX9-NEXT: v_add_u32_e32 v8, 1, v7 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s3, v3 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s15, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v7, v7, v8, vcc -; GFX9-NEXT: v_subrev_u32_e32 v8, s3, v3 +; GFX9-NEXT: v_subrev_u32_e32 v8, s15, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v8, v3, v8, vcc ; GFX9-NEXT: v_add_u32_e32 v3, 1, v7 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s3, v8 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s15, v8 ; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc -; GFX9-NEXT: v_subrev_u32_e32 v7, s3, v8 +; GFX9-NEXT: v_subrev_u32_e32 v7, s15, v8 ; GFX9-NEXT: v_cndmask_b32_e32 v7, v8, v7, vcc ; GFX9-NEXT: v_mov_b32_e32 v8, 0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: global_store_dwordx4 v8, v[0:3], s[4:5] -; GFX9-NEXT: global_store_dwordx4 v8, v[4:7], s[6:7] +; GFX9-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1] +; GFX9-NEXT: global_store_dwordx4 v8, v[4:7], s[2:3] ; GFX9-NEXT: s_endpgm ; ; GFX10-LABEL: udivrem_v4i32: ; GFX10: ; %bb.0: -; GFX10-NEXT: s_clause 0x1 -; GFX10-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x20 -; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x10 +; GFX10-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x10 ; GFX10-NEXT: v_mov_b32_e32 v4, 0x4f7ffffe +; GFX10-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0 ; GFX10-NEXT: v_mov_b32_e32 v8, 0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s8 -; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s9 -; GFX10-NEXT: v_cvt_f32_u32_e32 v2, s10 -; GFX10-NEXT: v_cvt_f32_u32_e32 v3, s11 -; GFX10-NEXT: s_sub_i32 s6, 0, s8 +; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s12 +; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s13 +; GFX10-NEXT: v_cvt_f32_u32_e32 v2, s14 +; GFX10-NEXT: v_cvt_f32_u32_e32 v3, s15 +; GFX10-NEXT: s_sub_i32 s0, 0, s12 ; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX10-NEXT: v_rcp_iflag_f32_e32 v1, v1 ; GFX10-NEXT: v_rcp_iflag_f32_e32 v2, v2 ; GFX10-NEXT: v_rcp_iflag_f32_e32 v3, v3 -; GFX10-NEXT: s_sub_i32 s7, 0, s9 -; GFX10-NEXT: s_sub_i32 s12, 0, s10 +; GFX10-NEXT: s_sub_i32 s1, 0, s13 +; GFX10-NEXT: s_sub_i32 s2, 0, s14 ; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX10-NEXT: v_mul_f32_e32 v1, v1, v4 ; GFX10-NEXT: v_mul_f32_e32 v2, v2, v4 @@ -948,12 +942,11 @@ ; GFX10-NEXT: v_cvt_u32_f32_e32 v1, v1 ; GFX10-NEXT: v_cvt_u32_f32_e32 v2, v2 ; GFX10-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GFX10-NEXT: v_mul_lo_u32 v4, s6, v0 -; GFX10-NEXT: v_mul_lo_u32 v5, s7, v1 -; GFX10-NEXT: v_mul_lo_u32 v6, s12, v2 -; GFX10-NEXT: s_sub_i32 s6, 0, s11 -; GFX10-NEXT: v_mul_lo_u32 v7, s6, v3 -; GFX10-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0 +; GFX10-NEXT: v_mul_lo_u32 v4, s0, v0 +; GFX10-NEXT: v_mul_lo_u32 v5, s1, v1 +; GFX10-NEXT: v_mul_lo_u32 v6, s2, v2 +; GFX10-NEXT: s_sub_i32 s0, 0, s15 +; GFX10-NEXT: v_mul_lo_u32 v7, s0, v3 ; GFX10-NEXT: v_mul_hi_u32 v4, v0, v4 ; GFX10-NEXT: v_mul_hi_u32 v5, v1, v5 ; GFX10-NEXT: v_mul_hi_u32 v6, v2, v6 @@ -962,34 +955,34 @@ ; GFX10-NEXT: v_add_nc_u32_e32 v1, v1, v5 ; GFX10-NEXT: v_add_nc_u32_e32 v2, v2, v6 ; GFX10-NEXT: v_add_nc_u32_e32 v3, v3, v7 -; GFX10-NEXT: v_mul_hi_u32 v0, s0, v0 -; GFX10-NEXT: v_mul_hi_u32 v1, s1, v1 -; GFX10-NEXT: v_mul_hi_u32 v2, s2, v2 -; GFX10-NEXT: v_mul_hi_u32 v3, s3, v3 -; GFX10-NEXT: v_mul_lo_u32 v4, v0, s8 -; GFX10-NEXT: v_mul_lo_u32 v5, v1, s9 -; GFX10-NEXT: v_mul_lo_u32 v6, v2, s10 +; GFX10-NEXT: v_mul_hi_u32 v0, s8, v0 +; GFX10-NEXT: v_mul_hi_u32 v1, s9, v1 +; GFX10-NEXT: v_mul_hi_u32 v2, s10, v2 +; GFX10-NEXT: v_mul_hi_u32 v3, s11, v3 +; GFX10-NEXT: v_mul_lo_u32 v4, v0, s12 +; GFX10-NEXT: v_mul_lo_u32 v5, v1, s13 +; GFX10-NEXT: v_mul_lo_u32 v6, v2, s14 ; GFX10-NEXT: v_add_nc_u32_e32 v9, 1, v0 -; GFX10-NEXT: v_mul_lo_u32 v7, v3, s11 +; GFX10-NEXT: v_mul_lo_u32 v7, v3, s15 ; GFX10-NEXT: v_add_nc_u32_e32 v10, 1, v1 ; GFX10-NEXT: v_add_nc_u32_e32 v11, 1, v2 ; GFX10-NEXT: v_add_nc_u32_e32 v12, 1, v3 -; GFX10-NEXT: v_sub_nc_u32_e32 v4, s0, v4 -; GFX10-NEXT: v_sub_nc_u32_e32 v5, s1, v5 -; GFX10-NEXT: v_sub_nc_u32_e32 v6, s2, v6 -; GFX10-NEXT: v_sub_nc_u32_e32 v7, s3, v7 -; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s8, v4 -; GFX10-NEXT: v_cmp_le_u32_e64 s0, s9, v5 -; GFX10-NEXT: v_cmp_le_u32_e64 s1, s10, v6 -; GFX10-NEXT: v_cmp_le_u32_e64 s2, s11, v7 +; GFX10-NEXT: v_sub_nc_u32_e32 v4, s8, v4 +; GFX10-NEXT: v_sub_nc_u32_e32 v5, s9, v5 +; GFX10-NEXT: v_sub_nc_u32_e32 v6, s10, v6 +; GFX10-NEXT: v_sub_nc_u32_e32 v7, s11, v7 +; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s12, v4 +; GFX10-NEXT: v_cmp_le_u32_e64 s0, s13, v5 +; GFX10-NEXT: v_cmp_le_u32_e64 s1, s14, v6 +; GFX10-NEXT: v_cmp_le_u32_e64 s2, s15, v7 ; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc_lo -; GFX10-NEXT: v_subrev_nc_u32_e32 v9, s8, v4 +; GFX10-NEXT: v_subrev_nc_u32_e32 v9, s12, v4 ; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v10, s0 -; GFX10-NEXT: v_subrev_nc_u32_e32 v10, s9, v5 +; GFX10-NEXT: v_subrev_nc_u32_e32 v10, s13, v5 ; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v11, s1 -; GFX10-NEXT: v_subrev_nc_u32_e32 v11, s10, v6 +; GFX10-NEXT: v_subrev_nc_u32_e32 v11, s14, v6 ; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v12, s2 -; GFX10-NEXT: v_subrev_nc_u32_e32 v12, s11, v7 +; GFX10-NEXT: v_subrev_nc_u32_e32 v12, s15, v7 ; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc_lo ; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v10, s0 ; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v11, s1 @@ -997,24 +990,23 @@ ; GFX10-NEXT: v_cndmask_b32_e64 v7, v7, v12, s2 ; GFX10-NEXT: v_add_nc_u32_e32 v10, 1, v1 ; GFX10-NEXT: v_add_nc_u32_e32 v11, 1, v2 -; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s8, v4 -; GFX10-NEXT: v_cmp_le_u32_e64 s0, s9, v5 -; GFX10-NEXT: v_cmp_le_u32_e64 s1, s10, v6 +; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s12, v4 +; GFX10-NEXT: v_cmp_le_u32_e64 s0, s13, v5 +; GFX10-NEXT: v_cmp_le_u32_e64 s1, s14, v6 ; GFX10-NEXT: v_add_nc_u32_e32 v12, 1, v3 -; GFX10-NEXT: v_cmp_le_u32_e64 s2, s11, v7 +; GFX10-NEXT: v_cmp_le_u32_e64 s2, s15, v7 ; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc_lo -; GFX10-NEXT: v_subrev_nc_u32_e32 v9, s8, v4 +; GFX10-NEXT: v_subrev_nc_u32_e32 v9, s12, v4 ; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v10, s0 -; GFX10-NEXT: v_subrev_nc_u32_e32 v10, s9, v5 +; GFX10-NEXT: v_subrev_nc_u32_e32 v10, s13, v5 ; GFX10-NEXT: v_cndmask_b32_e64 v2, v2, v11, s1 -; GFX10-NEXT: v_subrev_nc_u32_e32 v11, s10, v6 -; GFX10-NEXT: v_subrev_nc_u32_e32 v13, s11, v7 +; GFX10-NEXT: v_subrev_nc_u32_e32 v11, s14, v6 +; GFX10-NEXT: v_subrev_nc_u32_e32 v13, s15, v7 ; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v12, s2 ; GFX10-NEXT: v_cndmask_b32_e32 v4, v4, v9, vcc_lo ; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v10, s0 ; GFX10-NEXT: v_cndmask_b32_e64 v6, v6, v11, s1 ; GFX10-NEXT: v_cndmask_b32_e64 v7, v7, v13, s2 -; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: global_store_dwordx4 v8, v[0:3], s[4:5] ; GFX10-NEXT: global_store_dwordx4 v8, v[4:7], s[6:7] ; GFX10-NEXT: s_endpgm @@ -1028,20 +1020,19 @@ define amdgpu_kernel void @udivrem_v2i64(<2 x i64> addrspace(1)* %out0, <2 x i64> addrspace(1)* %out1, <2 x i64> %x, <2 x i64> %y) { ; GFX8-LABEL: udivrem_v2i64: ; GFX8: ; %bb.0: -; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x20 -; GFX8-NEXT: s_load_dwordx4 s[12:15], s[4:5], 0x10 +; GFX8-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x10 ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s9 -; GFX8-NEXT: v_cvt_f32_u32_e32 v1, s8 -; GFX8-NEXT: s_sub_u32 s0, 0, s8 +; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s13 +; GFX8-NEXT: v_cvt_f32_u32_e32 v1, s12 +; GFX8-NEXT: s_sub_u32 s0, 0, s12 ; GFX8-NEXT: s_cselect_b32 s1, 1, 0 ; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f800000, v0 ; GFX8-NEXT: v_add_f32_e32 v0, v0, v1 ; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX8-NEXT: s_and_b32 s1, s1, 1 ; GFX8-NEXT: s_cmp_lg_u32 s1, 0 -; GFX8-NEXT: s_subb_u32 s1, 0, s9 +; GFX8-NEXT: s_subb_u32 s1, 0, s13 ; GFX8-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 ; GFX8-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 ; GFX8-NEXT: v_trunc_f32_e32 v1, v1 @@ -1049,7 +1040,7 @@ ; GFX8-NEXT: v_add_f32_e32 v0, v2, v0 ; GFX8-NEXT: v_cvt_u32_f32_e32 v1, v1 ; GFX8-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX8-NEXT: s_sub_u32 s2, 0, s10 +; GFX8-NEXT: s_sub_u32 s2, 0, s14 ; GFX8-NEXT: v_mul_lo_u32 v2, s0, v1 ; GFX8-NEXT: v_mul_lo_u32 v3, s1, v0 ; GFX8-NEXT: v_mul_hi_u32 v5, s0, v0 @@ -1059,15 +1050,15 @@ ; GFX8-NEXT: v_mul_lo_u32 v3, v1, v4 ; GFX8-NEXT: v_mul_lo_u32 v5, v0, v2 ; GFX8-NEXT: v_mul_hi_u32 v6, v0, v4 +; GFX8-NEXT: v_mul_lo_u32 v7, v1, v2 ; GFX8-NEXT: v_mul_hi_u32 v4, v1, v4 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v5 ; GFX8-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc ; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v6 ; GFX8-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc -; GFX8-NEXT: v_mul_lo_u32 v6, v1, v2 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, v5, v3 ; GFX8-NEXT: v_mul_hi_u32 v5, v0, v2 -; GFX8-NEXT: v_add_u32_e32 v4, vcc, v6, v4 +; GFX8-NEXT: v_add_u32_e32 v4, vcc, v7, v4 ; GFX8-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc ; GFX8-NEXT: v_add_u32_e32 v4, vcc, v4, v5 ; GFX8-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc @@ -1083,7 +1074,7 @@ ; GFX8-NEXT: v_mul_lo_u32 v3, s0, v1 ; GFX8-NEXT: v_mul_hi_u32 v5, s0, v0 ; GFX8-NEXT: v_mul_lo_u32 v4, s0, v0 -; GFX8-NEXT: v_mov_b32_e32 v6, s9 +; GFX8-NEXT: v_mov_b32_e32 v6, s13 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v3 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v5 ; GFX8-NEXT: v_mul_lo_u32 v3, v1, v4 @@ -1109,64 +1100,64 @@ ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v4 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v3 ; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v1, v2, vcc -; GFX8-NEXT: v_mul_lo_u32 v2, s13, v0 -; GFX8-NEXT: v_mul_lo_u32 v3, s12, v1 -; GFX8-NEXT: v_mul_hi_u32 v5, s12, v0 -; GFX8-NEXT: v_mul_hi_u32 v0, s13, v0 -; GFX8-NEXT: v_mov_b32_e32 v4, s13 +; GFX8-NEXT: v_mul_lo_u32 v2, s9, v0 +; GFX8-NEXT: v_mul_lo_u32 v3, s8, v1 +; GFX8-NEXT: v_mul_hi_u32 v5, s8, v0 +; GFX8-NEXT: v_mul_hi_u32 v0, s9, v0 +; GFX8-NEXT: v_mov_b32_e32 v4, s9 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v3 ; GFX8-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v5 ; GFX8-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc -; GFX8-NEXT: v_mul_lo_u32 v5, s13, v1 +; GFX8-NEXT: v_mul_lo_u32 v5, s9, v1 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v3, v2 -; GFX8-NEXT: v_mul_hi_u32 v3, s12, v1 +; GFX8-NEXT: v_mul_hi_u32 v3, s8, v1 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, v5, v0 ; GFX8-NEXT: v_cndmask_b32_e64 v5, 0, 1, vcc ; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v3 ; GFX8-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc ; GFX8-NEXT: v_add_u32_e32 v3, vcc, v5, v3 -; GFX8-NEXT: v_mul_hi_u32 v1, s13, v1 +; GFX8-NEXT: v_mul_hi_u32 v1, s9, v1 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2 ; GFX8-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v3, v2 ; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v2 -; GFX8-NEXT: v_mul_lo_u32 v2, s9, v0 -; GFX8-NEXT: v_mul_lo_u32 v3, s8, v1 -; GFX8-NEXT: v_mul_hi_u32 v7, s8, v0 -; GFX8-NEXT: v_mul_lo_u32 v5, s8, v0 +; GFX8-NEXT: v_mul_lo_u32 v2, s13, v0 +; GFX8-NEXT: v_mul_lo_u32 v3, s12, v1 +; GFX8-NEXT: v_mul_hi_u32 v7, s12, v0 +; GFX8-NEXT: v_mul_lo_u32 v5, s12, v0 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v3 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v7 -; GFX8-NEXT: v_sub_u32_e32 v3, vcc, s12, v5 +; GFX8-NEXT: v_sub_u32_e32 v3, vcc, s8, v5 ; GFX8-NEXT: v_subb_u32_e64 v5, s[0:1], v4, v2, vcc -; GFX8-NEXT: v_sub_u32_e64 v2, s[0:1], s13, v2 -; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s9, v5 +; GFX8-NEXT: v_sub_u32_e64 v2, s[0:1], s9, v2 +; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s13, v5 ; GFX8-NEXT: v_cndmask_b32_e64 v4, 0, -1, s[0:1] -; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s8, v3 +; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s12, v3 ; GFX8-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[0:1] -; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], s9, v5 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], s13, v5 ; GFX8-NEXT: v_subb_u32_e32 v2, vcc, v2, v6, vcc ; GFX8-NEXT: v_cndmask_b32_e64 v4, v4, v7, s[0:1] -; GFX8-NEXT: v_subrev_u32_e32 v7, vcc, s8, v3 +; GFX8-NEXT: v_subrev_u32_e32 v7, vcc, s12, v3 ; GFX8-NEXT: v_subbrev_u32_e64 v8, s[0:1], 0, v2, vcc ; GFX8-NEXT: v_add_u32_e64 v9, s[0:1], 1, v0 ; GFX8-NEXT: v_addc_u32_e64 v10, s[0:1], 0, v1, s[0:1] -; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s9, v8 +; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s13, v8 ; GFX8-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[0:1] -; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s8, v7 +; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s12, v7 ; GFX8-NEXT: v_subb_u32_e32 v2, vcc, v2, v6, vcc ; GFX8-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[0:1] -; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], s9, v8 -; GFX8-NEXT: v_subrev_u32_e32 v6, vcc, s8, v7 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], s13, v8 +; GFX8-NEXT: v_subrev_u32_e32 v6, vcc, s12, v7 ; GFX8-NEXT: v_cndmask_b32_e64 v11, v11, v12, s[0:1] ; GFX8-NEXT: v_add_u32_e64 v12, s[0:1], 1, v9 ; GFX8-NEXT: v_subbrev_u32_e32 v2, vcc, 0, v2, vcc ; GFX8-NEXT: v_addc_u32_e64 v13, s[0:1], 0, v10, s[0:1] ; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, 0, v11 ; GFX8-NEXT: v_cndmask_b32_e32 v10, v10, v13, vcc -; GFX8-NEXT: v_cvt_f32_u32_e32 v13, s11 +; GFX8-NEXT: v_cvt_f32_u32_e32 v13, s15 ; GFX8-NEXT: v_cndmask_b32_e32 v9, v9, v12, vcc -; GFX8-NEXT: v_cvt_f32_u32_e32 v12, s10 +; GFX8-NEXT: v_cvt_f32_u32_e32 v12, s14 ; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, 0, v4 ; GFX8-NEXT: v_mul_f32_e32 v4, 0x4f800000, v13 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v9, vcc @@ -1186,7 +1177,7 @@ ; GFX8-NEXT: s_cselect_b32 s0, 1, 0 ; GFX8-NEXT: s_and_b32 s0, s0, 1 ; GFX8-NEXT: s_cmp_lg_u32 s0, 0 -; GFX8-NEXT: s_subb_u32 s3, 0, s11 +; GFX8-NEXT: s_subb_u32 s3, 0, s15 ; GFX8-NEXT: v_mul_lo_u32 v7, s3, v3 ; GFX8-NEXT: v_mul_lo_u32 v8, s2, v6 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc @@ -1222,7 +1213,7 @@ ; GFX8-NEXT: v_mul_lo_u32 v7, s2, v3 ; GFX8-NEXT: v_mul_hi_u32 v9, s2, v2 ; GFX8-NEXT: v_mul_lo_u32 v8, s2, v2 -; GFX8-NEXT: v_mov_b32_e32 v10, s11 +; GFX8-NEXT: v_mov_b32_e32 v10, s15 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v7 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v9 ; GFX8-NEXT: v_mul_lo_u32 v7, v3, v8 @@ -1248,51 +1239,51 @@ ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v8 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v7 ; GFX8-NEXT: v_addc_u32_e32 v3, vcc, v3, v6, vcc -; GFX8-NEXT: v_mul_lo_u32 v6, s15, v2 -; GFX8-NEXT: v_mul_lo_u32 v7, s14, v3 -; GFX8-NEXT: v_mul_hi_u32 v9, s14, v2 -; GFX8-NEXT: v_mul_hi_u32 v2, s15, v2 -; GFX8-NEXT: v_mov_b32_e32 v8, s15 +; GFX8-NEXT: v_mul_lo_u32 v6, s11, v2 +; GFX8-NEXT: v_mul_lo_u32 v7, s10, v3 +; GFX8-NEXT: v_mul_hi_u32 v9, s10, v2 +; GFX8-NEXT: v_mul_hi_u32 v2, s11, v2 +; GFX8-NEXT: v_mov_b32_e32 v8, s11 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v7 ; GFX8-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v9 ; GFX8-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc -; GFX8-NEXT: v_mul_lo_u32 v9, s15, v3 +; GFX8-NEXT: v_mul_lo_u32 v9, s11, v3 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v7, v6 -; GFX8-NEXT: v_mul_hi_u32 v7, s14, v3 +; GFX8-NEXT: v_mul_hi_u32 v7, s10, v3 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v9, v2 ; GFX8-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v7 ; GFX8-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc ; GFX8-NEXT: v_add_u32_e32 v7, vcc, v9, v7 -; GFX8-NEXT: v_mul_hi_u32 v3, s15, v3 +; GFX8-NEXT: v_mul_hi_u32 v3, s11, v3 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v6 ; GFX8-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v7, v6 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v6 -; GFX8-NEXT: v_mul_lo_u32 v6, s11, v2 -; GFX8-NEXT: v_mul_lo_u32 v7, s10, v3 -; GFX8-NEXT: v_mul_hi_u32 v11, s10, v2 -; GFX8-NEXT: v_mul_lo_u32 v9, s10, v2 +; GFX8-NEXT: v_mul_lo_u32 v6, s15, v2 +; GFX8-NEXT: v_mul_lo_u32 v7, s14, v3 +; GFX8-NEXT: v_mul_hi_u32 v11, s14, v2 +; GFX8-NEXT: v_mul_lo_u32 v9, s14, v2 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v7 ; GFX8-NEXT: v_add_u32_e32 v6, vcc, v6, v11 -; GFX8-NEXT: v_sub_u32_e32 v7, vcc, s14, v9 +; GFX8-NEXT: v_sub_u32_e32 v7, vcc, s10, v9 ; GFX8-NEXT: v_subb_u32_e64 v8, s[0:1], v8, v6, vcc -; GFX8-NEXT: v_sub_u32_e64 v6, s[0:1], s15, v6 -; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s11, v8 +; GFX8-NEXT: v_sub_u32_e64 v6, s[0:1], s11, v6 +; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s15, v8 ; GFX8-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[0:1] -; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s10, v7 +; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s14, v7 ; GFX8-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[0:1] -; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], s11, v8 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], s15, v8 ; GFX8-NEXT: v_subb_u32_e32 v6, vcc, v6, v10, vcc ; GFX8-NEXT: v_cndmask_b32_e64 v9, v9, v11, s[0:1] -; GFX8-NEXT: v_subrev_u32_e32 v11, vcc, s10, v7 +; GFX8-NEXT: v_subrev_u32_e32 v11, vcc, s14, v7 ; GFX8-NEXT: v_subbrev_u32_e64 v12, s[0:1], 0, v6, vcc -; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s11, v12 +; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s15, v12 ; GFX8-NEXT: v_cndmask_b32_e64 v13, 0, -1, s[0:1] -; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s10, v11 +; GFX8-NEXT: v_cmp_le_u32_e64 s[0:1], s14, v11 ; GFX8-NEXT: v_cndmask_b32_e64 v14, 0, -1, s[0:1] -; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], s11, v12 +; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], s15, v12 ; GFX8-NEXT: v_cndmask_b32_e64 v13, v13, v14, s[0:1] ; GFX8-NEXT: v_add_u32_e64 v14, s[0:1], 1, v2 ; GFX8-NEXT: v_subb_u32_e32 v6, vcc, v6, v10, vcc @@ -1300,7 +1291,7 @@ ; GFX8-NEXT: v_add_u32_e32 v10, vcc, 1, v14 ; GFX8-NEXT: v_addc_u32_e32 v16, vcc, 0, v15, vcc ; GFX8-NEXT: v_cmp_ne_u32_e32 vcc, 0, v13 -; GFX8-NEXT: v_subrev_u32_e64 v13, s[0:1], s10, v11 +; GFX8-NEXT: v_subrev_u32_e64 v13, s[0:1], s14, v11 ; GFX8-NEXT: v_subbrev_u32_e64 v6, s[0:1], 0, v6, s[0:1] ; GFX8-NEXT: v_cndmask_b32_e32 v10, v14, v10, vcc ; GFX8-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v9 @@ -1322,19 +1313,18 @@ ; ; GFX9-LABEL: udivrem_v2i64: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x20 -; GFX9-NEXT: s_load_dwordx4 s[12:15], s[4:5], 0x10 +; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x10 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s9 -; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s8 -; GFX9-NEXT: s_sub_u32 s0, 0, s8 +; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s13 +; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s12 +; GFX9-NEXT: s_sub_u32 s0, 0, s12 ; GFX9-NEXT: s_cselect_b32 s1, 1, 0 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f800000, v0 ; GFX9-NEXT: v_add_f32_e32 v0, v0, v1 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX9-NEXT: s_and_b32 s1, s1, 1 ; GFX9-NEXT: s_cmp_lg_u32 s1, 0 -; GFX9-NEXT: s_subb_u32 s1, 0, s9 +; GFX9-NEXT: s_subb_u32 s1, 0, s13 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 ; GFX9-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0 ; GFX9-NEXT: v_trunc_f32_e32 v1, v1 @@ -1342,8 +1332,8 @@ ; GFX9-NEXT: v_add_f32_e32 v0, v2, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX9-NEXT: v_cvt_f32_u32_e32 v14, s11 -; GFX9-NEXT: s_sub_u32 s2, 0, s10 +; GFX9-NEXT: v_cvt_f32_u32_e32 v14, s15 +; GFX9-NEXT: s_sub_u32 s2, 0, s14 ; GFX9-NEXT: v_mul_lo_u32 v2, s0, v1 ; GFX9-NEXT: v_mul_lo_u32 v3, s1, v0 ; GFX9-NEXT: v_mul_hi_u32 v4, s0, v0 @@ -1377,7 +1367,7 @@ ; GFX9-NEXT: v_mul_lo_u32 v3, s0, v1 ; GFX9-NEXT: v_mul_hi_u32 v4, s0, v0 ; GFX9-NEXT: v_mul_lo_u32 v5, s0, v0 -; GFX9-NEXT: v_mov_b32_e32 v7, s13 +; GFX9-NEXT: v_mov_b32_e32 v7, s9 ; GFX9-NEXT: v_add3_u32 v2, v2, v3, v4 ; GFX9-NEXT: v_mul_lo_u32 v3, v1, v5 ; GFX9-NEXT: v_mul_lo_u32 v4, v0, v2 @@ -1401,19 +1391,19 @@ ; GFX9-NEXT: v_add3_u32 v2, v5, v4, v2 ; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v3 ; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v2, vcc -; GFX9-NEXT: v_mul_lo_u32 v2, s13, v0 -; GFX9-NEXT: v_mul_lo_u32 v3, s12, v1 -; GFX9-NEXT: v_mul_hi_u32 v4, s12, v0 -; GFX9-NEXT: v_mul_hi_u32 v0, s13, v0 -; GFX9-NEXT: v_mov_b32_e32 v5, s9 +; GFX9-NEXT: v_mul_lo_u32 v2, s9, v0 +; GFX9-NEXT: v_mul_lo_u32 v3, s8, v1 +; GFX9-NEXT: v_mul_hi_u32 v4, s8, v0 +; GFX9-NEXT: v_mul_hi_u32 v0, s9, v0 +; GFX9-NEXT: v_mov_b32_e32 v5, s13 ; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v3 ; GFX9-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc ; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v2, v4 ; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc -; GFX9-NEXT: v_mul_lo_u32 v4, s13, v1 +; GFX9-NEXT: v_mul_lo_u32 v4, s9, v1 ; GFX9-NEXT: v_add_u32_e32 v2, v3, v2 -; GFX9-NEXT: v_mul_hi_u32 v3, s12, v1 -; GFX9-NEXT: v_mul_hi_u32 v1, s13, v1 +; GFX9-NEXT: v_mul_hi_u32 v3, s8, v1 +; GFX9-NEXT: v_mul_hi_u32 v1, s9, v1 ; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v4, v0 ; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc ; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v3 @@ -1422,35 +1412,35 @@ ; GFX9-NEXT: v_add_u32_e32 v3, v4, v3 ; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc ; GFX9-NEXT: v_add3_u32 v1, v3, v2, v1 -; GFX9-NEXT: v_mul_lo_u32 v2, s9, v0 -; GFX9-NEXT: v_mul_lo_u32 v3, s8, v1 -; GFX9-NEXT: v_mul_hi_u32 v4, s8, v0 -; GFX9-NEXT: v_mul_lo_u32 v6, s8, v0 +; GFX9-NEXT: v_mul_lo_u32 v2, s13, v0 +; GFX9-NEXT: v_mul_lo_u32 v3, s12, v1 +; GFX9-NEXT: v_mul_hi_u32 v4, s12, v0 +; GFX9-NEXT: v_mul_lo_u32 v6, s12, v0 ; GFX9-NEXT: v_add3_u32 v2, v2, v3, v4 -; GFX9-NEXT: v_sub_co_u32_e32 v3, vcc, s12, v6 +; GFX9-NEXT: v_sub_co_u32_e32 v3, vcc, s8, v6 ; GFX9-NEXT: v_subb_co_u32_e64 v6, s[0:1], v7, v2, vcc -; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s9, v6 -; GFX9-NEXT: v_sub_u32_e32 v2, s13, v2 +; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s13, v6 +; GFX9-NEXT: v_sub_u32_e32 v2, s9, v2 ; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, -1, s[0:1] -; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s8, v3 +; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s12, v3 ; GFX9-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[0:1] -; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s9, v6 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s13, v6 ; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v5, vcc ; GFX9-NEXT: v_cndmask_b32_e64 v4, v4, v7, s[0:1] -; GFX9-NEXT: v_subrev_co_u32_e32 v7, vcc, s8, v3 +; GFX9-NEXT: v_subrev_co_u32_e32 v7, vcc, s12, v3 ; GFX9-NEXT: v_subbrev_co_u32_e64 v8, s[0:1], 0, v2, vcc ; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v5, vcc -; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s10 +; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s14 ; GFX9-NEXT: v_add_co_u32_e64 v9, s[0:1], 1, v0 ; GFX9-NEXT: v_addc_co_u32_e64 v10, s[0:1], 0, v1, s[0:1] ; GFX9-NEXT: v_add_f32_e32 v5, v14, v5 -; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s9, v8 +; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s13, v8 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v5, v5 ; GFX9-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[0:1] -; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s8, v7 +; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s12, v7 ; GFX9-NEXT: v_cndmask_b32_e64 v12, 0, -1, s[0:1] -; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s9, v8 -; GFX9-NEXT: v_subrev_co_u32_e32 v15, vcc, s8, v7 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s13, v8 +; GFX9-NEXT: v_subrev_co_u32_e32 v15, vcc, s12, v7 ; GFX9-NEXT: v_cndmask_b32_e64 v11, v11, v12, s[0:1] ; GFX9-NEXT: v_subbrev_co_u32_e32 v2, vcc, 0, v2, vcc ; GFX9-NEXT: v_add_co_u32_e64 v12, s[0:1], 1, v9 @@ -1468,7 +1458,7 @@ ; GFX9-NEXT: s_cselect_b32 s0, 1, 0 ; GFX9-NEXT: s_and_b32 s0, s0, 1 ; GFX9-NEXT: s_cmp_lg_u32 s0, 0 -; GFX9-NEXT: s_subb_u32 s3, 0, s11 +; GFX9-NEXT: s_subb_u32 s3, 0, s15 ; GFX9-NEXT: v_mul_lo_u32 v13, s3, v5 ; GFX9-NEXT: v_mul_lo_u32 v14, s2, v12 ; GFX9-NEXT: v_mul_hi_u32 v16, s2, v5 @@ -1530,19 +1520,19 @@ ; GFX9-NEXT: v_add3_u32 v7, v10, v7, v8 ; GFX9-NEXT: v_add_co_u32_e64 v3, s[0:1], v5, v3 ; GFX9-NEXT: v_addc_co_u32_e64 v7, s[0:1], v9, v7, s[0:1] -; GFX9-NEXT: v_mul_lo_u32 v8, s15, v3 -; GFX9-NEXT: v_mul_lo_u32 v9, s14, v7 +; GFX9-NEXT: v_mul_lo_u32 v8, s11, v3 +; GFX9-NEXT: v_mul_lo_u32 v9, s10, v7 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v6, v2, vcc -; GFX9-NEXT: v_mul_hi_u32 v2, s14, v3 -; GFX9-NEXT: v_mul_hi_u32 v3, s15, v3 +; GFX9-NEXT: v_mul_hi_u32 v2, s10, v3 +; GFX9-NEXT: v_mul_hi_u32 v3, s11, v3 ; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, v8, v9 ; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc ; GFX9-NEXT: v_add_co_u32_e32 v2, vcc, v6, v2 ; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc -; GFX9-NEXT: v_mul_lo_u32 v6, s15, v7 +; GFX9-NEXT: v_mul_lo_u32 v6, s11, v7 ; GFX9-NEXT: v_add_u32_e32 v2, v8, v2 -; GFX9-NEXT: v_mul_hi_u32 v8, s14, v7 -; GFX9-NEXT: v_mul_hi_u32 v7, s15, v7 +; GFX9-NEXT: v_mul_hi_u32 v8, s10, v7 +; GFX9-NEXT: v_mul_hi_u32 v7, s11, v7 ; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, v6, v3 ; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc ; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, v3, v8 @@ -1551,30 +1541,30 @@ ; GFX9-NEXT: v_add_u32_e32 v6, v6, v8 ; GFX9-NEXT: v_cndmask_b32_e64 v3, 0, 1, vcc ; GFX9-NEXT: v_add3_u32 v3, v6, v3, v7 -; GFX9-NEXT: v_mul_lo_u32 v6, s11, v2 -; GFX9-NEXT: v_mul_lo_u32 v7, s10, v3 -; GFX9-NEXT: v_mul_hi_u32 v8, s10, v2 -; GFX9-NEXT: v_mul_lo_u32 v10, s10, v2 -; GFX9-NEXT: v_mov_b32_e32 v11, s15 -; GFX9-NEXT: v_mov_b32_e32 v9, s11 +; GFX9-NEXT: v_mul_lo_u32 v6, s15, v2 +; GFX9-NEXT: v_mul_lo_u32 v7, s14, v3 +; GFX9-NEXT: v_mul_hi_u32 v8, s14, v2 +; GFX9-NEXT: v_mul_lo_u32 v10, s14, v2 +; GFX9-NEXT: v_mov_b32_e32 v11, s11 +; GFX9-NEXT: v_mov_b32_e32 v9, s15 ; GFX9-NEXT: v_add3_u32 v6, v6, v7, v8 -; GFX9-NEXT: v_sub_co_u32_e32 v7, vcc, s14, v10 +; GFX9-NEXT: v_sub_co_u32_e32 v7, vcc, s10, v10 ; GFX9-NEXT: v_subb_co_u32_e64 v8, s[0:1], v11, v6, vcc -; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s11, v8 -; GFX9-NEXT: v_sub_u32_e32 v6, s15, v6 +; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s15, v8 +; GFX9-NEXT: v_sub_u32_e32 v6, s11, v6 ; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, -1, s[0:1] -; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s10, v7 +; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s14, v7 ; GFX9-NEXT: v_cndmask_b32_e64 v11, 0, -1, s[0:1] -; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s11, v8 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s15, v8 ; GFX9-NEXT: v_subb_co_u32_e32 v6, vcc, v6, v9, vcc ; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, v11, s[0:1] -; GFX9-NEXT: v_subrev_co_u32_e32 v11, vcc, s10, v7 +; GFX9-NEXT: v_subrev_co_u32_e32 v11, vcc, s14, v7 ; GFX9-NEXT: v_subbrev_co_u32_e64 v12, s[0:1], 0, v6, vcc -; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s11, v12 +; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s15, v12 ; GFX9-NEXT: v_cndmask_b32_e64 v13, 0, -1, s[0:1] -; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s10, v11 +; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s14, v11 ; GFX9-NEXT: v_cndmask_b32_e64 v14, 0, -1, s[0:1] -; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s11, v12 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s15, v12 ; GFX9-NEXT: v_cndmask_b32_e64 v13, v13, v14, s[0:1] ; GFX9-NEXT: v_add_co_u32_e64 v14, s[0:1], 1, v2 ; GFX9-NEXT: v_subb_co_u32_e32 v6, vcc, v6, v9, vcc @@ -1584,7 +1574,7 @@ ; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v13 ; GFX9-NEXT: v_cndmask_b32_e32 v9, v14, v9, vcc ; GFX9-NEXT: v_cndmask_b32_e32 v14, v15, v16, vcc -; GFX9-NEXT: v_subrev_co_u32_e64 v15, s[0:1], s10, v11 +; GFX9-NEXT: v_subrev_co_u32_e64 v15, s[0:1], s14, v11 ; GFX9-NEXT: v_subbrev_co_u32_e64 v6, s[0:1], 0, v6, s[0:1] ; GFX9-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v10 ; GFX9-NEXT: v_mov_b32_e32 v13, 0 @@ -1601,28 +1591,28 @@ ; ; GFX10-LABEL: udivrem_v2i64: ; GFX10: ; %bb.0: -; GFX10-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x20 +; GFX10-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x10 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s9 -; GFX10-NEXT: v_cvt_f32_u32_e32 v2, s11 -; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s8 -; GFX10-NEXT: v_cvt_f32_u32_e32 v3, s10 -; GFX10-NEXT: s_sub_u32 s6, 0, s8 +; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s13 +; GFX10-NEXT: v_cvt_f32_u32_e32 v2, s15 +; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s12 +; GFX10-NEXT: v_cvt_f32_u32_e32 v3, s14 +; GFX10-NEXT: s_sub_u32 s0, 0, s12 ; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f800000, v0 ; GFX10-NEXT: v_mul_f32_e32 v2, 0x4f800000, v2 -; GFX10-NEXT: s_cselect_b32 s0, 1, 0 -; GFX10-NEXT: s_and_b32 s0, s0, 1 +; GFX10-NEXT: s_cselect_b32 s1, 1, 0 +; GFX10-NEXT: s_and_b32 s1, s1, 1 ; GFX10-NEXT: v_add_f32_e32 v0, v0, v1 ; GFX10-NEXT: v_add_f32_e32 v1, v2, v3 -; GFX10-NEXT: s_cmp_lg_u32 s0, 0 -; GFX10-NEXT: s_subb_u32 s7, 0, s9 +; GFX10-NEXT: s_cmp_lg_u32 s1, 0 +; GFX10-NEXT: s_subb_u32 s1, 0, s13 ; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX10-NEXT: v_rcp_iflag_f32_e32 v1, v1 -; GFX10-NEXT: s_sub_u32 s12, 0, s10 -; GFX10-NEXT: s_cselect_b32 s0, 1, 0 -; GFX10-NEXT: s_and_b32 s0, s0, 1 -; GFX10-NEXT: s_cmp_lg_u32 s0, 0 -; GFX10-NEXT: s_subb_u32 s13, 0, s11 +; GFX10-NEXT: s_sub_u32 s2, 0, s14 +; GFX10-NEXT: s_cselect_b32 s3, 1, 0 +; GFX10-NEXT: s_and_b32 s3, s3, 1 +; GFX10-NEXT: s_cmp_lg_u32 s3, 0 +; GFX10-NEXT: s_subb_u32 s3, 0, s15 ; GFX10-NEXT: v_mul_f32_e32 v0, 0x5f7ffffc, v0 ; GFX10-NEXT: v_mul_f32_e32 v1, 0x5f7ffffc, v1 ; GFX10-NEXT: v_mul_f32_e32 v2, 0x2f800000, v0 @@ -1635,16 +1625,16 @@ ; GFX10-NEXT: v_cvt_u32_f32_e32 v3, v3 ; GFX10-NEXT: v_add_f32_e32 v0, v4, v0 ; GFX10-NEXT: v_add_f32_e32 v1, v5, v1 -; GFX10-NEXT: v_mul_lo_u32 v4, s6, v2 -; GFX10-NEXT: v_mul_lo_u32 v8, s12, v3 +; GFX10-NEXT: v_mul_lo_u32 v4, s0, v2 +; GFX10-NEXT: v_mul_lo_u32 v8, s2, v3 ; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX10-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX10-NEXT: v_mul_lo_u32 v5, s7, v0 -; GFX10-NEXT: v_mul_hi_u32 v6, s6, v0 -; GFX10-NEXT: v_mul_lo_u32 v9, s13, v1 -; GFX10-NEXT: v_mul_hi_u32 v10, s12, v1 -; GFX10-NEXT: v_mul_lo_u32 v7, s6, v0 -; GFX10-NEXT: v_mul_lo_u32 v11, s12, v1 +; GFX10-NEXT: v_mul_lo_u32 v5, s1, v0 +; GFX10-NEXT: v_mul_hi_u32 v6, s0, v0 +; GFX10-NEXT: v_mul_lo_u32 v9, s3, v1 +; GFX10-NEXT: v_mul_hi_u32 v10, s2, v1 +; GFX10-NEXT: v_mul_lo_u32 v7, s0, v0 +; GFX10-NEXT: v_mul_lo_u32 v11, s2, v1 ; GFX10-NEXT: v_add3_u32 v4, v5, v4, v6 ; GFX10-NEXT: v_add3_u32 v8, v9, v8, v10 ; GFX10-NEXT: v_mul_lo_u32 v5, v2, v7 @@ -1657,49 +1647,49 @@ ; GFX10-NEXT: v_mul_hi_u32 v10, v1, v11 ; GFX10-NEXT: v_mul_hi_u32 v11, v3, v11 ; GFX10-NEXT: v_mul_lo_u32 v16, v3, v8 -; GFX10-NEXT: v_add_co_u32 v5, s0, v5, v12 +; GFX10-NEXT: v_add_co_u32 v5, s6, v5, v12 ; GFX10-NEXT: v_mul_hi_u32 v14, v0, v4 -; GFX10-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v7, s0, v13, v7 -; GFX10-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v9, s0, v9, v15 -; GFX10-NEXT: v_cndmask_b32_e64 v15, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v11, s0, v16, v11 -; GFX10-NEXT: v_cndmask_b32_e64 v16, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v5, s0, v5, v6 +; GFX10-NEXT: v_cndmask_b32_e64 v12, 0, 1, s6 +; GFX10-NEXT: v_add_co_u32 v7, s6, v13, v7 +; GFX10-NEXT: v_cndmask_b32_e64 v13, 0, 1, s6 +; GFX10-NEXT: v_add_co_u32 v9, s6, v9, v15 +; GFX10-NEXT: v_cndmask_b32_e64 v15, 0, 1, s6 +; GFX10-NEXT: v_add_co_u32 v11, s6, v16, v11 +; GFX10-NEXT: v_cndmask_b32_e64 v16, 0, 1, s6 +; GFX10-NEXT: v_add_co_u32 v5, s6, v5, v6 ; GFX10-NEXT: v_mul_hi_u32 v17, v1, v8 -; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v6, s0, v7, v14 -; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v9, s0, v9, v10 -; GFX10-NEXT: v_cndmask_b32_e64 v9, 0, 1, s0 +; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s6 +; GFX10-NEXT: v_add_co_u32 v6, s6, v7, v14 +; GFX10-NEXT: v_cndmask_b32_e64 v7, 0, 1, s6 +; GFX10-NEXT: v_add_co_u32 v9, s6, v9, v10 +; GFX10-NEXT: v_cndmask_b32_e64 v9, 0, 1, s6 ; GFX10-NEXT: v_add_nc_u32_e32 v5, v12, v5 -; GFX10-NEXT: v_add_co_u32 v10, s0, v11, v17 +; GFX10-NEXT: v_add_co_u32 v10, s6, v11, v17 ; GFX10-NEXT: v_mul_hi_u32 v4, v2, v4 ; GFX10-NEXT: v_add_nc_u32_e32 v9, v15, v9 -; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0 -; GFX10-NEXT: v_add_co_u32 v5, s0, v6, v5 +; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, 1, s6 +; GFX10-NEXT: v_add_co_u32 v5, s6, v6, v5 ; GFX10-NEXT: v_add_nc_u32_e32 v7, v13, v7 -; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0 +; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s6 ; GFX10-NEXT: v_mul_hi_u32 v8, v3, v8 -; GFX10-NEXT: v_add_co_u32 v9, s0, v10, v9 +; GFX10-NEXT: v_add_co_u32 v9, s6, v10, v9 ; GFX10-NEXT: v_add_nc_u32_e32 v11, v16, v11 -; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0 +; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, 1, s6 ; GFX10-NEXT: v_add3_u32 v4, v7, v6, v4 ; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v5 -; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x10 +; GFX10-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0 ; GFX10-NEXT: v_add3_u32 v5, v11, v10, v8 ; GFX10-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v4, vcc_lo ; GFX10-NEXT: v_add_co_u32 v1, vcc_lo, v1, v9 ; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v5, vcc_lo -; GFX10-NEXT: v_mul_lo_u32 v6, s7, v0 -; GFX10-NEXT: v_mul_hi_u32 v7, s6, v0 -; GFX10-NEXT: v_mul_lo_u32 v5, s6, v2 -; GFX10-NEXT: v_mul_lo_u32 v9, s13, v1 -; GFX10-NEXT: v_mul_hi_u32 v10, s12, v1 -; GFX10-NEXT: v_mul_lo_u32 v11, s12, v3 -; GFX10-NEXT: v_mul_lo_u32 v4, s6, v0 -; GFX10-NEXT: v_mul_lo_u32 v8, s12, v1 +; GFX10-NEXT: v_mul_lo_u32 v6, s1, v0 +; GFX10-NEXT: v_mul_hi_u32 v7, s0, v0 +; GFX10-NEXT: v_mul_lo_u32 v5, s0, v2 +; GFX10-NEXT: v_mul_lo_u32 v9, s3, v1 +; GFX10-NEXT: v_mul_hi_u32 v10, s2, v1 +; GFX10-NEXT: v_mul_lo_u32 v11, s2, v3 +; GFX10-NEXT: v_mul_lo_u32 v4, s0, v0 +; GFX10-NEXT: v_mul_lo_u32 v8, s2, v1 ; GFX10-NEXT: v_add3_u32 v5, v6, v5, v7 ; GFX10-NEXT: v_add3_u32 v9, v9, v11, v10 ; GFX10-NEXT: v_mul_lo_u32 v12, v2, v4 @@ -1711,168 +1701,166 @@ ; GFX10-NEXT: v_mul_lo_u32 v15, v1, v9 ; GFX10-NEXT: v_mul_hi_u32 v7, v1, v8 ; GFX10-NEXT: v_mul_hi_u32 v8, v3, v8 -; GFX10-NEXT: v_add_co_u32 v10, s6, v12, v10 -; GFX10-NEXT: v_cndmask_b32_e64 v12, 0, 1, s6 -; GFX10-NEXT: v_add_co_u32 v4, s6, v11, v4 +; GFX10-NEXT: v_add_co_u32 v10, s0, v12, v10 +; GFX10-NEXT: v_cndmask_b32_e64 v12, 0, 1, s0 +; GFX10-NEXT: v_add_co_u32 v4, s0, v11, v4 ; GFX10-NEXT: v_mul_lo_u32 v16, v3, v9 -; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, 1, s6 -; GFX10-NEXT: v_add_co_u32 v6, s6, v6, v15 +; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0 +; GFX10-NEXT: v_add_co_u32 v6, s0, v6, v15 ; GFX10-NEXT: v_mul_hi_u32 v14, v0, v5 -; GFX10-NEXT: v_cndmask_b32_e64 v15, 0, 1, s6 -; GFX10-NEXT: v_add_co_u32 v10, s6, v10, v13 -; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, 1, s6 -; GFX10-NEXT: v_add_co_u32 v8, s6, v16, v8 -; GFX10-NEXT: v_cndmask_b32_e64 v13, 0, 1, s6 -; GFX10-NEXT: v_add_co_u32 v4, s6, v4, v14 +; GFX10-NEXT: v_cndmask_b32_e64 v15, 0, 1, s0 +; GFX10-NEXT: v_add_co_u32 v10, s0, v10, v13 +; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0 +; GFX10-NEXT: v_add_co_u32 v8, s0, v16, v8 +; GFX10-NEXT: v_cndmask_b32_e64 v13, 0, 1, s0 +; GFX10-NEXT: v_add_co_u32 v4, s0, v4, v14 ; GFX10-NEXT: v_add_nc_u32_e32 v10, v12, v10 -; GFX10-NEXT: v_cndmask_b32_e64 v14, 0, 1, s6 -; GFX10-NEXT: v_add_co_u32 v6, s6, v6, v7 +; GFX10-NEXT: v_cndmask_b32_e64 v14, 0, 1, s0 +; GFX10-NEXT: v_add_co_u32 v6, s0, v6, v7 ; GFX10-NEXT: v_mul_hi_u32 v5, v2, v5 -; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s6 -; GFX10-NEXT: v_add_co_u32 v4, s6, v4, v10 +; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0 +; GFX10-NEXT: v_add_co_u32 v4, s0, v4, v10 ; GFX10-NEXT: v_add_nc_u32_e32 v7, v11, v14 -; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, 1, s6 +; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, 1, s0 ; GFX10-NEXT: v_mul_hi_u32 v17, v1, v9 ; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v4 ; GFX10-NEXT: v_add_nc_u32_e32 v6, v15, v6 ; GFX10-NEXT: v_add3_u32 v5, v7, v10, v5 ; GFX10-NEXT: v_mul_hi_u32 v9, v3, v9 -; GFX10-NEXT: v_mov_b32_e32 v10, 0 -; GFX10-NEXT: v_add_co_u32 v8, s6, v8, v17 +; GFX10-NEXT: v_mul_hi_u32 v10, s9, v0 +; GFX10-NEXT: v_mov_b32_e32 v12, 0 +; GFX10-NEXT: v_add_co_u32 v8, s0, v8, v17 ; GFX10-NEXT: v_add_co_ci_u32_e32 v2, vcc_lo, v2, v5, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, 1, s6 -; GFX10-NEXT: v_add_co_u32 v4, s12, v8, v6 -; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: v_mul_lo_u32 v6, s1, v0 -; GFX10-NEXT: v_mul_lo_u32 v8, s0, v2 +; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, 1, s0 +; GFX10-NEXT: v_add_co_u32 v4, s0, v8, v6 +; GFX10-NEXT: v_mul_lo_u32 v6, s9, v0 +; GFX10-NEXT: v_mul_lo_u32 v8, s8, v2 +; GFX10-NEXT: v_mul_hi_u32 v0, s8, v0 ; GFX10-NEXT: v_add_nc_u32_e32 v7, v13, v11 -; GFX10-NEXT: v_mul_hi_u32 v11, s1, v0 -; GFX10-NEXT: v_mul_hi_u32 v0, s0, v0 -; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s12 -; GFX10-NEXT: v_mul_lo_u32 v12, s1, v2 +; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0 +; GFX10-NEXT: v_mul_lo_u32 v11, s9, v2 ; GFX10-NEXT: v_add_co_u32 v1, vcc_lo, v1, v4 -; GFX10-NEXT: v_add_co_u32 v6, s12, v6, v8 +; GFX10-NEXT: v_add_co_u32 v6, s0, v6, v8 ; GFX10-NEXT: v_add3_u32 v5, v7, v5, v9 -; GFX10-NEXT: v_mul_hi_u32 v7, s0, v2 -; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, s12 -; GFX10-NEXT: v_add_co_u32 v0, s13, v6, v0 -; GFX10-NEXT: v_add_co_u32 v9, s12, v12, v11 -; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s13 -; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s12 -; GFX10-NEXT: v_mul_hi_u32 v2, s1, v2 -; GFX10-NEXT: v_add_co_u32 v7, s12, v9, v7 +; GFX10-NEXT: v_mul_hi_u32 v7, s8, v2 +; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, s0 +; GFX10-NEXT: v_add_co_u32 v0, s1, v6, v0 +; GFX10-NEXT: v_add_co_u32 v9, s0, v11, v10 +; GFX10-NEXT: v_cndmask_b32_e64 v0, 0, 1, s1 +; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0 +; GFX10-NEXT: v_mul_hi_u32 v2, s9, v2 +; GFX10-NEXT: v_add_co_u32 v7, s0, v9, v7 ; GFX10-NEXT: v_add_nc_u32_e32 v0, v8, v0 -; GFX10-NEXT: v_cndmask_b32_e64 v9, 0, 1, s12 +; GFX10-NEXT: v_cndmask_b32_e64 v9, 0, 1, s0 ; GFX10-NEXT: v_add_co_ci_u32_e32 v3, vcc_lo, v3, v5, vcc_lo -; GFX10-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0 -; GFX10-NEXT: v_add_co_u32 v0, s12, v7, v0 +; GFX10-NEXT: v_add_co_u32 v0, s0, v7, v0 ; GFX10-NEXT: v_add_nc_u32_e32 v4, v6, v9 -; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s12 -; GFX10-NEXT: v_mul_lo_u32 v6, s3, v1 -; GFX10-NEXT: v_mul_lo_u32 v8, s2, v3 -; GFX10-NEXT: v_mul_lo_u32 v9, s9, v0 -; GFX10-NEXT: v_mul_hi_u32 v11, s8, v0 +; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0 +; GFX10-NEXT: v_mul_lo_u32 v6, s11, v1 +; GFX10-NEXT: v_mul_lo_u32 v8, s10, v3 +; GFX10-NEXT: v_mul_lo_u32 v9, s13, v0 +; GFX10-NEXT: v_mul_hi_u32 v10, s12, v0 ; GFX10-NEXT: v_add3_u32 v2, v4, v5, v2 -; GFX10-NEXT: v_mul_hi_u32 v7, s2, v1 -; GFX10-NEXT: v_mul_lo_u32 v13, s8, v0 -; GFX10-NEXT: v_mul_hi_u32 v1, s3, v1 -; GFX10-NEXT: v_mul_lo_u32 v4, s3, v3 -; GFX10-NEXT: v_mul_lo_u32 v12, s8, v2 -; GFX10-NEXT: v_add_co_u32 v6, s12, v6, v8 -; GFX10-NEXT: v_mul_hi_u32 v5, s2, v3 -; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, s12 -; GFX10-NEXT: v_mul_hi_u32 v3, s3, v3 -; GFX10-NEXT: v_add_co_u32 v1, s12, v4, v1 -; GFX10-NEXT: v_add3_u32 v9, v9, v12, v11 -; GFX10-NEXT: v_sub_co_u32 v11, vcc_lo, s0, v13 +; GFX10-NEXT: v_mul_hi_u32 v7, s10, v1 +; GFX10-NEXT: v_mul_hi_u32 v1, s11, v1 +; GFX10-NEXT: v_mul_lo_u32 v4, s11, v3 +; GFX10-NEXT: v_mul_lo_u32 v13, s12, v0 +; GFX10-NEXT: v_mul_lo_u32 v11, s12, v2 +; GFX10-NEXT: v_add_co_u32 v6, s0, v6, v8 +; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, 1, s0 +; GFX10-NEXT: v_mul_hi_u32 v5, s10, v3 +; GFX10-NEXT: v_mul_hi_u32 v3, s11, v3 +; GFX10-NEXT: v_add_co_u32 v1, s0, v4, v1 +; GFX10-NEXT: v_add3_u32 v9, v9, v11, v10 +; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, 1, s0 +; GFX10-NEXT: v_sub_co_u32 v10, vcc_lo, s8, v13 ; GFX10-NEXT: v_add_co_u32 v6, s0, v6, v7 -; GFX10-NEXT: v_sub_nc_u32_e32 v7, s1, v9 +; GFX10-NEXT: v_sub_nc_u32_e32 v7, s9, v9 ; GFX10-NEXT: v_cndmask_b32_e64 v6, 0, 1, s0 -; GFX10-NEXT: v_sub_co_ci_u32_e64 v9, s0, s1, v9, vcc_lo -; GFX10-NEXT: v_cmp_le_u32_e64 s0, s8, v11 -; GFX10-NEXT: v_subrev_co_ci_u32_e32 v7, vcc_lo, s9, v7, vcc_lo -; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s9, v9 +; GFX10-NEXT: v_sub_co_ci_u32_e64 v9, s0, s9, v9, vcc_lo +; GFX10-NEXT: v_cmp_le_u32_e64 s0, s12, v10 +; GFX10-NEXT: v_subrev_co_ci_u32_e32 v7, vcc_lo, s13, v7, vcc_lo +; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s13, v9 ; GFX10-NEXT: v_add_nc_u32_e32 v6, v8, v6 -; GFX10-NEXT: v_cndmask_b32_e64 v12, 0, -1, s0 -; GFX10-NEXT: v_cndmask_b32_e64 v4, 0, 1, s12 +; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, -1, s0 ; GFX10-NEXT: v_cndmask_b32_e64 v13, 0, -1, vcc_lo -; GFX10-NEXT: v_sub_co_u32 v14, vcc_lo, v11, s8 +; GFX10-NEXT: v_sub_co_u32 v14, vcc_lo, v10, s12 ; GFX10-NEXT: v_subrev_co_ci_u32_e64 v15, s0, 0, v7, vcc_lo ; GFX10-NEXT: v_add_co_u32 v1, s0, v1, v5 ; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, 1, s0 -; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s9, v9 -; GFX10-NEXT: v_subrev_co_ci_u32_e32 v7, vcc_lo, s9, v7, vcc_lo +; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s13, v9 +; GFX10-NEXT: v_subrev_co_ci_u32_e32 v7, vcc_lo, s13, v7, vcc_lo ; GFX10-NEXT: v_add_nc_u32_e32 v4, v4, v5 -; GFX10-NEXT: v_cndmask_b32_e64 v12, v13, v12, s0 -; GFX10-NEXT: v_cmp_le_u32_e64 s0, s9, v15 +; GFX10-NEXT: v_cndmask_b32_e64 v11, v13, v11, s0 +; GFX10-NEXT: v_cmp_le_u32_e64 s0, s13, v15 ; GFX10-NEXT: v_cndmask_b32_e64 v13, 0, -1, s0 -; GFX10-NEXT: v_cmp_le_u32_e64 s0, s8, v14 +; GFX10-NEXT: v_cmp_le_u32_e64 s0, s12, v14 ; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, -1, s0 ; GFX10-NEXT: v_add_co_u32 v6, s0, v1, v6 ; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, 1, s0 ; GFX10-NEXT: v_add_co_u32 v5, s0, v0, 1 ; GFX10-NEXT: v_add_co_ci_u32_e64 v16, s0, 0, v2, s0 -; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s9, v15 +; GFX10-NEXT: v_cmp_eq_u32_e64 s0, s13, v15 ; GFX10-NEXT: v_add3_u32 v3, v4, v1, v3 -; GFX10-NEXT: v_mul_hi_u32 v18, s10, v6 +; GFX10-NEXT: v_mul_hi_u32 v18, s14, v6 ; GFX10-NEXT: v_cndmask_b32_e64 v8, v13, v8, s0 -; GFX10-NEXT: v_mul_lo_u32 v13, s11, v6 -; GFX10-NEXT: v_mul_lo_u32 v17, s10, v3 +; GFX10-NEXT: v_mul_lo_u32 v13, s15, v6 +; GFX10-NEXT: v_mul_lo_u32 v17, s14, v3 ; GFX10-NEXT: v_add_co_u32 v1, s0, v5, 1 ; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v8 ; GFX10-NEXT: v_add_co_ci_u32_e64 v4, s0, 0, v16, s0 -; GFX10-NEXT: v_sub_co_u32 v19, s0, v14, s8 +; GFX10-NEXT: v_sub_co_u32 v19, s0, v14, s12 ; GFX10-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo -; GFX10-NEXT: v_mul_lo_u32 v5, s10, v6 +; GFX10-NEXT: v_mul_lo_u32 v5, s14, v6 ; GFX10-NEXT: v_cndmask_b32_e32 v4, v16, v4, vcc_lo ; GFX10-NEXT: v_add3_u32 v13, v13, v17, v18 -; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v12 +; GFX10-NEXT: v_cmp_ne_u32_e32 vcc_lo, 0, v11 ; GFX10-NEXT: v_subrev_co_ci_u32_e64 v7, s0, 0, v7, s0 ; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo ; GFX10-NEXT: v_cndmask_b32_e32 v1, v2, v4, vcc_lo -; GFX10-NEXT: v_sub_nc_u32_e32 v2, s3, v13 -; GFX10-NEXT: v_sub_co_u32 v12, s0, s2, v5 -; GFX10-NEXT: v_sub_co_ci_u32_e64 v16, s1, s3, v13, s0 -; GFX10-NEXT: v_subrev_co_ci_u32_e64 v2, s0, s11, v2, s0 -; GFX10-NEXT: v_cmp_le_u32_e64 s0, s10, v12 +; GFX10-NEXT: v_sub_nc_u32_e32 v2, s11, v13 +; GFX10-NEXT: v_sub_co_u32 v11, s0, s10, v5 +; GFX10-NEXT: v_sub_co_ci_u32_e64 v16, s1, s11, v13, s0 +; GFX10-NEXT: v_subrev_co_ci_u32_e64 v2, s0, s15, v2, s0 +; GFX10-NEXT: v_cmp_le_u32_e64 s0, s14, v11 ; GFX10-NEXT: v_cmp_ne_u32_e64 s1, 0, v8 -; GFX10-NEXT: v_cmp_le_u32_e64 s2, s11, v16 +; GFX10-NEXT: v_cmp_le_u32_e64 s2, s15, v16 ; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, -1, s0 -; GFX10-NEXT: v_sub_co_u32 v13, s0, v12, s10 +; GFX10-NEXT: v_sub_co_u32 v13, s0, v11, s14 ; GFX10-NEXT: v_cndmask_b32_e64 v4, v14, v19, s1 ; GFX10-NEXT: v_cndmask_b32_e64 v5, 0, -1, s2 ; GFX10-NEXT: v_subrev_co_ci_u32_e64 v14, s2, 0, v2, s0 ; GFX10-NEXT: v_cndmask_b32_e64 v7, v15, v7, s1 -; GFX10-NEXT: v_cmp_eq_u32_e64 s1, s11, v16 -; GFX10-NEXT: v_cndmask_b32_e32 v4, v11, v4, vcc_lo -; GFX10-NEXT: v_subrev_co_ci_u32_e64 v2, s0, s11, v2, s0 +; GFX10-NEXT: v_cmp_eq_u32_e64 s1, s15, v16 +; GFX10-NEXT: v_cndmask_b32_e32 v4, v10, v4, vcc_lo +; GFX10-NEXT: v_subrev_co_ci_u32_e64 v2, s0, s15, v2, s0 ; GFX10-NEXT: v_cndmask_b32_e64 v5, v5, v8, s1 -; GFX10-NEXT: v_cmp_le_u32_e64 s1, s11, v14 +; GFX10-NEXT: v_cmp_le_u32_e64 s1, s15, v14 ; GFX10-NEXT: v_cndmask_b32_e64 v8, 0, -1, s1 -; GFX10-NEXT: v_cmp_le_u32_e64 s1, s10, v13 -; GFX10-NEXT: v_cndmask_b32_e64 v11, 0, -1, s1 +; GFX10-NEXT: v_cmp_le_u32_e64 s1, s14, v13 +; GFX10-NEXT: v_cndmask_b32_e64 v10, 0, -1, s1 ; GFX10-NEXT: v_add_co_u32 v15, s1, v6, 1 ; GFX10-NEXT: v_add_co_ci_u32_e64 v17, s1, 0, v3, s1 -; GFX10-NEXT: v_cmp_eq_u32_e64 s1, s11, v14 -; GFX10-NEXT: v_cndmask_b32_e64 v8, v8, v11, s1 -; GFX10-NEXT: v_add_co_u32 v11, s1, v15, 1 +; GFX10-NEXT: v_cmp_eq_u32_e64 s1, s15, v14 +; GFX10-NEXT: v_cndmask_b32_e64 v8, v8, v10, s1 +; GFX10-NEXT: v_add_co_u32 v10, s1, v15, 1 ; GFX10-NEXT: v_add_co_ci_u32_e64 v18, s1, 0, v17, s1 ; GFX10-NEXT: v_cmp_ne_u32_e64 s0, 0, v8 -; GFX10-NEXT: v_sub_co_u32 v8, s1, v13, s10 +; GFX10-NEXT: v_sub_co_u32 v8, s1, v13, s14 ; GFX10-NEXT: v_subrev_co_ci_u32_e64 v2, s1, 0, v2, s1 -; GFX10-NEXT: v_cndmask_b32_e64 v11, v15, v11, s0 +; GFX10-NEXT: v_cndmask_b32_e64 v10, v15, v10, s0 ; GFX10-NEXT: v_cndmask_b32_e64 v15, v17, v18, s0 ; GFX10-NEXT: v_cmp_ne_u32_e64 s1, 0, v5 ; GFX10-NEXT: v_cndmask_b32_e64 v8, v13, v8, s0 ; GFX10-NEXT: v_cndmask_b32_e64 v13, v14, v2, s0 ; GFX10-NEXT: v_cndmask_b32_e32 v5, v9, v7, vcc_lo -; GFX10-NEXT: v_cndmask_b32_e64 v2, v6, v11, s1 +; GFX10-NEXT: v_cndmask_b32_e64 v2, v6, v10, s1 ; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v15, s1 -; GFX10-NEXT: v_cndmask_b32_e64 v6, v12, v8, s1 +; GFX10-NEXT: v_cndmask_b32_e64 v6, v11, v8, s1 ; GFX10-NEXT: v_cndmask_b32_e64 v7, v16, v13, s1 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: global_store_dwordx4 v10, v[0:3], s[4:5] -; GFX10-NEXT: global_store_dwordx4 v10, v[4:7], s[6:7] +; GFX10-NEXT: global_store_dwordx4 v12, v[0:3], s[4:5] +; GFX10-NEXT: global_store_dwordx4 v12, v[4:7], s[6:7] ; GFX10-NEXT: s_endpgm %div = udiv <2 x i64> %x, %y store <2 x i64> %div, <2 x i64> addrspace(1)* %out0 @@ -2307,57 +2295,55 @@ define amdgpu_kernel void @udivrem_v2i16(<2 x i16> addrspace(1)* %out0, <2 x i16> addrspace(1)* %out1, <2 x i16> %x, <2 x i16> %y) { ; GFX8-LABEL: udivrem_v2i16: ; GFX8: ; %bb.0: -; GFX8-NEXT: s_load_dword s0, s[4:5], 0x14 +; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10 ; GFX8-NEXT: s_mov_b32 s2, 0xffff +; GFX8-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: s_and_b32 s6, s0, s2 -; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s6 -; GFX8-NEXT: s_lshr_b32 s3, s0, 16 -; GFX8-NEXT: v_cvt_f32_u32_e32 v1, s3 -; GFX8-NEXT: s_sub_i32 s1, 0, s6 +; GFX8-NEXT: s_and_b32 s3, s1, s2 +; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s3 +; GFX8-NEXT: s_lshr_b32 s8, s1, 16 +; GFX8-NEXT: v_cvt_f32_u32_e32 v1, s8 +; GFX8-NEXT: s_sub_i32 s1, 0, s3 ; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x0 -; GFX8-NEXT: s_load_dword s0, s[4:5], 0x10 +; GFX8-NEXT: s_lshr_b32 s9, s0, 16 ; GFX8-NEXT: v_rcp_iflag_f32_e32 v1, v1 +; GFX8-NEXT: s_and_b32 s0, s0, s2 ; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX8-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX8-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ; GFX8-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: s_lshr_b32 s4, s0, 16 ; GFX8-NEXT: v_mul_lo_u32 v2, s1, v0 -; GFX8-NEXT: s_sub_i32 s1, 0, s3 -; GFX8-NEXT: s_and_b32 s0, s0, s2 +; GFX8-NEXT: s_sub_i32 s1, 0, s8 ; GFX8-NEXT: v_mul_lo_u32 v3, s1, v1 ; GFX8-NEXT: v_mul_hi_u32 v2, v0, v2 ; GFX8-NEXT: v_mul_hi_u32 v3, v1, v3 ; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2 ; GFX8-NEXT: v_mul_hi_u32 v0, s0, v0 ; GFX8-NEXT: v_add_u32_e32 v1, vcc, v1, v3 -; GFX8-NEXT: v_mul_hi_u32 v1, s4, v1 -; GFX8-NEXT: v_mul_lo_u32 v2, v0, s6 +; GFX8-NEXT: v_mul_hi_u32 v1, s9, v1 +; GFX8-NEXT: v_mul_lo_u32 v2, v0, s3 ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 1, v0 ; GFX8-NEXT: v_sub_u32_e32 v2, vcc, s0, v2 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s6, v2 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s3, v2 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX8-NEXT: v_subrev_u32_e64 v3, s[0:1], s6, v2 +; GFX8-NEXT: v_subrev_u32_e64 v3, s[0:1], s3, v2 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc ; GFX8-NEXT: v_add_u32_e32 v3, vcc, 1, v0 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s6, v2 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s3, v2 ; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX8-NEXT: v_mul_lo_u32 v3, v1, s3 -; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s6, v2 +; GFX8-NEXT: v_mul_lo_u32 v3, v1, s8 +; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s3, v2 ; GFX8-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc -; GFX8-NEXT: v_sub_u32_e32 v3, vcc, s4, v3 +; GFX8-NEXT: v_sub_u32_e32 v3, vcc, s9, v3 ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 1, v1 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s3, v3 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s8, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s3, v3 +; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s8, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc ; GFX8-NEXT: v_add_u32_e32 v4, vcc, 1, v1 -; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s3, v3 +; GFX8-NEXT: v_cmp_le_u32_e32 vcc, s8, v3 ; GFX8-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s3, v3 +; GFX8-NEXT: v_subrev_u32_e64 v4, s[0:1], s8, v3 ; GFX8-NEXT: v_and_b32_e32 v1, s2, v1 ; GFX8-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc ; GFX8-NEXT: v_lshlrev_b32_e32 v1, 16, v1 @@ -2365,36 +2351,34 @@ ; GFX8-NEXT: v_and_b32_e32 v0, s2, v3 ; GFX8-NEXT: v_lshlrev_b32_e32 v0, 16, v0 ; GFX8-NEXT: v_or_b32_sdwa v2, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_mov_b32_e32 v0, s8 -; GFX8-NEXT: v_mov_b32_e32 v1, s9 +; GFX8-NEXT: v_mov_b32_e32 v0, s4 +; GFX8-NEXT: v_mov_b32_e32 v1, s5 ; GFX8-NEXT: flat_store_dword v[0:1], v4 -; GFX8-NEXT: v_mov_b32_e32 v0, s10 -; GFX8-NEXT: v_mov_b32_e32 v1, s11 +; GFX8-NEXT: v_mov_b32_e32 v0, s6 +; GFX8-NEXT: v_mov_b32_e32 v1, s7 ; GFX8-NEXT: flat_store_dword v[0:1], v2 ; GFX8-NEXT: s_endpgm ; ; GFX9-LABEL: udivrem_v2i16: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dword s0, s[4:5], 0x14 -; GFX9-NEXT: s_mov_b32 s1, 0xffff +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10 +; GFX9-NEXT: s_mov_b32 s2, 0xffff ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_and_b32 s7, s0, s1 +; GFX9-NEXT: s_and_b32 s7, s1, s2 ; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s7 -; GFX9-NEXT: s_lshr_b32 s6, s0, 16 +; GFX9-NEXT: s_lshr_b32 s6, s1, 16 ; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s6 -; GFX9-NEXT: s_sub_i32 s2, 0, s7 +; GFX9-NEXT: s_sub_i32 s1, 0, s7 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX9-NEXT: s_load_dword s0, s[4:5], 0x10 -; GFX9-NEXT: v_rcp_iflag_f32_e32 v1, v1 ; GFX9-NEXT: s_sub_i32 s3, 0, s6 +; GFX9-NEXT: v_rcp_iflag_f32_e32 v1, v1 +; GFX9-NEXT: s_and_b32 s9, s0, s2 ; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX9-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1 ; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_and_b32 s9, s0, s1 -; GFX9-NEXT: v_mul_lo_u32 v2, s2, v0 ; GFX9-NEXT: s_lshr_b32 s8, s0, 16 +; GFX9-NEXT: v_mul_lo_u32 v2, s1, v0 ; GFX9-NEXT: v_mul_lo_u32 v3, s3, v1 ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; GFX9-NEXT: v_mul_hi_u32 v2, v0, v2 @@ -2440,15 +2424,14 @@ ; ; GFX10-LABEL: udivrem_v2i16: ; GFX10: ; %bb.0: -; GFX10-NEXT: s_load_dword s0, s[4:5], 0x14 -; GFX10-NEXT: s_mov_b32 s2, 0xffff +; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x10 +; GFX10-NEXT: s_mov_b32 s3, 0xffff ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_lshr_b32 s1, s0, 16 -; GFX10-NEXT: s_and_b32 s3, s0, s2 -; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s1 -; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s3 -; GFX10-NEXT: s_sub_i32 s6, 0, s1 -; GFX10-NEXT: s_load_dword s0, s[4:5], 0x10 +; GFX10-NEXT: s_lshr_b32 s2, s1, 16 +; GFX10-NEXT: s_and_b32 s1, s1, s3 +; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s2 +; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s1 +; GFX10-NEXT: s_sub_i32 s6, 0, s2 ; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX10-NEXT: v_rcp_iflag_f32_e32 v1, v1 ; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0 @@ -2456,38 +2439,37 @@ ; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX10-NEXT: v_cvt_u32_f32_e32 v1, v1 ; GFX10-NEXT: v_mul_lo_u32 v2, s6, v0 -; GFX10-NEXT: s_sub_i32 s6, 0, s3 +; GFX10-NEXT: s_sub_i32 s6, 0, s1 ; GFX10-NEXT: v_mul_lo_u32 v3, s6, v1 -; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_lshr_b32 s6, s0, 16 -; GFX10-NEXT: s_and_b32 s0, s0, s2 +; GFX10-NEXT: s_and_b32 s0, s0, s3 ; GFX10-NEXT: v_mul_hi_u32 v2, v0, v2 ; GFX10-NEXT: v_mul_hi_u32 v3, v1, v3 ; GFX10-NEXT: v_add_nc_u32_e32 v0, v0, v2 ; GFX10-NEXT: v_add_nc_u32_e32 v1, v1, v3 ; GFX10-NEXT: v_mul_hi_u32 v0, s6, v0 ; GFX10-NEXT: v_mul_hi_u32 v1, s0, v1 -; GFX10-NEXT: v_mul_lo_u32 v2, v0, s1 +; GFX10-NEXT: v_mul_lo_u32 v2, v0, s2 ; GFX10-NEXT: v_add_nc_u32_e32 v4, 1, v0 -; GFX10-NEXT: v_mul_lo_u32 v3, v1, s3 +; GFX10-NEXT: v_mul_lo_u32 v3, v1, s1 ; GFX10-NEXT: v_add_nc_u32_e32 v6, 1, v1 ; GFX10-NEXT: v_sub_nc_u32_e32 v2, s6, v2 ; GFX10-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0 ; GFX10-NEXT: v_sub_nc_u32_e32 v3, s0, v3 -; GFX10-NEXT: v_subrev_nc_u32_e32 v5, s1, v2 -; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s1, v2 -; GFX10-NEXT: v_cmp_le_u32_e64 s0, s3, v3 -; GFX10-NEXT: v_subrev_nc_u32_e32 v7, s3, v3 +; GFX10-NEXT: v_subrev_nc_u32_e32 v5, s2, v2 +; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s2, v2 +; GFX10-NEXT: v_cmp_le_u32_e64 s0, s1, v3 +; GFX10-NEXT: v_subrev_nc_u32_e32 v7, s1, v3 ; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo ; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo ; GFX10-NEXT: v_cndmask_b32_e64 v1, v1, v6, s0 ; GFX10-NEXT: v_cndmask_b32_e64 v3, v3, v7, s0 ; GFX10-NEXT: v_add_nc_u32_e32 v4, 1, v0 -; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s1, v2 -; GFX10-NEXT: v_subrev_nc_u32_e32 v5, s1, v2 +; GFX10-NEXT: v_cmp_le_u32_e32 vcc_lo, s2, v2 +; GFX10-NEXT: v_subrev_nc_u32_e32 v5, s2, v2 ; GFX10-NEXT: v_add_nc_u32_e32 v6, 1, v1 -; GFX10-NEXT: v_cmp_le_u32_e64 s0, s3, v3 -; GFX10-NEXT: v_subrev_nc_u32_e32 v7, s3, v3 +; GFX10-NEXT: v_cmp_le_u32_e64 s0, s1, v3 +; GFX10-NEXT: v_subrev_nc_u32_e32 v7, s1, v3 ; GFX10-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc_lo ; GFX10-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo ; GFX10-NEXT: v_mov_b32_e32 v4, 0xffff diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll --- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll +++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll @@ -2528,18 +2528,16 @@ ; GFX6-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; GFX6-NEXT: s_mov_b32 s8, 0xffff ; GFX6-NEXT: s_mov_b32 s3, 0xf000 -; GFX6-NEXT: s_mov_b32 s2, -1 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_and_b32 s9, s6, s8 -; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s9 -; GFX6-NEXT: s_lshr_b32 s9, s4, 16 -; GFX6-NEXT: s_and_b32 s4, s4, s8 +; GFX6-NEXT: s_and_b32 s2, s6, s8 +; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s2 +; GFX6-NEXT: s_and_b32 s9, s4, s8 ; GFX6-NEXT: s_lshr_b32 s6, s6, 16 -; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s4 +; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s9 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v2, v0 ; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s6 -; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s9 -; GFX6-NEXT: s_and_b32 s6, s7, s8 +; GFX6-NEXT: s_lshr_b32 s4, s4, 16 +; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4 ; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v5, v3 ; GFX6-NEXT: v_trunc_f32_e32 v2, v2 @@ -2548,36 +2546,38 @@ ; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 ; GFX6-NEXT: v_mul_f32_e32 v1, v4, v5 ; GFX6-NEXT: v_trunc_f32_e32 v1, v1 +; GFX6-NEXT: s_and_b32 s4, s7, s8 ; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc ; GFX6-NEXT: v_mad_f32 v2, -v1, v3, v4 -; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s6 -; GFX6-NEXT: s_lshr_b32 s4, s5, 16 -; GFX6-NEXT: s_lshr_b32 s10, s7, 16 -; GFX6-NEXT: s_and_b32 s5, s5, s8 +; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4 +; GFX6-NEXT: s_and_b32 s4, s5, s8 ; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX6-NEXT: v_cvt_f32_u32_e32 v5, s5 +; GFX6-NEXT: v_cvt_f32_u32_e32 v5, s4 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v6, v4 ; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v3 -; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s10 ; GFX6-NEXT: v_addc_u32_e32 v2, vcc, 0, v1, vcc ; GFX6-NEXT: v_mul_f32_e32 v1, v5, v6 -; GFX6-NEXT: v_cvt_f32_u32_e32 v6, s4 -; GFX6-NEXT: v_rcp_iflag_f32_e32 v7, v3 ; GFX6-NEXT: v_trunc_f32_e32 v1, v1 -; GFX6-NEXT: v_mad_f32 v5, -v1, v4, v5 +; GFX6-NEXT: s_lshr_b32 s4, s7, 16 +; GFX6-NEXT: v_mad_f32 v3, -v1, v4, v5 +; GFX6-NEXT: v_cvt_f32_u32_e32 v5, s4 +; GFX6-NEXT: s_lshr_b32 s4, s5, 16 +; GFX6-NEXT: v_cvt_f32_u32_e32 v6, s4 ; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v5|, v4 -; GFX6-NEXT: v_mul_f32_e32 v4, v6, v7 -; GFX6-NEXT: v_trunc_f32_e32 v4, v4 -; GFX6-NEXT: v_cvt_u32_f32_e32 v5, v4 +; GFX6-NEXT: v_rcp_iflag_f32_e32 v7, v5 +; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v4 +; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc -; GFX6-NEXT: v_mad_f32 v4, -v4, v3, v6 -; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v4|, v3 -; GFX6-NEXT: v_addc_u32_e32 v3, vcc, 0, v5, vcc -; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX6-NEXT: v_mul_f32_e32 v3, v6, v7 +; GFX6-NEXT: v_trunc_f32_e32 v3, v3 +; GFX6-NEXT: v_cvt_u32_f32_e32 v4, v3 +; GFX6-NEXT: v_mad_f32 v3, -v3, v5, v6 +; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v5 ; GFX6-NEXT: v_and_b32_e32 v1, s8, v1 -; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, 0, v4, vcc +; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; GFX6-NEXT: v_and_b32_e32 v0, s8, v0 +; GFX6-NEXT: s_mov_b32 s2, -1 ; GFX6-NEXT: v_or_b32_e32 v1, v1, v3 ; GFX6-NEXT: v_or_b32_e32 v0, v0, v2 ; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 @@ -2592,49 +2592,49 @@ ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_and_b32 s8, s6, s0 ; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s8 +; GFX9-NEXT: s_lshr_b32 s6, s6, 16 +; GFX9-NEXT: s_and_b32 s1, s4, s0 +; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s6 +; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s1 +; GFX9-NEXT: v_rcp_iflag_f32_e32 v4, v0 ; GFX9-NEXT: s_lshr_b32 s1, s4, 16 -; GFX9-NEXT: s_and_b32 s4, s4, s0 -; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s4 -; GFX9-NEXT: s_lshr_b32 s4, s6, 16 -; GFX9-NEXT: v_rcp_iflag_f32_e32 v3, v0 -; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s4 ; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s1 -; GFX9-NEXT: s_and_b32 s1, s7, s0 -; GFX9-NEXT: v_mul_f32_e32 v3, v1, v3 -; GFX9-NEXT: v_rcp_iflag_f32_e32 v6, v4 -; GFX9-NEXT: v_trunc_f32_e32 v3, v3 -; GFX9-NEXT: v_mad_f32 v1, -v3, v0, v1 -; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3 +; GFX9-NEXT: v_rcp_iflag_f32_e32 v6, v3 +; GFX9-NEXT: v_mul_f32_e32 v4, v1, v4 +; GFX9-NEXT: v_trunc_f32_e32 v4, v4 +; GFX9-NEXT: v_mad_f32 v1, -v4, v0, v1 ; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 ; GFX9-NEXT: v_mul_f32_e32 v1, v5, v6 ; GFX9-NEXT: v_trunc_f32_e32 v1, v1 -; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v3, vcc -; GFX9-NEXT: v_mad_f32 v3, -v1, v4, v5 +; GFX9-NEXT: s_and_b32 s1, s7, s0 +; GFX9-NEXT: v_cvt_u32_f32_e32 v7, v4 +; GFX9-NEXT: v_mad_f32 v4, -v1, v3, v5 ; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s1 -; GFX9-NEXT: s_lshr_b32 s6, s7, 16 ; GFX9-NEXT: s_and_b32 s0, s5, s0 +; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v7, vcc ; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1 ; GFX9-NEXT: v_cvt_f32_u32_e32 v6, s0 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v7, v5 -; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v4 -; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s6 -; GFX9-NEXT: s_lshr_b32 s8, s5, 16 +; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v4|, v3 ; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v1, vcc ; GFX9-NEXT: v_mul_f32_e32 v1, v6, v7 -; GFX9-NEXT: v_cvt_f32_u32_e32 v7, s8 -; GFX9-NEXT: v_rcp_iflag_f32_e32 v8, v4 ; GFX9-NEXT: v_trunc_f32_e32 v1, v1 -; GFX9-NEXT: v_mad_f32 v6, -v1, v5, v6 +; GFX9-NEXT: s_lshr_b32 s0, s7, 16 +; GFX9-NEXT: v_mad_f32 v4, -v1, v5, v6 +; GFX9-NEXT: v_cvt_f32_u32_e32 v6, s0 +; GFX9-NEXT: s_lshr_b32 s0, s5, 16 +; GFX9-NEXT: v_cvt_f32_u32_e32 v7, s0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v6|, v5 -; GFX9-NEXT: v_mul_f32_e32 v5, v7, v8 -; GFX9-NEXT: v_trunc_f32_e32 v5, v5 -; GFX9-NEXT: v_cvt_u32_f32_e32 v6, v5 +; GFX9-NEXT: v_rcp_iflag_f32_e32 v8, v6 +; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v4|, v5 ; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc -; GFX9-NEXT: v_mad_f32 v5, -v5, v4, v7 -; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v5|, v4 +; GFX9-NEXT: v_mul_f32_e32 v4, v7, v8 +; GFX9-NEXT: v_trunc_f32_e32 v4, v4 +; GFX9-NEXT: v_cvt_u32_f32_e32 v5, v4 +; GFX9-NEXT: v_mad_f32 v4, -v4, v6, v7 +; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v4|, v6 +; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v5, vcc ; GFX9-NEXT: v_mov_b32_e32 v5, 0xffff -; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v6, vcc ; GFX9-NEXT: v_and_b32_e32 v1, v5, v1 ; GFX9-NEXT: v_and_b32_e32 v0, v5, v0 ; GFX9-NEXT: v_lshl_or_b32 v1, v4, 16, v1 @@ -2745,65 +2745,69 @@ ; GFX6-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; GFX6-NEXT: s_mov_b32 s8, 0xffff ; GFX6-NEXT: s_mov_b32 s3, 0xf000 -; GFX6-NEXT: s_mov_b32 s2, -1 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_and_b32 s9, s6, s8 -; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s9 -; GFX6-NEXT: s_and_b32 s10, s4, s8 -; GFX6-NEXT: s_lshr_b32 s11, s6, 16 -; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s10 +; GFX6-NEXT: s_and_b32 s2, s6, s8 +; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s2 +; GFX6-NEXT: v_mov_b32_e32 v4, s6 +; GFX6-NEXT: v_alignbit_b32 v4, s7, v4, 16 +; GFX6-NEXT: s_and_b32 s9, s4, s8 +; GFX6-NEXT: v_and_b32_e32 v5, s8, v4 +; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s9 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v2, v0 -; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s11 -; GFX6-NEXT: s_lshr_b32 s9, s4, 16 -; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s9 +; GFX6-NEXT: v_cvt_f32_u32_e32 v5, v5 +; GFX6-NEXT: v_mov_b32_e32 v3, s4 +; GFX6-NEXT: v_alignbit_b32 v3, s5, v3, 16 +; GFX6-NEXT: v_and_b32_e32 v6, s8, v3 ; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2 -; GFX6-NEXT: v_rcp_iflag_f32_e32 v5, v3 +; GFX6-NEXT: v_cvt_f32_u32_e32 v6, v6 +; GFX6-NEXT: v_rcp_iflag_f32_e32 v7, v5 ; GFX6-NEXT: v_trunc_f32_e32 v2, v2 ; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1 ; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v2 ; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 -; GFX6-NEXT: v_mul_f32_e32 v1, v4, v5 +; GFX6-NEXT: v_mul_f32_e32 v1, v6, v7 ; GFX6-NEXT: v_trunc_f32_e32 v1, v1 ; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc ; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v1 -; GFX6-NEXT: v_mad_f32 v1, -v1, v3, v4 +; GFX6-NEXT: v_mad_f32 v1, -v1, v5, v6 ; GFX6-NEXT: v_mul_lo_u32 v0, v0, s6 -; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v3 +; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v5 ; GFX6-NEXT: s_and_b32 s6, s7, s8 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v2, vcc ; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s6 ; GFX6-NEXT: s_and_b32 s6, s5, s8 -; GFX6-NEXT: v_mul_lo_u32 v1, v1, s11 -; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s6 -; GFX6-NEXT: v_rcp_iflag_f32_e32 v4, v2 -; GFX6-NEXT: s_lshr_b32 s12, s7, 16 -; GFX6-NEXT: v_sub_i32_e32 v5, vcc, s9, v1 -; GFX6-NEXT: v_mul_f32_e32 v1, v3, v4 -; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s12 -; GFX6-NEXT: s_lshr_b32 s10, s5, 16 -; GFX6-NEXT: v_cvt_f32_u32_e32 v6, s10 -; GFX6-NEXT: v_trunc_f32_e32 v1, v1 -; GFX6-NEXT: v_rcp_iflag_f32_e32 v7, v4 +; GFX6-NEXT: v_mul_lo_u32 v1, v1, v4 +; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s6 +; GFX6-NEXT: v_rcp_iflag_f32_e32 v5, v2 ; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s4, v0 -; GFX6-NEXT: v_mad_f32 v3, -v1, v2, v3 +; GFX6-NEXT: s_lshr_b32 s4, s7, 16 +; GFX6-NEXT: v_sub_i32_e32 v3, vcc, v3, v1 +; GFX6-NEXT: v_mul_f32_e32 v1, v4, v5 +; GFX6-NEXT: v_cvt_f32_u32_e32 v5, s4 +; GFX6-NEXT: s_lshr_b32 s6, s5, 16 +; GFX6-NEXT: v_cvt_f32_u32_e32 v6, s6 +; GFX6-NEXT: v_trunc_f32_e32 v1, v1 +; GFX6-NEXT: v_rcp_iflag_f32_e32 v7, v5 +; GFX6-NEXT: v_mad_f32 v4, -v1, v2, v4 ; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2 +; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v4|, v2 ; GFX6-NEXT: v_mul_f32_e32 v2, v6, v7 ; GFX6-NEXT: v_trunc_f32_e32 v2, v2 -; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v2 +; GFX6-NEXT: v_cvt_u32_f32_e32 v4, v2 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc -; GFX6-NEXT: v_mad_f32 v2, -v2, v4, v6 -; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v4 -; GFX6-NEXT: v_addc_u32_e32 v2, vcc, 0, v3, vcc +; GFX6-NEXT: v_mad_f32 v2, -v2, v5, v6 +; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v5 +; GFX6-NEXT: v_addc_u32_e32 v2, vcc, 0, v4, vcc ; GFX6-NEXT: v_mul_lo_u32 v1, v1, s7 -; GFX6-NEXT: v_mul_lo_u32 v2, v2, s12 +; GFX6-NEXT: v_mul_lo_u32 v2, v2, s4 ; GFX6-NEXT: v_and_b32_e32 v0, s8, v0 +; GFX6-NEXT: s_mov_b32 s2, -1 ; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s5, v1 -; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s10, v2 +; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s6, v2 ; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX6-NEXT: v_and_b32_e32 v1, s8, v1 ; GFX6-NEXT: v_or_b32_e32 v1, v1, v2 -; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v5 +; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3 ; GFX6-NEXT: v_or_b32_e32 v0, v0, v2 ; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 ; GFX6-NEXT: s_endpgm @@ -2817,56 +2821,56 @@ ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: s_and_b32 s8, s6, s0 ; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s8 -; GFX9-NEXT: s_and_b32 s9, s4, s0 -; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s9 -; GFX9-NEXT: s_lshr_b32 s9, s6, 16 -; GFX9-NEXT: v_rcp_iflag_f32_e32 v3, v0 -; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s9 -; GFX9-NEXT: s_lshr_b32 s1, s4, 16 -; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s1 -; GFX9-NEXT: v_mul_f32_e32 v3, v1, v3 -; GFX9-NEXT: v_trunc_f32_e32 v3, v3 -; GFX9-NEXT: v_mad_f32 v1, -v3, v0, v1 -; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GFX9-NEXT: v_rcp_iflag_f32_e32 v6, v4 +; GFX9-NEXT: s_and_b32 s1, s4, s0 +; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s1 +; GFX9-NEXT: s_lshr_b32 s6, s6, 16 +; GFX9-NEXT: v_rcp_iflag_f32_e32 v4, v0 +; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s6 +; GFX9-NEXT: s_lshr_b32 s4, s4, 16 +; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s4 +; GFX9-NEXT: v_mul_f32_e32 v4, v1, v4 +; GFX9-NEXT: v_trunc_f32_e32 v4, v4 +; GFX9-NEXT: v_rcp_iflag_f32_e32 v6, v3 +; GFX9-NEXT: v_cvt_u32_f32_e32 v7, v4 +; GFX9-NEXT: v_mad_f32 v1, -v4, v0, v1 ; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 -; GFX9-NEXT: s_lshr_b32 s10, s7, 16 -; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v3, vcc ; GFX9-NEXT: v_mul_f32_e32 v1, v5, v6 -; GFX9-NEXT: v_mul_lo_u32 v0, v0, s6 +; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v7, vcc +; GFX9-NEXT: v_mul_lo_u32 v0, v0, s8 ; GFX9-NEXT: v_trunc_f32_e32 v1, v1 -; GFX9-NEXT: s_and_b32 s6, s7, s0 -; GFX9-NEXT: v_mad_f32 v3, -v1, v4, v5 -; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s6 +; GFX9-NEXT: s_and_b32 s8, s7, s0 +; GFX9-NEXT: v_mad_f32 v4, -v1, v3, v5 +; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s8 ; GFX9-NEXT: s_and_b32 s0, s5, s0 +; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1 ; GFX9-NEXT: v_cvt_f32_u32_e32 v6, s0 -; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v4 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v7, v5 -; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s10 -; GFX9-NEXT: s_lshr_b32 s8, s5, 16 -; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v4|, v3 +; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GFX9-NEXT: v_mul_f32_e32 v3, v6, v7 -; GFX9-NEXT: v_cvt_f32_u32_e32 v7, s8 -; GFX9-NEXT: v_rcp_iflag_f32_e32 v8, v4 +; GFX9-NEXT: v_mul_lo_u32 v1, v1, s6 ; GFX9-NEXT: v_trunc_f32_e32 v3, v3 -; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc -; GFX9-NEXT: v_mad_f32 v6, -v3, v5, v6 +; GFX9-NEXT: s_lshr_b32 s6, s7, 16 +; GFX9-NEXT: v_mad_f32 v4, -v3, v5, v6 +; GFX9-NEXT: v_cvt_f32_u32_e32 v6, s6 +; GFX9-NEXT: s_lshr_b32 s5, s5, 16 +; GFX9-NEXT: v_cvt_f32_u32_e32 v7, s5 ; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v6|, v5 -; GFX9-NEXT: v_mul_f32_e32 v5, v7, v8 -; GFX9-NEXT: v_trunc_f32_e32 v5, v5 -; GFX9-NEXT: v_cvt_u32_f32_e32 v6, v5 +; GFX9-NEXT: v_rcp_iflag_f32_e32 v8, v6 +; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v4|, v5 +; GFX9-NEXT: v_sub_u32_e32 v0, s1, v0 ; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc -; GFX9-NEXT: v_mad_f32 v5, -v5, v4, v7 -; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v5|, v4 -; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v6, vcc -; GFX9-NEXT: v_mul_lo_u32 v1, v1, s9 -; GFX9-NEXT: v_mul_lo_u32 v3, v3, s7 -; GFX9-NEXT: v_mul_lo_u32 v4, v4, s10 -; GFX9-NEXT: v_sub_u32_e32 v0, s4, v0 -; GFX9-NEXT: v_sub_u32_e32 v5, s1, v1 -; GFX9-NEXT: v_sub_u32_e32 v1, s5, v3 -; GFX9-NEXT: v_sub_u32_e32 v3, s8, v4 +; GFX9-NEXT: v_mul_f32_e32 v4, v7, v8 +; GFX9-NEXT: v_trunc_f32_e32 v4, v4 +; GFX9-NEXT: v_cvt_u32_f32_e32 v5, v4 +; GFX9-NEXT: v_mad_f32 v4, -v4, v6, v7 +; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v4|, v6 +; GFX9-NEXT: v_mul_lo_u32 v3, v3, s8 +; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v5, vcc +; GFX9-NEXT: v_mul_lo_u32 v4, v4, s6 +; GFX9-NEXT: v_sub_u32_e32 v5, s4, v1 +; GFX9-NEXT: v_sub_u32_e32 v1, s0, v3 +; GFX9-NEXT: v_sub_u32_e32 v3, s5, v4 ; GFX9-NEXT: v_mov_b32_e32 v4, 0xffff ; GFX9-NEXT: v_and_b32_e32 v1, v4, v1 ; GFX9-NEXT: v_and_b32_e32 v0, v4, v0 @@ -3274,70 +3278,75 @@ ; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2 ; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, |v0| ; GFX6-NEXT: v_cndmask_b32_e32 v0, 0, v3, vcc +; GFX6-NEXT: v_mov_b32_e32 v1, s4 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GFX6-NEXT: v_mov_b32_e32 v2, s6 +; GFX6-NEXT: v_alignbit_b32 v2, s7, v2, 16 +; GFX6-NEXT: v_bfe_i32 v3, v2, 0, 16 +; GFX6-NEXT: v_cvt_f32_i32_e32 v4, v3 +; GFX6-NEXT: v_alignbit_b32 v1, s5, v1, 16 +; GFX6-NEXT: v_bfe_i32 v5, v1, 0, 16 +; GFX6-NEXT: v_cvt_f32_i32_e32 v6, v5 +; GFX6-NEXT: v_rcp_iflag_f32_e32 v7, v4 ; GFX6-NEXT: v_mul_lo_u32 v0, v0, s6 -; GFX6-NEXT: s_ashr_i32 s6, s6, 16 -; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s6 +; GFX6-NEXT: v_xor_b32_e32 v3, v5, v3 +; GFX6-NEXT: v_ashrrev_i32_e32 v3, 30, v3 +; GFX6-NEXT: v_mul_f32_e32 v5, v6, v7 +; GFX6-NEXT: v_trunc_f32_e32 v5, v5 +; GFX6-NEXT: v_mad_f32 v6, -v5, v4, v6 +; GFX6-NEXT: v_cvt_i32_f32_e32 v5, v5 ; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s4, v0 -; GFX6-NEXT: s_ashr_i32 s4, s4, 16 -; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s4 -; GFX6-NEXT: v_rcp_iflag_f32_e32 v3, v1 -; GFX6-NEXT: s_xor_b32 s8, s4, s6 -; GFX6-NEXT: s_ashr_i32 s8, s8, 30 -; GFX6-NEXT: s_or_b32 s8, s8, 1 -; GFX6-NEXT: v_mul_f32_e32 v3, v2, v3 -; GFX6-NEXT: v_trunc_f32_e32 v3, v3 -; GFX6-NEXT: v_mad_f32 v2, -v3, v1, v2 -; GFX6-NEXT: v_cvt_i32_f32_e32 v3, v3 -; GFX6-NEXT: v_mov_b32_e32 v4, s8 -; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, |v1| -; GFX6-NEXT: v_cndmask_b32_e32 v1, 0, v4, vcc -; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v3 -; GFX6-NEXT: v_mul_lo_u32 v1, v1, s6 -; GFX6-NEXT: s_sext_i32_i16 s6, s7 +; GFX6-NEXT: v_or_b32_e32 v3, 1, v3 +; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v6|, |v4| +; GFX6-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v5 +; GFX6-NEXT: s_sext_i32_i16 s4, s7 +; GFX6-NEXT: v_mul_lo_u32 v2, v3, v2 +; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s4 +; GFX6-NEXT: s_sext_i32_i16 s6, s5 +; GFX6-NEXT: s_xor_b32 s4, s6, s4 +; GFX6-NEXT: v_sub_i32_e32 v1, vcc, v1, v2 ; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s6 -; GFX6-NEXT: v_sub_i32_e32 v3, vcc, s4, v1 -; GFX6-NEXT: s_sext_i32_i16 s4, s5 -; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4 -; GFX6-NEXT: v_rcp_iflag_f32_e32 v4, v2 -; GFX6-NEXT: s_xor_b32 s4, s4, s6 +; GFX6-NEXT: v_rcp_iflag_f32_e32 v4, v3 ; GFX6-NEXT: s_ashr_i32 s4, s4, 30 ; GFX6-NEXT: s_or_b32 s4, s4, 1 -; GFX6-NEXT: v_mul_f32_e32 v4, v1, v4 -; GFX6-NEXT: v_trunc_f32_e32 v4, v4 -; GFX6-NEXT: v_mad_f32 v1, -v4, v2, v1 ; GFX6-NEXT: v_mov_b32_e32 v5, s4 -; GFX6-NEXT: s_ashr_i32 s4, s7, 16 +; GFX6-NEXT: v_mul_f32_e32 v4, v2, v4 +; GFX6-NEXT: v_trunc_f32_e32 v4, v4 +; GFX6-NEXT: v_mad_f32 v2, -v4, v3, v2 ; GFX6-NEXT: v_cvt_i32_f32_e32 v4, v4 -; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, |v2| -; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s4 -; GFX6-NEXT: v_cndmask_b32_e32 v1, 0, v5, vcc -; GFX6-NEXT: s_ashr_i32 s6, s5, 16 -; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v4 -; GFX6-NEXT: v_cvt_f32_i32_e32 v4, s6 -; GFX6-NEXT: v_rcp_iflag_f32_e32 v5, v2 -; GFX6-NEXT: v_mul_lo_u32 v1, v1, s7 -; GFX6-NEXT: s_xor_b32 s7, s6, s4 -; GFX6-NEXT: s_ashr_i32 s7, s7, 30 +; GFX6-NEXT: s_ashr_i32 s4, s7, 16 +; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, |v3| +; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s4 +; GFX6-NEXT: v_cndmask_b32_e32 v2, 0, v5, vcc +; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v4 +; GFX6-NEXT: v_mul_lo_u32 v2, v2, s7 +; GFX6-NEXT: s_lshr_b32 s6, s7, 16 +; GFX6-NEXT: s_ashr_i32 s7, s5, 16 +; GFX6-NEXT: v_cvt_f32_i32_e32 v4, s7 +; GFX6-NEXT: v_rcp_iflag_f32_e32 v5, v3 +; GFX6-NEXT: s_xor_b32 s4, s7, s4 +; GFX6-NEXT: s_ashr_i32 s4, s4, 30 +; GFX6-NEXT: s_or_b32 s4, s4, 1 ; GFX6-NEXT: v_mul_f32_e32 v5, v4, v5 ; GFX6-NEXT: v_trunc_f32_e32 v5, v5 -; GFX6-NEXT: v_mad_f32 v4, -v5, v2, v4 +; GFX6-NEXT: v_mad_f32 v4, -v5, v3, v4 ; GFX6-NEXT: v_cvt_i32_f32_e32 v5, v5 -; GFX6-NEXT: s_or_b32 s7, s7, 1 -; GFX6-NEXT: v_mov_b32_e32 v6, s7 -; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v4|, |v2| -; GFX6-NEXT: v_cndmask_b32_e32 v2, 0, v6, vcc -; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v5 -; GFX6-NEXT: v_mul_lo_u32 v2, v2, s4 -; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s5, v1 +; GFX6-NEXT: v_mov_b32_e32 v6, s4 +; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v4|, |v3| +; GFX6-NEXT: v_cndmask_b32_e32 v3, 0, v6, vcc +; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v5 +; GFX6-NEXT: v_mul_lo_u32 v3, v3, s6 +; GFX6-NEXT: s_lshr_b32 s4, s5, 16 +; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s5, v2 +; GFX6-NEXT: v_sub_i32_e32 v3, vcc, s4, v3 ; GFX6-NEXT: s_mov_b32 s4, 0xffff -; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s6, v2 -; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX6-NEXT: v_and_b32_e32 v1, s4, v1 -; GFX6-NEXT: v_or_b32_e32 v1, v1, v2 -; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3 +; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; GFX6-NEXT: v_and_b32_e32 v0, s4, v0 -; GFX6-NEXT: v_or_b32_e32 v0, v0, v2 +; GFX6-NEXT: v_or_b32_e32 v0, v0, v1 +; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v3 +; GFX6-NEXT: v_and_b32_e32 v2, s4, v2 +; GFX6-NEXT: v_or_b32_e32 v1, v2, v1 ; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 ; GFX6-NEXT: s_endpgm ; @@ -3347,78 +3356,78 @@ ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 ; GFX9-NEXT: v_mov_b32_e32 v2, 0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_sext_i32_i16 s0, s6 -; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0 -; GFX9-NEXT: s_sext_i32_i16 s1, s4 -; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s1 -; GFX9-NEXT: s_xor_b32 s0, s1, s0 +; GFX9-NEXT: s_sext_i32_i16 s8, s6 +; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s8 +; GFX9-NEXT: s_sext_i32_i16 s9, s4 +; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s9 +; GFX9-NEXT: s_xor_b32 s0, s9, s8 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v3, v0 ; GFX9-NEXT: s_ashr_i32 s0, s0, 30 -; GFX9-NEXT: s_or_b32 s8, s0, 1 +; GFX9-NEXT: s_or_b32 s10, s0, 1 ; GFX9-NEXT: v_mul_f32_e32 v3, v1, v3 ; GFX9-NEXT: v_trunc_f32_e32 v3, v3 ; GFX9-NEXT: v_mad_f32 v1, -v3, v0, v1 ; GFX9-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0| ; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec -; GFX9-NEXT: s_cselect_b32 s0, s8, 0 -; GFX9-NEXT: s_ashr_i32 s9, s6, 16 +; GFX9-NEXT: s_cselect_b32 s0, s10, 0 +; GFX9-NEXT: s_ashr_i32 s6, s6, 16 ; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3 -; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s9 -; GFX9-NEXT: s_ashr_i32 s8, s4, 16 +; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s6 +; GFX9-NEXT: s_ashr_i32 s4, s4, 16 ; GFX9-NEXT: v_add_u32_e32 v1, s0, v3 -; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s8 +; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s4 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v4, v0 -; GFX9-NEXT: s_xor_b32 s0, s8, s9 +; GFX9-NEXT: s_xor_b32 s0, s4, s6 ; GFX9-NEXT: s_ashr_i32 s0, s0, 30 -; GFX9-NEXT: v_mul_lo_u32 v1, v1, s6 +; GFX9-NEXT: v_mul_lo_u32 v1, v1, s8 ; GFX9-NEXT: v_mul_f32_e32 v4, v3, v4 ; GFX9-NEXT: v_trunc_f32_e32 v4, v4 ; GFX9-NEXT: v_mad_f32 v3, -v4, v0, v3 -; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4 -; GFX9-NEXT: s_or_b32 s6, s0, 1 +; GFX9-NEXT: s_or_b32 s8, s0, 1 ; GFX9-NEXT: v_cmp_ge_f32_e64 s[0:1], |v3|, |v0| +; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4 ; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec -; GFX9-NEXT: s_cselect_b32 s0, s6, 0 +; GFX9-NEXT: s_cselect_b32 s0, s8, 0 +; GFX9-NEXT: s_sext_i32_i16 s8, s7 +; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s8 ; GFX9-NEXT: v_add_u32_e32 v0, s0, v4 -; GFX9-NEXT: s_sext_i32_i16 s0, s7 -; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s0 -; GFX9-NEXT: s_sext_i32_i16 s1, s5 -; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s1 -; GFX9-NEXT: s_xor_b32 s0, s1, s0 +; GFX9-NEXT: v_mul_lo_u32 v0, v0, s6 +; GFX9-NEXT: s_sext_i32_i16 s6, s5 +; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s6 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v5, v3 +; GFX9-NEXT: s_xor_b32 s0, s6, s8 ; GFX9-NEXT: s_ashr_i32 s0, s0, 30 -; GFX9-NEXT: s_or_b32 s6, s0, 1 -; GFX9-NEXT: v_mul_lo_u32 v0, v0, s9 +; GFX9-NEXT: s_or_b32 s10, s0, 1 ; GFX9-NEXT: v_mul_f32_e32 v5, v4, v5 ; GFX9-NEXT: v_trunc_f32_e32 v5, v5 ; GFX9-NEXT: v_mad_f32 v4, -v5, v3, v4 ; GFX9-NEXT: v_cmp_ge_f32_e64 s[0:1], |v4|, |v3| -; GFX9-NEXT: v_cvt_i32_f32_e32 v5, v5 ; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec -; GFX9-NEXT: s_cselect_b32 s0, s6, 0 -; GFX9-NEXT: s_ashr_i32 s6, s7, 16 -; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s6 +; GFX9-NEXT: s_cselect_b32 s0, s10, 0 +; GFX9-NEXT: s_ashr_i32 s7, s7, 16 +; GFX9-NEXT: v_cvt_i32_f32_e32 v5, v5 +; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s7 +; GFX9-NEXT: s_ashr_i32 s5, s5, 16 +; GFX9-NEXT: v_sub_u32_e32 v0, s4, v0 ; GFX9-NEXT: v_add_u32_e32 v3, s0, v5 -; GFX9-NEXT: v_mul_lo_u32 v3, v3, s7 -; GFX9-NEXT: s_ashr_i32 s7, s5, 16 -; GFX9-NEXT: v_cvt_f32_i32_e32 v5, s7 +; GFX9-NEXT: v_cvt_f32_i32_e32 v5, s5 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v6, v4 -; GFX9-NEXT: s_xor_b32 s0, s7, s6 +; GFX9-NEXT: s_xor_b32 s0, s5, s7 ; GFX9-NEXT: s_ashr_i32 s0, s0, 30 -; GFX9-NEXT: s_or_b32 s9, s0, 1 +; GFX9-NEXT: v_mul_lo_u32 v3, v3, s8 ; GFX9-NEXT: v_mul_f32_e32 v6, v5, v6 ; GFX9-NEXT: v_trunc_f32_e32 v6, v6 ; GFX9-NEXT: v_mad_f32 v5, -v6, v4, v5 ; GFX9-NEXT: v_cvt_i32_f32_e32 v6, v6 +; GFX9-NEXT: s_or_b32 s8, s0, 1 ; GFX9-NEXT: v_cmp_ge_f32_e64 s[0:1], |v5|, |v4| ; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec -; GFX9-NEXT: s_cselect_b32 s0, s9, 0 +; GFX9-NEXT: s_cselect_b32 s0, s8, 0 ; GFX9-NEXT: v_add_u32_e32 v4, s0, v6 -; GFX9-NEXT: v_mul_lo_u32 v4, v4, s6 -; GFX9-NEXT: v_sub_u32_e32 v5, s4, v1 -; GFX9-NEXT: v_sub_u32_e32 v1, s5, v3 -; GFX9-NEXT: v_sub_u32_e32 v0, s8, v0 -; GFX9-NEXT: v_sub_u32_e32 v3, s7, v4 +; GFX9-NEXT: v_mul_lo_u32 v4, v4, s7 +; GFX9-NEXT: v_sub_u32_e32 v5, s9, v1 +; GFX9-NEXT: v_sub_u32_e32 v1, s6, v3 +; GFX9-NEXT: v_sub_u32_e32 v3, s5, v4 ; GFX9-NEXT: v_mov_b32_e32 v4, 0xffff ; GFX9-NEXT: v_and_b32_e32 v1, v4, v1 ; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1 @@ -3820,21 +3829,20 @@ ; ; GFX6-LABEL: udiv_v3i16: ; GFX6: ; %bb.0: -; GFX6-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0xd -; GFX6-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 -; GFX6-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb +; GFX6-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xb +; GFX6-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; GFX6-NEXT: s_mov_b32 s8, 0xffff -; GFX6-NEXT: s_mov_b32 s7, 0xf000 +; GFX6-NEXT: s_mov_b32 s3, 0xf000 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_and_b32 s6, s2, s8 -; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s6 -; GFX6-NEXT: s_and_b32 s6, s0, s8 -; GFX6-NEXT: s_lshr_b32 s2, s2, 16 -; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s6 +; GFX6-NEXT: s_and_b32 s2, s6, s8 +; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s2 +; GFX6-NEXT: s_and_b32 s9, s4, s8 +; GFX6-NEXT: s_lshr_b32 s6, s6, 16 +; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s9 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v2, v0 -; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s2 -; GFX6-NEXT: s_lshr_b32 s0, s0, 16 -; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s0 +; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s6 +; GFX6-NEXT: s_lshr_b32 s4, s4, 16 +; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4 ; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v5, v3 ; GFX6-NEXT: v_trunc_f32_e32 v2, v2 @@ -3843,16 +3851,16 @@ ; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 ; GFX6-NEXT: v_mul_f32_e32 v1, v4, v5 ; GFX6-NEXT: v_trunc_f32_e32 v1, v1 -; GFX6-NEXT: s_and_b32 s0, s3, s8 +; GFX6-NEXT: s_and_b32 s4, s7, s8 ; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc ; GFX6-NEXT: v_mad_f32 v2, -v1, v3, v4 -; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s0 -; GFX6-NEXT: s_and_b32 s0, s1, s8 -; GFX6-NEXT: v_cvt_f32_u32_e32 v5, s0 +; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4 +; GFX6-NEXT: s_and_b32 s4, s5, s8 +; GFX6-NEXT: v_cvt_f32_u32_e32 v5, s4 ; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v6, v4 ; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v3 -; GFX6-NEXT: s_mov_b32 s6, -1 +; GFX6-NEXT: s_mov_b32 s2, -1 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; GFX6-NEXT: v_mul_f32_e32 v2, v5, v6 ; GFX6-NEXT: v_trunc_f32_e32 v2, v2 @@ -3863,55 +3871,54 @@ ; GFX6-NEXT: v_addc_u32_e32 v2, vcc, 0, v3, vcc ; GFX6-NEXT: v_and_b32_e32 v0, s8, v0 ; GFX6-NEXT: v_or_b32_e32 v0, v0, v1 -; GFX6-NEXT: buffer_store_short v2, off, s[4:7], 0 offset:4 -; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0 +; GFX6-NEXT: buffer_store_short v2, off, s[0:3], 0 offset:4 +; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX6-NEXT: s_endpgm ; ; GFX9-LABEL: udiv_v3i16: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 -; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24 -; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x2c +; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x2c +; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 ; GFX9-NEXT: s_mov_b32 s0, 0xffff -; GFX9-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_and_b32 s1, s2, s0 -; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s1 -; GFX9-NEXT: s_and_b32 s1, s6, s0 -; GFX9-NEXT: s_lshr_b32 s2, s2, 16 +; GFX9-NEXT: s_and_b32 s8, s6, s0 +; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s8 +; GFX9-NEXT: s_lshr_b32 s6, s6, 16 +; GFX9-NEXT: s_and_b32 s1, s4, s0 +; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s6 ; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s1 -; GFX9-NEXT: v_rcp_iflag_f32_e32 v3, v0 -; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s2 -; GFX9-NEXT: s_lshr_b32 s1, s6, 16 +; GFX9-NEXT: v_rcp_iflag_f32_e32 v4, v1 +; GFX9-NEXT: s_lshr_b32 s1, s4, 16 ; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s1 -; GFX9-NEXT: v_mul_f32_e32 v3, v2, v3 -; GFX9-NEXT: v_rcp_iflag_f32_e32 v6, v4 -; GFX9-NEXT: v_trunc_f32_e32 v3, v3 -; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2 -; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v3 -; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v0 +; GFX9-NEXT: v_rcp_iflag_f32_e32 v6, v3 +; GFX9-NEXT: v_mul_f32_e32 v4, v2, v4 +; GFX9-NEXT: v_trunc_f32_e32 v4, v4 +; GFX9-NEXT: v_mad_f32 v2, -v4, v1, v2 +; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v1 ; GFX9-NEXT: v_mul_f32_e32 v2, v5, v6 ; GFX9-NEXT: v_trunc_f32_e32 v2, v2 -; GFX9-NEXT: s_and_b32 s1, s3, s0 -; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v3, vcc -; GFX9-NEXT: v_mad_f32 v3, -v2, v4, v5 +; GFX9-NEXT: s_and_b32 s1, s7, s0 +; GFX9-NEXT: v_cvt_u32_f32_e32 v7, v4 +; GFX9-NEXT: v_mad_f32 v4, -v2, v3, v5 ; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s1 -; GFX9-NEXT: s_and_b32 s0, s7, s0 +; GFX9-NEXT: s_and_b32 s0, s5, s0 +; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v7, vcc ; GFX9-NEXT: v_cvt_f32_u32_e32 v6, s0 -; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v7, v5 -; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v4 -; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc +; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2 +; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v4|, v3 +; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1 ; GFX9-NEXT: v_mul_f32_e32 v3, v6, v7 ; GFX9-NEXT: v_trunc_f32_e32 v3, v3 ; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v3 +; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc ; GFX9-NEXT: v_mad_f32 v3, -v3, v5, v6 ; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v5 -; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0 ; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v4, vcc -; GFX9-NEXT: global_store_short v1, v3, s[4:5] offset:4 -; GFX9-NEXT: global_store_dword v1, v0, s[4:5] +; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1 +; GFX9-NEXT: global_store_short v0, v3, s[2:3] offset:4 +; GFX9-NEXT: global_store_dword v0, v1, s[2:3] ; GFX9-NEXT: s_endpgm %r = udiv <3 x i16> %x, %y store <3 x i16> %r, <3 x i16> addrspace(1)* %out @@ -3991,114 +3998,112 @@ ; ; GFX6-LABEL: urem_v3i16: ; GFX6: ; %bb.0: -; GFX6-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0xd -; GFX6-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 -; GFX6-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb +; GFX6-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xb +; GFX6-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; GFX6-NEXT: s_mov_b32 s8, 0xffff -; GFX6-NEXT: s_mov_b32 s7, 0xf000 +; GFX6-NEXT: s_mov_b32 s3, 0xf000 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_and_b32 s6, s2, s8 -; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s6 -; GFX6-NEXT: s_and_b32 s6, s0, s8 -; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s6 -; GFX6-NEXT: v_mov_b32_e32 v4, s2 -; GFX6-NEXT: v_rcp_iflag_f32_e32 v3, v0 -; GFX6-NEXT: v_alignbit_b32 v4, s3, v4, 16 -; GFX6-NEXT: v_and_b32_e32 v5, s8, v4 -; GFX6-NEXT: v_mov_b32_e32 v1, s0 -; GFX6-NEXT: v_mul_f32_e32 v3, v2, v3 -; GFX6-NEXT: v_trunc_f32_e32 v3, v3 -; GFX6-NEXT: v_cvt_u32_f32_e32 v6, v3 -; GFX6-NEXT: v_mad_f32 v2, -v3, v0, v2 -; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v0 -; GFX6-NEXT: v_cvt_f32_u32_e32 v2, v5 +; GFX6-NEXT: s_and_b32 s9, s6, s8 +; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s9 +; GFX6-NEXT: s_and_b32 s2, s4, s8 +; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s2 +; GFX6-NEXT: v_mov_b32_e32 v2, s6 +; GFX6-NEXT: v_rcp_iflag_f32_e32 v4, v0 +; GFX6-NEXT: v_alignbit_b32 v2, s7, v2, 16 +; GFX6-NEXT: v_and_b32_e32 v5, s8, v2 +; GFX6-NEXT: v_cvt_f32_u32_e32 v5, v5 +; GFX6-NEXT: v_mul_f32_e32 v4, v3, v4 +; GFX6-NEXT: v_trunc_f32_e32 v4, v4 +; GFX6-NEXT: v_cvt_u32_f32_e32 v6, v4 +; GFX6-NEXT: v_mad_f32 v3, -v4, v0, v3 +; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v0 +; GFX6-NEXT: v_mov_b32_e32 v1, s4 ; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v6, vcc -; GFX6-NEXT: v_mul_lo_u32 v0, v0, s2 -; GFX6-NEXT: v_alignbit_b32 v1, s1, v1, 16 +; GFX6-NEXT: v_mul_lo_u32 v0, v0, s6 +; GFX6-NEXT: v_alignbit_b32 v1, s5, v1, 16 ; GFX6-NEXT: v_and_b32_e32 v3, s8, v1 ; GFX6-NEXT: v_cvt_f32_u32_e32 v3, v3 -; GFX6-NEXT: v_rcp_iflag_f32_e32 v5, v2 -; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s0, v0 -; GFX6-NEXT: s_and_b32 s0, s3, s8 -; GFX6-NEXT: v_cvt_f32_u32_e32 v6, s0 -; GFX6-NEXT: v_mul_f32_e32 v5, v3, v5 -; GFX6-NEXT: s_and_b32 s0, s1, s8 -; GFX6-NEXT: v_trunc_f32_e32 v5, v5 -; GFX6-NEXT: v_cvt_f32_u32_e32 v7, s0 +; GFX6-NEXT: v_rcp_iflag_f32_e32 v4, v5 +; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s4, v0 +; GFX6-NEXT: s_and_b32 s4, s7, s8 +; GFX6-NEXT: v_cvt_f32_u32_e32 v6, s4 +; GFX6-NEXT: v_mul_f32_e32 v4, v3, v4 +; GFX6-NEXT: v_trunc_f32_e32 v4, v4 +; GFX6-NEXT: v_mad_f32 v3, -v4, v5, v3 +; GFX6-NEXT: v_cvt_u32_f32_e32 v4, v4 +; GFX6-NEXT: s_and_b32 s4, s5, s8 +; GFX6-NEXT: v_cvt_f32_u32_e32 v7, s4 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v8, v6 -; GFX6-NEXT: v_mad_f32 v3, -v5, v2, v3 -; GFX6-NEXT: v_cvt_u32_f32_e32 v5, v5 -; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2 +; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v5 +; GFX6-NEXT: v_addc_u32_e32 v3, vcc, 0, v4, vcc +; GFX6-NEXT: v_mul_lo_u32 v2, v3, v2 ; GFX6-NEXT: v_mul_f32_e32 v3, v7, v8 ; GFX6-NEXT: v_trunc_f32_e32 v3, v3 -; GFX6-NEXT: v_addc_u32_e32 v2, vcc, 0, v5, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, v2, v4 ; GFX6-NEXT: v_cvt_u32_f32_e32 v4, v3 ; GFX6-NEXT: v_mad_f32 v3, -v3, v6, v7 ; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v6 -; GFX6-NEXT: s_mov_b32 s6, -1 +; GFX6-NEXT: s_mov_b32 s2, -1 ; GFX6-NEXT: v_addc_u32_e32 v3, vcc, 0, v4, vcc -; GFX6-NEXT: v_mul_lo_u32 v3, v3, s3 +; GFX6-NEXT: v_mul_lo_u32 v3, v3, s7 ; GFX6-NEXT: v_sub_i32_e32 v1, vcc, v1, v2 ; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s1, v3 +; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s5, v3 ; GFX6-NEXT: v_and_b32_e32 v0, s8, v0 ; GFX6-NEXT: v_or_b32_e32 v0, v0, v1 -; GFX6-NEXT: buffer_store_short v2, off, s[4:7], 0 offset:4 -; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0 +; GFX6-NEXT: buffer_store_short v2, off, s[0:3], 0 offset:4 +; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX6-NEXT: s_endpgm ; ; GFX9-LABEL: urem_v3i16: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2c -; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 -; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x24 -; GFX9-NEXT: s_mov_b32 s0, 0xffff +; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x2c +; GFX9-NEXT: s_mov_b32 s2, 0xffff +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_and_b32 s1, s2, s0 -; GFX9-NEXT: s_and_b32 s8, s4, s0 +; GFX9-NEXT: s_and_b32 s8, s6, s2 ; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s8 +; GFX9-NEXT: s_and_b32 s3, s4, s2 +; GFX9-NEXT: s_lshr_b32 s6, s6, 16 +; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s6 +; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s3 +; GFX9-NEXT: v_rcp_iflag_f32_e32 v4, v0 ; GFX9-NEXT: s_lshr_b32 s4, s4, 16 -; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s4 -; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s1 -; GFX9-NEXT: v_rcp_iflag_f32_e32 v3, v0 -; GFX9-NEXT: s_lshr_b32 s2, s2, 16 -; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s2 -; GFX9-NEXT: v_rcp_iflag_f32_e32 v5, v2 -; GFX9-NEXT: v_mul_f32_e32 v3, v1, v3 -; GFX9-NEXT: v_trunc_f32_e32 v3, v3 -; GFX9-NEXT: v_mad_f32 v1, -v3, v0, v1 -; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 -; GFX9-NEXT: v_mul_f32_e32 v1, v4, v5 -; GFX9-NEXT: v_trunc_f32_e32 v1, v1 -; GFX9-NEXT: s_and_b32 s5, s5, s0 -; GFX9-NEXT: v_cvt_u32_f32_e32 v6, v3 -; GFX9-NEXT: v_mad_f32 v3, -v1, v2, v4 -; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s5 -; GFX9-NEXT: s_and_b32 s0, s3, s0 +; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s4 +; GFX9-NEXT: v_rcp_iflag_f32_e32 v5, v1 +; GFX9-NEXT: v_mul_f32_e32 v4, v2, v4 +; GFX9-NEXT: v_trunc_f32_e32 v4, v4 +; GFX9-NEXT: s_and_b32 s7, s7, s2 +; GFX9-NEXT: v_cvt_u32_f32_e32 v6, v4 +; GFX9-NEXT: v_mad_f32 v2, -v4, v0, v2 +; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s7 +; GFX9-NEXT: v_mul_f32_e32 v5, v3, v5 +; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v0 +; GFX9-NEXT: v_trunc_f32_e32 v2, v5 +; GFX9-NEXT: s_and_b32 s2, s5, s2 ; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v6, vcc -; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s0 +; GFX9-NEXT: v_mad_f32 v3, -v2, v1, v3 +; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2 +; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s2 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v6, v4 -; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2 -; GFX9-NEXT: v_mul_lo_u32 v0, v0, s8 +; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v1 +; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v2, vcc ; GFX9-NEXT: v_mul_f32_e32 v2, v5, v6 ; GFX9-NEXT: v_trunc_f32_e32 v2, v2 ; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v2 -; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc ; GFX9-NEXT: v_mad_f32 v2, -v2, v4, v5 ; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v4 +; GFX9-NEXT: v_mul_lo_u32 v0, v0, s8 ; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v3, vcc -; GFX9-NEXT: v_mul_lo_u32 v1, v1, s4 -; GFX9-NEXT: v_mul_lo_u32 v2, v2, s5 -; GFX9-NEXT: v_sub_u32_e32 v0, s1, v0 +; GFX9-NEXT: v_mul_lo_u32 v1, v1, s6 +; GFX9-NEXT: v_mul_lo_u32 v2, v2, s7 +; GFX9-NEXT: v_sub_u32_e32 v0, s3, v0 ; GFX9-NEXT: v_mov_b32_e32 v3, 0 -; GFX9-NEXT: v_sub_u32_e32 v1, s2, v1 -; GFX9-NEXT: v_sub_u32_e32 v2, s0, v2 +; GFX9-NEXT: v_sub_u32_e32 v1, s4, v1 +; GFX9-NEXT: v_sub_u32_e32 v2, s2, v2 ; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0 ; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0 -; GFX9-NEXT: global_store_short v3, v2, s[6:7] offset:4 -; GFX9-NEXT: global_store_dword v3, v0, s[6:7] +; GFX9-NEXT: global_store_short v3, v2, s[0:1] offset:4 +; GFX9-NEXT: global_store_dword v3, v0, s[0:1] ; GFX9-NEXT: s_endpgm %r = urem <3 x i16> %x, %y store <3 x i16> %r, <3 x i16> addrspace(1)* %out @@ -4184,19 +4189,18 @@ ; ; GFX6-LABEL: sdiv_v3i16: ; GFX6: ; %bb.0: -; GFX6-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0xd -; GFX6-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 -; GFX6-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb -; GFX6-NEXT: s_mov_b32 s7, 0xf000 -; GFX6-NEXT: s_mov_b32 s6, -1 +; GFX6-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xb +; GFX6-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GFX6-NEXT: s_mov_b32 s3, 0xf000 +; GFX6-NEXT: s_mov_b32 s2, -1 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_sext_i32_i16 s8, s2 +; GFX6-NEXT: s_sext_i32_i16 s8, s6 ; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s8 -; GFX6-NEXT: s_sext_i32_i16 s9, s0 +; GFX6-NEXT: s_sext_i32_i16 s9, s4 ; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s9 ; GFX6-NEXT: s_xor_b32 s8, s9, s8 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v2, v0 -; GFX6-NEXT: s_ashr_i32 s2, s2, 16 +; GFX6-NEXT: s_ashr_i32 s6, s6, 16 ; GFX6-NEXT: s_ashr_i32 s8, s8, 30 ; GFX6-NEXT: s_or_b32 s8, s8, 1 ; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2 @@ -4204,55 +4208,54 @@ ; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1 ; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2 ; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, |v0| -; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s2 +; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s6 ; GFX6-NEXT: v_mov_b32_e32 v3, s8 ; GFX6-NEXT: v_cndmask_b32_e32 v0, 0, v3, vcc -; GFX6-NEXT: s_ashr_i32 s0, s0, 16 +; GFX6-NEXT: s_ashr_i32 s4, s4, 16 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s0 +; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s4 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v3, v1 -; GFX6-NEXT: s_xor_b32 s0, s0, s2 -; GFX6-NEXT: s_ashr_i32 s0, s0, 30 -; GFX6-NEXT: s_or_b32 s0, s0, 1 +; GFX6-NEXT: s_xor_b32 s4, s4, s6 +; GFX6-NEXT: s_ashr_i32 s4, s4, 30 +; GFX6-NEXT: s_or_b32 s4, s4, 1 ; GFX6-NEXT: v_mul_f32_e32 v3, v2, v3 ; GFX6-NEXT: v_trunc_f32_e32 v3, v3 ; GFX6-NEXT: v_mad_f32 v2, -v3, v1, v2 -; GFX6-NEXT: v_mov_b32_e32 v4, s0 -; GFX6-NEXT: s_sext_i32_i16 s0, s3 +; GFX6-NEXT: v_mov_b32_e32 v4, s4 +; GFX6-NEXT: s_sext_i32_i16 s4, s7 ; GFX6-NEXT: v_cvt_i32_f32_e32 v3, v3 ; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, |v1| -; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s0 +; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s4 ; GFX6-NEXT: v_cndmask_b32_e32 v1, 0, v4, vcc -; GFX6-NEXT: s_sext_i32_i16 s1, s1 +; GFX6-NEXT: s_sext_i32_i16 s5, s5 ; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v3 -; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s1 +; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s5 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v4, v2 -; GFX6-NEXT: s_xor_b32 s0, s1, s0 -; GFX6-NEXT: s_ashr_i32 s0, s0, 30 -; GFX6-NEXT: s_or_b32 s0, s0, 1 +; GFX6-NEXT: s_xor_b32 s4, s5, s4 +; GFX6-NEXT: s_ashr_i32 s4, s4, 30 +; GFX6-NEXT: s_or_b32 s4, s4, 1 ; GFX6-NEXT: v_mul_f32_e32 v4, v3, v4 ; GFX6-NEXT: v_trunc_f32_e32 v4, v4 ; GFX6-NEXT: v_mad_f32 v3, -v4, v2, v3 ; GFX6-NEXT: v_cvt_i32_f32_e32 v4, v4 -; GFX6-NEXT: v_mov_b32_e32 v5, s0 +; GFX6-NEXT: v_mov_b32_e32 v5, s4 ; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v2| ; GFX6-NEXT: v_cndmask_b32_e32 v2, 0, v5, vcc ; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v4 ; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0 ; GFX6-NEXT: v_or_b32_e32 v0, v0, v1 -; GFX6-NEXT: buffer_store_short v2, off, s[4:7], 0 offset:4 -; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0 +; GFX6-NEXT: buffer_store_short v2, off, s[0:3], 0 offset:4 +; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX6-NEXT: s_endpgm ; ; GFX9-LABEL: sdiv_v3i16: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 -; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x2c -; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x24 +; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x2c +; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 ; GFX9-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_sext_i32_i16 s0, s2 +; GFX9-NEXT: s_sext_i32_i16 s0, s6 ; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0 ; GFX9-NEXT: s_sext_i32_i16 s1, s4 ; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s1 @@ -4266,44 +4269,44 @@ ; GFX9-NEXT: v_cmp_ge_f32_e64 s[0:1], |v2|, |v0| ; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec ; GFX9-NEXT: s_cselect_b32 s0, s8, 0 -; GFX9-NEXT: s_ashr_i32 s1, s2, 16 +; GFX9-NEXT: s_ashr_i32 s1, s6, 16 ; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3 ; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s1 -; GFX9-NEXT: s_ashr_i32 s2, s4, 16 +; GFX9-NEXT: s_ashr_i32 s4, s4, 16 ; GFX9-NEXT: v_add_u32_e32 v2, s0, v3 -; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s2 +; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s4 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v4, v0 -; GFX9-NEXT: s_xor_b32 s0, s2, s1 +; GFX9-NEXT: s_xor_b32 s0, s4, s1 ; GFX9-NEXT: s_ashr_i32 s0, s0, 30 -; GFX9-NEXT: s_or_b32 s2, s0, 1 +; GFX9-NEXT: s_or_b32 s4, s0, 1 ; GFX9-NEXT: v_mul_f32_e32 v4, v3, v4 ; GFX9-NEXT: v_trunc_f32_e32 v4, v4 ; GFX9-NEXT: v_mad_f32 v3, -v4, v0, v3 ; GFX9-NEXT: v_cmp_ge_f32_e64 s[0:1], |v3|, |v0| ; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec ; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4 -; GFX9-NEXT: s_sext_i32_i16 s1, s3 +; GFX9-NEXT: s_sext_i32_i16 s1, s7 ; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s1 -; GFX9-NEXT: s_cselect_b32 s0, s2, 0 +; GFX9-NEXT: s_cselect_b32 s0, s4, 0 ; GFX9-NEXT: v_add_u32_e32 v3, s0, v4 ; GFX9-NEXT: s_sext_i32_i16 s0, s5 ; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s0 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v5, v0 ; GFX9-NEXT: s_xor_b32 s0, s0, s1 ; GFX9-NEXT: s_ashr_i32 s0, s0, 30 -; GFX9-NEXT: s_or_b32 s2, s0, 1 +; GFX9-NEXT: s_or_b32 s4, s0, 1 ; GFX9-NEXT: v_mul_f32_e32 v5, v4, v5 ; GFX9-NEXT: v_trunc_f32_e32 v5, v5 ; GFX9-NEXT: v_mad_f32 v4, -v5, v0, v4 ; GFX9-NEXT: v_cvt_i32_f32_e32 v5, v5 ; GFX9-NEXT: v_cmp_ge_f32_e64 s[0:1], |v4|, |v0| ; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec -; GFX9-NEXT: s_cselect_b32 s0, s2, 0 +; GFX9-NEXT: s_cselect_b32 s0, s4, 0 ; GFX9-NEXT: v_add_u32_e32 v0, s0, v5 ; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2 ; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2 -; GFX9-NEXT: global_store_short v1, v0, s[6:7] offset:4 -; GFX9-NEXT: global_store_dword v1, v2, s[6:7] +; GFX9-NEXT: global_store_short v1, v0, s[2:3] offset:4 +; GFX9-NEXT: global_store_dword v1, v2, s[2:3] ; GFX9-NEXT: s_endpgm %r = sdiv <3 x i16> %x, %y store <3 x i16> %r, <3 x i16> addrspace(1)* %out @@ -4395,51 +4398,51 @@ ; ; GFX6-LABEL: srem_v3i16: ; GFX6: ; %bb.0: -; GFX6-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd -; GFX6-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0xb +; GFX6-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xb ; GFX6-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 +; GFX6-NEXT: s_mov_b32 s2, -1 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_sext_i32_i16 s2, s4 -; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s2 ; GFX6-NEXT: s_sext_i32_i16 s8, s6 -; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s8 -; GFX6-NEXT: s_xor_b32 s2, s8, s2 +; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s8 +; GFX6-NEXT: s_sext_i32_i16 s9, s4 +; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s9 +; GFX6-NEXT: s_xor_b32 s8, s9, s8 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v2, v0 -; GFX6-NEXT: s_ashr_i32 s2, s2, 30 -; GFX6-NEXT: s_or_b32 s2, s2, 1 -; GFX6-NEXT: v_mov_b32_e32 v3, s2 +; GFX6-NEXT: s_ashr_i32 s8, s8, 30 +; GFX6-NEXT: s_or_b32 s8, s8, 1 +; GFX6-NEXT: v_mov_b32_e32 v3, s8 ; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2 ; GFX6-NEXT: v_trunc_f32_e32 v2, v2 ; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1 ; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2 ; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, |v0| ; GFX6-NEXT: v_cndmask_b32_e32 v0, 0, v3, vcc -; GFX6-NEXT: v_mov_b32_e32 v1, s6 +; GFX6-NEXT: v_mov_b32_e32 v1, s4 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GFX6-NEXT: v_mov_b32_e32 v2, s4 -; GFX6-NEXT: v_alignbit_b32 v2, s5, v2, 16 +; GFX6-NEXT: v_mov_b32_e32 v2, s6 +; GFX6-NEXT: v_alignbit_b32 v2, s7, v2, 16 ; GFX6-NEXT: v_bfe_i32 v3, v2, 0, 16 ; GFX6-NEXT: v_cvt_f32_i32_e32 v4, v3 -; GFX6-NEXT: v_alignbit_b32 v1, s7, v1, 16 +; GFX6-NEXT: v_alignbit_b32 v1, s5, v1, 16 ; GFX6-NEXT: v_bfe_i32 v5, v1, 0, 16 ; GFX6-NEXT: v_cvt_f32_i32_e32 v6, v5 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v7, v4 -; GFX6-NEXT: v_mul_lo_u32 v0, v0, s4 +; GFX6-NEXT: v_mul_lo_u32 v0, v0, s6 ; GFX6-NEXT: v_xor_b32_e32 v3, v5, v3 -; GFX6-NEXT: s_sext_i32_i16 s4, s5 +; GFX6-NEXT: v_ashrrev_i32_e32 v3, 30, v3 ; GFX6-NEXT: v_mul_f32_e32 v5, v6, v7 ; GFX6-NEXT: v_trunc_f32_e32 v5, v5 -; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s6, v0 +; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s4, v0 ; GFX6-NEXT: v_mad_f32 v6, -v5, v4, v6 ; GFX6-NEXT: v_cvt_i32_f32_e32 v5, v5 -; GFX6-NEXT: v_ashrrev_i32_e32 v3, 30, v3 +; GFX6-NEXT: s_sext_i32_i16 s4, s7 ; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v6|, |v4| ; GFX6-NEXT: v_cvt_f32_i32_e32 v4, s4 ; GFX6-NEXT: v_or_b32_e32 v3, 1, v3 ; GFX6-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc ; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v5 -; GFX6-NEXT: s_sext_i32_i16 s6, s7 +; GFX6-NEXT: s_sext_i32_i16 s6, s5 ; GFX6-NEXT: v_mul_lo_u32 v2, v3, v2 ; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s6 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v5, v4 @@ -4454,11 +4457,10 @@ ; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v4| ; GFX6-NEXT: v_cndmask_b32_e32 v3, 0, v6, vcc ; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v5 -; GFX6-NEXT: v_mul_lo_u32 v3, v3, s5 +; GFX6-NEXT: v_mul_lo_u32 v3, v3, s7 ; GFX6-NEXT: v_sub_i32_e32 v1, vcc, v1, v2 -; GFX6-NEXT: s_mov_b32 s2, -1 -; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s7, v3 ; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s5, v3 ; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0 ; GFX6-NEXT: v_or_b32_e32 v0, v0, v1 ; GFX6-NEXT: buffer_store_short v2, off, s[0:3], 0 offset:4 @@ -4467,66 +4469,65 @@ ; ; GFX9-LABEL: srem_v3i16: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 -; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x2c +; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x2c ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_sext_i32_i16 s8, s2 +; GFX9-NEXT: s_sext_i32_i16 s8, s6 ; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s8 ; GFX9-NEXT: s_sext_i32_i16 s9, s4 ; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s9 -; GFX9-NEXT: s_xor_b32 s6, s9, s8 +; GFX9-NEXT: s_xor_b32 s2, s9, s8 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v2, v0 -; GFX9-NEXT: s_ashr_i32 s6, s6, 30 -; GFX9-NEXT: s_or_b32 s10, s6, 1 -; GFX9-NEXT: s_sext_i32_i16 s5, s5 +; GFX9-NEXT: s_ashr_i32 s2, s2, 30 +; GFX9-NEXT: s_or_b32 s10, s2, 1 +; GFX9-NEXT: s_sext_i32_i16 s7, s7 ; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2 ; GFX9-NEXT: v_trunc_f32_e32 v2, v2 ; GFX9-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GFX9-NEXT: v_cmp_ge_f32_e64 s[6:7], |v1|, |v0| -; GFX9-NEXT: s_and_b64 s[6:7], s[6:7], exec -; GFX9-NEXT: s_cselect_b32 s6, s10, 0 -; GFX9-NEXT: s_ashr_i32 s2, s2, 16 +; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, |v0| +; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec +; GFX9-NEXT: s_cselect_b32 s2, s10, 0 +; GFX9-NEXT: s_ashr_i32 s6, s6, 16 ; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2 -; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2 +; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s6 ; GFX9-NEXT: s_ashr_i32 s4, s4, 16 -; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 -; GFX9-NEXT: v_add_u32_e32 v1, s6, v2 +; GFX9-NEXT: s_sext_i32_i16 s5, s5 +; GFX9-NEXT: v_add_u32_e32 v1, s2, v2 ; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s4 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v3, v0 -; GFX9-NEXT: s_xor_b32 s6, s4, s2 -; GFX9-NEXT: s_ashr_i32 s6, s6, 30 +; GFX9-NEXT: s_xor_b32 s2, s4, s6 +; GFX9-NEXT: s_ashr_i32 s2, s2, 30 ; GFX9-NEXT: v_mul_lo_u32 v1, v1, s8 ; GFX9-NEXT: v_mul_f32_e32 v3, v2, v3 ; GFX9-NEXT: v_trunc_f32_e32 v3, v3 ; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2 +; GFX9-NEXT: s_or_b32 s8, s2, 1 ; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3 -; GFX9-NEXT: s_or_b32 s8, s6, 1 -; GFX9-NEXT: v_cmp_ge_f32_e64 s[6:7], |v2|, |v0| -; GFX9-NEXT: s_and_b64 s[6:7], s[6:7], exec -; GFX9-NEXT: s_cselect_b32 s6, s8, 0 -; GFX9-NEXT: v_add_u32_e32 v0, s6, v3 -; GFX9-NEXT: s_sext_i32_i16 s6, s3 -; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s6 +; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v2|, |v0| +; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s7 +; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec +; GFX9-NEXT: s_cselect_b32 s2, s8, 0 +; GFX9-NEXT: v_add_u32_e32 v0, s2, v3 ; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s5 -; GFX9-NEXT: v_mul_lo_u32 v0, v0, s2 -; GFX9-NEXT: s_xor_b32 s2, s5, s6 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v4, v2 +; GFX9-NEXT: s_xor_b32 s2, s5, s7 ; GFX9-NEXT: s_ashr_i32 s2, s2, 30 -; GFX9-NEXT: s_or_b32 s7, s2, 1 -; GFX9-NEXT: v_sub_u32_e32 v1, s9, v1 +; GFX9-NEXT: v_mul_lo_u32 v0, v0, s6 ; GFX9-NEXT: v_mul_f32_e32 v4, v3, v4 ; GFX9-NEXT: v_trunc_f32_e32 v4, v4 ; GFX9-NEXT: v_mad_f32 v3, -v4, v2, v3 ; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4 +; GFX9-NEXT: s_or_b32 s6, s2, 1 ; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v3|, |v2| ; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec -; GFX9-NEXT: s_cselect_b32 s2, s7, 0 +; GFX9-NEXT: s_cselect_b32 s2, s6, 0 ; GFX9-NEXT: v_add_u32_e32 v2, s2, v4 -; GFX9-NEXT: v_mul_lo_u32 v2, v2, s6 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; GFX9-NEXT: v_mul_lo_u32 v2, v2, s7 +; GFX9-NEXT: v_sub_u32_e32 v1, s9, v1 ; GFX9-NEXT: v_mov_b32_e32 v3, 0 ; GFX9-NEXT: v_sub_u32_e32 v0, s4, v0 -; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1 ; GFX9-NEXT: v_sub_u32_e32 v2, s5, v2 +; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1 ; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: global_store_short v3, v2, s[0:1] offset:4 @@ -5680,109 +5681,106 @@ ; ; GFX6-LABEL: udiv_v2i32_pow2_shl_denom: ; GFX6: ; %bb.0: -; GFX6-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0xd -; GFX6-NEXT: s_movk_i32 s4, 0x1000 -; GFX6-NEXT: s_mov_b32 s7, 0xf000 -; GFX6-NEXT: s_mov_b32 s6, -1 +; GFX6-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xb +; GFX6-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x9 +; GFX6-NEXT: s_movk_i32 s2, 0x1000 +; GFX6-NEXT: s_mov_b32 s0, 0x4f7ffffe +; GFX6-NEXT: s_mov_b32 s11, 0xf000 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_lshl_b32 s8, s4, s2 -; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s8 -; GFX6-NEXT: s_lshl_b32 s9, s4, s3 -; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s9 -; GFX6-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 -; GFX6-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0xb +; GFX6-NEXT: s_lshl_b32 s3, s2, s6 +; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s3 +; GFX6-NEXT: s_lshl_b32 s2, s2, s7 +; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s2 +; GFX6-NEXT: s_mov_b32 s10, -1 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX6-NEXT: s_mov_b32 s0, 0x4f7ffffe ; GFX6-NEXT: v_rcp_iflag_f32_e32 v1, v1 ; GFX6-NEXT: v_mul_f32_e32 v0, s0, v0 ; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX6-NEXT: v_mul_f32_e32 v1, s0, v1 -; GFX6-NEXT: s_sub_i32 s0, 0, s8 +; GFX6-NEXT: s_sub_i32 s0, 0, s3 ; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1 ; GFX6-NEXT: v_mul_lo_u32 v2, s0, v0 -; GFX6-NEXT: s_sub_i32 s0, 0, s9 +; GFX6-NEXT: s_sub_i32 s0, 0, s2 ; GFX6-NEXT: v_mul_lo_u32 v3, s0, v1 ; GFX6-NEXT: v_mul_hi_u32 v2, v0, v2 ; GFX6-NEXT: v_mul_hi_u32 v3, v1, v3 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, v2, v0 -; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: v_mul_hi_u32 v0, s2, v0 +; GFX6-NEXT: v_mul_hi_u32 v0, s4, v0 ; GFX6-NEXT: v_add_i32_e32 v1, vcc, v3, v1 -; GFX6-NEXT: v_mul_hi_u32 v1, s3, v1 -; GFX6-NEXT: v_mul_lo_u32 v2, v0, s8 +; GFX6-NEXT: v_mul_hi_u32 v1, s5, v1 +; GFX6-NEXT: v_mul_lo_u32 v2, v0, s3 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v0 -; GFX6-NEXT: v_mul_lo_u32 v4, v1, s9 -; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s2, v2 -; GFX6-NEXT: v_cmp_le_u32_e64 s[0:1], s8, v2 +; GFX6-NEXT: v_mul_lo_u32 v4, v1, s2 +; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s4, v2 +; GFX6-NEXT: v_cmp_le_u32_e64 s[0:1], s3, v2 ; GFX6-NEXT: v_cndmask_b32_e64 v0, v0, v3, s[0:1] -; GFX6-NEXT: v_subrev_i32_e32 v3, vcc, s8, v2 +; GFX6-NEXT: v_subrev_i32_e32 v3, vcc, s3, v2 ; GFX6-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[0:1] ; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v0 -; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s8, v2 +; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s3, v2 ; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s3, v4 +; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s5, v4 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v1 -; GFX6-NEXT: v_cmp_le_u32_e64 s[0:1], s9, v2 +; GFX6-NEXT: v_cmp_le_u32_e64 s[0:1], s2, v2 ; GFX6-NEXT: v_cndmask_b32_e64 v1, v1, v3, s[0:1] -; GFX6-NEXT: v_subrev_i32_e32 v3, vcc, s9, v2 +; GFX6-NEXT: v_subrev_i32_e32 v3, vcc, s2, v2 ; GFX6-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[0:1] ; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v1 -; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s9, v2 +; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s2, v2 ; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0 ; GFX6-NEXT: s_endpgm ; ; GFX9-LABEL: udiv_v2i32_pow2_shl_denom: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 -; GFX9-NEXT: s_movk_i32 s4, 0x1000 +; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x2c +; GFX9-NEXT: s_movk_i32 s2, 0x1000 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_lshl_b32 s5, s4, s3 -; GFX9-NEXT: s_lshl_b32 s4, s4, s2 -; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s4 -; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s5 +; GFX9-NEXT: s_lshl_b32 s6, s2, s6 +; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s6 +; GFX9-NEXT: s_lshl_b32 s7, s2, s7 +; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s7 ; GFX9-NEXT: s_mov_b32 s2, 0x4f7ffffe -; GFX9-NEXT: s_sub_i32 s3, 0, s5 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 +; GFX9-NEXT: s_sub_i32 s3, 0, s7 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v1, v1 ; GFX9-NEXT: v_mul_f32_e32 v0, s2, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX9-NEXT: v_mul_f32_e32 v1, s2, v1 -; GFX9-NEXT: s_sub_i32 s2, 0, s4 ; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GFX9-NEXT: s_sub_i32 s2, 0, s6 ; GFX9-NEXT: v_mul_lo_u32 v2, s2, v0 ; GFX9-NEXT: v_mul_lo_u32 v3, s3, v1 -; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2c +; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 ; GFX9-NEXT: v_mul_hi_u32 v2, v0, v2 -; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; GFX9-NEXT: v_mul_hi_u32 v3, v1, v3 ; GFX9-NEXT: v_add_u32_e32 v0, v0, v2 -; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: v_mul_hi_u32 v0, s2, v0 +; GFX9-NEXT: v_mul_hi_u32 v0, s4, v0 ; GFX9-NEXT: v_add_u32_e32 v1, v1, v3 -; GFX9-NEXT: v_mul_hi_u32 v1, s3, v1 +; GFX9-NEXT: v_mul_hi_u32 v1, s5, v1 ; GFX9-NEXT: v_mov_b32_e32 v2, 0 -; GFX9-NEXT: v_mul_lo_u32 v3, v0, s4 +; GFX9-NEXT: v_mul_lo_u32 v3, v0, s6 ; GFX9-NEXT: v_add_u32_e32 v5, 1, v0 -; GFX9-NEXT: v_mul_lo_u32 v4, v1, s5 +; GFX9-NEXT: v_mul_lo_u32 v4, v1, s7 ; GFX9-NEXT: v_add_u32_e32 v6, 1, v1 -; GFX9-NEXT: v_sub_u32_e32 v3, s2, v3 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s4, v3 +; GFX9-NEXT: v_sub_u32_e32 v3, s4, v3 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s6, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc -; GFX9-NEXT: v_subrev_u32_e32 v5, s4, v3 +; GFX9-NEXT: v_subrev_u32_e32 v5, s6, v3 +; GFX9-NEXT: v_sub_u32_e32 v4, s5, v4 ; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc -; GFX9-NEXT: v_sub_u32_e32 v4, s3, v4 +; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s7, v4 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s6, v3 +; GFX9-NEXT: v_subrev_u32_e32 v3, s7, v4 +; GFX9-NEXT: v_cndmask_b32_e64 v1, v1, v6, s[0:1] ; GFX9-NEXT: v_add_u32_e32 v5, 1, v0 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s4, v3 +; GFX9-NEXT: v_cndmask_b32_e64 v3, v4, v3, s[0:1] ; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s5, v4 -; GFX9-NEXT: v_subrev_u32_e32 v3, s5, v4 -; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v6, vcc -; GFX9-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc ; GFX9-NEXT: v_add_u32_e32 v4, 1, v1 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s5, v3 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s7, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] +; GFX9-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] ; GFX9-NEXT: s_endpgm %shl.y = shl <2 x i32> , %y %r = udiv <2 x i32> %x, %shl.y @@ -6020,18 +6018,18 @@ ; ; GFX6-LABEL: urem_v2i32_pow2_shl_denom: ; GFX6: ; %bb.0: -; GFX6-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0xd -; GFX6-NEXT: s_movk_i32 s4, 0x1000 +; GFX6-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xb +; GFX6-NEXT: s_movk_i32 s2, 0x1000 +; GFX6-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GFX6-NEXT: s_mov_b32 s3, 0xf000 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_lshl_b32 s6, s4, s2 +; GFX6-NEXT: s_lshl_b32 s6, s2, s6 ; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s6 -; GFX6-NEXT: s_lshl_b32 s7, s4, s3 +; GFX6-NEXT: s_lshl_b32 s7, s2, s7 ; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s7 ; GFX6-NEXT: s_mov_b32 s2, 0x4f7ffffe ; GFX6-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX6-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xb ; GFX6-NEXT: v_rcp_iflag_f32_e32 v1, v1 -; GFX6-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; GFX6-NEXT: v_mul_f32_e32 v0, s2, v0 ; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX6-NEXT: v_mul_f32_e32 v1, s2, v1 @@ -6039,13 +6037,11 @@ ; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1 ; GFX6-NEXT: v_mul_lo_u32 v2, s2, v0 ; GFX6-NEXT: s_sub_i32 s2, 0, s7 -; GFX6-NEXT: s_mov_b32 s3, 0xf000 ; GFX6-NEXT: v_mul_lo_u32 v3, s2, v1 ; GFX6-NEXT: v_mul_hi_u32 v2, v0, v2 ; GFX6-NEXT: s_mov_b32 s2, -1 ; GFX6-NEXT: v_mul_hi_u32 v3, v1, v3 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, v2, v0 -; GFX6-NEXT: s_waitcnt lgkmcnt(0) ; GFX6-NEXT: v_mul_hi_u32 v0, s4, v0 ; GFX6-NEXT: v_add_i32_e32 v1, vcc, v3, v1 ; GFX6-NEXT: v_mul_hi_u32 v1, s5, v1 @@ -6070,49 +6066,47 @@ ; ; GFX9-LABEL: urem_v2i32_pow2_shl_denom: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 -; GFX9-NEXT: s_movk_i32 s4, 0x1000 +; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x2c +; GFX9-NEXT: s_movk_i32 s2, 0x1000 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_lshl_b32 s5, s4, s3 -; GFX9-NEXT: s_lshl_b32 s4, s4, s2 -; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s4 -; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s5 -; GFX9-NEXT: s_mov_b32 s2, 0x4f7ffffe -; GFX9-NEXT: s_sub_i32 s3, 0, s5 +; GFX9-NEXT: s_lshl_b32 s3, s2, s7 +; GFX9-NEXT: s_lshl_b32 s2, s2, s6 +; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s2 +; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s3 +; GFX9-NEXT: s_mov_b32 s6, 0x4f7ffffe +; GFX9-NEXT: s_sub_i32 s7, 0, s3 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v1, v1 -; GFX9-NEXT: v_mul_f32_e32 v0, s2, v0 -; GFX9-NEXT: v_mul_f32_e32 v1, s2, v1 +; GFX9-NEXT: v_mul_f32_e32 v0, s6, v0 +; GFX9-NEXT: v_mul_f32_e32 v1, s6, v1 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX9-NEXT: s_sub_i32 s2, 0, s4 -; GFX9-NEXT: v_mul_lo_u32 v2, s2, v0 -; GFX9-NEXT: v_mul_lo_u32 v3, s3, v1 -; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2c +; GFX9-NEXT: s_sub_i32 s6, 0, s2 +; GFX9-NEXT: v_mul_lo_u32 v2, s6, v0 +; GFX9-NEXT: v_mul_lo_u32 v3, s7, v1 ; GFX9-NEXT: v_mul_hi_u32 v2, v0, v2 ; GFX9-NEXT: v_mul_hi_u32 v3, v1, v3 -; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; GFX9-NEXT: v_add_u32_e32 v0, v0, v2 ; GFX9-NEXT: v_add_u32_e32 v1, v1, v3 -; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: v_mul_hi_u32 v0, s2, v0 -; GFX9-NEXT: v_mul_hi_u32 v1, s3, v1 +; GFX9-NEXT: v_mul_hi_u32 v0, s4, v0 +; GFX9-NEXT: v_mul_hi_u32 v1, s5, v1 ; GFX9-NEXT: v_mov_b32_e32 v2, 0 -; GFX9-NEXT: v_mul_lo_u32 v0, v0, s4 -; GFX9-NEXT: v_mul_lo_u32 v1, v1, s5 -; GFX9-NEXT: v_sub_u32_e32 v0, s2, v0 -; GFX9-NEXT: v_sub_u32_e32 v1, s3, v1 -; GFX9-NEXT: v_subrev_u32_e32 v3, s4, v0 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s4, v0 -; GFX9-NEXT: v_subrev_u32_e32 v4, s5, v1 +; GFX9-NEXT: v_mul_lo_u32 v0, v0, s2 +; GFX9-NEXT: v_mul_lo_u32 v1, v1, s3 +; GFX9-NEXT: v_sub_u32_e32 v0, s4, v0 +; GFX9-NEXT: v_sub_u32_e32 v1, s5, v1 +; GFX9-NEXT: v_subrev_u32_e32 v3, s2, v0 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s2, v0 +; GFX9-NEXT: v_subrev_u32_e32 v4, s3, v1 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s5, v1 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s3, v1 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc -; GFX9-NEXT: v_subrev_u32_e32 v3, s4, v0 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s4, v0 -; GFX9-NEXT: v_subrev_u32_e32 v4, s5, v1 +; GFX9-NEXT: v_subrev_u32_e32 v3, s2, v0 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s2, v0 +; GFX9-NEXT: v_subrev_u32_e32 v4, s3, v1 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s5, v1 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s3, v1 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc ; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1] ; GFX9-NEXT: s_endpgm @@ -6492,133 +6486,130 @@ ; ; GFX6-LABEL: sdiv_v2i32_pow2_shl_denom: ; GFX6: ; %bb.0: -; GFX6-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0xd -; GFX6-NEXT: s_movk_i32 s10, 0x1000 +; GFX6-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xb +; GFX6-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x9 +; GFX6-NEXT: s_movk_i32 s2, 0x1000 ; GFX6-NEXT: s_mov_b32 s12, 0x4f7ffffe -; GFX6-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 -; GFX6-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0xb -; GFX6-NEXT: s_mov_b32 s7, 0xf000 +; GFX6-NEXT: s_mov_b32 s11, 0xf000 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_lshl_b32 s2, s10, s2 -; GFX6-NEXT: s_ashr_i32 s11, s2, 31 -; GFX6-NEXT: s_add_i32 s2, s2, s11 -; GFX6-NEXT: s_xor_b32 s2, s2, s11 -; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s2 -; GFX6-NEXT: s_lshl_b32 s0, s10, s3 -; GFX6-NEXT: s_sub_i32 s10, 0, s2 -; GFX6-NEXT: s_ashr_i32 s3, s0, 31 +; GFX6-NEXT: s_lshl_b32 s3, s2, s6 +; GFX6-NEXT: s_ashr_i32 s6, s3, 31 +; GFX6-NEXT: s_add_i32 s3, s3, s6 +; GFX6-NEXT: s_xor_b32 s3, s3, s6 +; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s3 +; GFX6-NEXT: s_lshl_b32 s0, s2, s7 +; GFX6-NEXT: s_sub_i32 s7, 0, s3 +; GFX6-NEXT: s_ashr_i32 s2, s0, 31 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX6-NEXT: s_add_i32 s0, s0, s3 -; GFX6-NEXT: s_ashr_i32 s1, s8, 31 -; GFX6-NEXT: s_mov_b32 s6, -1 +; GFX6-NEXT: s_add_i32 s0, s0, s2 +; GFX6-NEXT: s_ashr_i32 s1, s4, 31 +; GFX6-NEXT: s_mov_b32 s10, -1 ; GFX6-NEXT: v_mul_f32_e32 v0, s12, v0 ; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX6-NEXT: v_mul_lo_u32 v1, s10, v0 -; GFX6-NEXT: s_xor_b32 s10, s0, s3 -; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s10 -; GFX6-NEXT: s_add_i32 s0, s8, s1 +; GFX6-NEXT: v_mul_lo_u32 v1, s7, v0 +; GFX6-NEXT: s_xor_b32 s7, s0, s2 +; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s7 +; GFX6-NEXT: s_add_i32 s0, s4, s1 ; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1 ; GFX6-NEXT: s_xor_b32 s0, s0, s1 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v2, v2 -; GFX6-NEXT: s_xor_b32 s8, s1, s11 +; GFX6-NEXT: s_xor_b32 s4, s1, s6 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, v1, v0 ; GFX6-NEXT: v_mul_hi_u32 v0, s0, v0 ; GFX6-NEXT: v_mul_f32_e32 v1, s12, v2 ; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX6-NEXT: v_mul_lo_u32 v2, v0, s2 +; GFX6-NEXT: v_mul_lo_u32 v2, v0, s3 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v0 ; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s0, v2 -; GFX6-NEXT: v_cmp_le_u32_e64 s[0:1], s2, v2 +; GFX6-NEXT: v_cmp_le_u32_e64 s[0:1], s3, v2 ; GFX6-NEXT: v_cndmask_b32_e64 v0, v0, v3, s[0:1] -; GFX6-NEXT: v_subrev_i32_e32 v3, vcc, s2, v2 +; GFX6-NEXT: v_subrev_i32_e32 v3, vcc, s3, v2 ; GFX6-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[0:1] -; GFX6-NEXT: s_sub_i32 s0, 0, s10 +; GFX6-NEXT: s_sub_i32 s0, 0, s7 ; GFX6-NEXT: v_mul_lo_u32 v3, s0, v1 -; GFX6-NEXT: s_ashr_i32 s0, s9, 31 -; GFX6-NEXT: s_add_i32 s1, s9, s0 +; GFX6-NEXT: s_ashr_i32 s0, s5, 31 +; GFX6-NEXT: s_add_i32 s1, s5, s0 ; GFX6-NEXT: s_xor_b32 s1, s1, s0 ; GFX6-NEXT: v_mul_hi_u32 v3, v1, v3 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, 1, v0 +; GFX6-NEXT: s_xor_b32 s2, s0, s2 ; GFX6-NEXT: v_add_i32_e32 v1, vcc, v3, v1 ; GFX6-NEXT: v_mul_hi_u32 v1, s1, v1 -; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s2, v2 +; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s3, v2 ; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc -; GFX6-NEXT: s_xor_b32 s2, s0, s3 -; GFX6-NEXT: v_mul_lo_u32 v2, v1, s10 +; GFX6-NEXT: v_xor_b32_e32 v0, s4, v0 +; GFX6-NEXT: v_mul_lo_u32 v2, v1, s7 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v1 -; GFX6-NEXT: v_xor_b32_e32 v0, s8, v0 +; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s4, v0 ; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s1, v2 -; GFX6-NEXT: v_cmp_le_u32_e64 s[0:1], s10, v2 +; GFX6-NEXT: v_cmp_le_u32_e64 s[0:1], s7, v2 ; GFX6-NEXT: v_cndmask_b32_e64 v1, v1, v3, s[0:1] -; GFX6-NEXT: v_subrev_i32_e32 v3, vcc, s10, v2 -; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s8, v0 +; GFX6-NEXT: v_subrev_i32_e32 v3, vcc, s7, v2 ; GFX6-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[0:1] ; GFX6-NEXT: v_add_i32_e32 v3, vcc, 1, v1 -; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s10, v2 +; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s7, v2 ; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc ; GFX6-NEXT: v_xor_b32_e32 v1, s2, v1 ; GFX6-NEXT: v_subrev_i32_e32 v1, vcc, s2, v1 -; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0 ; GFX6-NEXT: s_endpgm ; ; GFX9-LABEL: sdiv_v2i32_pow2_shl_denom: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 -; GFX9-NEXT: s_movk_i32 s8, 0x1000 -; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24 -; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x2c +; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x2c +; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GFX9-NEXT: s_movk_i32 s0, 0x1000 ; GFX9-NEXT: s_mov_b32 s10, 0x4f7ffffe ; GFX9-NEXT: v_mov_b32_e32 v2, 0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_lshl_b32 s2, s8, s2 -; GFX9-NEXT: s_ashr_i32 s9, s2, 31 -; GFX9-NEXT: s_add_i32 s2, s2, s9 -; GFX9-NEXT: s_xor_b32 s2, s2, s9 -; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s2 -; GFX9-NEXT: s_lshl_b32 s0, s8, s3 -; GFX9-NEXT: s_ashr_i32 s1, s0, 31 -; GFX9-NEXT: s_add_i32 s0, s0, s1 +; GFX9-NEXT: s_lshl_b32 s1, s0, s6 +; GFX9-NEXT: s_ashr_i32 s6, s1, 31 +; GFX9-NEXT: s_add_i32 s1, s1, s6 +; GFX9-NEXT: s_xor_b32 s1, s1, s6 +; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s1 +; GFX9-NEXT: s_lshl_b32 s0, s0, s7 +; GFX9-NEXT: s_ashr_i32 s8, s0, 31 +; GFX9-NEXT: s_add_i32 s0, s0, s8 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX9-NEXT: s_xor_b32 s0, s0, s1 +; GFX9-NEXT: s_xor_b32 s0, s0, s8 ; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s0 -; GFX9-NEXT: s_sub_i32 s3, 0, s2 +; GFX9-NEXT: s_sub_i32 s9, 0, s1 ; GFX9-NEXT: v_mul_f32_e32 v0, s10, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v1, v1 -; GFX9-NEXT: s_sub_i32 s8, 0, s0 -; GFX9-NEXT: v_mul_lo_u32 v3, s3, v0 +; GFX9-NEXT: s_ashr_i32 s7, s4, 31 +; GFX9-NEXT: s_add_i32 s4, s4, s7 +; GFX9-NEXT: v_mul_lo_u32 v3, s9, v0 ; GFX9-NEXT: v_mul_f32_e32 v1, s10, v1 -; GFX9-NEXT: s_ashr_i32 s3, s6, 31 ; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1 +; GFX9-NEXT: s_xor_b32 s4, s4, s7 ; GFX9-NEXT: v_mul_hi_u32 v3, v0, v3 -; GFX9-NEXT: s_add_i32 s6, s6, s3 -; GFX9-NEXT: s_xor_b32 s6, s6, s3 -; GFX9-NEXT: s_xor_b32 s3, s3, s9 +; GFX9-NEXT: s_sub_i32 s10, 0, s0 +; GFX9-NEXT: s_ashr_i32 s9, s5, 31 +; GFX9-NEXT: s_add_i32 s5, s5, s9 ; GFX9-NEXT: v_add_u32_e32 v0, v0, v3 -; GFX9-NEXT: v_mul_hi_u32 v0, s6, v0 -; GFX9-NEXT: v_mul_lo_u32 v3, s8, v1 -; GFX9-NEXT: s_ashr_i32 s8, s7, 31 -; GFX9-NEXT: s_xor_b32 s1, s8, s1 -; GFX9-NEXT: v_mul_lo_u32 v4, v0, s2 +; GFX9-NEXT: v_mul_hi_u32 v0, s4, v0 +; GFX9-NEXT: v_mul_lo_u32 v3, s10, v1 +; GFX9-NEXT: s_xor_b32 s6, s7, s6 +; GFX9-NEXT: v_mul_lo_u32 v4, v0, s1 ; GFX9-NEXT: v_mul_hi_u32 v3, v1, v3 ; GFX9-NEXT: v_add_u32_e32 v5, 1, v0 -; GFX9-NEXT: v_sub_u32_e32 v4, s6, v4 -; GFX9-NEXT: s_add_i32 s6, s7, s8 -; GFX9-NEXT: s_xor_b32 s6, s6, s8 +; GFX9-NEXT: v_sub_u32_e32 v4, s4, v4 +; GFX9-NEXT: s_xor_b32 s4, s5, s9 ; GFX9-NEXT: v_add_u32_e32 v1, v1, v3 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s2, v4 -; GFX9-NEXT: v_mul_hi_u32 v1, s6, v1 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s1, v4 +; GFX9-NEXT: v_mul_hi_u32 v1, s4, v1 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v5, vcc -; GFX9-NEXT: v_subrev_u32_e32 v5, s2, v4 +; GFX9-NEXT: v_subrev_u32_e32 v5, s1, v4 ; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v5, vcc ; GFX9-NEXT: v_add_u32_e32 v3, 1, v0 -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s2, v4 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s1, v4 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc ; GFX9-NEXT: v_mul_lo_u32 v3, v1, s0 ; GFX9-NEXT: v_add_u32_e32 v4, 1, v1 -; GFX9-NEXT: v_xor_b32_e32 v0, s3, v0 -; GFX9-NEXT: v_subrev_u32_e32 v0, s3, v0 -; GFX9-NEXT: v_sub_u32_e32 v3, s6, v3 +; GFX9-NEXT: s_xor_b32 s1, s9, s8 +; GFX9-NEXT: v_xor_b32_e32 v0, s6, v0 +; GFX9-NEXT: v_sub_u32_e32 v3, s4, v3 ; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s0, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc ; GFX9-NEXT: v_subrev_u32_e32 v4, s0, v3 @@ -6627,8 +6618,9 @@ ; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s0, v3 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc ; GFX9-NEXT: v_xor_b32_e32 v1, s1, v1 +; GFX9-NEXT: v_subrev_u32_e32 v0, s6, v0 ; GFX9-NEXT: v_subrev_u32_e32 v1, s1, v1 -; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5] +; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] ; GFX9-NEXT: s_endpgm %shl.y = shl <2 x i32> , %y %r = sdiv <2 x i32> %x, %shl.y @@ -6951,120 +6943,117 @@ ; ; GFX6-LABEL: srem_v2i32_pow2_shl_denom: ; GFX6: ; %bb.0: -; GFX6-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0xd -; GFX6-NEXT: s_movk_i32 s6, 0x1000 -; GFX6-NEXT: s_mov_b32 s10, 0x4f7ffffe -; GFX6-NEXT: s_mov_b32 s7, 0xf000 +; GFX6-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xb +; GFX6-NEXT: s_movk_i32 s8, 0x1000 +; GFX6-NEXT: s_mov_b32 s11, 0x4f7ffffe +; GFX6-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_lshl_b32 s2, s6, s2 -; GFX6-NEXT: s_ashr_i32 s4, s2, 31 -; GFX6-NEXT: s_add_i32 s2, s2, s4 -; GFX6-NEXT: s_xor_b32 s2, s2, s4 -; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s2 -; GFX6-NEXT: s_lshl_b32 s3, s6, s3 -; GFX6-NEXT: s_ashr_i32 s6, s3, 31 -; GFX6-NEXT: s_add_i32 s3, s3, s6 +; GFX6-NEXT: s_lshl_b32 s2, s8, s6 +; GFX6-NEXT: s_ashr_i32 s3, s2, 31 +; GFX6-NEXT: s_add_i32 s2, s2, s3 +; GFX6-NEXT: s_xor_b32 s6, s2, s3 +; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s6 +; GFX6-NEXT: s_lshl_b32 s7, s8, s7 +; GFX6-NEXT: s_ashr_i32 s9, s7, 31 +; GFX6-NEXT: s_add_i32 s7, s7, s9 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v0, v0 -; GFX6-NEXT: s_sub_i32 s9, 0, s2 -; GFX6-NEXT: s_xor_b32 s3, s3, s6 -; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s3 -; GFX6-NEXT: v_mul_f32_e32 v0, s10, v0 +; GFX6-NEXT: s_sub_i32 s10, 0, s6 +; GFX6-NEXT: s_xor_b32 s7, s7, s9 +; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s7 +; GFX6-NEXT: v_mul_f32_e32 v0, s11, v0 ; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX6-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 -; GFX6-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb +; GFX6-NEXT: s_ashr_i32 s8, s4, 31 ; GFX6-NEXT: v_rcp_iflag_f32_e32 v2, v2 -; GFX6-NEXT: s_mov_b32 s6, -1 -; GFX6-NEXT: v_mul_lo_u32 v1, s9, v0 -; GFX6-NEXT: s_sub_i32 s9, 0, s3 -; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_ashr_i32 s8, s0, 31 -; GFX6-NEXT: s_add_i32 s0, s0, s8 +; GFX6-NEXT: s_add_i32 s4, s4, s8 +; GFX6-NEXT: v_mul_lo_u32 v1, s10, v0 +; GFX6-NEXT: s_xor_b32 s4, s4, s8 +; GFX6-NEXT: s_sub_i32 s10, 0, s7 +; GFX6-NEXT: s_ashr_i32 s9, s5, 31 ; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1 -; GFX6-NEXT: s_xor_b32 s0, s0, s8 +; GFX6-NEXT: s_mov_b32 s3, 0xf000 +; GFX6-NEXT: s_mov_b32 s2, -1 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, v1, v0 -; GFX6-NEXT: v_mul_f32_e32 v1, s10, v2 +; GFX6-NEXT: v_mul_f32_e32 v1, s11, v2 ; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX6-NEXT: v_mul_hi_u32 v0, s0, v0 -; GFX6-NEXT: v_mul_lo_u32 v2, s9, v1 -; GFX6-NEXT: v_mul_lo_u32 v0, v0, s2 +; GFX6-NEXT: v_mul_hi_u32 v0, s4, v0 +; GFX6-NEXT: v_mul_lo_u32 v2, s10, v1 +; GFX6-NEXT: v_mul_lo_u32 v0, v0, s6 ; GFX6-NEXT: v_mul_hi_u32 v2, v1, v2 -; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s0, v0 -; GFX6-NEXT: v_subrev_i32_e32 v3, vcc, s2, v0 -; GFX6-NEXT: s_ashr_i32 s0, s1, 31 -; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s2, v0 -; GFX6-NEXT: s_add_i32 s1, s1, s0 +; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s4, v0 +; GFX6-NEXT: v_subrev_i32_e32 v3, vcc, s6, v0 +; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s6, v0 +; GFX6-NEXT: s_add_i32 s4, s5, s9 ; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc -; GFX6-NEXT: s_xor_b32 s1, s1, s0 +; GFX6-NEXT: s_xor_b32 s4, s4, s9 ; GFX6-NEXT: v_add_i32_e32 v1, vcc, v2, v1 -; GFX6-NEXT: v_mul_hi_u32 v1, s1, v1 -; GFX6-NEXT: v_subrev_i32_e32 v3, vcc, s2, v0 -; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s2, v0 -; GFX6-NEXT: v_mul_lo_u32 v1, v1, s3 +; GFX6-NEXT: v_mul_hi_u32 v1, s4, v1 +; GFX6-NEXT: v_subrev_i32_e32 v3, vcc, s6, v0 +; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s6, v0 +; GFX6-NEXT: v_mul_lo_u32 v1, v1, s7 ; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc ; GFX6-NEXT: v_xor_b32_e32 v0, s8, v0 ; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s8, v0 -; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s1, v1 -; GFX6-NEXT: v_subrev_i32_e32 v2, vcc, s3, v1 -; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s3, v1 +; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s4, v1 +; GFX6-NEXT: v_subrev_i32_e32 v2, vcc, s7, v1 +; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s7, v1 ; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc -; GFX6-NEXT: v_subrev_i32_e32 v2, vcc, s3, v1 -; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s3, v1 +; GFX6-NEXT: v_subrev_i32_e32 v2, vcc, s7, v1 +; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s7, v1 ; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc -; GFX6-NEXT: v_xor_b32_e32 v1, s0, v1 -; GFX6-NEXT: v_subrev_i32_e32 v1, vcc, s0, v1 -; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; GFX6-NEXT: v_xor_b32_e32 v1, s9, v1 +; GFX6-NEXT: v_subrev_i32_e32 v1, vcc, s9, v1 +; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 ; GFX6-NEXT: s_endpgm ; ; GFX9-LABEL: srem_v2i32_pow2_shl_denom: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 -; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24 -; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x2c +; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x2c +; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 ; GFX9-NEXT: s_movk_i32 s0, 0x1000 ; GFX9-NEXT: s_mov_b32 s8, 0x4f7ffffe +; GFX9-NEXT: v_mov_b32_e32 v2, 0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_lshl_b32 s1, s0, s2 -; GFX9-NEXT: s_ashr_i32 s2, s1, 31 -; GFX9-NEXT: s_add_i32 s1, s1, s2 -; GFX9-NEXT: s_xor_b32 s1, s1, s2 +; GFX9-NEXT: s_lshl_b32 s1, s0, s6 +; GFX9-NEXT: s_ashr_i32 s6, s1, 31 +; GFX9-NEXT: s_add_i32 s1, s1, s6 +; GFX9-NEXT: s_xor_b32 s1, s1, s6 ; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s1 -; GFX9-NEXT: s_lshl_b32 s0, s0, s3 -; GFX9-NEXT: s_ashr_i32 s2, s0, 31 -; GFX9-NEXT: s_add_i32 s0, s0, s2 -; GFX9-NEXT: s_xor_b32 s0, s0, s2 +; GFX9-NEXT: s_lshl_b32 s0, s0, s7 +; GFX9-NEXT: s_ashr_i32 s6, s0, 31 +; GFX9-NEXT: s_add_i32 s0, s0, s6 +; GFX9-NEXT: s_xor_b32 s0, s0, s6 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0 ; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s0 -; GFX9-NEXT: s_sub_i32 s3, 0, s1 -; GFX9-NEXT: s_ashr_i32 s2, s6, 31 +; GFX9-NEXT: s_sub_i32 s7, 0, s1 +; GFX9-NEXT: s_ashr_i32 s6, s4, 31 ; GFX9-NEXT: v_mul_f32_e32 v0, s8, v0 ; GFX9-NEXT: v_rcp_iflag_f32_e32 v1, v1 ; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0 -; GFX9-NEXT: v_mov_b32_e32 v2, 0 +; GFX9-NEXT: s_add_i32 s4, s4, s6 +; GFX9-NEXT: s_xor_b32 s4, s4, s6 ; GFX9-NEXT: v_mul_f32_e32 v1, s8, v1 -; GFX9-NEXT: v_mul_lo_u32 v3, s3, v0 +; GFX9-NEXT: v_mul_lo_u32 v3, s7, v0 ; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1 -; GFX9-NEXT: s_add_i32 s3, s6, s2 -; GFX9-NEXT: s_sub_i32 s6, 0, s0 +; GFX9-NEXT: s_sub_i32 s7, 0, s0 ; GFX9-NEXT: v_mul_hi_u32 v3, v0, v3 -; GFX9-NEXT: v_mul_lo_u32 v4, s6, v1 -; GFX9-NEXT: s_xor_b32 s3, s3, s2 -; GFX9-NEXT: s_ashr_i32 s6, s7, 31 +; GFX9-NEXT: v_mul_lo_u32 v4, s7, v1 +; GFX9-NEXT: s_ashr_i32 s7, s5, 31 +; GFX9-NEXT: s_add_i32 s5, s5, s7 ; GFX9-NEXT: v_add_u32_e32 v0, v0, v3 ; GFX9-NEXT: v_mul_hi_u32 v3, v1, v4 -; GFX9-NEXT: v_mul_hi_u32 v0, s3, v0 -; GFX9-NEXT: s_add_i32 s7, s7, s6 -; GFX9-NEXT: s_xor_b32 s7, s7, s6 +; GFX9-NEXT: v_mul_hi_u32 v0, s4, v0 +; GFX9-NEXT: s_xor_b32 s5, s5, s7 ; GFX9-NEXT: v_add_u32_e32 v1, v1, v3 -; GFX9-NEXT: v_mul_hi_u32 v1, s7, v1 +; GFX9-NEXT: v_mul_hi_u32 v1, s5, v1 ; GFX9-NEXT: v_mul_lo_u32 v0, v0, s1 ; GFX9-NEXT: v_mul_lo_u32 v1, v1, s0 -; GFX9-NEXT: v_sub_u32_e32 v0, s3, v0 +; GFX9-NEXT: v_sub_u32_e32 v0, s4, v0 ; GFX9-NEXT: v_subrev_u32_e32 v3, s1, v0 ; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s1, v0 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc ; GFX9-NEXT: v_subrev_u32_e32 v3, s1, v0 ; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s1, v0 -; GFX9-NEXT: v_sub_u32_e32 v1, s7, v1 +; GFX9-NEXT: v_sub_u32_e32 v1, s5, v1 ; GFX9-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc ; GFX9-NEXT: v_subrev_u32_e32 v3, s0, v1 ; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s0, v1 @@ -7072,11 +7061,11 @@ ; GFX9-NEXT: v_subrev_u32_e32 v3, s0, v1 ; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s0, v1 ; GFX9-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc -; GFX9-NEXT: v_xor_b32_e32 v0, s2, v0 -; GFX9-NEXT: v_xor_b32_e32 v1, s6, v1 -; GFX9-NEXT: v_subrev_u32_e32 v0, s2, v0 -; GFX9-NEXT: v_subrev_u32_e32 v1, s6, v1 -; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5] +; GFX9-NEXT: v_xor_b32_e32 v0, s6, v0 +; GFX9-NEXT: v_xor_b32_e32 v1, s7, v1 +; GFX9-NEXT: v_subrev_u32_e32 v0, s6, v0 +; GFX9-NEXT: v_subrev_u32_e32 v1, s7, v1 +; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] ; GFX9-NEXT: s_endpgm %shl.y = shl <2 x i32> , %y %r = srem <2 x i32> %x, %shl.y @@ -7703,16 +7692,15 @@ ; ; GFX6-LABEL: udiv_v2i64_pow2_shl_denom: ; GFX6: ; %bb.0: -; GFX6-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x11 -; GFX6-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0xd +; GFX6-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0xd ; GFX6-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; GFX6-NEXT: s_mov_b32 s3, 0xf000 ; GFX6-NEXT: s_mov_b32 s2, -1 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_add_i32 s4, s4, 12 -; GFX6-NEXT: s_add_i32 s6, s6, 12 -; GFX6-NEXT: s_lshr_b64 s[4:5], s[8:9], s4 -; GFX6-NEXT: s_lshr_b64 s[6:7], s[10:11], s6 +; GFX6-NEXT: s_add_i32 s8, s8, 12 +; GFX6-NEXT: s_add_i32 s9, s10, 12 +; GFX6-NEXT: s_lshr_b64 s[4:5], s[4:5], s8 +; GFX6-NEXT: s_lshr_b64 s[6:7], s[6:7], s9 ; GFX6-NEXT: v_mov_b32_e32 v0, s4 ; GFX6-NEXT: v_mov_b32_e32 v1, s5 ; GFX6-NEXT: v_mov_b32_e32 v2, s6 @@ -7722,20 +7710,19 @@ ; ; GFX9-LABEL: udiv_v2i64_pow2_shl_denom: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x44 -; GFX9-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x34 -; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GFX9-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x34 ; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_add_i32 s0, s4, 12 -; GFX9-NEXT: s_add_i32 s4, s6, 12 -; GFX9-NEXT: s_lshr_b64 s[0:1], s[8:9], s0 -; GFX9-NEXT: s_lshr_b64 s[4:5], s[10:11], s4 -; GFX9-NEXT: v_mov_b32_e32 v0, s0 -; GFX9-NEXT: v_mov_b32_e32 v1, s1 +; GFX9-NEXT: s_add_i32 s2, s8, 12 +; GFX9-NEXT: s_add_i32 s8, s10, 12 +; GFX9-NEXT: s_lshr_b64 s[2:3], s[4:5], s2 +; GFX9-NEXT: s_lshr_b64 s[4:5], s[6:7], s8 +; GFX9-NEXT: v_mov_b32_e32 v0, s2 +; GFX9-NEXT: v_mov_b32_e32 v1, s3 ; GFX9-NEXT: v_mov_b32_e32 v2, s4 ; GFX9-NEXT: v_mov_b32_e32 v3, s5 -; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] +; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; GFX9-NEXT: s_endpgm %shl.y = shl <2 x i64> , %y %r = udiv <2 x i64> %x, %shl.y @@ -8135,49 +8122,47 @@ ; ; GFX6-LABEL: urem_v2i64_pow2_shl_denom: ; GFX6: ; %bb.0: -; GFX6-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 -; GFX6-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0xd -; GFX6-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x11 +; GFX6-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x9 +; GFX6-NEXT: s_load_dwordx8 s[0:7], s[0:1], 0xd ; GFX6-NEXT: s_mov_b64 s[12:13], 0x1000 -; GFX6-NEXT: s_mov_b32 s7, 0xf000 -; GFX6-NEXT: s_mov_b32 s6, -1 +; GFX6-NEXT: s_mov_b32 s11, 0xf000 +; GFX6-NEXT: s_mov_b32 s10, -1 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_lshl_b64 s[2:3], s[12:13], s2 -; GFX6-NEXT: s_lshl_b64 s[0:1], s[12:13], s0 -; GFX6-NEXT: s_add_u32 s0, s0, -1 -; GFX6-NEXT: s_addc_u32 s1, s1, -1 -; GFX6-NEXT: s_and_b64 s[0:1], s[8:9], s[0:1] -; GFX6-NEXT: s_add_u32 s2, s2, -1 -; GFX6-NEXT: s_addc_u32 s3, s3, -1 -; GFX6-NEXT: s_and_b64 s[2:3], s[10:11], s[2:3] +; GFX6-NEXT: s_lshl_b64 s[6:7], s[12:13], s6 +; GFX6-NEXT: s_lshl_b64 s[4:5], s[12:13], s4 +; GFX6-NEXT: s_add_u32 s4, s4, -1 +; GFX6-NEXT: s_addc_u32 s5, s5, -1 +; GFX6-NEXT: s_and_b64 s[0:1], s[0:1], s[4:5] +; GFX6-NEXT: s_add_u32 s4, s6, -1 +; GFX6-NEXT: s_addc_u32 s5, s7, -1 +; GFX6-NEXT: s_and_b64 s[2:3], s[2:3], s[4:5] ; GFX6-NEXT: v_mov_b32_e32 v0, s0 ; GFX6-NEXT: v_mov_b32_e32 v1, s1 ; GFX6-NEXT: v_mov_b32_e32 v2, s2 ; GFX6-NEXT: v_mov_b32_e32 v3, s3 -; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 +; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0 ; GFX6-NEXT: s_endpgm ; ; GFX9-LABEL: urem_v2i64_pow2_shl_denom: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x44 -; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 -; GFX9-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x34 -; GFX9-NEXT: s_mov_b64 s[0:1], 0x1000 +; GFX9-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x34 +; GFX9-NEXT: s_mov_b64 s[2:3], 0x1000 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; GFX9-NEXT: v_mov_b32_e32 v4, 0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_lshl_b64 s[6:7], s[0:1], s6 -; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], s4 -; GFX9-NEXT: s_add_u32 s0, s0, -1 -; GFX9-NEXT: s_addc_u32 s1, s1, -1 -; GFX9-NEXT: s_and_b64 s[0:1], s[8:9], s[0:1] -; GFX9-NEXT: s_add_u32 s4, s6, -1 -; GFX9-NEXT: s_addc_u32 s5, s7, -1 -; GFX9-NEXT: s_and_b64 s[4:5], s[10:11], s[4:5] -; GFX9-NEXT: v_mov_b32_e32 v0, s0 -; GFX9-NEXT: v_mov_b32_e32 v1, s1 +; GFX9-NEXT: s_lshl_b64 s[10:11], s[2:3], s10 +; GFX9-NEXT: s_lshl_b64 s[2:3], s[2:3], s8 +; GFX9-NEXT: s_add_u32 s2, s2, -1 +; GFX9-NEXT: s_addc_u32 s3, s3, -1 +; GFX9-NEXT: s_and_b64 s[2:3], s[4:5], s[2:3] +; GFX9-NEXT: s_add_u32 s4, s10, -1 +; GFX9-NEXT: s_addc_u32 s5, s11, -1 +; GFX9-NEXT: s_and_b64 s[4:5], s[6:7], s[4:5] +; GFX9-NEXT: v_mov_b32_e32 v0, s2 +; GFX9-NEXT: v_mov_b32_e32 v1, s3 ; GFX9-NEXT: v_mov_b32_e32 v2, s4 ; GFX9-NEXT: v_mov_b32_e32 v3, s5 -; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] +; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; GFX9-NEXT: s_endpgm %shl.y = shl <2 x i64> , %y %r = urem <2 x i64> %x, %shl.y @@ -9091,42 +9076,42 @@ ; ; GFX6-LABEL: sdiv_v2i64_pow2_shl_denom: ; GFX6: ; %bb.0: -; GFX6-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x11 -; GFX6-NEXT: s_mov_b64 s[2:3], 0x1000 +; GFX6-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0xd +; GFX6-NEXT: s_mov_b64 s[12:13], 0x1000 ; GFX6-NEXT: s_mov_b32 s18, 0x4f800000 ; GFX6-NEXT: s_mov_b32 s19, 0x5f7ffffc ; GFX6-NEXT: s_mov_b32 s20, 0x2f800000 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_lshl_b64 s[8:9], s[2:3], s6 -; GFX6-NEXT: s_lshl_b64 s[2:3], s[2:3], s4 -; GFX6-NEXT: s_ashr_i32 s12, s3, 31 -; GFX6-NEXT: s_add_u32 s2, s2, s12 -; GFX6-NEXT: s_mov_b32 s13, s12 -; GFX6-NEXT: s_addc_u32 s3, s3, s12 -; GFX6-NEXT: s_xor_b64 s[10:11], s[2:3], s[12:13] -; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s10 -; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s11 +; GFX6-NEXT: s_lshl_b64 s[8:9], s[12:13], s8 +; GFX6-NEXT: s_lshl_b64 s[2:3], s[12:13], s10 +; GFX6-NEXT: s_ashr_i32 s14, s9, 31 +; GFX6-NEXT: s_add_u32 s8, s8, s14 +; GFX6-NEXT: s_mov_b32 s15, s14 +; GFX6-NEXT: s_addc_u32 s9, s9, s14 +; GFX6-NEXT: s_xor_b64 s[12:13], s[8:9], s[14:15] +; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s12 +; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s13 ; GFX6-NEXT: s_mov_b32 s21, 0xcf800000 -; GFX6-NEXT: s_sub_u32 s6, 0, s10 -; GFX6-NEXT: s_subb_u32 s7, 0, s11 +; GFX6-NEXT: s_sub_u32 s10, 0, s12 +; GFX6-NEXT: s_subb_u32 s11, 0, s13 ; GFX6-NEXT: v_mac_f32_e32 v0, s18, v1 ; GFX6-NEXT: v_rcp_f32_e32 v0, v0 -; GFX6-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 -; GFX6-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0xd +; GFX6-NEXT: s_ashr_i32 s16, s5, 31 +; GFX6-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x9 +; GFX6-NEXT: s_add_u32 s0, s4, s16 ; GFX6-NEXT: v_mul_f32_e32 v0, s19, v0 ; GFX6-NEXT: v_mul_f32_e32 v1, s20, v0 ; GFX6-NEXT: v_trunc_f32_e32 v1, v1 ; GFX6-NEXT: v_mac_f32_e32 v0, s21, v1 ; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1 ; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v0 -; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_ashr_i32 s14, s1, 31 -; GFX6-NEXT: s_add_u32 s0, s0, s14 -; GFX6-NEXT: v_mul_lo_u32 v0, s6, v1 -; GFX6-NEXT: v_mul_hi_u32 v3, s6, v2 -; GFX6-NEXT: v_mul_lo_u32 v4, s7, v2 -; GFX6-NEXT: v_mul_lo_u32 v5, s6, v2 -; GFX6-NEXT: s_mov_b32 s15, s14 +; GFX6-NEXT: s_mov_b32 s17, s16 +; GFX6-NEXT: s_addc_u32 s1, s5, s16 +; GFX6-NEXT: v_mul_lo_u32 v0, s10, v1 +; GFX6-NEXT: v_mul_hi_u32 v3, s10, v2 +; GFX6-NEXT: v_mul_lo_u32 v4, s11, v2 +; GFX6-NEXT: v_mul_lo_u32 v5, s10, v2 +; GFX6-NEXT: s_xor_b64 s[4:5], s[0:1], s[16:17] ; GFX6-NEXT: v_add_i32_e32 v0, vcc, v3, v0 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, v0, v4 ; GFX6-NEXT: v_mul_lo_u32 v0, v2, v3 @@ -9138,8 +9123,7 @@ ; GFX6-NEXT: v_addc_u32_e32 v4, vcc, 0, v6, vcc ; GFX6-NEXT: v_mul_lo_u32 v6, v1, v5 ; GFX6-NEXT: v_mul_hi_u32 v5, v1, v5 -; GFX6-NEXT: s_addc_u32 s1, s1, s14 -; GFX6-NEXT: s_xor_b64 s[16:17], s[0:1], s[14:15] +; GFX6-NEXT: s_xor_b64 s[14:15], s[16:17], s[14:15] ; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v6 ; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v4, v5, vcc ; GFX6-NEXT: v_mov_b32_e32 v0, 0 @@ -9148,13 +9132,12 @@ ; GFX6-NEXT: v_addc_u32_e32 v4, vcc, 0, v5, vcc ; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v3 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v4, vcc -; GFX6-NEXT: v_mul_lo_u32 v3, s6, v1 -; GFX6-NEXT: v_mul_hi_u32 v4, s6, v2 -; GFX6-NEXT: v_mul_lo_u32 v5, s7, v2 -; GFX6-NEXT: s_xor_b64 s[14:15], s[14:15], s[12:13] -; GFX6-NEXT: s_ashr_i32 s12, s9, 31 +; GFX6-NEXT: v_mul_lo_u32 v3, s10, v1 +; GFX6-NEXT: v_mul_hi_u32 v4, s10, v2 +; GFX6-NEXT: v_mul_lo_u32 v5, s11, v2 +; GFX6-NEXT: s_mov_b32 s11, 0xf000 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, v4, v3 -; GFX6-NEXT: v_mul_lo_u32 v4, s6, v2 +; GFX6-NEXT: v_mul_lo_u32 v4, s10, v2 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, v5, v3 ; GFX6-NEXT: v_mul_lo_u32 v7, v2, v3 ; GFX6-NEXT: v_mul_hi_u32 v8, v2, v4 @@ -9172,57 +9155,59 @@ ; GFX6-NEXT: v_addc_u32_e32 v4, vcc, 0, v5, vcc ; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v3 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v4, vcc -; GFX6-NEXT: v_mul_lo_u32 v3, s16, v1 -; GFX6-NEXT: v_mul_hi_u32 v4, s16, v2 -; GFX6-NEXT: v_mul_hi_u32 v5, s16, v1 -; GFX6-NEXT: v_mul_hi_u32 v6, s17, v1 -; GFX6-NEXT: v_mul_lo_u32 v1, s17, v1 +; GFX6-NEXT: v_mul_lo_u32 v3, s4, v1 +; GFX6-NEXT: v_mul_hi_u32 v4, s4, v2 +; GFX6-NEXT: v_mul_hi_u32 v5, s4, v1 +; GFX6-NEXT: v_mul_hi_u32 v6, s5, v1 +; GFX6-NEXT: v_mul_lo_u32 v1, s5, v1 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, v4, v3 ; GFX6-NEXT: v_addc_u32_e32 v4, vcc, 0, v5, vcc -; GFX6-NEXT: v_mul_lo_u32 v5, s17, v2 -; GFX6-NEXT: v_mul_hi_u32 v2, s17, v2 -; GFX6-NEXT: s_add_u32 s8, s8, s12 -; GFX6-NEXT: s_mov_b32 s13, s12 +; GFX6-NEXT: v_mul_lo_u32 v5, s5, v2 +; GFX6-NEXT: v_mul_hi_u32 v2, s5, v2 +; GFX6-NEXT: s_mov_b32 s10, -1 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v5 ; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v4, v2, vcc ; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v6, v0, vcc ; GFX6-NEXT: v_add_i32_e32 v1, vcc, v2, v1 ; GFX6-NEXT: v_addc_u32_e32 v2, vcc, 0, v3, vcc -; GFX6-NEXT: v_mul_lo_u32 v3, s10, v2 -; GFX6-NEXT: v_mul_hi_u32 v4, s10, v1 -; GFX6-NEXT: v_mul_lo_u32 v5, s11, v1 -; GFX6-NEXT: v_mov_b32_e32 v6, s11 -; GFX6-NEXT: s_addc_u32 s9, s9, s12 +; GFX6-NEXT: v_mul_lo_u32 v3, s12, v2 +; GFX6-NEXT: v_mul_hi_u32 v4, s12, v1 +; GFX6-NEXT: v_mul_lo_u32 v5, s13, v1 +; GFX6-NEXT: v_mov_b32_e32 v6, s13 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, v4, v3 -; GFX6-NEXT: v_mul_lo_u32 v4, s10, v1 +; GFX6-NEXT: v_mul_lo_u32 v4, s12, v1 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v5 -; GFX6-NEXT: v_sub_i32_e32 v5, vcc, s17, v3 -; GFX6-NEXT: v_sub_i32_e32 v4, vcc, s16, v4 +; GFX6-NEXT: v_sub_i32_e32 v5, vcc, s5, v3 +; GFX6-NEXT: v_sub_i32_e32 v4, vcc, s4, v4 ; GFX6-NEXT: v_subb_u32_e64 v5, s[0:1], v5, v6, vcc -; GFX6-NEXT: v_subrev_i32_e64 v6, s[0:1], s10, v4 +; GFX6-NEXT: v_subrev_i32_e64 v6, s[0:1], s12, v4 ; GFX6-NEXT: v_subbrev_u32_e64 v5, s[0:1], 0, v5, s[0:1] -; GFX6-NEXT: v_cmp_le_u32_e64 s[0:1], s11, v5 +; GFX6-NEXT: v_cmp_le_u32_e64 s[0:1], s13, v5 ; GFX6-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[0:1] -; GFX6-NEXT: v_cmp_le_u32_e64 s[0:1], s10, v6 +; GFX6-NEXT: v_cmp_le_u32_e64 s[0:1], s12, v6 ; GFX6-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[0:1] -; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], s11, v5 +; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], s13, v5 ; GFX6-NEXT: v_cndmask_b32_e64 v5, v7, v6, s[0:1] ; GFX6-NEXT: v_add_i32_e64 v6, s[0:1], 2, v1 ; GFX6-NEXT: v_addc_u32_e64 v7, s[0:1], 0, v2, s[0:1] ; GFX6-NEXT: v_add_i32_e64 v8, s[0:1], 1, v1 ; GFX6-NEXT: v_addc_u32_e64 v9, s[0:1], 0, v2, s[0:1] +; GFX6-NEXT: s_ashr_i32 s4, s3, 31 ; GFX6-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v5 -; GFX6-NEXT: s_xor_b64 s[8:9], s[8:9], s[12:13] +; GFX6-NEXT: s_add_u32 s2, s2, s4 ; GFX6-NEXT: v_cndmask_b32_e64 v5, v9, v7, s[0:1] -; GFX6-NEXT: v_mov_b32_e32 v7, s17 -; GFX6-NEXT: v_cvt_f32_u32_e32 v9, s8 -; GFX6-NEXT: v_cvt_f32_u32_e32 v10, s9 +; GFX6-NEXT: v_mov_b32_e32 v7, s5 +; GFX6-NEXT: s_mov_b32 s5, s4 +; GFX6-NEXT: s_addc_u32 s3, s3, s4 +; GFX6-NEXT: s_xor_b64 s[2:3], s[2:3], s[4:5] +; GFX6-NEXT: v_cvt_f32_u32_e32 v9, s2 +; GFX6-NEXT: v_cvt_f32_u32_e32 v10, s3 ; GFX6-NEXT: v_subb_u32_e32 v3, vcc, v7, v3, vcc -; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s11, v3 +; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s13, v3 ; GFX6-NEXT: v_cndmask_b32_e64 v7, 0, -1, vcc -; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s10, v4 +; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s12, v4 ; GFX6-NEXT: v_cndmask_b32_e64 v4, 0, -1, vcc -; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s11, v3 +; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s13, v3 ; GFX6-NEXT: v_mac_f32_e32 v9, s18, v10 ; GFX6-NEXT: v_cndmask_b32_e32 v3, v7, v4, vcc ; GFX6-NEXT: v_rcp_f32_e32 v4, v9 @@ -9235,13 +9220,13 @@ ; GFX6-NEXT: v_mac_f32_e32 v4, s21, v5 ; GFX6-NEXT: v_cvt_u32_f32_e32 v4, v4 ; GFX6-NEXT: v_cvt_u32_f32_e32 v5, v5 -; GFX6-NEXT: s_sub_u32 s0, 0, s8 +; GFX6-NEXT: s_sub_u32 s0, 0, s2 ; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc ; GFX6-NEXT: v_mul_hi_u32 v3, s0, v4 ; GFX6-NEXT: v_mul_lo_u32 v6, s0, v5 -; GFX6-NEXT: s_subb_u32 s1, 0, s9 +; GFX6-NEXT: s_subb_u32 s1, 0, s3 ; GFX6-NEXT: v_mul_lo_u32 v7, s1, v4 -; GFX6-NEXT: s_ashr_i32 s10, s3, 31 +; GFX6-NEXT: s_ashr_i32 s12, s7, 31 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v6 ; GFX6-NEXT: v_mul_lo_u32 v6, s0, v4 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v7 @@ -9254,7 +9239,7 @@ ; GFX6-NEXT: v_addc_u32_e32 v8, vcc, 0, v9, vcc ; GFX6-NEXT: v_mul_lo_u32 v9, v5, v6 ; GFX6-NEXT: v_mul_hi_u32 v6, v5, v6 -; GFX6-NEXT: s_mov_b32 s11, s10 +; GFX6-NEXT: s_mov_b32 s13, s12 ; GFX6-NEXT: v_xor_b32_e32 v1, s14, v1 ; GFX6-NEXT: v_add_i32_e32 v7, vcc, v7, v9 ; GFX6-NEXT: v_addc_u32_e32 v6, vcc, v8, v6, vcc @@ -9267,7 +9252,6 @@ ; GFX6-NEXT: v_mul_hi_u32 v6, s0, v3 ; GFX6-NEXT: v_mul_lo_u32 v7, s1, v3 ; GFX6-NEXT: v_xor_b32_e32 v2, s15, v2 -; GFX6-NEXT: s_mov_b32 s7, 0xf000 ; GFX6-NEXT: v_add_i32_e32 v5, vcc, v6, v5 ; GFX6-NEXT: v_mul_lo_u32 v6, s0, v3 ; GFX6-NEXT: v_add_i32_e32 v5, vcc, v7, v5 @@ -9285,46 +9269,45 @@ ; GFX6-NEXT: v_addc_u32_e32 v7, vcc, v7, v0, vcc ; GFX6-NEXT: v_add_i32_e32 v5, vcc, v6, v5 ; GFX6-NEXT: v_addc_u32_e32 v6, vcc, 0, v7, vcc -; GFX6-NEXT: s_add_u32 s0, s2, s10 +; GFX6-NEXT: s_add_u32 s0, s6, s12 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v5 -; GFX6-NEXT: s_addc_u32 s1, s3, s10 +; GFX6-NEXT: s_addc_u32 s1, s7, s12 ; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v4, v6, vcc -; GFX6-NEXT: s_xor_b64 s[2:3], s[0:1], s[10:11] -; GFX6-NEXT: v_mul_lo_u32 v5, s2, v4 -; GFX6-NEXT: v_mul_hi_u32 v6, s2, v3 -; GFX6-NEXT: v_mul_hi_u32 v8, s2, v4 -; GFX6-NEXT: v_mul_hi_u32 v9, s3, v4 -; GFX6-NEXT: v_mul_lo_u32 v4, s3, v4 +; GFX6-NEXT: s_xor_b64 s[6:7], s[0:1], s[12:13] +; GFX6-NEXT: v_mul_lo_u32 v5, s6, v4 +; GFX6-NEXT: v_mul_hi_u32 v6, s6, v3 +; GFX6-NEXT: v_mul_hi_u32 v8, s6, v4 +; GFX6-NEXT: v_mul_hi_u32 v9, s7, v4 +; GFX6-NEXT: v_mul_lo_u32 v4, s7, v4 ; GFX6-NEXT: v_add_i32_e32 v5, vcc, v6, v5 ; GFX6-NEXT: v_addc_u32_e32 v6, vcc, 0, v8, vcc -; GFX6-NEXT: v_mul_lo_u32 v8, s3, v3 -; GFX6-NEXT: v_mul_hi_u32 v3, s3, v3 +; GFX6-NEXT: v_mul_lo_u32 v8, s7, v3 +; GFX6-NEXT: v_mul_hi_u32 v3, s7, v3 ; GFX6-NEXT: v_mov_b32_e32 v7, s15 -; GFX6-NEXT: s_mov_b32 s6, -1 ; GFX6-NEXT: v_add_i32_e32 v5, vcc, v5, v8 ; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v6, v3, vcc ; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v9, v0, vcc ; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v4 ; GFX6-NEXT: v_addc_u32_e32 v4, vcc, 0, v0, vcc -; GFX6-NEXT: v_mul_lo_u32 v5, s8, v4 -; GFX6-NEXT: v_mul_hi_u32 v6, s8, v3 +; GFX6-NEXT: v_mul_lo_u32 v5, s2, v4 +; GFX6-NEXT: v_mul_hi_u32 v6, s2, v3 ; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s14, v1 ; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v2, v7, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, s9, v3 +; GFX6-NEXT: v_mul_lo_u32 v2, s3, v3 ; GFX6-NEXT: v_add_i32_e32 v5, vcc, v6, v5 -; GFX6-NEXT: v_mov_b32_e32 v7, s9 +; GFX6-NEXT: v_mov_b32_e32 v7, s3 ; GFX6-NEXT: v_add_i32_e32 v2, vcc, v5, v2 -; GFX6-NEXT: v_mul_lo_u32 v5, s8, v3 -; GFX6-NEXT: v_sub_i32_e32 v6, vcc, s3, v2 -; GFX6-NEXT: v_sub_i32_e32 v5, vcc, s2, v5 +; GFX6-NEXT: v_mul_lo_u32 v5, s2, v3 +; GFX6-NEXT: v_sub_i32_e32 v6, vcc, s7, v2 +; GFX6-NEXT: v_sub_i32_e32 v5, vcc, s6, v5 ; GFX6-NEXT: v_subb_u32_e64 v6, s[0:1], v6, v7, vcc -; GFX6-NEXT: v_subrev_i32_e64 v7, s[0:1], s8, v5 +; GFX6-NEXT: v_subrev_i32_e64 v7, s[0:1], s2, v5 ; GFX6-NEXT: v_subbrev_u32_e64 v6, s[0:1], 0, v6, s[0:1] -; GFX6-NEXT: v_cmp_le_u32_e64 s[0:1], s9, v6 +; GFX6-NEXT: v_cmp_le_u32_e64 s[0:1], s3, v6 ; GFX6-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[0:1] -; GFX6-NEXT: v_cmp_le_u32_e64 s[0:1], s8, v7 +; GFX6-NEXT: v_cmp_le_u32_e64 s[0:1], s2, v7 ; GFX6-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[0:1] -; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], s9, v6 +; GFX6-NEXT: v_cmp_eq_u32_e64 s[0:1], s3, v6 ; GFX6-NEXT: v_cndmask_b32_e64 v6, v8, v7, s[0:1] ; GFX6-NEXT: v_add_i32_e64 v7, s[0:1], 2, v3 ; GFX6-NEXT: v_addc_u32_e64 v8, s[0:1], 0, v4, s[0:1] @@ -9332,59 +9315,58 @@ ; GFX6-NEXT: v_addc_u32_e64 v10, s[0:1], 0, v4, s[0:1] ; GFX6-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v6 ; GFX6-NEXT: v_cndmask_b32_e64 v6, v10, v8, s[0:1] -; GFX6-NEXT: v_mov_b32_e32 v8, s3 +; GFX6-NEXT: v_mov_b32_e32 v8, s7 ; GFX6-NEXT: v_subb_u32_e32 v2, vcc, v8, v2, vcc -; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s9, v2 +; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s3, v2 ; GFX6-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc -; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s8, v5 +; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s2, v5 ; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, -1, vcc -; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s9, v2 +; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s3, v2 ; GFX6-NEXT: v_cndmask_b32_e32 v2, v8, v5, vcc ; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v2 ; GFX6-NEXT: v_cndmask_b32_e32 v2, v4, v6, vcc ; GFX6-NEXT: v_cndmask_b32_e64 v4, v9, v7, s[0:1] ; GFX6-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc -; GFX6-NEXT: s_xor_b64 s[0:1], s[10:11], s[12:13] +; GFX6-NEXT: s_xor_b64 s[0:1], s[12:13], s[4:5] ; GFX6-NEXT: v_xor_b32_e32 v3, s0, v3 ; GFX6-NEXT: v_xor_b32_e32 v4, s1, v2 ; GFX6-NEXT: v_mov_b32_e32 v5, s1 ; GFX6-NEXT: v_subrev_i32_e32 v2, vcc, s0, v3 ; GFX6-NEXT: v_subb_u32_e32 v3, vcc, v4, v5, vcc -; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 +; GFX6-NEXT: s_waitcnt lgkmcnt(0) +; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0 ; GFX6-NEXT: s_endpgm ; ; GFX9-LABEL: sdiv_v2i64_pow2_shl_denom: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x44 +; GFX9-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x34 ; GFX9-NEXT: s_mov_b64 s[2:3], 0x1000 ; GFX9-NEXT: s_mov_b32 s16, 0x4f800000 ; GFX9-NEXT: s_mov_b32 s17, 0x5f7ffffc ; GFX9-NEXT: s_mov_b32 s18, 0x2f800000 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_lshl_b64 s[8:9], s[2:3], s6 -; GFX9-NEXT: s_lshl_b64 s[2:3], s[2:3], s4 +; GFX9-NEXT: s_lshl_b64 s[10:11], s[2:3], s10 +; GFX9-NEXT: s_lshl_b64 s[2:3], s[2:3], s8 ; GFX9-NEXT: s_ashr_i32 s12, s3, 31 ; GFX9-NEXT: s_add_u32 s2, s2, s12 ; GFX9-NEXT: s_mov_b32 s13, s12 ; GFX9-NEXT: s_addc_u32 s3, s3, s12 -; GFX9-NEXT: s_xor_b64 s[10:11], s[2:3], s[12:13] -; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s10 -; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s11 +; GFX9-NEXT: s_xor_b64 s[8:9], s[2:3], s[12:13] +; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s8 +; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s9 ; GFX9-NEXT: s_mov_b32 s19, 0xcf800000 -; GFX9-NEXT: s_sub_u32 s2, 0, s10 -; GFX9-NEXT: s_subb_u32 s3, 0, s11 +; GFX9-NEXT: s_sub_u32 s2, 0, s8 +; GFX9-NEXT: s_subb_u32 s3, 0, s9 ; GFX9-NEXT: v_mac_f32_e32 v0, s16, v1 ; GFX9-NEXT: v_rcp_f32_e32 v0, v0 -; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x34 +; GFX9-NEXT: s_ashr_i32 s14, s5, 31 +; GFX9-NEXT: s_mov_b32 s15, s14 ; GFX9-NEXT: v_mul_f32_e32 v0, s17, v0 ; GFX9-NEXT: v_mul_f32_e32 v1, s18, v0 ; GFX9-NEXT: v_trunc_f32_e32 v1, v1 ; GFX9-NEXT: v_mac_f32_e32 v0, s19, v1 ; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1 ; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v0 -; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_ashr_i32 s14, s5, 31 -; GFX9-NEXT: s_mov_b32 s15, s14 ; GFX9-NEXT: v_mul_lo_u32 v0, s2, v1 ; GFX9-NEXT: v_mul_hi_u32 v3, s2, v2 ; GFX9-NEXT: v_mul_lo_u32 v5, s3, v2 @@ -9449,44 +9431,44 @@ ; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v6, v0, vcc ; GFX9-NEXT: v_add_co_u32_e32 v1, vcc, v2, v1 ; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v3, vcc -; GFX9-NEXT: v_mul_lo_u32 v3, s10, v2 -; GFX9-NEXT: v_mul_hi_u32 v4, s10, v1 -; GFX9-NEXT: v_mul_lo_u32 v5, s11, v1 -; GFX9-NEXT: v_mov_b32_e32 v6, s11 +; GFX9-NEXT: v_mul_lo_u32 v3, s8, v2 +; GFX9-NEXT: v_mul_hi_u32 v4, s8, v1 +; GFX9-NEXT: v_mul_lo_u32 v5, s9, v1 +; GFX9-NEXT: v_mov_b32_e32 v6, s9 ; GFX9-NEXT: v_add_u32_e32 v3, v4, v3 -; GFX9-NEXT: v_mul_lo_u32 v4, s10, v1 +; GFX9-NEXT: v_mul_lo_u32 v4, s8, v1 ; GFX9-NEXT: v_add_u32_e32 v3, v3, v5 ; GFX9-NEXT: v_sub_u32_e32 v5, s5, v3 ; GFX9-NEXT: v_sub_co_u32_e32 v4, vcc, s4, v4 ; GFX9-NEXT: v_subb_co_u32_e64 v5, s[0:1], v5, v6, vcc -; GFX9-NEXT: v_subrev_co_u32_e64 v6, s[0:1], s10, v4 +; GFX9-NEXT: v_subrev_co_u32_e64 v6, s[0:1], s8, v4 ; GFX9-NEXT: v_subbrev_co_u32_e64 v5, s[0:1], 0, v5, s[0:1] -; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s11, v5 +; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s9, v5 ; GFX9-NEXT: v_cndmask_b32_e64 v7, 0, -1, s[0:1] -; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s10, v6 +; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s8, v6 ; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, -1, s[0:1] -; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s11, v5 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s9, v5 ; GFX9-NEXT: v_cndmask_b32_e64 v5, v7, v6, s[0:1] ; GFX9-NEXT: v_add_co_u32_e64 v6, s[0:1], 2, v1 ; GFX9-NEXT: v_addc_co_u32_e64 v7, s[0:1], 0, v2, s[0:1] ; GFX9-NEXT: v_add_co_u32_e64 v8, s[0:1], 1, v1 ; GFX9-NEXT: v_addc_co_u32_e64 v9, s[0:1], 0, v2, s[0:1] -; GFX9-NEXT: s_ashr_i32 s4, s9, 31 +; GFX9-NEXT: s_ashr_i32 s4, s11, 31 ; GFX9-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v5 -; GFX9-NEXT: s_add_u32 s8, s8, s4 +; GFX9-NEXT: s_add_u32 s10, s10, s4 ; GFX9-NEXT: v_cndmask_b32_e64 v5, v9, v7, s[0:1] ; GFX9-NEXT: v_mov_b32_e32 v7, s5 ; GFX9-NEXT: s_mov_b32 s5, s4 -; GFX9-NEXT: s_addc_u32 s9, s9, s4 -; GFX9-NEXT: s_xor_b64 s[8:9], s[8:9], s[4:5] -; GFX9-NEXT: v_cvt_f32_u32_e32 v9, s8 -; GFX9-NEXT: v_cvt_f32_u32_e32 v10, s9 +; GFX9-NEXT: s_addc_u32 s11, s11, s4 +; GFX9-NEXT: s_xor_b64 s[10:11], s[10:11], s[4:5] +; GFX9-NEXT: v_cvt_f32_u32_e32 v9, s10 +; GFX9-NEXT: v_cvt_f32_u32_e32 v10, s11 ; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v7, v3, vcc -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s11, v3 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s9, v3 ; GFX9-NEXT: v_cndmask_b32_e64 v7, 0, -1, vcc -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s10, v4 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s8, v4 ; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, -1, vcc -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, s11, v3 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, s9, v3 ; GFX9-NEXT: v_mac_f32_e32 v9, s16, v10 ; GFX9-NEXT: v_cndmask_b32_e32 v3, v7, v4, vcc ; GFX9-NEXT: v_rcp_f32_e32 v4, v9 @@ -9499,8 +9481,8 @@ ; GFX9-NEXT: v_mac_f32_e32 v4, s19, v5 ; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v4 ; GFX9-NEXT: v_cvt_u32_f32_e32 v5, v5 -; GFX9-NEXT: s_sub_u32 s0, 0, s8 -; GFX9-NEXT: s_subb_u32 s1, 0, s9 +; GFX9-NEXT: s_sub_u32 s0, 0, s10 +; GFX9-NEXT: s_subb_u32 s1, 0, s11 ; GFX9-NEXT: v_mul_hi_u32 v6, s0, v4 ; GFX9-NEXT: v_mul_lo_u32 v7, s0, v5 ; GFX9-NEXT: v_mul_lo_u32 v8, s1, v4 @@ -9517,8 +9499,8 @@ ; GFX9-NEXT: v_addc_co_u32_e32 v8, vcc, 0, v9, vcc ; GFX9-NEXT: v_mul_lo_u32 v9, v5, v3 ; GFX9-NEXT: v_mul_hi_u32 v3, v5, v3 -; GFX9-NEXT: s_ashr_i32 s10, s7, 31 -; GFX9-NEXT: s_mov_b32 s11, s10 +; GFX9-NEXT: s_ashr_i32 s8, s7, 31 +; GFX9-NEXT: s_mov_b32 s9, s8 ; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, v7, v9 ; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, v8, v3, vcc ; GFX9-NEXT: v_addc_co_u32_e32 v7, vcc, v10, v0, vcc @@ -9530,7 +9512,7 @@ ; GFX9-NEXT: v_mul_hi_u32 v6, s0, v3 ; GFX9-NEXT: v_mul_lo_u32 v7, s1, v3 ; GFX9-NEXT: v_mul_lo_u32 v8, s0, v3 -; GFX9-NEXT: s_add_u32 s0, s6, s10 +; GFX9-NEXT: s_add_u32 s0, s6, s8 ; GFX9-NEXT: v_add_u32_e32 v5, v6, v5 ; GFX9-NEXT: v_add_u32_e32 v5, v5, v7 ; GFX9-NEXT: v_mul_lo_u32 v9, v3, v5 @@ -9548,9 +9530,9 @@ ; GFX9-NEXT: v_add_co_u32_e32 v5, vcc, v7, v5 ; GFX9-NEXT: v_addc_co_u32_e32 v6, vcc, 0, v6, vcc ; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, v3, v5 -; GFX9-NEXT: s_addc_u32 s1, s7, s10 +; GFX9-NEXT: s_addc_u32 s1, s7, s8 ; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v4, v6, vcc -; GFX9-NEXT: s_xor_b64 s[6:7], s[0:1], s[10:11] +; GFX9-NEXT: s_xor_b64 s[6:7], s[0:1], s[8:9] ; GFX9-NEXT: v_mul_lo_u32 v5, s6, v4 ; GFX9-NEXT: v_mul_hi_u32 v6, s6, v3 ; GFX9-NEXT: v_mul_hi_u32 v8, s6, v4 @@ -9567,26 +9549,26 @@ ; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v9, v0, vcc ; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, v3, v4 ; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v5, vcc -; GFX9-NEXT: v_mul_lo_u32 v5, s8, v4 -; GFX9-NEXT: v_mul_hi_u32 v6, s8, v3 -; GFX9-NEXT: v_mul_lo_u32 v8, s9, v3 +; GFX9-NEXT: v_mul_lo_u32 v5, s10, v4 +; GFX9-NEXT: v_mul_hi_u32 v6, s10, v3 +; GFX9-NEXT: v_mul_lo_u32 v8, s11, v3 ; GFX9-NEXT: v_mov_b32_e32 v7, s13 ; GFX9-NEXT: v_subrev_co_u32_e32 v1, vcc, s12, v1 ; GFX9-NEXT: v_add_u32_e32 v5, v6, v5 -; GFX9-NEXT: v_mul_lo_u32 v6, s8, v3 +; GFX9-NEXT: v_mul_lo_u32 v6, s10, v3 ; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v7, vcc ; GFX9-NEXT: v_add_u32_e32 v5, v5, v8 ; GFX9-NEXT: v_sub_u32_e32 v7, s7, v5 -; GFX9-NEXT: v_mov_b32_e32 v8, s9 +; GFX9-NEXT: v_mov_b32_e32 v8, s11 ; GFX9-NEXT: v_sub_co_u32_e32 v6, vcc, s6, v6 ; GFX9-NEXT: v_subb_co_u32_e64 v7, s[0:1], v7, v8, vcc -; GFX9-NEXT: v_subrev_co_u32_e64 v8, s[0:1], s8, v6 +; GFX9-NEXT: v_subrev_co_u32_e64 v8, s[0:1], s10, v6 ; GFX9-NEXT: v_subbrev_co_u32_e64 v7, s[0:1], 0, v7, s[0:1] -; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s9, v7 +; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s11, v7 ; GFX9-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[0:1] -; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s8, v8 +; GFX9-NEXT: v_cmp_le_u32_e64 s[0:1], s10, v8 ; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[0:1] -; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s9, v7 +; GFX9-NEXT: v_cmp_eq_u32_e64 s[0:1], s11, v7 ; GFX9-NEXT: v_cndmask_b32_e64 v7, v9, v8, s[0:1] ; GFX9-NEXT: v_add_co_u32_e64 v8, s[0:1], 2, v3 ; GFX9-NEXT: v_addc_co_u32_e64 v9, s[0:1], 0, v4, s[0:1] @@ -9596,16 +9578,16 @@ ; GFX9-NEXT: v_cndmask_b32_e64 v7, v11, v9, s[0:1] ; GFX9-NEXT: v_mov_b32_e32 v9, s7 ; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, v9, v5, vcc -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s9, v5 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s11, v5 ; GFX9-NEXT: v_cndmask_b32_e64 v9, 0, -1, vcc -; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s8, v6 +; GFX9-NEXT: v_cmp_le_u32_e32 vcc, s10, v6 ; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, -1, vcc -; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, s9, v5 +; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, s11, v5 ; GFX9-NEXT: v_cndmask_b32_e32 v5, v9, v6, vcc ; GFX9-NEXT: v_cmp_ne_u32_e32 vcc, 0, v5 ; GFX9-NEXT: v_cndmask_b32_e64 v5, v10, v8, s[0:1] ; GFX9-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc -; GFX9-NEXT: s_xor_b64 s[0:1], s[10:11], s[4:5] +; GFX9-NEXT: s_xor_b64 s[0:1], s[8:9], s[4:5] ; GFX9-NEXT: v_cndmask_b32_e32 v4, v4, v7, vcc ; GFX9-NEXT: v_xor_b32_e32 v3, s0, v3 ; GFX9-NEXT: v_xor_b32_e32 v4, s1, v4 @@ -10272,19 +10254,19 @@ ; ; GFX6-LABEL: srem_v2i64_pow2_shl_denom: ; GFX6: ; %bb.0: -; GFX6-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x11 +; GFX6-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0xd ; GFX6-NEXT: s_mov_b64 s[2:3], 0x1000 ; GFX6-NEXT: s_mov_b32 s18, 0x4f800000 ; GFX6-NEXT: s_mov_b32 s19, 0x5f7ffffc ; GFX6-NEXT: s_mov_b32 s20, 0x2f800000 ; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_lshl_b64 s[14:15], s[2:3], s6 -; GFX6-NEXT: s_lshl_b64 s[2:3], s[2:3], s4 -; GFX6-NEXT: s_ashr_i32 s4, s3, 31 -; GFX6-NEXT: s_add_u32 s2, s2, s4 -; GFX6-NEXT: s_mov_b32 s5, s4 -; GFX6-NEXT: s_addc_u32 s3, s3, s4 -; GFX6-NEXT: s_xor_b64 s[16:17], s[2:3], s[4:5] +; GFX6-NEXT: s_lshl_b64 s[14:15], s[2:3], s10 +; GFX6-NEXT: s_lshl_b64 s[2:3], s[2:3], s8 +; GFX6-NEXT: s_ashr_i32 s8, s3, 31 +; GFX6-NEXT: s_add_u32 s2, s2, s8 +; GFX6-NEXT: s_mov_b32 s9, s8 +; GFX6-NEXT: s_addc_u32 s3, s3, s8 +; GFX6-NEXT: s_xor_b64 s[16:17], s[2:3], s[8:9] ; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s16 ; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s17 ; GFX6-NEXT: s_mov_b32 s21, 0xcf800000 @@ -10292,24 +10274,22 @@ ; GFX6-NEXT: s_subb_u32 s3, 0, s17 ; GFX6-NEXT: v_mac_f32_e32 v0, s18, v1 ; GFX6-NEXT: v_rcp_f32_e32 v0, v0 -; GFX6-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 -; GFX6-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0xd -; GFX6-NEXT: s_mov_b32 s7, 0xf000 -; GFX6-NEXT: s_mov_b32 s6, -1 +; GFX6-NEXT: s_ashr_i32 s12, s5, 31 +; GFX6-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x9 +; GFX6-NEXT: s_add_u32 s0, s4, s12 ; GFX6-NEXT: v_mul_f32_e32 v0, s19, v0 ; GFX6-NEXT: v_mul_f32_e32 v1, s20, v0 ; GFX6-NEXT: v_trunc_f32_e32 v1, v1 ; GFX6-NEXT: v_mac_f32_e32 v0, s21, v1 ; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1 ; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v0 -; GFX6-NEXT: s_waitcnt lgkmcnt(0) -; GFX6-NEXT: s_ashr_i32 s12, s9, 31 -; GFX6-NEXT: s_add_u32 s0, s8, s12 +; GFX6-NEXT: s_mov_b32 s13, s12 +; GFX6-NEXT: s_addc_u32 s1, s5, s12 ; GFX6-NEXT: v_mul_lo_u32 v0, s2, v1 ; GFX6-NEXT: v_mul_hi_u32 v3, s2, v2 ; GFX6-NEXT: v_mul_lo_u32 v4, s3, v2 ; GFX6-NEXT: v_mul_lo_u32 v5, s2, v2 -; GFX6-NEXT: s_mov_b32 s13, s12 +; GFX6-NEXT: s_xor_b64 s[4:5], s[0:1], s[12:13] ; GFX6-NEXT: v_add_i32_e32 v0, vcc, v3, v0 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, v0, v4 ; GFX6-NEXT: v_mul_lo_u32 v0, v2, v3 @@ -10321,8 +10301,8 @@ ; GFX6-NEXT: v_addc_u32_e32 v4, vcc, 0, v6, vcc ; GFX6-NEXT: v_mul_lo_u32 v6, v1, v5 ; GFX6-NEXT: v_mul_hi_u32 v5, v1, v5 -; GFX6-NEXT: s_addc_u32 s1, s9, s12 -; GFX6-NEXT: s_xor_b64 s[8:9], s[0:1], s[12:13] +; GFX6-NEXT: s_mov_b32 s11, 0xf000 +; GFX6-NEXT: s_mov_b32 s10, -1 ; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v6 ; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v4, v5, vcc ; GFX6-NEXT: v_mov_b32_e32 v0, 0 @@ -10353,15 +10333,15 @@ ; GFX6-NEXT: v_addc_u32_e32 v4, vcc, 0, v5, vcc ; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v3 ; GFX6-NEXT: v_addc_u32_e32 v1, vcc, v1, v4, vcc -; GFX6-NEXT: v_mul_lo_u32 v3, s8, v1 -; GFX6-NEXT: v_mul_hi_u32 v4, s8, v2 -; GFX6-NEXT: v_mul_hi_u32 v5, s8, v1 -; GFX6-NEXT: v_mul_hi_u32 v6, s9, v1 -; GFX6-NEXT: v_mul_lo_u32 v1, s9, v1 +; GFX6-NEXT: v_mul_lo_u32 v3, s4, v1 +; GFX6-NEXT: v_mul_hi_u32 v4, s4, v2 +; GFX6-NEXT: v_mul_hi_u32 v5, s4, v1 +; GFX6-NEXT: v_mul_hi_u32 v6, s5, v1 +; GFX6-NEXT: v_mul_lo_u32 v1, s5, v1 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, v4, v3 ; GFX6-NEXT: v_addc_u32_e32 v4, vcc, 0, v5, vcc -; GFX6-NEXT: v_mul_lo_u32 v5, s9, v2 -; GFX6-NEXT: v_mul_hi_u32 v2, s9, v2 +; GFX6-NEXT: v_mul_lo_u32 v5, s5, v2 +; GFX6-NEXT: v_mul_hi_u32 v2, s5, v2 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v5 ; GFX6-NEXT: v_addc_u32_e32 v2, vcc, v4, v2, vcc ; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v6, v0, vcc @@ -10373,9 +10353,9 @@ ; GFX6-NEXT: v_mul_lo_u32 v1, s16, v1 ; GFX6-NEXT: v_add_i32_e32 v2, vcc, v3, v2 ; GFX6-NEXT: v_add_i32_e32 v2, vcc, v2, v4 -; GFX6-NEXT: v_sub_i32_e32 v3, vcc, s9, v2 +; GFX6-NEXT: v_sub_i32_e32 v3, vcc, s5, v2 ; GFX6-NEXT: v_mov_b32_e32 v4, s17 -; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s8, v1 +; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s4, v1 ; GFX6-NEXT: v_subb_u32_e64 v3, s[0:1], v3, v4, vcc ; GFX6-NEXT: v_subrev_i32_e64 v5, s[0:1], s16, v1 ; GFX6-NEXT: v_subbrev_u32_e64 v6, s[2:3], 0, v3, s[0:1] @@ -10390,14 +10370,14 @@ ; GFX6-NEXT: v_subbrev_u32_e64 v3, s[0:1], 0, v3, s[0:1] ; GFX6-NEXT: s_ashr_i32 s2, s15, 31 ; GFX6-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v7 -; GFX6-NEXT: s_add_u32 s8, s14, s2 +; GFX6-NEXT: s_add_u32 s4, s14, s2 ; GFX6-NEXT: v_cndmask_b32_e64 v3, v6, v3, s[0:1] -; GFX6-NEXT: v_mov_b32_e32 v6, s9 +; GFX6-NEXT: v_mov_b32_e32 v6, s5 ; GFX6-NEXT: s_mov_b32 s3, s2 -; GFX6-NEXT: s_addc_u32 s9, s15, s2 -; GFX6-NEXT: s_xor_b64 s[8:9], s[8:9], s[2:3] -; GFX6-NEXT: v_cvt_f32_u32_e32 v7, s8 -; GFX6-NEXT: v_cvt_f32_u32_e32 v8, s9 +; GFX6-NEXT: s_addc_u32 s5, s15, s2 +; GFX6-NEXT: s_xor_b64 s[4:5], s[4:5], s[2:3] +; GFX6-NEXT: v_cvt_f32_u32_e32 v7, s4 +; GFX6-NEXT: v_cvt_f32_u32_e32 v8, s5 ; GFX6-NEXT: v_subb_u32_e32 v2, vcc, v6, v2, vcc ; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s17, v2 ; GFX6-NEXT: v_mac_f32_e32 v7, s18, v8 @@ -10416,13 +10396,13 @@ ; GFX6-NEXT: v_mac_f32_e32 v4, s21, v5 ; GFX6-NEXT: v_cvt_u32_f32_e32 v4, v4 ; GFX6-NEXT: v_cvt_u32_f32_e32 v5, v5 -; GFX6-NEXT: s_sub_u32 s0, 0, s8 +; GFX6-NEXT: s_sub_u32 s0, 0, s4 ; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc ; GFX6-NEXT: v_mul_hi_u32 v3, s0, v4 ; GFX6-NEXT: v_mul_lo_u32 v6, s0, v5 -; GFX6-NEXT: s_subb_u32 s1, 0, s9 +; GFX6-NEXT: s_subb_u32 s1, 0, s5 ; GFX6-NEXT: v_mul_lo_u32 v7, s1, v4 -; GFX6-NEXT: s_ashr_i32 s14, s11, 31 +; GFX6-NEXT: s_ashr_i32 s14, s7, 31 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v6 ; GFX6-NEXT: v_mul_lo_u32 v6, s0, v4 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v7 @@ -10465,58 +10445,58 @@ ; GFX6-NEXT: v_addc_u32_e32 v7, vcc, v7, v0, vcc ; GFX6-NEXT: v_add_i32_e32 v5, vcc, v6, v5 ; GFX6-NEXT: v_addc_u32_e32 v6, vcc, 0, v7, vcc -; GFX6-NEXT: s_add_u32 s0, s10, s14 +; GFX6-NEXT: s_add_u32 s0, s6, s14 ; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v5 -; GFX6-NEXT: s_addc_u32 s1, s11, s14 +; GFX6-NEXT: s_addc_u32 s1, s7, s14 ; GFX6-NEXT: v_addc_u32_e32 v4, vcc, v4, v6, vcc -; GFX6-NEXT: s_xor_b64 s[10:11], s[0:1], s[14:15] -; GFX6-NEXT: v_mul_lo_u32 v5, s10, v4 -; GFX6-NEXT: v_mul_hi_u32 v6, s10, v3 -; GFX6-NEXT: v_mul_hi_u32 v8, s10, v4 -; GFX6-NEXT: v_mul_hi_u32 v9, s11, v4 -; GFX6-NEXT: v_mul_lo_u32 v4, s11, v4 +; GFX6-NEXT: s_xor_b64 s[6:7], s[0:1], s[14:15] +; GFX6-NEXT: v_mul_lo_u32 v5, s6, v4 +; GFX6-NEXT: v_mul_hi_u32 v6, s6, v3 +; GFX6-NEXT: v_mul_hi_u32 v8, s6, v4 +; GFX6-NEXT: v_mul_hi_u32 v9, s7, v4 +; GFX6-NEXT: v_mul_lo_u32 v4, s7, v4 ; GFX6-NEXT: v_add_i32_e32 v5, vcc, v6, v5 ; GFX6-NEXT: v_addc_u32_e32 v6, vcc, 0, v8, vcc -; GFX6-NEXT: v_mul_lo_u32 v8, s11, v3 -; GFX6-NEXT: v_mul_hi_u32 v3, s11, v3 +; GFX6-NEXT: v_mul_lo_u32 v8, s7, v3 +; GFX6-NEXT: v_mul_hi_u32 v3, s7, v3 ; GFX6-NEXT: v_mov_b32_e32 v7, s12 ; GFX6-NEXT: v_add_i32_e32 v5, vcc, v5, v8 ; GFX6-NEXT: v_addc_u32_e32 v3, vcc, v6, v3, vcc ; GFX6-NEXT: v_addc_u32_e32 v0, vcc, v9, v0, vcc ; GFX6-NEXT: v_add_i32_e32 v3, vcc, v3, v4 ; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v0, vcc -; GFX6-NEXT: v_mul_lo_u32 v4, s8, v0 -; GFX6-NEXT: v_mul_hi_u32 v5, s8, v3 +; GFX6-NEXT: v_mul_lo_u32 v4, s4, v0 +; GFX6-NEXT: v_mul_hi_u32 v5, s4, v3 ; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s12, v1 ; GFX6-NEXT: v_subb_u32_e32 v1, vcc, v2, v7, vcc -; GFX6-NEXT: v_mul_lo_u32 v2, s9, v3 -; GFX6-NEXT: v_mul_lo_u32 v3, s8, v3 +; GFX6-NEXT: v_mul_lo_u32 v2, s5, v3 +; GFX6-NEXT: v_mul_lo_u32 v3, s4, v3 ; GFX6-NEXT: v_add_i32_e32 v4, vcc, v5, v4 ; GFX6-NEXT: v_add_i32_e32 v2, vcc, v4, v2 -; GFX6-NEXT: v_sub_i32_e32 v4, vcc, s11, v2 -; GFX6-NEXT: v_mov_b32_e32 v5, s9 -; GFX6-NEXT: v_sub_i32_e32 v3, vcc, s10, v3 +; GFX6-NEXT: v_sub_i32_e32 v4, vcc, s7, v2 +; GFX6-NEXT: v_mov_b32_e32 v5, s5 +; GFX6-NEXT: v_sub_i32_e32 v3, vcc, s6, v3 ; GFX6-NEXT: v_subb_u32_e64 v4, s[0:1], v4, v5, vcc -; GFX6-NEXT: v_subrev_i32_e64 v6, s[0:1], s8, v3 +; GFX6-NEXT: v_subrev_i32_e64 v6, s[0:1], s4, v3 ; GFX6-NEXT: v_subbrev_u32_e64 v7, s[2:3], 0, v4, s[0:1] -; GFX6-NEXT: v_cmp_le_u32_e64 s[2:3], s9, v7 +; GFX6-NEXT: v_cmp_le_u32_e64 s[2:3], s5, v7 ; GFX6-NEXT: v_cndmask_b32_e64 v8, 0, -1, s[2:3] -; GFX6-NEXT: v_cmp_le_u32_e64 s[2:3], s8, v6 +; GFX6-NEXT: v_cmp_le_u32_e64 s[2:3], s4, v6 ; GFX6-NEXT: v_subb_u32_e64 v4, s[0:1], v4, v5, s[0:1] ; GFX6-NEXT: v_cndmask_b32_e64 v9, 0, -1, s[2:3] -; GFX6-NEXT: v_cmp_eq_u32_e64 s[2:3], s9, v7 -; GFX6-NEXT: v_subrev_i32_e64 v5, s[0:1], s8, v6 +; GFX6-NEXT: v_cmp_eq_u32_e64 s[2:3], s5, v7 +; GFX6-NEXT: v_subrev_i32_e64 v5, s[0:1], s4, v6 ; GFX6-NEXT: v_cndmask_b32_e64 v8, v8, v9, s[2:3] ; GFX6-NEXT: v_subbrev_u32_e64 v4, s[0:1], 0, v4, s[0:1] ; GFX6-NEXT: v_cmp_ne_u32_e64 s[0:1], 0, v8 ; GFX6-NEXT: v_cndmask_b32_e64 v4, v7, v4, s[0:1] -; GFX6-NEXT: v_mov_b32_e32 v7, s11 +; GFX6-NEXT: v_mov_b32_e32 v7, s7 ; GFX6-NEXT: v_subb_u32_e32 v2, vcc, v7, v2, vcc -; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s9, v2 +; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s5, v2 ; GFX6-NEXT: v_cndmask_b32_e64 v7, 0, -1, vcc -; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s8, v3 +; GFX6-NEXT: v_cmp_le_u32_e32 vcc, s4, v3 ; GFX6-NEXT: v_cndmask_b32_e64 v8, 0, -1, vcc -; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s9, v2 +; GFX6-NEXT: v_cmp_eq_u32_e32 vcc, s5, v2 ; GFX6-NEXT: v_cndmask_b32_e32 v7, v7, v8, vcc ; GFX6-NEXT: v_cmp_ne_u32_e32 vcc, 0, v7 ; GFX6-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc @@ -10527,24 +10507,25 @@ ; GFX6-NEXT: v_mov_b32_e32 v5, s14 ; GFX6-NEXT: v_subrev_i32_e32 v2, vcc, s14, v3 ; GFX6-NEXT: v_subb_u32_e32 v3, vcc, v4, v5, vcc -; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 +; GFX6-NEXT: s_waitcnt lgkmcnt(0) +; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0 ; GFX6-NEXT: s_endpgm ; ; GFX9-LABEL: srem_v2i64_pow2_shl_denom: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x44 +; GFX9-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x34 ; GFX9-NEXT: s_mov_b64 s[2:3], 0x1000 ; GFX9-NEXT: s_mov_b32 s16, 0x4f800000 ; GFX9-NEXT: s_mov_b32 s17, 0x5f7ffffc ; GFX9-NEXT: s_mov_b32 s18, 0x2f800000 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_lshl_b64 s[10:11], s[2:3], s6 -; GFX9-NEXT: s_lshl_b64 s[2:3], s[2:3], s4 -; GFX9-NEXT: s_ashr_i32 s4, s3, 31 -; GFX9-NEXT: s_add_u32 s2, s2, s4 -; GFX9-NEXT: s_mov_b32 s5, s4 -; GFX9-NEXT: s_addc_u32 s3, s3, s4 -; GFX9-NEXT: s_xor_b64 s[12:13], s[2:3], s[4:5] +; GFX9-NEXT: s_lshl_b64 s[10:11], s[2:3], s10 +; GFX9-NEXT: s_lshl_b64 s[2:3], s[2:3], s8 +; GFX9-NEXT: s_ashr_i32 s8, s3, 31 +; GFX9-NEXT: s_add_u32 s2, s2, s8 +; GFX9-NEXT: s_mov_b32 s9, s8 +; GFX9-NEXT: s_addc_u32 s3, s3, s8 +; GFX9-NEXT: s_xor_b64 s[12:13], s[2:3], s[8:9] ; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s12 ; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s13 ; GFX9-NEXT: s_mov_b32 s19, 0xcf800000 @@ -10552,16 +10533,14 @@ ; GFX9-NEXT: s_subb_u32 s3, 0, s13 ; GFX9-NEXT: v_mac_f32_e32 v0, s16, v1 ; GFX9-NEXT: v_rcp_f32_e32 v0, v0 -; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x34 +; GFX9-NEXT: s_ashr_i32 s8, s5, 31 +; GFX9-NEXT: s_mov_b32 s9, s8 ; GFX9-NEXT: v_mul_f32_e32 v0, s17, v0 ; GFX9-NEXT: v_mul_f32_e32 v1, s18, v0 ; GFX9-NEXT: v_trunc_f32_e32 v1, v1 ; GFX9-NEXT: v_mac_f32_e32 v0, s19, v1 ; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1 ; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v0 -; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_ashr_i32 s8, s5, 31 -; GFX9-NEXT: s_mov_b32 s9, s8 ; GFX9-NEXT: v_mul_lo_u32 v0, s2, v1 ; GFX9-NEXT: v_mul_hi_u32 v3, s2, v2 ; GFX9-NEXT: v_mul_lo_u32 v5, s3, v2 diff --git a/llvm/test/CodeGen/AMDGPU/ashr.v2i16.ll b/llvm/test/CodeGen/AMDGPU/ashr.v2i16.ll --- a/llvm/test/CodeGen/AMDGPU/ashr.v2i16.ll +++ b/llvm/test/CodeGen/AMDGPU/ashr.v2i16.ll @@ -29,16 +29,15 @@ } ; GCN-LABEL: {{^}}v_ashr_v2i16: -; GCN: {{buffer|flat|global}}_load_dword [[LHS:v[0-9]+]] -; GCN: {{buffer|flat|global}}_load_dword [[RHS:v[0-9]+]] -; GFX9: v_pk_ashrrev_i16 [[RESULT:v[0-9]+]], [[RHS]], [[LHS]] +; GCN: {{buffer|flat|global}}_load_dwordx2 v[[[LHS:[0-9]+]]:[[RHS:[0-9]+]]] +; GFX9: v_pk_ashrrev_i16 [[RESULT:v[0-9]+]], v[[RHS]], v[[LHS]] ; VI: v_ashrrev_i16_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; VI: v_ashrrev_i16_sdwa v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1 ; VI: v_or_b32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; CI-DAG: v_bfe_i32 v{{[0-9]+}}, v{{[0-9]+}}, 0, 16 -; CI: v_ashrrev_i32_e32 v{{[0-9]+}}, 16, [[LHS]] +; CI: v_ashrrev_i32_e32 v{{[0-9]+}}, 16, v[[LHS]] ; CI: v_lshrrev_b32_e32 v{{[0-9]+}}, 16, v{{[0-9]+}} ; CI: v_ashrrev_i32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; CI: v_ashrrev_i32_e32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} @@ -117,8 +116,7 @@ } ; GCN-LABEL: {{^}}v_ashr_v4i16: -; GCN: {{buffer|flat|global}}_load_dwordx2 -; GCN: {{buffer|flat|global}}_load_dwordx2 +; GCN: {{buffer|flat|global}}_load_dwordx4 ; GFX9: v_pk_ashrrev_i16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} ; GFX9: v_pk_ashrrev_i16 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} diff --git a/llvm/test/CodeGen/AMDGPU/fmax_legacy.ll b/llvm/test/CodeGen/AMDGPU/fmax_legacy.ll --- a/llvm/test/CodeGen/AMDGPU/fmax_legacy.ll +++ b/llvm/test/CodeGen/AMDGPU/fmax_legacy.ll @@ -157,8 +157,8 @@ %gep.0 = getelementptr <1 x float>, <1 x float> addrspace(1)* %in, i32 %tid %gep.1 = getelementptr <1 x float>, <1 x float> addrspace(1)* %gep.0, i32 1 - %a = load <1 x float>, <1 x float> addrspace(1)* %gep.0 - %b = load <1 x float>, <1 x float> addrspace(1)* %gep.1 + %a = load volatile <1 x float>, <1 x float> addrspace(1)* %gep.0 + %b = load volatile <1 x float>, <1 x float> addrspace(1)* %gep.1 %cmp = fcmp ogt <1 x float> %a, %b %val = select <1 x i1> %cmp, <1 x float> %a, <1 x float> %b diff --git a/llvm/test/CodeGen/AMDGPU/fmin_legacy.ll b/llvm/test/CodeGen/AMDGPU/fmin_legacy.ll --- a/llvm/test/CodeGen/AMDGPU/fmin_legacy.ll +++ b/llvm/test/CodeGen/AMDGPU/fmin_legacy.ll @@ -187,8 +187,8 @@ %gep.0 = getelementptr <1 x float>, <1 x float> addrspace(1)* %in, i32 %tid %gep.1 = getelementptr <1 x float>, <1 x float> addrspace(1)* %gep.0, i32 1 - %a = load <1 x float>, <1 x float> addrspace(1)* %gep.0 - %b = load <1 x float>, <1 x float> addrspace(1)* %gep.1 + %a = load volatile <1 x float>, <1 x float> addrspace(1)* %gep.0 + %b = load volatile <1 x float>, <1 x float> addrspace(1)* %gep.1 %cmp = fcmp ult <1 x float> %a, %b %val = select <1 x i1> %cmp, <1 x float> %a, <1 x float> %b @@ -214,8 +214,8 @@ %gep.0 = getelementptr <2 x float>, <2 x float> addrspace(1)* %in, i32 %tid %gep.1 = getelementptr <2 x float>, <2 x float> addrspace(1)* %gep.0, i32 1 - %a = load <2 x float>, <2 x float> addrspace(1)* %gep.0 - %b = load <2 x float>, <2 x float> addrspace(1)* %gep.1 + %a = load volatile <2 x float>, <2 x float> addrspace(1)* %gep.0 + %b = load volatile <2 x float>, <2 x float> addrspace(1)* %gep.1 %cmp = fcmp ult <2 x float> %a, %b %val = select <2 x i1> %cmp, <2 x float> %a, <2 x float> %b diff --git a/llvm/test/CodeGen/AMDGPU/fshl.ll b/llvm/test/CodeGen/AMDGPU/fshl.ll --- a/llvm/test/CodeGen/AMDGPU/fshl.ll +++ b/llvm/test/CodeGen/AMDGPU/fshl.ll @@ -161,47 +161,45 @@ define amdgpu_kernel void @fshl_v2i32(<2 x i32> addrspace(1)* %in, <2 x i32> %x, <2 x i32> %y, <2 x i32> %z) { ; SI-LABEL: fshl_v2i32: ; SI: ; %bb.0: ; %entry -; SI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0xd -; SI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 -; SI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0xb +; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xb +; SI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x9 ; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xf -; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_mov_b32 s11, 0xf000 +; SI-NEXT: s_mov_b32 s10, -1 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: v_mov_b32_e32 v0, s3 -; SI-NEXT: s_mov_b32 s6, -1 -; SI-NEXT: v_alignbit_b32 v0, s9, v0, 1 +; SI-NEXT: v_mov_b32_e32 v0, s7 +; SI-NEXT: v_alignbit_b32 v0, s5, v0, 1 ; SI-NEXT: s_not_b32 s1, s1 -; SI-NEXT: s_lshr_b32 s3, s9, 1 +; SI-NEXT: s_lshr_b32 s2, s5, 1 ; SI-NEXT: v_mov_b32_e32 v1, s1 -; SI-NEXT: v_alignbit_b32 v1, s3, v0, v1 -; SI-NEXT: v_mov_b32_e32 v0, s2 +; SI-NEXT: v_alignbit_b32 v1, s2, v0, v1 +; SI-NEXT: v_mov_b32_e32 v0, s6 ; SI-NEXT: s_not_b32 s0, s0 -; SI-NEXT: v_alignbit_b32 v0, s8, v0, 1 -; SI-NEXT: s_lshr_b32 s1, s8, 1 +; SI-NEXT: v_alignbit_b32 v0, s4, v0, 1 +; SI-NEXT: s_lshr_b32 s1, s4, 1 ; SI-NEXT: v_mov_b32_e32 v2, s0 ; SI-NEXT: v_alignbit_b32 v0, s1, v0, v2 -; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0 ; SI-NEXT: s_endpgm ; ; VI-LABEL: fshl_v2i32: ; VI: ; %bb.0: ; %entry -; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 -; VI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x2c -; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x3c +; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x2c +; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x3c ; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: v_mov_b32_e32 v0, s3 -; VI-NEXT: s_lshr_b32 s3, s5, 1 +; VI-NEXT: v_mov_b32_e32 v0, s7 +; VI-NEXT: s_not_b32 s3, s3 +; VI-NEXT: s_lshr_b32 s7, s5, 1 ; VI-NEXT: v_alignbit_b32 v0, s5, v0, 1 -; VI-NEXT: s_not_b32 s5, s7 -; VI-NEXT: v_mov_b32_e32 v1, s5 -; VI-NEXT: v_alignbit_b32 v1, s3, v0, v1 -; VI-NEXT: v_mov_b32_e32 v0, s2 -; VI-NEXT: s_not_b32 s3, s6 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: v_alignbit_b32 v1, s7, v0, v1 +; VI-NEXT: v_mov_b32_e32 v0, s6 +; VI-NEXT: s_not_b32 s2, s2 ; VI-NEXT: v_alignbit_b32 v0, s4, v0, 1 -; VI-NEXT: s_lshr_b32 s2, s4, 1 -; VI-NEXT: v_mov_b32_e32 v2, s3 -; VI-NEXT: v_alignbit_b32 v0, s2, v0, v2 +; VI-NEXT: s_lshr_b32 s3, s4, 1 +; VI-NEXT: v_mov_b32_e32 v2, s2 +; VI-NEXT: v_alignbit_b32 v0, s3, v0, v2 ; VI-NEXT: v_mov_b32_e32 v3, s1 ; VI-NEXT: v_mov_b32_e32 v2, s0 ; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] @@ -209,25 +207,24 @@ ; ; GFX9-LABEL: fshl_v2i32: ; GFX9: ; %bb.0: ; %entry -; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 -; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x2c -; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x24 +; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x2c +; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 ; GFX9-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x3c ; GFX9-NEXT: v_mov_b32_e32 v2, 0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v0, s3 -; GFX9-NEXT: v_alignbit_b32 v0, s5, v0, 1 +; GFX9-NEXT: v_mov_b32_e32 v0, s7 ; GFX9-NEXT: s_lshr_b32 s0, s5, 1 ; GFX9-NEXT: s_not_b32 s1, s9 +; GFX9-NEXT: v_alignbit_b32 v0, s5, v0, 1 ; GFX9-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-NEXT: v_alignbit_b32 v1, s0, v0, v1 -; GFX9-NEXT: v_mov_b32_e32 v0, s2 +; GFX9-NEXT: v_mov_b32_e32 v0, s6 ; GFX9-NEXT: s_not_b32 s1, s8 ; GFX9-NEXT: v_alignbit_b32 v0, s4, v0, 1 ; GFX9-NEXT: s_lshr_b32 s0, s4, 1 ; GFX9-NEXT: v_mov_b32_e32 v3, s1 ; GFX9-NEXT: v_alignbit_b32 v0, s0, v0, v3 -; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7] +; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] ; GFX9-NEXT: s_endpgm ; ; R600-LABEL: fshl_v2i32: @@ -250,21 +247,20 @@ ; ; GFX10-LABEL: fshl_v2i32: ; GFX10: ; %bb.0: ; %entry -; GFX10-NEXT: s_clause 0x3 -; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2c -; GFX10-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 -; GFX10-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x3c +; GFX10-NEXT: s_clause 0x2 +; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x2c +; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x3c ; GFX10-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x24 ; GFX10-NEXT: v_mov_b32_e32 v2, 0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_lshr_b32 s0, s3, 1 -; GFX10-NEXT: v_alignbit_b32 v0, s3, s5, 1 -; GFX10-NEXT: v_alignbit_b32 v3, s2, s4, 1 -; GFX10-NEXT: s_not_b32 s1, s7 -; GFX10-NEXT: s_lshr_b32 s2, s2, 1 -; GFX10-NEXT: s_not_b32 s3, s6 +; GFX10-NEXT: v_alignbit_b32 v0, s5, s7, 1 +; GFX10-NEXT: v_alignbit_b32 v3, s4, s6, 1 +; GFX10-NEXT: s_lshr_b32 s0, s5, 1 +; GFX10-NEXT: s_not_b32 s1, s3 +; GFX10-NEXT: s_lshr_b32 s3, s4, 1 +; GFX10-NEXT: s_not_b32 s2, s2 ; GFX10-NEXT: v_alignbit_b32 v1, s0, v0, s1 -; GFX10-NEXT: v_alignbit_b32 v0, s2, v3, s3 +; GFX10-NEXT: v_alignbit_b32 v0, s3, v3, s2 ; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] ; GFX10-NEXT: s_endpgm entry: @@ -276,27 +272,25 @@ define amdgpu_kernel void @fshl_v2i32_imm(<2 x i32> addrspace(1)* %in, <2 x i32> %x, <2 x i32> %y) { ; SI-LABEL: fshl_v2i32_imm: ; SI: ; %bb.0: ; %entry -; SI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd -; SI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0xb +; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xb ; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; SI-NEXT: s_mov_b32 s3, 0xf000 ; SI-NEXT: s_mov_b32 s2, -1 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: v_mov_b32_e32 v0, s5 -; SI-NEXT: v_alignbit_b32 v1, s7, v0, 23 -; SI-NEXT: v_mov_b32_e32 v0, s4 -; SI-NEXT: v_alignbit_b32 v0, s6, v0, 25 +; SI-NEXT: v_mov_b32_e32 v0, s7 +; SI-NEXT: v_mov_b32_e32 v2, s6 +; SI-NEXT: v_alignbit_b32 v1, s5, v0, 23 +; SI-NEXT: v_alignbit_b32 v0, s4, v2, 25 ; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 ; SI-NEXT: s_endpgm ; ; VI-LABEL: fshl_v2i32_imm: ; VI: ; %bb.0: ; %entry -; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 -; VI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x2c +; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x2c ; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: v_mov_b32_e32 v0, s3 -; VI-NEXT: v_mov_b32_e32 v2, s2 +; VI-NEXT: v_mov_b32_e32 v0, s7 +; VI-NEXT: v_mov_b32_e32 v2, s6 ; VI-NEXT: v_alignbit_b32 v1, s5, v0, 23 ; VI-NEXT: v_alignbit_b32 v0, s4, v2, 25 ; VI-NEXT: v_mov_b32_e32 v3, s1 @@ -306,16 +300,15 @@ ; ; GFX9-LABEL: fshl_v2i32_imm: ; GFX9: ; %bb.0: ; %entry -; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 -; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x2c -; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x24 +; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x2c +; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 ; GFX9-NEXT: v_mov_b32_e32 v2, 0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v0, s3 -; GFX9-NEXT: v_mov_b32_e32 v3, s2 +; GFX9-NEXT: v_mov_b32_e32 v0, s7 +; GFX9-NEXT: v_mov_b32_e32 v3, s6 ; GFX9-NEXT: v_alignbit_b32 v1, s5, v0, 23 ; GFX9-NEXT: v_alignbit_b32 v0, s4, v3, 25 -; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7] +; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] ; GFX9-NEXT: s_endpgm ; ; R600-LABEL: fshl_v2i32_imm: @@ -334,15 +327,14 @@ ; ; GFX10-LABEL: fshl_v2i32_imm: ; GFX10: ; %bb.0: ; %entry -; GFX10-NEXT: s_clause 0x2 -; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2c -; GFX10-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 -; GFX10-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x24 +; GFX10-NEXT: s_clause 0x1 +; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x2c +; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 ; GFX10-NEXT: v_mov_b32_e32 v2, 0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: v_alignbit_b32 v1, s3, s5, 23 -; GFX10-NEXT: v_alignbit_b32 v0, s2, s4, 25 -; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7] +; GFX10-NEXT: v_alignbit_b32 v1, s5, s7, 23 +; GFX10-NEXT: v_alignbit_b32 v0, s4, s6, 25 +; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] ; GFX10-NEXT: s_endpgm entry: %0 = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> ) @@ -353,69 +345,67 @@ define amdgpu_kernel void @fshl_v4i32(<4 x i32> addrspace(1)* %in, <4 x i32> %x, <4 x i32> %y, <4 x i32> %z) { ; SI-LABEL: fshl_v4i32: ; SI: ; %bb.0: ; %entry -; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x11 -; SI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x9 -; SI-NEXT: s_load_dwordx4 s[12:15], s[0:1], 0xd -; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x15 -; SI-NEXT: s_mov_b32 s11, 0xf000 +; SI-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0xd +; SI-NEXT: s_load_dwordx4 s[12:15], s[0:1], 0x15 +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: v_mov_b32_e32 v0, s7 -; SI-NEXT: s_mov_b32 s10, -1 -; SI-NEXT: v_alignbit_b32 v0, s15, v0, 1 -; SI-NEXT: s_not_b32 s3, s3 -; SI-NEXT: s_lshr_b32 s7, s15, 1 -; SI-NEXT: v_mov_b32_e32 v1, s3 +; SI-NEXT: v_mov_b32_e32 v0, s11 +; SI-NEXT: s_not_b32 s11, s15 +; SI-NEXT: v_alignbit_b32 v0, s7, v0, 1 +; SI-NEXT: s_lshr_b32 s7, s7, 1 +; SI-NEXT: v_mov_b32_e32 v1, s11 ; SI-NEXT: v_alignbit_b32 v3, s7, v0, v1 -; SI-NEXT: v_mov_b32_e32 v0, s6 -; SI-NEXT: s_not_b32 s2, s2 -; SI-NEXT: v_alignbit_b32 v0, s14, v0, 1 -; SI-NEXT: s_lshr_b32 s3, s14, 1 -; SI-NEXT: v_mov_b32_e32 v1, s2 -; SI-NEXT: v_alignbit_b32 v2, s3, v0, v1 -; SI-NEXT: v_mov_b32_e32 v0, s5 -; SI-NEXT: s_not_b32 s1, s1 -; SI-NEXT: v_alignbit_b32 v0, s13, v0, 1 -; SI-NEXT: s_lshr_b32 s2, s13, 1 -; SI-NEXT: v_mov_b32_e32 v1, s1 -; SI-NEXT: v_alignbit_b32 v1, s2, v0, v1 -; SI-NEXT: v_mov_b32_e32 v0, s4 -; SI-NEXT: s_not_b32 s0, s0 -; SI-NEXT: v_alignbit_b32 v0, s12, v0, 1 -; SI-NEXT: s_lshr_b32 s1, s12, 1 -; SI-NEXT: v_mov_b32_e32 v4, s0 -; SI-NEXT: v_alignbit_b32 v0, s1, v0, v4 -; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0 +; SI-NEXT: v_mov_b32_e32 v0, s10 +; SI-NEXT: s_not_b32 s7, s14 +; SI-NEXT: v_alignbit_b32 v0, s6, v0, 1 +; SI-NEXT: s_lshr_b32 s6, s6, 1 +; SI-NEXT: v_mov_b32_e32 v1, s7 +; SI-NEXT: v_alignbit_b32 v2, s6, v0, v1 +; SI-NEXT: v_mov_b32_e32 v0, s9 +; SI-NEXT: s_not_b32 s6, s13 +; SI-NEXT: v_alignbit_b32 v0, s5, v0, 1 +; SI-NEXT: s_lshr_b32 s5, s5, 1 +; SI-NEXT: v_mov_b32_e32 v1, s6 +; SI-NEXT: v_alignbit_b32 v1, s5, v0, v1 +; SI-NEXT: v_mov_b32_e32 v0, s8 +; SI-NEXT: s_not_b32 s5, s12 +; SI-NEXT: v_alignbit_b32 v0, s4, v0, 1 +; SI-NEXT: s_lshr_b32 s4, s4, 1 +; SI-NEXT: v_mov_b32_e32 v4, s5 +; SI-NEXT: v_alignbit_b32 v0, s4, v0, v4 +; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 ; SI-NEXT: s_endpgm ; ; VI-LABEL: fshl_v4i32: ; VI: ; %bb.0: ; %entry -; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x44 -; VI-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x34 +; VI-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x34 ; VI-NEXT: s_load_dwordx4 s[12:15], s[0:1], 0x54 ; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: v_mov_b32_e32 v0, s7 -; VI-NEXT: s_lshr_b32 s2, s11, 1 +; VI-NEXT: v_mov_b32_e32 v0, s11 ; VI-NEXT: s_not_b32 s3, s15 -; VI-NEXT: v_alignbit_b32 v0, s11, v0, 1 +; VI-NEXT: s_lshr_b32 s2, s7, 1 +; VI-NEXT: v_alignbit_b32 v0, s7, v0, 1 ; VI-NEXT: v_mov_b32_e32 v1, s3 ; VI-NEXT: v_alignbit_b32 v3, s2, v0, v1 -; VI-NEXT: v_mov_b32_e32 v0, s6 +; VI-NEXT: v_mov_b32_e32 v0, s10 ; VI-NEXT: s_not_b32 s3, s14 -; VI-NEXT: v_alignbit_b32 v0, s10, v0, 1 -; VI-NEXT: s_lshr_b32 s2, s10, 1 +; VI-NEXT: v_alignbit_b32 v0, s6, v0, 1 +; VI-NEXT: s_lshr_b32 s2, s6, 1 ; VI-NEXT: v_mov_b32_e32 v1, s3 ; VI-NEXT: v_alignbit_b32 v2, s2, v0, v1 -; VI-NEXT: v_mov_b32_e32 v0, s5 +; VI-NEXT: v_mov_b32_e32 v0, s9 ; VI-NEXT: s_not_b32 s3, s13 -; VI-NEXT: v_alignbit_b32 v0, s9, v0, 1 -; VI-NEXT: s_lshr_b32 s2, s9, 1 +; VI-NEXT: v_alignbit_b32 v0, s5, v0, 1 +; VI-NEXT: s_lshr_b32 s2, s5, 1 ; VI-NEXT: v_mov_b32_e32 v1, s3 ; VI-NEXT: v_alignbit_b32 v1, s2, v0, v1 -; VI-NEXT: v_mov_b32_e32 v0, s4 +; VI-NEXT: v_mov_b32_e32 v0, s8 ; VI-NEXT: s_not_b32 s3, s12 -; VI-NEXT: v_alignbit_b32 v0, s8, v0, 1 -; VI-NEXT: s_lshr_b32 s2, s8, 1 +; VI-NEXT: v_alignbit_b32 v0, s4, v0, 1 +; VI-NEXT: s_lshr_b32 s2, s4, 1 ; VI-NEXT: v_mov_b32_e32 v4, s3 ; VI-NEXT: v_alignbit_b32 v0, s2, v0, v4 ; VI-NEXT: v_mov_b32_e32 v5, s1 @@ -425,34 +415,33 @@ ; ; GFX9-LABEL: fshl_v4i32: ; GFX9: ; %bb.0: ; %entry -; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x44 -; GFX9-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x34 -; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GFX9-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x34 ; GFX9-NEXT: s_load_dwordx4 s[12:15], s[0:1], 0x54 +; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 ; GFX9-NEXT: v_mov_b32_e32 v4, 0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v0, s7 -; GFX9-NEXT: v_alignbit_b32 v0, s11, v0, 1 -; GFX9-NEXT: s_lshr_b32 s0, s11, 1 ; GFX9-NEXT: s_not_b32 s1, s15 +; GFX9-NEXT: v_mov_b32_e32 v0, s11 +; GFX9-NEXT: s_lshr_b32 s0, s7, 1 +; GFX9-NEXT: v_alignbit_b32 v0, s7, v0, 1 ; GFX9-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-NEXT: v_alignbit_b32 v3, s0, v0, v1 -; GFX9-NEXT: v_mov_b32_e32 v0, s6 +; GFX9-NEXT: v_mov_b32_e32 v0, s10 ; GFX9-NEXT: s_not_b32 s1, s14 -; GFX9-NEXT: v_alignbit_b32 v0, s10, v0, 1 -; GFX9-NEXT: s_lshr_b32 s0, s10, 1 +; GFX9-NEXT: v_alignbit_b32 v0, s6, v0, 1 +; GFX9-NEXT: s_lshr_b32 s0, s6, 1 ; GFX9-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-NEXT: v_alignbit_b32 v2, s0, v0, v1 -; GFX9-NEXT: v_mov_b32_e32 v0, s5 +; GFX9-NEXT: v_mov_b32_e32 v0, s9 ; GFX9-NEXT: s_not_b32 s1, s13 -; GFX9-NEXT: v_alignbit_b32 v0, s9, v0, 1 -; GFX9-NEXT: s_lshr_b32 s0, s9, 1 +; GFX9-NEXT: v_alignbit_b32 v0, s5, v0, 1 +; GFX9-NEXT: s_lshr_b32 s0, s5, 1 ; GFX9-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-NEXT: v_alignbit_b32 v1, s0, v0, v1 -; GFX9-NEXT: v_mov_b32_e32 v0, s4 +; GFX9-NEXT: v_mov_b32_e32 v0, s8 ; GFX9-NEXT: s_not_b32 s1, s12 -; GFX9-NEXT: v_alignbit_b32 v0, s8, v0, 1 -; GFX9-NEXT: s_lshr_b32 s0, s8, 1 +; GFX9-NEXT: v_alignbit_b32 v0, s4, v0, 1 +; GFX9-NEXT: s_lshr_b32 s0, s4, 1 ; GFX9-NEXT: v_mov_b32_e32 v5, s1 ; GFX9-NEXT: v_alignbit_b32 v0, s0, v0, v5 ; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] @@ -486,30 +475,29 @@ ; ; GFX10-LABEL: fshl_v4i32: ; GFX10: ; %bb.0: ; %entry -; GFX10-NEXT: s_clause 0x3 -; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x34 -; GFX10-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x44 +; GFX10-NEXT: s_clause 0x1 +; GFX10-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x34 ; GFX10-NEXT: s_load_dwordx4 s[12:15], s[0:1], 0x54 -; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 ; GFX10-NEXT: v_mov_b32_e32 v4, 0 +; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_lshr_b32 s0, s7, 1 ; GFX10-NEXT: v_alignbit_b32 v0, s7, s11, 1 ; GFX10-NEXT: v_alignbit_b32 v1, s6, s10, 1 ; GFX10-NEXT: v_alignbit_b32 v5, s5, s9, 1 ; GFX10-NEXT: v_alignbit_b32 v6, s4, s8, 1 -; GFX10-NEXT: s_not_b32 s1, s15 +; GFX10-NEXT: s_lshr_b32 s2, s7, 1 +; GFX10-NEXT: s_not_b32 s3, s15 ; GFX10-NEXT: s_lshr_b32 s6, s6, 1 ; GFX10-NEXT: s_not_b32 s7, s14 ; GFX10-NEXT: s_lshr_b32 s5, s5, 1 ; GFX10-NEXT: s_not_b32 s9, s13 ; GFX10-NEXT: s_lshr_b32 s4, s4, 1 ; GFX10-NEXT: s_not_b32 s8, s12 -; GFX10-NEXT: v_alignbit_b32 v3, s0, v0, s1 +; GFX10-NEXT: v_alignbit_b32 v3, s2, v0, s3 ; GFX10-NEXT: v_alignbit_b32 v2, s6, v1, s7 ; GFX10-NEXT: v_alignbit_b32 v1, s5, v5, s9 ; GFX10-NEXT: v_alignbit_b32 v0, s4, v6, s8 -; GFX10-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] +; GFX10-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; GFX10-NEXT: s_endpgm entry: %0 = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i32> %z) @@ -520,58 +508,55 @@ define amdgpu_kernel void @fshl_v4i32_imm(<4 x i32> addrspace(1)* %in, <4 x i32> %x, <4 x i32> %y) { ; SI-LABEL: fshl_v4i32_imm: ; SI: ; %bb.0: ; %entry -; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x11 -; SI-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0xd +; SI-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0xd ; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; SI-NEXT: s_mov_b32 s3, 0xf000 ; SI-NEXT: s_mov_b32 s2, -1 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: v_mov_b32_e32 v0, s7 -; SI-NEXT: v_alignbit_b32 v3, s11, v0, 31 -; SI-NEXT: v_mov_b32_e32 v0, s6 -; SI-NEXT: v_alignbit_b32 v2, s10, v0, 23 -; SI-NEXT: v_mov_b32_e32 v0, s5 -; SI-NEXT: v_alignbit_b32 v1, s9, v0, 25 -; SI-NEXT: v_mov_b32_e32 v0, s4 -; SI-NEXT: v_alignbit_b32 v0, s8, v0, 31 +; SI-NEXT: v_mov_b32_e32 v0, s11 +; SI-NEXT: v_mov_b32_e32 v1, s10 +; SI-NEXT: v_alignbit_b32 v3, s7, v0, 31 +; SI-NEXT: v_mov_b32_e32 v0, s9 +; SI-NEXT: v_alignbit_b32 v2, s6, v1, 23 +; SI-NEXT: v_alignbit_b32 v1, s5, v0, 25 +; SI-NEXT: v_mov_b32_e32 v0, s8 +; SI-NEXT: v_alignbit_b32 v0, s4, v0, 31 ; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 ; SI-NEXT: s_endpgm ; ; VI-LABEL: fshl_v4i32_imm: ; VI: ; %bb.0: ; %entry -; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x44 -; VI-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x34 +; VI-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x34 ; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: v_mov_b32_e32 v0, s7 -; VI-NEXT: v_mov_b32_e32 v1, s6 -; VI-NEXT: v_alignbit_b32 v3, s11, v0, 31 -; VI-NEXT: v_mov_b32_e32 v0, s5 -; VI-NEXT: v_alignbit_b32 v2, s10, v1, 23 -; VI-NEXT: v_alignbit_b32 v1, s9, v0, 25 -; VI-NEXT: v_mov_b32_e32 v0, s4 +; VI-NEXT: v_mov_b32_e32 v0, s11 +; VI-NEXT: v_mov_b32_e32 v1, s10 +; VI-NEXT: v_mov_b32_e32 v4, s9 +; VI-NEXT: v_alignbit_b32 v3, s7, v0, 31 +; VI-NEXT: v_alignbit_b32 v2, s6, v1, 23 +; VI-NEXT: v_alignbit_b32 v1, s5, v4, 25 +; VI-NEXT: v_mov_b32_e32 v0, s8 ; VI-NEXT: v_mov_b32_e32 v5, s1 -; VI-NEXT: v_alignbit_b32 v0, s8, v0, 31 +; VI-NEXT: v_alignbit_b32 v0, s4, v0, 31 ; VI-NEXT: v_mov_b32_e32 v4, s0 ; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3] ; VI-NEXT: s_endpgm ; ; GFX9-LABEL: fshl_v4i32_imm: ; GFX9: ; %bb.0: ; %entry -; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x44 -; GFX9-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x34 -; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GFX9-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x34 ; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v0, s7 -; GFX9-NEXT: v_mov_b32_e32 v1, s6 -; GFX9-NEXT: v_alignbit_b32 v3, s11, v0, 31 -; GFX9-NEXT: v_mov_b32_e32 v0, s5 -; GFX9-NEXT: v_alignbit_b32 v2, s10, v1, 23 -; GFX9-NEXT: v_alignbit_b32 v1, s9, v0, 25 -; GFX9-NEXT: v_mov_b32_e32 v0, s4 -; GFX9-NEXT: v_alignbit_b32 v0, s8, v0, 31 -; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v0, s11 +; GFX9-NEXT: v_mov_b32_e32 v1, s10 +; GFX9-NEXT: v_alignbit_b32 v3, s7, v0, 31 +; GFX9-NEXT: v_mov_b32_e32 v0, s9 +; GFX9-NEXT: v_alignbit_b32 v2, s6, v1, 23 +; GFX9-NEXT: v_alignbit_b32 v1, s5, v0, 25 +; GFX9-NEXT: v_mov_b32_e32 v0, s8 +; GFX9-NEXT: v_alignbit_b32 v0, s4, v0, 31 +; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; GFX9-NEXT: s_endpgm ; ; R600-LABEL: fshl_v4i32_imm: @@ -594,9 +579,8 @@ ; ; GFX10-LABEL: fshl_v4i32_imm: ; GFX10: ; %bb.0: ; %entry -; GFX10-NEXT: s_clause 0x2 -; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x34 -; GFX10-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x44 +; GFX10-NEXT: s_clause 0x1 +; GFX10-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x34 ; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 ; GFX10-NEXT: v_mov_b32_e32 v4, 0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) diff --git a/llvm/test/CodeGen/AMDGPU/fshr.ll b/llvm/test/CodeGen/AMDGPU/fshr.ll --- a/llvm/test/CodeGen/AMDGPU/fshr.ll +++ b/llvm/test/CodeGen/AMDGPU/fshr.ll @@ -156,35 +156,33 @@ define amdgpu_kernel void @fshr_v2i32(<2 x i32> addrspace(1)* %in, <2 x i32> %x, <2 x i32> %y, <2 x i32> %z) { ; SI-LABEL: fshr_v2i32: ; SI: ; %bb.0: ; %entry -; SI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0xd +; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xb ; SI-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0xf -; SI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 -; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb -; SI-NEXT: s_mov_b32 s7, 0xf000 +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: v_mov_b32_e32 v0, s3 +; SI-NEXT: v_mov_b32_e32 v0, s7 ; SI-NEXT: v_mov_b32_e32 v1, s9 +; SI-NEXT: v_alignbit_b32 v1, s5, v0, v1 +; SI-NEXT: v_mov_b32_e32 v0, s6 ; SI-NEXT: v_mov_b32_e32 v2, s8 -; SI-NEXT: v_alignbit_b32 v1, s1, v0, v1 -; SI-NEXT: v_mov_b32_e32 v0, s2 -; SI-NEXT: s_mov_b32 s6, -1 -; SI-NEXT: v_alignbit_b32 v0, s0, v0, v2 -; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; SI-NEXT: v_alignbit_b32 v0, s4, v0, v2 +; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 ; SI-NEXT: s_endpgm ; ; VI-LABEL: fshr_v2i32: ; VI: ; %bb.0: ; %entry -; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 -; VI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x3c -; VI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x2c +; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x2c +; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x3c ; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: v_mov_b32_e32 v0, s3 -; VI-NEXT: v_mov_b32_e32 v1, s5 -; VI-NEXT: v_alignbit_b32 v1, s7, v0, v1 +; VI-NEXT: v_mov_b32_e32 v0, s7 +; VI-NEXT: v_mov_b32_e32 v1, s3 +; VI-NEXT: v_mov_b32_e32 v2, s6 +; VI-NEXT: v_alignbit_b32 v1, s5, v0, v1 ; VI-NEXT: v_mov_b32_e32 v0, s2 -; VI-NEXT: v_mov_b32_e32 v2, s4 -; VI-NEXT: v_alignbit_b32 v0, s6, v0, v2 +; VI-NEXT: v_alignbit_b32 v0, s4, v2, v0 ; VI-NEXT: v_mov_b32_e32 v3, s1 ; VI-NEXT: v_mov_b32_e32 v2, s0 ; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] @@ -192,18 +190,17 @@ ; ; GFX9-LABEL: fshr_v2i32: ; GFX9: ; %bb.0: ; %entry -; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 -; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x3c -; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x2c +; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x2c +; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x3c ; GFX9-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x24 ; GFX9-NEXT: v_mov_b32_e32 v2, 0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v0, s3 -; GFX9-NEXT: v_mov_b32_e32 v1, s5 -; GFX9-NEXT: v_alignbit_b32 v1, s7, v0, v1 -; GFX9-NEXT: v_mov_b32_e32 v0, s2 -; GFX9-NEXT: v_mov_b32_e32 v3, s4 -; GFX9-NEXT: v_alignbit_b32 v0, s6, v0, v3 +; GFX9-NEXT: v_mov_b32_e32 v0, s7 +; GFX9-NEXT: v_mov_b32_e32 v1, s3 +; GFX9-NEXT: v_alignbit_b32 v1, s5, v0, v1 +; GFX9-NEXT: v_mov_b32_e32 v0, s6 +; GFX9-NEXT: v_mov_b32_e32 v3, s2 +; GFX9-NEXT: v_alignbit_b32 v0, s4, v0, v3 ; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9] ; GFX9-NEXT: s_endpgm ; @@ -223,10 +220,9 @@ ; ; GFX10-LABEL: fshr_v2i32: ; GFX10: ; %bb.0: ; %entry -; GFX10-NEXT: s_clause 0x3 +; GFX10-NEXT: s_clause 0x2 ; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x3c -; GFX10-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x2c -; GFX10-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x34 +; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x2c ; GFX10-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x24 ; GFX10-NEXT: v_mov_b32_e32 v3, 0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) @@ -245,27 +241,25 @@ define amdgpu_kernel void @fshr_v2i32_imm(<2 x i32> addrspace(1)* %in, <2 x i32> %x, <2 x i32> %y) { ; SI-LABEL: fshr_v2i32_imm: ; SI: ; %bb.0: ; %entry -; SI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xd -; SI-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0xb +; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xb ; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; SI-NEXT: s_mov_b32 s3, 0xf000 ; SI-NEXT: s_mov_b32 s2, -1 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: v_mov_b32_e32 v0, s5 -; SI-NEXT: v_alignbit_b32 v1, s7, v0, 9 -; SI-NEXT: v_mov_b32_e32 v0, s4 -; SI-NEXT: v_alignbit_b32 v0, s6, v0, 7 +; SI-NEXT: v_mov_b32_e32 v0, s7 +; SI-NEXT: v_mov_b32_e32 v2, s6 +; SI-NEXT: v_alignbit_b32 v1, s5, v0, 9 +; SI-NEXT: v_alignbit_b32 v0, s4, v2, 7 ; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 ; SI-NEXT: s_endpgm ; ; VI-LABEL: fshr_v2i32_imm: ; VI: ; %bb.0: ; %entry -; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 -; VI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x2c +; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x2c ; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: v_mov_b32_e32 v0, s3 -; VI-NEXT: v_mov_b32_e32 v2, s2 +; VI-NEXT: v_mov_b32_e32 v0, s7 +; VI-NEXT: v_mov_b32_e32 v2, s6 ; VI-NEXT: v_alignbit_b32 v1, s5, v0, 9 ; VI-NEXT: v_alignbit_b32 v0, s4, v2, 7 ; VI-NEXT: v_mov_b32_e32 v3, s1 @@ -275,16 +269,15 @@ ; ; GFX9-LABEL: fshr_v2i32_imm: ; GFX9: ; %bb.0: ; %entry -; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 -; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x2c -; GFX9-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x24 +; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x2c +; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 ; GFX9-NEXT: v_mov_b32_e32 v2, 0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v0, s3 -; GFX9-NEXT: v_mov_b32_e32 v3, s2 +; GFX9-NEXT: v_mov_b32_e32 v0, s7 +; GFX9-NEXT: v_mov_b32_e32 v3, s6 ; GFX9-NEXT: v_alignbit_b32 v1, s5, v0, 9 ; GFX9-NEXT: v_alignbit_b32 v0, s4, v3, 7 -; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7] +; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] ; GFX9-NEXT: s_endpgm ; ; R600-LABEL: fshr_v2i32_imm: @@ -303,15 +296,14 @@ ; ; GFX10-LABEL: fshr_v2i32_imm: ; GFX10: ; %bb.0: ; %entry -; GFX10-NEXT: s_clause 0x2 -; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2c -; GFX10-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 -; GFX10-NEXT: s_load_dwordx2 s[6:7], s[0:1], 0x24 +; GFX10-NEXT: s_clause 0x1 +; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x2c +; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 ; GFX10-NEXT: v_mov_b32_e32 v2, 0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: v_alignbit_b32 v1, s3, s5, 9 -; GFX10-NEXT: v_alignbit_b32 v0, s2, s4, 7 -; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7] +; GFX10-NEXT: v_alignbit_b32 v1, s5, s7, 9 +; GFX10-NEXT: v_alignbit_b32 v0, s4, s6, 7 +; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3] ; GFX10-NEXT: s_endpgm entry: %0 = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> ) @@ -322,47 +314,45 @@ define amdgpu_kernel void @fshr_v4i32(<4 x i32> addrspace(1)* %in, <4 x i32> %x, <4 x i32> %y, <4 x i32> %z) { ; SI-LABEL: fshr_v4i32: ; SI: ; %bb.0: ; %entry -; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x11 -; SI-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x15 -; SI-NEXT: s_load_dwordx2 s[12:13], s[0:1], 0x9 -; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0xd -; SI-NEXT: s_mov_b32 s15, 0xf000 +; SI-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0xd +; SI-NEXT: s_load_dwordx4 s[12:15], s[0:1], 0x15 +; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; SI-NEXT: s_mov_b32 s3, 0xf000 +; SI-NEXT: s_mov_b32 s2, -1 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: v_mov_b32_e32 v0, s7 -; SI-NEXT: v_mov_b32_e32 v1, s11 -; SI-NEXT: v_mov_b32_e32 v4, s8 -; SI-NEXT: v_alignbit_b32 v3, s3, v0, v1 -; SI-NEXT: v_mov_b32_e32 v0, s6 -; SI-NEXT: v_mov_b32_e32 v1, s10 -; SI-NEXT: v_alignbit_b32 v2, s2, v0, v1 -; SI-NEXT: v_mov_b32_e32 v0, s5 -; SI-NEXT: v_mov_b32_e32 v1, s9 -; SI-NEXT: v_alignbit_b32 v1, s1, v0, v1 -; SI-NEXT: v_mov_b32_e32 v0, s4 -; SI-NEXT: s_mov_b32 s14, -1 -; SI-NEXT: v_alignbit_b32 v0, s0, v0, v4 -; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[12:15], 0 +; SI-NEXT: v_mov_b32_e32 v0, s11 +; SI-NEXT: v_mov_b32_e32 v1, s15 +; SI-NEXT: v_alignbit_b32 v3, s7, v0, v1 +; SI-NEXT: v_mov_b32_e32 v0, s10 +; SI-NEXT: v_mov_b32_e32 v1, s14 +; SI-NEXT: v_alignbit_b32 v2, s6, v0, v1 +; SI-NEXT: v_mov_b32_e32 v0, s9 +; SI-NEXT: v_mov_b32_e32 v1, s13 +; SI-NEXT: v_alignbit_b32 v1, s5, v0, v1 +; SI-NEXT: v_mov_b32_e32 v0, s8 +; SI-NEXT: v_mov_b32_e32 v4, s12 +; SI-NEXT: v_alignbit_b32 v0, s4, v0, v4 +; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 ; SI-NEXT: s_endpgm ; ; VI-LABEL: fshr_v4i32: ; VI: ; %bb.0: ; %entry -; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x44 -; VI-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x54 -; VI-NEXT: s_load_dwordx4 s[12:15], s[0:1], 0x34 +; VI-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x34 +; VI-NEXT: s_load_dwordx4 s[12:15], s[0:1], 0x54 ; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: v_mov_b32_e32 v0, s7 -; VI-NEXT: v_mov_b32_e32 v1, s11 -; VI-NEXT: v_alignbit_b32 v3, s15, v0, v1 -; VI-NEXT: v_mov_b32_e32 v0, s6 -; VI-NEXT: v_mov_b32_e32 v1, s10 -; VI-NEXT: v_alignbit_b32 v2, s14, v0, v1 -; VI-NEXT: v_mov_b32_e32 v0, s5 -; VI-NEXT: v_mov_b32_e32 v1, s9 -; VI-NEXT: v_alignbit_b32 v1, s13, v0, v1 -; VI-NEXT: v_mov_b32_e32 v0, s4 -; VI-NEXT: v_mov_b32_e32 v4, s8 -; VI-NEXT: v_alignbit_b32 v0, s12, v0, v4 +; VI-NEXT: v_mov_b32_e32 v0, s11 +; VI-NEXT: v_mov_b32_e32 v1, s15 +; VI-NEXT: v_mov_b32_e32 v2, s10 +; VI-NEXT: v_alignbit_b32 v3, s7, v0, v1 +; VI-NEXT: v_mov_b32_e32 v0, s14 +; VI-NEXT: v_alignbit_b32 v2, s6, v2, v0 +; VI-NEXT: v_mov_b32_e32 v0, s9 +; VI-NEXT: v_mov_b32_e32 v1, s13 +; VI-NEXT: v_alignbit_b32 v1, s5, v0, v1 +; VI-NEXT: v_mov_b32_e32 v0, s8 +; VI-NEXT: v_mov_b32_e32 v4, s12 +; VI-NEXT: v_alignbit_b32 v0, s4, v0, v4 ; VI-NEXT: v_mov_b32_e32 v5, s1 ; VI-NEXT: v_mov_b32_e32 v4, s0 ; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3] @@ -370,25 +360,24 @@ ; ; GFX9-LABEL: fshr_v4i32: ; GFX9: ; %bb.0: ; %entry -; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x44 -; GFX9-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x54 -; GFX9-NEXT: s_load_dwordx4 s[12:15], s[0:1], 0x34 -; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GFX9-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x34 +; GFX9-NEXT: s_load_dwordx4 s[12:15], s[0:1], 0x54 ; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v0, s7 -; GFX9-NEXT: v_mov_b32_e32 v1, s11 -; GFX9-NEXT: v_alignbit_b32 v3, s15, v0, v1 -; GFX9-NEXT: v_mov_b32_e32 v0, s6 -; GFX9-NEXT: v_mov_b32_e32 v1, s10 -; GFX9-NEXT: v_alignbit_b32 v2, s14, v0, v1 -; GFX9-NEXT: v_mov_b32_e32 v0, s5 -; GFX9-NEXT: v_mov_b32_e32 v1, s9 -; GFX9-NEXT: v_alignbit_b32 v1, s13, v0, v1 -; GFX9-NEXT: v_mov_b32_e32 v0, s4 -; GFX9-NEXT: v_mov_b32_e32 v5, s8 -; GFX9-NEXT: v_alignbit_b32 v0, s12, v0, v5 -; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v0, s11 +; GFX9-NEXT: v_mov_b32_e32 v1, s15 +; GFX9-NEXT: v_alignbit_b32 v3, s7, v0, v1 +; GFX9-NEXT: v_mov_b32_e32 v0, s10 +; GFX9-NEXT: v_mov_b32_e32 v1, s14 +; GFX9-NEXT: v_alignbit_b32 v2, s6, v0, v1 +; GFX9-NEXT: v_mov_b32_e32 v0, s9 +; GFX9-NEXT: v_mov_b32_e32 v1, s13 +; GFX9-NEXT: v_alignbit_b32 v1, s5, v0, v1 +; GFX9-NEXT: v_mov_b32_e32 v0, s8 +; GFX9-NEXT: v_mov_b32_e32 v5, s12 +; GFX9-NEXT: v_alignbit_b32 v0, s4, v0, v5 +; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; GFX9-NEXT: s_endpgm ; ; R600-LABEL: fshr_v4i32: @@ -411,21 +400,20 @@ ; ; GFX10-LABEL: fshr_v4i32: ; GFX10: ; %bb.0: ; %entry -; GFX10-NEXT: s_clause 0x3 -; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x54 -; GFX10-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x44 -; GFX10-NEXT: s_load_dwordx4 s[12:15], s[0:1], 0x34 +; GFX10-NEXT: s_clause 0x2 +; GFX10-NEXT: s_load_dwordx4 s[12:15], s[0:1], 0x54 +; GFX10-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x34 ; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 ; GFX10-NEXT: v_mov_b32_e32 v6, 0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: v_mov_b32_e32 v0, s7 -; GFX10-NEXT: v_mov_b32_e32 v1, s6 -; GFX10-NEXT: v_mov_b32_e32 v4, s5 -; GFX10-NEXT: v_mov_b32_e32 v5, s4 -; GFX10-NEXT: v_alignbit_b32 v3, s15, s11, v0 -; GFX10-NEXT: v_alignbit_b32 v2, s14, s10, v1 -; GFX10-NEXT: v_alignbit_b32 v1, s13, s9, v4 -; GFX10-NEXT: v_alignbit_b32 v0, s12, s8, v5 +; GFX10-NEXT: v_mov_b32_e32 v0, s15 +; GFX10-NEXT: v_mov_b32_e32 v1, s14 +; GFX10-NEXT: v_mov_b32_e32 v4, s13 +; GFX10-NEXT: v_mov_b32_e32 v5, s12 +; GFX10-NEXT: v_alignbit_b32 v3, s7, s11, v0 +; GFX10-NEXT: v_alignbit_b32 v2, s6, s10, v1 +; GFX10-NEXT: v_alignbit_b32 v1, s5, s9, v4 +; GFX10-NEXT: v_alignbit_b32 v0, s4, s8, v5 ; GFX10-NEXT: global_store_dwordx4 v6, v[0:3], s[2:3] ; GFX10-NEXT: s_endpgm entry: @@ -437,58 +425,55 @@ define amdgpu_kernel void @fshr_v4i32_imm(<4 x i32> addrspace(1)* %in, <4 x i32> %x, <4 x i32> %y) { ; SI-LABEL: fshr_v4i32_imm: ; SI: ; %bb.0: ; %entry -; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x11 -; SI-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0xd +; SI-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0xd ; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; SI-NEXT: s_mov_b32 s3, 0xf000 ; SI-NEXT: s_mov_b32 s2, -1 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: v_mov_b32_e32 v0, s7 -; SI-NEXT: v_alignbit_b32 v3, s11, v0, 1 -; SI-NEXT: v_mov_b32_e32 v0, s6 -; SI-NEXT: v_alignbit_b32 v2, s10, v0, 9 -; SI-NEXT: v_mov_b32_e32 v0, s5 -; SI-NEXT: v_alignbit_b32 v1, s9, v0, 7 -; SI-NEXT: v_mov_b32_e32 v0, s4 -; SI-NEXT: v_alignbit_b32 v0, s8, v0, 1 +; SI-NEXT: v_mov_b32_e32 v0, s11 +; SI-NEXT: v_mov_b32_e32 v1, s10 +; SI-NEXT: v_alignbit_b32 v3, s7, v0, 1 +; SI-NEXT: v_mov_b32_e32 v0, s9 +; SI-NEXT: v_alignbit_b32 v2, s6, v1, 9 +; SI-NEXT: v_alignbit_b32 v1, s5, v0, 7 +; SI-NEXT: v_mov_b32_e32 v0, s8 +; SI-NEXT: v_alignbit_b32 v0, s4, v0, 1 ; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 ; SI-NEXT: s_endpgm ; ; VI-LABEL: fshr_v4i32_imm: ; VI: ; %bb.0: ; %entry -; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x44 -; VI-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x34 +; VI-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x34 ; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: v_mov_b32_e32 v0, s7 -; VI-NEXT: v_mov_b32_e32 v1, s6 -; VI-NEXT: v_alignbit_b32 v3, s11, v0, 1 -; VI-NEXT: v_mov_b32_e32 v0, s5 -; VI-NEXT: v_alignbit_b32 v2, s10, v1, 9 -; VI-NEXT: v_alignbit_b32 v1, s9, v0, 7 -; VI-NEXT: v_mov_b32_e32 v0, s4 +; VI-NEXT: v_mov_b32_e32 v0, s11 +; VI-NEXT: v_mov_b32_e32 v1, s10 +; VI-NEXT: v_mov_b32_e32 v4, s9 +; VI-NEXT: v_alignbit_b32 v3, s7, v0, 1 +; VI-NEXT: v_alignbit_b32 v2, s6, v1, 9 +; VI-NEXT: v_alignbit_b32 v1, s5, v4, 7 +; VI-NEXT: v_mov_b32_e32 v0, s8 ; VI-NEXT: v_mov_b32_e32 v5, s1 -; VI-NEXT: v_alignbit_b32 v0, s8, v0, 1 +; VI-NEXT: v_alignbit_b32 v0, s4, v0, 1 ; VI-NEXT: v_mov_b32_e32 v4, s0 ; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3] ; VI-NEXT: s_endpgm ; ; GFX9-LABEL: fshr_v4i32_imm: ; GFX9: ; %bb.0: ; %entry -; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x44 -; GFX9-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x34 -; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GFX9-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x34 ; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v0, s7 -; GFX9-NEXT: v_mov_b32_e32 v1, s6 -; GFX9-NEXT: v_alignbit_b32 v3, s11, v0, 1 -; GFX9-NEXT: v_mov_b32_e32 v0, s5 -; GFX9-NEXT: v_alignbit_b32 v2, s10, v1, 9 -; GFX9-NEXT: v_alignbit_b32 v1, s9, v0, 7 -; GFX9-NEXT: v_mov_b32_e32 v0, s4 -; GFX9-NEXT: v_alignbit_b32 v0, s8, v0, 1 -; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v0, s11 +; GFX9-NEXT: v_mov_b32_e32 v1, s10 +; GFX9-NEXT: v_alignbit_b32 v3, s7, v0, 1 +; GFX9-NEXT: v_mov_b32_e32 v0, s9 +; GFX9-NEXT: v_alignbit_b32 v2, s6, v1, 9 +; GFX9-NEXT: v_alignbit_b32 v1, s5, v0, 7 +; GFX9-NEXT: v_mov_b32_e32 v0, s8 +; GFX9-NEXT: v_alignbit_b32 v0, s4, v0, 1 +; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1] ; GFX9-NEXT: s_endpgm ; ; R600-LABEL: fshr_v4i32_imm: @@ -509,9 +494,8 @@ ; ; GFX10-LABEL: fshr_v4i32_imm: ; GFX10: ; %bb.0: ; %entry -; GFX10-NEXT: s_clause 0x2 -; GFX10-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x34 -; GFX10-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x44 +; GFX10-NEXT: s_clause 0x1 +; GFX10-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0x34 ; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 ; GFX10-NEXT: v_mov_b32_e32 v4, 0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) diff --git a/llvm/test/CodeGen/AMDGPU/half.ll b/llvm/test/CodeGen/AMDGPU/half.ll --- a/llvm/test/CodeGen/AMDGPU/half.ll +++ b/llvm/test/CodeGen/AMDGPU/half.ll @@ -1897,8 +1897,7 @@ define amdgpu_kernel void @fadd_v2f16(<2 x half> addrspace(1)* %out, <2 x half> %a, <2 x half> %b) #0 { ; SI-LABEL: fadd_v2f16: ; SI: ; %bb.0: -; SI-NEXT: s_load_dword s0, s[4:5], 0x2 -; SI-NEXT: s_load_dword s1, s[4:5], 0x3 +; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2 ; SI-NEXT: s_waitcnt lgkmcnt(0) ; SI-NEXT: s_lshr_b32 s2, s0, 16 ; SI-NEXT: v_cvt_f32_f16_e32 v0, s0 @@ -1921,20 +1920,19 @@ ; ; VI-LABEL: fadd_v2f16: ; VI: ; %bb.0: -; VI-NEXT: s_load_dword s2, s[4:5], 0xc -; VI-NEXT: s_load_dword s3, s[4:5], 0x8 -; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0 +; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8 +; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_lshr_b32 s4, s2, 16 -; VI-NEXT: s_lshr_b32 s5, s3, 16 -; VI-NEXT: v_mov_b32_e32 v0, s4 -; VI-NEXT: v_mov_b32_e32 v1, s5 -; VI-NEXT: v_add_f16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD -; VI-NEXT: v_mov_b32_e32 v1, s2 -; VI-NEXT: v_add_f16_e32 v1, s3, v1 -; VI-NEXT: v_or_b32_e32 v2, v1, v0 -; VI-NEXT: v_mov_b32_e32 v0, s0 -; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: s_lshr_b32 s4, s1, 16 +; VI-NEXT: s_lshr_b32 s5, s0, 16 +; VI-NEXT: v_mov_b32_e32 v0, s1 +; VI-NEXT: v_mov_b32_e32 v1, s4 +; VI-NEXT: v_mov_b32_e32 v2, s5 +; VI-NEXT: v_add_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD +; VI-NEXT: v_add_f16_e32 v0, s0, v0 +; VI-NEXT: v_or_b32_e32 v2, v0, v1 +; VI-NEXT: v_mov_b32_e32 v0, s2 +; VI-NEXT: v_mov_b32_e32 v1, s3 ; VI-NEXT: flat_store_dword v[0:1], v2 ; VI-NEXT: s_endpgm %add = fadd <2 x half> %a, %b @@ -1949,20 +1947,14 @@ ; SI-NEXT: s_waitcnt lgkmcnt(0) ; SI-NEXT: v_mov_b32_e32 v0, s2 ; SI-NEXT: v_mov_b32_e32 v1, s3 -; SI-NEXT: s_add_u32 s2, s2, 8 -; SI-NEXT: s_addc_u32 s3, s3, 0 -; SI-NEXT: v_mov_b32_e32 v2, s2 -; SI-NEXT: v_mov_b32_e32 v3, s3 -; SI-NEXT: flat_load_dwordx2 v[0:1], v[0:1] -; SI-NEXT: flat_load_dwordx2 v[2:3], v[2:3] +; SI-NEXT: flat_load_dwordx4 v[0:3], v[0:1] ; SI-NEXT: v_mov_b32_e32 v4, s0 ; SI-NEXT: v_mov_b32_e32 v5, s1 -; SI-NEXT: s_waitcnt vmcnt(1) +; SI-NEXT: s_waitcnt vmcnt(0) ; SI-NEXT: v_cvt_f32_f16_e32 v6, v0 ; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; SI-NEXT: v_cvt_f32_f16_e32 v7, v1 ; SI-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; SI-NEXT: s_waitcnt vmcnt(0) ; SI-NEXT: v_cvt_f32_f16_e32 v8, v2 ; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v2 ; SI-NEXT: v_cvt_f32_f16_e32 v9, v3 @@ -1990,14 +1982,9 @@ ; VI: ; %bb.0: ; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_add_u32 s4, s2, 8 ; VI-NEXT: v_mov_b32_e32 v0, s2 -; VI-NEXT: s_addc_u32 s5, s3, 0 -; VI-NEXT: v_mov_b32_e32 v2, s4 ; VI-NEXT: v_mov_b32_e32 v1, s3 -; VI-NEXT: v_mov_b32_e32 v3, s5 -; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1] -; VI-NEXT: flat_load_dwordx2 v[2:3], v[2:3] +; VI-NEXT: flat_load_dwordx4 v[0:3], v[0:1] ; VI-NEXT: v_mov_b32_e32 v4, s0 ; VI-NEXT: v_mov_b32_e32 v5, s1 ; VI-NEXT: s_waitcnt vmcnt(0) diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.ll @@ -186,8 +186,7 @@ ; GCN-LABEL: {{^}}test_mfma_f32_4x4x4f16: ; GCN: s_load_dwordx4 -; GCN: s_load_dwordx2 -; GCN: s_load_dwordx2 +; GCN: s_load_dwordx4 ; GFX908_A-COUNT-4: v_accvgpr_write_b32 a{{[0-9]+}}, v{{[0-9]+}} ; GFX908_A: v_mfma_f32_4x4x4f16 [[RES:a\[[0-9]+:[0-9]+\]]], {{v\[[0-9]+:[0-9]+\]}}, {{v\[[0-9]+:[0-9]+\]}}, a[{{[0-9]+:[0-9]+}}] cbsz:1 abid:2 blgp:3 ; GFX908-COUNT-4: v_accvgpr_read_b32 diff --git a/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll b/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll --- a/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll +++ b/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll @@ -7,20 +7,18 @@ define amdgpu_kernel void @s_lshr_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> %lhs, <2 x i16> %rhs) #0 { ; GFX9-LABEL: s_lshr_v2i16: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dword s4, s[0:1], 0x2c -; GFX9-NEXT: s_load_dword s5, s[0:1], 0x30 -; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2c +; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24 ; GFX9-NEXT: v_mov_b32_e32 v0, 0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v1, s4 -; GFX9-NEXT: v_pk_lshrrev_b16 v1, s5, v1 -; GFX9-NEXT: global_store_dword v0, v1, s[2:3] +; GFX9-NEXT: v_mov_b32_e32 v1, s2 +; GFX9-NEXT: v_pk_lshrrev_b16 v1, s3, v1 +; GFX9-NEXT: global_store_dword v0, v1, s[4:5] ; GFX9-NEXT: s_endpgm ; ; VI-LABEL: s_lshr_v2i16: ; VI: ; %bb.0: -; VI-NEXT: s_load_dword s2, s[0:1], 0x2c -; VI-NEXT: s_load_dword s3, s[0:1], 0x30 +; VI-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2c ; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_and_b32 s4, s2, 0xffff @@ -38,33 +36,31 @@ ; ; CI-LABEL: s_lshr_v2i16: ; CI: ; %bb.0: -; CI-NEXT: s_load_dword s2, s[0:1], 0xb -; CI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9 -; CI-NEXT: s_load_dword s0, s[0:1], 0xc -; CI-NEXT: s_mov_b32 s7, 0xf000 -; CI-NEXT: s_mov_b32 s6, -1 +; CI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xb +; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; CI-NEXT: s_mov_b32 s3, 0xf000 +; CI-NEXT: s_mov_b32 s2, -1 ; CI-NEXT: s_waitcnt lgkmcnt(0) -; CI-NEXT: s_and_b32 s1, s2, 0xffff -; CI-NEXT: s_lshr_b32 s2, s2, 16 -; CI-NEXT: s_lshr_b32 s3, s0, 16 -; CI-NEXT: s_lshr_b32 s2, s2, s3 -; CI-NEXT: s_lshl_b32 s2, s2, 16 -; CI-NEXT: s_lshr_b32 s0, s1, s0 -; CI-NEXT: s_or_b32 s0, s0, s2 -; CI-NEXT: v_mov_b32_e32 v0, s0 -; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0 +; CI-NEXT: s_and_b32 s6, s4, 0xffff +; CI-NEXT: s_lshr_b32 s4, s4, 16 +; CI-NEXT: s_lshr_b32 s7, s5, 16 +; CI-NEXT: s_lshr_b32 s4, s4, s7 +; CI-NEXT: s_lshl_b32 s4, s4, 16 +; CI-NEXT: s_lshr_b32 s5, s6, s5 +; CI-NEXT: s_or_b32 s4, s5, s4 +; CI-NEXT: v_mov_b32_e32 v0, s4 +; CI-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; CI-NEXT: s_endpgm ; ; GFX10-LABEL: s_lshr_v2i16: ; GFX10: ; %bb.0: -; GFX10-NEXT: s_clause 0x2 -; GFX10-NEXT: s_load_dword s4, s[0:1], 0x2c -; GFX10-NEXT: s_load_dword s5, s[0:1], 0x30 -; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GFX10-NEXT: s_clause 0x1 +; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2c +; GFX10-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24 ; GFX10-NEXT: v_mov_b32_e32 v0, 0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: v_pk_lshrrev_b16 v1, s5, s4 -; GFX10-NEXT: global_store_dword v0, v1, s[2:3] +; GFX10-NEXT: v_pk_lshrrev_b16 v1, s3, s2 +; GFX10-NEXT: global_store_dword v0, v1, s[4:5] ; GFX10-NEXT: s_endpgm %result = lshr <2 x i16> %lhs, %rhs store <2 x i16> %result, <2 x i16> addrspace(1)* %out @@ -75,35 +71,31 @@ ; GFX9-LABEL: v_lshr_v2i16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 -; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: global_load_dword v1, v0, s[2:3] -; GFX9-NEXT: global_load_dword v2, v0, s[2:3] offset:4 +; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[2:3] ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_pk_lshrrev_b16 v1, v2, v1 -; GFX9-NEXT: global_store_dword v0, v1, s[0:1] +; GFX9-NEXT: v_pk_lshrrev_b16 v0, v1, v0 +; GFX9-NEXT: global_store_dword v2, v0, s[0:1] ; GFX9-NEXT: s_endpgm ; ; VI-LABEL: v_lshr_v2i16: ; VI: ; %bb.0: ; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 -; VI-NEXT: v_lshlrev_b32_e32 v4, 2, v0 +; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0 ; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: v_mov_b32_e32 v1, s3 -; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v4 -; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc -; VI-NEXT: v_add_u32_e32 v2, vcc, 4, v0 -; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc -; VI-NEXT: flat_load_dword v5, v[0:1] -; VI-NEXT: flat_load_dword v2, v[2:3] -; VI-NEXT: v_mov_b32_e32 v1, s1 -; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v4 +; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc +; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1] +; VI-NEXT: v_mov_b32_e32 v3, s1 +; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v2 +; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc ; VI-NEXT: s_waitcnt vmcnt(0) -; VI-NEXT: v_lshrrev_b16_e32 v3, v2, v5 -; VI-NEXT: v_lshrrev_b16_sdwa v2, v2, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1 -; VI-NEXT: v_or_b32_e32 v2, v3, v2 -; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: v_lshrrev_b16_e32 v4, v1, v0 +; VI-NEXT: v_lshrrev_b16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1 +; VI-NEXT: v_or_b32_e32 v0, v4, v0 +; VI-NEXT: flat_store_dword v[2:3], v0 ; VI-NEXT: s_endpgm ; ; CI-LABEL: v_lshr_v2i16: @@ -115,13 +107,11 @@ ; CI-NEXT: v_mov_b32_e32 v1, 0 ; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_mov_b64 s[4:5], s[2:3] -; CI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 -; CI-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 +; CI-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64 ; CI-NEXT: s_mov_b64 s[2:3], s[6:7] -; CI-NEXT: s_waitcnt vmcnt(1) +; CI-NEXT: s_waitcnt vmcnt(0) ; CI-NEXT: v_lshrrev_b32_e32 v4, 16, v2 ; CI-NEXT: v_and_b32_e32 v2, 0xffff, v2 -; CI-NEXT: s_waitcnt vmcnt(0) ; CI-NEXT: v_lshrrev_b32_e32 v5, 16, v3 ; CI-NEXT: v_lshrrev_b32_e32 v2, v3, v2 ; CI-NEXT: v_lshrrev_b32_e32 v3, v5, v4 @@ -133,14 +123,12 @@ ; GFX10-LABEL: v_lshr_v2i16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 -; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX10-NEXT: v_lshlrev_b32_e32 v2, 2, v0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_clause 0x1 -; GFX10-NEXT: global_load_dword v1, v0, s[2:3] -; GFX10-NEXT: global_load_dword v2, v0, s[2:3] offset:4 +; GFX10-NEXT: global_load_dwordx2 v[0:1], v2, s[2:3] ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_pk_lshrrev_b16 v1, v2, v1 -; GFX10-NEXT: global_store_dword v0, v1, s[0:1] +; GFX10-NEXT: v_pk_lshrrev_b16 v0, v1, v0 +; GFX10-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-NEXT: s_endpgm %tid = call i32 @llvm.amdgcn.workitem.id.x() %tid.ext = sext i32 %tid to i64 @@ -458,8 +446,7 @@ ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX9-NEXT: v_lshlrev_b32_e32 v4, 3, v0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[0:1], v4, s[2:3] -; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3] offset:8 +; GFX9-NEXT: global_load_dwordx4 v[0:3], v4, s[2:3] ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_pk_lshrrev_b16 v1, v3, v1 ; GFX9-NEXT: v_pk_lshrrev_b16 v0, v2, v0 @@ -474,10 +461,7 @@ ; VI-NEXT: v_mov_b32_e32 v1, s3 ; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v4 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc -; VI-NEXT: v_add_u32_e32 v2, vcc, 8, v0 -; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc -; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1] -; VI-NEXT: flat_load_dwordx2 v[2:3], v[2:3] +; VI-NEXT: flat_load_dwordx4 v[0:3], v[0:1] ; VI-NEXT: v_mov_b32_e32 v5, s1 ; VI-NEXT: v_add_u32_e32 v4, vcc, s0, v4 ; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc @@ -496,31 +480,29 @@ ; CI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 ; CI-NEXT: s_mov_b32 s7, 0xf000 ; CI-NEXT: s_mov_b32 s6, 0 -; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v0 -; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: v_lshlrev_b32_e32 v4, 3, v0 +; CI-NEXT: v_mov_b32_e32 v5, 0 ; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_mov_b64 s[4:5], s[2:3] -; CI-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64 -; CI-NEXT: buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8 +; CI-NEXT: buffer_load_dwordx4 v[0:3], v[4:5], s[4:7], 0 addr64 ; CI-NEXT: s_mov_b32 s4, 0xffff ; CI-NEXT: s_mov_b64 s[2:3], s[6:7] -; CI-NEXT: s_waitcnt vmcnt(1) -; CI-NEXT: v_lshrrev_b32_e32 v6, 16, v2 -; CI-NEXT: v_and_b32_e32 v2, s4, v2 -; CI-NEXT: v_lshrrev_b32_e32 v7, 16, v3 -; CI-NEXT: v_and_b32_e32 v3, s4, v3 ; CI-NEXT: s_waitcnt vmcnt(0) -; CI-NEXT: v_lshrrev_b32_e32 v8, 16, v4 -; CI-NEXT: v_lshrrev_b32_e32 v9, 16, v5 -; CI-NEXT: v_lshrrev_b32_e32 v3, v5, v3 -; CI-NEXT: v_lshrrev_b32_e32 v5, v9, v7 -; CI-NEXT: v_lshrrev_b32_e32 v2, v4, v2 -; CI-NEXT: v_lshrrev_b32_e32 v4, v8, v6 -; CI-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; CI-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; CI-NEXT: v_or_b32_e32 v3, v3, v5 -; CI-NEXT: v_or_b32_e32 v2, v2, v4 -; CI-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64 +; CI-NEXT: v_lshrrev_b32_e32 v6, 16, v0 +; CI-NEXT: v_and_b32_e32 v0, s4, v0 +; CI-NEXT: v_lshrrev_b32_e32 v7, 16, v1 +; CI-NEXT: v_and_b32_e32 v1, s4, v1 +; CI-NEXT: v_lshrrev_b32_e32 v8, 16, v2 +; CI-NEXT: v_lshrrev_b32_e32 v9, 16, v3 +; CI-NEXT: v_lshrrev_b32_e32 v1, v3, v1 +; CI-NEXT: v_lshrrev_b32_e32 v3, v9, v7 +; CI-NEXT: v_lshrrev_b32_e32 v0, v2, v0 +; CI-NEXT: v_lshrrev_b32_e32 v2, v8, v6 +; CI-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; CI-NEXT: v_or_b32_e32 v1, v1, v3 +; CI-NEXT: v_or_b32_e32 v0, v0, v2 +; CI-NEXT: buffer_store_dwordx2 v[0:1], v[4:5], s[0:3], 0 addr64 ; CI-NEXT: s_endpgm ; ; GFX10-LABEL: v_lshr_v4i16: @@ -528,9 +510,7 @@ ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_clause 0x1 -; GFX10-NEXT: global_load_dwordx2 v[0:1], v4, s[2:3] -; GFX10-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3] offset:8 +; GFX10-NEXT: global_load_dwordx4 v[0:3], v4, s[2:3] ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_pk_lshrrev_b16 v1, v3, v1 ; GFX10-NEXT: v_pk_lshrrev_b16 v0, v2, v0 diff --git a/llvm/test/CodeGen/AMDGPU/memory_clause.ll b/llvm/test/CodeGen/AMDGPU/memory_clause.ll --- a/llvm/test/CodeGen/AMDGPU/memory_clause.ll +++ b/llvm/test/CodeGen/AMDGPU/memory_clause.ll @@ -71,50 +71,42 @@ define amdgpu_kernel void @scalar_clause(<4 x i32> addrspace(1)* noalias nocapture readonly %arg, <4 x i32> addrspace(1)* noalias nocapture %arg1) { ; GCN-LABEL: scalar_clause: ; GCN: ; %bb.0: ; %bb -; GCN-NEXT: s_load_dwordx2 s[12:13], s[0:1], 0x24 +; GCN-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 ; GCN-NEXT: s_load_dwordx2 s[16:17], s[0:1], 0x2c -; GCN-NEXT: v_mov_b32_e32 v12, 0 +; GCN-NEXT: v_mov_b32_e32 v16, 0 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_load_dwordx4 s[0:3], s[12:13], 0x0 -; GCN-NEXT: s_load_dwordx4 s[4:7], s[12:13], 0x10 -; GCN-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x20 -; GCN-NEXT: s_nop 0 -; GCN-NEXT: s_load_dwordx4 s[12:15], s[12:13], 0x30 +; GCN-NEXT: s_load_dwordx16 s[0:15], s[2:3], 0x0 ; GCN-NEXT: s_waitcnt lgkmcnt(0) ; GCN-NEXT: v_mov_b32_e32 v0, s0 ; GCN-NEXT: v_mov_b32_e32 v1, s1 ; GCN-NEXT: v_mov_b32_e32 v2, s2 ; GCN-NEXT: v_mov_b32_e32 v3, s3 ; GCN-NEXT: v_mov_b32_e32 v4, s4 -; GCN-NEXT: v_mov_b32_e32 v8, s8 ; GCN-NEXT: v_mov_b32_e32 v5, s5 ; GCN-NEXT: v_mov_b32_e32 v6, s6 ; GCN-NEXT: v_mov_b32_e32 v7, s7 +; GCN-NEXT: v_mov_b32_e32 v8, s8 ; GCN-NEXT: v_mov_b32_e32 v9, s9 ; GCN-NEXT: v_mov_b32_e32 v10, s10 ; GCN-NEXT: v_mov_b32_e32 v11, s11 -; GCN-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17] -; GCN-NEXT: global_store_dwordx4 v12, v[4:7], s[16:17] offset:16 -; GCN-NEXT: global_store_dwordx4 v12, v[8:11], s[16:17] offset:32 -; GCN-NEXT: v_mov_b32_e32 v0, s12 -; GCN-NEXT: v_mov_b32_e32 v1, s13 -; GCN-NEXT: v_mov_b32_e32 v2, s14 -; GCN-NEXT: v_mov_b32_e32 v3, s15 -; GCN-NEXT: global_store_dwordx4 v12, v[0:3], s[16:17] offset:48 +; GCN-NEXT: v_mov_b32_e32 v12, s12 +; GCN-NEXT: v_mov_b32_e32 v13, s13 +; GCN-NEXT: v_mov_b32_e32 v14, s14 +; GCN-NEXT: v_mov_b32_e32 v15, s15 +; GCN-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17] +; GCN-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16 +; GCN-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32 +; GCN-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48 ; GCN-NEXT: s_endpgm ; ; GCN-SCRATCH-LABEL: scalar_clause: ; GCN-SCRATCH: ; %bb.0: ; %bb ; GCN-SCRATCH-NEXT: s_clause 0x1 -; GCN-SCRATCH-NEXT: s_load_dwordx2 s[12:13], s[0:1], 0x24 +; GCN-SCRATCH-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 ; GCN-SCRATCH-NEXT: s_load_dwordx2 s[16:17], s[0:1], 0x2c ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v16, 0 ; GCN-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) -; GCN-SCRATCH-NEXT: s_clause 0x3 -; GCN-SCRATCH-NEXT: s_load_dwordx4 s[0:3], s[12:13], 0x0 -; GCN-SCRATCH-NEXT: s_load_dwordx4 s[4:7], s[12:13], 0x10 -; GCN-SCRATCH-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x20 -; GCN-SCRATCH-NEXT: s_load_dwordx4 s[12:15], s[12:13], 0x30 +; GCN-SCRATCH-NEXT: s_load_dwordx16 s[0:15], s[2:3], 0x0 ; GCN-SCRATCH-NEXT: s_waitcnt lgkmcnt(0) ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v0, s0 ; GCN-SCRATCH-NEXT: v_mov_b32_e32 v1, s1 diff --git a/llvm/test/CodeGen/AMDGPU/min.ll b/llvm/test/CodeGen/AMDGPU/min.ll --- a/llvm/test/CodeGen/AMDGPU/min.ll +++ b/llvm/test/CodeGen/AMDGPU/min.ll @@ -107,8 +107,8 @@ } ; FUNC-LABEL: {{^}}s_test_imin_sle_v2i16: -; GCN: s_load_dword s -; GCN: s_load_dword s +; GCN: s_load_dwordx2 s +; GCN: s_load_dwordx2 s ; SI: s_ashr_i32 ; SI: s_sext_i32_i16 diff --git a/llvm/test/CodeGen/AMDGPU/mul_int24.ll b/llvm/test/CodeGen/AMDGPU/mul_int24.ll --- a/llvm/test/CodeGen/AMDGPU/mul_int24.ll +++ b/llvm/test/CodeGen/AMDGPU/mul_int24.ll @@ -709,13 +709,12 @@ ; SI-NEXT: ; %bb.1: ; %bb7 ; SI-NEXT: s_endpgm ; SI-NEXT: .LBB8_2: ; %bb11 -; SI-NEXT: s_load_dword s2, s[0:1], 0xd -; SI-NEXT: s_load_dword s4, s[0:1], 0xf +; SI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xd ; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; SI-NEXT: s_mov_b32 s3, 0xf000 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: s_bfe_i32 s2, s2, 0x180000 -; SI-NEXT: s_bfe_i32 s4, s4, 0x180000 +; SI-NEXT: s_bfe_i32 s2, s4, 0x180000 +; SI-NEXT: s_bfe_i32 s4, s6, 0x180000 ; SI-NEXT: s_mul_i32 s4, s2, s4 ; SI-NEXT: s_mov_b32 s2, -1 ; SI-NEXT: v_mov_b32_e32 v0, s4 @@ -732,14 +731,13 @@ ; VI-NEXT: ; %bb.1: ; %bb7 ; VI-NEXT: s_endpgm ; VI-NEXT: .LBB8_2: ; %bb11 -; VI-NEXT: s_load_dword s4, s[0:1], 0x34 -; VI-NEXT: s_load_dword s5, s[0:1], 0x3c +; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x34 ; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; VI-NEXT: s_mov_b32 s3, 0xf000 ; VI-NEXT: s_mov_b32 s2, -1 ; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: s_bfe_i32 s4, s4, 0x180000 -; VI-NEXT: s_bfe_i32 s5, s5, 0x180000 +; VI-NEXT: s_bfe_i32 s5, s6, 0x180000 ; VI-NEXT: s_mul_i32 s4, s4, s5 ; VI-NEXT: v_mov_b32_e32 v0, s4 ; VI-NEXT: v_mov_b32_e32 v1, s4 @@ -755,18 +753,17 @@ ; GFX9-NEXT: ; %bb.1: ; %bb7 ; GFX9-NEXT: s_endpgm ; GFX9-NEXT: .LBB8_2: ; %bb11 -; GFX9-NEXT: s_load_dword s2, s[0:1], 0x34 -; GFX9-NEXT: s_load_dword s3, s[0:1], 0x3c -; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24 -; GFX9-NEXT: s_mov_b32 s7, 0xf000 -; GFX9-NEXT: s_mov_b32 s6, -1 +; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x34 +; GFX9-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x24 +; GFX9-NEXT: s_mov_b32 s11, 0xf000 +; GFX9-NEXT: s_mov_b32 s10, -1 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: s_bfe_i32 s0, s2, 0x180000 -; GFX9-NEXT: s_bfe_i32 s1, s3, 0x180000 +; GFX9-NEXT: s_bfe_i32 s0, s4, 0x180000 +; GFX9-NEXT: s_bfe_i32 s1, s6, 0x180000 ; GFX9-NEXT: s_mul_i32 s0, s0, s1 ; GFX9-NEXT: v_mov_b32_e32 v0, s0 ; GFX9-NEXT: v_mov_b32_e32 v1, s0 -; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0 ; GFX9-NEXT: s_endpgm ; ; EG-LABEL: simplify_i24_crash: diff --git a/llvm/test/CodeGen/AMDGPU/sdiv.ll b/llvm/test/CodeGen/AMDGPU/sdiv.ll --- a/llvm/test/CodeGen/AMDGPU/sdiv.ll +++ b/llvm/test/CodeGen/AMDGPU/sdiv.ll @@ -440,15 +440,15 @@ ; GCN-NEXT: v_mul_lo_u32 v6, v4, v2 ; GCN-NEXT: v_mul_lo_u32 v10, v5, v3 ; GCN-NEXT: v_add_i32_e32 v7, vcc, 1, v4 -; GCN-NEXT: v_subrev_i32_e32 v0, vcc, v6, v0 -; GCN-NEXT: v_subrev_i32_e32 v1, vcc, v10, v1 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v6 +; GCN-NEXT: v_sub_i32_e32 v1, vcc, v1, v10 ; GCN-NEXT: v_add_i32_e32 v11, vcc, 1, v5 ; GCN-NEXT: v_cmp_ge_u32_e64 s[0:1], v0, v2 ; GCN-NEXT: v_cmp_ge_u32_e64 s[2:3], v1, v3 ; GCN-NEXT: v_cndmask_b32_e64 v4, v4, v7, s[0:1] -; GCN-NEXT: v_subrev_i32_e32 v6, vcc, v2, v0 +; GCN-NEXT: v_sub_i32_e32 v6, vcc, v0, v2 ; GCN-NEXT: v_cndmask_b32_e64 v5, v5, v11, s[2:3] -; GCN-NEXT: v_subrev_i32_e32 v7, vcc, v3, v1 +; GCN-NEXT: v_sub_i32_e32 v7, vcc, v1, v3 ; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v6, s[0:1] ; GCN-NEXT: v_add_i32_e32 v6, vcc, 1, v4 ; GCN-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[2:3] @@ -514,15 +514,15 @@ ; TONGA-NEXT: v_mul_lo_u32 v6, v4, v2 ; TONGA-NEXT: v_mul_lo_u32 v10, v5, v3 ; TONGA-NEXT: v_add_u32_e32 v7, vcc, 1, v4 -; TONGA-NEXT: v_subrev_u32_e32 v0, vcc, v6, v0 -; TONGA-NEXT: v_subrev_u32_e32 v1, vcc, v10, v1 +; TONGA-NEXT: v_sub_u32_e32 v0, vcc, v0, v6 +; TONGA-NEXT: v_sub_u32_e32 v1, vcc, v1, v10 ; TONGA-NEXT: v_add_u32_e32 v11, vcc, 1, v5 ; TONGA-NEXT: v_cmp_ge_u32_e64 s[0:1], v0, v2 ; TONGA-NEXT: v_cmp_ge_u32_e64 s[2:3], v1, v3 ; TONGA-NEXT: v_cndmask_b32_e64 v4, v4, v7, s[0:1] -; TONGA-NEXT: v_subrev_u32_e32 v6, vcc, v2, v0 +; TONGA-NEXT: v_sub_u32_e32 v6, vcc, v0, v2 ; TONGA-NEXT: v_cndmask_b32_e64 v5, v5, v11, s[2:3] -; TONGA-NEXT: v_subrev_u32_e32 v7, vcc, v3, v1 +; TONGA-NEXT: v_sub_u32_e32 v7, vcc, v1, v3 ; TONGA-NEXT: v_cndmask_b32_e64 v0, v0, v6, s[0:1] ; TONGA-NEXT: v_add_u32_e32 v6, vcc, 1, v4 ; TONGA-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[2:3] @@ -533,8 +533,8 @@ ; TONGA-NEXT: v_cndmask_b32_e32 v1, v5, v7, vcc ; TONGA-NEXT: v_xor_b32_e32 v0, v0, v8 ; TONGA-NEXT: v_xor_b32_e32 v1, v1, v9 -; TONGA-NEXT: v_sub_u32_e32 v0, vcc, v0, v8 -; TONGA-NEXT: v_sub_u32_e32 v1, vcc, v1, v9 +; TONGA-NEXT: v_subrev_u32_e32 v0, vcc, v8, v0 +; TONGA-NEXT: v_subrev_u32_e32 v1, vcc, v9, v1 ; TONGA-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; TONGA-NEXT: s_endpgm ; @@ -614,27 +614,26 @@ ; ; EG-LABEL: sdiv_v2i32: ; EG: ; %bb.0: -; EG-NEXT: ALU 0, @10, KC0[CB0:0-32], KC1[] -; EG-NEXT: TEX 1 @6 -; EG-NEXT: ALU 51, @11, KC0[CB0:0-32], KC1[] +; EG-NEXT: ALU 0, @8, KC0[CB0:0-32], KC1[] +; EG-NEXT: TEX 0 @6 +; EG-NEXT: ALU 51, @9, KC0[CB0:0-32], KC1[] ; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1 ; EG-NEXT: CF_END ; EG-NEXT: PAD ; EG-NEXT: Fetch clause starting at 6: -; EG-NEXT: VTX_READ_64 T1.XY, T0.X, 8, #1 -; EG-NEXT: VTX_READ_64 T0.XY, T0.X, 0, #1 -; EG-NEXT: ALU clause starting at 10: +; EG-NEXT: VTX_READ_128 T0.XYZW, T0.X, 0, #1 +; EG-NEXT: ALU clause starting at 8: ; EG-NEXT: MOV * T0.X, KC0[2].Z, -; EG-NEXT: ALU clause starting at 11: -; EG-NEXT: SETGT_INT * T0.W, 0.0, T1.Y, -; EG-NEXT: ADD_INT T1.W, T1.Y, PV.W, -; EG-NEXT: SETGT_INT * T2.W, 0.0, T1.X, -; EG-NEXT: XOR_INT * T1.W, PV.W, T0.W, -; EG-NEXT: SUB_INT T0.Z, 0.0, PV.W, -; EG-NEXT: ADD_INT T3.W, T1.X, T2.W, -; EG-NEXT: RECIP_UINT * T1.X, PV.W, +; EG-NEXT: ALU clause starting at 9: +; EG-NEXT: SETGT_INT * T1.W, 0.0, T0.W, +; EG-NEXT: ADD_INT T0.W, T0.W, PV.W, +; EG-NEXT: SETGT_INT * T2.W, 0.0, T0.Z, +; EG-NEXT: XOR_INT * T0.W, PV.W, T1.W, +; EG-NEXT: SUB_INT T1.Z, 0.0, PV.W, +; EG-NEXT: ADD_INT T3.W, T0.Z, T2.W, +; EG-NEXT: RECIP_UINT * T0.Z, PV.W, ; EG-NEXT: XOR_INT T3.W, PV.W, T2.W, -; EG-NEXT: MULLO_INT * T0.Z, PV.Z, PS, +; EG-NEXT: MULLO_INT * T1.X, PV.Z, PS, ; EG-NEXT: SUB_INT T4.W, 0.0, PV.W, ; EG-NEXT: RECIP_UINT * T1.Y, PV.W, ; EG-NEXT: SETGT_INT T5.W, 0.0, T0.X, @@ -645,18 +644,18 @@ ; EG-NEXT: ADD_INT T1.Y, T1.Y, PS, ; EG-NEXT: XOR_INT T1.Z, PV.W, T5.W, ; EG-NEXT: ADD_INT T4.W, T0.Y, PV.Z, BS:VEC_120/SCL_212 -; EG-NEXT: MULHI * T0.X, T1.X, T0.Z, -; EG-NEXT: ADD_INT T0.Z, T1.X, PS, +; EG-NEXT: MULHI * T0.X, T0.Z, T1.X, +; EG-NEXT: ADD_INT T0.Z, T0.Z, PS, ; EG-NEXT: XOR_INT T4.W, PV.W, T2.Z, ; EG-NEXT: MULHI * T0.X, PV.Z, PV.Y, ; EG-NEXT: MULHI * T0.Y, PV.W, PV.Z, -; EG-NEXT: MULLO_INT * T0.Z, PS, T1.W, +; EG-NEXT: MULLO_INT * T0.Z, PS, T0.W, ; EG-NEXT: SUB_INT T4.W, T4.W, PS, ; EG-NEXT: MULLO_INT * T0.Z, T0.X, T3.W, ; EG-NEXT: SUB_INT T1.Y, T1.Z, PS, ; EG-NEXT: ADD_INT T0.Z, T0.Y, 1, -; EG-NEXT: SETGE_UINT T6.W, PV.W, T1.W, -; EG-NEXT: SUB_INT * T7.W, PV.W, T1.W, +; EG-NEXT: SETGE_UINT T6.W, PV.W, T0.W, +; EG-NEXT: SUB_INT * T7.W, PV.W, T0.W, ; EG-NEXT: CNDE_INT T1.X, PV.W, T4.W, PS, BS:VEC_021/SCL_122 ; EG-NEXT: CNDE_INT T0.Y, PV.W, T0.Y, PV.Z, ; EG-NEXT: ADD_INT T0.Z, T0.X, 1, @@ -665,9 +664,9 @@ ; EG-NEXT: CNDE_INT T1.Y, PV.W, T1.Y, PS, ; EG-NEXT: CNDE_INT T0.Z, PV.W, T0.X, PV.Z, ; EG-NEXT: ADD_INT T4.W, PV.Y, 1, -; EG-NEXT: SETGE_UINT * T1.W, PV.X, T1.W, +; EG-NEXT: SETGE_UINT * T0.W, PV.X, T0.W, ; EG-NEXT: CNDE_INT T0.Y, PS, T0.Y, PV.W, -; EG-NEXT: XOR_INT T1.Z, T2.Z, T0.W, BS:VEC_021/SCL_122 +; EG-NEXT: XOR_INT T1.Z, T2.Z, T1.W, BS:VEC_021/SCL_122 ; EG-NEXT: ADD_INT T0.W, PV.Z, 1, ; EG-NEXT: SETGE_UINT * T1.W, PV.Y, T3.W, ; EG-NEXT: CNDE_INT T0.Z, PS, T0.Z, PV.W, diff --git a/llvm/test/CodeGen/AMDGPU/sdiv64.ll b/llvm/test/CodeGen/AMDGPU/sdiv64.ll --- a/llvm/test/CodeGen/AMDGPU/sdiv64.ll +++ b/llvm/test/CodeGen/AMDGPU/sdiv64.ll @@ -854,94 +854,92 @@ define amdgpu_kernel void @s_test_sdiv24_v2i64(<2 x i64> addrspace(1)* %out, <2 x i64> %x, <2 x i64> %y) { ; GCN-LABEL: s_test_sdiv24_v2i64: ; GCN: ; %bb.0: -; GCN-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xd -; GCN-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x11 +; GCN-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0xd ; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; GCN-NEXT: s_mov_b32 s3, 0xf000 ; GCN-NEXT: s_mov_b32 s2, -1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_ashr_i64 s[4:5], s[4:5], 40 ; GCN-NEXT: s_ashr_i64 s[8:9], s[8:9], 40 ; GCN-NEXT: v_cvt_f32_i32_e32 v0, s8 +; GCN-NEXT: s_ashr_i64 s[4:5], s[4:5], 40 ; GCN-NEXT: v_cvt_f32_i32_e32 v1, s4 ; GCN-NEXT: s_xor_b32 s4, s4, s8 -; GCN-NEXT: s_ashr_i32 s4, s4, 30 ; GCN-NEXT: v_rcp_iflag_f32_e32 v2, v0 +; GCN-NEXT: s_ashr_i32 s4, s4, 30 ; GCN-NEXT: s_or_b32 s4, s4, 1 ; GCN-NEXT: v_mov_b32_e32 v3, s4 -; GCN-NEXT: s_ashr_i64 s[10:11], s[10:11], 40 ; GCN-NEXT: v_mul_f32_e32 v2, v1, v2 ; GCN-NEXT: v_trunc_f32_e32 v2, v2 ; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1 ; GCN-NEXT: v_cvt_i32_f32_e32 v2, v2 ; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, |v0| +; GCN-NEXT: s_ashr_i64 s[10:11], s[10:11], 40 ; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v3, vcc -; GCN-NEXT: s_ashr_i64 s[6:7], s[6:7], 40 ; GCN-NEXT: v_add_i32_e32 v0, vcc, v0, v2 ; GCN-NEXT: v_cvt_f32_i32_e32 v2, s10 +; GCN-NEXT: s_ashr_i64 s[6:7], s[6:7], 40 ; GCN-NEXT: v_cvt_f32_i32_e32 v3, s6 ; GCN-NEXT: s_xor_b32 s4, s6, s10 -; GCN-NEXT: s_ashr_i32 s4, s4, 30 ; GCN-NEXT: v_rcp_iflag_f32_e32 v4, v2 +; GCN-NEXT: s_ashr_i32 s4, s4, 30 ; GCN-NEXT: s_or_b32 s4, s4, 1 ; GCN-NEXT: v_mov_b32_e32 v5, s4 -; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 ; GCN-NEXT: v_mul_f32_e32 v4, v3, v4 ; GCN-NEXT: v_trunc_f32_e32 v4, v4 ; GCN-NEXT: v_mad_f32 v3, -v4, v2, v3 ; GCN-NEXT: v_cvt_i32_f32_e32 v4, v4 ; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v2| ; GCN-NEXT: v_cndmask_b32_e32 v2, 0, v5, vcc -; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 +; GCN-NEXT: v_bfe_i32 v0, v0, 0, 24 ; GCN-NEXT: v_add_i32_e32 v2, vcc, v2, v4 ; GCN-NEXT: v_bfe_i32 v2, v2, 0, 24 +; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0 ; GCN-NEXT: v_ashrrev_i32_e32 v3, 31, v2 ; GCN-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 ; GCN-NEXT: s_endpgm ; ; GCN-IR-LABEL: s_test_sdiv24_v2i64: ; GCN-IR: ; %bb.0: -; GCN-IR-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0xd -; GCN-IR-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x11 +; GCN-IR-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0xd ; GCN-IR-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; GCN-IR-NEXT: s_mov_b32 s3, 0xf000 ; GCN-IR-NEXT: s_mov_b32 s2, -1 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) -; GCN-IR-NEXT: s_ashr_i64 s[4:5], s[4:5], 40 ; GCN-IR-NEXT: s_ashr_i64 s[8:9], s[8:9], 40 ; GCN-IR-NEXT: v_cvt_f32_i32_e32 v0, s8 +; GCN-IR-NEXT: s_ashr_i64 s[4:5], s[4:5], 40 ; GCN-IR-NEXT: v_cvt_f32_i32_e32 v1, s4 ; GCN-IR-NEXT: s_xor_b32 s4, s4, s8 -; GCN-IR-NEXT: s_ashr_i32 s4, s4, 30 ; GCN-IR-NEXT: v_rcp_iflag_f32_e32 v2, v0 +; GCN-IR-NEXT: s_ashr_i32 s4, s4, 30 ; GCN-IR-NEXT: s_or_b32 s4, s4, 1 ; GCN-IR-NEXT: v_mov_b32_e32 v3, s4 -; GCN-IR-NEXT: s_ashr_i64 s[10:11], s[10:11], 40 ; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2 ; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 ; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1 ; GCN-IR-NEXT: v_cvt_i32_f32_e32 v2, v2 ; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, |v0| +; GCN-IR-NEXT: s_ashr_i64 s[10:11], s[10:11], 40 ; GCN-IR-NEXT: v_cndmask_b32_e32 v0, 0, v3, vcc -; GCN-IR-NEXT: s_ashr_i64 s[6:7], s[6:7], 40 ; GCN-IR-NEXT: v_add_i32_e32 v0, vcc, v0, v2 ; GCN-IR-NEXT: v_cvt_f32_i32_e32 v2, s10 +; GCN-IR-NEXT: s_ashr_i64 s[6:7], s[6:7], 40 ; GCN-IR-NEXT: v_cvt_f32_i32_e32 v3, s6 ; GCN-IR-NEXT: s_xor_b32 s4, s6, s10 -; GCN-IR-NEXT: s_ashr_i32 s4, s4, 30 ; GCN-IR-NEXT: v_rcp_iflag_f32_e32 v4, v2 +; GCN-IR-NEXT: s_ashr_i32 s4, s4, 30 ; GCN-IR-NEXT: s_or_b32 s4, s4, 1 ; GCN-IR-NEXT: v_mov_b32_e32 v5, s4 -; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 ; GCN-IR-NEXT: v_mul_f32_e32 v4, v3, v4 ; GCN-IR-NEXT: v_trunc_f32_e32 v4, v4 ; GCN-IR-NEXT: v_mad_f32 v3, -v4, v2, v3 ; GCN-IR-NEXT: v_cvt_i32_f32_e32 v4, v4 ; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, |v2| ; GCN-IR-NEXT: v_cndmask_b32_e32 v2, 0, v5, vcc -; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 +; GCN-IR-NEXT: v_bfe_i32 v0, v0, 0, 24 ; GCN-IR-NEXT: v_add_i32_e32 v2, vcc, v2, v4 ; GCN-IR-NEXT: v_bfe_i32 v2, v2, 0, 24 +; GCN-IR-NEXT: v_ashrrev_i32_e32 v1, 31, v0 ; GCN-IR-NEXT: v_ashrrev_i32_e32 v3, 31, v2 ; GCN-IR-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 ; GCN-IR-NEXT: s_endpgm diff --git a/llvm/test/CodeGen/AMDGPU/select-vectors.ll b/llvm/test/CodeGen/AMDGPU/select-vectors.ll --- a/llvm/test/CodeGen/AMDGPU/select-vectors.ll +++ b/llvm/test/CodeGen/AMDGPU/select-vectors.ll @@ -216,16 +216,14 @@ } ; GCN-LABEL: {{^}}s_select_v2f32: -; GCN-DAG: s_load_dwordx2 s{{\[}}[[ALO:[0-9]+]]:[[AHI:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0xb|0x2c}} -; GCN-DAG: s_load_dwordx2 s{{\[}}[[BLO:[0-9]+]]:[[BHI:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0xd|0x34}} +; GCN-DAG: s_load_dwordx4 s{{\[}}[[ALO:[0-9]+]]:[[BHI:[0-9]+]]{{\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0xb|0x2c}} -; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s[[AHI]] ; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s[[BHI]] ; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s[[ALO]] ; GCN-DAG: s_cmp_eq_u32 s{{[0-9]+}}, 0{{$}} ; GCN-DAG: v_cndmask_b32_e32 -; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s[[BLO]] +; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}} ; GCN-DAG: v_cndmask_b32_e32 ; GCN: buffer_store_dwordx2 define amdgpu_kernel void @s_select_v2f32(<2 x float> addrspace(1)* %out, <2 x float> %a, <2 x float> %b, i32 %c) #0 { @@ -251,8 +249,7 @@ } ; GCN-LABEL: {{^}}s_select_v4f32: -; GCN: s_load_dwordx4 -; GCN: s_load_dwordx4 +; GCN: s_load_dwordx8 ; GCN: s_cmp_eq_u32 s{{[0-9]+}}, 0{{$}} ; GCN: v_cndmask_b32_e32 diff --git a/llvm/test/CodeGen/AMDGPU/shift-i128.ll b/llvm/test/CodeGen/AMDGPU/shift-i128.ll --- a/llvm/test/CodeGen/AMDGPU/shift-i128.ll +++ b/llvm/test/CodeGen/AMDGPU/shift-i128.ll @@ -451,64 +451,63 @@ define amdgpu_kernel void @s_shl_v2i128ss(<2 x i128> %lhs, <2 x i128> %rhs) { ; GCN-LABEL: s_shl_v2i128ss: ; GCN: ; %bb.0: -; GCN-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x8 -; GCN-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x0 +; GCN-NEXT: s_load_dwordx16 s[4:19], s[4:5], 0x0 ; GCN-NEXT: v_mov_b32_e32 v10, 16 ; GCN-NEXT: v_mov_b32_e32 v8, 0 ; GCN-NEXT: v_mov_b32_e32 v11, 0 +; GCN-NEXT: v_mov_b32_e32 v9, 0 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: v_cmp_lt_u64_e64 s[0:1], s[8:9], 64 -; GCN-NEXT: v_cmp_eq_u64_e64 s[2:3], s[10:11], 0 -; GCN-NEXT: s_sub_i32 s6, 64, s8 -; GCN-NEXT: s_sub_i32 s4, s8, 64 -; GCN-NEXT: s_lshr_b64 s[6:7], s[16:17], s6 -; GCN-NEXT: s_lshl_b64 s[24:25], s[18:19], s8 -; GCN-NEXT: s_lshl_b64 s[4:5], s[16:17], s4 -; GCN-NEXT: s_or_b64 s[6:7], s[24:25], s[6:7] +; GCN-NEXT: v_cmp_lt_u64_e64 s[0:1], s[12:13], 64 +; GCN-NEXT: v_cmp_eq_u64_e64 s[2:3], s[14:15], 0 +; GCN-NEXT: s_sub_i32 s22, 64, s12 +; GCN-NEXT: s_sub_i32 s20, s12, 64 +; GCN-NEXT: s_lshr_b64 s[22:23], s[4:5], s22 +; GCN-NEXT: s_lshl_b64 s[24:25], s[6:7], s12 +; GCN-NEXT: s_lshl_b64 s[20:21], s[4:5], s20 +; GCN-NEXT: s_or_b64 s[22:23], s[24:25], s[22:23] ; GCN-NEXT: s_and_b64 vcc, s[2:3], s[0:1] -; GCN-NEXT: s_or_b64 s[0:1], s[8:9], s[10:11] -; GCN-NEXT: v_mov_b32_e32 v0, s5 -; GCN-NEXT: v_mov_b32_e32 v1, s7 +; GCN-NEXT: s_or_b64 s[0:1], s[12:13], s[14:15] +; GCN-NEXT: v_mov_b32_e32 v0, s21 +; GCN-NEXT: v_mov_b32_e32 v1, s23 ; GCN-NEXT: v_cmp_eq_u64_e64 s[0:1], s[0:1], 0 ; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc -; GCN-NEXT: v_mov_b32_e32 v1, s19 +; GCN-NEXT: v_mov_b32_e32 v1, s7 ; GCN-NEXT: v_cndmask_b32_e64 v3, v0, v1, s[0:1] -; GCN-NEXT: v_mov_b32_e32 v0, s4 -; GCN-NEXT: v_mov_b32_e32 v1, s6 +; GCN-NEXT: v_mov_b32_e32 v0, s20 +; GCN-NEXT: v_mov_b32_e32 v1, s22 ; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc -; GCN-NEXT: v_mov_b32_e32 v1, s18 +; GCN-NEXT: v_mov_b32_e32 v1, s6 ; GCN-NEXT: v_cndmask_b32_e64 v2, v0, v1, s[0:1] -; GCN-NEXT: v_cmp_lt_u64_e64 s[0:1], s[12:13], 64 -; GCN-NEXT: v_cmp_eq_u64_e64 s[2:3], s[14:15], 0 -; GCN-NEXT: s_sub_i32 s6, 64, s12 -; GCN-NEXT: s_sub_i32 s4, s12, 64 -; GCN-NEXT: s_lshr_b64 s[6:7], s[20:21], s6 -; GCN-NEXT: s_lshl_b64 s[10:11], s[22:23], s12 -; GCN-NEXT: s_lshl_b64 s[4:5], s[20:21], s4 -; GCN-NEXT: s_or_b64 s[6:7], s[10:11], s[6:7] +; GCN-NEXT: v_cmp_lt_u64_e64 s[0:1], s[16:17], 64 +; GCN-NEXT: v_cmp_eq_u64_e64 s[2:3], s[18:19], 0 +; GCN-NEXT: s_sub_i32 s13, 64, s16 +; GCN-NEXT: s_sub_i32 s6, s16, 64 +; GCN-NEXT: s_lshr_b64 s[14:15], s[8:9], s13 +; GCN-NEXT: s_lshl_b64 s[20:21], s[10:11], s16 +; GCN-NEXT: s_lshl_b64 s[6:7], s[8:9], s6 +; GCN-NEXT: s_or_b64 s[14:15], s[20:21], s[14:15] ; GCN-NEXT: s_and_b64 s[0:1], s[2:3], s[0:1] -; GCN-NEXT: s_or_b64 s[2:3], s[12:13], s[14:15] -; GCN-NEXT: v_mov_b32_e32 v0, s5 -; GCN-NEXT: v_mov_b32_e32 v1, s7 +; GCN-NEXT: s_or_b64 s[2:3], s[16:17], s[18:19] +; GCN-NEXT: v_mov_b32_e32 v0, s7 +; GCN-NEXT: v_mov_b32_e32 v1, s15 ; GCN-NEXT: v_cmp_eq_u64_e64 s[2:3], s[2:3], 0 ; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[0:1] -; GCN-NEXT: v_mov_b32_e32 v1, s23 +; GCN-NEXT: v_mov_b32_e32 v1, s11 ; GCN-NEXT: v_cndmask_b32_e64 v7, v0, v1, s[2:3] -; GCN-NEXT: v_mov_b32_e32 v0, s4 -; GCN-NEXT: v_mov_b32_e32 v1, s6 +; GCN-NEXT: v_mov_b32_e32 v0, s6 +; GCN-NEXT: v_mov_b32_e32 v1, s14 ; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[0:1] -; GCN-NEXT: v_mov_b32_e32 v1, s22 +; GCN-NEXT: v_mov_b32_e32 v1, s10 ; GCN-NEXT: v_cndmask_b32_e64 v6, v0, v1, s[2:3] -; GCN-NEXT: s_lshl_b64 s[2:3], s[16:17], s8 +; GCN-NEXT: s_lshl_b64 s[2:3], s[4:5], s12 ; GCN-NEXT: v_mov_b32_e32 v0, s3 ; GCN-NEXT: v_cndmask_b32_e32 v1, 0, v0, vcc ; GCN-NEXT: v_mov_b32_e32 v0, s2 -; GCN-NEXT: s_lshl_b64 s[2:3], s[20:21], s12 +; GCN-NEXT: s_lshl_b64 s[2:3], s[8:9], s16 ; GCN-NEXT: v_mov_b32_e32 v4, s3 ; GCN-NEXT: v_cndmask_b32_e64 v5, 0, v4, s[0:1] ; GCN-NEXT: v_mov_b32_e32 v4, s2 ; GCN-NEXT: v_cndmask_b32_e64 v4, 0, v4, s[0:1] -; GCN-NEXT: v_mov_b32_e32 v9, 0 ; GCN-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc ; GCN-NEXT: flat_store_dwordx4 v[10:11], v[4:7] ; GCN-NEXT: flat_store_dwordx4 v[8:9], v[0:3] @@ -521,64 +520,63 @@ define amdgpu_kernel void @s_lshr_v2i128_ss(<2 x i128> %lhs, <2 x i128> %rhs) { ; GCN-LABEL: s_lshr_v2i128_ss: ; GCN: ; %bb.0: -; GCN-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x8 -; GCN-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x0 +; GCN-NEXT: s_load_dwordx16 s[4:19], s[4:5], 0x0 ; GCN-NEXT: v_mov_b32_e32 v10, 16 ; GCN-NEXT: v_mov_b32_e32 v8, 0 ; GCN-NEXT: v_mov_b32_e32 v11, 0 +; GCN-NEXT: v_mov_b32_e32 v9, 0 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: v_cmp_lt_u64_e64 s[0:1], s[8:9], 64 -; GCN-NEXT: v_cmp_eq_u64_e64 s[2:3], s[10:11], 0 -; GCN-NEXT: s_sub_i32 s6, 64, s8 -; GCN-NEXT: s_sub_i32 s4, s8, 64 -; GCN-NEXT: s_lshl_b64 s[6:7], s[18:19], s6 -; GCN-NEXT: s_lshr_b64 s[24:25], s[16:17], s8 -; GCN-NEXT: s_lshr_b64 s[4:5], s[18:19], s4 -; GCN-NEXT: s_or_b64 s[6:7], s[24:25], s[6:7] +; GCN-NEXT: v_cmp_lt_u64_e64 s[0:1], s[12:13], 64 +; GCN-NEXT: v_cmp_eq_u64_e64 s[2:3], s[14:15], 0 +; GCN-NEXT: s_sub_i32 s22, 64, s12 +; GCN-NEXT: s_sub_i32 s20, s12, 64 +; GCN-NEXT: s_lshl_b64 s[22:23], s[6:7], s22 +; GCN-NEXT: s_lshr_b64 s[24:25], s[4:5], s12 +; GCN-NEXT: s_lshr_b64 s[20:21], s[6:7], s20 +; GCN-NEXT: s_or_b64 s[22:23], s[24:25], s[22:23] ; GCN-NEXT: s_and_b64 vcc, s[2:3], s[0:1] -; GCN-NEXT: s_or_b64 s[0:1], s[8:9], s[10:11] -; GCN-NEXT: v_mov_b32_e32 v0, s5 -; GCN-NEXT: v_mov_b32_e32 v1, s7 +; GCN-NEXT: s_or_b64 s[0:1], s[12:13], s[14:15] +; GCN-NEXT: v_mov_b32_e32 v0, s21 +; GCN-NEXT: v_mov_b32_e32 v1, s23 ; GCN-NEXT: v_cmp_eq_u64_e64 s[0:1], s[0:1], 0 ; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc -; GCN-NEXT: v_mov_b32_e32 v1, s17 +; GCN-NEXT: v_mov_b32_e32 v1, s5 ; GCN-NEXT: v_cndmask_b32_e64 v1, v0, v1, s[0:1] -; GCN-NEXT: v_mov_b32_e32 v0, s4 -; GCN-NEXT: v_mov_b32_e32 v2, s6 +; GCN-NEXT: v_mov_b32_e32 v0, s20 +; GCN-NEXT: v_mov_b32_e32 v2, s22 ; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GCN-NEXT: v_mov_b32_e32 v2, s16 +; GCN-NEXT: v_mov_b32_e32 v2, s4 ; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v2, s[0:1] -; GCN-NEXT: v_cmp_lt_u64_e64 s[0:1], s[12:13], 64 -; GCN-NEXT: v_cmp_eq_u64_e64 s[2:3], s[14:15], 0 -; GCN-NEXT: s_sub_i32 s6, 64, s12 -; GCN-NEXT: s_sub_i32 s4, s12, 64 -; GCN-NEXT: s_lshl_b64 s[6:7], s[22:23], s6 -; GCN-NEXT: s_lshr_b64 s[10:11], s[20:21], s12 -; GCN-NEXT: s_lshr_b64 s[4:5], s[22:23], s4 -; GCN-NEXT: s_or_b64 s[6:7], s[10:11], s[6:7] +; GCN-NEXT: v_cmp_lt_u64_e64 s[0:1], s[16:17], 64 +; GCN-NEXT: v_cmp_eq_u64_e64 s[2:3], s[18:19], 0 +; GCN-NEXT: s_sub_i32 s13, 64, s16 +; GCN-NEXT: s_sub_i32 s4, s16, 64 +; GCN-NEXT: s_lshl_b64 s[14:15], s[10:11], s13 +; GCN-NEXT: s_lshr_b64 s[20:21], s[8:9], s16 +; GCN-NEXT: s_lshr_b64 s[4:5], s[10:11], s4 +; GCN-NEXT: s_or_b64 s[14:15], s[20:21], s[14:15] ; GCN-NEXT: s_and_b64 s[0:1], s[2:3], s[0:1] -; GCN-NEXT: s_or_b64 s[2:3], s[12:13], s[14:15] +; GCN-NEXT: s_or_b64 s[2:3], s[16:17], s[18:19] ; GCN-NEXT: v_mov_b32_e32 v2, s5 -; GCN-NEXT: v_mov_b32_e32 v3, s7 +; GCN-NEXT: v_mov_b32_e32 v3, s15 ; GCN-NEXT: v_cmp_eq_u64_e64 s[2:3], s[2:3], 0 ; GCN-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[0:1] -; GCN-NEXT: v_mov_b32_e32 v3, s21 +; GCN-NEXT: v_mov_b32_e32 v3, s9 ; GCN-NEXT: v_cndmask_b32_e64 v5, v2, v3, s[2:3] ; GCN-NEXT: v_mov_b32_e32 v2, s4 -; GCN-NEXT: v_mov_b32_e32 v3, s6 +; GCN-NEXT: v_mov_b32_e32 v3, s14 ; GCN-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[0:1] -; GCN-NEXT: v_mov_b32_e32 v3, s20 +; GCN-NEXT: v_mov_b32_e32 v3, s8 ; GCN-NEXT: v_cndmask_b32_e64 v4, v2, v3, s[2:3] -; GCN-NEXT: s_lshr_b64 s[2:3], s[18:19], s8 +; GCN-NEXT: s_lshr_b64 s[2:3], s[6:7], s12 ; GCN-NEXT: v_mov_b32_e32 v2, s3 ; GCN-NEXT: v_cndmask_b32_e32 v3, 0, v2, vcc ; GCN-NEXT: v_mov_b32_e32 v2, s2 -; GCN-NEXT: s_lshr_b64 s[2:3], s[22:23], s12 +; GCN-NEXT: s_lshr_b64 s[2:3], s[10:11], s16 ; GCN-NEXT: v_mov_b32_e32 v6, s3 ; GCN-NEXT: v_cndmask_b32_e64 v7, 0, v6, s[0:1] ; GCN-NEXT: v_mov_b32_e32 v6, s2 ; GCN-NEXT: v_cndmask_b32_e64 v6, 0, v6, s[0:1] -; GCN-NEXT: v_mov_b32_e32 v9, 0 ; GCN-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc ; GCN-NEXT: flat_store_dwordx4 v[10:11], v[4:7] ; GCN-NEXT: flat_store_dwordx4 v[8:9], v[0:3] @@ -591,61 +589,61 @@ define amdgpu_kernel void @s_ashr_v2i128_ss(<2 x i128> %lhs, <2 x i128> %rhs) { ; GCN-LABEL: s_ashr_v2i128_ss: ; GCN: ; %bb.0: -; GCN-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x8 -; GCN-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x0 +; GCN-NEXT: s_load_dwordx16 s[4:19], s[4:5], 0x0 ; GCN-NEXT: v_mov_b32_e32 v10, 16 ; GCN-NEXT: v_mov_b32_e32 v8, 0 ; GCN-NEXT: v_mov_b32_e32 v11, 0 +; GCN-NEXT: v_mov_b32_e32 v9, 0 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: v_cmp_lt_u64_e64 s[0:1], s[8:9], 64 -; GCN-NEXT: v_cmp_eq_u64_e64 s[2:3], s[10:11], 0 -; GCN-NEXT: s_sub_i32 s6, 64, s8 -; GCN-NEXT: s_sub_i32 s4, s8, 64 -; GCN-NEXT: s_lshl_b64 s[6:7], s[18:19], s6 -; GCN-NEXT: s_lshr_b64 s[24:25], s[16:17], s8 -; GCN-NEXT: s_ashr_i64 s[4:5], s[18:19], s4 -; GCN-NEXT: s_or_b64 s[6:7], s[24:25], s[6:7] +; GCN-NEXT: v_cmp_lt_u64_e64 s[0:1], s[12:13], 64 +; GCN-NEXT: v_cmp_eq_u64_e64 s[2:3], s[14:15], 0 +; GCN-NEXT: s_sub_i32 s22, 64, s12 +; GCN-NEXT: s_sub_i32 s20, s12, 64 +; GCN-NEXT: s_lshl_b64 s[22:23], s[6:7], s22 +; GCN-NEXT: s_lshr_b64 s[24:25], s[4:5], s12 +; GCN-NEXT: s_ashr_i64 s[20:21], s[6:7], s20 +; GCN-NEXT: s_or_b64 s[22:23], s[24:25], s[22:23] ; GCN-NEXT: s_and_b64 vcc, s[2:3], s[0:1] -; GCN-NEXT: s_or_b64 s[0:1], s[8:9], s[10:11] -; GCN-NEXT: v_mov_b32_e32 v0, s5 -; GCN-NEXT: v_mov_b32_e32 v1, s7 +; GCN-NEXT: s_or_b64 s[0:1], s[12:13], s[14:15] +; GCN-NEXT: v_mov_b32_e32 v0, s21 +; GCN-NEXT: v_mov_b32_e32 v1, s23 ; GCN-NEXT: v_cmp_eq_u64_e64 s[0:1], s[0:1], 0 ; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc -; GCN-NEXT: v_mov_b32_e32 v1, s17 +; GCN-NEXT: v_mov_b32_e32 v1, s5 ; GCN-NEXT: v_cndmask_b32_e64 v1, v0, v1, s[0:1] -; GCN-NEXT: v_mov_b32_e32 v0, s4 -; GCN-NEXT: v_mov_b32_e32 v2, s6 +; GCN-NEXT: v_mov_b32_e32 v0, s20 +; GCN-NEXT: v_mov_b32_e32 v2, s22 ; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc -; GCN-NEXT: v_mov_b32_e32 v2, s16 +; GCN-NEXT: v_mov_b32_e32 v2, s4 ; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v2, s[0:1] -; GCN-NEXT: v_cmp_lt_u64_e64 s[0:1], s[12:13], 64 -; GCN-NEXT: v_cmp_eq_u64_e64 s[2:3], s[14:15], 0 -; GCN-NEXT: s_sub_i32 s6, 64, s12 -; GCN-NEXT: s_sub_i32 s4, s12, 64 -; GCN-NEXT: s_lshl_b64 s[6:7], s[22:23], s6 -; GCN-NEXT: s_lshr_b64 s[10:11], s[20:21], s12 -; GCN-NEXT: s_ashr_i64 s[4:5], s[22:23], s4 -; GCN-NEXT: s_or_b64 s[6:7], s[10:11], s[6:7] +; GCN-NEXT: v_cmp_lt_u64_e64 s[0:1], s[16:17], 64 +; GCN-NEXT: v_cmp_eq_u64_e64 s[2:3], s[18:19], 0 +; GCN-NEXT: s_sub_i32 s13, 64, s16 +; GCN-NEXT: s_sub_i32 s4, s16, 64 +; GCN-NEXT: s_lshl_b64 s[14:15], s[10:11], s13 +; GCN-NEXT: s_lshr_b64 s[20:21], s[8:9], s16 +; GCN-NEXT: s_ashr_i64 s[4:5], s[10:11], s4 +; GCN-NEXT: s_or_b64 s[14:15], s[20:21], s[14:15] ; GCN-NEXT: s_and_b64 s[0:1], s[2:3], s[0:1] -; GCN-NEXT: s_or_b64 s[2:3], s[12:13], s[14:15] +; GCN-NEXT: s_or_b64 s[2:3], s[16:17], s[18:19] ; GCN-NEXT: v_mov_b32_e32 v2, s5 -; GCN-NEXT: v_mov_b32_e32 v3, s7 +; GCN-NEXT: v_mov_b32_e32 v3, s15 ; GCN-NEXT: v_cmp_eq_u64_e64 s[2:3], s[2:3], 0 ; GCN-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[0:1] -; GCN-NEXT: v_mov_b32_e32 v3, s21 +; GCN-NEXT: v_mov_b32_e32 v3, s9 ; GCN-NEXT: v_cndmask_b32_e64 v5, v2, v3, s[2:3] ; GCN-NEXT: v_mov_b32_e32 v2, s4 -; GCN-NEXT: v_mov_b32_e32 v3, s6 +; GCN-NEXT: v_mov_b32_e32 v3, s14 ; GCN-NEXT: v_cndmask_b32_e64 v2, v2, v3, s[0:1] -; GCN-NEXT: v_mov_b32_e32 v3, s20 +; GCN-NEXT: v_mov_b32_e32 v3, s8 ; GCN-NEXT: v_cndmask_b32_e64 v4, v2, v3, s[2:3] -; GCN-NEXT: s_ashr_i32 s4, s19, 31 -; GCN-NEXT: s_ashr_i64 s[2:3], s[18:19], s8 +; GCN-NEXT: s_ashr_i32 s4, s7, 31 +; GCN-NEXT: s_ashr_i64 s[2:3], s[6:7], s12 ; GCN-NEXT: v_mov_b32_e32 v2, s4 ; GCN-NEXT: v_mov_b32_e32 v3, s3 ; GCN-NEXT: v_mov_b32_e32 v6, s2 -; GCN-NEXT: s_ashr_i32 s4, s23, 31 -; GCN-NEXT: s_ashr_i64 s[2:3], s[22:23], s12 +; GCN-NEXT: s_ashr_i32 s4, s11, 31 +; GCN-NEXT: s_ashr_i64 s[2:3], s[10:11], s16 ; GCN-NEXT: v_cndmask_b32_e32 v3, v2, v3, vcc ; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v6, vcc ; GCN-NEXT: v_mov_b32_e32 v6, s4 @@ -653,7 +651,6 @@ ; GCN-NEXT: v_mov_b32_e32 v12, s2 ; GCN-NEXT: v_cndmask_b32_e64 v7, v6, v7, s[0:1] ; GCN-NEXT: v_cndmask_b32_e64 v6, v6, v12, s[0:1] -; GCN-NEXT: v_mov_b32_e32 v9, 0 ; GCN-NEXT: flat_store_dwordx4 v[10:11], v[4:7] ; GCN-NEXT: flat_store_dwordx4 v[8:9], v[0:3] ; GCN-NEXT: s_endpgm diff --git a/llvm/test/CodeGen/AMDGPU/shl.ll b/llvm/test/CodeGen/AMDGPU/shl.ll --- a/llvm/test/CodeGen/AMDGPU/shl.ll +++ b/llvm/test/CodeGen/AMDGPU/shl.ll @@ -31,8 +31,7 @@ ; VI: ; %bb.0: ; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0 -; VI-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x8 +; VI-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0 ; VI-NEXT: s_mov_b32 s3, 0xf000 ; VI-NEXT: s_mov_b32 s2, -1 ; VI-NEXT: s_waitcnt lgkmcnt(0) @@ -45,20 +44,19 @@ ; ; EG-LABEL: shl_v2i32: ; EG: ; %bb.0: -; EG-NEXT: ALU 0, @10, KC0[CB0:0-32], KC1[] -; EG-NEXT: TEX 1 @6 -; EG-NEXT: ALU 3, @11, KC0[CB0:0-32], KC1[] +; EG-NEXT: ALU 0, @8, KC0[CB0:0-32], KC1[] +; EG-NEXT: TEX 0 @6 +; EG-NEXT: ALU 3, @9, KC0[CB0:0-32], KC1[] ; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1 ; EG-NEXT: CF_END ; EG-NEXT: PAD ; EG-NEXT: Fetch clause starting at 6: -; EG-NEXT: VTX_READ_64 T1.XY, T0.X, 8, #1 -; EG-NEXT: VTX_READ_64 T0.XY, T0.X, 0, #1 -; EG-NEXT: ALU clause starting at 10: +; EG-NEXT: VTX_READ_128 T0.XYZW, T0.X, 0, #1 +; EG-NEXT: ALU clause starting at 8: ; EG-NEXT: MOV * T0.X, KC0[2].Z, -; EG-NEXT: ALU clause starting at 11: -; EG-NEXT: LSHL * T0.Y, T0.Y, T1.Y, -; EG-NEXT: LSHL T0.X, T0.X, T1.X, +; EG-NEXT: ALU clause starting at 9: +; EG-NEXT: LSHL * T0.Y, T0.Y, T0.W, +; EG-NEXT: LSHL T0.X, T0.X, T0.Z, ; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, ; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00) %b_ptr = getelementptr <2 x i32>, <2 x i32> addrspace(1)* %in, i32 1 @@ -94,22 +92,21 @@ ; ; VI-LABEL: shl_v4i32: ; VI: ; %bb.0: -; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; VI-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x24 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0 -; VI-NEXT: s_load_dwordx4 s[8:11], s[2:3], 0x10 -; VI-NEXT: s_mov_b32 s3, 0xf000 -; VI-NEXT: s_mov_b32 s2, -1 +; VI-NEXT: s_load_dwordx8 s[0:7], s[10:11], 0x0 +; VI-NEXT: s_mov_b32 s11, 0xf000 +; VI-NEXT: s_mov_b32 s10, -1 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_lshl_b32 s7, s7, s11 -; VI-NEXT: s_lshl_b32 s6, s6, s10 -; VI-NEXT: s_lshl_b32 s5, s5, s9 -; VI-NEXT: s_lshl_b32 s4, s4, s8 -; VI-NEXT: v_mov_b32_e32 v0, s4 -; VI-NEXT: v_mov_b32_e32 v1, s5 -; VI-NEXT: v_mov_b32_e32 v2, s6 -; VI-NEXT: v_mov_b32_e32 v3, s7 -; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 +; VI-NEXT: s_lshl_b32 s3, s3, s7 +; VI-NEXT: s_lshl_b32 s2, s2, s6 +; VI-NEXT: s_lshl_b32 s1, s1, s5 +; VI-NEXT: s_lshl_b32 s0, s0, s4 +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: v_mov_b32_e32 v2, s2 +; VI-NEXT: v_mov_b32_e32 v3, s3 +; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0 ; VI-NEXT: s_endpgm ; ; EG-LABEL: shl_v4i32: @@ -637,31 +634,29 @@ ; SI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 ; SI-NEXT: s_mov_b32 s7, 0xf000 ; SI-NEXT: s_mov_b32 s6, 0 -; SI-NEXT: v_lshlrev_b32_e32 v0, 3, v0 -; SI-NEXT: v_mov_b32_e32 v1, 0 +; SI-NEXT: v_lshlrev_b32_e32 v4, 3, v0 +; SI-NEXT: v_mov_b32_e32 v5, 0 ; SI-NEXT: s_waitcnt lgkmcnt(0) ; SI-NEXT: s_mov_b64 s[4:5], s[2:3] -; SI-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64 -; SI-NEXT: buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8 +; SI-NEXT: buffer_load_dwordx4 v[0:3], v[4:5], s[4:7], 0 addr64 ; SI-NEXT: s_mov_b32 s4, 0xffff ; SI-NEXT: s_mov_b64 s[2:3], s[6:7] -; SI-NEXT: s_waitcnt vmcnt(1) -; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v2 -; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v3 ; SI-NEXT: s_waitcnt vmcnt(0) -; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v4 -; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v5 -; SI-NEXT: v_lshlrev_b32_e32 v3, v5, v3 -; SI-NEXT: v_lshlrev_b32_e32 v2, v4, v2 -; SI-NEXT: v_lshlrev_b32_e32 v4, v9, v7 -; SI-NEXT: v_lshlrev_b32_e32 v5, v8, v6 -; SI-NEXT: v_and_b32_e32 v3, s4, v3 -; SI-NEXT: v_and_b32_e32 v2, s4, v2 -; SI-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; SI-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; SI-NEXT: v_or_b32_e32 v3, v3, v4 -; SI-NEXT: v_or_b32_e32 v2, v2, v5 -; SI-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64 +; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v0 +; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v1 +; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v2 +; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v3 +; SI-NEXT: v_lshlrev_b32_e32 v1, v3, v1 +; SI-NEXT: v_lshlrev_b32_e32 v0, v2, v0 +; SI-NEXT: v_lshlrev_b32_e32 v2, v9, v7 +; SI-NEXT: v_lshlrev_b32_e32 v3, v8, v6 +; SI-NEXT: v_and_b32_e32 v1, s4, v1 +; SI-NEXT: v_and_b32_e32 v0, s4, v0 +; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; SI-NEXT: v_or_b32_e32 v1, v1, v2 +; SI-NEXT: v_or_b32_e32 v0, v0, v3 +; SI-NEXT: buffer_store_dwordx2 v[0:1], v[4:5], s[0:3], 0 addr64 ; SI-NEXT: s_endpgm ; ; VI-LABEL: shl_v4i16: @@ -672,10 +667,7 @@ ; VI-NEXT: v_mov_b32_e32 v1, s3 ; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v4 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc -; VI-NEXT: v_add_u32_e32 v2, vcc, 8, v0 -; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc -; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1] -; VI-NEXT: flat_load_dwordx2 v[2:3], v[2:3] +; VI-NEXT: flat_load_dwordx4 v[0:3], v[0:1] ; VI-NEXT: v_mov_b32_e32 v5, s1 ; VI-NEXT: v_add_u32_e32 v4, vcc, s0, v4 ; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc @@ -691,45 +683,40 @@ ; ; EG-LABEL: shl_v4i16: ; EG: ; %bb.0: -; EG-NEXT: ALU 2, @12, KC0[CB0:0-32], KC1[] -; EG-NEXT: TEX 0 @8 -; EG-NEXT: ALU 3, @15, KC0[], KC1[] -; EG-NEXT: TEX 0 @10 -; EG-NEXT: ALU 49, @19, KC0[CB0:0-32], KC1[] +; EG-NEXT: ALU 2, @8, KC0[CB0:0-32], KC1[] +; EG-NEXT: TEX 0 @6 +; EG-NEXT: ALU 53, @11, KC0[CB0:0-32], KC1[] ; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T10.XY, T0.X, 1 ; EG-NEXT: CF_END ; EG-NEXT: PAD -; EG-NEXT: Fetch clause starting at 8: -; EG-NEXT: VTX_READ_64 T10.XY, T0.X, 0, #1 -; EG-NEXT: Fetch clause starting at 10: -; EG-NEXT: VTX_READ_64 T10.XY, T0.X, 8, #1 -; EG-NEXT: ALU clause starting at 12: +; EG-NEXT: Fetch clause starting at 6: +; EG-NEXT: VTX_READ_128 T10.XYZW, T0.X, 0, #1 +; EG-NEXT: ALU clause starting at 8: ; EG-NEXT: LSHL * T0.W, T0.X, literal.x, ; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00) ; EG-NEXT: ADD_INT * T0.X, KC0[2].Z, PV.W, -; EG-NEXT: ALU clause starting at 15: +; EG-NEXT: ALU clause starting at 11: ; EG-NEXT: MOV T4.X, T10.X, ; EG-NEXT: MOV * T5.X, T10.Y, -; EG-NEXT: MOV T0.Y, PV.X, -; EG-NEXT: MOV * T0.Z, PS, -; EG-NEXT: ALU clause starting at 19: -; EG-NEXT: MOV T2.X, T10.X, -; EG-NEXT: MOV * T3.X, T10.Y, -; EG-NEXT: MOV T0.X, T6.X, -; EG-NEXT: MOV * T1.Y, PV.X, +; EG-NEXT: MOV T0.X, PV.X, +; EG-NEXT: MOV T0.Y, PS, +; EG-NEXT: MOV * T2.X, T10.Z, +; EG-NEXT: MOV T3.X, T10.W, +; EG-NEXT: MOV * T0.Z, T6.X, +; EG-NEXT: MOV * T1.Y, T2.X, ; EG-NEXT: AND_INT T1.W, PV.Y, literal.x, -; EG-NEXT: AND_INT * T2.W, T0.Y, literal.x, +; EG-NEXT: AND_INT * T2.W, T0.X, literal.x, ; EG-NEXT: 65535(9.183409e-41), 0(0.000000e+00) ; EG-NEXT: LSHL * T1.W, PS, PV.W, ; EG-NEXT: AND_INT T1.W, PV.W, literal.x, -; EG-NEXT: AND_INT * T2.W, T0.X, literal.y, +; EG-NEXT: AND_INT * T2.W, T0.Z, literal.y, ; EG-NEXT: 65535(9.183409e-41), -65536(nan) ; EG-NEXT: OR_INT * T1.W, PS, PV.W, -; EG-NEXT: MOV T0.X, T3.X, -; EG-NEXT: MOV * T6.X, PV.W, -; EG-NEXT: MOV T1.Z, PS, +; EG-NEXT: MOV * T0.Z, T3.X, +; EG-NEXT: MOV * T6.X, T1.W, +; EG-NEXT: MOV T1.Z, PV.X, ; EG-NEXT: LSHR T1.W, T1.Y, literal.x, -; EG-NEXT: LSHR * T2.W, T0.Y, literal.x, +; EG-NEXT: LSHR * T2.W, T0.X, literal.x, ; EG-NEXT: 16(2.242078e-44), 0(0.000000e+00) ; EG-NEXT: LSHL T1.W, PS, PV.W, ; EG-NEXT: AND_INT * T2.W, PV.Z, literal.x, @@ -738,23 +725,23 @@ ; EG-NEXT: 16(2.242078e-44), 0(0.000000e+00) ; EG-NEXT: OR_INT * T1.W, T2.W, PV.W, ; EG-NEXT: MOV T6.X, PV.W, -; EG-NEXT: MOV T0.Y, T7.X, -; EG-NEXT: AND_INT T1.W, T0.X, literal.x, BS:VEC_120/SCL_212 -; EG-NEXT: AND_INT * T2.W, T0.Z, literal.x, +; EG-NEXT: MOV * T0.X, T7.X, +; EG-NEXT: AND_INT T1.W, T0.Z, literal.x, +; EG-NEXT: AND_INT * T2.W, T0.Y, literal.x, ; EG-NEXT: 65535(9.183409e-41), 0(0.000000e+00) ; EG-NEXT: LSHL T1.W, PS, PV.W, -; EG-NEXT: AND_INT * T2.W, PV.Y, literal.x, +; EG-NEXT: AND_INT * T2.W, T0.X, literal.x, ; EG-NEXT: -65536(nan), 0(0.000000e+00) ; EG-NEXT: AND_INT * T1.W, PV.W, literal.x, ; EG-NEXT: 65535(9.183409e-41), 0(0.000000e+00) ; EG-NEXT: OR_INT * T1.W, T2.W, PV.W, ; EG-NEXT: MOV * T7.X, PV.W, -; EG-NEXT: MOV T0.Y, PV.X, -; EG-NEXT: LSHR T1.W, T0.X, literal.x, -; EG-NEXT: LSHR * T2.W, T0.Z, literal.x, +; EG-NEXT: MOV T0.X, PV.X, +; EG-NEXT: LSHR T1.W, T0.Z, literal.x, +; EG-NEXT: LSHR * T2.W, T0.Y, literal.x, ; EG-NEXT: 16(2.242078e-44), 0(0.000000e+00) ; EG-NEXT: LSHL * T1.W, PS, PV.W, -; EG-NEXT: AND_INT T0.Z, T0.Y, literal.x, +; EG-NEXT: AND_INT T0.Z, T0.X, literal.x, ; EG-NEXT: LSHL T1.W, PV.W, literal.y, ; EG-NEXT: ADD_INT * T0.W, KC0[2].Y, T0.W, ; EG-NEXT: 65535(9.183409e-41), 16(2.242078e-44) @@ -864,20 +851,19 @@ ; ; VI-LABEL: shl_v2i64: ; VI: ; %bb.0: -; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; VI-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x24 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0 -; VI-NEXT: s_load_dwordx4 s[8:11], s[2:3], 0x10 -; VI-NEXT: s_mov_b32 s3, 0xf000 -; VI-NEXT: s_mov_b32 s2, -1 +; VI-NEXT: s_load_dwordx8 s[0:7], s[10:11], 0x0 +; VI-NEXT: s_mov_b32 s11, 0xf000 +; VI-NEXT: s_mov_b32 s10, -1 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_lshl_b64 s[6:7], s[6:7], s10 -; VI-NEXT: s_lshl_b64 s[4:5], s[4:5], s8 -; VI-NEXT: v_mov_b32_e32 v0, s4 -; VI-NEXT: v_mov_b32_e32 v1, s5 -; VI-NEXT: v_mov_b32_e32 v2, s6 -; VI-NEXT: v_mov_b32_e32 v3, s7 -; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 +; VI-NEXT: s_lshl_b64 s[2:3], s[2:3], s6 +; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], s4 +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: v_mov_b32_e32 v2, s2 +; VI-NEXT: v_mov_b32_e32 v3, s3 +; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0 ; VI-NEXT: s_endpgm ; ; EG-LABEL: shl_v2i64: @@ -956,8 +942,7 @@ ; VI: ; %bb.0: ; VI-NEXT: s_load_dwordx4 s[16:19], s[0:1], 0x24 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_load_dwordx8 s[0:7], s[18:19], 0x0 -; VI-NEXT: s_load_dwordx8 s[8:15], s[18:19], 0x20 +; VI-NEXT: s_load_dwordx16 s[0:15], s[18:19], 0x0 ; VI-NEXT: s_mov_b32 s19, 0xf000 ; VI-NEXT: s_mov_b32 s18, -1 ; VI-NEXT: s_waitcnt lgkmcnt(0) diff --git a/llvm/test/CodeGen/AMDGPU/shl.v2i16.ll b/llvm/test/CodeGen/AMDGPU/shl.v2i16.ll --- a/llvm/test/CodeGen/AMDGPU/shl.v2i16.ll +++ b/llvm/test/CodeGen/AMDGPU/shl.v2i16.ll @@ -7,8 +7,7 @@ define amdgpu_kernel void @s_shl_v2i16(<2 x i16> addrspace(1)* %out, <2 x i16> %lhs, <2 x i16> %rhs) #0 { ; GFX9-LABEL: s_shl_v2i16: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dword s2, s[0:1], 0x2c -; GFX9-NEXT: s_load_dword s3, s[0:1], 0x30 +; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2c ; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24 ; GFX9-NEXT: s_mov_b32 s7, 0xf000 ; GFX9-NEXT: s_mov_b32 s6, -1 @@ -20,29 +19,27 @@ ; ; VI-LABEL: s_shl_v2i16: ; VI: ; %bb.0: -; VI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24 -; VI-NEXT: s_load_dword s2, s[0:1], 0x2c -; VI-NEXT: s_load_dword s0, s[0:1], 0x30 -; VI-NEXT: s_mov_b32 s1, 0xffff -; VI-NEXT: s_mov_b32 s7, 0xf000 -; VI-NEXT: s_mov_b32 s6, -1 +; VI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x2c +; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 +; VI-NEXT: s_mov_b32 s6, 0xffff +; VI-NEXT: s_mov_b32 s3, 0xf000 +; VI-NEXT: s_mov_b32 s2, -1 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_and_b32 s3, s2, s1 -; VI-NEXT: s_lshr_b32 s2, s2, 16 -; VI-NEXT: s_lshr_b32 s8, s0, 16 -; VI-NEXT: s_lshl_b32 s2, s2, s8 -; VI-NEXT: s_lshl_b32 s0, s3, s0 -; VI-NEXT: s_lshl_b32 s2, s2, 16 -; VI-NEXT: s_and_b32 s0, s0, s1 -; VI-NEXT: s_or_b32 s0, s0, s2 -; VI-NEXT: v_mov_b32_e32 v0, s0 -; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0 +; VI-NEXT: s_and_b32 s7, s4, s6 +; VI-NEXT: s_lshr_b32 s4, s4, 16 +; VI-NEXT: s_lshr_b32 s8, s5, 16 +; VI-NEXT: s_lshl_b32 s4, s4, s8 +; VI-NEXT: s_lshl_b32 s5, s7, s5 +; VI-NEXT: s_lshl_b32 s4, s4, 16 +; VI-NEXT: s_and_b32 s5, s5, s6 +; VI-NEXT: s_or_b32 s4, s5, s4 +; VI-NEXT: v_mov_b32_e32 v0, s4 +; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; VI-NEXT: s_endpgm ; ; CI-LABEL: s_shl_v2i16: ; CI: ; %bb.0: -; CI-NEXT: s_load_dword s4, s[0:1], 0xb -; CI-NEXT: s_load_dword s5, s[0:1], 0xc +; CI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0xb ; CI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; CI-NEXT: s_mov_b32 s3, 0xf000 ; CI-NEXT: s_mov_b32 s2, -1 @@ -60,9 +57,8 @@ ; ; GFX10-LABEL: s_shl_v2i16: ; GFX10: ; %bb.0: -; GFX10-NEXT: s_clause 0x2 -; GFX10-NEXT: s_load_dword s2, s[0:1], 0x2c -; GFX10-NEXT: s_load_dword s3, s[0:1], 0x30 +; GFX10-NEXT: s_clause 0x1 +; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2c ; GFX10-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24 ; GFX10-NEXT: s_mov_b32 s7, 0x31016000 ; GFX10-NEXT: s_mov_b32 s6, -1 @@ -79,35 +75,31 @@ ; GFX9-LABEL: v_shl_v2i16: ; GFX9: ; %bb.0: ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 -; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: global_load_dword v1, v0, s[2:3] -; GFX9-NEXT: global_load_dword v2, v0, s[2:3] offset:4 +; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[2:3] ; GFX9-NEXT: s_waitcnt vmcnt(0) -; GFX9-NEXT: v_pk_lshlrev_b16 v1, v2, v1 -; GFX9-NEXT: global_store_dword v0, v1, s[0:1] +; GFX9-NEXT: v_pk_lshlrev_b16 v0, v1, v0 +; GFX9-NEXT: global_store_dword v2, v0, s[0:1] ; GFX9-NEXT: s_endpgm ; ; VI-LABEL: v_shl_v2i16: ; VI: ; %bb.0: ; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 -; VI-NEXT: v_lshlrev_b32_e32 v4, 2, v0 +; VI-NEXT: v_lshlrev_b32_e32 v2, 2, v0 ; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: v_mov_b32_e32 v1, s3 -; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v4 -; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc -; VI-NEXT: v_add_u32_e32 v2, vcc, 4, v0 -; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc -; VI-NEXT: flat_load_dword v5, v[0:1] -; VI-NEXT: flat_load_dword v2, v[2:3] -; VI-NEXT: v_mov_b32_e32 v1, s1 -; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v4 +; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v2 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc +; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1] +; VI-NEXT: v_mov_b32_e32 v3, s1 +; VI-NEXT: v_add_u32_e32 v2, vcc, s0, v2 +; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v3, vcc ; VI-NEXT: s_waitcnt vmcnt(0) -; VI-NEXT: v_lshlrev_b16_e32 v3, v2, v5 -; VI-NEXT: v_lshlrev_b16_sdwa v2, v2, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1 -; VI-NEXT: v_or_b32_e32 v2, v3, v2 -; VI-NEXT: flat_store_dword v[0:1], v2 +; VI-NEXT: v_lshlrev_b16_e32 v4, v1, v0 +; VI-NEXT: v_lshlrev_b16_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1 +; VI-NEXT: v_or_b32_e32 v0, v4, v0 +; VI-NEXT: flat_store_dword v[2:3], v0 ; VI-NEXT: s_endpgm ; ; CI-LABEL: v_shl_v2i16: @@ -119,17 +111,15 @@ ; CI-NEXT: v_mov_b32_e32 v1, 0 ; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_mov_b64 s[4:5], s[2:3] -; CI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64 -; CI-NEXT: buffer_load_dword v3, v[0:1], s[4:7], 0 addr64 offset:4 +; CI-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64 ; CI-NEXT: s_mov_b64 s[2:3], s[6:7] -; CI-NEXT: s_waitcnt vmcnt(1) -; CI-NEXT: v_lshrrev_b32_e32 v4, 16, v2 ; CI-NEXT: s_waitcnt vmcnt(0) +; CI-NEXT: v_lshrrev_b32_e32 v4, 16, v2 ; CI-NEXT: v_lshrrev_b32_e32 v5, 16, v3 ; CI-NEXT: v_lshlrev_b32_e32 v2, v3, v2 ; CI-NEXT: v_lshlrev_b32_e32 v3, v5, v4 -; CI-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; CI-NEXT: v_and_b32_e32 v2, 0xffff, v2 +; CI-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; CI-NEXT: v_or_b32_e32 v2, v2, v3 ; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64 ; CI-NEXT: s_endpgm @@ -137,14 +127,12 @@ ; GFX10-LABEL: v_shl_v2i16: ; GFX10: ; %bb.0: ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 -; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0 +; GFX10-NEXT: v_lshlrev_b32_e32 v2, 2, v0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_clause 0x1 -; GFX10-NEXT: global_load_dword v1, v0, s[2:3] -; GFX10-NEXT: global_load_dword v2, v0, s[2:3] offset:4 +; GFX10-NEXT: global_load_dwordx2 v[0:1], v2, s[2:3] ; GFX10-NEXT: s_waitcnt vmcnt(0) -; GFX10-NEXT: v_pk_lshlrev_b16 v1, v2, v1 -; GFX10-NEXT: global_store_dword v0, v1, s[0:1] +; GFX10-NEXT: v_pk_lshlrev_b16 v0, v1, v0 +; GFX10-NEXT: global_store_dword v2, v0, s[0:1] ; GFX10-NEXT: s_endpgm %tid = call i32 @llvm.amdgcn.workitem.id.x() %tid.ext = sext i32 %tid to i64 @@ -464,8 +452,7 @@ ; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX9-NEXT: v_lshlrev_b32_e32 v4, 3, v0 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: global_load_dwordx2 v[0:1], v4, s[2:3] -; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3] offset:8 +; GFX9-NEXT: global_load_dwordx4 v[0:3], v4, s[2:3] ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_pk_lshlrev_b16 v1, v3, v1 ; GFX9-NEXT: v_pk_lshlrev_b16 v0, v2, v0 @@ -480,10 +467,7 @@ ; VI-NEXT: v_mov_b32_e32 v1, s3 ; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v4 ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc -; VI-NEXT: v_add_u32_e32 v2, vcc, 8, v0 -; VI-NEXT: v_addc_u32_e32 v3, vcc, 0, v1, vcc -; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1] -; VI-NEXT: flat_load_dwordx2 v[2:3], v[2:3] +; VI-NEXT: flat_load_dwordx4 v[0:3], v[0:1] ; VI-NEXT: v_mov_b32_e32 v5, s1 ; VI-NEXT: v_add_u32_e32 v4, vcc, s0, v4 ; VI-NEXT: v_addc_u32_e32 v5, vcc, 0, v5, vcc @@ -502,31 +486,29 @@ ; CI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9 ; CI-NEXT: s_mov_b32 s7, 0xf000 ; CI-NEXT: s_mov_b32 s6, 0 -; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v0 -; CI-NEXT: v_mov_b32_e32 v1, 0 +; CI-NEXT: v_lshlrev_b32_e32 v4, 3, v0 +; CI-NEXT: v_mov_b32_e32 v5, 0 ; CI-NEXT: s_waitcnt lgkmcnt(0) ; CI-NEXT: s_mov_b64 s[4:5], s[2:3] -; CI-NEXT: buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64 -; CI-NEXT: buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8 +; CI-NEXT: buffer_load_dwordx4 v[0:3], v[4:5], s[4:7], 0 addr64 ; CI-NEXT: s_mov_b32 s4, 0xffff ; CI-NEXT: s_mov_b64 s[2:3], s[6:7] -; CI-NEXT: s_waitcnt vmcnt(1) -; CI-NEXT: v_lshrrev_b32_e32 v6, 16, v2 -; CI-NEXT: v_lshrrev_b32_e32 v7, 16, v3 ; CI-NEXT: s_waitcnt vmcnt(0) -; CI-NEXT: v_lshrrev_b32_e32 v8, 16, v4 -; CI-NEXT: v_lshrrev_b32_e32 v9, 16, v5 -; CI-NEXT: v_lshlrev_b32_e32 v3, v5, v3 -; CI-NEXT: v_lshlrev_b32_e32 v2, v4, v2 -; CI-NEXT: v_lshlrev_b32_e32 v4, v9, v7 -; CI-NEXT: v_lshlrev_b32_e32 v5, v8, v6 -; CI-NEXT: v_and_b32_e32 v3, s4, v3 -; CI-NEXT: v_and_b32_e32 v2, s4, v2 -; CI-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; CI-NEXT: v_lshlrev_b32_e32 v5, 16, v5 -; CI-NEXT: v_or_b32_e32 v3, v3, v4 -; CI-NEXT: v_or_b32_e32 v2, v2, v5 -; CI-NEXT: buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64 +; CI-NEXT: v_lshrrev_b32_e32 v6, 16, v0 +; CI-NEXT: v_lshrrev_b32_e32 v7, 16, v1 +; CI-NEXT: v_lshrrev_b32_e32 v8, 16, v2 +; CI-NEXT: v_lshrrev_b32_e32 v9, 16, v3 +; CI-NEXT: v_lshlrev_b32_e32 v1, v3, v1 +; CI-NEXT: v_lshlrev_b32_e32 v0, v2, v0 +; CI-NEXT: v_lshlrev_b32_e32 v2, v9, v7 +; CI-NEXT: v_lshlrev_b32_e32 v3, v8, v6 +; CI-NEXT: v_and_b32_e32 v1, s4, v1 +; CI-NEXT: v_and_b32_e32 v0, s4, v0 +; CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; CI-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; CI-NEXT: v_or_b32_e32 v1, v1, v2 +; CI-NEXT: v_or_b32_e32 v0, v0, v3 +; CI-NEXT: buffer_store_dwordx2 v[0:1], v[4:5], s[0:3], 0 addr64 ; CI-NEXT: s_endpgm ; ; GFX10-LABEL: v_shl_v4i16: @@ -534,9 +516,7 @@ ; GFX10-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; GFX10-NEXT: v_lshlrev_b32_e32 v4, 3, v0 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-NEXT: s_clause 0x1 -; GFX10-NEXT: global_load_dwordx2 v[0:1], v4, s[2:3] -; GFX10-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3] offset:8 +; GFX10-NEXT: global_load_dwordx4 v[0:3], v4, s[2:3] ; GFX10-NEXT: s_waitcnt vmcnt(0) ; GFX10-NEXT: v_pk_lshlrev_b16 v1, v3, v1 ; GFX10-NEXT: v_pk_lshlrev_b16 v0, v2, v0 diff --git a/llvm/test/CodeGen/AMDGPU/sra.ll b/llvm/test/CodeGen/AMDGPU/sra.ll --- a/llvm/test/CodeGen/AMDGPU/sra.ll +++ b/llvm/test/CodeGen/AMDGPU/sra.ll @@ -46,20 +46,19 @@ ; ; EG-LABEL: ashr_v2i32: ; EG: ; %bb.0: -; EG-NEXT: ALU 0, @10, KC0[CB0:0-32], KC1[] -; EG-NEXT: TEX 1 @6 -; EG-NEXT: ALU 3, @11, KC0[CB0:0-32], KC1[] +; EG-NEXT: ALU 0, @8, KC0[CB0:0-32], KC1[] +; EG-NEXT: TEX 0 @6 +; EG-NEXT: ALU 3, @9, KC0[CB0:0-32], KC1[] ; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1 ; EG-NEXT: CF_END ; EG-NEXT: PAD ; EG-NEXT: Fetch clause starting at 6: -; EG-NEXT: VTX_READ_64 T1.XY, T0.X, 8, #1 -; EG-NEXT: VTX_READ_64 T0.XY, T0.X, 0, #1 -; EG-NEXT: ALU clause starting at 10: +; EG-NEXT: VTX_READ_128 T0.XYZW, T0.X, 0, #1 +; EG-NEXT: ALU clause starting at 8: ; EG-NEXT: MOV * T0.X, KC0[2].Z, -; EG-NEXT: ALU clause starting at 11: -; EG-NEXT: ASHR * T0.Y, T0.Y, T1.Y, -; EG-NEXT: ASHR T0.X, T0.X, T1.X, +; EG-NEXT: ALU clause starting at 9: +; EG-NEXT: ASHR * T0.Y, T0.Y, T0.W, +; EG-NEXT: ASHR T0.X, T0.X, T0.Z, ; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, ; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00) %b_ptr = getelementptr <2 x i32>, <2 x i32> addrspace(1)* %in, i32 1 @@ -193,25 +192,23 @@ ; ; EG-LABEL: ashr_v2i16: ; EG: ; %bb.0: -; EG-NEXT: ALU 1, @10, KC0[CB0:0-32], KC1[] -; EG-NEXT: TEX 1 @6 -; EG-NEXT: ALU 14, @12, KC0[CB0:0-32], KC1[] +; EG-NEXT: ALU 0, @8, KC0[CB0:0-32], KC1[] +; EG-NEXT: TEX 0 @6 +; EG-NEXT: ALU 14, @9, KC0[CB0:0-32], KC1[] ; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T6.X, T7.X, 1 ; EG-NEXT: CF_END ; EG-NEXT: PAD ; EG-NEXT: Fetch clause starting at 6: -; EG-NEXT: VTX_READ_32 T7.X, T7.X, 0, #1 -; EG-NEXT: VTX_READ_32 T6.X, T6.X, 4, #1 -; EG-NEXT: ALU clause starting at 10: +; EG-NEXT: VTX_READ_64 T6.XY, T6.X, 0, #1 +; EG-NEXT: ALU clause starting at 8: ; EG-NEXT: MOV * T6.X, KC0[2].Z, -; EG-NEXT: MOV * T7.X, PV.X, -; EG-NEXT: ALU clause starting at 12: -; EG-NEXT: LSHR * T0.W, T7.X, literal.x, +; EG-NEXT: ALU clause starting at 9: +; EG-NEXT: LSHR * T0.W, T6.X, literal.x, ; EG-NEXT: 16(2.242078e-44), 0(0.000000e+00) ; EG-NEXT: BFE_INT T0.Y, PV.W, 0.0, literal.x, -; EG-NEXT: LSHR T0.Z, T6.X, literal.x, -; EG-NEXT: BFE_INT T0.W, T7.X, 0.0, literal.x, BS:VEC_120/SCL_212 -; EG-NEXT: AND_INT * T1.W, T6.X, literal.y, +; EG-NEXT: LSHR T0.Z, T6.Y, literal.x, +; EG-NEXT: BFE_INT T0.W, T6.X, 0.0, literal.x, +; EG-NEXT: AND_INT * T1.W, T6.Y, literal.y, ; EG-NEXT: 16(2.242078e-44), 65535(9.183409e-41) ; EG-NEXT: ASHR T0.W, PV.W, PS, ; EG-NEXT: ASHR * T1.W, PV.Y, PV.Z, @@ -253,10 +250,10 @@ ; SI-NEXT: v_ashrrev_i32_e32 v1, 16, v1 ; SI-NEXT: v_lshrrev_b32_e32 v6, 16, v2 ; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v3 -; SI-NEXT: v_ashrrev_i32_e32 v1, v7, v1 -; SI-NEXT: v_ashrrev_i32_e32 v3, v3, v5 -; SI-NEXT: v_ashrrev_i32_e32 v0, v6, v0 -; SI-NEXT: v_ashrrev_i32_e32 v2, v2, v4 +; SI-NEXT: v_ashr_i32_e32 v1, v1, v7 +; SI-NEXT: v_ashr_i32_e32 v3, v5, v3 +; SI-NEXT: v_ashr_i32_e32 v0, v0, v6 +; SI-NEXT: v_ashr_i32_e32 v2, v4, v2 ; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; SI-NEXT: v_and_b32_e32 v3, s2, v3 ; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 @@ -291,28 +288,23 @@ ; ; EG-LABEL: ashr_v4i16: ; EG: ; %bb.0: -; EG-NEXT: ALU 0, @12, KC0[CB0:0-32], KC1[] -; EG-NEXT: TEX 0 @8 -; EG-NEXT: ALU 3, @13, KC0[], KC1[] -; EG-NEXT: TEX 0 @10 -; EG-NEXT: ALU 54, @17, KC0[CB0:0-32], KC1[] +; EG-NEXT: ALU 0, @8, KC0[CB0:0-32], KC1[] +; EG-NEXT: TEX 0 @6 +; EG-NEXT: ALU 58, @9, KC0[CB0:0-32], KC1[] ; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T10.XY, T9.X, 1 ; EG-NEXT: CF_END ; EG-NEXT: PAD -; EG-NEXT: Fetch clause starting at 8: -; EG-NEXT: VTX_READ_64 T10.XY, T9.X, 0, #1 -; EG-NEXT: Fetch clause starting at 10: -; EG-NEXT: VTX_READ_64 T9.XY, T9.X, 8, #1 -; EG-NEXT: ALU clause starting at 12: +; EG-NEXT: Fetch clause starting at 6: +; EG-NEXT: VTX_READ_128 T9.XYZW, T9.X, 0, #1 +; EG-NEXT: ALU clause starting at 8: ; EG-NEXT: MOV * T9.X, KC0[2].Z, -; EG-NEXT: ALU clause starting at 13: -; EG-NEXT: MOV T4.X, T10.X, -; EG-NEXT: MOV * T5.X, T10.Y, +; EG-NEXT: ALU clause starting at 9: +; EG-NEXT: MOV T4.X, T9.X, +; EG-NEXT: MOV * T5.X, T9.Y, ; EG-NEXT: MOV T0.Y, PV.X, ; EG-NEXT: MOV * T0.Z, PS, -; EG-NEXT: ALU clause starting at 17: -; EG-NEXT: MOV T2.X, T9.X, -; EG-NEXT: MOV * T3.X, T9.Y, +; EG-NEXT: MOV T2.X, T9.Z, +; EG-NEXT: MOV * T3.X, T9.W, ; EG-NEXT: MOV * T0.W, T6.X, ; EG-NEXT: MOV T1.Y, T2.X, ; EG-NEXT: BFE_INT * T1.W, T0.Y, 0.0, literal.x, diff --git a/llvm/test/CodeGen/AMDGPU/srl.ll b/llvm/test/CodeGen/AMDGPU/srl.ll --- a/llvm/test/CodeGen/AMDGPU/srl.ll +++ b/llvm/test/CodeGen/AMDGPU/srl.ll @@ -85,8 +85,7 @@ ; VI: ; %bb.0: ; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x0 -; VI-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x8 +; VI-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0 ; VI-NEXT: s_mov_b32 s3, 0xf000 ; VI-NEXT: s_mov_b32 s2, -1 ; VI-NEXT: s_waitcnt lgkmcnt(0) @@ -99,20 +98,19 @@ ; ; EG-LABEL: lshr_v2i32: ; EG: ; %bb.0: -; EG-NEXT: ALU 0, @10, KC0[CB0:0-32], KC1[] -; EG-NEXT: TEX 1 @6 -; EG-NEXT: ALU 3, @11, KC0[CB0:0-32], KC1[] +; EG-NEXT: ALU 0, @8, KC0[CB0:0-32], KC1[] +; EG-NEXT: TEX 0 @6 +; EG-NEXT: ALU 3, @9, KC0[CB0:0-32], KC1[] ; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T0.XY, T1.X, 1 ; EG-NEXT: CF_END ; EG-NEXT: PAD ; EG-NEXT: Fetch clause starting at 6: -; EG-NEXT: VTX_READ_64 T1.XY, T0.X, 8, #1 -; EG-NEXT: VTX_READ_64 T0.XY, T0.X, 0, #1 -; EG-NEXT: ALU clause starting at 10: +; EG-NEXT: VTX_READ_128 T0.XYZW, T0.X, 0, #1 +; EG-NEXT: ALU clause starting at 8: ; EG-NEXT: MOV * T0.X, KC0[2].Z, -; EG-NEXT: ALU clause starting at 11: -; EG-NEXT: LSHR * T0.Y, T0.Y, T1.Y, -; EG-NEXT: LSHR T0.X, T0.X, T1.X, +; EG-NEXT: ALU clause starting at 9: +; EG-NEXT: LSHR * T0.Y, T0.Y, T0.W, +; EG-NEXT: LSHR T0.X, T0.X, T0.Z, ; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x, ; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00) %b_ptr = getelementptr <2 x i32>, <2 x i32> addrspace(1)* %in, i32 1 @@ -148,22 +146,21 @@ ; ; VI-LABEL: lshr_v4i32: ; VI: ; %bb.0: -; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24 +; VI-NEXT: s_load_dwordx4 s[8:11], s[0:1], 0x24 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0 -; VI-NEXT: s_load_dwordx4 s[8:11], s[2:3], 0x10 -; VI-NEXT: s_mov_b32 s3, 0xf000 -; VI-NEXT: s_mov_b32 s2, -1 +; VI-NEXT: s_load_dwordx8 s[0:7], s[10:11], 0x0 +; VI-NEXT: s_mov_b32 s11, 0xf000 +; VI-NEXT: s_mov_b32 s10, -1 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_lshr_b32 s7, s7, s11 -; VI-NEXT: s_lshr_b32 s6, s6, s10 -; VI-NEXT: s_lshr_b32 s5, s5, s9 -; VI-NEXT: s_lshr_b32 s4, s4, s8 -; VI-NEXT: v_mov_b32_e32 v0, s4 -; VI-NEXT: v_mov_b32_e32 v1, s5 -; VI-NEXT: v_mov_b32_e32 v2, s6 -; VI-NEXT: v_mov_b32_e32 v3, s7 -; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 +; VI-NEXT: s_lshr_b32 s3, s3, s7 +; VI-NEXT: s_lshr_b32 s2, s2, s6 +; VI-NEXT: s_lshr_b32 s1, s1, s5 +; VI-NEXT: s_lshr_b32 s0, s0, s4 +; VI-NEXT: v_mov_b32_e32 v0, s0 +; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: v_mov_b32_e32 v2, s2 +; VI-NEXT: v_mov_b32_e32 v3, s3 +; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0 ; VI-NEXT: s_endpgm ; ; EG-LABEL: lshr_v4i32: @@ -289,8 +286,7 @@ ; VI: ; %bb.0: ; VI-NEXT: s_load_dwordx4 s[16:19], s[0:1], 0x24 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_load_dwordx8 s[0:7], s[18:19], 0x0 -; VI-NEXT: s_load_dwordx8 s[8:15], s[18:19], 0x20 +; VI-NEXT: s_load_dwordx16 s[0:15], s[18:19], 0x0 ; VI-NEXT: s_mov_b32 s19, 0xf000 ; VI-NEXT: s_mov_b32 s18, -1 ; VI-NEXT: s_waitcnt lgkmcnt(0) diff --git a/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll b/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll --- a/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll +++ b/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll @@ -165,21 +165,19 @@ define amdgpu_kernel void @s_test_sub_v2i16_kernarg(<2 x i16> addrspace(1)* %out, <2 x i16> %a, <2 x i16> %b) #1 { ; GFX9-LABEL: s_test_sub_v2i16_kernarg: ; GFX9: ; %bb.0: -; GFX9-NEXT: s_load_dword s2, s[0:1], 0x30 -; GFX9-NEXT: s_load_dword s3, s[0:1], 0x2c +; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2c ; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24 ; GFX9-NEXT: s_mov_b32 s7, 0xf000 ; GFX9-NEXT: s_mov_b32 s6, -1 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NEXT: v_mov_b32_e32 v0, s2 -; GFX9-NEXT: v_pk_sub_i16 v0, s3, v0 +; GFX9-NEXT: v_mov_b32_e32 v0, s3 +; GFX9-NEXT: v_pk_sub_i16 v0, s2, v0 ; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0 ; GFX9-NEXT: s_endpgm ; ; VI-LABEL: s_test_sub_v2i16_kernarg: ; VI: ; %bb.0: -; VI-NEXT: s_load_dword s4, s[0:1], 0x2c -; VI-NEXT: s_load_dword s5, s[0:1], 0x30 +; VI-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x2c ; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; VI-NEXT: s_mov_b32 s3, 0xf000 ; VI-NEXT: s_mov_b32 s2, -1 @@ -197,9 +195,8 @@ ; ; GFX10-LABEL: s_test_sub_v2i16_kernarg: ; GFX10: ; %bb.0: -; GFX10-NEXT: s_clause 0x2 -; GFX10-NEXT: s_load_dword s2, s[0:1], 0x2c -; GFX10-NEXT: s_load_dword s3, s[0:1], 0x30 +; GFX10-NEXT: s_clause 0x1 +; GFX10-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x2c ; GFX10-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x24 ; GFX10-NEXT: s_mov_b32 s7, 0x31016000 ; GFX10-NEXT: s_mov_b32 s6, -1 diff --git a/llvm/test/CodeGen/AMDGPU/udiv.ll b/llvm/test/CodeGen/AMDGPU/udiv.ll --- a/llvm/test/CodeGen/AMDGPU/udiv.ll +++ b/llvm/test/CodeGen/AMDGPU/udiv.ll @@ -547,50 +547,49 @@ ; ; EG-LABEL: udiv_v2i32: ; EG: ; %bb.0: -; EG-NEXT: ALU 0, @10, KC0[CB0:0-32], KC1[] -; EG-NEXT: TEX 1 @6 -; EG-NEXT: ALU 33, @11, KC0[CB0:0-32], KC1[] +; EG-NEXT: ALU 0, @8, KC0[CB0:0-32], KC1[] +; EG-NEXT: TEX 0 @6 +; EG-NEXT: ALU 33, @9, KC0[CB0:0-32], KC1[] ; EG-NEXT: MEM_RAT_CACHELESS STORE_RAW T1.XY, T0.X, 1 ; EG-NEXT: CF_END ; EG-NEXT: PAD ; EG-NEXT: Fetch clause starting at 6: -; EG-NEXT: VTX_READ_64 T1.XY, T0.X, 8, #1 -; EG-NEXT: VTX_READ_64 T0.XY, T0.X, 0, #1 -; EG-NEXT: ALU clause starting at 10: +; EG-NEXT: VTX_READ_128 T0.XYZW, T0.X, 0, #1 +; EG-NEXT: ALU clause starting at 8: ; EG-NEXT: MOV * T0.X, KC0[2].Z, -; EG-NEXT: ALU clause starting at 11: -; EG-NEXT: SUB_INT T0.W, 0.0, T1.Y, -; EG-NEXT: RECIP_UINT * T0.Z, T1.Y, -; EG-NEXT: MULLO_INT * T0.W, PV.W, PS, -; EG-NEXT: SUB_INT T1.W, 0.0, T1.X, -; EG-NEXT: RECIP_UINT * T1.Z, T1.X, +; EG-NEXT: ALU clause starting at 9: +; EG-NEXT: SUB_INT T1.W, 0.0, T0.W, +; EG-NEXT: RECIP_UINT * T1.X, T0.W, +; EG-NEXT: MULLO_INT * T1.Y, PV.W, PS, +; EG-NEXT: SUB_INT T1.W, 0.0, T0.Z, +; EG-NEXT: RECIP_UINT * T1.Z, T0.Z, ; EG-NEXT: MULLO_INT * T1.W, PV.W, PS, ; EG-NEXT: MULHI * T1.W, T1.Z, PS, ; EG-NEXT: ADD_INT T1.W, T1.Z, PS, -; EG-NEXT: MULHI * T0.W, T0.Z, T0.W, -; EG-NEXT: ADD_INT T0.W, T0.Z, PS, -; EG-NEXT: MULHI * T0.Z, T0.X, PV.W, -; EG-NEXT: MULHI * T0.W, T0.Y, PV.W, -; EG-NEXT: MULLO_INT * T1.Z, PS, T1.Y, +; EG-NEXT: MULHI * T1.Y, T1.X, T1.Y, +; EG-NEXT: ADD_INT T2.W, T1.X, PS, +; EG-NEXT: MULHI * T1.X, T0.X, PV.W, +; EG-NEXT: MULHI * T1.Y, T0.Y, PV.W, +; EG-NEXT: MULLO_INT * T1.Z, PS, T0.W, ; EG-NEXT: SUB_INT T1.W, T0.Y, PS, -; EG-NEXT: MULLO_INT * T0.Y, T0.Z, T1.X, +; EG-NEXT: MULLO_INT * T0.Y, T1.X, T0.Z, ; EG-NEXT: SUB_INT T0.Y, T0.X, PS, -; EG-NEXT: ADD_INT T1.Z, T0.W, 1, -; EG-NEXT: SETGE_UINT T2.W, PV.W, T1.Y, -; EG-NEXT: SUB_INT * T3.W, PV.W, T1.Y, +; EG-NEXT: ADD_INT T1.Z, T1.Y, 1, +; EG-NEXT: SETGE_UINT T2.W, PV.W, T0.W, +; EG-NEXT: SUB_INT * T3.W, PV.W, T0.W, ; EG-NEXT: CNDE_INT T0.X, PV.W, T1.W, PS, -; EG-NEXT: CNDE_INT T2.Y, PV.W, T0.W, PV.Z, BS:VEC_021/SCL_122 -; EG-NEXT: ADD_INT T1.Z, T0.Z, 1, -; EG-NEXT: SETGE_UINT T0.W, PV.Y, T1.X, -; EG-NEXT: SUB_INT * T1.W, PV.Y, T1.X, -; EG-NEXT: CNDE_INT T0.Y, PV.W, T0.Y, PS, BS:VEC_021/SCL_122 -; EG-NEXT: CNDE_INT T0.Z, PV.W, T0.Z, PV.Z, -; EG-NEXT: ADD_INT T0.W, PV.Y, 1, -; EG-NEXT: SETGE_UINT * T1.W, PV.X, T1.Y, -; EG-NEXT: CNDE_INT T1.Y, PS, T2.Y, PV.W, +; EG-NEXT: CNDE_INT T1.Y, PV.W, T1.Y, PV.Z, +; EG-NEXT: ADD_INT T1.Z, T1.X, 1, +; EG-NEXT: SETGE_UINT T1.W, PV.Y, T0.Z, +; EG-NEXT: SUB_INT * T2.W, PV.Y, T0.Z, +; EG-NEXT: CNDE_INT T0.Y, PV.W, T0.Y, PS, +; EG-NEXT: CNDE_INT T1.Z, PV.W, T1.X, PV.Z, +; EG-NEXT: ADD_INT T1.W, PV.Y, 1, +; EG-NEXT: SETGE_UINT * T0.W, PV.X, T0.W, +; EG-NEXT: CNDE_INT T1.Y, PS, T1.Y, PV.W, ; EG-NEXT: ADD_INT T0.W, PV.Z, 1, -; EG-NEXT: SETGE_UINT * T1.W, PV.Y, T1.X, -; EG-NEXT: CNDE_INT T1.X, PS, T0.Z, PV.W, +; EG-NEXT: SETGE_UINT * T1.W, PV.Y, T0.Z, +; EG-NEXT: CNDE_INT T1.X, PS, T1.Z, PV.W, ; EG-NEXT: LSHR * T0.X, KC0[2].Y, literal.x, ; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00) %b_ptr = getelementptr <2 x i32>, <2 x i32> addrspace(1)* %in, i32 1 diff --git a/llvm/test/CodeGen/AMDGPU/urem64.ll b/llvm/test/CodeGen/AMDGPU/urem64.ll --- a/llvm/test/CodeGen/AMDGPU/urem64.ll +++ b/llvm/test/CodeGen/AMDGPU/urem64.ll @@ -491,86 +491,84 @@ define amdgpu_kernel void @s_test_urem31_v2i64(<2 x i64> addrspace(1)* %out, <2 x i64> %x, <2 x i64> %y) { ; GCN-LABEL: s_test_urem31_v2i64: ; GCN: ; %bb.0: -; GCN-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x11 -; GCN-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x9 -; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0xd -; GCN-NEXT: s_mov_b32 s11, 0xf000 -; GCN-NEXT: s_mov_b32 s10, -1 +; GCN-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0xd +; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GCN-NEXT: v_mov_b32_e32 v1, 0 +; GCN-NEXT: s_mov_b32 s3, 0xf000 +; GCN-NEXT: s_mov_b32 s2, -1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_lshr_b32 s0, s5, 1 -; GCN-NEXT: v_cvt_f32_u32_e32 v0, s0 -; GCN-NEXT: s_lshr_b32 s1, s1, 1 -; GCN-NEXT: v_cvt_f32_u32_e32 v1, s1 -; GCN-NEXT: s_lshr_b32 s2, s3, 1 -; GCN-NEXT: v_rcp_iflag_f32_e32 v2, v0 -; GCN-NEXT: s_lshr_b32 s3, s7, 1 -; GCN-NEXT: v_cvt_f32_u32_e32 v4, s3 -; GCN-NEXT: v_cvt_f32_u32_e32 v3, s2 -; GCN-NEXT: v_mul_f32_e32 v2, v1, v2 +; GCN-NEXT: s_lshr_b32 s4, s5, 1 +; GCN-NEXT: s_lshr_b32 s5, s9, 1 +; GCN-NEXT: v_cvt_f32_u32_e32 v0, s5 +; GCN-NEXT: v_cvt_f32_u32_e32 v2, s4 +; GCN-NEXT: s_lshr_b32 s6, s11, 1 +; GCN-NEXT: v_cvt_f32_u32_e32 v3, s6 +; GCN-NEXT: v_rcp_iflag_f32_e32 v4, v0 +; GCN-NEXT: s_lshr_b32 s7, s7, 1 +; GCN-NEXT: v_cvt_f32_u32_e32 v5, s7 +; GCN-NEXT: v_rcp_iflag_f32_e32 v6, v3 +; GCN-NEXT: v_mul_f32_e32 v4, v2, v4 +; GCN-NEXT: v_trunc_f32_e32 v4, v4 +; GCN-NEXT: v_cvt_u32_f32_e32 v7, v4 +; GCN-NEXT: v_mad_f32 v2, -v4, v0, v2 +; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v0 +; GCN-NEXT: v_mul_f32_e32 v2, v5, v6 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v7, vcc +; GCN-NEXT: v_mul_lo_u32 v0, v0, s5 ; GCN-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-NEXT: v_cvt_u32_f32_e32 v5, v2 -; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GCN-NEXT: v_rcp_iflag_f32_e32 v2, v4 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 -; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v5, vcc -; GCN-NEXT: v_mul_lo_u32 v0, v0, s0 -; GCN-NEXT: v_mul_f32_e32 v2, v3, v2 -; GCN-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-NEXT: v_cvt_u32_f32_e32 v5, v2 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, s1, v0 -; GCN-NEXT: v_mad_f32 v2, -v2, v4, v3 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v4 -; GCN-NEXT: v_addc_u32_e32 v2, vcc, 0, v5, vcc -; GCN-NEXT: v_mul_lo_u32 v2, v2, s3 -; GCN-NEXT: v_mov_b32_e32 v1, 0 -; GCN-NEXT: s_brev_b32 s0, -2 -; GCN-NEXT: v_and_b32_e32 v0, s0, v0 -; GCN-NEXT: v_sub_i32_e32 v2, vcc, s2, v2 -; GCN-NEXT: v_and_b32_e32 v2, s0, v2 +; GCN-NEXT: v_cvt_u32_f32_e32 v4, v2 +; GCN-NEXT: v_mad_f32 v2, -v2, v3, v5 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, s4, v0 +; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v3 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, 0, v4, vcc +; GCN-NEXT: v_mul_lo_u32 v2, v2, s6 +; GCN-NEXT: s_brev_b32 s4, -2 +; GCN-NEXT: v_and_b32_e32 v0, s4, v0 ; GCN-NEXT: v_mov_b32_e32 v3, v1 -; GCN-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, s7, v2 +; GCN-NEXT: v_and_b32_e32 v2, s4, v2 +; GCN-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 ; GCN-NEXT: s_endpgm ; ; GCN-IR-LABEL: s_test_urem31_v2i64: ; GCN-IR: ; %bb.0: -; GCN-IR-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x11 -; GCN-IR-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x9 -; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0xd -; GCN-IR-NEXT: s_mov_b32 s11, 0xf000 -; GCN-IR-NEXT: s_mov_b32 s10, -1 +; GCN-IR-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0xd +; GCN-IR-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 +; GCN-IR-NEXT: s_mov_b32 s3, 0xf000 +; GCN-IR-NEXT: s_mov_b32 s2, -1 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) -; GCN-IR-NEXT: s_lshr_b32 s0, s5, 1 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s0 -; GCN-IR-NEXT: s_lshr_b32 s1, s1, 1 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, s1 -; GCN-IR-NEXT: s_lshr_b32 s2, s3, 1 -; GCN-IR-NEXT: v_rcp_iflag_f32_e32 v2, v0 -; GCN-IR-NEXT: s_lshr_b32 s3, s7, 1 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v4, s3 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v3, s2 -; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2 +; GCN-IR-NEXT: s_lshr_b32 s4, s5, 1 +; GCN-IR-NEXT: s_lshr_b32 s5, s9, 1 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s5 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v2, s4 +; GCN-IR-NEXT: s_lshr_b32 s6, s11, 1 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v3, s6 +; GCN-IR-NEXT: v_rcp_iflag_f32_e32 v4, v0 +; GCN-IR-NEXT: s_lshr_b32 s7, s7, 1 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v5, s7 +; GCN-IR-NEXT: v_rcp_iflag_f32_e32 v6, v3 +; GCN-IR-NEXT: v_mul_f32_e32 v4, v2, v4 +; GCN-IR-NEXT: v_trunc_f32_e32 v4, v4 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v7, v4 +; GCN-IR-NEXT: v_mad_f32 v2, -v4, v0, v2 +; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v0 +; GCN-IR-NEXT: v_mul_f32_e32 v2, v5, v6 +; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v7, vcc +; GCN-IR-NEXT: v_mul_lo_u32 v0, v0, s5 ; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-IR-NEXT: v_cvt_u32_f32_e32 v5, v2 -; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GCN-IR-NEXT: v_rcp_iflag_f32_e32 v2, v4 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 -; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v5, vcc -; GCN-IR-NEXT: v_mul_lo_u32 v0, v0, s0 -; GCN-IR-NEXT: v_mul_f32_e32 v2, v3, v2 -; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-IR-NEXT: v_cvt_u32_f32_e32 v5, v2 -; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, s1, v0 -; GCN-IR-NEXT: v_mad_f32 v2, -v2, v4, v3 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v4 -; GCN-IR-NEXT: v_addc_u32_e32 v2, vcc, 0, v5, vcc -; GCN-IR-NEXT: v_mul_lo_u32 v2, v2, s3 -; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 -; GCN-IR-NEXT: s_brev_b32 s0, -2 -; GCN-IR-NEXT: v_and_b32_e32 v0, s0, v0 -; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, s2, v2 -; GCN-IR-NEXT: v_and_b32_e32 v2, s0, v2 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v4, v2 +; GCN-IR-NEXT: v_mad_f32 v2, -v2, v3, v5 +; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, s4, v0 +; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v3 +; GCN-IR-NEXT: v_addc_u32_e32 v2, vcc, 0, v4, vcc +; GCN-IR-NEXT: v_mul_lo_u32 v2, v2, s6 +; GCN-IR-NEXT: s_brev_b32 s4, -2 +; GCN-IR-NEXT: v_and_b32_e32 v0, s4, v0 ; GCN-IR-NEXT: v_mov_b32_e32 v3, v1 -; GCN-IR-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0 +; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, s7, v2 +; GCN-IR-NEXT: v_and_b32_e32 v2, s4, v2 +; GCN-IR-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 ; GCN-IR-NEXT: s_endpgm %1 = lshr <2 x i64> %x, %2 = lshr <2 x i64> %y, @@ -639,86 +637,84 @@ define amdgpu_kernel void @s_test_urem23_64_v2i64(<2 x i64> addrspace(1)* %out, <2 x i64> %x, <2 x i64> %y) { ; GCN-LABEL: s_test_urem23_64_v2i64: ; GCN: ; %bb.0: -; GCN-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x11 -; GCN-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x9 -; GCN-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0xd -; GCN-NEXT: s_mov_b32 s11, 0xf000 -; GCN-NEXT: s_mov_b32 s10, -1 +; GCN-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0xd +; GCN-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GCN-NEXT: v_mov_b32_e32 v1, 0 +; GCN-NEXT: s_mov_b32 s3, 0xf000 +; GCN-NEXT: s_mov_b32 s2, -1 ; GCN-NEXT: s_waitcnt lgkmcnt(0) -; GCN-NEXT: s_lshr_b32 s0, s5, 1 -; GCN-NEXT: v_cvt_f32_u32_e32 v0, s0 -; GCN-NEXT: s_lshr_b32 s1, s1, 1 -; GCN-NEXT: v_cvt_f32_u32_e32 v1, s1 -; GCN-NEXT: s_lshr_b32 s2, s3, 9 -; GCN-NEXT: v_rcp_iflag_f32_e32 v2, v0 -; GCN-NEXT: s_lshr_b32 s3, s7, 9 -; GCN-NEXT: v_cvt_f32_u32_e32 v4, s3 -; GCN-NEXT: v_cvt_f32_u32_e32 v3, s2 -; GCN-NEXT: v_mul_f32_e32 v2, v1, v2 +; GCN-NEXT: s_lshr_b32 s4, s5, 1 +; GCN-NEXT: s_lshr_b32 s5, s9, 1 +; GCN-NEXT: v_cvt_f32_u32_e32 v0, s5 +; GCN-NEXT: v_cvt_f32_u32_e32 v2, s4 +; GCN-NEXT: s_lshr_b32 s6, s11, 9 +; GCN-NEXT: v_cvt_f32_u32_e32 v3, s6 +; GCN-NEXT: v_rcp_iflag_f32_e32 v4, v0 +; GCN-NEXT: s_lshr_b32 s7, s7, 9 +; GCN-NEXT: v_cvt_f32_u32_e32 v5, s7 +; GCN-NEXT: v_rcp_iflag_f32_e32 v6, v3 +; GCN-NEXT: v_mul_f32_e32 v4, v2, v4 +; GCN-NEXT: v_trunc_f32_e32 v4, v4 +; GCN-NEXT: v_cvt_u32_f32_e32 v7, v4 +; GCN-NEXT: v_mad_f32 v2, -v4, v0, v2 +; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v0 +; GCN-NEXT: v_mul_f32_e32 v2, v5, v6 +; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v7, vcc +; GCN-NEXT: v_mul_lo_u32 v0, v0, s5 ; GCN-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-NEXT: v_cvt_u32_f32_e32 v5, v2 -; GCN-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GCN-NEXT: v_rcp_iflag_f32_e32 v2, v4 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 -; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v5, vcc -; GCN-NEXT: v_mul_lo_u32 v0, v0, s0 -; GCN-NEXT: v_mul_f32_e32 v2, v3, v2 -; GCN-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-NEXT: v_cvt_u32_f32_e32 v5, v2 -; GCN-NEXT: v_sub_i32_e32 v0, vcc, s1, v0 -; GCN-NEXT: v_mad_f32 v2, -v2, v4, v3 -; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v4 -; GCN-NEXT: v_addc_u32_e32 v2, vcc, 0, v5, vcc -; GCN-NEXT: v_mul_lo_u32 v2, v2, s3 -; GCN-NEXT: v_mov_b32_e32 v1, 0 -; GCN-NEXT: s_brev_b32 s0, -2 -; GCN-NEXT: v_and_b32_e32 v0, s0, v0 -; GCN-NEXT: v_sub_i32_e32 v2, vcc, s2, v2 -; GCN-NEXT: v_and_b32_e32 v2, s0, v2 +; GCN-NEXT: v_cvt_u32_f32_e32 v4, v2 +; GCN-NEXT: v_mad_f32 v2, -v2, v3, v5 +; GCN-NEXT: v_sub_i32_e32 v0, vcc, s4, v0 +; GCN-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v3 +; GCN-NEXT: v_addc_u32_e32 v2, vcc, 0, v4, vcc +; GCN-NEXT: v_mul_lo_u32 v2, v2, s6 +; GCN-NEXT: s_brev_b32 s4, -2 +; GCN-NEXT: v_and_b32_e32 v0, s4, v0 ; GCN-NEXT: v_mov_b32_e32 v3, v1 -; GCN-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0 +; GCN-NEXT: v_sub_i32_e32 v2, vcc, s7, v2 +; GCN-NEXT: v_and_b32_e32 v2, s4, v2 +; GCN-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 ; GCN-NEXT: s_endpgm ; ; GCN-IR-LABEL: s_test_urem23_64_v2i64: ; GCN-IR: ; %bb.0: -; GCN-IR-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x11 -; GCN-IR-NEXT: s_load_dwordx2 s[8:9], s[0:1], 0x9 -; GCN-IR-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0xd -; GCN-IR-NEXT: s_mov_b32 s11, 0xf000 -; GCN-IR-NEXT: s_mov_b32 s10, -1 +; GCN-IR-NEXT: s_load_dwordx8 s[4:11], s[0:1], 0xd +; GCN-IR-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 +; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 +; GCN-IR-NEXT: s_mov_b32 s3, 0xf000 +; GCN-IR-NEXT: s_mov_b32 s2, -1 ; GCN-IR-NEXT: s_waitcnt lgkmcnt(0) -; GCN-IR-NEXT: s_lshr_b32 s0, s5, 1 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s0 -; GCN-IR-NEXT: s_lshr_b32 s1, s1, 1 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v1, s1 -; GCN-IR-NEXT: s_lshr_b32 s2, s3, 9 -; GCN-IR-NEXT: v_rcp_iflag_f32_e32 v2, v0 -; GCN-IR-NEXT: s_lshr_b32 s3, s7, 9 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v4, s3 -; GCN-IR-NEXT: v_cvt_f32_u32_e32 v3, s2 -; GCN-IR-NEXT: v_mul_f32_e32 v2, v1, v2 +; GCN-IR-NEXT: s_lshr_b32 s4, s5, 1 +; GCN-IR-NEXT: s_lshr_b32 s5, s9, 1 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v0, s5 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v2, s4 +; GCN-IR-NEXT: s_lshr_b32 s6, s11, 9 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v3, s6 +; GCN-IR-NEXT: v_rcp_iflag_f32_e32 v4, v0 +; GCN-IR-NEXT: s_lshr_b32 s7, s7, 9 +; GCN-IR-NEXT: v_cvt_f32_u32_e32 v5, s7 +; GCN-IR-NEXT: v_rcp_iflag_f32_e32 v6, v3 +; GCN-IR-NEXT: v_mul_f32_e32 v4, v2, v4 +; GCN-IR-NEXT: v_trunc_f32_e32 v4, v4 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v7, v4 +; GCN-IR-NEXT: v_mad_f32 v2, -v4, v0, v2 +; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v0 +; GCN-IR-NEXT: v_mul_f32_e32 v2, v5, v6 +; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v7, vcc +; GCN-IR-NEXT: v_mul_lo_u32 v0, v0, s5 ; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-IR-NEXT: v_cvt_u32_f32_e32 v5, v2 -; GCN-IR-NEXT: v_mad_f32 v1, -v2, v0, v1 -; GCN-IR-NEXT: v_rcp_iflag_f32_e32 v2, v4 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0 -; GCN-IR-NEXT: v_addc_u32_e32 v0, vcc, 0, v5, vcc -; GCN-IR-NEXT: v_mul_lo_u32 v0, v0, s0 -; GCN-IR-NEXT: v_mul_f32_e32 v2, v3, v2 -; GCN-IR-NEXT: v_trunc_f32_e32 v2, v2 -; GCN-IR-NEXT: v_cvt_u32_f32_e32 v5, v2 -; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, s1, v0 -; GCN-IR-NEXT: v_mad_f32 v2, -v2, v4, v3 -; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v4 -; GCN-IR-NEXT: v_addc_u32_e32 v2, vcc, 0, v5, vcc -; GCN-IR-NEXT: v_mul_lo_u32 v2, v2, s3 -; GCN-IR-NEXT: v_mov_b32_e32 v1, 0 -; GCN-IR-NEXT: s_brev_b32 s0, -2 -; GCN-IR-NEXT: v_and_b32_e32 v0, s0, v0 -; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, s2, v2 -; GCN-IR-NEXT: v_and_b32_e32 v2, s0, v2 +; GCN-IR-NEXT: v_cvt_u32_f32_e32 v4, v2 +; GCN-IR-NEXT: v_mad_f32 v2, -v2, v3, v5 +; GCN-IR-NEXT: v_sub_i32_e32 v0, vcc, s4, v0 +; GCN-IR-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v3 +; GCN-IR-NEXT: v_addc_u32_e32 v2, vcc, 0, v4, vcc +; GCN-IR-NEXT: v_mul_lo_u32 v2, v2, s6 +; GCN-IR-NEXT: s_brev_b32 s4, -2 +; GCN-IR-NEXT: v_and_b32_e32 v0, s4, v0 ; GCN-IR-NEXT: v_mov_b32_e32 v3, v1 -; GCN-IR-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0 +; GCN-IR-NEXT: v_sub_i32_e32 v2, vcc, s7, v2 +; GCN-IR-NEXT: v_and_b32_e32 v2, s4, v2 +; GCN-IR-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 ; GCN-IR-NEXT: s_endpgm %1 = lshr <2 x i64> %x, %2 = lshr <2 x i64> %y, diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/4x2xhalf.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/4x2xhalf.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/4x2xhalf.ll @@ -0,0 +1,96 @@ +; RUN: opt -mtriple=nvptx64-nvidia-cuda -load-store-vectorizer -S -o - %s | FileCheck %s + +define void @ldg_f16(half* nocapture align 16 %rd0) { + %in1b = bitcast half* %rd0 to <2 x half>* + %load1 = load <2 x half>, <2 x half>* %in1b, align 4 + %p1 = fcmp ogt <2 x half> %load1, zeroinitializer + %s1 = select <2 x i1> %p1, <2 x half> %load1, <2 x half> zeroinitializer + store <2 x half> %s1, <2 x half>* %in1b, align 4 + %in2 = getelementptr half, half* %rd0, i64 2 + %in2b = bitcast half* %in2 to <2 x half>* + %load2 = load <2 x half>, <2 x half>* %in2b, align 4 + %p2 = fcmp ogt <2 x half> %load2, zeroinitializer + %s2 = select <2 x i1> %p2, <2 x half> %load2, <2 x half> zeroinitializer + store <2 x half> %s2, <2 x half>* %in2b, align 4 + %in3 = getelementptr half, half* %rd0, i64 4 + %in3b = bitcast half* %in3 to <2 x half>* + %load3 = load <2 x half>, <2 x half>* %in3b, align 4 + %p3 = fcmp ogt <2 x half> %load3, zeroinitializer + %s3 = select <2 x i1> %p3, <2 x half> %load3, <2 x half> zeroinitializer + store <2 x half> %s3, <2 x half>* %in3b, align 4 + %in4 = getelementptr half, half* %rd0, i64 6 + %in4b = bitcast half* %in4 to <2 x half>* + %load4 = load <2 x half>, <2 x half>* %in4b, align 4 + %p4 = fcmp ogt <2 x half> %load4, zeroinitializer + %s4 = select <2 x i1> %p4, <2 x half> %load4, <2 x half> zeroinitializer + store <2 x half> %s4, <2 x half>* %in4b, align 4 + ret void + +; CHECK-LABEL: @ldg_f16 +; CHECK: %[[LD:.*]] = load <8 x half>, <8 x half>* +; CHECK: shufflevector <8 x half> %[[LD]], <8 x half> poison, <2 x i32> +; CHECK: shufflevector <8 x half> %[[LD]], <8 x half> poison, <2 x i32> +; CHECK: shufflevector <8 x half> %[[LD]], <8 x half> poison, <2 x i32> +; CHECK: shufflevector <8 x half> %[[LD]], <8 x half> poison, <2 x i32> +; CHECK: store <8 x half> +} + +define void @no_nonpow2_vector(half* nocapture align 16 %rd0) { + %in1b = bitcast half* %rd0 to <3 x half>* + %load1 = load <3 x half>, <3 x half>* %in1b, align 4 + %p1 = fcmp ogt <3 x half> %load1, zeroinitializer + %s1 = select <3 x i1> %p1, <3 x half> %load1, <3 x half> zeroinitializer + store <3 x half> %s1, <3 x half>* %in1b, align 4 + %in2 = getelementptr half, half* %rd0, i64 3 + %in2b = bitcast half* %in2 to <3 x half>* + %load2 = load <3 x half>, <3 x half>* %in2b, align 4 + %p2 = fcmp ogt <3 x half> %load2, zeroinitializer + %s2 = select <3 x i1> %p2, <3 x half> %load2, <3 x half> zeroinitializer + store <3 x half> %s2, <3 x half>* %in2b, align 4 + %in3 = getelementptr half, half* %rd0, i64 6 + %in3b = bitcast half* %in3 to <3 x half>* + %load3 = load <3 x half>, <3 x half>* %in3b, align 4 + %p3 = fcmp ogt <3 x half> %load3, zeroinitializer + %s3 = select <3 x i1> %p3, <3 x half> %load3, <3 x half> zeroinitializer + store <3 x half> %s3, <3 x half>* %in3b, align 4 + %in4 = getelementptr half, half* %rd0, i64 9 + %in4b = bitcast half* %in4 to <3 x half>* + %load4 = load <3 x half>, <3 x half>* %in4b, align 4 + %p4 = fcmp ogt <3 x half> %load4, zeroinitializer + %s4 = select <3 x i1> %p4, <3 x half> %load4, <3 x half> zeroinitializer + store <3 x half> %s4, <3 x half>* %in4b, align 4 + ret void + +; CHECK-LABEL: @no_nonpow2_vector +; CHECK-NOT: shufflevector +} + +define void @no_pointer_vector(half** nocapture align 16 %rd0) { + %in1b = bitcast half** %rd0 to <2 x half*>* + %load1 = load <2 x half*>, <2 x half*>* %in1b, align 4 + %p1 = icmp ne <2 x half*> %load1, zeroinitializer + %s1 = select <2 x i1> %p1, <2 x half*> %load1, <2 x half*> zeroinitializer + store <2 x half*> %s1, <2 x half*>* %in1b, align 4 + %in2 = getelementptr half*, half** %rd0, i64 2 + %in2b = bitcast half** %in2 to <2 x half*>* + %load2 = load <2 x half*>, <2 x half*>* %in2b, align 4 + %p2 = icmp ne <2 x half*> %load2, zeroinitializer + %s2 = select <2 x i1> %p2, <2 x half*> %load2, <2 x half*> zeroinitializer + store <2 x half*> %s2, <2 x half*>* %in2b, align 4 + %in3 = getelementptr half*, half** %rd0, i64 4 + %in3b = bitcast half** %in3 to <2 x half*>* + %load3 = load <2 x half*>, <2 x half*>* %in3b, align 4 + %p3 = icmp ne <2 x half*> %load3, zeroinitializer + %s3 = select <2 x i1> %p3, <2 x half*> %load3, <2 x half*> zeroinitializer + store <2 x half*> %s3, <2 x half*>* %in3b, align 4 + %in4 = getelementptr half*, half** %rd0, i64 6 + %in4b = bitcast half** %in4 to <2 x half*>* + %load4 = load <2 x half*>, <2 x half*>* %in4b, align 4 + %p4 = icmp ne <2 x half*> %load4, zeroinitializer + %s4 = select <2 x i1> %p4, <2 x half*> %load4, <2 x half*> zeroinitializer + store <2 x half*> %s4, <2 x half*>* %in4b, align 4 + ret void + +; CHECK-LABEL: @no_pointer_vector +; CHECK-NOT: shufflevector +}