Index: include/llvm/CodeGen/TargetInstrInfo.h =================================================================== --- include/llvm/CodeGen/TargetInstrInfo.h +++ include/llvm/CodeGen/TargetInstrInfo.h @@ -429,6 +429,13 @@ RegSubRegPair(unsigned Reg = 0, unsigned SubReg = 0) : Reg(Reg), SubReg(SubReg) {} + + bool operator==(const RegSubRegPair& P) const { + return Reg == P.Reg && SubReg == P.SubReg; + } + bool operator!=(const RegSubRegPair& P) const { + return !(*this == P); + } }; /// A pair composed of a pair of a register and a sub-register index, Index: lib/Target/AMDGPU/GCNDPPCombine.cpp =================================================================== --- lib/Target/AMDGPU/GCNDPPCombine.cpp +++ lib/Target/AMDGPU/GCNDPPCombine.cpp @@ -12,26 +12,32 @@ // // $old = ... // $dpp_value = V_MOV_B32_dpp $old, $vgpr_to_be_read_from_other_lane, -// dpp_controls..., $bound_ctrl +// dpp_controls, $row_mask, $bank_mask, $bound_ctrl // $res = VALU $dpp_value, ... // // to // -// $res = VALU_DPP $folded_old, $vgpr_to_be_read_from_other_lane, ..., -// dpp_controls..., $folded_bound_ctrl +// $res = VALU_DPP $combined_old, $vgpr_to_be_read_from_other_lane, ..., +// dpp_controls..., $row_mask, $bank_mask, $combined_bound_ctrl // // Combining rules : // -// $bound_ctrl is DPP_BOUND_ZERO, $old is any -// $bound_ctrl is DPP_BOUND_OFF, $old is 0 +// if $row_mask and $bank_mask are all enabled (0xF) and +// $old == $vgpr_to_be_read_from_other_lane +// -> $combined_old = $old +// $combined_bound_ctrl = $bound_ctrl // -// ->$folded_old = undef, $folded_bound_ctrl = DPP_BOUND_ZERO -// $bound_ctrl is DPP_BOUND_OFF, $old is undef +// if $row_mask and $bank_mask are all enabled (0xF) and +// $bound_ctrl==DPP_BOUND_ZERO and $old==any or +// $bound_ctrl==DPP_BOUND_OFF and $old==0 +// -> $combined_old = undef, +// $combined_bound_ctrl = DPP_BOUND_ZERO // -// ->$folded_old = undef, $folded_bound_ctrl = DPP_BOUND_OFF -// $bound_ctrl is DPP_BOUND_OFF, $old is foldable +// if $old==undef or $old==identity value for the VALU op +// -> $combined_old = undef/folded identity value, +// $combined_bound_ctrl = DPP_BOUND_OFF // -// ->$folded_old = folded value, $folded_bound_ctrl = DPP_BOUND_OFF +// Othervise cancel. //===----------------------------------------------------------------------===// #include "AMDGPU.h" @@ -199,6 +205,7 @@ break; } DPPInst.add(*Src0); + DPPInst->getOperand(NumOperands).setIsKill(false); ++NumOperands; if (auto *Mod1 = TII->getNamedOperand(OrigMI, @@ -249,8 +256,18 @@ assert(OldOpndValue.isImm()); switch (OrigMI.getOpcode()) { default: break; + case AMDGPU::V_ADD_U32_e32: + case AMDGPU::V_ADD_I32_e32: + case AMDGPU::V_AND_B32_e32: + case AMDGPU::V_SUBREV_U32_e32: + case AMDGPU::V_SUBREV_I32_e32: + if (OldOpndValue.getImm() == 0) + return OldOpndVGPR; + break; + case AMDGPU::V_OR_B32_e32: case AMDGPU::V_MAX_U32_e32: - if (OldOpndValue.getImm() == std::numeric_limits::max()) + if (static_cast(OldOpndValue.getImm()) == + std::numeric_limits::max()) return OldOpndVGPR; break; case AMDGPU::V_MAX_I32_e32: @@ -274,25 +291,13 @@ return RegSubRegPair(); } -// Cases to combine: -// $bound_ctrl is DPP_BOUND_ZERO, $old is any -// $bound_ctrl is DPP_BOUND_OFF, $old is 0 -// -> $old = undef, $bound_ctrl = DPP_BOUND_ZERO - -// $bound_ctrl is DPP_BOUND_OFF, $old is undef -// -> $old = undef, $bound_ctrl = DPP_BOUND_OFF - -// $bound_ctrl is DPP_BOUND_OFF, $old is foldable -// -> $old = folded value, $bound_ctrl = DPP_BOUND_OFF - MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI, MachineInstr &MovMI, RegSubRegPair OldOpndVGPR, MachineOperand *OldOpndValue, bool BoundCtrlZero) const { assert(OldOpndVGPR.Reg); - if (!BoundCtrlZero && OldOpndValue) { - assert(OldOpndValue->isImm()); + if (!BoundCtrlZero && OldOpndValue && OldOpndValue->isImm()) { OldOpndVGPR = foldOldOpnd(OrigMI, OldOpndVGPR, *OldOpndValue); if (!OldOpndVGPR.Reg) { LLVM_DEBUG(dbgs() << " failed: old immediate cannot be folded\n"); @@ -316,11 +321,27 @@ bool GCNDPPCombine::combineDPPMov(MachineInstr &MovMI) const { assert(MovMI.getOpcode() == AMDGPU::V_MOV_B32_dpp); + LLVM_DEBUG(dbgs() << "\nDPP combine: " << MovMI); + + auto *DstOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst); + assert(DstOpnd && DstOpnd->isReg()); + auto DPPMovReg = DstOpnd->getReg(); + if (!isEXECMaskConstantBetweenDefAndUses(DPPMovReg, *MRI)) { + LLVM_DEBUG(dbgs() << " failed: EXEC mask should remain the same" + " for all uses\n"); + return false; + } + + auto *RowMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::row_mask); + assert(RowMaskOpnd && RowMaskOpnd->isImm()); + auto *BankMaskOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::bank_mask); + assert(BankMaskOpnd && BankMaskOpnd->isImm()); + const bool MaskAllLanes = RowMaskOpnd->getImm() == 0xF && + BankMaskOpnd->getImm() == 0xF; + auto *BCZOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::bound_ctrl); assert(BCZOpnd && BCZOpnd->isImm()); - bool BoundCtrlZero = 0 != BCZOpnd->getImm(); - - LLVM_DEBUG(dbgs() << "\nDPP combine: " << MovMI); + bool BoundCtrlZero = MaskAllLanes && 0 != BCZOpnd->getImm(); auto *OldOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::old); assert(OldOpnd && OldOpnd->isReg()); @@ -332,14 +353,24 @@ OldOpndVGPR.Reg = AMDGPU::NoRegister; // should be undef, ignore old opnd OldOpndValue = nullptr; } else { - if (!OldOpndValue->isImm()) { - LLVM_DEBUG(dbgs() << " failed: old operand isn't an imm or undef\n"); + if (OldOpndValue->getParent()->getParent() != MovMI.getParent()) { + LLVM_DEBUG(dbgs() << " failed: old reg def and mov should" + " be in the same BB\n"); return false; } - if (OldOpndValue->getImm() == 0) { - OldOpndVGPR.Reg = AMDGPU::NoRegister; // should be undef - OldOpndValue = nullptr; - BoundCtrlZero = true; + if (OldOpndValue->isImm()) { + if (MaskAllLanes && OldOpndValue->getImm() == 0) { + OldOpndVGPR.Reg = AMDGPU::NoRegister; // should be undef + OldOpndValue = nullptr; + BoundCtrlZero = true; + } + } else { + auto *DPPSrcOpnd = TII->getNamedOperand(MovMI, AMDGPU::OpName::src0); + assert(DPPSrcOpnd && DPPSrcOpnd->isReg()); + if (!MaskAllLanes || getRegSubRegPair(*DPPSrcOpnd) != OldOpndVGPR) { + LLVM_DEBUG(dbgs() << " failed: the DPP mov isn't combinable\n"); + return false; + } } } } @@ -348,7 +379,7 @@ if (!OldOpndValue) dbgs() << "undef"; else - dbgs() << OldOpndValue->getImm(); + dbgs() << *OldOpndValue; dbgs() << ", bound_ctrl=" << BoundCtrlZero << '\n'); std::vector OrigMIs, DPPMIs; @@ -362,11 +393,12 @@ OrigMIs.push_back(&MovMI); bool Rollback = true; - for (auto &Use : MRI->use_nodbg_operands( - TII->getNamedOperand(MovMI, AMDGPU::OpName::vdst)->getReg())) { + for (auto &Use : MRI->use_nodbg_operands(DPPMovReg)) { Rollback = true; auto &OrigMI = *Use.getParent(); + LLVM_DEBUG(dbgs() << " try: " << OrigMI); + auto OrigOp = OrigMI.getOpcode(); if (TII->isVOP3(OrigOp)) { if (!TII->hasVALU32BitEncoding(OrigOp)) { Index: lib/Target/AMDGPU/SIInstrInfo.h =================================================================== --- lib/Target/AMDGPU/SIInstrInfo.h +++ lib/Target/AMDGPU/SIInstrInfo.h @@ -956,6 +956,12 @@ MachineInstr *getVRegSubRegDef(const TargetInstrInfo::RegSubRegPair &P, MachineRegisterInfo &MRI); +/// \brief Return true if EXEC mask isnt' changed between the def and +/// all uses of VReg. Currently if def and uses are in different BBs - +/// simply return false. Should be run on SSA. +bool isEXECMaskConstantBetweenDefAndUses(unsigned VReg, + MachineRegisterInfo &MRI); + namespace AMDGPU { LLVM_READONLY Index: lib/Target/AMDGPU/SIInstrInfo.cpp =================================================================== --- lib/Target/AMDGPU/SIInstrInfo.cpp +++ lib/Target/AMDGPU/SIInstrInfo.cpp @@ -5591,3 +5591,29 @@ } return nullptr; } + +bool llvm::isEXECMaskConstantBetweenDefAndUses(unsigned VReg, + MachineRegisterInfo &MRI) { + assert(MRI.isSSA() && "Must be run on SSA"); + auto *TRI = MRI.getTargetRegisterInfo(); + + auto *DefI = MRI.getVRegDef(VReg); + auto *BB = DefI->getParent(); + + DenseSet Uses; + for (auto &Use : MRI.use_nodbg_operands(VReg)) { + auto *I = Use.getParent(); + if (I->getParent() != BB) + return false; + Uses.insert(I); + } + + auto E = BB->end(); + for (auto I = std::next(DefI->getIterator()); I != E; ++I) { + Uses.erase(&*I); + // don't check the last use + if (Uses.empty() || I->modifiesRegister(AMDGPU::EXEC, TRI)) + break; + } + return Uses.empty(); +} Index: test/CodeGen/AMDGPU/dpp_combine.ll =================================================================== --- test/CodeGen/AMDGPU/dpp_combine.ll +++ test/CodeGen/AMDGPU/dpp_combine.ll @@ -2,25 +2,36 @@ ; VOP2 with literal cannot be combined ; CHECK-LABEL: {{^}}dpp_combine_i32_literal: -; CHECK: v_mov_b32_dpp [[OLD:v[0-9]+]], {{v[0-9]+}} quad_perm:[1,0,0,0] row_mask:0x2 bank_mask:0x1 bound_ctrl:0 +; CHECK: v_mov_b32_dpp [[OLD:v[0-9]+]], {{v[0-9]+}} quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:0 ; CHECK: v_add_u32_e32 {{v[0-9]+}}, vcc, 42, [[OLD]] define amdgpu_kernel void @dpp_combine_i32_literal(i32 addrspace(1)* %out, i32 %in) { - %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 2, i32 1, i1 1) #0 + %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 15, i32 15, i1 1) #0 %res = add nsw i32 %dpp, 42 store i32 %res, i32 addrspace(1)* %out ret void } ; CHECK-LABEL: {{^}}dpp_combine_i32_bz: -; CHECK: v_add_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0 +; CHECK: v_add_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:0 define amdgpu_kernel void @dpp_combine_i32_bz(i32 addrspace(1)* %out, i32 %in) { %x = tail call i32 @llvm.amdgcn.workitem.id.x() - %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 1, i32 1, i1 1) #0 + %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 15, i32 15, i1 1) #0 %res = add nsw i32 %dpp, %x store i32 %res, i32 addrspace(1)* %out ret void } +; CHECK-LABEL: {{^}}dpp_combine_i32_mov: +; CHECK: v_add_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf +define amdgpu_kernel void @dpp_combine_i32_mov(i32 addrspace(1)* %out, i32 %in) { + %x = tail call i32 @llvm.amdgcn.workitem.id.x() + %dpp = call i32 @llvm.amdgcn.mov.dpp.i32(i32 %in, i32 1, i32 15, i32 15, i1 1) #0 + %res = add nsw i32 %dpp, %x + store i32 %res, i32 addrspace(1)* %out + ret void +} + + ; CHECK-LABEL: {{^}}dpp_combine_i32_boff_undef: ; CHECK: v_add_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 define amdgpu_kernel void @dpp_combine_i32_boff_undef(i32 addrspace(1)* %out, i32 %in) { @@ -32,15 +43,36 @@ } ; CHECK-LABEL: {{^}}dpp_combine_i32_boff_0: -; CHECK: v_add_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0 +; CHECK: v_add_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:0 define amdgpu_kernel void @dpp_combine_i32_boff_0(i32 addrspace(1)* %out, i32 %in) { %x = tail call i32 @llvm.amdgcn.workitem.id.x() - %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 0, i32 %in, i32 1, i32 1, i32 1, i1 0) #0 + %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 0, i32 %in, i32 1, i32 15, i32 15, i1 0) #0 %res = add nsw i32 %dpp, %x store i32 %res, i32 addrspace(1)* %out ret void } +; CHECK-LABEL: {{^}}dpp_combine_i32_boff_or: +; CHECK: v_mov_b32_e32 [[OLD:v[0-9]+]], -1 +; CHECK: v_or_b32_dpp [[OLD]], {{v[0-9]+}}, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 +define amdgpu_kernel void @dpp_combine_i32_boff_or(i32 addrspace(1)* %out, i32 %in) { + %x = tail call i32 @llvm.amdgcn.workitem.id.x() + %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 4294967295, i32 %in, i32 1, i32 1, i32 1, i1 0) #0 + %res = or i32 %dpp, %x + store i32 %res, i32 addrspace(1)* %out + ret void +} + +; CHECK-LABEL: {{^}}dpp_combine_i32_boff_and: +; CHECK: v_and_b32_dpp {{v[0-9]+}}, {{v[0-9]+}}, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 +define amdgpu_kernel void @dpp_combine_i32_boff_and(i32 addrspace(1)* %out, i32 %in) { + %x = tail call i32 @llvm.amdgcn.workitem.id.x() + %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 0, i32 %in, i32 1, i32 1, i32 1, i1 0) #0 + %res = and i32 %dpp, %x + store i32 %res, i32 addrspace(1)* %out + ret void +} + ; CHECK-LABEL: {{^}}dpp_combine_i32_boff_max: ; CHECK: v_bfrev_b32_e32 [[OLD:v[0-9]+]], -2 ; CHECK: v_max_i32_dpp [[OLD]], {{v[0-9]+}}, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 @@ -81,21 +113,21 @@ } ; CHECK-LABEL: {{^}}dpp_combine_i32_commute: -; CHECK: v_subrev_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0 quad_perm:[2,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0 +; CHECK: v_subrev_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0 quad_perm:[2,0,0,0] row_mask:0x1 bank_mask:0x1 define amdgpu_kernel void @dpp_combine_i32_commute(i32 addrspace(1)* %out, i32 %in) { %x = tail call i32 @llvm.amdgcn.workitem.id.x() - %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 2, i32 1, i32 1, i1 1) #0 + %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 0, i32 %in, i32 2, i32 1, i32 1, i1 1) #0 %res = sub nsw i32 %x, %dpp store i32 %res, i32 addrspace(1)* %out ret void } ; CHECK-LABEL: {{^}}dpp_combine_f32: -; CHECK: v_add_f32_dpp {{v[0-9]+}}, {{v[0-9]+}}, v0 quad_perm:[3,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0 +; CHECK: v_add_f32_dpp {{v[0-9]+}}, {{v[0-9]+}}, v0 quad_perm:[3,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:0 define amdgpu_kernel void @dpp_combine_f32(i32 addrspace(1)* %out, i32 %in) { %x = tail call i32 @llvm.amdgcn.workitem.id.x() - %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 3, i32 1, i32 1, i1 1) #0 + %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 3, i32 15, i32 15, i1 1) #0 %dpp.f32 = bitcast i32 %dpp to float %x.f32 = bitcast i32 %x to float %res.f32 = fadd float %x.f32, %dpp.f32 @@ -105,11 +137,11 @@ } ; CHECK-LABEL: {{^}}dpp_combine_test_f32_mods: -; CHECK: v_mul_f32_dpp {{v[0-9]+}}, |{{v[0-9]+}}|, -v0 quad_perm:[0,1,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0 +; CHECK: v_mul_f32_dpp {{v[0-9]+}}, |{{v[0-9]+}}|, -v0 quad_perm:[0,1,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:0 define amdgpu_kernel void @dpp_combine_test_f32_mods(i32 addrspace(1)* %out, i32 %in) { %x = tail call i32 @llvm.amdgcn.workitem.id.x() - %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 4, i32 1, i32 1, i1 1) #0 + %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 4, i32 15, i32 15, i1 1) #0 %x.f32 = bitcast i32 %x to float %x.f32.neg = fsub float -0.000000e+00, %x.f32 @@ -126,11 +158,11 @@ } ; CHECK-LABEL: {{^}}dpp_combine_mac: -; CHECK: v_mac_f32_dpp v0, {{v[0-9]+}}, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0 +; CHECK: v_mac_f32_dpp v0, {{v[0-9]+}}, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:0 define amdgpu_kernel void @dpp_combine_mac(float addrspace(1)* %out, i32 %in) { %x = tail call i32 @llvm.amdgcn.workitem.id.x() %y = tail call i32 @llvm.amdgcn.workitem.id.y() - %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 1, i32 1, i1 1) #0 + %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 15, i32 15, i1 1) #0 %dpp.f32 = bitcast i32 %dpp to float %x.f32 = bitcast i32 %x to float %y.f32 = bitcast i32 %y to float @@ -142,29 +174,26 @@ } ; CHECK-LABEL: {{^}}dpp_combine_sequence: -define amdgpu_kernel void @dpp_combine_sequence(i32 addrspace(1)* %out, i32 %in, i1 %cmp) { +define amdgpu_kernel void @dpp_combine_sequence(i32 addrspace(1)* %out1, i32 addrspace(1)* %out2, i32 %in) { %x = tail call i32 @llvm.amdgcn.workitem.id.x() - %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 1, i32 1, i1 1) #0 - br i1 %cmp, label %bb1, label %bb2 -bb1: -; CHECK: v_add_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0 + %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 15, i32 15, i1 1) #0 + +; CHECK: v_add_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:0 %resadd = add nsw i32 %dpp, %x - br label %bb3 -bb2: -; CHECK: v_subrev_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0 + +; CHECK: v_subrev_u32_dpp {{v[0-9]+}}, vcc, {{v[0-9]+}}, v0 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:0 %ressub = sub nsw i32 %x, %dpp - br label %bb3 -bb3: - %res = phi i32 [%resadd, %bb1], [%ressub, %bb2] - store i32 %res, i32 addrspace(1)* %out + + store i32 %resadd, i32 addrspace(1)* %out1 + store i32 %ressub, i32 addrspace(1)* %out2 ret void } ; CHECK-LABEL: {{^}}dpp_combine_sequence_negative: -; CHECK: v_mov_b32_dpp v1, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:0 +; CHECK: v_mov_b32_dpp v1, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:0 define amdgpu_kernel void @dpp_combine_sequence_negative(i32 addrspace(1)* %out, i32 %in, i1 %cmp) { %x = tail call i32 @llvm.amdgcn.workitem.id.x() - %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 1, i32 1, i1 1) #0 + %dpp = call i32 @llvm.amdgcn.update.dpp.i32(i32 undef, i32 %in, i32 1, i32 15, i32 15, i1 1) #0 br i1 %cmp, label %bb1, label %bb2 bb1: %resadd = add nsw i32 %dpp, %x @@ -180,6 +209,7 @@ declare i32 @llvm.amdgcn.workitem.id.x() declare i32 @llvm.amdgcn.workitem.id.y() +declare i32 @llvm.amdgcn.mov.dpp.i32(i32, i32, i32, i32, i1) #0 declare i32 @llvm.amdgcn.update.dpp.i32(i32, i32, i32, i32, i32, i1) #0 attributes #0 = { nounwind readnone convergent } Index: test/CodeGen/AMDGPU/dpp_combine_subregs.mir =================================================================== --- test/CodeGen/AMDGPU/dpp_combine_subregs.mir +++ test/CodeGen/AMDGPU/dpp_combine_subregs.mir @@ -36,8 +36,7 @@ ... # CHECK-LABEL: name: add_old_subreg -# CHECK: [[OLD:\%[0-9]+]]:vgpr_32 = IMPLICIT_DEF -# CHECK: %5:vgpr_32 = V_ADD_U32_dpp [[OLD]], %1, %0.sub1, 1, 1, 1, 1, implicit $exec +# CHECK: %5:vgpr_32 = V_ADD_U32_dpp %3.sub1, %1, %0.sub1, 1, 1, 1, 0, implicit $exec name: add_old_subreg tracksRegLiveness: true @@ -93,10 +92,10 @@ ... # CHECK-LABEL: name: add_f32_e64 -# CHECK: %3:vgpr_32 = V_MOV_B32_dpp undef %2, %1, 1, 1, 1, 1, implicit $exec +# CHECK: %3:vgpr_32 = V_MOV_B32_dpp undef %2, %1, 1, 15, 15, 1, implicit $exec # CHECK: %4:vgpr_32 = V_ADD_F32_e64 0, %3, 0, %0, 0, 1, implicit $exec -# CHECK: %6:vgpr_32 = V_ADD_F32_dpp %2, 0, %1, 0, %0, 1, 1, 1, 1, implicit $exec -# CHECK: %7:vgpr_32 = V_ADD_F32_dpp %2, 1, %1, 2, %0, 1, 1, 1, 1, implicit $exec +# CHECK: %6:vgpr_32 = V_ADD_F32_dpp %2, 0, %1, 0, %0, 1, 15, 15, 1, implicit $exec +# CHECK: %7:vgpr_32 = V_ADD_F32_dpp %2, 1, %1, 2, %0, 1, 15, 15, 1, implicit $exec # CHECK: %9:vgpr_32 = V_ADD_F32_e64 4, %8, 8, %0, 0, 0, implicit $exec name: add_f32_e64 @@ -123,12 +122,12 @@ %0:vgpr_32 = COPY $vgpr0 %1:vgpr_32 = COPY $vgpr1 %2:vgpr_32 = IMPLICIT_DEF - %3:vgpr_32 = V_MOV_B32_dpp undef %2, %1, 1, 1, 1, 1, implicit $exec + %3:vgpr_32 = V_MOV_B32_dpp undef %2, %1, 1, 15, 15, 1, implicit $exec ; this shouldn't be combined as omod is set %4:vgpr_32 = V_ADD_F32_e64 0, %3, 0, %0, 0, 1, implicit $exec - %5:vgpr_32 = V_MOV_B32_dpp undef %2, %1, 1, 1, 1, 1, implicit $exec + %5:vgpr_32 = V_MOV_B32_dpp undef %2, %1, 1, 15, 15, 1, implicit $exec ; this should be combined as all modifiers are default %6:vgpr_32 = V_ADD_F32_e64 0, %5, 0, %0, 0, 0, implicit $exec @@ -136,8 +135,102 @@ ; this should be combined as modifiers other than abs|neg are default %7:vgpr_32 = V_ADD_F32_e64 1, %5, 2, %0, 0, 0, implicit $exec - %8:vgpr_32 = V_MOV_B32_dpp undef %2, %1, 1, 1, 1, 1, implicit $exec + %8:vgpr_32 = V_MOV_B32_dpp undef %2, %1, 1, 15, 15, 1, implicit $exec ; this shouldn't be combined as modifiers aren't abs|neg %9:vgpr_32 = V_ADD_F32_e64 4, %8, 8, %0, 0, 0, implicit $exec ... + +# old reg def is in diff BB - cannot combine +# CHECK-LABEL: name: add_negative_old_in_diff_bb +# CHECK: %3:vgpr_32 = V_MOV_B32_dpp %2, %1, 1, 1, 1, 0, implicit $exec + +name: add_negative_old_in_diff_bb +tracksRegLiveness: true +registers: + - { id: 0, class: vreg_64 } + - { id: 1, class: vgpr_32 } + - { id: 2, class: vgpr_32 } + - { id: 3, class: vgpr_32 } + - { id: 4, class: vgpr_32 } + +liveins: + - { reg: '$vgpr0', virtual-reg: '%0' } + - { reg: '$vgpr1', virtual-reg: '%1' } +body: | + bb.0: + successors: %bb.1 + liveins: $vgpr0, $vgpr1 + + %0:vreg_64 = COPY $vgpr0 + %1:vgpr_32 = COPY $vgpr1 + %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + S_BRANCH %bb.1 + + bb.1: + %3:vgpr_32 = V_MOV_B32_dpp %2, %1, 1, 1, 1, 0, implicit $exec + %4:vgpr_32 = V_ADD_U32_e32 %3, %0.sub1, implicit $exec +... + +# old reg def is in diff BB but bound_ctrl:0 - can combine +# CHECK-LABEL: name: add_positive_old_in_diff_bb_bctrl_zero +# CHECK: %4:vgpr_32 = V_ADD_U32_dpp %5, %1, %0.sub1, 1, 15, 15, 1, implicit $exec + +name: add_positive_old_in_diff_bb_bctrl_zero +tracksRegLiveness: true +registers: + - { id: 0, class: vreg_64 } + - { id: 1, class: vgpr_32 } + - { id: 2, class: vgpr_32 } + - { id: 3, class: vgpr_32 } + - { id: 4, class: vgpr_32 } + +liveins: + - { reg: '$vgpr0', virtual-reg: '%0' } + - { reg: '$vgpr1', virtual-reg: '%1' } +body: | + bb.0: + successors: %bb.1 + liveins: $vgpr0, $vgpr1 + + %0:vreg_64 = COPY $vgpr0 + %1:vgpr_32 = COPY $vgpr1 + %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + S_BRANCH %bb.1 + + bb.1: + %3:vgpr_32 = V_MOV_B32_dpp %2, %1, 1, 15, 15, 1, implicit $exec + %4:vgpr_32 = V_ADD_U32_e32 %3, %0.sub1, implicit $exec + +... + +# EXEC mask changed between def and use - cannot combine +# CHECK-LABEL: name: negative_exec_changed +# CHECK: %4:vgpr_32 = V_MOV_B32_dpp %2, %1, 1, 1, 1, 0, implicit $exec + +name: negative_exec_changed +tracksRegLiveness: true +registers: + - { id: 0, class: vreg_64 } + - { id: 1, class: vgpr_32 } + - { id: 2, class: vgpr_32 } + - { id: 3, class: vreg_64 } + - { id: 4, class: vgpr_32 } + - { id: 5, class: vgpr_32 } + - { id: 6, class: vgpr_32 } + - { id: 7, class: sreg_64 } + +liveins: + - { reg: '$vgpr0', virtual-reg: '%0' } + - { reg: '$vgpr1', virtual-reg: '%1' } +body: | + bb.0: + liveins: $vgpr0, $vgpr1 + + %0:vreg_64 = COPY $vgpr0 + %1:vgpr_32 = COPY $vgpr1 + %2:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + %4:vgpr_32 = V_MOV_B32_dpp %2, %1, 1, 1, 1, 0, implicit $exec + %5:vgpr_32 = V_ADD_U32_e32 %4, %0.sub0, implicit $exec + %7:sreg_64 = COPY $exec, implicit-def $exec + %6:vgpr_32 = V_ADD_U32_e32 %4, %0.sub1, implicit $exec