diff --git a/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h b/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h --- a/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h +++ b/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h @@ -77,11 +77,12 @@ struct PhysRegSUOper { SUnit *SU; int OpIdx; - unsigned Reg; + unsigned RegUnit; - PhysRegSUOper(SUnit *su, int op, unsigned R): SU(su), OpIdx(op), Reg(R) {} + PhysRegSUOper(SUnit *su, int op, unsigned R) + : SU(su), OpIdx(op), RegUnit(R) {} - unsigned getSparseSetIndex() const { return Reg; } + unsigned getSparseSetIndex() const { return RegUnit; } }; /// Use a SparseMultiSet to track physical registers. Storage is only diff --git a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp --- a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp +++ b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp @@ -211,7 +211,8 @@ for (const MachineOperand &MO : ExitMI->all_uses()) { Register Reg = MO.getReg(); if (Reg.isPhysical()) { - Uses.insert(PhysRegSUOper(&ExitSU, -1, Reg)); + for (MCRegUnit Unit : TRI->regunits(Reg)) + Uses.insert(PhysRegSUOper(&ExitSU, -1, Unit)); } else if (Reg.isVirtual() && MO.readsReg()) { addVRegUseDeps(&ExitSU, MO.getOperandNo()); } @@ -222,8 +223,11 @@ // uses all the registers that are livein to the successor blocks. for (const MachineBasicBlock *Succ : BB->successors()) { for (const auto &LI : Succ->liveins()) { - if (!Uses.contains(LI.PhysReg)) - Uses.insert(PhysRegSUOper(&ExitSU, -1, LI.PhysReg)); + // TODO: Use LI.LaneMask to refine this. + for (MCRegUnit Unit : TRI->regunits(LI.PhysReg)) { + if (!Uses.contains(Unit)) + Uses.insert(PhysRegSUOper(&ExitSU, -1, Unit)); + } } } } @@ -244,8 +248,8 @@ const MCInstrDesc &DefMIDesc = SU->getInstr()->getDesc(); bool ImplicitPseudoDef = (OperIdx >= DefMIDesc.getNumOperands() && !DefMIDesc.hasImplicitDefOfPhysReg(Reg)); - for (MCRegAliasIterator Alias(Reg, TRI, true); Alias.isValid(); ++Alias) { - for (Reg2SUnitsMap::iterator I = Uses.find(*Alias); I != Uses.end(); ++I) { + for (MCRegUnit Unit : TRI->regunits(Reg)) { + for (Reg2SUnitsMap::iterator I = Uses.find(Unit); I != Uses.end(); ++I) { SUnit *UseSU = I->SU; if (UseSU == SU) continue; @@ -262,11 +266,14 @@ // Set the hasPhysRegDefs only for physreg defs that have a use within // the scheduling region. SU->hasPhysRegDefs = true; + UseInstr = UseSU->getInstr(); + Register UseReg = UseInstr->getOperand(UseOpIdx).getReg(); const MCInstrDesc &UseMIDesc = UseInstr->getDesc(); - ImplicitPseudoUse = (UseOpIdx >= ((int)UseMIDesc.getNumOperands()) && - !UseMIDesc.hasImplicitUseOfPhysReg(*Alias)); - Dep = SDep(SU, SDep::Data, *Alias); + ImplicitPseudoUse = UseOpIdx >= ((int)UseMIDesc.getNumOperands()) && + !UseMIDesc.hasImplicitUseOfPhysReg(UseReg); + + Dep = SDep(SU, SDep::Data, UseReg); } if (!ImplicitPseudoDef && !ImplicitPseudoUse) { Dep.setLatency(SchedModel.computeOperandLatency(SU->getInstr(), OperIdx, @@ -300,15 +307,16 @@ // TODO: Using a latency of 1 here for output dependencies assumes // there's no cost for reusing registers. SDep::Kind Kind = MO.isUse() ? SDep::Anti : SDep::Output; - for (MCRegAliasIterator Alias(Reg, TRI, true); Alias.isValid(); ++Alias) { - for (Reg2SUnitsMap::iterator I = Defs.find(*Alias); I != Defs.end(); ++I) { + for (MCRegUnit Unit : TRI->regunits(Reg)) { + for (Reg2SUnitsMap::iterator I = Defs.find(Unit); I != Defs.end(); ++I) { SUnit *DefSU = I->SU; if (DefSU == &ExitSU) continue; MachineInstr *DefInstr = DefSU->getInstr(); - if (DefSU != SU && (Kind != SDep::Output || !MO.isDead() || - !DefInstr->registerDefIsDead(*Alias))) { - SDep Dep(SU, Kind, /*Reg=*/*Alias); + MachineOperand &DefMO = DefInstr->getOperand(I->OpIdx); + if (DefSU != SU && + (Kind != SDep::Output || !MO.isDead() || !DefMO.isDead())) { + SDep Dep(SU, Kind, DefMO.getReg()); if (Kind != SDep::Anti) { Dep.setLatency( SchedModel.computeOutputLatency(MI, OperIdx, DefInstr)); @@ -324,37 +332,42 @@ // Either insert a new Reg2SUnits entry with an empty SUnits list, or // retrieve the existing SUnits list for this register's uses. // Push this SUnit on the use list. - Uses.insert(PhysRegSUOper(SU, OperIdx, Reg)); + for (MCRegUnit Unit : TRI->regunits(Reg)) + Uses.insert(PhysRegSUOper(SU, OperIdx, Unit)); if (RemoveKillFlags) MO.setIsKill(false); } else { addPhysRegDataDeps(SU, OperIdx); // Clear previous uses and defs of this register and its subregisters. - for (MCPhysReg SubReg : TRI->subregs_inclusive(Reg)) { - Uses.eraseAll(SubReg); + for (MCRegUnit Unit : TRI->regunits(Reg)) { + Uses.eraseAll(Unit); if (!MO.isDead()) - Defs.eraseAll(SubReg); + Defs.eraseAll(Unit); } + if (MO.isDead() && SU->isCall) { // Calls will not be reordered because of chain dependencies (see // below). Since call operands are dead, calls may continue to be added // to the DefList making dependence checking quadratic in the size of // the block. Instead, we leave only one call at the back of the // DefList. - Reg2SUnitsMap::RangePair P = Defs.equal_range(Reg); - Reg2SUnitsMap::iterator B = P.first; - Reg2SUnitsMap::iterator I = P.second; - for (bool isBegin = I == B; !isBegin; /* empty */) { - isBegin = (--I) == B; - if (!I->SU->isCall) - break; - I = Defs.erase(I); + for (MCRegUnit Unit : TRI->regunits(Reg)) { + Reg2SUnitsMap::RangePair P = Defs.equal_range(Unit); + Reg2SUnitsMap::iterator B = P.first; + Reg2SUnitsMap::iterator I = P.second; + for (bool isBegin = I == B; !isBegin; /* empty */) { + isBegin = (--I) == B; + if (!I->SU->isCall) + break; + I = Defs.erase(I); + } } } // Defs are pushed in the order they are visited and never reordered. - Defs.insert(PhysRegSUOper(SU, OperIdx, Reg)); + for (MCRegUnit Unit : TRI->regunits(Reg)) + Defs.insert(PhysRegSUOper(SU, OperIdx, Unit)); } } diff --git a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll --- a/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll +++ b/llvm/test/CodeGen/AMDGPU/fcopysign.f16.ll @@ -1363,11 +1363,11 @@ ; VI-NEXT: flat_load_dwordx2 v[1:2], v[1:2] ; VI-NEXT: s_waitcnt vmcnt(0) ; VI-NEXT: v_mov_b32_e32 v1, s7 -; VI-NEXT: s_movk_i32 s0, 0x7fff ; VI-NEXT: flat_load_ushort v3, v[0:1] -; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; VI-NEXT: s_movk_i32 s0, 0x7fff ; VI-NEXT: v_mov_b32_e32 v0, s4 ; VI-NEXT: v_mov_b32_e32 v1, s5 +; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2 ; VI-NEXT: s_waitcnt vmcnt(0) ; VI-NEXT: v_bfi_b32 v2, s0, v3, v2 ; VI-NEXT: flat_store_short v[0:1], v2 diff --git a/llvm/test/CodeGen/AMDGPU/load-global-i16.ll b/llvm/test/CodeGen/AMDGPU/load-global-i16.ll --- a/llvm/test/CodeGen/AMDGPU/load-global-i16.ll +++ b/llvm/test/CodeGen/AMDGPU/load-global-i16.ll @@ -3788,13 +3788,13 @@ ; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v33, 16, v2 ; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v34, 0xffff, v3 ; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v32, 0xffff, v2 +; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v39, 16, v1 +; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v37, 16, v0 ; GCN-NOHSA-VI-NEXT: buffer_store_dword v32, off, s[88:91], 0 offset:4 ; 4-byte Folded Spill ; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(0) ; GCN-NOHSA-VI-NEXT: buffer_store_dword v33, off, s[88:91], 0 offset:8 ; 4-byte Folded Spill ; GCN-NOHSA-VI-NEXT: buffer_store_dword v34, off, s[88:91], 0 offset:12 ; 4-byte Folded Spill ; GCN-NOHSA-VI-NEXT: buffer_store_dword v35, off, s[88:91], 0 offset:16 ; 4-byte Folded Spill -; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v39, 16, v1 -; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v37, 16, v0 ; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v38, 0xffff, v1 ; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v36, 0xffff, v0 ; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v3, 16, v29 diff --git a/llvm/test/CodeGen/AMDGPU/schedule-physregdeps.mir b/llvm/test/CodeGen/AMDGPU/schedule-physregdeps.mir --- a/llvm/test/CodeGen/AMDGPU/schedule-physregdeps.mir +++ b/llvm/test/CodeGen/AMDGPU/schedule-physregdeps.mir @@ -4,15 +4,11 @@ # CHECK: SU(0): $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec # CHECK: Successors: # CHECK-NEXT: SU(2): Out Latency=1 -# CHECK-NEXT: SU(4): Out Latency=1 # CHECK-NEXT: SU(2): Data Latency=1 Reg=$vgpr0 -# CHECK-NEXT: SU(4): Data Latency=1 Reg=$vgpr0_vgpr1 # CHECK: SU(1): $vgpr1 = V_MOV_B32_e32 $sgpr0, implicit $exec # CHECK: Successors: # CHECK-NEXT: SU(3): Out Latency=1 -# CHECK-NEXT: SU(4): Out Latency=1 # CHECK-NEXT: SU(3): Data Latency=1 Reg=$vgpr1 -# CHECK-NEXT: SU(4): Data Latency=1 Reg=$vgpr0_vgpr1 # CHECK: SU(2): $vgpr0 = V_ADD_CO_U32_e32 $sgpr2, $vgpr0, implicit-def $vcc, implicit $exec # CHECK: Predecessors: # CHECK-NEXT: SU(0): Out Latency=1 @@ -22,7 +18,6 @@ # CHECK-NEXT: SU(4): Data Latency=1 Reg=$vgpr0_vgpr1 # CHECK-NEXT: SU(3): Out Latency=1 # CHECK-NEXT: SU(3): Data Latency=1 Reg=$vcc -# CHECK-NEXT: SU(4): Anti Latency=0 # CHECK: SU(3): $vgpr1 = V_ADDC_U32_e32 0, $vgpr1, implicit-def dead $vcc, implicit $vcc, implicit $exec # CHECK: Predecessors: # CHECK-NEXT: SU(2): Out Latency=1 @@ -32,19 +27,12 @@ # CHECK: Successors: # CHECK-NEXT: SU(4): Out Latency=1 # CHECK-NEXT: SU(4): Data Latency=1 Reg=$vgpr0_vgpr1 -# CHECK-NEXT: SU(4): Anti Latency=0 # CHECK: SU(4): $vgpr0_vgpr1 = FLAT_LOAD_DWORDX2 renamable $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr # CHECK: Predecessors: # CHECK-NEXT: SU(3): Out Latency=1 # CHECK-NEXT: SU(3): Data Latency=1 Reg=$vgpr0_vgpr1 -# CHECK-NEXT: SU(3): Anti Latency=0 # CHECK-NEXT: SU(2): Out Latency=1 # CHECK-NEXT: SU(2): Data Latency=1 Reg=$vgpr0_vgpr1 -# CHECK-NEXT: SU(2): Anti Latency=0 -# CHECK-NEXT: SU(1): Out Latency=1 -# CHECK-NEXT: SU(1): Data Latency=1 Reg=$vgpr0_vgpr1 -# CHECK-NEXT: SU(0): Out Latency=1 -# CHECK-NEXT: SU(0): Data Latency=1 Reg=$vgpr0_vgpr1 # CHECK: Successors: # CHECK-NEXT: ExitSU: Ord Latency=3 Artificial diff --git a/llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting-explicit-regs.ll b/llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting-explicit-regs.ll --- a/llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting-explicit-regs.ll +++ b/llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting-explicit-regs.ll @@ -35,8 +35,8 @@ ; Z15-LABEL: __int128_and_f: ; Z15: # %bb.0: # %entry ; Z15-NEXT: vl %v0, 0(%r3), 3 -; Z15-NEXT: vrepg %v6, %v0, 1 ; Z15-NEXT: vlr %v4, %v0 +; Z15-NEXT: vrepg %v6, %v0, 1 ; Z15-NEXT: #APP ; Z15-NEXT: #NO_APP ; Z15-NEXT: vmrhg %v0, %v4, %v6 @@ -260,8 +260,8 @@ define <4 x i32> @vec128_and_f(<4 x i32> %cc_dep1) { ; CHECK-LABEL: vec128_and_f: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vrepg %v3, %v24, 1 ; CHECK-NEXT: vlr %v1, %v24 +; CHECK-NEXT: vrepg %v3, %v24, 1 ; CHECK-NEXT: #APP ; CHECK-NEXT: #NO_APP ; CHECK-NEXT: vmrhg %v24, %v1, %v3 diff --git a/llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting.ll b/llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting.ll --- a/llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting.ll +++ b/llvm/test/CodeGen/SystemZ/inline-asm-fp-int-casting.ll @@ -249,8 +249,8 @@ define <4 x i32> @vec128_and_f(<4 x i32> %cc_dep1) { ; CHECK-LABEL: vec128_and_f: ; CHECK: # %bb.0: # %entry -; CHECK-NEXT: vrepg %v2, %v24, 1 ; CHECK-NEXT: vlr %v0, %v24 +; CHECK-NEXT: vrepg %v2, %v24, 1 ; CHECK-NEXT: #APP ; CHECK-NEXT: #NO_APP ; CHECK-NEXT: vmrhg %v24, %v0, %v2 diff --git a/llvm/test/CodeGen/Thumb2/mve-vldst4.ll b/llvm/test/CodeGen/Thumb2/mve-vldst4.ll --- a/llvm/test/CodeGen/Thumb2/mve-vldst4.ll +++ b/llvm/test/CodeGen/Thumb2/mve-vldst4.ll @@ -47,8 +47,8 @@ ; CHECK-NEXT: vmovx.f16 s8, s13 ; CHECK-NEXT: vins.f16 s20, s22 ; CHECK-NEXT: vins.f16 s16, s18 -; CHECK-NEXT: vins.f16 s25, s6 ; CHECK-NEXT: vmov.f32 s2, s5 +; CHECK-NEXT: vins.f16 s25, s6 ; CHECK-NEXT: vmov.f32 s3, s17 ; CHECK-NEXT: vins.f16 s0, s15 ; CHECK-NEXT: vmovx.f16 s9, s21 diff --git a/llvm/test/CodeGen/Thumb2/mve-vst3.ll b/llvm/test/CodeGen/Thumb2/mve-vst3.ll --- a/llvm/test/CodeGen/Thumb2/mve-vst3.ll +++ b/llvm/test/CodeGen/Thumb2/mve-vst3.ll @@ -1358,8 +1358,8 @@ ; CHECK-NEXT: vldrw.u32 q3, [r0] ; CHECK-NEXT: vmov r2, s2 ; CHECK-NEXT: vmov.f32 s16, s12 -; CHECK-NEXT: vins.f16 s16, s24 ; CHECK-NEXT: vmov.f32 s0, s13 +; CHECK-NEXT: vins.f16 s16, s24 ; CHECK-NEXT: vmov.16 q4[4], r2 ; CHECK-NEXT: vins.f16 s0, s25 ; CHECK-NEXT: vmov.f32 s19, s0