diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h @@ -662,7 +662,9 @@ static bool isWWMRegSpillOpcode(uint16_t Opcode) { return Opcode == AMDGPU::SI_SPILL_WWM_V32_SAVE || - Opcode == AMDGPU::SI_SPILL_WWM_V32_RESTORE; + Opcode == AMDGPU::SI_SPILL_WWM_AV32_SAVE || + Opcode == AMDGPU::SI_SPILL_WWM_V32_RESTORE || + Opcode == AMDGPU::SI_SPILL_WWM_AV32_RESTORE; } static bool isDPP(const MachineInstr &MI) { diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -1587,11 +1587,15 @@ } } -static unsigned getWWMRegSpillSaveOpcode(unsigned Size) { +static unsigned getWWMRegSpillSaveOpcode(unsigned Size, + bool IsVectorSuperClass) { // Currently, there is only 32-bit WWM register spills needed. if (Size != 4) llvm_unreachable("unknown wwm register spill size"); + if (IsVectorSuperClass) + return AMDGPU::SI_SPILL_WWM_AV32_SAVE; + return AMDGPU::SI_SPILL_WWM_V32_SAVE; } @@ -1600,11 +1604,13 @@ unsigned Size, const SIRegisterInfo &TRI, const SIMachineFunctionInfo &MFI) { + bool IsVectorSuperClass = TRI.isVectorSuperClass(RC); + // Choose the right opcode if spilling a WWM register. if (MFI.checkFlag(Reg, AMDGPU::VirtRegFlag::WWM_REG)) - return getWWMRegSpillSaveOpcode(Size); + return getWWMRegSpillSaveOpcode(Size, IsVectorSuperClass); - if (TRI.isVectorSuperClass(RC)) + if (IsVectorSuperClass) return getAVSpillSaveOpcode(Size); return TRI.isAGPRClass(RC) ? getAGPRSpillSaveOpcode(Size) @@ -1807,11 +1813,15 @@ } } -static unsigned getWWMRegSpillRestoreOpcode(unsigned Size) { +static unsigned getWWMRegSpillRestoreOpcode(unsigned Size, + bool IsVectorSuperClass) { // Currently, there is only 32-bit WWM register spills needed. if (Size != 4) llvm_unreachable("unknown wwm register spill size"); + if (IsVectorSuperClass) + return AMDGPU::SI_SPILL_WWM_AV32_RESTORE; + return AMDGPU::SI_SPILL_WWM_V32_RESTORE; } @@ -1819,11 +1829,13 @@ getVectorRegSpillRestoreOpcode(Register Reg, const TargetRegisterClass *RC, unsigned Size, const SIRegisterInfo &TRI, const SIMachineFunctionInfo &MFI) { + bool IsVectorSuperClass = TRI.isVectorSuperClass(RC); + // Choose the right opcode if restoring a WWM register. if (MFI.checkFlag(Reg, AMDGPU::VirtRegFlag::WWM_REG)) - return getWWMRegSpillRestoreOpcode(Size); + return getWWMRegSpillRestoreOpcode(Size, IsVectorSuperClass); - if (TRI.isVectorSuperClass(RC)) + if (IsVectorSuperClass) return getAVSpillRestoreOpcode(Size); return TRI.isAGPRClass(RC) ? getAGPRSpillRestoreOpcode(Size) diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td --- a/llvm/lib/Target/AMDGPU/SIInstructions.td +++ b/llvm/lib/Target/AMDGPU/SIInstructions.td @@ -952,8 +952,10 @@ defm SI_SPILL_AV512 : SI_SPILL_VGPR ; defm SI_SPILL_AV1024 : SI_SPILL_VGPR ; -let isConvergent = 1 in -defm SI_SPILL_WWM_V32 : SI_SPILL_VGPR ; +let isConvergent = 1 in { + defm SI_SPILL_WWM_V32 : SI_SPILL_VGPR ; + defm SI_SPILL_WWM_AV32 : SI_SPILL_VGPR ; +} def SI_PC_ADD_REL_OFFSET : SPseudoInstSI < (outs SReg_64:$dst), diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp --- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp @@ -1065,6 +1065,8 @@ case AMDGPU::SI_SPILL_AV32_RESTORE: case AMDGPU::SI_SPILL_WWM_V32_SAVE: case AMDGPU::SI_SPILL_WWM_V32_RESTORE: + case AMDGPU::SI_SPILL_WWM_AV32_SAVE: + case AMDGPU::SI_SPILL_WWM_AV32_RESTORE: return 1; default: llvm_unreachable("Invalid spill opcode"); } @@ -2144,7 +2146,8 @@ case AMDGPU::SI_SPILL_AV96_SAVE: case AMDGPU::SI_SPILL_AV64_SAVE: case AMDGPU::SI_SPILL_AV32_SAVE: - case AMDGPU::SI_SPILL_WWM_V32_SAVE: { + case AMDGPU::SI_SPILL_WWM_V32_SAVE: + case AMDGPU::SI_SPILL_WWM_AV32_SAVE: { const MachineOperand *VData = TII->getNamedOperand(*MI, AMDGPU::OpName::vdata); assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() == @@ -2211,7 +2214,8 @@ case AMDGPU::SI_SPILL_AV384_RESTORE: case AMDGPU::SI_SPILL_AV512_RESTORE: case AMDGPU::SI_SPILL_AV1024_RESTORE: - case AMDGPU::SI_SPILL_WWM_V32_RESTORE: { + case AMDGPU::SI_SPILL_WWM_V32_RESTORE: + case AMDGPU::SI_SPILL_WWM_AV32_RESTORE: { const MachineOperand *VData = TII->getNamedOperand(*MI, AMDGPU::OpName::vdata); assert(TII->getNamedOperand(*MI, AMDGPU::OpName::soffset)->getReg() ==