AMDGPU/GlobalISel: Select wqm, softwqm and wwm intrinsics

[kerbowa]

LGTM

[arsenm]

3b93945587a15bb92dd618214037b73197a6dc57

llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h

const RegisterBank *getArtifactRegBank(
     const TargetRegisterInfo &TRI) const;
 
   /// tblgen-erated 'select' implementation.
   bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
 
   MachineOperand getSubOperand64(MachineOperand &MO,
                                  const TargetRegisterClass &SubRC,
                                  unsigned SubIdx) const;
+
+  bool constrainCopyLikeIntrin(MachineInstr &MI, unsigned NewOpc) const;
   bool selectCOPY(MachineInstr &I) const;
   bool selectPHI(MachineInstr &I) const;
   bool selectG_TRUNC(MachineInstr &I) const;
   bool selectG_SZA_EXT(MachineInstr &I) const;
   bool selectG_CONSTANT(MachineInstr &I) const;
   bool selectG_AND_OR_XOR(MachineInstr &I) const;
   bool selectG_ADD_SUB(MachineInstr &I) const;
   bool selectG_UADDO_USUBO_UADDE_USUBE(MachineInstr &I) const;

llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp

if (RC) {
     return RC->hasSuperClassEq(TRI.getBoolRC()) &&
            Ty.isValid() && Ty.getSizeInBits() == 1;
   }
 
   const RegisterBank *RB = RegClassOrBank.get<const RegisterBank *>();
   return RB->getID() == AMDGPU::VCCRegBankID;
 }
 
+bool AMDGPUInstructionSelector::constrainCopyLikeIntrin(MachineInstr &MI,
+                                                        unsigned NewOpc) const {
+  MI.setDesc(TII.get(NewOpc));
+  MI.RemoveOperand(1); // Remove intrinsic ID.
+  MI.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true));
+
+  MachineOperand &Dst = MI.getOperand(0);
+  MachineOperand &Src = MI.getOperand(1);
+
+  // TODO: This should be legalized to s32 if needed
+  if (MRI->getType(Dst.getReg()) == LLT::scalar(1))
+    return false;
+
+  const TargetRegisterClass *DstRC
+    = TRI.getConstrainedRegClassForOperand(Dst, *MRI);
+  const TargetRegisterClass *SrcRC
+    = TRI.getConstrainedRegClassForOperand(Src, *MRI);
+  if (!DstRC || DstRC != SrcRC)
+    return false;
+
+  return RBI.constrainGenericRegister(Dst.getReg(), *DstRC, *MRI) &&
+         RBI.constrainGenericRegister(Src.getReg(), *SrcRC, *MRI);
+}
+
 bool AMDGPUInstructionSelector::selectCOPY(MachineInstr &I) const {
   const DebugLoc &DL = I.getDebugLoc();
   MachineBasicBlock *BB = I.getParent();
   I.setDesc(TII.get(TargetOpcode::COPY));
 
   const MachineOperand &Src = I.getOperand(1);
   MachineOperand &Dst = I.getOperand(0);
   Register DstReg = Dst.getReg();
case Intrinsic::amdgcn_if_break: {
 
     I.eraseFromParent();
 
     for (Register Reg : { DstReg, Src0Reg, Src1Reg })
       MRI->setRegClass(Reg, TRI.getWaveMaskRegClass());
 
     return true;
   }
+  case Intrinsic::amdgcn_wqm:
+    return constrainCopyLikeIntrin(I, AMDGPU::WQM);
+  case Intrinsic::amdgcn_softwqm:
+    return constrainCopyLikeIntrin(I, AMDGPU::SOFT_WQM);
+  case Intrinsic::amdgcn_wwm:
+    return constrainCopyLikeIntrin(I, AMDGPU::WWM);
   default:
     return selectImpl(I, *CoverageInfo);
   }
 }
 
 static int getV_CMPOpcode(CmpInst::Predicate P, unsigned Size) {
   if (Size != 32 && Size != 64)
     return -1;

llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp

case AMDGPU::G_INTRINSIC: {
     case Intrinsic::amdgcn_alignbyte:
     case Intrinsic::amdgcn_fdot2:
     case Intrinsic::amdgcn_sdot2:
     case Intrinsic::amdgcn_udot2:
     case Intrinsic::amdgcn_sdot4:
     case Intrinsic::amdgcn_udot4:
     case Intrinsic::amdgcn_sdot8:
     case Intrinsic::amdgcn_udot8:
-    case Intrinsic::amdgcn_wwm:
-    case Intrinsic::amdgcn_wqm:
       return getDefaultMappingVOP(MI);
     case Intrinsic::amdgcn_ds_swizzle:
     case Intrinsic::amdgcn_ds_permute:
     case Intrinsic::amdgcn_ds_bpermute:
     case Intrinsic::amdgcn_update_dpp:
+    case Intrinsic::amdgcn_wwm:
+    case Intrinsic::amdgcn_wqm:
+    case Intrinsic::amdgcn_softwqm:
       return getDefaultMappingAllVGPR(MI);
     case Intrinsic::amdgcn_kernarg_segment_ptr:
     case Intrinsic::amdgcn_s_getpc:
     case Intrinsic::amdgcn_groupstaticsize: {
       unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
       OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size);
       break;
     }

llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.softwqm.ll

+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -march=amdgcn -mcpu=hawaii -stop-after=instruction-select -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+define amdgpu_ps float @softwqm_f32(float %val) {
+  ; GCN-LABEL: name: softwqm_f32
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $vgpr0
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GCN:   [[SOFT_WQM:%[0-9]+]]:vgpr_32 = SOFT_WQM [[COPY]], implicit $exec
+  ; GCN:   $vgpr0 = COPY [[SOFT_WQM]]
+  ; GCN:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %ret = call float @llvm.amdgcn.softwqm.f32(float %val)
+  ret float %ret
+}
+
+define amdgpu_ps float @softwqm_v2f16(float %arg) {
+  ; GCN-LABEL: name: softwqm_v2f16
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $vgpr0
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GCN:   [[SOFT_WQM:%[0-9]+]]:vgpr_32 = SOFT_WQM [[COPY]], implicit $exec
+  ; GCN:   $vgpr0 = COPY [[SOFT_WQM]]
+  ; GCN:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %val = bitcast float %arg to <2 x half>
+  %ret = call <2 x half> @llvm.amdgcn.softwqm.v2f16(<2 x half> %val)
+  %bc = bitcast <2 x half> %ret to float
+  ret float %bc
+}
+
+define amdgpu_ps <2 x float> @softwqm_f64(double %val) {
+  ; GCN-LABEL: name: softwqm_f64
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $vgpr0, $vgpr1
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GCN:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GCN:   [[SOFT_WQM:%[0-9]+]]:vreg_64 = SOFT_WQM [[REG_SEQUENCE]], implicit $exec
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[SOFT_WQM]].sub0
+  ; GCN:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[SOFT_WQM]].sub1
+  ; GCN:   $vgpr0 = COPY [[COPY2]]
+  ; GCN:   $vgpr1 = COPY [[COPY3]]
+  ; GCN:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
+  %ret = call double @llvm.amdgcn.softwqm.f64(double %val)
+  %bitcast = bitcast double %ret to <2 x float>
+  ret <2 x float> %bitcast
+}
+
+; TODO
+; define amdgpu_ps float @softwqm_i1_vcc(float %val) {
+;   %vcc = fcmp oeq float %val, 0.0
+;   %ret = call i1 @llvm.amdgcn.softwqm.i1(i1 %vcc)
+;   %select = select i1 %ret, float 1.0, float 0.0
+;   ret float %select
+; }
+
+define amdgpu_ps <3 x float> @softwqm_v3f32(<3 x float> %val) {
+  ; GCN-LABEL: name: softwqm_v3f32
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $vgpr0, $vgpr1, $vgpr2
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GCN:   [[REG_SEQUENCE:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2
+  ; GCN:   [[SOFT_WQM:%[0-9]+]]:vreg_96 = SOFT_WQM [[REG_SEQUENCE]], implicit $exec
+  ; GCN:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[SOFT_WQM]].sub0
+  ; GCN:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[SOFT_WQM]].sub1
+  ; GCN:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[SOFT_WQM]].sub2
+  ; GCN:   $vgpr0 = COPY [[COPY3]]
+  ; GCN:   $vgpr1 = COPY [[COPY4]]
+  ; GCN:   $vgpr2 = COPY [[COPY5]]
+  ; GCN:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
+  %ret = call <3 x float> @llvm.amdgcn.softwqm.v3f32(<3 x float> %val)
+  ret <3 x float> %ret
+}
+
+declare i1 @llvm.amdgcn.softwqm.i1(i1) #0
+declare float @llvm.amdgcn.softwqm.f32(float) #0
+declare <2 x half> @llvm.amdgcn.softwqm.v2f16(<2 x half>) #0
+declare <3 x float> @llvm.amdgcn.softwqm.v3f32(<3 x float>) #0
+declare double @llvm.amdgcn.softwqm.f64(double) #0
+
+attributes #0 = { nounwind readnone speculatable }

llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wqm.ll

+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -march=amdgcn -mcpu=hawaii -stop-after=instruction-select -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+define amdgpu_ps float @wqm_f32(float %val) {
+  ; GCN-LABEL: name: wqm_f32
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $vgpr0
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GCN:   [[WQM:%[0-9]+]]:vgpr_32 = WQM [[COPY]], implicit $exec
+  ; GCN:   $vgpr0 = COPY [[WQM]]
+  ; GCN:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %ret = call float @llvm.amdgcn.wqm.f32(float %val)
+  ret float %ret
+}
+
+define amdgpu_ps float @wqm_v2f16(float %arg) {
+  ; GCN-LABEL: name: wqm_v2f16
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $vgpr0
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GCN:   [[WQM:%[0-9]+]]:vgpr_32 = WQM [[COPY]], implicit $exec
+  ; GCN:   $vgpr0 = COPY [[WQM]]
+  ; GCN:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %val = bitcast float %arg to <2 x half>
+  %ret = call <2 x half> @llvm.amdgcn.wqm.v2f16(<2 x half> %val)
+  %bc = bitcast <2 x half> %ret to float
+  ret float %bc
+}
+
+define amdgpu_ps <2 x float> @wqm_f64(double %val) {
+  ; GCN-LABEL: name: wqm_f64
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $vgpr0, $vgpr1
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GCN:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GCN:   [[WQM:%[0-9]+]]:vreg_64 = WQM [[REG_SEQUENCE]], implicit $exec
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[WQM]].sub0
+  ; GCN:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[WQM]].sub1
+  ; GCN:   $vgpr0 = COPY [[COPY2]]
+  ; GCN:   $vgpr1 = COPY [[COPY3]]
+  ; GCN:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
+  %ret = call double @llvm.amdgcn.wqm.f64(double %val)
+  %bitcast = bitcast double %ret to <2 x float>
+  ret <2 x float> %bitcast
+}
+
+; TODO
+; define amdgpu_ps float @wqm_i1_vcc(float %val) {
+;   %vcc = fcmp oeq float %val, 0.0
+;   %ret = call i1 @llvm.amdgcn.wqm.i1(i1 %vcc)
+;   %select = select i1 %ret, float 1.0, float 0.0
+;   ret float %select
+; }
+
+define amdgpu_ps <3 x float> @wqm_v3f32(<3 x float> %val) {
+  ; GCN-LABEL: name: wqm_v3f32
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $vgpr0, $vgpr1, $vgpr2
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GCN:   [[REG_SEQUENCE:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2
+  ; GCN:   [[WQM:%[0-9]+]]:vreg_96 = WQM [[REG_SEQUENCE]], implicit $exec
+  ; GCN:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[WQM]].sub0
+  ; GCN:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[WQM]].sub1
+  ; GCN:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[WQM]].sub2
+  ; GCN:   $vgpr0 = COPY [[COPY3]]
+  ; GCN:   $vgpr1 = COPY [[COPY4]]
+  ; GCN:   $vgpr2 = COPY [[COPY5]]
+  ; GCN:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
+  %ret = call <3 x float> @llvm.amdgcn.wqm.v3f32(<3 x float> %val)
+  ret <3 x float> %ret
+}
+
+declare i1 @llvm.amdgcn.wqm.i1(i1) #0
+declare float @llvm.amdgcn.wqm.f32(float) #0
+declare <2 x half> @llvm.amdgcn.wqm.v2f16(<2 x half>) #0
+declare <3 x float> @llvm.amdgcn.wqm.v3f32(<3 x float>) #0
+declare double @llvm.amdgcn.wqm.f64(double) #0
+
+attributes #0 = { nounwind readnone speculatable }

llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.wwm.ll

+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+; RUN: llc -global-isel -march=amdgcn -mcpu=hawaii -stop-after=instruction-select -verify-machineinstrs < %s | FileCheck -check-prefix=GCN %s
+
+define amdgpu_ps float @wwm_f32(float %val) {
+  ; GCN-LABEL: name: wwm_f32
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $vgpr0
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GCN:   [[WWM:%[0-9]+]]:vgpr_32 = WWM [[COPY]], implicit $exec
+  ; GCN:   $vgpr0 = COPY [[WWM]]
+  ; GCN:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %ret = call float @llvm.amdgcn.wwm.f32(float %val)
+  ret float %ret
+}
+
+define amdgpu_ps float @wwm_v2f16(float %arg) {
+  ; GCN-LABEL: name: wwm_v2f16
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $vgpr0
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GCN:   [[WWM:%[0-9]+]]:vgpr_32 = WWM [[COPY]], implicit $exec
+  ; GCN:   $vgpr0 = COPY [[WWM]]
+  ; GCN:   SI_RETURN_TO_EPILOG implicit $vgpr0
+  %val = bitcast float %arg to <2 x half>
+  %ret = call <2 x half> @llvm.amdgcn.wwm.v2f16(<2 x half> %val)
+  %bc = bitcast <2 x half> %ret to float
+  ret float %bc
+}
+
+define amdgpu_ps <2 x float> @wwm_f64(double %val) {
+  ; GCN-LABEL: name: wwm_f64
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $vgpr0, $vgpr1
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GCN:   [[REG_SEQUENCE:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1
+  ; GCN:   [[WWM:%[0-9]+]]:vreg_64 = WWM [[REG_SEQUENCE]], implicit $exec
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY [[WWM]].sub0
+  ; GCN:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[WWM]].sub1
+  ; GCN:   $vgpr0 = COPY [[COPY2]]
+  ; GCN:   $vgpr1 = COPY [[COPY3]]
+  ; GCN:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1
+  %ret = call double @llvm.amdgcn.wwm.f64(double %val)
+  %bitcast = bitcast double %ret to <2 x float>
+  ret <2 x float> %bitcast
+}
+
+; TODO
+; define amdgpu_ps float @wwm_i1_vcc(float %val) {
+;   %vcc = fcmp oeq float %val, 0.0
+;   %ret = call i1 @llvm.amdgcn.wwm.i1(i1 %vcc)
+;   %select = select i1 %ret, float 1.0, float 0.0
+;   ret float %select
+; }
+
+define amdgpu_ps <3 x float> @wwm_v3f32(<3 x float> %val) {
+  ; GCN-LABEL: name: wwm_v3f32
+  ; GCN: bb.1 (%ir-block.0):
+  ; GCN:   liveins: $vgpr0, $vgpr1, $vgpr2
+  ; GCN:   [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+  ; GCN:   [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+  ; GCN:   [[COPY2:%[0-9]+]]:vgpr_32 = COPY $vgpr2
+  ; GCN:   [[REG_SEQUENCE:%[0-9]+]]:vreg_96 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY1]], %subreg.sub1, [[COPY2]], %subreg.sub2
+  ; GCN:   [[WWM:%[0-9]+]]:vreg_96 = WWM [[REG_SEQUENCE]], implicit $exec
+  ; GCN:   [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[WWM]].sub0
+  ; GCN:   [[COPY4:%[0-9]+]]:vgpr_32 = COPY [[WWM]].sub1
+  ; GCN:   [[COPY5:%[0-9]+]]:vgpr_32 = COPY [[WWM]].sub2
+  ; GCN:   $vgpr0 = COPY [[COPY3]]
+  ; GCN:   $vgpr1 = COPY [[COPY4]]
+  ; GCN:   $vgpr2 = COPY [[COPY5]]
+  ; GCN:   SI_RETURN_TO_EPILOG implicit $vgpr0, implicit $vgpr1, implicit $vgpr2
+  %ret = call <3 x float> @llvm.amdgcn.wwm.v3f32(<3 x float> %val)
+  ret <3 x float> %ret
+}
+
+declare i1 @llvm.amdgcn.wwm.i1(i1) #0
+declare float @llvm.amdgcn.wwm.f32(float) #0
+declare <2 x half> @llvm.amdgcn.wwm.v2f16(<2 x half>) #0
+declare <3 x float> @llvm.amdgcn.wwm.v3f32(<3 x float>) #0
+declare double @llvm.amdgcn.wwm.f64(double) #0
+
+attributes #0 = { nounwind readnone speculatable }