diff --git a/.arcconfig b/.arcconfig --- a/.arcconfig +++ b/.arcconfig @@ -4,5 +4,6 @@ "conduit_uri" : "https://reviews.llvm.org/", "base": "git:HEAD^", "arc.land.onto.default": "main", - "arc.land.onto": ["main"] + "arc.land.onto": ["main"], + "history.immutable" : false } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMFMAClustering.h b/llvm/lib/Target/AMDGPU/AMDGPUMFMAClustering.h new file mode 100644 --- /dev/null +++ b/llvm/lib/Target/AMDGPU/AMDGPUMFMAClustering.h @@ -0,0 +1,21 @@ +//===- AMDGPUMFMAClustering.h - AMDGPU MFMA Clustering ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUMFMACLUSTERING_H +#define LLVM_LIB_TARGET_AMDGPU_AMDGPUMFMACLUSTERING_H + +#include "llvm/CodeGen/ScheduleDAGMutation.h" +#include + +namespace llvm { + +std::unique_ptr createMFMAClusterDAGMutation(); + +} // namespace llvm + +#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUMFMACLUSTERING_H diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMFMAClustering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMFMAClustering.cpp new file mode 100644 --- /dev/null +++ b/llvm/lib/Target/AMDGPU/AMDGPUMFMAClustering.cpp @@ -0,0 +1,171 @@ +//===--- AMDGPUMFMAClusting.cpp - AMDGPU MFMA Clustering -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file This file contains a DAG scheduling mutation to cluster MFMA +/// instructions. +// +//===----------------------------------------------------------------------===// + +#include "AMDGPUMFMAClustering.h" +#include "AMDGPUTargetMachine.h" +#include "MCTargetDesc/AMDGPUMCTargetDesc.h" +#include "SIInstrInfo.h" +#include "SIMachineFunctionInfo.h" +#include "llvm/CodeGen/MachineScheduler.h" + +using namespace llvm; + +#define DEBUG_TYPE "amdgpu-subtarget" + +namespace { + +static cl::opt EnableMFMACluster("amdgpu-mfma-cluster", + cl::desc("Enable MFMA clustering"), + cl::init(false)); + +static cl::opt + MFMAClusterSize("amdgpu-mfma-cluster-size", cl::init(5), cl::Hidden, + cl::desc("The maximum number of MFMA insts to " + "attempt to cluster together.")); + +class MFMAClusterDAGMutation : public ScheduleDAGMutation { + const SIInstrInfo *TII; + ScheduleDAGMI *DAG; + +public: + MFMAClusterDAGMutation() = default; + void apply(ScheduleDAGInstrs *DAGInstrs) override; +}; + +static void collectMFMASUnits(SmallVectorImpl &MFMASUnits, + const SIInstrInfo *TII, ScheduleDAGInstrs *DAG) { + for (SUnit &SU : DAG->SUnits) { + MachineInstr &MAI = *SU.getInstr(); + if (!TII->isMAI(MAI) || + MAI.getOpcode() == AMDGPU::V_ACCVGPR_WRITE_B32_e64 || + MAI.getOpcode() == AMDGPU::V_ACCVGPR_READ_B32_e64) + continue; + + MFMASUnits.push_back(&SU); + + LLVM_DEBUG(dbgs() << "Found MFMA: "; DAG->dumpNode(SU);); + } +} + +static void propagateDeps(DenseMap &SUnit2ClusterInfo, + llvm::ArrayRef ClusterPreds, + llvm::ArrayRef ClusterSuccs, SUnit *MFMAOpa, + ScheduleDAGInstrs *DAG) { + + for (auto Node : SUnit2ClusterInfo) { + if (Node.second != MFMAOpa->NodeNum) + continue; // Only add the combined succs to the current cluster + + LLVM_DEBUG(dbgs() << "Copying Deps To SU(" << Node.first << ")\n"); + + for (const SDep &Succ : ClusterSuccs) { + LLVM_DEBUG(dbgs() << "Copying Succ SU(" << Succ.getSUnit()->NodeNum + << ")\n"); + DAG->addEdge(Succ.getSUnit(), + SDep(&DAG->SUnits[Node.first], SDep::Artificial)); + } + + for (const SDep &Pred : ClusterPreds) { + LLVM_DEBUG(dbgs() << "Copying Pred SU(" << Pred.getSUnit()->NodeNum + << ")\n"); + if (Pred.getSUnit()->NodeNum == MFMAOpa->NodeNum) + continue; + DAG->addEdge(&DAG->SUnits[Node.first], + SDep(Pred.getSUnit(), SDep::Artificial)); + } + } +} + +static void clusterNeighboringMFMAs(llvm::ArrayRef MFMASUnits, + ScheduleDAGInstrs *DAG) { + + DenseMap SUnit2ClusterInfo; + + for (unsigned Idx = 0, End = MFMASUnits.size(); Idx < (End - 1); ++Idx) { + if (SUnit2ClusterInfo.count(MFMASUnits[Idx]->NodeNum)) + continue; // We don't want to cluster against a different cluster + + auto MFMAOpa = MFMASUnits[Idx]; + SmallVector ClusterSuccs(MFMAOpa->Succs); + SmallVector ClusterPreds(MFMAOpa->Preds); + unsigned NextIdx = Idx + 1; + unsigned ClusterSize = 1; + + // Attempt to cluster all the remaining MFMASunits with MFMAOpa. + // Clustering in this manner allows for nicely handling the preds and + // succs s.t. they dont get interspersed in the cluster. + for (; NextIdx < End; ++NextIdx) { + if (ClusterSize >= MFMAClusterSize) + break; + + // Only add independent MFMAs that have not been previously clustered + if (SUnit2ClusterInfo.count(MFMASUnits[NextIdx]->NodeNum) || + DAG->IsReachable(MFMASUnits[NextIdx], MFMAOpa) || + DAG->IsReachable(MFMAOpa, MFMASUnits[NextIdx])) + continue; + + auto MFMAOpb = MFMASUnits[NextIdx]; + if (MFMAOpa->NodeNum > MFMAOpb->NodeNum) + std::swap(MFMAOpa, MFMAOpb); + + if (!DAG->addEdge(MFMAOpb, SDep(MFMAOpa, SDep::Cluster))) + continue; + + LLVM_DEBUG(dbgs() << "Cluster MFMA SU(" << MFMAOpa->NodeNum << ") - SU(" + << MFMAOpb->NodeNum << ")\n"); + + SUnit2ClusterInfo[MFMAOpb->NodeNum] = MFMAOpa->NodeNum; + SUnit2ClusterInfo[MFMAOpa->NodeNum] = MFMAOpa->NodeNum; + ++ClusterSize; + // Aggregate the deps over each inst in the cluster + ClusterPreds.append(MFMAOpb->Preds); + ClusterSuccs.append(MFMAOpb->Succs); + } + + propagateDeps(SUnit2ClusterInfo, ClusterPreds, ClusterSuccs, MFMAOpa, DAG); + } +} + +void MFMAClusterDAGMutation::apply(ScheduleDAGInstrs *DAGInstrs) { + const GCNSubtarget &ST = DAGInstrs->MF.getSubtarget(); + TII = ST.getInstrInfo(); + const SIMachineFunctionInfo *MFI = + DAGInstrs->MF.getInfo(); + // The purpose of clustering is to aid with multi wave scheduling. + // If our occupancy doesn't support multi waves, bypass clustering + if (!ST.hasMAIInsts() || MFI->getOccupancy() < 2) + return; + DAG = static_cast(DAGInstrs); + const TargetSchedModel *TSchedModel = DAGInstrs->getSchedModel(); + if (!TSchedModel || DAG->SUnits.empty()) + return; + + SmallVector MFMASUnits; + collectMFMASUnits(MFMASUnits, TII, DAG); + + if (MFMASUnits.size() < 2) + return; + + clusterNeighboringMFMAs(MFMASUnits, DAG); +} + +} // namespace + +namespace llvm { + +std::unique_ptr createMFMAClusterDAGMutation() { + return EnableMFMACluster ? std::make_unique() + : nullptr; +} + +} // end namespace llvm diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -16,6 +16,7 @@ #include "AMDGPU.h" #include "AMDGPUAliasAnalysis.h" #include "AMDGPUExportClustering.h" +#include "AMDGPUMFMAClustering.h" #include "AMDGPUMacroFusion.h" #include "AMDGPUTargetObjectFile.h" #include "AMDGPUTargetTransformInfo.h" @@ -398,6 +399,7 @@ ScheduleDAGMILive *DAG = new GCNScheduleDAGMILive(C, std::make_unique(C)); DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI)); + DAG->addMutation(createMFMAClusterDAGMutation()); DAG->addMutation(createAMDGPUMacroFusionDAGMutation()); DAG->addMutation(createAMDGPUExportClusteringDAGMutation()); return DAG; @@ -879,6 +881,7 @@ const GCNSubtarget &ST = C->MF->getSubtarget(); DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI)); DAG->addMutation(ST.createFillMFMAShadowMutation(DAG->TII)); + DAG->addMutation(createMFMAClusterDAGMutation()); return DAG; } diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt --- a/llvm/lib/Target/AMDGPU/CMakeLists.txt +++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt @@ -75,6 +75,7 @@ AMDGPUMachineModuleInfo.cpp AMDGPUMacroFusion.cpp AMDGPUMCInstLower.cpp + AMDGPUMFMAClustering.cpp AMDGPUMIRFormatter.cpp AMDGPUOpenCLEnqueuedBlockLowering.cpp AMDGPUPerfHintAnalysis.cpp diff --git a/llvm/test/CodeGen/AMDGPU/mfma-cluster.mir b/llvm/test/CodeGen/AMDGPU/mfma-cluster.mir new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/mfma-cluster.mir @@ -0,0 +1,247 @@ +# REQUIRES : asserts +# RUN: llc -march=amdgcn -mcpu=gfx90a -run-pass=machine-scheduler %s -o - --debug-only=amdgpu-subtarget,machine-scheduler 2>&1 | FileCheck -check-prefix=DEFAULT %s +# RUN: llc -march=amdgcn -mcpu=gfx90a -run-pass=machine-scheduler %s -o - -amdgpu-mfma-cluster=1 --debug-only=amdgpu-subtarget,machine-scheduler 2>&1 | FileCheck -check-prefix=PRERA %s +# RUN: llc -march=amdgcn -mcpu=gfx90a -run-pass=machine-scheduler %s -o - -amdgpu-mfma-cluster=1 -amdgpu-mfma-cluster-size=2 --debug-only=amdgpu-subtarget,machine-scheduler 2>&1 | FileCheck -check-prefix=TWOLIMIT %s +# RUN: llc -march=amdgcn -mcpu=gfx90a -start-before=si-post-ra-bundler %s -o - -amdgpu-mfma-cluster=1 --debug-only=amdgpu-subtarget,machine-scheduler 2>&1| FileCheck -check-prefix=POSTRA %s + + +--- +name: no_cluster +tracksRegLiveness: true +body: | + bb.0: + liveins: $sgpr0, $vgpr10_vgpr11 + $vgpr1 = V_MOV_B32_e32 1, implicit $exec + $vgpr0 = V_MOV_B32_e32 1, implicit $exec + $vgpr8 = V_MOV_B32_e32 0, implicit $exec + $vgpr9 = V_MOV_B32_e32 9, implicit $exec + $vgpr1 = V_ADD_F16_e32 $vgpr1, $vgpr0, implicit $mode, implicit $exec + GLOBAL_STORE_DWORD $vgpr10_vgpr11, $vgpr1, 0, 0, implicit $exec + $vgpr2 = V_MOV_B32_e32 1, implicit $exec + $vgpr3 = DS_READ_U16_gfx9 $vgpr2, 0, 0, implicit $exec + $vgpr4 = GLOBAL_LOAD_USHORT $vgpr0_vgpr1, 0, 0, implicit $exec + $vgpr5 = V_XOR_B32_e32 $vgpr1, $vgpr0, implicit $exec + $vgpr6 = V_MUL_LO_U32_e64 $vgpr1, $sgpr0, implicit $exec + $vgpr7 = GLOBAL_LOAD_USHORT $vgpr8_vgpr9, 0, 0, implicit $exec +... + +# PRERA: *** Final schedule for %bb.0 *** +# PRERA-NEXT: SU(0): $vgpr1 = V_MOV_B32_e32 1, implicit $exec +# PRERA-NEXT: SU(1): $vgpr0 = V_MOV_B32_e32 1, implicit $exec +# PRERA-NEXT: SU(4): $vgpr1 = V_ADD_F16_e32 $vgpr1, $vgpr0, implicit $mode, implicit $exec +# PRERA-NEXT: SU(5): GLOBAL_STORE_DWORD $vgpr10_vgpr11, $vgpr1, 0, 0, implicit $exec +# PRERA-NEXT: SU(6): $vgpr2 = V_MOV_B32_e32 1, implicit $exec +# PRERA-NEXT: SU(7): $vgpr3 = DS_READ_U16_gfx9 $vgpr2, 0, 0, implicit $exec +# PRERA-NEXT: SU(8): $vgpr4 = GLOBAL_LOAD_USHORT $vgpr0_vgpr1, 0, 0, implicit $exec +# PRERA-NEXT: SU(2): $vgpr8 = V_MOV_B32_e32 0, implicit $exec +# PRERA-NEXT: SU(3): $vgpr9 = V_MOV_B32_e32 9, implicit $exec +# PRERA-NEXT: SU(11): $vgpr7 = GLOBAL_LOAD_USHORT $vgpr8_vgpr9, 0, 0, implicit $exec +# PRERA-NEXT: SU(9): $vgpr5 = V_XOR_B32_e32 $vgpr1, $vgpr0, implicit $exec +# PRERA-NEXT: SU(10): $vgpr6 = V_MUL_LO_U32_e64 $vgpr1, $sgpr0, implicit $exec + +# DEFAULT: *** Final schedule for %bb.0 *** +# DEFAULT-NEXT: SU(0): $vgpr1 = V_MOV_B32_e32 1, implicit $exec +# DEFAULT-NEXT: SU(1): $vgpr0 = V_MOV_B32_e32 1, implicit $exec +# DEFAULT-NEXT: SU(4): $vgpr1 = V_ADD_F16_e32 $vgpr1, $vgpr0, implicit $mode, implicit $exec +# DEFAULT-NEXT: SU(5): GLOBAL_STORE_DWORD $vgpr10_vgpr11, $vgpr1, 0, 0, implicit $exec +# DEFAULT-NEXT: SU(6): $vgpr2 = V_MOV_B32_e32 1, implicit $exec +# DEFAULT-NEXT: SU(7): $vgpr3 = DS_READ_U16_gfx9 $vgpr2, 0, 0, implicit $exec +# DEFAULT-NEXT: SU(8): $vgpr4 = GLOBAL_LOAD_USHORT $vgpr0_vgpr1, 0, 0, implicit $exec +# DEFAULT-NEXT: SU(2): $vgpr8 = V_MOV_B32_e32 0, implicit $exec +# DEFAULT-NEXT: SU(3): $vgpr9 = V_MOV_B32_e32 9, implicit $exec +# DEFAULT-NEXT: SU(11): $vgpr7 = GLOBAL_LOAD_USHORT $vgpr8_vgpr9, 0, 0, implicit $exec +# DEFAULT-NEXT: SU(9): $vgpr5 = V_XOR_B32_e32 $vgpr1, $vgpr0, implicit $exec +# DEFAULT-NEXT: SU(10): $vgpr6 = V_MUL_LO_U32_e64 $vgpr1, $sgpr0, implicit $exec + +--- +name: basic_cluster +tracksRegLiveness: true +body: | + bb.0: + liveins: $agpr0_agpr1_agpr2_agpr3, $agpr4_agpr5_agpr6_agpr7, $agpr8_agpr9_agpr10_agpr11, $agpr12_agpr13_agpr14_agpr15 + $vgpr1 = V_MOV_B32_e32 1, implicit $exec + $vgpr0 = V_MOV_B32_e32 1, implicit $exec + $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + $vgpr2 = V_MOV_B32_e32 1, implicit $exec + $vgpr3 = V_MOV_B32_e32 1, implicit $exec + $vgpr4 = V_MOV_B32_e32 1, implicit $exec + $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_4X4X1F32_e64 $vgpr3, $vgpr4, $agpr4_agpr5_agpr6_agpr7, 0, 0, 0, implicit $mode, implicit $exec + $vgpr5 = V_XOR_B32_e32 $vgpr1, $vgpr0, implicit $exec + $vgpr5 = V_MOV_B32_e32 1, implicit $exec + $vgpr6 = V_MOV_B32_e32 1, implicit $exec + $agpr8_agpr9_agpr10_agpr11 = V_MFMA_F32_4X4X1F32_e64 $vgpr3, $vgpr4, $agpr8_agpr9_agpr10_agpr11, 0, 0, 0, implicit $mode, implicit $exec + $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_4X4X1F32_e64 $vgpr5, $vgpr6, $agpr4_agpr5_agpr6_agpr7, 0, 0, 0, implicit $mode, implicit $exec + $agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec +... + +# DEFAULT: *** Final schedule for %bb.0 *** +# DEFAULT-NEXT: SU(0): $vgpr1 = V_MOV_B32_e32 1, implicit $exec +# DEFAULT-NEXT: SU(1): $vgpr0 = V_MOV_B32_e32 1, implicit $exec +# DEFAULT-NEXT: SU(2): $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec +# DEFAULT-NEXT: SU(4): $vgpr3 = V_MOV_B32_e32 1, implicit $exec +# DEFAULT-NEXT: SU(5): $vgpr4 = V_MOV_B32_e32 1, implicit $exec +# DEFAULT-NEXT: SU(6): $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_4X4X1F32_e64 $vgpr3, $vgpr4, $agpr4_agpr5_agpr6_agpr7, 0, 0, 0, implicit $mode, implicit $exec +# DEFAULT-NEXT: SU(7): $vgpr5 = V_XOR_B32_e32 $vgpr1, $vgpr0, implicit $exec +# DEFAULT-NEXT: SU(10): $agpr8_agpr9_agpr10_agpr11 = V_MFMA_F32_4X4X1F32_e64 $vgpr3, $vgpr4, $agpr8_agpr9_agpr10_agpr11, 0, 0, 0, implicit $mode, implicit $exec +# DEFAULT-NEXT: SU(8): $vgpr5 = V_MOV_B32_e32 1, implicit $exec +# DEFAULT-NEXT: SU(9): $vgpr6 = V_MOV_B32_e32 1, implicit $exec +# DEFAULT-NEXT: SU(11): $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_4X4X1F32_e64 $vgpr5, $vgpr6, $agpr4_agpr5_agpr6_agpr7, 0, 0, 0, implicit $mode, implicit $exec +# DEFAULT-NEXT: SU(12): $agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec +# DEFAULT-NEXT: SU(3): $vgpr2 = V_MOV_B32_e32 1, implicit $exec + +# PRERA: Cluster MFMA SU(2) - SU(6) +# PRERA-NEXT: Cluster MFMA SU(2) - SU(10) +# PRERA-NEXT: Cluster MFMA SU(2) - SU(12) +# PRERA: *** Final schedule for %bb.0 *** +# PRERA-NEXT: SU(4): $vgpr3 = V_MOV_B32_e32 1, implicit $exec +# PRERA-NEXT: SU(0): $vgpr1 = V_MOV_B32_e32 1, implicit $exec +# PRERA-NEXT: SU(1): $vgpr0 = V_MOV_B32_e32 1, implicit $exec +# PRERA-NEXT: SU(5): $vgpr4 = V_MOV_B32_e32 1, implicit $exec +# PRERA-NEXT: SU(6): $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_4X4X1F32_e64 $vgpr3, $vgpr4, $agpr4_agpr5_agpr6_agpr7, 0, 0, 0, implicit $mode, implicit $exec +# PRERA-NEXT: SU(10): $agpr8_agpr9_agpr10_agpr11 = V_MFMA_F32_4X4X1F32_e64 $vgpr3, $vgpr4, $agpr8_agpr9_agpr10_agpr11, 0, 0, 0, implicit $mode, implicit $exec +# PRERA-NEXT: SU(2): $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec +# PRERA-NEXT: SU(7): $vgpr5 = V_XOR_B32_e32 $vgpr1, $vgpr0, implicit $exec +# PRERA-NEXT: SU(12): $agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec +# PRERA-NEXT: SU(8): $vgpr5 = V_MOV_B32_e32 1, implicit $exec +# PRERA-NEXT: SU(9): $vgpr6 = V_MOV_B32_e32 1, implicit $exec +# PRERA-NEXT: SU(11): $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_4X4X1F32_e64 $vgpr5, $vgpr6, $agpr4_agpr5_agpr6_agpr7, 0, 0, 0, implicit $mode, implicit $exec +# PRERA-NEXT: SU(3): $vgpr2 = V_MOV_B32_e32 1, implicit $exec + +# TWOLIMIT: Cluster MFMA SU(2) - SU(6) +# TWOLIMIT: Cluster MFMA SU(10) - SU(11) +# TWOLIMIT: *** Final schedule for %bb.0 *** +# TWOLIMIT-NEXT: SU(0): $vgpr1 = V_MOV_B32_e32 1, implicit $exec +# TWOLIMIT-NEXT: SU(4): $vgpr3 = V_MOV_B32_e32 1, implicit $exec +# TWOLIMIT-NEXT: SU(1): $vgpr0 = V_MOV_B32_e32 1, implicit $exec +# TWOLIMIT-NEXT: SU(5): $vgpr4 = V_MOV_B32_e32 1, implicit $exec +# TWOLIMIT-NEXT: SU(2): $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec +# TWOLIMIT-NEXT: SU(6): $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_4X4X1F32_e64 $vgpr3, $vgpr4, $agpr4_agpr5_agpr6_agpr7, 0, 0, 0, implicit $mode, implicit $exec +# TWOLIMIT-NEXT: SU(7): $vgpr5 = V_XOR_B32_e32 $vgpr1, $vgpr0, implicit $exec +# TWOLIMIT-NEXT: SU(8): $vgpr5 = V_MOV_B32_e32 1, implicit $exec +# TWOLIMIT-NEXT: SU(9): $vgpr6 = V_MOV_B32_e32 1, implicit $exec +# TWOLIMIT-NEXT: SU(10): $agpr8_agpr9_agpr10_agpr11 = V_MFMA_F32_4X4X1F32_e64 $vgpr3, $vgpr4, $agpr8_agpr9_agpr10_agpr11, 0, 0, 0, implicit $mode, implicit $exec +# TWOLIMIT-NEXT: SU(11): $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_4X4X1F32_e64 $vgpr5, $vgpr6, $agpr4_agpr5_agpr6_agpr7, 0, 0, 0, implicit $mode, implicit $exec +# TWOLIMIT-NEXT: SU(12): $agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec + + +# POSTRA: Cluster MFMA SU(2) - SU(6) +# POSTRA-NEXT: Cluster MFMA SU(2) - SU(10) +# POSTRA-NEXT: Cluster MFMA SU(2) - SU(12) +# POSTRA: *** Final schedule for %bb.0 *** +# POSTRA-NEXT: SU(0): $vgpr1 = V_MOV_B32_e32 1, implicit $exec +# POSTRA-NEXT: SU(1): $vgpr0 = V_MOV_B32_e32 1, implicit $exec +# POSTRA-NEXT: SU(4): $vgpr3 = V_MOV_B32_e32 1, implicit $exec +# POSTRA-NEXT: SU(7): $vgpr5 = V_XOR_B32_e32 $vgpr1, $vgpr0, implicit $exec +# POSTRA-NEXT: SU(8): $vgpr5 = V_MOV_B32_e32 1, implicit $exec +# POSTRA-NEXT: SU(5): $vgpr4 = V_MOV_B32_e32 1, implicit $exec +# POSTRA-NEXT: SU(2): $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec +# POSTRA-NEXT: SU(9): $vgpr6 = V_MOV_B32_e32 1, implicit $exec +# POSTRA-NEXT: SU(12): $agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec +# POSTRA-NEXT: SU(3): $vgpr2 = V_MOV_B32_e32 1, implicit $exec +# POSTRA-NEXT: SU(6): $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_4X4X1F32_e64 $vgpr3, $vgpr4, $agpr4_agpr5_agpr6_agpr7, 0, 0, 0, implicit $mode, implicit $exec +# POSTRA-NEXT: SU(10): $agpr8_agpr9_agpr10_agpr11 = V_MFMA_F32_4X4X1F32_e64 $vgpr3, $vgpr4, $agpr8_agpr9_agpr10_agpr11, 0, 0, 0, implicit $mode, implicit $exec +# POSTRA-NEXT: SU(11): $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_4X4X1F32_e64 $vgpr5, $vgpr6, $agpr4_agpr5_agpr6_agpr7, 0, 0, 0, implicit $mode, implicit $exec + +--- +name: complex_cluster +tracksRegLiveness: true +body: | + bb.0: + liveins: $agpr0_agpr1_agpr2_agpr3, $agpr4_agpr5_agpr6_agpr7, $agpr8_agpr9_agpr10_agpr11, $agpr12_agpr13_agpr14_agpr15, $sgpr0, $vgpr10_vgpr11 + $vgpr1 = V_MOV_B32_e32 1, implicit $exec + $vgpr0 = V_MOV_B32_e32 1, implicit $exec + $vgpr8 = V_MOV_B32_e32 0, implicit $exec + $vgpr9 = V_MOV_B32_e32 9, implicit $exec + $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec + $vgpr2 = V_MOV_B32_e32 1, implicit $exec + $vgpr3 = V_MOV_B32_e32 1, implicit $exec + $vgpr4 = V_MOV_B32_e32 1, implicit $exec + $vgpr1 = V_ADD_F16_e32 $vgpr1, $vgpr0, implicit $mode, implicit $exec + GLOBAL_STORE_DWORD $vgpr10_vgpr11, $vgpr1, 0, 0, implicit $exec + $vgpr3 = DS_READ_U16_gfx9 $vgpr2, 0, 0, implicit $exec + $vgpr4 = GLOBAL_LOAD_USHORT $vgpr0_vgpr1, 0, 0, implicit $exec + $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_4X4X1F32_e64 $vgpr3, $vgpr4, $agpr4_agpr5_agpr6_agpr7, 0, 0, 0, implicit $mode, implicit $exec + $vgpr5 = V_XOR_B32_e32 $vgpr1, $vgpr0, implicit $exec + $vgpr5 = V_MOV_B32_e32 1, implicit $exec + $vgpr6 = V_MOV_B32_e32 1, implicit $exec + $agpr8_agpr9_agpr10_agpr11 = V_MFMA_F32_4X4X1F32_e64 $vgpr3, $vgpr4, $agpr8_agpr9_agpr10_agpr11, 0, 0, 0, implicit $mode, implicit $exec + $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_4X4X1F32_e64 $vgpr5, $vgpr6, $agpr4_agpr5_agpr6_agpr7, 0, 0, 0, implicit $mode, implicit $exec + $vgpr5 = V_XOR_B32_e32 $vgpr1, $vgpr0, implicit $exec + $vgpr6 = V_MUL_LO_U32_e64 $vgpr1, $sgpr0, implicit $exec + $agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec +... + +# DEFAULT: *** Final schedule for %bb.0 *** +# DEFAULT-NEXT: SU(0): $vgpr1 = V_MOV_B32_e32 1, implicit $exec +# DEFAULT-NEXT: SU(1): $vgpr0 = V_MOV_B32_e32 1, implicit $exec +# DEFAULT-NEXT: SU(4): $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec +# DEFAULT-NEXT: SU(8): $vgpr1 = V_ADD_F16_e32 $vgpr1, $vgpr0, implicit $mode, implicit $exec +# DEFAULT-NEXT: SU(9): GLOBAL_STORE_DWORD $vgpr10_vgpr11, $vgpr1, 0, 0, implicit $exec +# DEFAULT-NEXT: SU(5): $vgpr2 = V_MOV_B32_e32 1, implicit $exec +# DEFAULT-NEXT: SU(6): $vgpr3 = V_MOV_B32_e32 1, implicit $exec +# DEFAULT-NEXT: SU(10): $vgpr3 = DS_READ_U16_gfx9 $vgpr2, 0, 0, implicit $exec +# DEFAULT-NEXT: SU(20): $agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec +# DEFAULT-NEXT: SU(7): $vgpr4 = V_MOV_B32_e32 1, implicit $exec +# DEFAULT-NEXT: SU(11): $vgpr4 = GLOBAL_LOAD_USHORT $vgpr0_vgpr1, 0, 0, implicit $exec +# DEFAULT-NEXT: SU(12): $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_4X4X1F32_e64 $vgpr3, $vgpr4, $agpr4_agpr5_agpr6_agpr7, 0, 0, 0, implicit $mode, implicit $exec +# DEFAULT-NEXT: SU(13): $vgpr5 = V_XOR_B32_e32 $vgpr1, $vgpr0, implicit $exec +# DEFAULT-NEXT: SU(16): $agpr8_agpr9_agpr10_agpr11 = V_MFMA_F32_4X4X1F32_e64 $vgpr3, $vgpr4, $agpr8_agpr9_agpr10_agpr11, 0, 0, 0, implicit $mode, implicit $exec +# DEFAULT-NEXT: SU(14): $vgpr5 = V_MOV_B32_e32 1, implicit $exec +# DEFAULT-NEXT: SU(15): $vgpr6 = V_MOV_B32_e32 1, implicit $exec +# DEFAULT-NEXT: SU(17): $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_4X4X1F32_e64 $vgpr5, $vgpr6, $agpr4_agpr5_agpr6_agpr7, 0, 0, 0, implicit $mode, implicit $exec +# DEFAULT-NEXT: SU(18): $vgpr5 = V_XOR_B32_e32 $vgpr1, $vgpr0, implicit $exec +# DEFAULT-NEXT: SU(19): $vgpr6 = V_MUL_LO_U32_e64 $vgpr1, $sgpr0, implicit $exec +# DEFAULT-NEXT: SU(2): $vgpr8 = V_MOV_B32_e32 0, implicit $exec +# DEFAULT-NEXT: SU(3): $vgpr9 = V_MOV_B32_e32 9, implicit $exec + +# PRERA: Cluster MFMA SU(12) - SU(16) +# PRERA-NEXT: Cluster MFMA SU(12) - SU(20) +# PRERA: *** Final schedule for %bb.0 *** +# PRERA-NEXT: SU(0): $vgpr1 = V_MOV_B32_e32 1, implicit $exec +# PRERA-NEXT: SU(1): $vgpr0 = V_MOV_B32_e32 1, implicit $exec +# PRERA-NEXT: SU(4): $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec +# PRERA-NEXT: SU(8): $vgpr1 = V_ADD_F16_e32 $vgpr1, $vgpr0, implicit $mode, implicit $exec +# PRERA-NEXT: SU(9): GLOBAL_STORE_DWORD $vgpr10_vgpr11, $vgpr1, 0, 0, implicit $exec +# PRERA-NEXT: SU(5): $vgpr2 = V_MOV_B32_e32 1, implicit $exec +# PRERA-NEXT: SU(6): $vgpr3 = V_MOV_B32_e32 1, implicit $exec +# PRERA-NEXT: SU(10): $vgpr3 = DS_READ_U16_gfx9 $vgpr2, 0, 0, implicit $exec +# PRERA-NEXT: SU(7): $vgpr4 = V_MOV_B32_e32 1, implicit $exec +# PRERA-NEXT: SU(11): $vgpr4 = GLOBAL_LOAD_USHORT $vgpr0_vgpr1, 0, 0, implicit $exec +# PRERA-NEXT: SU(16): $agpr8_agpr9_agpr10_agpr11 = V_MFMA_F32_4X4X1F32_e64 $vgpr3, $vgpr4, $agpr8_agpr9_agpr10_agpr11, 0, 0, 0, implicit $mode, implicit $exec +# PRERA-NEXT: SU(12): $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_4X4X1F32_e64 $vgpr3, $vgpr4, $agpr4_agpr5_agpr6_agpr7, 0, 0, 0, implicit $mode, implicit $exec +# PRERA-NEXT: SU(13): $vgpr5 = V_XOR_B32_e32 $vgpr1, $vgpr0, implicit $exec +# PRERA-NEXT: SU(20): $agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec +# PRERA-NEXT: SU(14): $vgpr5 = V_MOV_B32_e32 1, implicit $exec +# PRERA-NEXT: SU(15): $vgpr6 = V_MOV_B32_e32 1, implicit $exec +# PRERA-NEXT: SU(17): $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_4X4X1F32_e64 $vgpr5, $vgpr6, $agpr4_agpr5_agpr6_agpr7, 0, 0, 0, implicit $mode, implicit $exec +# PRERA-NEXT: SU(18): $vgpr5 = V_XOR_B32_e32 $vgpr1, $vgpr0, implicit $exec +# PRERA-NEXT: SU(19): $vgpr6 = V_MUL_LO_U32_e64 $vgpr1, $sgpr0, implicit $exec +# PRERA-NEXT: SU(2): $vgpr8 = V_MOV_B32_e32 0, implicit $exec +# PRERA-NEXT: SU(3): $vgpr9 = V_MOV_B32_e32 9, implicit $exec + + +# POSTRA: Cluster MFMA SU(12) - SU(16) +# POSTRA-NEXT: Cluster MFMA SU(12) - SU(20) +# POSTRA: *** Final schedule for %bb.0 *** +# POSTRA-NEXT: SU(0): $vgpr1 = V_MOV_B32_e32 1, implicit $exec +# POSTRA-NEXT: SU(1): $vgpr0 = V_MOV_B32_e32 1, implicit $exec +# POSTRA-NEXT: SU(5): $vgpr2 = V_MOV_B32_e32 1, implicit $exec +# POSTRA-NEXT: SU(6): $vgpr3 = V_MOV_B32_e32 1, implicit $exec +# POSTRA-NEXT: SU(7): $vgpr4 = V_MOV_B32_e32 1, implicit $exec +# POSTRA-NEXT: SU(4): $agpr0_agpr1_agpr2_agpr3 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr0_agpr1_agpr2_agpr3, 0, 0, 0, implicit $mode, implicit $exec +# POSTRA-NEXT: SU(8): $vgpr1 = V_ADD_F16_e32 $vgpr1, $vgpr0, implicit $mode, implicit $exec +# POSTRA-NEXT: SU(9): GLOBAL_STORE_DWORD $vgpr10_vgpr11, $vgpr1, 0, 0, implicit $exec +# POSTRA-NEXT: SU(10): $vgpr3 = DS_READ_U16_gfx9 $vgpr2, 0, 0, implicit $exec +# POSTRA-NEXT: SU(11): $vgpr4 = GLOBAL_LOAD_USHORT $vgpr0_vgpr1, 0, 0, implicit $exec +# POSTRA-NEXT: SU(13): $vgpr5 = V_XOR_B32_e32 $vgpr1, $vgpr0, implicit $exec +# POSTRA-NEXT: SU(14): $vgpr5 = V_MOV_B32_e32 1, implicit $exec +# POSTRA-NEXT: SU(15): $vgpr6 = V_MOV_B32_e32 1, implicit $exec +# POSTRA-NEXT: SU(2): $vgpr8 = V_MOV_B32_e32 0, implicit $exec +# POSTRA-NEXT: SU(3): $vgpr9 = V_MOV_B32_e32 9, implicit $exec +# POSTRA-NEXT: SU(12): $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_4X4X1F32_e64 $vgpr3, $vgpr4, $agpr4_agpr5_agpr6_agpr7, 0, 0, 0, implicit $mode, implicit $exec +# POSTRA-NEXT: SU(20): $agpr12_agpr13_agpr14_agpr15 = V_MFMA_F32_4X4X1F32_e64 $vgpr1, $vgpr0, $agpr12_agpr13_agpr14_agpr15, 0, 0, 0, implicit $mode, implicit $exec +# POSTRA-NEXT: SU(16): $agpr8_agpr9_agpr10_agpr11 = V_MFMA_F32_4X4X1F32_e64 $vgpr3, $vgpr4, $agpr8_agpr9_agpr10_agpr11, 0, 0, 0, implicit $mode, implicit $exec +# POSTRA-NEXT: SU(17): $agpr4_agpr5_agpr6_agpr7 = V_MFMA_F32_4X4X1F32_e64 $vgpr5, $vgpr6, $agpr4_agpr5_agpr6_agpr7, 0, 0, 0, implicit $mode, implicit $exec +# POSTRA-NEXT: SU(18): $vgpr5 = V_XOR_B32_e32 $vgpr1, $vgpr0, implicit $exec +# POSTRA-NEXT: SU(19): $vgpr6 = V_MUL_LO_U32_e64 $vgpr1, $sgpr0, implicit $exec +