This is an archive of the discontinued LLVM Phabricator instance.

Differential D88739

[PowerPC] Add assemble disassemble intrinsics for MMA
ClosedPublic

Authored by saghir on Oct 2 2020, 7:52 AM.

Details

Reviewers
bsaleil
Group Reviewers
Restricted Project
Commits
rGf3202b30b8e6: [PowerPC] Add assemble disassemble intrinsics for MMA
Summary

This patch adds support for assemble disassemble intrinsics
for MMA.

Diff Detail

Event Timeline

saghir created this revision.Oct 2 2020, 7:52 AM
Herald added a project: Restricted Project. · View Herald TranscriptOct 2 2020, 7:52 AM
Herald added subscribers: llvm-commits, shchenz, kbarton and 2 others. · View Herald Transcript
saghir requested review of this revision.Oct 2 2020, 7:52 AM
Herald added a subscriber: jdoerfert. · View Herald TranscriptOct 2 2020, 7:52 AM
saghir added reviewers: bsaleil, Restricted Project.Oct 2 2020, 7:53 AM
Harbormaster completed remote builds in B73793: Diff 295828.Oct 2 2020, 8:16 AM
saghir updated this revision to Diff 295856.Oct 2 2020, 10:07 AM

Formatting change

Harbormaster completed remote builds in B73804: Diff 295856.Oct 2 2020, 10:28 AM
bsaleil accepted this revision.Oct 5 2020, 2:07 PM

LGTM

This revision is now accepted and ready to land.Oct 5 2020, 2:07 PM
This revision was landed with ongoing or failed builds.Oct 13 2020, 11:22 AM
Closed by commit rGf3202b30b8e6: [PowerPC] Add assemble disassemble intrinsics for MMA (authored by saghir). · Explain Why
This revision was automatically updated to reflect the committed changes.
saghir added a commit: rGf3202b30b8e6: [PowerPC] Add assemble disassemble intrinsics for MMA.

Revision Contents

PathSize
llvm/
include/
llvm/
IR/
1 line
29 lines
lib/
IR/
7 lines
Target/
PowerPC/
23 lines
17 lines
19 lines
test/
CodeGen/
PowerPC/
250 lines
utils/
TableGen/
4 lines

Diff 297919

llvm/include/llvm/IR/Intrinsics.td

Show First 20 Lines • Show All 254 Lines • ▼ Show 20 Lines
def llvm_v2i1_ty : LLVMType<v2i1>; // 2 x i1 def llvm_v2i1_ty : LLVMType<v2i1>; // 2 x i1
def llvm_v4i1_ty : LLVMType<v4i1>; // 4 x i1 def llvm_v4i1_ty : LLVMType<v4i1>; // 4 x i1
def llvm_v8i1_ty : LLVMType<v8i1>; // 8 x i1 def llvm_v8i1_ty : LLVMType<v8i1>; // 8 x i1
def llvm_v16i1_ty : LLVMType<v16i1>; // 16 x i1 def llvm_v16i1_ty : LLVMType<v16i1>; // 16 x i1
def llvm_v32i1_ty : LLVMType<v32i1>; // 32 x i1 def llvm_v32i1_ty : LLVMType<v32i1>; // 32 x i1
def llvm_v64i1_ty : LLVMType<v64i1>; // 64 x i1 def llvm_v64i1_ty : LLVMType<v64i1>; // 64 x i1
def llvm_v128i1_ty : LLVMType<v128i1>; // 128 x i1 def llvm_v128i1_ty : LLVMType<v128i1>; // 128 x i1
def llvm_v256i1_ty : LLVMType<v256i1>; // 256 x i1
def llvm_v512i1_ty : LLVMType<v512i1>; // 512 x i1 def llvm_v512i1_ty : LLVMType<v512i1>; // 512 x i1
def llvm_v1024i1_ty : LLVMType<v1024i1>; //1024 x i1 def llvm_v1024i1_ty : LLVMType<v1024i1>; //1024 x i1
def llvm_v1i8_ty : LLVMType<v1i8>; // 1 x i8 def llvm_v1i8_ty : LLVMType<v1i8>; // 1 x i8
def llvm_v2i8_ty : LLVMType<v2i8>; // 2 x i8 def llvm_v2i8_ty : LLVMType<v2i8>; // 2 x i8
def llvm_v4i8_ty : LLVMType<v4i8>; // 4 x i8 def llvm_v4i8_ty : LLVMType<v4i8>; // 4 x i8
def llvm_v8i8_ty : LLVMType<v8i8>; // 8 x i8 def llvm_v8i8_ty : LLVMType<v8i8>; // 8 x i8
def llvm_v16i8_ty : LLVMType<v16i8>; // 16 x i8 def llvm_v16i8_ty : LLVMType<v16i8>; // 16 x i8
▲ Show 20 Lines • Show All 1,321 Lines • Show Last 20 Lines

llvm/include/llvm/IR/IntrinsicsPowerPC.td

Show First 20 Lines • Show All 1,358 Lines • ▼ Show 20 Lines
def int_ppc_cfence : Intrinsic<[], [llvm_anyint_ty], []>; def int_ppc_cfence : Intrinsic<[], [llvm_anyint_ty], []>;
// PowerPC set FPSCR Intrinsic Definitions. // PowerPC set FPSCR Intrinsic Definitions.
def int_ppc_setrnd : GCCBuiltin<"__builtin_setrnd">, def int_ppc_setrnd : GCCBuiltin<"__builtin_setrnd">,
Intrinsic<[llvm_double_ty], [llvm_i32_ty], []>; Intrinsic<[llvm_double_ty], [llvm_i32_ty], []>;
} }
let TargetPrefix = "ppc" in {
def int_ppc_mma_assemble_acc :
Intrinsic<[llvm_v512i1_ty],
[llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty],
[IntrNoMem]>;
def int_ppc_mma_disassemble_acc :
Intrinsic<[llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty],
[llvm_v512i1_ty], [IntrNoMem]>;
def int_ppc_mma_assemble_pair :
Intrinsic<[llvm_v256i1_ty],
[llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>;
def int_ppc_mma_disassemble_pair :
Intrinsic<[llvm_v16i8_ty, llvm_v16i8_ty],
[llvm_v256i1_ty], [IntrNoMem]>;
def int_ppc_mma_xxmtacc :
Intrinsic<[llvm_v512i1_ty], [llvm_v512i1_ty], [IntrNoMem]>;
def int_ppc_mma_xxmfacc :
Intrinsic<[llvm_v512i1_ty], [llvm_v512i1_ty], [IntrNoMem]>;
def int_ppc_mma_xxsetaccz :
Intrinsic<[llvm_v512i1_ty], [], [IntrNoMem]>;
}

llvm/lib/IR/Function.cpp

Show First 20 Lines • Show All 828 Lines • ▼ Show 20 Linesenum IIT_Info {
IIT_F128 = 41, IIT_F128 = 41,
IIT_VEC_ELEMENT = 42, IIT_VEC_ELEMENT = 42,
IIT_SCALABLE_VEC = 43, IIT_SCALABLE_VEC = 43,
IIT_SUBDIVIDE2_ARG = 44, IIT_SUBDIVIDE2_ARG = 44,
IIT_SUBDIVIDE4_ARG = 45, IIT_SUBDIVIDE4_ARG = 45,
IIT_VEC_OF_BITCASTS_TO_INT = 46, IIT_VEC_OF_BITCASTS_TO_INT = 46,
IIT_V128 = 47, IIT_V128 = 47,
IIT_BF16 = 48, IIT_BF16 = 48,
IIT_STRUCT9 = 49 IIT_STRUCT9 = 49,
IIT_V256 = 50
}; };
static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos, static void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
IIT_Info LastInfo, IIT_Info LastInfo,
SmallVectorImpl<Intrinsic::IITDescriptor> &OutputTable) { SmallVectorImpl<Intrinsic::IITDescriptor> &OutputTable) {
using namespace Intrinsic; using namespace Intrinsic;
bool IsScalableVector = (LastInfo == IIT_SCALABLE_VEC); bool IsScalableVector = (LastInfo == IIT_SCALABLE_VEC);
▲ Show 20 Lines • Show All 77 Lines • ▼ Show 20 Linesstatic void DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
case IIT_V64: case IIT_V64:
OutputTable.push_back(IITDescriptor::getVector(64, IsScalableVector)); OutputTable.push_back(IITDescriptor::getVector(64, IsScalableVector));
DecodeIITType(NextElt, Infos, Info, OutputTable); DecodeIITType(NextElt, Infos, Info, OutputTable);
return; return;
case IIT_V128: case IIT_V128:
OutputTable.push_back(IITDescriptor::getVector(128, IsScalableVector)); OutputTable.push_back(IITDescriptor::getVector(128, IsScalableVector));
DecodeIITType(NextElt, Infos, Info, OutputTable); DecodeIITType(NextElt, Infos, Info, OutputTable);
return; return;
case IIT_V256:
OutputTable.push_back(IITDescriptor::getVector(256, IsScalableVector));
DecodeIITType(NextElt, Infos, Info, OutputTable);
return;
case IIT_V512: case IIT_V512:
OutputTable.push_back(IITDescriptor::getVector(512, IsScalableVector)); OutputTable.push_back(IITDescriptor::getVector(512, IsScalableVector));
DecodeIITType(NextElt, Infos, Info, OutputTable); DecodeIITType(NextElt, Infos, Info, OutputTable);
return; return;
case IIT_V1024: case IIT_V1024:
OutputTable.push_back(IITDescriptor::getVector(1024, IsScalableVector)); OutputTable.push_back(IITDescriptor::getVector(1024, IsScalableVector));
DecodeIITType(NextElt, Infos, Info, OutputTable); DecodeIITType(NextElt, Infos, Info, OutputTable);
return; return;
▲ Show 20 Lines • Show All 820 Lines • Show Last 20 Lines

llvm/lib/Target/PowerPC/PPCISelLowering.cpp

  • This file is larger than 256 KB, so syntax highlighting is disabled by default.
Show First 20 Lines • Show All 10,410 Lines • ▼ Show 20 Lines
/// lower, do it, otherwise return null. /// lower, do it, otherwise return null.
SDValue PPCTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SDValue PPCTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
SelectionDAG &DAG) const { SelectionDAG &DAG) const {
unsigned IntrinsicID = unsigned IntrinsicID =
cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue(); cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
SDLoc dl(Op); SDLoc dl(Op);
if (IntrinsicID == Intrinsic::thread_pointer) { switch (IntrinsicID) {
case Intrinsic::thread_pointer:
// Reads the thread pointer register, used for __builtin_thread_pointer. // Reads the thread pointer register, used for __builtin_thread_pointer.
if (Subtarget.isPPC64()) if (Subtarget.isPPC64())
return DAG.getRegister(PPC::X13, MVT::i64); return DAG.getRegister(PPC::X13, MVT::i64);
return DAG.getRegister(PPC::R2, MVT::i32); return DAG.getRegister(PPC::R2, MVT::i32);
case Intrinsic::ppc_mma_disassemble_acc:
case Intrinsic::ppc_mma_disassemble_pair: {
int NumVecs = 2;
SDValue WideVec = Op.getOperand(1);
if (IntrinsicID == Intrinsic::ppc_mma_disassemble_acc) {
NumVecs = 4;
WideVec = DAG.getNode(PPCISD::XXMFACC, dl, MVT::v512i1, WideVec);
}
SmallVector<SDValue, 4> RetOps;
for (int VecNo = 0; VecNo < NumVecs; VecNo++) {
SDValue Extract = DAG.getNode(
PPCISD::EXTRACT_VSX_REG, dl, MVT::v16i8, WideVec,
DAG.getConstant(Subtarget.isLittleEndian() ? NumVecs - 1 - VecNo
: VecNo,
dl, MVT::i64));
RetOps.push_back(Extract);
}
return DAG.getMergeValues(RetOps, dl);
}
} }
// If this is a lowered altivec predicate compare, CompareOpc is set to the // If this is a lowered altivec predicate compare, CompareOpc is set to the
// opcode number of the comparison. // opcode number of the comparison.
int CompareOpc; int CompareOpc;
bool isDot; bool isDot;
if (!getVectorCompareInfo(Op, CompareOpc, isDot, Subtarget)) if (!getVectorCompareInfo(Op, CompareOpc, isDot, Subtarget))
return SDValue(); // Don't custom lower most intrinsics. return SDValue(); // Don't custom lower most intrinsics.
▲ Show 20 Lines • Show All 6,468 Lines • Show Last 20 Lines

llvm/lib/Target/PowerPC/PPCInstrInfo.cpp

Show First 20 Lines • Show All 666 Lines • ▼ Show 20 Linesbool PPCInstrInfo::isReallyTriviallyReMaterializable(const MachineInstr &MI,
case PPC::V_SET0B: case PPC::V_SET0B:
case PPC::V_SET0H: case PPC::V_SET0H:
case PPC::V_SET0: case PPC::V_SET0:
case PPC::V_SETALLONESB: case PPC::V_SETALLONESB:
case PPC::V_SETALLONESH: case PPC::V_SETALLONESH:
case PPC::V_SETALLONES: case PPC::V_SETALLONES:
case PPC::CRSET: case PPC::CRSET:
case PPC::CRUNSET: case PPC::CRUNSET:
case PPC::XXSETACCZ:
return true; return true;
} }
return false; return false;
} }
unsigned PPCInstrInfo::isStoreToStackSlot(const MachineInstr &MI, unsigned PPCInstrInfo::isStoreToStackSlot(const MachineInstr &MI,
int &FrameIndex) const { int &FrameIndex) const {
unsigned Opcode = MI.getOpcode(); unsigned Opcode = MI.getOpcode();
▲ Show 20 Lines • Show All 652 Lines • ▼ Show 20 Lines else if (PPC::VSRCRegClass.contains(DestReg, SrcReg))
// can go to either pipeline. // can go to either pipeline.
// We'll always use xxlor here, because in practically all cases where // We'll always use xxlor here, because in practically all cases where
// copies are generated, they are close enough to some use that the // copies are generated, they are close enough to some use that the
// lower-latency form is preferable. // lower-latency form is preferable.
Opc = PPC::XXLOR; Opc = PPC::XXLOR;
else if (PPC::VSFRCRegClass.contains(DestReg, SrcReg) || else if (PPC::VSFRCRegClass.contains(DestReg, SrcReg) ||
PPC::VSSRCRegClass.contains(DestReg, SrcReg)) PPC::VSSRCRegClass.contains(DestReg, SrcReg))
Opc = (Subtarget.hasP9Vector()) ? PPC::XSCPSGNDP : PPC::XXLORf; Opc = (Subtarget.hasP9Vector()) ? PPC::XSCPSGNDP : PPC::XXLORf;
else if (Subtarget.pairedVectorMemops() &&
PPC::VSRpRCRegClass.contains(DestReg, SrcReg)) {
if (SrcReg > PPC::VSRp15)
SrcReg = PPC::V0 + (SrcReg - PPC::VSRp16) * 2;
else
SrcReg = PPC::VSL0 + (SrcReg - PPC::VSRp0) * 2;
if (DestReg > PPC::VSRp15)
DestReg = PPC::V0 + (DestReg - PPC::VSRp16) * 2;
else
DestReg = PPC::VSL0 + (DestReg - PPC::VSRp0) * 2;
BuildMI(MBB, I, DL, get(PPC::XXLOR), DestReg).
addReg(SrcReg).addReg(SrcReg, getKillRegState(KillSrc));
BuildMI(MBB, I, DL, get(PPC::XXLOR), DestReg + 1).
addReg(SrcReg + 1).addReg(SrcReg + 1, getKillRegState(KillSrc));
return;
}
else if (PPC::CRBITRCRegClass.contains(DestReg, SrcReg)) else if (PPC::CRBITRCRegClass.contains(DestReg, SrcReg))
Opc = PPC::CROR; Opc = PPC::CROR;
else if (PPC::SPERCRegClass.contains(DestReg, SrcReg)) else if (PPC::SPERCRegClass.contains(DestReg, SrcReg))
Opc = PPC::EVOR; Opc = PPC::EVOR;
else else
llvm_unreachable("Impossible reg-to-reg copy"); llvm_unreachable("Impossible reg-to-reg copy");
const MCInstrDesc &MCID = get(Opc); const MCInstrDesc &MCID = get(Opc);
▲ Show 20 Lines • Show All 3,495 Lines • Show Last 20 Lines

llvm/lib/Target/PowerPC/PPCInstrPrefix.td

Show First 20 Lines • Show All 1,258 Lines • ▼ Show 20 Lines
} }
// End of class definitions. // End of class definitions.
//----------------------------------------------------------------------------- //-----------------------------------------------------------------------------
let Predicates = [MMA] in { let Predicates = [MMA] in {
def XXMFACC : def XXMFACC :
XForm_AT3<31, 0, 177, (outs acc:$ASo), (ins acc:$AS), "xxmfacc $AS", XForm_AT3<31, 0, 177, (outs acc:$ASo), (ins acc:$AS), "xxmfacc $AS",
IIC_VecGeneral, []>, RegConstraint<"$ASo = $AS">, IIC_VecGeneral,
NoEncode<"$ASo">; [(set v512i1:$ASo, (int_ppc_mma_xxmfacc v512i1:$AS))]>,
RegConstraint<"$ASo = $AS">, NoEncode<"$ASo">;
def XXMTACC : def XXMTACC :
XForm_AT3<31, 1, 177, (outs acc:$AT), (ins acc:$ATi), "xxmtacc $AT", XForm_AT3<31, 1, 177, (outs acc:$AT), (ins acc:$ATi), "xxmtacc $AT",
IIC_VecGeneral, []>, RegConstraint<"$ATi = $AT">, IIC_VecGeneral,
NoEncode<"$ATi">; [(set v512i1:$AT, (int_ppc_mma_xxmtacc v512i1:$ATi))]>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
def KILL_PAIR : PPCPostRAExpPseudo<(outs vsrprc:$XTp), (ins vsrprc:$XSp), def KILL_PAIR : PPCPostRAExpPseudo<(outs vsrprc:$XTp), (ins vsrprc:$XSp),
"#KILL_PAIR", []>, "#KILL_PAIR", []>,
RegConstraint<"$XTp = $XSp">; RegConstraint<"$XTp = $XSp">;
def BUILD_UACC : PPCPostRAExpPseudo<(outs acc:$AT), (ins uacc:$AS), def BUILD_UACC : PPCPostRAExpPseudo<(outs acc:$AT), (ins uacc:$AS),
"#BUILD_UACC $AT, $AS", []>; "#BUILD_UACC $AT, $AS", []>;
// We define XXSETACCZ as rematerializable to undo CSE of that intrinsic in // We define XXSETACCZ as rematerializable to undo CSE of that intrinsic in
// the backend. We avoid CSE here because it generates a copy of the acc // the backend. We avoid CSE here because it generates a copy of the acc
// register and this copy is more expensive than calling the intrinsic again. // register and this copy is more expensive than calling the intrinsic again.
let isAsCheapAsAMove = 1, isReMaterializable = 1 in { let isAsCheapAsAMove = 1, isReMaterializable = 1 in {
def XXSETACCZ : def XXSETACCZ :
XForm_AT3<31, 3, 177, (outs acc:$AT), (ins), "xxsetaccz $AT", XForm_AT3<31, 3, 177, (outs acc:$AT), (ins), "xxsetaccz $AT", IIC_VecGeneral,
IIC_VecGeneral, []>; [(set v512i1:$AT, (int_ppc_mma_xxsetaccz))]>;
} }
def XVI8GER4SPP : def XVI8GER4SPP :
XX3Form_AT3_XAB6<59, 99, (outs acc:$AT), (ins acc:$ATi, vsrc:$XA, vsrc:$XB), XX3Form_AT3_XAB6<59, 99, (outs acc:$AT), (ins acc:$ATi, vsrc:$XA, vsrc:$XB),
"xvi8ger4spp $AT, $XA, $XB", IIC_VecGeneral, []>, "xvi8ger4spp $AT, $XA, $XB", IIC_VecGeneral, []>,
RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">; RegConstraint<"$ATi = $AT">, NoEncode<"$ATi">;
} }
let Predicates = [MMA, PrefixInstrs] in { let Predicates = [MMA, PrefixInstrs] in {
▲ Show 20 Lines • Show All 71 Lines • ▼ Show 20 Linesdef Extracts {
dag Vec3 = (v4i32 (EXTRACT_SUBREG Pair1, sub_vsx1)); dag Vec3 = (v4i32 (EXTRACT_SUBREG Pair1, sub_vsx1));
} }
let Predicates = [MMA] in { let Predicates = [MMA] in {
def : Pat<(v512i1 (PPCAccBuild v4i32:$vs1, v4i32:$vs0, v4i32:$vs3, v4i32:$vs2)), def : Pat<(v512i1 (PPCAccBuild v4i32:$vs1, v4i32:$vs0, v4i32:$vs3, v4i32:$vs2)),
(XXMTACC Concats.VecsToVecQuad)>; (XXMTACC Concats.VecsToVecQuad)>;
def : Pat<(v256i1 (PPCPairBuild v4i32:$vs1, v4i32:$vs0)), def : Pat<(v256i1 (PPCPairBuild v4i32:$vs1, v4i32:$vs0)),
Concats.VecsToVecPair0>; Concats.VecsToVecPair0>;
def : Pat<(v512i1 (int_ppc_mma_assemble_acc v16i8:$vs1, v16i8:$vs0,
v16i8:$vs3, v16i8:$vs2)),
(XXMTACC Concats.VecsToVecQuad)>;
def : Pat<(v256i1 (int_ppc_mma_assemble_pair v16i8:$vs1, v16i8:$vs0)),
Concats.VecsToVecPair0>;
def : Pat<(v512i1 (PPCxxmfacc v512i1:$AS)), (XXMFACC acc:$AS)>; def : Pat<(v512i1 (PPCxxmfacc v512i1:$AS)), (XXMFACC acc:$AS)>;
def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, (i64 0))), def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, (i64 0))),
Extracts.Vec0>; Extracts.Vec0>;
def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, (i64 1))), def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, (i64 1))),
Extracts.Vec1>; Extracts.Vec1>;
def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, (i64 2))), def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, (i64 2))),
Extracts.Vec2>; Extracts.Vec2>;
def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, (i64 3))), def : Pat<(v4i32 (PPCAccExtractVsx acc:$v, (i64 3))),
▲ Show 20 Lines • Show All 846 Lines • Show Last 20 Lines

llvm/test/CodeGen/PowerPC/mma-intrinsics.ll

  • This file was added.
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -verify-machineinstrs -mtriple=powerpc64le-unknown-linux-gnu \
; RUN: -mcpu=pwr10 -ppc-asm-full-reg-names \
; RUN: -ppc-vsr-nums-as-vr < %s | FileCheck %s
; RUN: llc -verify-machineinstrs -mtriple=powerpc64-unknown-linux-gnu \
; RUN: -mcpu=pwr10 -ppc-asm-full-reg-names \
; RUN: -ppc-vsr-nums-as-vr < %s | FileCheck %s --check-prefix=CHECK-BE
; assemble_acc
declare <512 x i1> @llvm.ppc.mma.assemble.acc(<16 x i8>, <16 x i8>, <16 x i8>, <16 x i8>)
define void @ass_acc(<512 x i1>* %ptr, <16 x i8> %vc) {
; CHECK-LABEL: ass_acc:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: vmr v3, v2
; CHECK-NEXT: xxlor vs0, v2, v2
; CHECK-NEXT: xxlor vs1, v3, v3
; CHECK-NEXT: xxlor vs2, v2, v2
; CHECK-NEXT: xxlor vs3, v3, v3
; CHECK-NEXT: xxmtacc acc0
; CHECK-NEXT: xxmfacc acc0
; CHECK-NEXT: stxv vs0, 48(r3)
; CHECK-NEXT: stxv vs1, 32(r3)
; CHECK-NEXT: stxv vs2, 16(r3)
; CHECK-NEXT: stxv vs3, 0(r3)
; CHECK-NEXT: blr
;
; CHECK-BE-LABEL: ass_acc:
; CHECK-BE: # %bb.0: # %entry
; CHECK-BE-NEXT: vmr v3, v2
; CHECK-BE-NEXT: xxlor vs0, v2, v2
; CHECK-BE-NEXT: xxlor vs1, v3, v3
; CHECK-BE-NEXT: xxlor vs2, v2, v2
; CHECK-BE-NEXT: xxlor vs3, v3, v3
; CHECK-BE-NEXT: xxmtacc acc0
; CHECK-BE-NEXT: xxmfacc acc0
; CHECK-BE-NEXT: stxv vs1, 16(r3)
; CHECK-BE-NEXT: stxv vs0, 0(r3)
; CHECK-BE-NEXT: stxv vs3, 48(r3)
; CHECK-BE-NEXT: stxv vs2, 32(r3)
; CHECK-BE-NEXT: blr
; CHECK-O0-LABEL: ass_acc:
; CHECK-O0: # %bb.0: # %entry
; CHECK-BE-O0-LABEL: ass_acc:
; CHECK-BE-O0: # %bb.0: # %entry
entry:
%0 = tail call <512 x i1> @llvm.ppc.mma.assemble.acc(<16 x i8> %vc, <16 x i8> %vc, <16 x i8> %vc, <16 x i8> %vc)
store <512 x i1> %0, <512 x i1>* %ptr, align 64
ret void
}
; assemble_pair
declare <256 x i1> @llvm.ppc.mma.assemble.pair(<16 x i8>, <16 x i8>)
define void @ass_pair(<256 x i1>* %ptr, <16 x i8> %vc) {
; CHECK-LABEL: ass_pair:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: vmr v3, v2
; CHECK-NEXT: stxv v2, 16(r3)
; CHECK-NEXT: stxv v3, 0(r3)
; CHECK-NEXT: blr
;
; CHECK-BE-LABEL: ass_pair:
; CHECK-BE: # %bb.0: # %entry
; CHECK-BE-NEXT: vmr v3, v2
; CHECK-BE-NEXT: stxv v2, 16(r3)
; CHECK-BE-NEXT: stxv v2, 0(r3)
; CHECK-BE-NEXT: blr
entry:
%0 = tail call <256 x i1> @llvm.ppc.mma.assemble.pair(<16 x i8> %vc, <16 x i8> %vc)
store <256 x i1> %0, <256 x i1>* %ptr, align 32
ret void
}
; xxmtacc
declare <512 x i1> @llvm.ppc.mma.xxmtacc(<512 x i1>)
define void @int_xxmtacc(<512 x i1>* %ptr, <16 x i8> %vc) {
; CHECK-LABEL: int_xxmtacc:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: vmr v3, v2
; CHECK-NEXT: xxlor vs0, v2, v2
; CHECK-NEXT: xxlor vs1, v3, v3
; CHECK-NEXT: xxlor vs2, v2, v2
; CHECK-NEXT: xxlor vs3, v3, v3
; CHECK-NEXT: xxmtacc acc0
; CHECK-NEXT: xxmtacc acc0
; CHECK-NEXT: xxmfacc acc0
; CHECK-NEXT: stxv vs0, 48(r3)
; CHECK-NEXT: stxv vs1, 32(r3)
; CHECK-NEXT: stxv vs2, 16(r3)
; CHECK-NEXT: stxv vs3, 0(r3)
; CHECK-NEXT: blr
;
; CHECK-BE-LABEL: int_xxmtacc:
; CHECK-BE: # %bb.0: # %entry
; CHECK-BE-NEXT: vmr v3, v2
; CHECK-BE-NEXT: xxlor vs0, v2, v2
; CHECK-BE-NEXT: xxlor vs1, v3, v3
; CHECK-BE-NEXT: xxlor vs2, v2, v2
; CHECK-BE-NEXT: xxlor vs3, v3, v3
; CHECK-BE-NEXT: xxmtacc acc0
; CHECK-BE-NEXT: xxmtacc acc0
; CHECK-BE-NEXT: xxmfacc acc0
; CHECK-BE-NEXT: stxv vs1, 16(r3)
; CHECK-BE-NEXT: stxv vs0, 0(r3)
; CHECK-BE-NEXT: stxv vs3, 48(r3)
; CHECK-BE-NEXT: stxv vs2, 32(r3)
; CHECK-BE-NEXT: blr
entry:
; One xxmtacc is generated from the call to assemble.acc then one xxmtacc is
; generated from the call to xxmtacc then one xxmfacc is generated for the store
%0 = tail call <512 x i1> @llvm.ppc.mma.assemble.acc(<16 x i8> %vc, <16 x i8> %vc, <16 x i8> %vc, <16 x i8> %vc)
%1 = tail call <512 x i1> @llvm.ppc.mma.xxmtacc(<512 x i1> %0)
store <512 x i1> %1, <512 x i1>* %ptr, align 64
ret void
}
; xxmfacc
declare <512 x i1> @llvm.ppc.mma.xxmfacc(<512 x i1>)
define void @int_xxmfacc(<512 x i1>* %ptr, <16 x i8> %vc) {
; CHECK-LABEL: int_xxmfacc:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: vmr v3, v2
; CHECK-NEXT: xxlor vs0, v2, v2
; CHECK-NEXT: xxlor vs1, v3, v3
; CHECK-NEXT: xxlor vs2, v2, v2
; CHECK-NEXT: xxlor vs3, v3, v3
; CHECK-NEXT: xxmtacc acc0
; CHECK-NEXT: xxmfacc acc0
; CHECK-NEXT: xxmfacc acc0
; CHECK-NEXT: stxv vs0, 48(r3)
; CHECK-NEXT: stxv vs1, 32(r3)
; CHECK-NEXT: stxv vs2, 16(r3)
; CHECK-NEXT: stxv vs3, 0(r3)
; CHECK-NEXT: blr
;
; CHECK-BE-LABEL: int_xxmfacc:
; CHECK-BE: # %bb.0: # %entry
; CHECK-BE-NEXT: vmr v3, v2
; CHECK-BE-NEXT: xxlor vs0, v2, v2
; CHECK-BE-NEXT: xxlor vs1, v3, v3
; CHECK-BE-NEXT: xxlor vs2, v2, v2
; CHECK-BE-NEXT: xxlor vs3, v3, v3
; CHECK-BE-NEXT: xxmtacc acc0
; CHECK-BE-NEXT: xxmfacc acc0
; CHECK-BE-NEXT: xxmfacc acc0
; CHECK-BE-NEXT: stxv vs1, 16(r3)
; CHECK-BE-NEXT: stxv vs0, 0(r3)
; CHECK-BE-NEXT: stxv vs3, 48(r3)
; CHECK-BE-NEXT: stxv vs2, 32(r3)
; CHECK-BE-NEXT: blr
entry:
; One xxmtacc is generated from the call to assemble.acc then one xxmfacc is
; generated from the call to xxmfacc then one xxmfacc is generated for the store
%0 = tail call <512 x i1> @llvm.ppc.mma.assemble.acc(<16 x i8> %vc, <16 x i8> %vc, <16 x i8> %vc, <16 x i8> %vc)
%1 = tail call <512 x i1> @llvm.ppc.mma.xxmfacc(<512 x i1> %0)
store <512 x i1> %1, <512 x i1>* %ptr, align 64
ret void
}
; xxsetaccz
declare <512 x i1> @llvm.ppc.mma.xxsetaccz()
define void @int_xxsetaccz(<512 x i1>* %ptr) {
; CHECK-LABEL: int_xxsetaccz:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xxsetaccz acc0
; CHECK-NEXT: xxmfacc acc0
; CHECK-NEXT: stxv vs0, 48(r3)
; CHECK-NEXT: stxv vs1, 32(r3)
; CHECK-NEXT: stxv vs2, 16(r3)
; CHECK-NEXT: stxv vs3, 0(r3)
; CHECK-NEXT: blr
;
; CHECK-BE-LABEL: int_xxsetaccz:
; CHECK-BE: # %bb.0: # %entry
; CHECK-BE-NEXT: xxsetaccz acc0
; CHECK-BE-NEXT: xxmfacc acc0
; CHECK-BE-NEXT: stxv vs1, 16(r3)
; CHECK-BE-NEXT: stxv vs0, 0(r3)
; CHECK-BE-NEXT: stxv vs3, 48(r3)
; CHECK-BE-NEXT: stxv vs2, 32(r3)
; CHECK-BE-NEXT: blr
entry:
%0 = tail call <512 x i1> @llvm.ppc.mma.xxsetaccz()
store <512 x i1> %0, <512 x i1>* %ptr, align 64
ret void
}
; disassemble_acc
declare { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.ppc.mma.disassemble.acc(<512 x i1>)
define void @disass_acc(<16 x i8>* %ptr1, <16 x i8>* %ptr2, <16 x i8>* %ptr3, <16 x i8>* %ptr4) {
; CHECK-LABEL: disass_acc:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xxsetaccz acc0
; CHECK-NEXT: xxmfacc acc0
; CHECK-NEXT: stxv vs3, 0(r3)
; CHECK-NEXT: stxv vs2, 0(r4)
; CHECK-NEXT: stxv vs1, 0(r5)
; CHECK-NEXT: stxv vs0, 0(r6)
; CHECK-NEXT: blr
;
; CHECK-BE-LABEL: disass_acc:
; CHECK-BE: # %bb.0: # %entry
; CHECK-BE-NEXT: xxsetaccz acc0
; CHECK-BE-NEXT: xxmfacc acc0
; CHECK-BE-NEXT: stxv vs0, 0(r3)
; CHECK-BE-NEXT: stxv vs1, 0(r4)
; CHECK-BE-NEXT: stxv vs2, 0(r5)
; CHECK-BE-NEXT: stxv vs3, 0(r6)
; CHECK-BE-NEXT: blr
entry:
%0 = tail call <512 x i1> @llvm.ppc.mma.xxsetaccz()
%1 = tail call { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } @llvm.ppc.mma.disassemble.acc(<512 x i1> %0)
%2 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %1, 0
%3 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %1, 1
%4 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %1, 2
%5 = extractvalue { <16 x i8>, <16 x i8>, <16 x i8>, <16 x i8> } %1, 3
store <16 x i8> %2, <16 x i8>* %ptr1, align 16
store <16 x i8> %3, <16 x i8>* %ptr2, align 16
store <16 x i8> %4, <16 x i8>* %ptr3, align 16
store <16 x i8> %5, <16 x i8>* %ptr4, align 16
ret void
}
; disassemble_pair
declare { <16 x i8>, <16 x i8> } @llvm.ppc.mma.disassemble.pair(<256 x i1>)
define void @disass_pair(<256 x i1>* %ptr1, <16 x i8>* %ptr2, <16 x i8>* %ptr3) {
; CHECK-LABEL: disass_pair:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: lxv vs1, 0(r3)
; CHECK-NEXT: lxv vs0, 16(r3)
; CHECK-NEXT: stxv vs1, 0(r4)
; CHECK-NEXT: stxv vs0, 0(r5)
; CHECK-NEXT: blr
;
; CHECK-BE-LABEL: disass_pair:
; CHECK-BE: # %bb.0: # %entry
; CHECK-BE-NEXT: lxv vs1, 16(r3)
; CHECK-BE-NEXT: lxv vs0, 0(r3)
; CHECK-BE-NEXT: stxv vs0, 0(r4)
; CHECK-BE-NEXT: stxv vs1, 0(r5)
; CHECK-BE-NEXT: blr
entry:
%0 = load <256 x i1>, <256 x i1>* %ptr1, align 32
%1 = tail call { <16 x i8>, <16 x i8> } @llvm.ppc.mma.disassemble.pair(<256 x i1> %0)
%2 = extractvalue { <16 x i8>, <16 x i8> } %1, 0
%3 = extractvalue { <16 x i8>, <16 x i8> } %1, 1
store <16 x i8> %2, <16 x i8>* %ptr2, align 16
store <16 x i8> %3, <16 x i8>* %ptr3, align 16
ret void
}

llvm/utils/TableGen/IntrinsicEmitter.cpp

Show First 20 Lines • Show All 241 Lines • ▼ Show 20 Linesenum IIT_Info {
IIT_F128 = 41, IIT_F128 = 41,
IIT_VEC_ELEMENT = 42, IIT_VEC_ELEMENT = 42,
IIT_SCALABLE_VEC = 43, IIT_SCALABLE_VEC = 43,
IIT_SUBDIVIDE2_ARG = 44, IIT_SUBDIVIDE2_ARG = 44,
IIT_SUBDIVIDE4_ARG = 45, IIT_SUBDIVIDE4_ARG = 45,
IIT_VEC_OF_BITCASTS_TO_INT = 46, IIT_VEC_OF_BITCASTS_TO_INT = 46,
IIT_V128 = 47, IIT_V128 = 47,
IIT_BF16 = 48, IIT_BF16 = 48,
IIT_STRUCT9 = 49 IIT_STRUCT9 = 49,
IIT_V256 = 50
}; };
static void EncodeFixedValueType(MVT::SimpleValueType VT, static void EncodeFixedValueType(MVT::SimpleValueType VT,
std::vector<unsigned char> &Sig) { std::vector<unsigned char> &Sig) {
if (MVT(VT).isInteger()) { if (MVT(VT).isInteger()) {
unsigned BitWidth = MVT(VT).getSizeInBits(); unsigned BitWidth = MVT(VT).getSizeInBits();
switch (BitWidth) { switch (BitWidth) {
default: PrintFatalError("unhandled integer type width in intrinsic!"); default: PrintFatalError("unhandled integer type width in intrinsic!");
▲ Show 20 Lines • Show All 121 Lines • ▼ Show 20 Lines if (MVT(VT).isVector()) {
case 1: Sig.push_back(IIT_V1); break; case 1: Sig.push_back(IIT_V1); break;
case 2: Sig.push_back(IIT_V2); break; case 2: Sig.push_back(IIT_V2); break;
case 4: Sig.push_back(IIT_V4); break; case 4: Sig.push_back(IIT_V4); break;
case 8: Sig.push_back(IIT_V8); break; case 8: Sig.push_back(IIT_V8); break;
case 16: Sig.push_back(IIT_V16); break; case 16: Sig.push_back(IIT_V16); break;
case 32: Sig.push_back(IIT_V32); break; case 32: Sig.push_back(IIT_V32); break;
case 64: Sig.push_back(IIT_V64); break; case 64: Sig.push_back(IIT_V64); break;
case 128: Sig.push_back(IIT_V128); break; case 128: Sig.push_back(IIT_V128); break;
case 256: Sig.push_back(IIT_V256); break;
case 512: Sig.push_back(IIT_V512); break; case 512: Sig.push_back(IIT_V512); break;
case 1024: Sig.push_back(IIT_V1024); break; case 1024: Sig.push_back(IIT_V1024); break;
} }
return EncodeFixedValueType(VVT.getVectorElementType().SimpleTy, Sig); return EncodeFixedValueType(VVT.getVectorElementType().SimpleTy, Sig);
} }
EncodeFixedValueType(VT, Sig); EncodeFixedValueType(VT, Sig);
▲ Show 20 Lines • Show All 616 Lines • Show Last 20 Lines