This is an archive of the discontinued LLVM Phabricator instance.

Differential D30513

[NVPTX] Reduce amount of boilerplate code used to select load instruction opcode.
ClosedPublic

Authored by tra on Mar 1 2017, 3:04 PM.

Details

Reviewers
jlebar
Commits
rGee7dd12ff415: [NVPTX] Reduce amount of boilerplate code used to select load instruction…
rL296785: [NVPTX] Reduce amount of boilerplate code used to select load instruction…
Summary

Make opcode selection code for the load instruction a bit easier
to read and maintain.

This patch also catches number of f16 load variants that were not
handled before.

Diff Detail

Repository
rL LLVM

Event Timeline

tra created this revision.Mar 1 2017, 3:04 PM
Herald added a subscriber: jholewinski. · View Herald TranscriptMar 1 2017, 3:04 PM
tra added a parent revision: D30512: [NVPTX] Added LDU/LDG intrinsics for f16..Mar 1 2017, 3:05 PM
jlebar edited edge metadata.Mar 1 2017, 3:06 PM

As I've said before, red *is* my favorite color. :)

This patch also catches number of f16 load variants that were not handled before.

Do we want to add tests so we don't regress this?

lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp
795 ↗(On Diff #90239)

Do we want to use optional<unsigned> here instead so you can't forget to check for the default case?

tra updated this revision to Diff 90256.Mar 1 2017, 4:30 PM

pickOpcodeForVT now returns Optional<unsigned>.

It's also used for all loads/stores now.

tra marked an inline comment as done.Mar 1 2017, 4:33 PM
In D30513#690102, @jlebar wrote:

This patch also catches number of f16 load variants that were not handled before.

Do we want to add tests so we don't regress this?

Some of these instructions need additional help during lowering, so there's no easy way to emit them now.
I'll send another patch which will fix f16 load/store lowering and will include better tests.

jlebar accepted this revision.Mar 1 2017, 4:34 PM

OK, sgtm.

This revision is now accepted and ready to land.Mar 1 2017, 4:34 PM
Closed by commit rL296785: [NVPTX] Reduce amount of boilerplate code used to select load instruction… (authored by tra). · Explain WhyMar 2 2017, 11:26 AM
This revision was automatically updated to reflect the committed changes.

Revision Contents

PathSize
llvm/
trunk/
lib/
Target/
NVPTX/
2368 lines

Diff 90363

llvm/trunk/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp

Show First 20 Lines • Show All 784 Lines • ▼ Show 20 Lines case ADDRESS_SPACE_PARAM:
break; break;
} }
ReplaceNode(N, CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0), ReplaceNode(N, CurDAG->getMachineNode(Opc, SDLoc(N), N->getValueType(0),
Src)); Src));
return; return;
} }
} }
// Helper function template to reduce amount of boilerplate code for
// opcode selection.
static Optional<unsigned> pickOpcodeForVT(
MVT::SimpleValueType VT, unsigned Opcode_i8, unsigned Opcode_i16,
unsigned Opcode_i32, Optional<unsigned> Opcode_i64, unsigned Opcode_f16,
unsigned Opcode_f16x2, unsigned Opcode_f32, Optional<unsigned> Opcode_f64) {
switch (VT) {
case MVT::i1:
case MVT::i8:
return Opcode_i8;
case MVT::i16:
return Opcode_i16;
case MVT::i32:
return Opcode_i32;
case MVT::i64:
return Opcode_i64;
case MVT::f16:
return Opcode_f16;
case MVT::v2f16:
return Opcode_f16x2;
case MVT::f32:
return Opcode_f32;
case MVT::f64:
return Opcode_f64;
default:
return None;
}
}
bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) { bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
SDLoc dl(N); SDLoc dl(N);
LoadSDNode *LD = cast<LoadSDNode>(N); LoadSDNode *LD = cast<LoadSDNode>(N);
EVT LoadedVT = LD->getMemoryVT(); EVT LoadedVT = LD->getMemoryVT();
SDNode *NVPTXLD = nullptr; SDNode *NVPTXLD = nullptr;
// do not support pre/post inc/dec // do not support pre/post inc/dec
if (LD->isIndexed()) if (LD->isIndexed())
▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Linesbool NVPTXDAGToDAGISel::tryLoad(SDNode *N) {
else else
fromType = NVPTX::PTXLdStInstCode::Unsigned; fromType = NVPTX::PTXLdStInstCode::Unsigned;
// Create the machine instruction DAG // Create the machine instruction DAG
SDValue Chain = N->getOperand(0); SDValue Chain = N->getOperand(0);
SDValue N1 = N->getOperand(1); SDValue N1 = N->getOperand(1);
SDValue Addr; SDValue Addr;
SDValue Offset, Base; SDValue Offset, Base;
unsigned Opcode; Optional<unsigned> Opcode;
MVT::SimpleValueType TargetVT = LD->getSimpleValueType(0).SimpleTy; MVT::SimpleValueType TargetVT = LD->getSimpleValueType(0).SimpleTy;
if (SelectDirectAddr(N1, Addr)) { if (SelectDirectAddr(N1, Addr)) {
switch (TargetVT) { Opcode = pickOpcodeForVT(
case MVT::i8: TargetVT, NVPTX::LD_i8_avar, NVPTX::LD_i16_avar, NVPTX::LD_i32_avar,
Opcode = NVPTX::LD_i8_avar; NVPTX::LD_i64_avar, NVPTX::LD_f16_avar, NVPTX::LD_f16x2_avar,
break; NVPTX::LD_f32_avar, NVPTX::LD_f64_avar);
case MVT::i16: if (!Opcode)
Opcode = NVPTX::LD_i16_avar;
break;
case MVT::i32:
Opcode = NVPTX::LD_i32_avar;
break;
case MVT::i64:
Opcode = NVPTX::LD_i64_avar;
break;
case MVT::f16:
Opcode = NVPTX::LD_f16_avar;
break;
case MVT::v2f16:
Opcode = NVPTX::LD_f16x2_avar;
break;
case MVT::f32:
Opcode = NVPTX::LD_f32_avar;
break;
case MVT::f64:
Opcode = NVPTX::LD_f64_avar;
break;
default:
return false; return false;
}
SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl), SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl),
getI32Imm(vecType, dl), getI32Imm(fromType, dl), getI32Imm(vecType, dl), getI32Imm(fromType, dl),
getI32Imm(fromTypeWidth, dl), Addr, Chain }; getI32Imm(fromTypeWidth, dl), Addr, Chain };
NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); NVPTXLD = CurDAG->getMachineNode(Opcode.getValue(), dl, TargetVT,
MVT::Other, Ops);
} else if (TM.is64Bit() ? SelectADDRsi64(N1.getNode(), N1, Base, Offset) } else if (TM.is64Bit() ? SelectADDRsi64(N1.getNode(), N1, Base, Offset)
: SelectADDRsi(N1.getNode(), N1, Base, Offset)) { : SelectADDRsi(N1.getNode(), N1, Base, Offset)) {
switch (TargetVT) { Opcode = pickOpcodeForVT(TargetVT, NVPTX::LD_i8_asi, NVPTX::LD_i16_asi,
case MVT::i8: NVPTX::LD_i32_asi, NVPTX::LD_i64_asi,
Opcode = NVPTX::LD_i8_asi; NVPTX::LD_f16_asi, NVPTX::LD_f16x2_asi,
break; NVPTX::LD_f32_asi, NVPTX::LD_f64_asi);
case MVT::i16: if (!Opcode)
Opcode = NVPTX::LD_i16_asi;
break;
case MVT::i32:
Opcode = NVPTX::LD_i32_asi;
break;
case MVT::i64:
Opcode = NVPTX::LD_i64_asi;
break;
case MVT::f16:
Opcode = NVPTX::LD_f16_asi;
break;
case MVT::v2f16:
Opcode = NVPTX::LD_f16x2_asi;
break;
case MVT::f32:
Opcode = NVPTX::LD_f32_asi;
break;
case MVT::f64:
Opcode = NVPTX::LD_f64_asi;
break;
default:
return false; return false;
}
SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl), SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl),
getI32Imm(vecType, dl), getI32Imm(fromType, dl), getI32Imm(vecType, dl), getI32Imm(fromType, dl),
getI32Imm(fromTypeWidth, dl), Base, Offset, Chain }; getI32Imm(fromTypeWidth, dl), Base, Offset, Chain };
NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); NVPTXLD = CurDAG->getMachineNode(Opcode.getValue(), dl, TargetVT,
MVT::Other, Ops);
} else if (TM.is64Bit() ? SelectADDRri64(N1.getNode(), N1, Base, Offset) } else if (TM.is64Bit() ? SelectADDRri64(N1.getNode(), N1, Base, Offset)
: SelectADDRri(N1.getNode(), N1, Base, Offset)) { : SelectADDRri(N1.getNode(), N1, Base, Offset)) {
if (TM.is64Bit()) { if (TM.is64Bit())
switch (TargetVT) { Opcode = pickOpcodeForVT(
case MVT::i8: TargetVT, NVPTX::LD_i8_ari_64, NVPTX::LD_i16_ari_64,
Opcode = NVPTX::LD_i8_ari_64; NVPTX::LD_i32_ari_64, NVPTX::LD_i64_ari_64, NVPTX::LD_f16_ari_64,
break; NVPTX::LD_f16x2_ari_64, NVPTX::LD_f32_ari_64, NVPTX::LD_f64_ari_64);
case MVT::i16: else
Opcode = NVPTX::LD_i16_ari_64; Opcode = pickOpcodeForVT(
break; TargetVT, NVPTX::LD_i8_ari, NVPTX::LD_i16_ari, NVPTX::LD_i32_ari,
case MVT::i32: NVPTX::LD_i64_ari, NVPTX::LD_f16_ari, NVPTX::LD_f16x2_ari,
Opcode = NVPTX::LD_i32_ari_64; NVPTX::LD_f32_ari, NVPTX::LD_f64_ari);
break; if (!Opcode)
case MVT::i64:
Opcode = NVPTX::LD_i64_ari_64;
break;
case MVT::f16:
Opcode = NVPTX::LD_f16_ari_64;
break;
case MVT::v2f16:
Opcode = NVPTX::LD_f16x2_ari_64;
break;
case MVT::f32:
Opcode = NVPTX::LD_f32_ari_64;
break;
case MVT::f64:
Opcode = NVPTX::LD_f64_ari_64;
break;
default:
return false;
}
} else {
switch (TargetVT) {
case MVT::i8:
Opcode = NVPTX::LD_i8_ari;
break;
case MVT::i16:
Opcode = NVPTX::LD_i16_ari;
break;
case MVT::i32:
Opcode = NVPTX::LD_i32_ari;
break;
case MVT::i64:
Opcode = NVPTX::LD_i64_ari;
break;
case MVT::f16:
Opcode = NVPTX::LD_f16_ari;
break;
case MVT::v2f16:
Opcode = NVPTX::LD_f16x2_ari;
break;
case MVT::f32:
Opcode = NVPTX::LD_f32_ari;
break;
case MVT::f64:
Opcode = NVPTX::LD_f64_ari;
break;
default:
return false; return false;
}
}
SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl), SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl),
getI32Imm(vecType, dl), getI32Imm(fromType, dl), getI32Imm(vecType, dl), getI32Imm(fromType, dl),
getI32Imm(fromTypeWidth, dl), Base, Offset, Chain }; getI32Imm(fromTypeWidth, dl), Base, Offset, Chain };
NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); NVPTXLD = CurDAG->getMachineNode(Opcode.getValue(), dl, TargetVT,
MVT::Other, Ops);
} else { } else {
if (TM.is64Bit()) { if (TM.is64Bit())
switch (TargetVT) { Opcode = pickOpcodeForVT(
case MVT::i8: TargetVT, NVPTX::LD_i8_areg_64, NVPTX::LD_i16_areg_64,
Opcode = NVPTX::LD_i8_areg_64; NVPTX::LD_i32_areg_64, NVPTX::LD_i64_areg_64, NVPTX::LD_f16_areg_64,
break; NVPTX::LD_f16x2_areg_64, NVPTX::LD_f32_areg_64,
case MVT::i16: NVPTX::LD_f64_areg_64);
Opcode = NVPTX::LD_i16_areg_64; else
break; Opcode = pickOpcodeForVT(
case MVT::i32: TargetVT, NVPTX::LD_i8_areg, NVPTX::LD_i16_areg, NVPTX::LD_i32_areg,
Opcode = NVPTX::LD_i32_areg_64; NVPTX::LD_i64_areg, NVPTX::LD_f16_areg, NVPTX::LD_f16x2_areg,
break; NVPTX::LD_f32_areg, NVPTX::LD_f64_areg);
case MVT::i64: if (!Opcode)
Opcode = NVPTX::LD_i64_areg_64;
break;
case MVT::f16:
Opcode = NVPTX::LD_f16_areg_64;
break;
case MVT::v2f16:
Opcode = NVPTX::LD_f16x2_areg_64;
break;
case MVT::f32:
Opcode = NVPTX::LD_f32_areg_64;
break;
case MVT::f64:
Opcode = NVPTX::LD_f64_areg_64;
break;
default:
return false;
}
} else {
switch (TargetVT) {
case MVT::i8:
Opcode = NVPTX::LD_i8_areg;
break;
case MVT::i16:
Opcode = NVPTX::LD_i16_areg;
break;
case MVT::i32:
Opcode = NVPTX::LD_i32_areg;
break;
case MVT::i64:
Opcode = NVPTX::LD_i64_areg;
break;
case MVT::f16:
Opcode = NVPTX::LD_f16_areg;
break;
case MVT::v2f16:
Opcode = NVPTX::LD_f16x2_areg;
break;
case MVT::f32:
Opcode = NVPTX::LD_f32_areg;
break;
case MVT::f64:
Opcode = NVPTX::LD_f64_areg;
break;
default:
return false; return false;
}
}
SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl), SDValue Ops[] = { getI32Imm(isVolatile, dl), getI32Imm(codeAddrSpace, dl),
getI32Imm(vecType, dl), getI32Imm(fromType, dl), getI32Imm(vecType, dl), getI32Imm(fromType, dl),
getI32Imm(fromTypeWidth, dl), N1, Chain }; getI32Imm(fromTypeWidth, dl), N1, Chain };
NVPTXLD = CurDAG->getMachineNode(Opcode, dl, TargetVT, MVT::Other, Ops); NVPTXLD = CurDAG->getMachineNode(Opcode.getValue(), dl, TargetVT,
MVT::Other, Ops);
} }
if (!NVPTXLD) if (!NVPTXLD)
return false; return false;
MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1); cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1);
ReplaceNode(N, NVPTXLD); ReplaceNode(N, NVPTXLD);
return true; return true;
} }
bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) { bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) {
SDValue Chain = N->getOperand(0); SDValue Chain = N->getOperand(0);
SDValue Op1 = N->getOperand(1); SDValue Op1 = N->getOperand(1);
SDValue Addr, Offset, Base; SDValue Addr, Offset, Base;
unsigned Opcode; Optional<unsigned> Opcode;
SDLoc DL(N); SDLoc DL(N);
SDNode *LD; SDNode *LD;
MemSDNode *MemSD = cast<MemSDNode>(N); MemSDNode *MemSD = cast<MemSDNode>(N);
EVT LoadedVT = MemSD->getMemoryVT(); EVT LoadedVT = MemSD->getMemoryVT();
if (!LoadedVT.isSimple()) if (!LoadedVT.isSimple())
return false; return false;
▲ Show 20 Lines • Show All 61 Lines • ▼ Show 20 Lines if (EltVT == MVT::v2f16) {
FromTypeWidth = 32; FromTypeWidth = 32;
} }
if (SelectDirectAddr(Op1, Addr)) { if (SelectDirectAddr(Op1, Addr)) {
switch (N->getOpcode()) { switch (N->getOpcode()) {
default: default:
return false; return false;
case NVPTXISD::LoadV2: case NVPTXISD::LoadV2:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::LDV_i8_v2_avar, NVPTX::LDV_i16_v2_avar,
return false; NVPTX::LDV_i32_v2_avar, NVPTX::LDV_i64_v2_avar,
case MVT::i8: NVPTX::LDV_f16_v2_avar, NVPTX::LDV_f16x2_v2_avar,
Opcode = NVPTX::LDV_i8_v2_avar; NVPTX::LDV_f32_v2_avar, NVPTX::LDV_f64_v2_avar);
break;
case MVT::i16:
Opcode = NVPTX::LDV_i16_v2_avar;
break;
case MVT::i32:
Opcode = NVPTX::LDV_i32_v2_avar;
break;
case MVT::i64:
Opcode = NVPTX::LDV_i64_v2_avar;
break;
case MVT::f16:
Opcode = NVPTX::LDV_f16_v2_avar;
break;
case MVT::f32:
Opcode = NVPTX::LDV_f32_v2_avar;
break;
case MVT::f64:
Opcode = NVPTX::LDV_f64_v2_avar;
break;
}
break; break;
case NVPTXISD::LoadV4: case NVPTXISD::LoadV4:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode =
default: pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy, NVPTX::LDV_i8_v4_avar,
return false; NVPTX::LDV_i16_v4_avar, NVPTX::LDV_i32_v4_avar, None,
case MVT::i8: NVPTX::LDV_f16_v4_avar, NVPTX::LDV_f16x2_v4_avar,
Opcode = NVPTX::LDV_i8_v4_avar; NVPTX::LDV_f32_v4_avar, None);
break;
case MVT::i16:
Opcode = NVPTX::LDV_i16_v4_avar;
break;
case MVT::i32:
Opcode = NVPTX::LDV_i32_v4_avar;
break;
case MVT::f16:
Opcode = NVPTX::LDV_f16_v4_avar;
break;
case MVT::f32:
Opcode = NVPTX::LDV_f32_v4_avar;
break; break;
} }
break; if (!Opcode)
} return false;
SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL), SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
getI32Imm(VecType, DL), getI32Imm(FromType, DL), getI32Imm(VecType, DL), getI32Imm(FromType, DL),
getI32Imm(FromTypeWidth, DL), Addr, Chain }; getI32Imm(FromTypeWidth, DL), Addr, Chain };
LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); LD = CurDAG->getMachineNode(Opcode.getValue(), DL, N->getVTList(), Ops);
} else if (TM.is64Bit() ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset) } else if (TM.is64Bit() ? SelectADDRsi64(Op1.getNode(), Op1, Base, Offset)
: SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) { : SelectADDRsi(Op1.getNode(), Op1, Base, Offset)) {
switch (N->getOpcode()) { switch (N->getOpcode()) {
default: default:
return false; return false;
case NVPTXISD::LoadV2: case NVPTXISD::LoadV2:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::LDV_i8_v2_asi, NVPTX::LDV_i16_v2_asi,
return false; NVPTX::LDV_i32_v2_asi, NVPTX::LDV_i64_v2_asi,
case MVT::i8: NVPTX::LDV_f16_v2_asi, NVPTX::LDV_f16x2_v2_asi,
Opcode = NVPTX::LDV_i8_v2_asi; NVPTX::LDV_f32_v2_asi, NVPTX::LDV_f64_v2_asi);
break;
case MVT::i16:
Opcode = NVPTX::LDV_i16_v2_asi;
break;
case MVT::i32:
Opcode = NVPTX::LDV_i32_v2_asi;
break;
case MVT::i64:
Opcode = NVPTX::LDV_i64_v2_asi;
break;
case MVT::f16:
Opcode = NVPTX::LDV_f16_v2_asi;
break;
case MVT::f32:
Opcode = NVPTX::LDV_f32_v2_asi;
break;
case MVT::f64:
Opcode = NVPTX::LDV_f64_v2_asi;
break;
}
break; break;
case NVPTXISD::LoadV4: case NVPTXISD::LoadV4:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode =
default: pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy, NVPTX::LDV_i8_v4_asi,
return false; NVPTX::LDV_i16_v4_asi, NVPTX::LDV_i32_v4_asi, None,
case MVT::i8: NVPTX::LDV_f16_v4_asi, NVPTX::LDV_f16x2_v4_asi,
Opcode = NVPTX::LDV_i8_v4_asi; NVPTX::LDV_f32_v4_asi, None);
break;
case MVT::i16:
Opcode = NVPTX::LDV_i16_v4_asi;
break;
case MVT::i32:
Opcode = NVPTX::LDV_i32_v4_asi;
break;
case MVT::f16:
Opcode = NVPTX::LDV_f16_v4_asi;
break;
case MVT::f32:
Opcode = NVPTX::LDV_f32_v4_asi;
break;
}
break; break;
} }
if (!Opcode)
return false;
SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL), SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
getI32Imm(VecType, DL), getI32Imm(FromType, DL), getI32Imm(VecType, DL), getI32Imm(FromType, DL),
getI32Imm(FromTypeWidth, DL), Base, Offset, Chain }; getI32Imm(FromTypeWidth, DL), Base, Offset, Chain };
LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); LD = CurDAG->getMachineNode(Opcode.getValue(), DL, N->getVTList(), Ops);
} else if (TM.is64Bit() ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset) } else if (TM.is64Bit() ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
: SelectADDRri(Op1.getNode(), Op1, Base, Offset)) { : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
if (TM.is64Bit()) { if (TM.is64Bit()) {
switch (N->getOpcode()) { switch (N->getOpcode()) {
default: default:
return false; return false;
case NVPTXISD::LoadV2: case NVPTXISD::LoadV2:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(
default: EltVT.getSimpleVT().SimpleTy, NVPTX::LDV_i8_v2_ari_64,
return false; NVPTX::LDV_i16_v2_ari_64, NVPTX::LDV_i32_v2_ari_64,
case MVT::i8: NVPTX::LDV_i64_v2_ari_64, NVPTX::LDV_f16_v2_ari_64,
Opcode = NVPTX::LDV_i8_v2_ari_64; NVPTX::LDV_f16x2_v2_ari_64, NVPTX::LDV_f32_v2_ari_64,
break; NVPTX::LDV_f64_v2_ari_64);
case MVT::i16:
Opcode = NVPTX::LDV_i16_v2_ari_64;
break;
case MVT::i32:
Opcode = NVPTX::LDV_i32_v2_ari_64;
break;
case MVT::i64:
Opcode = NVPTX::LDV_i64_v2_ari_64;
break;
case MVT::f16:
Opcode = NVPTX::LDV_f16_v2_ari_64;
break;
case MVT::f32:
Opcode = NVPTX::LDV_f32_v2_ari_64;
break;
case MVT::f64:
Opcode = NVPTX::LDV_f64_v2_ari_64;
break;
}
break; break;
case NVPTXISD::LoadV4: case NVPTXISD::LoadV4:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(
default: EltVT.getSimpleVT().SimpleTy, NVPTX::LDV_i8_v4_ari_64,
return false; NVPTX::LDV_i16_v4_ari_64, NVPTX::LDV_i32_v4_ari_64, None,
case MVT::i8: NVPTX::LDV_f16_v4_ari_64, NVPTX::LDV_f16x2_v4_ari_64,
Opcode = NVPTX::LDV_i8_v4_ari_64; NVPTX::LDV_f32_v4_ari_64, None);
break;
case MVT::i16:
Opcode = NVPTX::LDV_i16_v4_ari_64;
break;
case MVT::i32:
Opcode = NVPTX::LDV_i32_v4_ari_64;
break;
case MVT::f16:
Opcode = NVPTX::LDV_f16_v4_ari_64;
break;
case MVT::f32:
Opcode = NVPTX::LDV_f32_v4_ari_64;
break;
}
break; break;
} }
} else { } else {
switch (N->getOpcode()) { switch (N->getOpcode()) {
default: default:
return false; return false;
case NVPTXISD::LoadV2: case NVPTXISD::LoadV2:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::LDV_i8_v2_ari, NVPTX::LDV_i16_v2_ari,
return false; NVPTX::LDV_i32_v2_ari, NVPTX::LDV_i64_v2_ari,
case MVT::i8: NVPTX::LDV_f16_v2_ari, NVPTX::LDV_f16x2_v2_ari,
Opcode = NVPTX::LDV_i8_v2_ari; NVPTX::LDV_f32_v2_ari, NVPTX::LDV_f64_v2_ari);
break;
case MVT::i16:
Opcode = NVPTX::LDV_i16_v2_ari;
break;
case MVT::i32:
Opcode = NVPTX::LDV_i32_v2_ari;
break;
case MVT::i64:
Opcode = NVPTX::LDV_i64_v2_ari;
break;
case MVT::f16:
Opcode = NVPTX::LDV_f16_v2_ari;
break;
case MVT::f32:
Opcode = NVPTX::LDV_f32_v2_ari;
break;
case MVT::f64:
Opcode = NVPTX::LDV_f64_v2_ari;
break;
}
break; break;
case NVPTXISD::LoadV4: case NVPTXISD::LoadV4:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode =
default: pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy, NVPTX::LDV_i8_v4_ari,
return false; NVPTX::LDV_i16_v4_ari, NVPTX::LDV_i32_v4_ari, None,
case MVT::i8: NVPTX::LDV_f16_v4_ari, NVPTX::LDV_f16x2_v4_ari,
Opcode = NVPTX::LDV_i8_v4_ari; NVPTX::LDV_f32_v4_ari, None);
break;
case MVT::i16:
Opcode = NVPTX::LDV_i16_v4_ari;
break;
case MVT::i32:
Opcode = NVPTX::LDV_i32_v4_ari;
break;
case MVT::f16:
Opcode = NVPTX::LDV_f16_v4_ari;
break;
case MVT::f32:
Opcode = NVPTX::LDV_f32_v4_ari;
break; break;
} }
break;
} }
} if (!Opcode)
return false;
SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL), SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
getI32Imm(VecType, DL), getI32Imm(FromType, DL), getI32Imm(VecType, DL), getI32Imm(FromType, DL),
getI32Imm(FromTypeWidth, DL), Base, Offset, Chain }; getI32Imm(FromTypeWidth, DL), Base, Offset, Chain };
LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); LD = CurDAG->getMachineNode(Opcode.getValue(), DL, N->getVTList(), Ops);
} else { } else {
if (TM.is64Bit()) { if (TM.is64Bit()) {
switch (N->getOpcode()) { switch (N->getOpcode()) {
default: default:
return false; return false;
case NVPTXISD::LoadV2: case NVPTXISD::LoadV2:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(
default: EltVT.getSimpleVT().SimpleTy, NVPTX::LDV_i8_v2_areg_64,
return false; NVPTX::LDV_i16_v2_areg_64, NVPTX::LDV_i32_v2_areg_64,
case MVT::i8: NVPTX::LDV_i64_v2_areg_64, NVPTX::LDV_f16_v2_areg_64,
Opcode = NVPTX::LDV_i8_v2_areg_64; NVPTX::LDV_f16x2_v2_areg_64, NVPTX::LDV_f32_v2_areg_64,
break; NVPTX::LDV_f64_v2_areg_64);
case MVT::i16:
Opcode = NVPTX::LDV_i16_v2_areg_64;
break;
case MVT::i32:
Opcode = NVPTX::LDV_i32_v2_areg_64;
break;
case MVT::i64:
Opcode = NVPTX::LDV_i64_v2_areg_64;
break;
case MVT::f16:
Opcode = NVPTX::LDV_f16_v2_areg_64;
break;
case MVT::f32:
Opcode = NVPTX::LDV_f32_v2_areg_64;
break;
case MVT::f64:
Opcode = NVPTX::LDV_f64_v2_areg_64;
break;
}
break; break;
case NVPTXISD::LoadV4: case NVPTXISD::LoadV4:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(
default: EltVT.getSimpleVT().SimpleTy, NVPTX::LDV_i8_v4_areg_64,
return false; NVPTX::LDV_i16_v4_areg_64, NVPTX::LDV_i32_v4_areg_64, None,
case MVT::i8: NVPTX::LDV_f16_v4_areg_64, NVPTX::LDV_f16x2_v4_areg_64,
Opcode = NVPTX::LDV_i8_v4_areg_64; NVPTX::LDV_f32_v4_areg_64, None);
break;
case MVT::i16:
Opcode = NVPTX::LDV_i16_v4_areg_64;
break;
case MVT::i32:
Opcode = NVPTX::LDV_i32_v4_areg_64;
break;
case MVT::f16:
Opcode = NVPTX::LDV_f16_v4_areg_64;
break;
case MVT::f32:
Opcode = NVPTX::LDV_f32_v4_areg_64;
break;
}
break; break;
} }
} else { } else {
switch (N->getOpcode()) { switch (N->getOpcode()) {
default: default:
return false; return false;
case NVPTXISD::LoadV2: case NVPTXISD::LoadV2:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode =
default: pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy, NVPTX::LDV_i8_v2_areg,
return false; NVPTX::LDV_i16_v2_areg, NVPTX::LDV_i32_v2_areg,
case MVT::i8: NVPTX::LDV_i64_v2_areg, NVPTX::LDV_f16_v2_areg,
Opcode = NVPTX::LDV_i8_v2_areg; NVPTX::LDV_f16x2_v2_areg, NVPTX::LDV_f32_v2_areg,
break; NVPTX::LDV_f64_v2_areg);
case MVT::i16:
Opcode = NVPTX::LDV_i16_v2_areg;
break;
case MVT::i32:
Opcode = NVPTX::LDV_i32_v2_areg;
break;
case MVT::i64:
Opcode = NVPTX::LDV_i64_v2_areg;
break;
case MVT::f16:
Opcode = NVPTX::LDV_f16_v2_areg;
break;
case MVT::f32:
Opcode = NVPTX::LDV_f32_v2_areg;
break;
case MVT::f64:
Opcode = NVPTX::LDV_f64_v2_areg;
break;
}
break; break;
case NVPTXISD::LoadV4: case NVPTXISD::LoadV4:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(
default: EltVT.getSimpleVT().SimpleTy, NVPTX::LDV_i8_v4_areg,
return false; NVPTX::LDV_i16_v4_areg, NVPTX::LDV_i32_v4_areg, None,
case MVT::i8: NVPTX::LDV_f16_v4_areg, NVPTX::LDV_f16x2_v4_areg,
Opcode = NVPTX::LDV_i8_v4_areg; NVPTX::LDV_f32_v4_areg, None);
break;
case MVT::i16:
Opcode = NVPTX::LDV_i16_v4_areg;
break;
case MVT::i32:
Opcode = NVPTX::LDV_i32_v4_areg;
break;
case MVT::f16:
Opcode = NVPTX::LDV_f16_v4_areg;
break;
case MVT::f32:
Opcode = NVPTX::LDV_f32_v4_areg;
break; break;
} }
break;
} }
} if (!Opcode)
return false;
SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL), SDValue Ops[] = { getI32Imm(IsVolatile, DL), getI32Imm(CodeAddrSpace, DL),
getI32Imm(VecType, DL), getI32Imm(FromType, DL), getI32Imm(VecType, DL), getI32Imm(FromType, DL),
getI32Imm(FromTypeWidth, DL), Op1, Chain }; getI32Imm(FromTypeWidth, DL), Op1, Chain };
LD = CurDAG->getMachineNode(Opcode, DL, N->getVTList(), Ops); LD = CurDAG->getMachineNode(Opcode.getValue(), DL, N->getVTList(), Ops);
} }
MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1); cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
ReplaceNode(N, LD); ReplaceNode(N, LD);
return true; return true;
Show All 26 Lines case Intrinsic::nvvm_ldu_global_p:
IsLDG = false; IsLDG = false;
break; break;
} }
} else { } else {
Op1 = N->getOperand(1); Op1 = N->getOperand(1);
Mem = cast<MemSDNode>(N); Mem = cast<MemSDNode>(N);
} }
unsigned Opcode; Optional<unsigned> Opcode;
SDLoc DL(N); SDLoc DL(N);
SDNode *LD; SDNode *LD;
SDValue Base, Offset, Addr; SDValue Base, Offset, Addr;
EVT EltVT = Mem->getMemoryVT(); EVT EltVT = Mem->getMemoryVT();
unsigned NumElts = 1; unsigned NumElts = 1;
if (EltVT.isVector()) { if (EltVT.isVector()) {
NumElts = EltVT.getVectorNumElements(); NumElts = EltVT.getVectorNumElements();
Show All 11 Linesbool NVPTXDAGToDAGISel::tryLDGLDU(SDNode *N) {
InstVTs.push_back(MVT::Other); InstVTs.push_back(MVT::Other);
SDVTList InstVTList = CurDAG->getVTList(InstVTs); SDVTList InstVTList = CurDAG->getVTList(InstVTs);
if (SelectDirectAddr(Op1, Addr)) { if (SelectDirectAddr(Op1, Addr)) {
switch (N->getOpcode()) { switch (N->getOpcode()) {
default: default:
return false; return false;
case ISD::INTRINSIC_W_CHAIN: case ISD::INTRINSIC_W_CHAIN:
if (IsLDG) { if (IsLDG)
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::INT_PTX_LDG_GLOBAL_i8avar,
return false; NVPTX::INT_PTX_LDG_GLOBAL_i16avar,
case MVT::i8: NVPTX::INT_PTX_LDG_GLOBAL_i32avar,
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8avar; NVPTX::INT_PTX_LDG_GLOBAL_i64avar,
break; NVPTX::INT_PTX_LDG_GLOBAL_f16avar,
case MVT::i16: NVPTX::INT_PTX_LDG_GLOBAL_f16x2avar,
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16avar; NVPTX::INT_PTX_LDG_GLOBAL_f32avar,
break; NVPTX::INT_PTX_LDG_GLOBAL_f64avar);
case MVT::i32: else
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32avar; Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
break; NVPTX::INT_PTX_LDU_GLOBAL_i8avar,
case MVT::i64: NVPTX::INT_PTX_LDU_GLOBAL_i16avar,
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64avar; NVPTX::INT_PTX_LDU_GLOBAL_i32avar,
break; NVPTX::INT_PTX_LDU_GLOBAL_i64avar,
case MVT::f32: NVPTX::INT_PTX_LDU_GLOBAL_f16avar,
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32avar; NVPTX::INT_PTX_LDU_GLOBAL_f16x2avar,
break; NVPTX::INT_PTX_LDU_GLOBAL_f32avar,
case MVT::f64: NVPTX::INT_PTX_LDU_GLOBAL_f64avar);
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64avar;
break;
}
} else {
switch (EltVT.getSimpleVT().SimpleTy) {
default:
return false;
case MVT::i8:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8avar;
break;
case MVT::i16:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16avar;
break;
case MVT::i32:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32avar;
break;
case MVT::i64:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64avar;
break;
case MVT::f32:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32avar;
break;
case MVT::f64:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64avar;
break;
}
}
break; break;
case NVPTXISD::LDGV2: case NVPTXISD::LDGV2:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar,
return false; NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar,
case MVT::i8: NVPTX::INT_PTX_LDG_G_v2i32_ELE_avar,
Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_avar; NVPTX::INT_PTX_LDG_G_v2i64_ELE_avar,
break; NVPTX::INT_PTX_LDG_G_v2f16_ELE_avar,
case MVT::i16: NVPTX::INT_PTX_LDG_G_v2f16x2_ELE_avar,
Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_avar; NVPTX::INT_PTX_LDG_G_v2f32_ELE_avar,
break; NVPTX::INT_PTX_LDG_G_v2f64_ELE_avar);
case MVT::i32:
Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_avar;
break;
case MVT::i64:
Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_avar;
break;
case MVT::f32:
Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_avar;
break;
case MVT::f64:
Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_avar;
break;
}
break; break;
case NVPTXISD::LDUV2: case NVPTXISD::LDUV2:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar,
return false; NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar,
case MVT::i8: NVPTX::INT_PTX_LDU_G_v2i32_ELE_avar,
Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_avar; NVPTX::INT_PTX_LDU_G_v2i64_ELE_avar,
break; NVPTX::INT_PTX_LDU_G_v2f16_ELE_avar,
case MVT::i16: NVPTX::INT_PTX_LDU_G_v2f16x2_ELE_avar,
Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_avar; NVPTX::INT_PTX_LDU_G_v2f32_ELE_avar,
break; NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar);
case MVT::i32:
Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_avar;
break;
case MVT::i64:
Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_avar;
break;
case MVT::f32:
Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_avar;
break;
case MVT::f64:
Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_avar;
break;
}
break; break;
case NVPTXISD::LDGV4: case NVPTXISD::LDGV4:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar,
return false; NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar,
case MVT::i8: NVPTX::INT_PTX_LDG_G_v4i32_ELE_avar, None,
Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_avar; NVPTX::INT_PTX_LDG_G_v4f16_ELE_avar,
break; NVPTX::INT_PTX_LDG_G_v4f16x2_ELE_avar,
case MVT::i16: NVPTX::INT_PTX_LDG_G_v4f32_ELE_avar, None);
Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_avar;
break;
case MVT::i32:
Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_avar;
break;
case MVT::f32:
Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_avar;
break;
}
break; break;
case NVPTXISD::LDUV4: case NVPTXISD::LDUV4:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar,
return false; NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar,
case MVT::i8: NVPTX::INT_PTX_LDU_G_v4i32_ELE_avar, None,
Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_avar; NVPTX::INT_PTX_LDU_G_v4f16_ELE_avar,
break; NVPTX::INT_PTX_LDU_G_v4f16x2_ELE_avar,
case MVT::i16: NVPTX::INT_PTX_LDU_G_v4f32_ELE_avar, None);
Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_avar;
break;
case MVT::i32:
Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_avar;
break;
case MVT::f32:
Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_avar;
break; break;
} }
break; if (!Opcode)
} return false;
SDValue Ops[] = { Addr, Chain }; SDValue Ops[] = { Addr, Chain };
LD = CurDAG->getMachineNode(Opcode, DL, InstVTList, Ops); LD = CurDAG->getMachineNode(Opcode.getValue(), DL, InstVTList, Ops);
} else if (TM.is64Bit() ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset) } else if (TM.is64Bit() ? SelectADDRri64(Op1.getNode(), Op1, Base, Offset)
: SelectADDRri(Op1.getNode(), Op1, Base, Offset)) { : SelectADDRri(Op1.getNode(), Op1, Base, Offset)) {
if (TM.is64Bit()) { if (TM.is64Bit()) {
switch (N->getOpcode()) { switch (N->getOpcode()) {
default: default:
return false; return false;
case ISD::LOAD: case ISD::LOAD:
case ISD::INTRINSIC_W_CHAIN: case ISD::INTRINSIC_W_CHAIN:
if (IsLDG) { if (IsLDG)
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::INT_PTX_LDG_GLOBAL_i8ari64,
return false; NVPTX::INT_PTX_LDG_GLOBAL_i16ari64,
case MVT::i8: NVPTX::INT_PTX_LDG_GLOBAL_i32ari64,
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari64; NVPTX::INT_PTX_LDG_GLOBAL_i64ari64,
break; NVPTX::INT_PTX_LDG_GLOBAL_f16ari64,
case MVT::i16: NVPTX::INT_PTX_LDG_GLOBAL_f16x2ari64,
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari64; NVPTX::INT_PTX_LDG_GLOBAL_f32ari64,
break; NVPTX::INT_PTX_LDG_GLOBAL_f64ari64);
case MVT::i32: else
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari64; Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
break; NVPTX::INT_PTX_LDU_GLOBAL_i8ari64,
case MVT::i64: NVPTX::INT_PTX_LDU_GLOBAL_i16ari64,
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari64; NVPTX::INT_PTX_LDU_GLOBAL_i32ari64,
break; NVPTX::INT_PTX_LDU_GLOBAL_i64ari64,
case MVT::f32: NVPTX::INT_PTX_LDU_GLOBAL_f16ari64,
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari64; NVPTX::INT_PTX_LDU_GLOBAL_f16x2ari64,
break; NVPTX::INT_PTX_LDU_GLOBAL_f32ari64,
case MVT::f64: NVPTX::INT_PTX_LDU_GLOBAL_f64ari64);
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari64;
break;
}
} else {
switch (EltVT.getSimpleVT().SimpleTy) {
default:
return false;
case MVT::i8:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari64;
break;
case MVT::i16:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari64;
break;
case MVT::i32:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari64;
break;
case MVT::i64:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari64;
break;
case MVT::f32:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari64;
break;
case MVT::f64:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari64;
break;
}
}
break; break;
case NVPTXISD::LoadV2: case NVPTXISD::LoadV2:
case NVPTXISD::LDGV2: case NVPTXISD::LDGV2:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64,
return false; NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64,
case MVT::i8: NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari64,
Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari64; NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari64,
break; NVPTX::INT_PTX_LDG_G_v2f16_ELE_ari64,
case MVT::i16: NVPTX::INT_PTX_LDG_G_v2f16x2_ELE_ari64,
Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari64; NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari64,
break; NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari64);
case MVT::i32:
Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari64;
break;
case MVT::i64:
Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari64;
break;
case MVT::f32:
Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari64;
break;
case MVT::f64:
Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari64;
break;
}
break; break;
case NVPTXISD::LDUV2: case NVPTXISD::LDUV2:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64,
return false; NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64,
case MVT::i8: NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari64,
Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari64; NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari64,
break; NVPTX::INT_PTX_LDU_G_v2f16_ELE_ari64,
case MVT::i16: NVPTX::INT_PTX_LDU_G_v2f16x2_ELE_ari64,
Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari64; NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari64,
break; NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64);
case MVT::i32:
Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari64;
break;
case MVT::i64:
Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari64;
break;
case MVT::f32:
Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari64;
break;
case MVT::f64:
Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari64;
break;
}
break; break;
case NVPTXISD::LoadV4: case NVPTXISD::LoadV4:
case NVPTXISD::LDGV4: case NVPTXISD::LDGV4:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64,
return false; NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64,
case MVT::i8: NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari64, None,
Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari64; NVPTX::INT_PTX_LDG_G_v4f16_ELE_ari64,
break; NVPTX::INT_PTX_LDG_G_v4f16x2_ELE_ari64,
case MVT::i16: NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari64, None);
Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari64;
break;
case MVT::i32:
Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari64;
break;
case MVT::f32:
Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari64;
break;
}
break; break;
case NVPTXISD::LDUV4: case NVPTXISD::LDUV4:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64,
return false; NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64,
case MVT::i8: NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari64, None,
Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari64; NVPTX::INT_PTX_LDU_G_v4f16_ELE_ari64,
break; NVPTX::INT_PTX_LDU_G_v4f16x2_ELE_ari64,
case MVT::i16: NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari64, None);
Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari64;
break;
case MVT::i32:
Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari64;
break;
case MVT::f32:
Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari64;
break;
}
break; break;
} }
} else { } else {
switch (N->getOpcode()) { switch (N->getOpcode()) {
default: default:
return false; return false;
case ISD::LOAD: case ISD::LOAD:
case ISD::INTRINSIC_W_CHAIN: case ISD::INTRINSIC_W_CHAIN:
if (IsLDG) { if (IsLDG)
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::INT_PTX_LDG_GLOBAL_i8ari,
return false; NVPTX::INT_PTX_LDG_GLOBAL_i16ari,
case MVT::i8: NVPTX::INT_PTX_LDG_GLOBAL_i32ari,
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8ari; NVPTX::INT_PTX_LDG_GLOBAL_i64ari,
break; NVPTX::INT_PTX_LDG_GLOBAL_f16ari,
case MVT::i16: NVPTX::INT_PTX_LDG_GLOBAL_f16x2ari,
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16ari; NVPTX::INT_PTX_LDG_GLOBAL_f32ari,
break; NVPTX::INT_PTX_LDG_GLOBAL_f64ari);
case MVT::i32: else
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32ari; Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
break; NVPTX::INT_PTX_LDU_GLOBAL_i8ari,
case MVT::i64: NVPTX::INT_PTX_LDU_GLOBAL_i16ari,
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64ari; NVPTX::INT_PTX_LDU_GLOBAL_i32ari,
break; NVPTX::INT_PTX_LDU_GLOBAL_i64ari,
case MVT::f32: NVPTX::INT_PTX_LDU_GLOBAL_f16ari,
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32ari; NVPTX::INT_PTX_LDU_GLOBAL_f16x2ari,
break; NVPTX::INT_PTX_LDU_GLOBAL_f32ari,
case MVT::f64: NVPTX::INT_PTX_LDU_GLOBAL_f64ari);
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64ari;
break;
}
} else {
switch (EltVT.getSimpleVT().SimpleTy) {
default:
return false;
case MVT::i8:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8ari;
break;
case MVT::i16:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16ari;
break;
case MVT::i32:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32ari;
break;
case MVT::i64:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64ari;
break;
case MVT::f32:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32ari;
break;
case MVT::f64:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64ari;
break;
}
}
break; break;
case NVPTXISD::LoadV2: case NVPTXISD::LoadV2:
case NVPTXISD::LDGV2: case NVPTXISD::LDGV2:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32,
return false; NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32,
case MVT::i8: NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari32,
Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_ari32; NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari32,
break; NVPTX::INT_PTX_LDG_G_v2f16_ELE_ari32,
case MVT::i16: NVPTX::INT_PTX_LDG_G_v2f16x2_ELE_ari32,
Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_ari32; NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari32,
break; NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari32);
case MVT::i32:
Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_ari32;
break;
case MVT::i64:
Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_ari32;
break;
case MVT::f32:
Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_ari32;
break;
case MVT::f64:
Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_ari32;
break;
}
break; break;
case NVPTXISD::LDUV2: case NVPTXISD::LDUV2:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32,
return false; NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32,
case MVT::i8: NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari32,
Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_ari32; NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari32,
break; NVPTX::INT_PTX_LDU_G_v2f16_ELE_ari32,
case MVT::i16: NVPTX::INT_PTX_LDU_G_v2f16x2_ELE_ari32,
Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_ari32; NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari32,
break; NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32);
case MVT::i32:
Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_ari32;
break;
case MVT::i64:
Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_ari32;
break;
case MVT::f32:
Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_ari32;
break;
case MVT::f64:
Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_ari32;
break;
}
break; break;
case NVPTXISD::LoadV4: case NVPTXISD::LoadV4:
case NVPTXISD::LDGV4: case NVPTXISD::LDGV4:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32,
return false; NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32,
case MVT::i8: NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari32, None,
Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_ari32; NVPTX::INT_PTX_LDG_G_v4f16_ELE_ari32,
break; NVPTX::INT_PTX_LDG_G_v4f16x2_ELE_ari32,
case MVT::i16: NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari32, None);
Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_ari32;
break;
case MVT::i32:
Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_ari32;
break;
case MVT::f32:
Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_ari32;
break;
}
break; break;
case NVPTXISD::LDUV4: case NVPTXISD::LDUV4:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32,
return false; NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32,
case MVT::i8: NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari32, None,
Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_ari32; NVPTX::INT_PTX_LDU_G_v4f16_ELE_ari32,
break; NVPTX::INT_PTX_LDU_G_v4f16x2_ELE_ari32,
case MVT::i16: NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari32, None);
Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_ari32;
break;
case MVT::i32:
Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_ari32;
break;
case MVT::f32:
Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_ari32;
break;
}
break; break;
} }
} }
if (!Opcode)
return false;
SDValue Ops[] = { Base, Offset, Chain }; SDValue Ops[] = {Base, Offset, Chain};
LD = CurDAG->getMachineNode(Opcode.getValue(), DL, InstVTList, Ops);
LD = CurDAG->getMachineNode(Opcode, DL, InstVTList, Ops);
} else { } else {
if (TM.is64Bit()) { if (TM.is64Bit()) {
switch (N->getOpcode()) { switch (N->getOpcode()) {
default: default:
return false; return false;
case ISD::LOAD: case ISD::LOAD:
case ISD::INTRINSIC_W_CHAIN: case ISD::INTRINSIC_W_CHAIN:
if (IsLDG) { if (IsLDG)
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::INT_PTX_LDG_GLOBAL_i8areg64,
return false; NVPTX::INT_PTX_LDG_GLOBAL_i16areg64,
case MVT::i8: NVPTX::INT_PTX_LDG_GLOBAL_i32areg64,
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg64; NVPTX::INT_PTX_LDG_GLOBAL_i64areg64,
break; NVPTX::INT_PTX_LDG_GLOBAL_f16areg64,
case MVT::i16: NVPTX::INT_PTX_LDG_GLOBAL_f16x2areg64,
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg64; NVPTX::INT_PTX_LDG_GLOBAL_f32areg64,
break; NVPTX::INT_PTX_LDG_GLOBAL_f64areg64);
case MVT::i32: else
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg64; Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
break; NVPTX::INT_PTX_LDU_GLOBAL_i8areg64,
case MVT::i64: NVPTX::INT_PTX_LDU_GLOBAL_i16areg64,
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg64; NVPTX::INT_PTX_LDU_GLOBAL_i32areg64,
break; NVPTX::INT_PTX_LDU_GLOBAL_i64areg64,
case MVT::f32: NVPTX::INT_PTX_LDU_GLOBAL_f16areg64,
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg64; NVPTX::INT_PTX_LDU_GLOBAL_f16x2areg64,
break; NVPTX::INT_PTX_LDU_GLOBAL_f32areg64,
case MVT::f64: NVPTX::INT_PTX_LDU_GLOBAL_f64areg64);
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg64;
break;
}
} else {
switch (EltVT.getSimpleVT().SimpleTy) {
default:
return false;
case MVT::i8:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg64;
break;
case MVT::i16:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg64;
break;
case MVT::i32:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg64;
break;
case MVT::i64:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg64;
break;
case MVT::f32:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg64;
break;
case MVT::f64:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg64;
break;
}
}
break; break;
case NVPTXISD::LoadV2: case NVPTXISD::LoadV2:
case NVPTXISD::LDGV2: case NVPTXISD::LDGV2:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64,
return false; NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64,
case MVT::i8: NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg64,
Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg64; NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg64,
break; NVPTX::INT_PTX_LDG_G_v2f16_ELE_areg64,
case MVT::i16: NVPTX::INT_PTX_LDG_G_v2f16x2_ELE_areg64,
Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg64; NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg64,
break; NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg64);
case MVT::i32:
Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg64;
break;
case MVT::i64:
Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg64;
break;
case MVT::f32:
Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg64;
break;
case MVT::f64:
Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg64;
break;
}
break; break;
case NVPTXISD::LDUV2: case NVPTXISD::LDUV2:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64,
return false; NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64,
case MVT::i8: NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg64,
Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg64; NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg64,
break; NVPTX::INT_PTX_LDU_G_v2f16_ELE_areg64,
case MVT::i16: NVPTX::INT_PTX_LDU_G_v2f16x2_ELE_areg64,
Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg64; NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg64,
break; NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64);
case MVT::i32:
Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg64;
break;
case MVT::i64:
Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg64;
break;
case MVT::f32:
Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg64;
break;
case MVT::f64:
Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg64;
break;
}
break; break;
case NVPTXISD::LoadV4: case NVPTXISD::LoadV4:
case NVPTXISD::LDGV4: case NVPTXISD::LDGV4:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64,
return false; NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64,
case MVT::i8: NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg64, None,
Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg64; NVPTX::INT_PTX_LDG_G_v4f16_ELE_areg64,
break; NVPTX::INT_PTX_LDG_G_v4f16x2_ELE_areg64,
case MVT::i16: NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg64, None);
Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg64;
break;
case MVT::i32:
Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg64;
break;
case MVT::f32:
Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg64;
break;
}
break; break;
case NVPTXISD::LDUV4: case NVPTXISD::LDUV4:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64,
return false; NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64,
case MVT::i8: NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg64, None,
Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg64; NVPTX::INT_PTX_LDU_G_v4f16_ELE_areg64,
break; NVPTX::INT_PTX_LDU_G_v4f16x2_ELE_areg64,
case MVT::i16: NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg64, None);
Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg64;
break;
case MVT::i32:
Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg64;
break;
case MVT::f32:
Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg64;
break;
}
break; break;
} }
} else { } else {
switch (N->getOpcode()) { switch (N->getOpcode()) {
default: default:
return false; return false;
case ISD::LOAD: case ISD::LOAD:
case ISD::INTRINSIC_W_CHAIN: case ISD::INTRINSIC_W_CHAIN:
if (IsLDG) { if (IsLDG)
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::INT_PTX_LDG_GLOBAL_i8areg,
return false; NVPTX::INT_PTX_LDG_GLOBAL_i16areg,
case MVT::i8: NVPTX::INT_PTX_LDG_GLOBAL_i32areg,
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i8areg; NVPTX::INT_PTX_LDG_GLOBAL_i64areg,
break; NVPTX::INT_PTX_LDG_GLOBAL_f16areg,
case MVT::i16: NVPTX::INT_PTX_LDG_GLOBAL_f16x2areg,
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i16areg; NVPTX::INT_PTX_LDG_GLOBAL_f32areg,
break; NVPTX::INT_PTX_LDG_GLOBAL_f64areg);
case MVT::i32: else
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i32areg; Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
break; NVPTX::INT_PTX_LDU_GLOBAL_i8areg,
case MVT::i64: NVPTX::INT_PTX_LDU_GLOBAL_i16areg,
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_i64areg; NVPTX::INT_PTX_LDU_GLOBAL_i32areg,
break; NVPTX::INT_PTX_LDU_GLOBAL_i64areg,
case MVT::f32: NVPTX::INT_PTX_LDU_GLOBAL_f16areg,
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f32areg; NVPTX::INT_PTX_LDU_GLOBAL_f16x2areg,
break; NVPTX::INT_PTX_LDU_GLOBAL_f32areg,
case MVT::f64: NVPTX::INT_PTX_LDU_GLOBAL_f64areg);
Opcode = NVPTX::INT_PTX_LDG_GLOBAL_f64areg;
break;
}
} else {
switch (EltVT.getSimpleVT().SimpleTy) {
default:
return false;
case MVT::i8:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i8areg;
break;
case MVT::i16:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i16areg;
break;
case MVT::i32:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i32areg;
break;
case MVT::i64:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_i64areg;
break;
case MVT::f32:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f32areg;
break;
case MVT::f64:
Opcode = NVPTX::INT_PTX_LDU_GLOBAL_f64areg;
break;
}
}
break; break;
case NVPTXISD::LoadV2: case NVPTXISD::LoadV2:
case NVPTXISD::LDGV2: case NVPTXISD::LDGV2:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32,
return false; NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32,
case MVT::i8: NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg32,
Opcode = NVPTX::INT_PTX_LDG_G_v2i8_ELE_areg32; NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg32,
break; NVPTX::INT_PTX_LDG_G_v2f16_ELE_areg32,
case MVT::i16: NVPTX::INT_PTX_LDG_G_v2f16x2_ELE_areg32,
Opcode = NVPTX::INT_PTX_LDG_G_v2i16_ELE_areg32; NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg32,
break; NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg32);
case MVT::i32:
Opcode = NVPTX::INT_PTX_LDG_G_v2i32_ELE_areg32;
break;
case MVT::i64:
Opcode = NVPTX::INT_PTX_LDG_G_v2i64_ELE_areg32;
break;
case MVT::f32:
Opcode = NVPTX::INT_PTX_LDG_G_v2f32_ELE_areg32;
break;
case MVT::f64:
Opcode = NVPTX::INT_PTX_LDG_G_v2f64_ELE_areg32;
break;
}
break; break;
case NVPTXISD::LDUV2: case NVPTXISD::LDUV2:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32,
return false; NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32,
case MVT::i8: NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg32,
Opcode = NVPTX::INT_PTX_LDU_G_v2i8_ELE_areg32; NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg32,
break; NVPTX::INT_PTX_LDU_G_v2f16_ELE_areg32,
case MVT::i16: NVPTX::INT_PTX_LDU_G_v2f16x2_ELE_areg32,
Opcode = NVPTX::INT_PTX_LDU_G_v2i16_ELE_areg32; NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg32,
break; NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32);
case MVT::i32:
Opcode = NVPTX::INT_PTX_LDU_G_v2i32_ELE_areg32;
break;
case MVT::i64:
Opcode = NVPTX::INT_PTX_LDU_G_v2i64_ELE_areg32;
break;
case MVT::f32:
Opcode = NVPTX::INT_PTX_LDU_G_v2f32_ELE_areg32;
break;
case MVT::f64:
Opcode = NVPTX::INT_PTX_LDU_G_v2f64_ELE_areg32;
break;
}
break; break;
case NVPTXISD::LoadV4: case NVPTXISD::LoadV4:
case NVPTXISD::LDGV4: case NVPTXISD::LDGV4:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32,
return false; NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32,
case MVT::i8: NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg32, None,
Opcode = NVPTX::INT_PTX_LDG_G_v4i8_ELE_areg32; NVPTX::INT_PTX_LDG_G_v4f16_ELE_areg32,
break; NVPTX::INT_PTX_LDG_G_v4f16x2_ELE_areg32,
case MVT::i16: NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg32, None);
Opcode = NVPTX::INT_PTX_LDG_G_v4i16_ELE_areg32;
break;
case MVT::i32:
Opcode = NVPTX::INT_PTX_LDG_G_v4i32_ELE_areg32;
break;
case MVT::f32:
Opcode = NVPTX::INT_PTX_LDG_G_v4f32_ELE_areg32;
break;
}
break; break;
case NVPTXISD::LDUV4: case NVPTXISD::LDUV4:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32,
return false; NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32,
case MVT::i8: NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg32, None,
Opcode = NVPTX::INT_PTX_LDU_G_v4i8_ELE_areg32; NVPTX::INT_PTX_LDU_G_v4f16_ELE_areg32,
break; NVPTX::INT_PTX_LDU_G_v4f16x2_ELE_areg32,
case MVT::i16: NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg32, None);
Opcode = NVPTX::INT_PTX_LDU_G_v4i16_ELE_areg32;
break;
case MVT::i32:
Opcode = NVPTX::INT_PTX_LDU_G_v4i32_ELE_areg32;
break;
case MVT::f32:
Opcode = NVPTX::INT_PTX_LDU_G_v4f32_ELE_areg32;
break;
}
break; break;
} }
} }
if (!Opcode)
return false;
SDValue Ops[] = { Op1, Chain }; SDValue Ops[] = { Op1, Chain };
LD = CurDAG->getMachineNode(Opcode, DL, InstVTList, Ops); LD = CurDAG->getMachineNode(Opcode.getValue(), DL, InstVTList, Ops);
} }
MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
MemRefs0[0] = Mem->getMemOperand(); MemRefs0[0] = Mem->getMemOperand();
cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1); cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1);
// For automatic generation of LDG (through SelectLoad[Vector], not the // For automatic generation of LDG (through SelectLoad[Vector], not the
// intrinsics), we may have an extending load like: // intrinsics), we may have an extending load like:
▲ Show 20 Lines • Show All 83 Lines • ▼ Show 20 Lines else
toType = NVPTX::PTXLdStInstCode::Unsigned; toType = NVPTX::PTXLdStInstCode::Unsigned;
// Create the machine instruction DAG // Create the machine instruction DAG
SDValue Chain = N->getOperand(0); SDValue Chain = N->getOperand(0);
SDValue N1 = N->getOperand(1); SDValue N1 = N->getOperand(1);
SDValue N2 = N->getOperand(2); SDValue N2 = N->getOperand(2);
SDValue Addr; SDValue Addr;
SDValue Offset, Base; SDValue Offset, Base;
unsigned Opcode; Optional<unsigned> Opcode;
MVT::SimpleValueType SourceVT = N1.getNode()->getSimpleValueType(0).SimpleTy; MVT::SimpleValueType SourceVT = N1.getNode()->getSimpleValueType(0).SimpleTy;
if (SelectDirectAddr(N2, Addr)) { if (SelectDirectAddr(N2, Addr)) {
switch (SourceVT) { Opcode = pickOpcodeForVT(SourceVT, NVPTX::ST_i8_avar, NVPTX::ST_i16_avar,
case MVT::i8: NVPTX::ST_i32_avar, NVPTX::ST_i64_avar,
Opcode = NVPTX::ST_i8_avar; NVPTX::ST_f16_avar, NVPTX::ST_f16x2_avar,
break; NVPTX::ST_f32_avar, NVPTX::ST_f64_avar);
case MVT::i16: if (!Opcode)
Opcode = NVPTX::ST_i16_avar;
break;
case MVT::i32:
Opcode = NVPTX::ST_i32_avar;
break;
case MVT::i64:
Opcode = NVPTX::ST_i64_avar;
break;
case MVT::f16:
Opcode = NVPTX::ST_f16_avar;
break;
case MVT::v2f16:
Opcode = NVPTX::ST_f16x2_avar;
break;
case MVT::f32:
Opcode = NVPTX::ST_f32_avar;
break;
case MVT::f64:
Opcode = NVPTX::ST_f64_avar;
break;
default:
return false; return false;
}
SDValue Ops[] = { N1, getI32Imm(isVolatile, dl), SDValue Ops[] = { N1, getI32Imm(isVolatile, dl),
getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl), getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl),
getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), Addr, getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), Addr,
Chain }; Chain };
NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); NVPTXST = CurDAG->getMachineNode(Opcode.getValue(), dl, MVT::Other, Ops);
} else if (TM.is64Bit() ? SelectADDRsi64(N2.getNode(), N2, Base, Offset) } else if (TM.is64Bit() ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
: SelectADDRsi(N2.getNode(), N2, Base, Offset)) { : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
switch (SourceVT) { Opcode = pickOpcodeForVT(SourceVT, NVPTX::ST_i8_asi, NVPTX::ST_i16_asi,
case MVT::i8: NVPTX::ST_i32_asi, NVPTX::ST_i64_asi,
Opcode = NVPTX::ST_i8_asi; NVPTX::ST_f16_asi, NVPTX::ST_f16x2_asi,
break; NVPTX::ST_f32_asi, NVPTX::ST_f64_asi);
case MVT::i16: if (!Opcode)
Opcode = NVPTX::ST_i16_asi;
break;
case MVT::i32:
Opcode = NVPTX::ST_i32_asi;
break;
case MVT::i64:
Opcode = NVPTX::ST_i64_asi;
break;
case MVT::f16:
Opcode = NVPTX::ST_f16_asi;
break;
case MVT::v2f16:
Opcode = NVPTX::ST_f16x2_asi;
break;
case MVT::f32:
Opcode = NVPTX::ST_f32_asi;
break;
case MVT::f64:
Opcode = NVPTX::ST_f64_asi;
break;
default:
return false; return false;
}
SDValue Ops[] = { N1, getI32Imm(isVolatile, dl), SDValue Ops[] = { N1, getI32Imm(isVolatile, dl),
getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl), getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl),
getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), Base, getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), Base,
Offset, Chain }; Offset, Chain };
NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); NVPTXST = CurDAG->getMachineNode(Opcode.getValue(), dl, MVT::Other, Ops);
} else if (TM.is64Bit() ? SelectADDRri64(N2.getNode(), N2, Base, Offset) } else if (TM.is64Bit() ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
: SelectADDRri(N2.getNode(), N2, Base, Offset)) { : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
if (TM.is64Bit()) { if (TM.is64Bit())
switch (SourceVT) { Opcode = pickOpcodeForVT(
case MVT::i8: SourceVT, NVPTX::ST_i8_ari_64, NVPTX::ST_i16_ari_64,
Opcode = NVPTX::ST_i8_ari_64; NVPTX::ST_i32_ari_64, NVPTX::ST_i64_ari_64, NVPTX::ST_f16_ari_64,
break; NVPTX::ST_f16x2_ari_64, NVPTX::ST_f32_ari_64, NVPTX::ST_f64_ari_64);
case MVT::i16: else
Opcode = NVPTX::ST_i16_ari_64; Opcode = pickOpcodeForVT(SourceVT, NVPTX::ST_i8_ari, NVPTX::ST_i16_ari,
break; NVPTX::ST_i32_ari, NVPTX::ST_i64_ari,
case MVT::i32: NVPTX::ST_f16_ari, NVPTX::ST_f16x2_ari,
Opcode = NVPTX::ST_i32_ari_64; NVPTX::ST_f32_ari, NVPTX::ST_f64_ari);
break; if (!Opcode)
case MVT::i64:
Opcode = NVPTX::ST_i64_ari_64;
break;
case MVT::f16:
Opcode = NVPTX::ST_f16_ari_64;
break;
case MVT::v2f16:
Opcode = NVPTX::ST_f16x2_ari_64;
break;
case MVT::f32:
Opcode = NVPTX::ST_f32_ari_64;
break;
case MVT::f64:
Opcode = NVPTX::ST_f64_ari_64;
break;
default:
return false;
}
} else {
switch (SourceVT) {
case MVT::i8:
Opcode = NVPTX::ST_i8_ari;
break;
case MVT::i16:
Opcode = NVPTX::ST_i16_ari;
break;
case MVT::i32:
Opcode = NVPTX::ST_i32_ari;
break;
case MVT::i64:
Opcode = NVPTX::ST_i64_ari;
break;
case MVT::f16:
Opcode = NVPTX::ST_f16_ari;
break;
case MVT::v2f16:
Opcode = NVPTX::ST_f16x2_ari;
break;
case MVT::f32:
Opcode = NVPTX::ST_f32_ari;
break;
case MVT::f64:
Opcode = NVPTX::ST_f64_ari;
break;
default:
return false; return false;
}
}
SDValue Ops[] = { N1, getI32Imm(isVolatile, dl), SDValue Ops[] = { N1, getI32Imm(isVolatile, dl),
getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl), getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl),
getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), Base, getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), Base,
Offset, Chain }; Offset, Chain };
NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); NVPTXST = CurDAG->getMachineNode(Opcode.getValue(), dl, MVT::Other, Ops);
} else {
if (TM.is64Bit()) {
switch (SourceVT) {
case MVT::i8:
Opcode = NVPTX::ST_i8_areg_64;
break;
case MVT::i16:
Opcode = NVPTX::ST_i16_areg_64;
break;
case MVT::i32:
Opcode = NVPTX::ST_i32_areg_64;
break;
case MVT::i64:
Opcode = NVPTX::ST_i64_areg_64;
break;
case MVT::f16:
Opcode = NVPTX::ST_f16_areg_64;
break;
case MVT::v2f16:
Opcode = NVPTX::ST_f16x2_areg_64;
break;
case MVT::f32:
Opcode = NVPTX::ST_f32_areg_64;
break;
case MVT::f64:
Opcode = NVPTX::ST_f64_areg_64;
break;
default:
return false;
}
} else { } else {
switch (SourceVT) { if (TM.is64Bit())
case MVT::i8: Opcode =
Opcode = NVPTX::ST_i8_areg; pickOpcodeForVT(SourceVT, NVPTX::ST_i8_areg_64, NVPTX::ST_i16_areg_64,
break; NVPTX::ST_i32_areg_64, NVPTX::ST_i64_areg_64,
case MVT::i16: NVPTX::ST_f16_areg_64, NVPTX::ST_f16x2_areg_64,
Opcode = NVPTX::ST_i16_areg; NVPTX::ST_f32_areg_64, NVPTX::ST_f64_areg_64);
break; else
case MVT::i32: Opcode = pickOpcodeForVT(SourceVT, NVPTX::ST_i8_areg, NVPTX::ST_i16_areg,
Opcode = NVPTX::ST_i32_areg; NVPTX::ST_i32_areg, NVPTX::ST_i64_areg,
break; NVPTX::ST_f16_areg, NVPTX::ST_f16x2_areg,
case MVT::i64: NVPTX::ST_f32_areg, NVPTX::ST_f64_areg);
Opcode = NVPTX::ST_i64_areg; if (!Opcode)
break;
case MVT::f16:
Opcode = NVPTX::ST_f16_areg;
break;
case MVT::v2f16:
Opcode = NVPTX::ST_f16x2_areg;
break;
case MVT::f32:
Opcode = NVPTX::ST_f32_areg;
break;
case MVT::f64:
Opcode = NVPTX::ST_f64_areg;
break;
default:
return false; return false;
}
}
SDValue Ops[] = { N1, getI32Imm(isVolatile, dl), SDValue Ops[] = { N1, getI32Imm(isVolatile, dl),
getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl), getI32Imm(codeAddrSpace, dl), getI32Imm(vecType, dl),
getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), N2, getI32Imm(toType, dl), getI32Imm(toTypeWidth, dl), N2,
Chain }; Chain };
NVPTXST = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); NVPTXST = CurDAG->getMachineNode(Opcode.getValue(), dl, MVT::Other, Ops);
} }
if (!NVPTXST) if (!NVPTXST)
return false; return false;
MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1); cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1);
ReplaceNode(N, NVPTXST); ReplaceNode(N, NVPTXST);
return true; return true;
} }
bool NVPTXDAGToDAGISel::tryStoreVector(SDNode *N) { bool NVPTXDAGToDAGISel::tryStoreVector(SDNode *N) {
SDValue Chain = N->getOperand(0); SDValue Chain = N->getOperand(0);
SDValue Op1 = N->getOperand(1); SDValue Op1 = N->getOperand(1);
SDValue Addr, Offset, Base; SDValue Addr, Offset, Base;
unsigned Opcode; Optional<unsigned> Opcode;
SDLoc DL(N); SDLoc DL(N);
SDNode *ST; SDNode *ST;
EVT EltVT = Op1.getValueType(); EVT EltVT = Op1.getValueType();
MemSDNode *MemSD = cast<MemSDNode>(N); MemSDNode *MemSD = cast<MemSDNode>(N);
EVT StoreVT = MemSD->getMemoryVT(); EVT StoreVT = MemSD->getMemoryVT();
// Address Space Setting // Address Space Setting
unsigned CodeAddrSpace = getCodeAddrSpace(MemSD); unsigned CodeAddrSpace = getCodeAddrSpace(MemSD);
▲ Show 20 Lines • Show All 62 Lines • ▼ Show 20 Linesbool NVPTXDAGToDAGISel::tryStoreVector(SDNode *N) {
StOps.push_back(getI32Imm(ToType, DL)); StOps.push_back(getI32Imm(ToType, DL));
StOps.push_back(getI32Imm(ToTypeWidth, DL)); StOps.push_back(getI32Imm(ToTypeWidth, DL));
if (SelectDirectAddr(N2, Addr)) { if (SelectDirectAddr(N2, Addr)) {
switch (N->getOpcode()) { switch (N->getOpcode()) {
default: default:
return false; return false;
case NVPTXISD::StoreV2: case NVPTXISD::StoreV2:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::STV_i8_v2_avar, NVPTX::STV_i16_v2_avar,
return false; NVPTX::STV_i32_v2_avar, NVPTX::STV_i64_v2_avar,
case MVT::i8: NVPTX::STV_f16_v2_avar, NVPTX::STV_f16x2_v2_avar,
Opcode = NVPTX::STV_i8_v2_avar; NVPTX::STV_f32_v2_avar, NVPTX::STV_f64_v2_avar);
break;
case MVT::i16:
Opcode = NVPTX::STV_i16_v2_avar;
break;
case MVT::i32:
Opcode = NVPTX::STV_i32_v2_avar;
break;
case MVT::i64:
Opcode = NVPTX::STV_i64_v2_avar;
break;
case MVT::f16:
Opcode = NVPTX::STV_f16_v2_avar;
break;
case MVT::f32:
Opcode = NVPTX::STV_f32_v2_avar;
break;
case MVT::f64:
Opcode = NVPTX::STV_f64_v2_avar;
break;
}
break; break;
case NVPTXISD::StoreV4: case NVPTXISD::StoreV4:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode =
default: pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy, NVPTX::STV_i8_v4_avar,
return false; NVPTX::STV_i16_v4_avar, NVPTX::STV_i32_v4_avar, None,
case MVT::i8: NVPTX::STV_f16_v4_avar, NVPTX::STV_f16x2_v4_avar,
Opcode = NVPTX::STV_i8_v4_avar; NVPTX::STV_f32_v4_avar, None);
break;
case MVT::i16:
Opcode = NVPTX::STV_i16_v4_avar;
break;
case MVT::i32:
Opcode = NVPTX::STV_i32_v4_avar;
break;
case MVT::f32:
Opcode = NVPTX::STV_f32_v4_avar;
break;
}
break; break;
} }
StOps.push_back(Addr); StOps.push_back(Addr);
} else if (TM.is64Bit() ? SelectADDRsi64(N2.getNode(), N2, Base, Offset) } else if (TM.is64Bit() ? SelectADDRsi64(N2.getNode(), N2, Base, Offset)
: SelectADDRsi(N2.getNode(), N2, Base, Offset)) { : SelectADDRsi(N2.getNode(), N2, Base, Offset)) {
switch (N->getOpcode()) { switch (N->getOpcode()) {
default: default:
return false; return false;
case NVPTXISD::StoreV2: case NVPTXISD::StoreV2:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::STV_i8_v2_asi, NVPTX::STV_i16_v2_asi,
return false; NVPTX::STV_i32_v2_asi, NVPTX::STV_i64_v2_asi,
case MVT::i8: NVPTX::STV_f16_v2_asi, NVPTX::STV_f16x2_v2_asi,
Opcode = NVPTX::STV_i8_v2_asi; NVPTX::STV_f32_v2_asi, NVPTX::STV_f64_v2_asi);
break;
case MVT::i16:
Opcode = NVPTX::STV_i16_v2_asi;
break;
case MVT::i32:
Opcode = NVPTX::STV_i32_v2_asi;
break;
case MVT::i64:
Opcode = NVPTX::STV_i64_v2_asi;
break;
case MVT::f16:
Opcode = NVPTX::STV_f16_v2_asi;
break;
case MVT::f32:
Opcode = NVPTX::STV_f32_v2_asi;
break;
case MVT::f64:
Opcode = NVPTX::STV_f64_v2_asi;
break;
}
break; break;
case NVPTXISD::StoreV4: case NVPTXISD::StoreV4:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode =
default: pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy, NVPTX::STV_i8_v4_asi,
return false; NVPTX::STV_i16_v4_asi, NVPTX::STV_i32_v4_asi, None,
case MVT::i8: NVPTX::STV_f16_v4_asi, NVPTX::STV_f16x2_v4_asi,
Opcode = NVPTX::STV_i8_v4_asi; NVPTX::STV_f32_v4_asi, None);
break;
case MVT::i16:
Opcode = NVPTX::STV_i16_v4_asi;
break;
case MVT::i32:
Opcode = NVPTX::STV_i32_v4_asi;
break;
case MVT::f16:
Opcode = NVPTX::STV_f16_v4_asi;
break;
case MVT::f32:
Opcode = NVPTX::STV_f32_v4_asi;
break;
}
break; break;
} }
StOps.push_back(Base); StOps.push_back(Base);
StOps.push_back(Offset); StOps.push_back(Offset);
} else if (TM.is64Bit() ? SelectADDRri64(N2.getNode(), N2, Base, Offset) } else if (TM.is64Bit() ? SelectADDRri64(N2.getNode(), N2, Base, Offset)
: SelectADDRri(N2.getNode(), N2, Base, Offset)) { : SelectADDRri(N2.getNode(), N2, Base, Offset)) {
if (TM.is64Bit()) { if (TM.is64Bit()) {
switch (N->getOpcode()) { switch (N->getOpcode()) {
default: default:
return false; return false;
case NVPTXISD::StoreV2: case NVPTXISD::StoreV2:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(
default: EltVT.getSimpleVT().SimpleTy, NVPTX::STV_i8_v2_ari_64,
return false; NVPTX::STV_i16_v2_ari_64, NVPTX::STV_i32_v2_ari_64,
case MVT::i8: NVPTX::STV_i64_v2_ari_64, NVPTX::STV_f16_v2_ari_64,
Opcode = NVPTX::STV_i8_v2_ari_64; NVPTX::STV_f16x2_v2_ari_64, NVPTX::STV_f32_v2_ari_64,
break; NVPTX::STV_f64_v2_ari_64);
case MVT::i16:
Opcode = NVPTX::STV_i16_v2_ari_64;
break;
case MVT::i32:
Opcode = NVPTX::STV_i32_v2_ari_64;
break;
case MVT::i64:
Opcode = NVPTX::STV_i64_v2_ari_64;
break;
case MVT::f16:
Opcode = NVPTX::STV_f16_v2_ari_64;
break;
case MVT::f32:
Opcode = NVPTX::STV_f32_v2_ari_64;
break;
case MVT::f64:
Opcode = NVPTX::STV_f64_v2_ari_64;
break;
}
break; break;
case NVPTXISD::StoreV4: case NVPTXISD::StoreV4:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(
default: EltVT.getSimpleVT().SimpleTy, NVPTX::STV_i8_v4_ari_64,
return false; NVPTX::STV_i16_v4_ari_64, NVPTX::STV_i32_v4_ari_64, None,
case MVT::i8: NVPTX::STV_f16_v4_ari_64, NVPTX::STV_f16x2_v4_ari_64,
Opcode = NVPTX::STV_i8_v4_ari_64; NVPTX::STV_f32_v4_ari_64, None);
break;
case MVT::i16:
Opcode = NVPTX::STV_i16_v4_ari_64;
break;
case MVT::i32:
Opcode = NVPTX::STV_i32_v4_ari_64;
break;
case MVT::f16:
Opcode = NVPTX::STV_f16_v4_ari_64;
break;
case MVT::f32:
Opcode = NVPTX::STV_f32_v4_ari_64;
break;
}
break; break;
} }
} else { } else {
switch (N->getOpcode()) { switch (N->getOpcode()) {
default: default:
return false; return false;
case NVPTXISD::StoreV2: case NVPTXISD::StoreV2:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy,
default: NVPTX::STV_i8_v2_ari, NVPTX::STV_i16_v2_ari,
return false; NVPTX::STV_i32_v2_ari, NVPTX::STV_i64_v2_ari,
case MVT::i8: NVPTX::STV_f16_v2_ari, NVPTX::STV_f16x2_v2_ari,
Opcode = NVPTX::STV_i8_v2_ari; NVPTX::STV_f32_v2_ari, NVPTX::STV_f64_v2_ari);
break;
case MVT::i16:
Opcode = NVPTX::STV_i16_v2_ari;
break;
case MVT::i32:
Opcode = NVPTX::STV_i32_v2_ari;
break;
case MVT::i64:
Opcode = NVPTX::STV_i64_v2_ari;
break;
case MVT::f16:
Opcode = NVPTX::STV_f16_v2_ari;
break;
case MVT::f32:
Opcode = NVPTX::STV_f32_v2_ari;
break;
case MVT::f64:
Opcode = NVPTX::STV_f64_v2_ari;
break;
}
break; break;
case NVPTXISD::StoreV4: case NVPTXISD::StoreV4:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode =
default: pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy, NVPTX::STV_i8_v4_ari,
return false; NVPTX::STV_i16_v4_ari, NVPTX::STV_i32_v4_ari, None,
case MVT::i8: NVPTX::STV_f16_v4_ari, NVPTX::STV_f16x2_v4_ari,
Opcode = NVPTX::STV_i8_v4_ari; NVPTX::STV_f32_v4_ari, None);
break;
case MVT::i16:
Opcode = NVPTX::STV_i16_v4_ari;
break;
case MVT::i32:
Opcode = NVPTX::STV_i32_v4_ari;
break;
case MVT::f16:
Opcode = NVPTX::STV_f16_v4_ari;
break;
case MVT::f32:
Opcode = NVPTX::STV_f32_v4_ari;
break;
}
break; break;
} }
} }
StOps.push_back(Base); StOps.push_back(Base);
StOps.push_back(Offset); StOps.push_back(Offset);
} else { } else {
if (TM.is64Bit()) { if (TM.is64Bit()) {
switch (N->getOpcode()) { switch (N->getOpcode()) {
default: default:
return false; return false;
case NVPTXISD::StoreV2: case NVPTXISD::StoreV2:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(
default: EltVT.getSimpleVT().SimpleTy, NVPTX::STV_i8_v2_areg_64,
return false; NVPTX::STV_i16_v2_areg_64, NVPTX::STV_i32_v2_areg_64,
case MVT::i8: NVPTX::STV_i64_v2_areg_64, NVPTX::STV_f16_v2_areg_64,
Opcode = NVPTX::STV_i8_v2_areg_64; NVPTX::STV_f16x2_v2_areg_64, NVPTX::STV_f32_v2_areg_64,
break; NVPTX::STV_f64_v2_areg_64);
case MVT::i16:
Opcode = NVPTX::STV_i16_v2_areg_64;
break;
case MVT::i32:
Opcode = NVPTX::STV_i32_v2_areg_64;
break;
case MVT::i64:
Opcode = NVPTX::STV_i64_v2_areg_64;
break;
case MVT::f16:
Opcode = NVPTX::STV_f16_v2_areg_64;
break;
case MVT::f32:
Opcode = NVPTX::STV_f32_v2_areg_64;
break;
case MVT::f64:
Opcode = NVPTX::STV_f64_v2_areg_64;
break;
}
break; break;
case NVPTXISD::StoreV4: case NVPTXISD::StoreV4:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(
default: EltVT.getSimpleVT().SimpleTy, NVPTX::STV_i8_v4_areg_64,
return false; NVPTX::STV_i16_v4_areg_64, NVPTX::STV_i32_v4_areg_64, None,
case MVT::i8: NVPTX::STV_f16_v4_areg_64, NVPTX::STV_f16x2_v4_areg_64,
Opcode = NVPTX::STV_i8_v4_areg_64; NVPTX::STV_f32_v4_areg_64, None);
break;
case MVT::i16:
Opcode = NVPTX::STV_i16_v4_areg_64;
break;
case MVT::i32:
Opcode = NVPTX::STV_i32_v4_areg_64;
break;
case MVT::f16:
Opcode = NVPTX::STV_f16_v4_areg_64;
break;
case MVT::f32:
Opcode = NVPTX::STV_f32_v4_areg_64;
break;
}
break; break;
} }
} else { } else {
switch (N->getOpcode()) { switch (N->getOpcode()) {
default: default:
return false; return false;
case NVPTXISD::StoreV2: case NVPTXISD::StoreV2:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode =
default: pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy, NVPTX::STV_i8_v2_areg,
return false; NVPTX::STV_i16_v2_areg, NVPTX::STV_i32_v2_areg,
case MVT::i8: NVPTX::STV_i64_v2_areg, NVPTX::STV_f16_v2_areg,
Opcode = NVPTX::STV_i8_v2_areg; NVPTX::STV_f16x2_v2_areg, NVPTX::STV_f32_v2_areg,
break; NVPTX::STV_f64_v2_areg);
case MVT::i16:
Opcode = NVPTX::STV_i16_v2_areg;
break;
case MVT::i32:
Opcode = NVPTX::STV_i32_v2_areg;
break;
case MVT::i64:
Opcode = NVPTX::STV_i64_v2_areg;
break;
case MVT::f16:
Opcode = NVPTX::STV_f16_v2_areg;
break;
case MVT::f32:
Opcode = NVPTX::STV_f32_v2_areg;
break;
case MVT::f64:
Opcode = NVPTX::STV_f64_v2_areg;
break;
}
break; break;
case NVPTXISD::StoreV4: case NVPTXISD::StoreV4:
switch (EltVT.getSimpleVT().SimpleTy) { Opcode =
default: pickOpcodeForVT(EltVT.getSimpleVT().SimpleTy, NVPTX::STV_i8_v4_areg,
return false; NVPTX::STV_i16_v4_areg, NVPTX::STV_i32_v4_areg, None,
case MVT::i8: NVPTX::STV_f16_v4_areg, NVPTX::STV_f16x2_v4_areg,
Opcode = NVPTX::STV_i8_v4_areg; NVPTX::STV_f32_v4_areg, None);
break;
case MVT::i16:
Opcode = NVPTX::STV_i16_v4_areg;
break;
case MVT::i32:
Opcode = NVPTX::STV_i32_v4_areg;
break;
case MVT::f16:
Opcode = NVPTX::STV_f16_v4_areg;
break;
case MVT::f32:
Opcode = NVPTX::STV_f32_v4_areg;
break;
}
break; break;
} }
} }
StOps.push_back(N2); StOps.push_back(N2);
} }
if (!Opcode)
return false;
StOps.push_back(Chain); StOps.push_back(Chain);
ST = CurDAG->getMachineNode(Opcode, DL, MVT::Other, StOps); ST = CurDAG->getMachineNode(Opcode.getValue(), DL, MVT::Other, StOps);
MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
cast<MachineSDNode>(ST)->setMemRefs(MemRefs0, MemRefs0 + 1); cast<MachineSDNode>(ST)->setMemRefs(MemRefs0, MemRefs0 + 1);
ReplaceNode(N, ST); ReplaceNode(N, ST);
return true; return true;
} }
Show All 18 Linesbool NVPTXDAGToDAGISel::tryLoadParam(SDNode *Node) {
case NVPTXISD::LoadParamV4: case NVPTXISD::LoadParamV4:
VecSize = 4; VecSize = 4;
break; break;
} }
EVT EltVT = Node->getValueType(0); EVT EltVT = Node->getValueType(0);
EVT MemVT = Mem->getMemoryVT(); EVT MemVT = Mem->getMemoryVT();
unsigned Opc = 0; Optional<unsigned> Opcode;
switch (VecSize) { switch (VecSize) {
default: default:
return false; return false;
case 1: case 1:
switch (MemVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(MemVT.getSimpleVT().SimpleTy,
default: NVPTX::LoadParamMemI8, NVPTX::LoadParamMemI16,
return false; NVPTX::LoadParamMemI32, NVPTX::LoadParamMemI64,
case MVT::i1: NVPTX::LoadParamMemF16, NVPTX::LoadParamMemF16x2,
Opc = NVPTX::LoadParamMemI8; NVPTX::LoadParamMemF32, NVPTX::LoadParamMemF64);
break;
case MVT::i8:
Opc = NVPTX::LoadParamMemI8;
break;
case MVT::i16:
Opc = NVPTX::LoadParamMemI16;
break;
case MVT::i32:
Opc = NVPTX::LoadParamMemI32;
break;
case MVT::i64:
Opc = NVPTX::LoadParamMemI64;
break;
case MVT::f16:
Opc = NVPTX::LoadParamMemF16;
break;
case MVT::v2f16:
Opc = NVPTX::LoadParamMemF16x2;
break;
case MVT::f32:
Opc = NVPTX::LoadParamMemF32;
break;
case MVT::f64:
Opc = NVPTX::LoadParamMemF64;
break;
}
break; break;
case 2: case 2:
switch (MemVT.getSimpleVT().SimpleTy) { Opcode =
default: pickOpcodeForVT(MemVT.getSimpleVT().SimpleTy, NVPTX::LoadParamMemV2I8,
return false; NVPTX::LoadParamMemV2I16, NVPTX::LoadParamMemV2I32,
case MVT::i1: NVPTX::LoadParamMemV2I64, NVPTX::LoadParamMemV2F16,
Opc = NVPTX::LoadParamMemV2I8; NVPTX::LoadParamMemV2F16x2, NVPTX::LoadParamMemV2F32,
break; NVPTX::LoadParamMemV2F64);
case MVT::i8:
Opc = NVPTX::LoadParamMemV2I8;
break;
case MVT::i16:
Opc = NVPTX::LoadParamMemV2I16;
break;
case MVT::i32:
Opc = NVPTX::LoadParamMemV2I32;
break;
case MVT::i64:
Opc = NVPTX::LoadParamMemV2I64;
break;
case MVT::f16:
Opc = NVPTX::LoadParamMemV2F16;
break;
case MVT::v2f16:
Opc = NVPTX::LoadParamMemV2F16x2;
break;
case MVT::f32:
Opc = NVPTX::LoadParamMemV2F32;
break;
case MVT::f64:
Opc = NVPTX::LoadParamMemV2F64;
break;
}
break; break;
case 4: case 4:
switch (MemVT.getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(
default: MemVT.getSimpleVT().SimpleTy, NVPTX::LoadParamMemV4I8,
return false; NVPTX::LoadParamMemV4I16, NVPTX::LoadParamMemV4I32, None,
case MVT::i1: NVPTX::LoadParamMemV4F16, NVPTX::LoadParamMemV4F16x2,
Opc = NVPTX::LoadParamMemV4I8; NVPTX::LoadParamMemV4F32, None);
break;
case MVT::i8:
Opc = NVPTX::LoadParamMemV4I8;
break;
case MVT::i16:
Opc = NVPTX::LoadParamMemV4I16;
break;
case MVT::i32:
Opc = NVPTX::LoadParamMemV4I32;
break;
case MVT::f16:
Opc = NVPTX::LoadParamMemV4F16;
break;
case MVT::v2f16:
Opc = NVPTX::LoadParamMemV4F16x2;
break;
case MVT::f32:
Opc = NVPTX::LoadParamMemV4F32;
break;
}
break; break;
} }
if (!Opcode)
return false;
SDVTList VTs; SDVTList VTs;
if (VecSize == 1) { if (VecSize == 1) {
VTs = CurDAG->getVTList(EltVT, MVT::Other, MVT::Glue); VTs = CurDAG->getVTList(EltVT, MVT::Other, MVT::Glue);
} else if (VecSize == 2) { } else if (VecSize == 2) {
VTs = CurDAG->getVTList(EltVT, EltVT, MVT::Other, MVT::Glue); VTs = CurDAG->getVTList(EltVT, EltVT, MVT::Other, MVT::Glue);
} else { } else {
EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue }; EVT EVTs[] = { EltVT, EltVT, EltVT, EltVT, MVT::Other, MVT::Glue };
VTs = CurDAG->getVTList(EVTs); VTs = CurDAG->getVTList(EVTs);
} }
unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue(); unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
SmallVector<SDValue, 2> Ops; SmallVector<SDValue, 2> Ops;
Ops.push_back(CurDAG->getTargetConstant(OffsetVal, DL, MVT::i32)); Ops.push_back(CurDAG->getTargetConstant(OffsetVal, DL, MVT::i32));
Ops.push_back(Chain); Ops.push_back(Chain);
Ops.push_back(Flag); Ops.push_back(Flag);
ReplaceNode(Node, CurDAG->getMachineNode(Opc, DL, VTs, Ops)); ReplaceNode(Node, CurDAG->getMachineNode(Opcode.getValue(), DL, VTs, Ops));
return true; return true;
} }
bool NVPTXDAGToDAGISel::tryStoreRetval(SDNode *N) { bool NVPTXDAGToDAGISel::tryStoreRetval(SDNode *N) {
SDLoc DL(N); SDLoc DL(N);
SDValue Chain = N->getOperand(0); SDValue Chain = N->getOperand(0);
SDValue Offset = N->getOperand(1); SDValue Offset = N->getOperand(1);
unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue(); unsigned OffsetVal = cast<ConstantSDNode>(Offset)->getZExtValue();
Show All 20 Linesbool NVPTXDAGToDAGISel::tryStoreRetval(SDNode *N) {
for (unsigned i = 0; i < NumElts; ++i) for (unsigned i = 0; i < NumElts; ++i)
Ops.push_back(N->getOperand(i + 2)); Ops.push_back(N->getOperand(i + 2));
Ops.push_back(CurDAG->getTargetConstant(OffsetVal, DL, MVT::i32)); Ops.push_back(CurDAG->getTargetConstant(OffsetVal, DL, MVT::i32));
Ops.push_back(Chain); Ops.push_back(Chain);
// Determine target opcode // Determine target opcode
// If we have an i1, use an 8-bit store. The lowering code in // If we have an i1, use an 8-bit store. The lowering code in
// NVPTXISelLowering will have already emitted an upcast. // NVPTXISelLowering will have already emitted an upcast.
unsigned Opcode = 0; Optional<unsigned> Opcode = 0;
switch (NumElts) { switch (NumElts) {
default: default:
return false; return false;
case 1: case 1:
switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(Mem->getMemoryVT().getSimpleVT().SimpleTy,
default: NVPTX::StoreRetvalI8, NVPTX::StoreRetvalI16,
return false; NVPTX::StoreRetvalI32, NVPTX::StoreRetvalI64,
case MVT::i1: NVPTX::StoreRetvalF16, NVPTX::StoreRetvalF16x2,
Opcode = NVPTX::StoreRetvalI8; NVPTX::StoreRetvalF32, NVPTX::StoreRetvalF64);
break;
case MVT::i8:
Opcode = NVPTX::StoreRetvalI8;
break;
case MVT::i16:
Opcode = NVPTX::StoreRetvalI16;
break;
case MVT::i32:
Opcode = NVPTX::StoreRetvalI32;
break;
case MVT::i64:
Opcode = NVPTX::StoreRetvalI64;
break;
case MVT::f16:
Opcode = NVPTX::StoreRetvalF16;
break;
case MVT::v2f16:
Opcode = NVPTX::StoreRetvalF16x2;
break;
case MVT::f32:
Opcode = NVPTX::StoreRetvalF32;
break;
case MVT::f64:
Opcode = NVPTX::StoreRetvalF64;
break;
}
break; break;
case 2: case 2:
switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(Mem->getMemoryVT().getSimpleVT().SimpleTy,
default: NVPTX::StoreRetvalV2I8, NVPTX::StoreRetvalV2I16,
return false; NVPTX::StoreRetvalV2I32, NVPTX::StoreRetvalV2I64,
case MVT::i1: NVPTX::StoreRetvalV2F16, NVPTX::StoreRetvalV2F16x2,
Opcode = NVPTX::StoreRetvalV2I8; NVPTX::StoreRetvalV2F32, NVPTX::StoreRetvalV2F64);
break;
case MVT::i8:
Opcode = NVPTX::StoreRetvalV2I8;
break;
case MVT::i16:
Opcode = NVPTX::StoreRetvalV2I16;
break;
case MVT::i32:
Opcode = NVPTX::StoreRetvalV2I32;
break;
case MVT::i64:
Opcode = NVPTX::StoreRetvalV2I64;
break;
case MVT::f16:
Opcode = NVPTX::StoreRetvalV2F16;
break;
case MVT::v2f16:
Opcode = NVPTX::StoreRetvalV2F16x2;
break;
case MVT::f32:
Opcode = NVPTX::StoreRetvalV2F32;
break;
case MVT::f64:
Opcode = NVPTX::StoreRetvalV2F64;
break;
}
break; break;
case 4: case 4:
switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(Mem->getMemoryVT().getSimpleVT().SimpleTy,
default: NVPTX::StoreRetvalV4I8, NVPTX::StoreRetvalV4I16,
return false; NVPTX::StoreRetvalV4I32, None,
case MVT::i1: NVPTX::StoreRetvalV4F16, NVPTX::StoreRetvalV4F16x2,
Opcode = NVPTX::StoreRetvalV4I8; NVPTX::StoreRetvalV4F32, None);
break;
case MVT::i8:
Opcode = NVPTX::StoreRetvalV4I8;
break;
case MVT::i16:
Opcode = NVPTX::StoreRetvalV4I16;
break;
case MVT::i32:
Opcode = NVPTX::StoreRetvalV4I32;
break;
case MVT::f16:
Opcode = NVPTX::StoreRetvalV4F16;
break;
case MVT::v2f16:
Opcode = NVPTX::StoreRetvalV4F16x2;
break;
case MVT::f32:
Opcode = NVPTX::StoreRetvalV4F32;
break;
}
break; break;
} }
if (!Opcode)
return false;
SDNode *Ret = CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops); SDNode *Ret = CurDAG->getMachineNode(Opcode.getValue(), DL, MVT::Other, Ops);
MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1); cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
ReplaceNode(N, Ret); ReplaceNode(N, Ret);
return true; return true;
} }
Show All 32 Linesbool NVPTXDAGToDAGISel::tryStoreParam(SDNode *N) {
Ops.push_back(CurDAG->getTargetConstant(ParamVal, DL, MVT::i32)); Ops.push_back(CurDAG->getTargetConstant(ParamVal, DL, MVT::i32));
Ops.push_back(CurDAG->getTargetConstant(OffsetVal, DL, MVT::i32)); Ops.push_back(CurDAG->getTargetConstant(OffsetVal, DL, MVT::i32));
Ops.push_back(Chain); Ops.push_back(Chain);
Ops.push_back(Flag); Ops.push_back(Flag);
// Determine target opcode // Determine target opcode
// If we have an i1, use an 8-bit store. The lowering code in // If we have an i1, use an 8-bit store. The lowering code in
// NVPTXISelLowering will have already emitted an upcast. // NVPTXISelLowering will have already emitted an upcast.
unsigned Opcode = 0; Optional<unsigned> Opcode = 0;
switch (N->getOpcode()) { switch (N->getOpcode()) {
default: default:
switch (NumElts) { switch (NumElts) {
default: default:
return false; return false;
case 1: case 1:
switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(Mem->getMemoryVT().getSimpleVT().SimpleTy,
default: NVPTX::StoreParamI8, NVPTX::StoreParamI16,
return false; NVPTX::StoreParamI32, NVPTX::StoreParamI64,
case MVT::i1: NVPTX::StoreParamF16, NVPTX::StoreParamF16x2,
Opcode = NVPTX::StoreParamI8; NVPTX::StoreParamF32, NVPTX::StoreParamF64);
break;
case MVT::i8:
Opcode = NVPTX::StoreParamI8;
break;
case MVT::i16:
Opcode = NVPTX::StoreParamI16;
break;
case MVT::i32:
Opcode = NVPTX::StoreParamI32;
break;
case MVT::i64:
Opcode = NVPTX::StoreParamI64;
break;
case MVT::f16:
Opcode = NVPTX::StoreParamF16;
break;
case MVT::v2f16:
Opcode = NVPTX::StoreParamF16x2;
break;
case MVT::f32:
Opcode = NVPTX::StoreParamF32;
break;
case MVT::f64:
Opcode = NVPTX::StoreParamF64;
break;
}
break; break;
case 2: case 2:
switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(Mem->getMemoryVT().getSimpleVT().SimpleTy,
default: NVPTX::StoreParamV2I8, NVPTX::StoreParamV2I16,
return false; NVPTX::StoreParamV2I32, NVPTX::StoreParamV2I64,
case MVT::i1: NVPTX::StoreParamV2F16, NVPTX::StoreParamV2F16x2,
Opcode = NVPTX::StoreParamV2I8; NVPTX::StoreParamV2F32, NVPTX::StoreParamV2F64);
break;
case MVT::i8:
Opcode = NVPTX::StoreParamV2I8;
break;
case MVT::i16:
Opcode = NVPTX::StoreParamV2I16;
break;
case MVT::i32:
Opcode = NVPTX::StoreParamV2I32;
break;
case MVT::i64:
Opcode = NVPTX::StoreParamV2I64;
break;
case MVT::f16:
Opcode = NVPTX::StoreParamV2F16;
break;
case MVT::v2f16:
Opcode = NVPTX::StoreParamV2F16x2;
break;
case MVT::f32:
Opcode = NVPTX::StoreParamV2F32;
break;
case MVT::f64:
Opcode = NVPTX::StoreParamV2F64;
break;
}
break; break;
case 4: case 4:
switch (Mem->getMemoryVT().getSimpleVT().SimpleTy) { Opcode = pickOpcodeForVT(Mem->getMemoryVT().getSimpleVT().SimpleTy,
default: NVPTX::StoreParamV4I8, NVPTX::StoreParamV4I16,
return false; NVPTX::StoreParamV4I32, None,
case MVT::i1: NVPTX::StoreParamV4F16, NVPTX::StoreParamV4F16x2,
Opcode = NVPTX::StoreParamV4I8; NVPTX::StoreParamV4F32, None);
break;
case MVT::i8:
Opcode = NVPTX::StoreParamV4I8;
break;
case MVT::i16:
Opcode = NVPTX::StoreParamV4I16;
break;
case MVT::i32:
Opcode = NVPTX::StoreParamV4I32;
break;
case MVT::f16:
Opcode = NVPTX::StoreParamV4F16;
break;
case MVT::v2f16:
Opcode = NVPTX::StoreParamV4F16x2;
break;
case MVT::f32:
Opcode = NVPTX::StoreParamV4F32;
break;
}
break; break;
} }
if (!Opcode)
return false;
break; break;
// Special case: if we have a sign-extend/zero-extend node, insert the // Special case: if we have a sign-extend/zero-extend node, insert the
// conversion instruction first, and use that as the value operand to // conversion instruction first, and use that as the value operand to
// the selected StoreParam node. // the selected StoreParam node.
case NVPTXISD::StoreParamU32: { case NVPTXISD::StoreParamU32: {
Opcode = NVPTX::StoreParamI32; Opcode = NVPTX::StoreParamI32;
SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, DL, SDValue CvtNone = CurDAG->getTargetConstant(NVPTX::PTXCvtMode::NONE, DL,
MVT::i32); MVT::i32);
Show All 10 Lines SDNode *Cvt = CurDAG->getMachineNode(NVPTX::CVT_s32_s16, DL,
MVT::i32, Ops[0], CvtNone); MVT::i32, Ops[0], CvtNone);
Ops[0] = SDValue(Cvt, 0); Ops[0] = SDValue(Cvt, 0);
break; break;
} }
} }
SDVTList RetVTs = CurDAG->getVTList(MVT::Other, MVT::Glue); SDVTList RetVTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
SDNode *Ret = SDNode *Ret =
CurDAG->getMachineNode(Opcode, DL, RetVTs, Ops); CurDAG->getMachineNode(Opcode.getValue(), DL, RetVTs, Ops);
MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1);
MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand();
cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1); cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1);
ReplaceNode(N, Ret); ReplaceNode(N, Ret);
return true; return true;
} }
▲ Show 20 Lines • Show All 2,116 Lines • Show Last 20 Lines