Changeset View
Changeset View
Standalone View
Standalone View
llvm/trunk/lib/Target/AMDGPU/SIFoldOperands.cpp
Show First 20 Lines • Show All 44 Lines • ▼ Show 20 Line(s) | 42 | FoldCandidate(MachineInstr *MI, unsigned OpNo, MachineOperand *FoldOp, | |||
---|---|---|---|---|---|
45 | UseMI(MI), OpToFold(nullptr), ShrinkOpcode(ShrinkOp), UseOpNo(OpNo), | 45 | UseMI(MI), OpToFold(nullptr), ShrinkOpcode(ShrinkOp), UseOpNo(OpNo), | ||
46 | Kind(FoldOp->getType()), | 46 | Kind(FoldOp->getType()), | ||
47 | Commuted(Commuted_) { | 47 | Commuted(Commuted_) { | ||
48 | if (FoldOp->isImm()) { | 48 | if (FoldOp->isImm()) { | ||
49 | ImmToFold = FoldOp->getImm(); | 49 | ImmToFold = FoldOp->getImm(); | ||
50 | } else if (FoldOp->isFI()) { | 50 | } else if (FoldOp->isFI()) { | ||
51 | FrameIndexToFold = FoldOp->getIndex(); | 51 | FrameIndexToFold = FoldOp->getIndex(); | ||
52 | } else { | 52 | } else { | ||
53 | assert(FoldOp->isReg()); | 53 | assert(FoldOp->isReg() || FoldOp->isGlobal()); | ||
54 | OpToFold = FoldOp; | 54 | OpToFold = FoldOp; | ||
55 | } | 55 | } | ||
56 | } | 56 | } | ||
57 | 57 | | |||
58 | bool isFI() const { | 58 | bool isFI() const { | ||
59 | return Kind == MachineOperand::MO_FrameIndex; | 59 | return Kind == MachineOperand::MO_FrameIndex; | ||
60 | } | 60 | } | ||
61 | 61 | | |||
62 | bool isImm() const { | 62 | bool isImm() const { | ||
63 | return Kind == MachineOperand::MO_Immediate; | 63 | return Kind == MachineOperand::MO_Immediate; | ||
64 | } | 64 | } | ||
65 | 65 | | |||
66 | bool isReg() const { | 66 | bool isReg() const { | ||
67 | return Kind == MachineOperand::MO_Register; | 67 | return Kind == MachineOperand::MO_Register; | ||
68 | } | 68 | } | ||
69 | 69 | | |||
70 | bool isGlobal() const { return Kind == MachineOperand::MO_GlobalAddress; } | ||||
71 | | ||||
70 | bool isCommuted() const { | 72 | bool isCommuted() const { | ||
71 | return Commuted; | 73 | return Commuted; | ||
72 | } | 74 | } | ||
73 | 75 | | |||
74 | bool needsShrink() const { | 76 | bool needsShrink() const { | ||
75 | return ShrinkOpcode != -1; | 77 | return ShrinkOpcode != -1; | ||
76 | } | 78 | } | ||
77 | 79 | | |||
▲ Show 20 Lines • Show All 147 Lines • ▼ Show 20 Line(s) | 215 | case AMDGPU::OPERAND_REG_INLINE_C_V2INT16: | |||
225 | } | 227 | } | ||
226 | break; | 228 | break; | ||
227 | default: | 229 | default: | ||
228 | break; | 230 | break; | ||
229 | } | 231 | } | ||
230 | } | 232 | } | ||
231 | } | 233 | } | ||
232 | 234 | | |||
233 | if ((Fold.isImm() || Fold.isFI()) && Fold.needsShrink()) { | 235 | if ((Fold.isImm() || Fold.isFI() || Fold.isGlobal()) && Fold.needsShrink()) { | ||
234 | MachineBasicBlock *MBB = MI->getParent(); | 236 | MachineBasicBlock *MBB = MI->getParent(); | ||
235 | auto Liveness = MBB->computeRegisterLiveness(&TRI, AMDGPU::VCC, MI); | 237 | auto Liveness = MBB->computeRegisterLiveness(&TRI, AMDGPU::VCC, MI); | ||
236 | if (Liveness != MachineBasicBlock::LQR_Dead) | 238 | if (Liveness != MachineBasicBlock::LQR_Dead) | ||
237 | return false; | 239 | return false; | ||
238 | 240 | | |||
239 | MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); | 241 | MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo(); | ||
240 | int Op32 = Fold.getShrinkOpcode(); | 242 | int Op32 = Fold.getShrinkOpcode(); | ||
241 | MachineOperand &Dst0 = MI->getOperand(0); | 243 | MachineOperand &Dst0 = MI->getOperand(0); | ||
Show All 30 Lines | |||||
272 | 274 | | |||
273 | assert(!Fold.needsShrink() && "not handled"); | 275 | assert(!Fold.needsShrink() && "not handled"); | ||
274 | 276 | | |||
275 | if (Fold.isImm()) { | 277 | if (Fold.isImm()) { | ||
276 | Old.ChangeToImmediate(Fold.ImmToFold); | 278 | Old.ChangeToImmediate(Fold.ImmToFold); | ||
277 | return true; | 279 | return true; | ||
278 | } | 280 | } | ||
279 | 281 | | |||
282 | if (Fold.isGlobal()) { | ||||
283 | Old.ChangeToGA(Fold.OpToFold->getGlobal(), Fold.OpToFold->getOffset(), | ||||
284 | Fold.OpToFold->getTargetFlags()); | ||||
285 | return true; | ||||
286 | } | ||||
287 | | ||||
280 | if (Fold.isFI()) { | 288 | if (Fold.isFI()) { | ||
281 | Old.ChangeToFrameIndex(Fold.FrameIndexToFold); | 289 | Old.ChangeToFrameIndex(Fold.FrameIndexToFold); | ||
282 | return true; | 290 | return true; | ||
283 | } | 291 | } | ||
284 | 292 | | |||
285 | MachineOperand *New = Fold.OpToFold; | 293 | MachineOperand *New = Fold.OpToFold; | ||
286 | Old.substVirtReg(New->getReg(), New->getSubReg(), TRI); | 294 | Old.substVirtReg(New->getReg(), New->getSubReg(), TRI); | ||
287 | Old.setIsUndef(New->isUndef()); | 295 | Old.setIsUndef(New->isUndef()); | ||
▲ Show 20 Lines • Show All 75 Lines • ▼ Show 20 Line(s) | 312 | if (!TII->isOperandLegal(*MI, OpNo, OpToFold)) { | |||
363 | if (!CanCommute || | 371 | if (!CanCommute || | ||
364 | !TII->commuteInstruction(*MI, false, CommuteIdx0, CommuteIdx1)) | 372 | !TII->commuteInstruction(*MI, false, CommuteIdx0, CommuteIdx1)) | ||
365 | return false; | 373 | return false; | ||
366 | 374 | | |||
367 | if (!TII->isOperandLegal(*MI, CommuteOpNo, OpToFold)) { | 375 | if (!TII->isOperandLegal(*MI, CommuteOpNo, OpToFold)) { | ||
368 | if ((Opc == AMDGPU::V_ADD_I32_e64 || | 376 | if ((Opc == AMDGPU::V_ADD_I32_e64 || | ||
369 | Opc == AMDGPU::V_SUB_I32_e64 || | 377 | Opc == AMDGPU::V_SUB_I32_e64 || | ||
370 | Opc == AMDGPU::V_SUBREV_I32_e64) && // FIXME | 378 | Opc == AMDGPU::V_SUBREV_I32_e64) && // FIXME | ||
371 | (OpToFold->isImm() || OpToFold->isFI())) { | 379 | (OpToFold->isImm() || OpToFold->isFI() || OpToFold->isGlobal())) { | ||
372 | MachineRegisterInfo &MRI = MI->getParent()->getParent()->getRegInfo(); | 380 | MachineRegisterInfo &MRI = MI->getParent()->getParent()->getRegInfo(); | ||
373 | 381 | | |||
374 | // Verify the other operand is a VGPR, otherwise we would violate the | 382 | // Verify the other operand is a VGPR, otherwise we would violate the | ||
375 | // constant bus restriction. | 383 | // constant bus restriction. | ||
376 | unsigned OtherIdx = CommuteOpNo == CommuteIdx0 ? CommuteIdx1 : CommuteIdx0; | 384 | unsigned OtherIdx = CommuteOpNo == CommuteIdx0 ? CommuteIdx1 : CommuteIdx0; | ||
377 | MachineOperand &OtherOp = MI->getOperand(OtherIdx); | 385 | MachineOperand &OtherOp = MI->getOperand(OtherIdx); | ||
378 | if (!OtherOp.isReg() || | 386 | if (!OtherOp.isReg() || | ||
379 | !TII->getRegisterInfo().isVGPR(MRI, OtherOp.getReg())) | 387 | !TII->getRegisterInfo().isVGPR(MRI, OtherOp.getReg())) | ||
▲ Show 20 Lines • Show All 98 Lines • ▼ Show 20 Line(s) | 475 | if (frameIndexMayFold(TII, *UseMI, UseOpIdx, OpToFold)) { | |||
478 | 486 | | |||
479 | // A frame index will resolve to a positive constant, so it should always be | 487 | // A frame index will resolve to a positive constant, so it should always be | ||
480 | // safe to fold the addressing mode, even pre-GFX9. | 488 | // safe to fold the addressing mode, even pre-GFX9. | ||
481 | UseMI->getOperand(UseOpIdx).ChangeToFrameIndex(OpToFold.getIndex()); | 489 | UseMI->getOperand(UseOpIdx).ChangeToFrameIndex(OpToFold.getIndex()); | ||
482 | SOff->setReg(MFI->getStackPtrOffsetReg()); | 490 | SOff->setReg(MFI->getStackPtrOffsetReg()); | ||
483 | return; | 491 | return; | ||
484 | } | 492 | } | ||
485 | 493 | | |||
486 | bool FoldingImmLike = OpToFold.isImm() || OpToFold.isFI(); | 494 | bool FoldingImmLike = | ||
495 | OpToFold.isImm() || OpToFold.isFI() || OpToFold.isGlobal(); | ||||
487 | 496 | | |||
488 | if (FoldingImmLike && UseMI->isCopy()) { | 497 | if (FoldingImmLike && UseMI->isCopy()) { | ||
489 | unsigned DestReg = UseMI->getOperand(0).getReg(); | 498 | unsigned DestReg = UseMI->getOperand(0).getReg(); | ||
490 | const TargetRegisterClass *DestRC | 499 | const TargetRegisterClass *DestRC | ||
491 | = TargetRegisterInfo::isVirtualRegister(DestReg) ? | 500 | = TargetRegisterInfo::isVirtualRegister(DestReg) ? | ||
492 | MRI->getRegClass(DestReg) : | 501 | MRI->getRegClass(DestReg) : | ||
493 | TRI->getPhysRegClass(DestReg); | 502 | TRI->getPhysRegClass(DestReg); | ||
494 | 503 | | |||
▲ Show 20 Lines • Show All 384 Lines • ▼ Show 20 Line(s) | 887 | void SIFoldOperands::foldInstOperand(MachineInstr &MI, | |||
879 | MachineOperand &OpToFold) const { | 888 | MachineOperand &OpToFold) const { | ||
880 | // We need mutate the operands of new mov instructions to add implicit | 889 | // We need mutate the operands of new mov instructions to add implicit | ||
881 | // uses of EXEC, but adding them invalidates the use_iterator, so defer | 890 | // uses of EXEC, but adding them invalidates the use_iterator, so defer | ||
882 | // this. | 891 | // this. | ||
883 | SmallVector<MachineInstr *, 4> CopiesToReplace; | 892 | SmallVector<MachineInstr *, 4> CopiesToReplace; | ||
884 | SmallVector<FoldCandidate, 4> FoldList; | 893 | SmallVector<FoldCandidate, 4> FoldList; | ||
885 | MachineOperand &Dst = MI.getOperand(0); | 894 | MachineOperand &Dst = MI.getOperand(0); | ||
886 | 895 | | |||
887 | bool FoldingImm = OpToFold.isImm() || OpToFold.isFI(); | 896 | bool FoldingImm = OpToFold.isImm() || OpToFold.isFI() || OpToFold.isGlobal(); | ||
888 | if (FoldingImm) { | 897 | if (FoldingImm) { | ||
889 | unsigned NumLiteralUses = 0; | 898 | unsigned NumLiteralUses = 0; | ||
890 | MachineOperand *NonInlineUse = nullptr; | 899 | MachineOperand *NonInlineUse = nullptr; | ||
891 | int NonInlineUseOpNo = -1; | 900 | int NonInlineUseOpNo = -1; | ||
892 | 901 | | |||
893 | MachineRegisterInfo::use_iterator NextUse; | 902 | MachineRegisterInfo::use_iterator NextUse; | ||
894 | for (MachineRegisterInfo::use_iterator | 903 | for (MachineRegisterInfo::use_iterator | ||
895 | Use = MRI->use_begin(Dst.getReg()), E = MRI->use_end(); | 904 | Use = MRI->use_begin(Dst.getReg()), E = MRI->use_end(); | ||
▲ Show 20 Lines • Show All 331 Lines • ▼ Show 20 Line(s) | 1234 | if (!TII->isFoldableCopy(MI)) { | |||
1227 | // instruction, and not the omod multiply. | 1236 | // instruction, and not the omod multiply. | ||
1228 | if (IsIEEEMode || (!HasNSZ && !MI.getFlag(MachineInstr::FmNsz)) || | 1237 | if (IsIEEEMode || (!HasNSZ && !MI.getFlag(MachineInstr::FmNsz)) || | ||
1229 | !tryFoldOMod(MI)) | 1238 | !tryFoldOMod(MI)) | ||
1230 | tryFoldClamp(MI); | 1239 | tryFoldClamp(MI); | ||
1231 | continue; | 1240 | continue; | ||
1232 | } | 1241 | } | ||
1233 | 1242 | | |||
1234 | MachineOperand &OpToFold = MI.getOperand(1); | 1243 | MachineOperand &OpToFold = MI.getOperand(1); | ||
1235 | bool FoldingImm = OpToFold.isImm() || OpToFold.isFI(); | 1244 | bool FoldingImm = | ||
1245 | OpToFold.isImm() || OpToFold.isFI() || OpToFold.isGlobal(); | ||||
1236 | 1246 | | |||
1237 | // FIXME: We could also be folding things like TargetIndexes. | 1247 | // FIXME: We could also be folding things like TargetIndexes. | ||
1238 | if (!FoldingImm && !OpToFold.isReg()) | 1248 | if (!FoldingImm && !OpToFold.isReg()) | ||
1239 | continue; | 1249 | continue; | ||
1240 | 1250 | | |||
1241 | if (OpToFold.isReg() && | 1251 | if (OpToFold.isReg() && | ||
1242 | !TargetRegisterInfo::isVirtualRegister(OpToFold.getReg())) | 1252 | !TargetRegisterInfo::isVirtualRegister(OpToFold.getReg())) | ||
1243 | continue; | 1253 | continue; | ||
Show All 17 Lines |