diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td --- a/llvm/lib/Target/AMDGPU/BUFInstructions.td +++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -488,15 +488,15 @@ } class MUBUF_Offset_Load_Pat : Pat < - (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset))), - (load_vt (inst v4i32:$srsrc, i32:$soffset, i16:$offset)) + (load_vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset))), + (load_vt (inst v4i32:$srsrc, i32:$soffset, i32:$offset)) >; class MUBUF_Addr64_Load_Pat : Pat < - (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset))), - (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i16:$offset)) + (load_vt (ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset))), + (load_vt (inst i64:$vaddr, v4i32:$srsrc, i32:$soffset, i32:$offset)) >; multiclass MUBUF_Pseudo_Load_Pats { @@ -580,12 +580,12 @@ def _OFFSET : MUBUF_Store_Pseudo , + i32:$offset))]>, MUBUFAddr64Table<0, NAME>; def _ADDR64 : MUBUF_Store_Pseudo , + i32:$offset))]>, MUBUFAddr64Table<1, NAME>; def _OFFEN : MUBUF_Store_Pseudo ; @@ -743,13 +743,13 @@ let FPAtomic = isFP in { def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo , MUBUFAddr64Table <0, NAME # "_RTN">; def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo , MUBUFAddr64Table <1, NAME # "_RTN">; @@ -1395,13 +1395,13 @@ let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in { def : GCNPat< - (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), vt:$vdata_in)), + (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset), vt:$vdata_in)), (!cast(Inst # "_OFFSET" # InstSuffix) getVregSrcForVT.ret:$vdata_in, SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset) >; def : GCNPat< - (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), + (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), vt:$vdata_in)), (!cast(Inst # "_ADDR64" # InstSuffix) getVregSrcForVT.ret:$vdata_in, VReg_64:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset) @@ -1428,7 +1428,7 @@ getVregSrcForVT.ret:$vdata_in, SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset); def : GCNPat< - (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset), data_vt:$vdata_in)), + (vt (Op (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset), data_vt:$vdata_in)), !if(!eq(RtnMode, "ret"), (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS OffsetResDag, getVregSrcForVT.ret)), !if(!eq(vt, i32), sub0, sub0_sub1)), @@ -1439,7 +1439,7 @@ getVregSrcForVT.ret:$vdata_in, VReg_64:$vaddr, SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset); def : GCNPat< - (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), + (vt (Op (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), data_vt:$vdata_in)), !if(!eq(RtnMode, "ret"), (EXTRACT_SUBREG (vt (COPY_TO_REGCLASS Addr64ResDag, getVregSrcForVT.ret)), @@ -1685,19 +1685,19 @@ class MUBUFLoad_PatternADDR64 : GCNPat < (vt (constant_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, - i16:$offset))), + i32:$offset))), (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) >; multiclass MUBUFLoad_Atomic_Pattern { def : GCNPat < - (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset))), + (vt (atomic_ld (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset))), (Instr_ADDR64 $vaddr, $srsrc, $soffset, $offset) >; def : GCNPat < - (vt (atomic_ld (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset))), + (vt (atomic_ld (MUBUFOffset v4i32:$rsrc, i32:$soffset, i32:$offset))), (Instr_OFFSET $rsrc, $soffset, (as_i16imm $offset)) >; } @@ -1718,7 +1718,7 @@ PatFrag ld> { def : GCNPat < - (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset))), + (vt (ld (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset))), (Instr_OFFSET $srsrc, $soffset, $offset) >; } @@ -1741,12 +1741,12 @@ ValueType vt, PatFrag ld> { def : GCNPat < (vt (ld (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, - i32:$soffset, u16imm:$offset))), + i32:$soffset, u32imm:$offset))), (InstrOffen $vaddr, $srsrc, $soffset, $offset, 0, 0) >; def : GCNPat < - (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset))), + (vt (ld (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u32imm:$offset))), (InstrOffset $srsrc, $soffset, $offset, 0, 0) >; } @@ -1756,12 +1756,12 @@ MUBUF_Pseudo InstrOffset, ValueType vt, PatFrag ld_frag> { def : GCNPat < - (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u16imm:$offset), vt:$in), + (ld_frag (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, i32:$soffset, u32imm:$offset), vt:$in), (InstrOffen $vaddr, $srsrc, $soffset, $offset, $in) >; def : GCNPat < - (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u16imm:$offset), vt:$in), + (ld_frag (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, u32imm:$offset), vt:$in), (InstrOffset $srsrc, $soffset, $offset, $in) >; } @@ -1807,12 +1807,12 @@ ValueType vt, PatFrag atomic_st> { // Store follows atomic op convention so address is first def : GCNPat < - (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i16:$offset), vt:$val), + (atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), vt:$val), (Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset) >; def : GCNPat < - (atomic_st (MUBUFOffset v4i32:$rsrc, i32:$soffset, i16:$offset), vt:$val), + (atomic_st (MUBUFOffset v4i32:$rsrc, i32:$soffset, i32:$offset), vt:$val), (Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset)) >; } @@ -1830,7 +1830,7 @@ PatFrag st> { def : GCNPat < - (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i16:$offset)), + (st vt:$vdata, (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset)), (Instr_OFFSET $vdata, $srsrc, $soffset, $offset) >; } @@ -1844,13 +1844,13 @@ RegisterClass rc = VGPR_32> { def : GCNPat < (st vt:$value, (MUBUFScratchOffen v4i32:$srsrc, i32:$vaddr, - i32:$soffset, u16imm:$offset)), + i32:$soffset, u32imm:$offset)), (InstrOffen rc:$value, $vaddr, $srsrc, $soffset, $offset, 0, 0) >; def : GCNPat < (st vt:$value, (MUBUFScratchOffset v4i32:$srsrc, i32:$soffset, - u16imm:$offset)), + u32imm:$offset)), (InstrOffset rc:$value, $srsrc, $soffset, $offset, 0, 0) >; } diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td --- a/llvm/lib/Target/AMDGPU/DSInstructions.td +++ b/llvm/lib/Target/AMDGPU/DSInstructions.td @@ -436,7 +436,7 @@ (ins VGPR_32:$addr, data_op:$data0, offset:$offset), " $vdst, $addr, $data0$offset", [(set i32:$vdst, - (node (DS1Addr1Offset i32:$addr, i16:$offset), i32:$data0))] > { + (node (DS1Addr1Offset i32:$addr, i32:$offset), i32:$data0))] > { let mayLoad = 0; let mayStore = 0; @@ -740,7 +740,7 @@ >; class DSReadPat : GCNPat < - (vt (frag (DS1Addr1Offset i32:$ptr, i16:$offset))), + (vt (frag (DS1Addr1Offset i32:$ptr, i32:$offset))), (inst $ptr, offset:$offset, (i1 gds)) >; @@ -756,7 +756,7 @@ } class DSReadPat_D16 : GCNPat < - (frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$in), + (frag (DS1Addr1Offset i32:$ptr, i32:$offset), vt:$in), (inst $ptr, offset:$offset, (i1 0), $in) >; @@ -800,7 +800,7 @@ } class DSWritePat : GCNPat < - (frag vt:$value, (DS1Addr1Offset i32:$ptr, i16:$offset)), + (frag vt:$value, (DS1Addr1Offset i32:$ptr, i32:$offset)), (inst $ptr, getVregSrcForVT.ret:$value, offset:$offset, (i1 gds)) >; @@ -817,7 +817,7 @@ // Irritatingly, atomic_store reverses the order of operands from a // normal store. class DSAtomicWritePat : GCNPat < - (frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$value), + (frag (DS1Addr1Offset i32:$ptr, i32:$offset), vt:$value), (inst $ptr, getVregSrcForVT.ret:$value, offset:$offset, (i1 0)) >; @@ -965,7 +965,7 @@ } // End AddedComplexity = 100 class DSAtomicRetPat : GCNPat <(frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$value), + bit gds=0> : GCNPat <(frag (DS1Addr1Offset i32:$ptr, i32:$offset), vt:$value), (inst $ptr, getVregSrcForVT.ret:$value, offset:$offset, (i1 gds))> { let AddedComplexity = complexity; } @@ -1014,7 +1014,7 @@ // Caution, the order of src and cmp is the *opposite* of the BUFFER_ATOMIC_CMPSWAP opcode. class DSAtomicCmpXChgSwapped : GCNPat< - (frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$cmp, vt:$swap), + (frag (DS1Addr1Offset i32:$ptr, i32:$offset), vt:$cmp, vt:$swap), (inst $ptr, getVregSrcForVT.ret:$cmp, getVregSrcForVT.ret:$swap, offset:$offset, (i1 gds))> { let AddedComplexity = complexity; } @@ -1046,7 +1046,7 @@ // The order of src and cmp agrees with the BUFFER_ATOMIC_CMPSWAP opcode. class DSAtomicCmpXChg : GCNPat< - (frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$cmp, vt:$swap), + (frag (DS1Addr1Offset i32:$ptr, i32:$offset), vt:$cmp, vt:$swap), (inst $ptr, getVregSrcForVT.ret:$swap, getVregSrcForVT.ret:$cmp, offset:$offset, (i1 gds))> { let AddedComplexity = complexity; } @@ -1124,7 +1124,7 @@ class DSAtomicRetPatIntrinsic : GCNPat < - (vt (frag (DS1Addr1Offset i32:$ptr, i16:$offset), vt:$value)), + (vt (frag (DS1Addr1Offset i32:$ptr, i32:$offset), vt:$value)), (inst $ptr, getVregSrcForVT.ret:$value, offset:$offset, (i1 gds))> { } diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td @@ -1220,7 +1220,7 @@ let OperandType = "OPERAND_IMMEDIATE" in { def flat_offset : CustomOperand; -def offset : NamedIntOperand; +def offset : NamedIntOperand; def offset0 : NamedIntOperand; def offset1 : NamedIntOperand;